1598 lines
72 KiB
Plaintext
1598 lines
72 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "94f47e49-b4d8-4816-90e6-441baa97223a",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"/opt/miniconda/envs/focallora4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
|
||
" from .autonotebook import tqdm as notebook_tqdm\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"🥷 Injecting Spy Interceptor into Flash Attention 2...\n",
|
||
"✅ Spy Interceptor installed successfully!\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"\n",
|
||
"import os\n",
|
||
"import json\n",
|
||
"import csv\n",
|
||
"import argparse\n",
|
||
"import math\n",
|
||
"import random\n",
|
||
"import re\n",
|
||
"from typing import List, Tuple, Dict\n",
|
||
"\n",
|
||
"import torch\n",
|
||
"import numpy as np\n",
|
||
"from tqdm import tqdm\n",
|
||
"from torch.utils.data import Dataset, DataLoader\n",
|
||
"from evallib import quick_eval_mmlu, quick_eval_asr_util\n",
|
||
"from transformers import (\n",
|
||
" AutoConfig, AutoTokenizer, AutoModelForCausalLM,\n",
|
||
" BitsAndBytesConfig, get_linear_schedule_with_warmup,\n",
|
||
")\n",
|
||
"from transformers.models.llama.modeling_llama import ALL_ATTENTION_FUNCTIONS\n",
|
||
"from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training, PeftModel\n",
|
||
"from lib.tokenize_data_mask import apply_chat_tokenize_with_strip_and_mark\n",
|
||
"from lib.printcolor import print_tok_color\n",
|
||
"# -----------------------------------------------------------------------------\n",
|
||
"# 1. Flash Attention Spy Interceptor (核心機制)\n",
|
||
"# -----------------------------------------------------------------------------\n",
|
||
"print(\"🥷 Injecting Spy Interceptor into Flash Attention 2...\")\n",
|
||
"\n",
|
||
"if \"flash_attention_2\" not in ALL_ATTENTION_FUNCTIONS:\n",
|
||
" raise RuntimeError(\"Current environment does not support Flash Attention 2!\")\n",
|
||
"\n",
|
||
"orig_flash_attn = ALL_ATTENTION_FUNCTIONS[\"flash_attention_2\"]\n",
|
||
"\n",
|
||
"def wrapped_flash_attn(module, query, key, value, attention_mask, scaling, **kwargs):\n",
|
||
" \"\"\"\n",
|
||
" 攔截 Flash Attention，並在 Breakpoint 驗證手算結果與原始 FA2 結果的一致性。\n",
|
||
" \"\"\"\n",
|
||
" # -------------------------------------------------------------------------\n",
|
||
" # [TASK A] 間諜行動：偷算 Last Token Weight\n",
|
||
" # -------------------------------------------------------------------------\n",
|
||
" # 1. 切片 (只看最後一個 Token)\n",
|
||
" q_last = query[..., -1:, :] \n",
|
||
" \n",
|
||
" # 2. 處理 GQA (手動 Expand Key 和 Value)\n",
|
||
" num_heads = query.shape[1]\n",
|
||
" num_kv_heads = key.shape[1]\n",
|
||
" \n",
|
||
" # 用於後面計算 Manual Output\n",
|
||
" k_expanded = key\n",
|
||
" v_expanded = value \n",
|
||
"\n",
|
||
" if num_heads != num_kv_heads:\n",
|
||
" n_rep = num_heads // num_kv_heads\n",
|
||
" k_expanded = key.repeat_interleave(n_rep, dim=1)\n",
|
||
" v_expanded = value.repeat_interleave(n_rep, dim=1) # Value 也要 Expand 才能乘\n",
|
||
" \n",
|
||
" # 3. 手動計算 Tiny Attention Weight [Batch, Heads, 1, Seq]\n",
|
||
" # Q: [B, H, 1, D], K: [B, H, S, D] -> QK^T: [B, H, 1, S]\n",
|
||
" attn_weights_tiny = torch.matmul(q_last, k_expanded.transpose(2, 3)) * scaling\n",
|
||
" \n",
|
||
" # 4. 處理 Mask\n",
|
||
" if attention_mask is not None:\n",
|
||
" if attention_mask.dim() == 4:\n",
|
||
" if attention_mask.size(2) > 1:\n",
|
||
" mask_slice = attention_mask[..., -1:, :]\n",
|
||
" attn_weights_tiny = attn_weights_tiny + mask_slice\n",
|
||
" else:\n",
|
||
" attn_weights_tiny = attn_weights_tiny + attention_mask\n",
|
||
" elif attention_mask.dim() == 2:\n",
|
||
" mask_expanded = attention_mask[:, None, None, :]\n",
|
||
" if attention_mask.dtype == torch.bool or (attention_mask.max() <= 1.0 and attention_mask.min() >= 0.0):\n",
|
||
" min_dtype = torch.finfo(attn_weights_tiny.dtype).min\n",
|
||
" attn_weights_tiny = attn_weights_tiny.masked_fill(mask_expanded == 0, min_dtype)\n",
|
||
" else:\n",
|
||
" attn_weights_tiny = attn_weights_tiny + mask_expanded\n",
|
||
" \n",
|
||
" # 5. Softmax (得到機率分佈 P)\n",
|
||
" # 注意：這裡轉成 float32 做 softmax 以求精確，實際 FA2 內部可能是 fp16/bf16\n",
|
||
" attn_probs = torch.nn.functional.softmax(attn_weights_tiny, dim=-1, dtype=torch.float32).to(query.dtype)\n",
|
||
"\n",
|
||
" # 填入 Config (原本的邏輯)\n",
|
||
" retrieve_map = getattr(module.config, \"retrieve_attn_map\", None)\n",
|
||
" if retrieve_map is not None and hasattr(module, \"layer_idx\") and module.layer_idx in retrieve_map:\n",
|
||
" target_heads_dict = retrieve_map[module.layer_idx]\n",
|
||
" for h_idx in target_heads_dict.keys():\n",
|
||
" # 存入 config 用於 Loss 計算\n",
|
||
" target_heads_dict[h_idx] = attn_probs[:, h_idx, :, :]\n",
|
||
"\n",
|
||
" # -------------------------------------------------------------------------\n",
|
||
" # [DEBUG] 驗證環節：手算 Output vs Flash Attention Output\n",
|
||
" # -------------------------------------------------------------------------\n",
|
||
" \n",
|
||
" # 1. 執行原始 Flash Attention\n",
|
||
" orig_result = orig_flash_attn(module, query, key, value, attention_mask, scaling=scaling, **kwargs)\n",
|
||
" debug=False\n",
|
||
" if debug:\n",
|
||
" orig_tensor = orig_result[0]\n",
|
||
" # 2. 手動計算 Output ( O = Attention_Probs * V )\n",
|
||
" # attn_probs: [B, H, 1, S]\n",
|
||
" # v_expanded: [B, H, S, D]\n",
|
||
" # manual_out: [B, H, 1, D]\n",
|
||
" manual_out = torch.matmul(attn_probs, v_expanded)\n",
|
||
" \n",
|
||
" # 3. 對齊形狀以便比較\n",
|
||
" # FA2 output 通常是 [Batch, Seq, Heads, Dim]\n",
|
||
" # 我們取最後一個 token: [Batch, 1, Heads, Dim]\n",
|
||
" orig_out_last = orig_tensor[:, -1:, :, :]\n",
|
||
" \n",
|
||
" # 手算結果原本是 [Batch, Heads, 1, Dim]，轉置成 [Batch, 1, Heads, Dim]\n",
|
||
" manual_out_check = manual_out.transpose(1, 2)\n",
|
||
" \n",
|
||
" # 4. 計算誤差\n",
|
||
" diff = (orig_out_last - manual_out_check).abs()\n",
|
||
" max_diff = diff.max().item()\n",
|
||
" mean_diff = diff.mean().item()\n",
|
||
" \n",
|
||
" print(f\"\\n🕵️ [Layer {getattr(module, 'layer_idx', '?')}] Validation:\")\n",
|
||
" print(f\" Max Diff: {max_diff:.8f}\")\n",
|
||
" print(f\" Mean Diff: {mean_diff:.8f}\")\n",
|
||
" \n",
|
||
" # 如果誤差過大 (例如 > 1e-3)，自動暫停檢查\n",
|
||
" # 注意：BF16 下 FlashAttention 和標準 Matmul 會有一定精度差異是正常的\n",
|
||
" if max_diff > 1e-2: \n",
|
||
" print(\"⚠️ Warning: Large difference detected!\")\n",
|
||
" \n",
|
||
" # 呼叫 breakpoint 讓你進去檢查\n",
|
||
" # 在 pdb 輸入: \n",
|
||
" # p manual_out_check[0,0,0,:5] \n",
|
||
" # p orig_out_last[0,0,0,:5]\n",
|
||
" # 來對比數值\n",
|
||
" breakpoint() \n",
|
||
"\n",
|
||
" return orig_result\n",
|
||
"\n",
|
||
"# 替換全局函數\n",
|
||
"ALL_ATTENTION_FUNCTIONS[\"flash_attention_2\"] = wrapped_flash_attn\n",
|
||
"print(\"✅ Spy Interceptor installed successfully!\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 2,
|
||
"id": "7e2904a3-3abf-44b0-9d78-43710a3c4d62",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"SEED = 42\n",
|
||
"random.seed(SEED)\n",
|
||
"np.random.seed(SEED)\n",
|
||
"torch.manual_seed(SEED)\n",
|
||
"\n",
|
||
"def get_lora_targets(model, layers: List[int]) -> List[str]:\n",
|
||
" mtype = (getattr(model.config, \"model_type\", \"\") or \"\").lower()\n",
|
||
" if \"llama\" in mtype or \"mistral\" in mtype:\n",
|
||
" return [f\"model.layers.{i}.self_attn.{p}\" for i in layers for p in (\"q_proj\", \"k_proj\")]\n",
|
||
" # Fallback / Other architectures\n",
|
||
" cand = []\n",
|
||
" for name, _ in model.named_modules():\n",
|
||
" if any(f\".{i}.\" in name for i in layers) and name.split(\".\")[-1] in {\"q_proj\", \"k_proj\"}:\n",
|
||
" cand.append(name)\n",
|
||
" return cand\n",
|
||
"\n",
|
||
"def load_model(model_path: str):\n",
|
||
" cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=True)\n",
|
||
" tok = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=True,padding_side=\"left\")\n",
|
||
" \n",
|
||
" if tok.pad_token_id is None:\n",
|
||
" tok.pad_token = tok.eos_token\n",
|
||
" tok.pad_token_id = tok.eos_token_id\n",
|
||
" \n",
|
||
" bnb_cfg = BitsAndBytesConfig(\n",
|
||
" load_in_4bit=True,\n",
|
||
" bnb_4bit_compute_dtype=torch.float16,\n",
|
||
" bnb_4bit_use_double_quant=True,\n",
|
||
" bnb_4bit_quant_type=\"nf4\",\n",
|
||
" )\n",
|
||
" model = AutoModelForCausalLM.from_pretrained(\n",
|
||
" model_path,\n",
|
||
" config=cfg,\n",
|
||
" quantization_config=bnb_cfg,\n",
|
||
" device_map=\"auto\",\n",
|
||
" trust_remote_code=True,\n",
|
||
" attn_implementation=\"flash_attention_2\",\n",
|
||
" )\n",
|
||
" return model, tok, tok # use same tokenizer for simplicity"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "28290215-6860-4220-a02b-35e882da579a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def get_target_structure(heads_file: str, topk_spec: str) -> Dict[int, List[int]]:\n",
|
||
" \"\"\"Load heads from file and parse into {layer: [heads]} structure.\"\"\"\n",
|
||
" if not os.path.exists(heads_file):\n",
|
||
" raise FileNotFoundError(f\"Heads file not found: {heads_file}\")\n",
|
||
" \n",
|
||
" with open(heads_file, \"r\") as f:\n",
|
||
" ihead_list = json.load(f)\n",
|
||
" if type(ihead_list[0]) == str:\n",
|
||
" all_heads = [(str(tag), 0.0) for tag in ihead_list]\n",
|
||
" else:\n",
|
||
" all_heads = ihead_list\n",
|
||
" \n",
|
||
" # Filter Top-K\n",
|
||
" count = len(all_heads)\n",
|
||
" if topk_spec.endswith(\"p\"):\n",
|
||
" count = max(1, math.ceil(float(topk_spec[:-1]) / 100.0 * len(all_heads)))\n",
|
||
" else:\n",
|
||
" count = int(topk_spec)\n",
|
||
" \n",
|
||
" target_heads = all_heads[:min(count, len(all_heads))]\n",
|
||
" \n",
|
||
" # Parse into structure\n",
|
||
" structure = {}\n",
|
||
" for tag, _ in target_heads:\n",
|
||
" try:\n",
|
||
" # tag format: \"L22_H7\"\n",
|
||
" parts = tag.split('_')\n",
|
||
" l = int(parts[0][1:])\n",
|
||
" h = int(parts[1][1:])\n",
|
||
" structure.setdefault(l, []).append(h)\n",
|
||
" except Exception:\n",
|
||
" continue\n",
|
||
" \n",
|
||
" print(f\"🎯 Selected {len(target_heads)} heads from {topk_spec}.\")\n",
|
||
" return structure"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 4,
|
||
"id": "211437db-b128-4d9c-8251-d38b43e3bb35",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"🎯 Selected 223 heads from 21.75p.\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"model_path = \"/data/local/models/Qwen3-8B\"\n",
|
||
"head_path=\"/data/local/hujk/ISH/head_scoring/qwen3-8b_sep/heads_sorted/all_roc_inst_0.1.json\"\n",
|
||
"\n",
|
||
"model_path = \"/data/local/models/Llama-3.1-8B-Instruct\"\n",
|
||
"head_path=\"/data/local/hujk/ISH/head_scoring/llama31-8b_sep/heads_sorted/all_roc_inst_0.1.json\"\n",
|
||
"target_heads = get_target_structure(head_path,\"21.75p\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 6,
|
||
"id": "44d6094d-74e8-47a1-bac7-fa012b90ede7",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"{13: [7,\n",
|
||
" 14,\n",
|
||
" 8,\n",
|
||
" 13,\n",
|
||
" 29,\n",
|
||
" 22,\n",
|
||
" 20,\n",
|
||
" 10,\n",
|
||
" 6,\n",
|
||
" 9,\n",
|
||
" 19,\n",
|
||
" 2,\n",
|
||
" 24,\n",
|
||
" 28,\n",
|
||
" 26,\n",
|
||
" 16,\n",
|
||
" 0,\n",
|
||
" 25,\n",
|
||
" 12,\n",
|
||
" 23,\n",
|
||
" 17],\n",
|
||
" 16: [3, 16, 4, 22, 14, 6, 17, 27, 24, 19, 15, 12, 5, 25],\n",
|
||
" 9: [24, 26, 31, 27, 22, 20, 13, 12, 29, 28, 30, 18, 1, 9, 2, 23],\n",
|
||
" 15: [23, 12, 14, 21, 27, 15, 5, 10, 29, 0, 13, 22, 3, 20, 24, 16, 6, 8],\n",
|
||
" 17: [31, 20, 30, 25, 22, 21, 7, 27, 14, 4, 28, 13, 5],\n",
|
||
" 14: [29, 7, 13, 3, 6, 23, 0, 28, 18, 1, 24, 10, 12, 14, 2, 15, 21, 27],\n",
|
||
" 5: [29, 4, 15, 30, 31, 3, 16, 17, 25],\n",
|
||
" 12: [9, 24, 26, 4, 30, 15, 31, 16, 8, 21, 27, 25, 11, 10, 22, 28, 0, 23],\n",
|
||
" 7: [14, 9, 28, 31, 11, 10, 17, 8, 29, 18, 24, 12, 23],\n",
|
||
" 8: [25, 4, 15, 31, 13, 24, 19, 30, 27, 20, 8],\n",
|
||
" 18: [25, 29, 11, 10, 28, 24, 27, 19, 17, 12, 5, 18, 0, 2, 9],\n",
|
||
" 6: [7, 29, 22, 30, 20, 13, 0, 4],\n",
|
||
" 10: [11, 20, 24, 19, 9, 0, 16, 30, 18, 1, 22, 25, 21, 13, 17, 23, 10],\n",
|
||
" 19: [21, 20, 3, 22],\n",
|
||
" 11: [23, 25, 9, 26, 21, 24, 10, 27, 11, 6],\n",
|
||
" 20: [30, 23, 0, 9, 21, 24, 15],\n",
|
||
" 22: [9, 4, 17],\n",
|
||
" 4: [15, 31, 4],\n",
|
||
" 26: [2],\n",
|
||
" 21: [8],\n",
|
||
" 3: [30, 28],\n",
|
||
" 23: [17]}"
|
||
]
|
||
},
|
||
"execution_count": 6,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"target_heads"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "f1bbf423-65a8-401f-ad5a-8a644bc53952",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Loading checkpoint shards: 100% 4/4 [00:12<00:00, 3.21s/it]\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"model, tok, tok2 = load_model(model_path)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 7,
|
||
"id": "66dda564-e757-4784-9a72-48e2add45151",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"def get_lora_targets(model, layers: List[int]) -> List[str]:\n",
|
||
" mtype = (getattr(model.config, \"model_type\", \"\") or \"\").lower()\n",
|
||
" if \"llama\" in mtype or \"mistral\" in mtype:\n",
|
||
" return [f\"model.layers.{i}.self_attn.{p}\" for i in layers for p in (\"q_proj\", \"k_proj\")]\n",
|
||
" # Fallback / Other architectures\n",
|
||
" cand = []\n",
|
||
" for name, _ in model.named_modules():\n",
|
||
" if any(f\".{i}.\" in name for i in layers) and name.split(\".\")[-1] in {\"q_proj\", \"k_proj\"}:\n",
|
||
" cand.append(name)\n",
|
||
" return cand"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 9,
|
||
"id": "f4a9304c-0e5c-4c8d-b270-0cbc4f20e3bd",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"ls =get_lora_targets(model,target_heads)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 10,
|
||
"id": "27374e33-9478-489b-a4a6-14a51644a445",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"['model.layers.10.self_attn.k_proj',\n",
|
||
" 'model.layers.10.self_attn.q_proj',\n",
|
||
" 'model.layers.11.self_attn.k_proj',\n",
|
||
" 'model.layers.11.self_attn.q_proj',\n",
|
||
" 'model.layers.12.self_attn.k_proj',\n",
|
||
" 'model.layers.12.self_attn.q_proj',\n",
|
||
" 'model.layers.13.self_attn.k_proj',\n",
|
||
" 'model.layers.13.self_attn.q_proj',\n",
|
||
" 'model.layers.14.self_attn.k_proj',\n",
|
||
" 'model.layers.14.self_attn.q_proj',\n",
|
||
" 'model.layers.15.self_attn.k_proj',\n",
|
||
" 'model.layers.15.self_attn.q_proj',\n",
|
||
" 'model.layers.16.self_attn.k_proj',\n",
|
||
" 'model.layers.16.self_attn.q_proj',\n",
|
||
" 'model.layers.17.self_attn.k_proj',\n",
|
||
" 'model.layers.17.self_attn.q_proj',\n",
|
||
" 'model.layers.18.self_attn.k_proj',\n",
|
||
" 'model.layers.18.self_attn.q_proj',\n",
|
||
" 'model.layers.19.self_attn.k_proj',\n",
|
||
" 'model.layers.19.self_attn.q_proj',\n",
|
||
" 'model.layers.20.self_attn.k_proj',\n",
|
||
" 'model.layers.20.self_attn.q_proj',\n",
|
||
" 'model.layers.21.self_attn.k_proj',\n",
|
||
" 'model.layers.21.self_attn.q_proj',\n",
|
||
" 'model.layers.22.self_attn.k_proj',\n",
|
||
" 'model.layers.22.self_attn.q_proj',\n",
|
||
" 'model.layers.23.self_attn.k_proj',\n",
|
||
" 'model.layers.23.self_attn.q_proj',\n",
|
||
" 'model.layers.26.self_attn.k_proj',\n",
|
||
" 'model.layers.26.self_attn.q_proj',\n",
|
||
" 'model.layers.3.self_attn.k_proj',\n",
|
||
" 'model.layers.3.self_attn.q_proj',\n",
|
||
" 'model.layers.4.self_attn.k_proj',\n",
|
||
" 'model.layers.4.self_attn.q_proj',\n",
|
||
" 'model.layers.5.self_attn.k_proj',\n",
|
||
" 'model.layers.5.self_attn.q_proj',\n",
|
||
" 'model.layers.6.self_attn.k_proj',\n",
|
||
" 'model.layers.6.self_attn.q_proj',\n",
|
||
" 'model.layers.7.self_attn.k_proj',\n",
|
||
" 'model.layers.7.self_attn.q_proj',\n",
|
||
" 'model.layers.8.self_attn.k_proj',\n",
|
||
" 'model.layers.8.self_attn.q_proj',\n",
|
||
" 'model.layers.9.self_attn.k_proj',\n",
|
||
" 'model.layers.9.self_attn.q_proj']"
|
||
]
|
||
},
|
||
"execution_count": 10,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"sorted(ls)"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 22,
|
||
"id": "62ce6fe8-1a01-4dd6-a682-830e12eb7c11",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"0:===========================================\n",
|
||
"\n",
|
||
"1:===========================================\n",
|
||
"model\n",
|
||
"2:===========================================\n",
|
||
"model.embed_tokens\n",
|
||
"3:===========================================\n",
|
||
"model.layers\n",
|
||
"4:===========================================\n",
|
||
"model.layers.0\n",
|
||
"5:===========================================\n",
|
||
"model.layers.0.self_attn\n",
|
||
"6:===========================================\n",
|
||
"model.layers.0.self_attn.q_proj\n",
|
||
"7:===========================================\n",
|
||
"model.layers.0.self_attn.k_proj\n",
|
||
"8:===========================================\n",
|
||
"model.layers.0.self_attn.v_proj\n",
|
||
"9:===========================================\n",
|
||
"model.layers.0.self_attn.o_proj\n",
|
||
"10:===========================================\n",
|
||
"model.layers.0.self_attn.q_norm\n",
|
||
"11:===========================================\n",
|
||
"model.layers.0.self_attn.k_norm\n",
|
||
"12:===========================================\n",
|
||
"model.layers.0.mlp\n",
|
||
"13:===========================================\n",
|
||
"model.layers.0.mlp.gate_proj\n",
|
||
"14:===========================================\n",
|
||
"model.layers.0.mlp.up_proj\n",
|
||
"15:===========================================\n",
|
||
"model.layers.0.mlp.down_proj\n",
|
||
"16:===========================================\n",
|
||
"model.layers.0.mlp.act_fn\n",
|
||
"17:===========================================\n",
|
||
"model.layers.0.input_layernorm\n",
|
||
"18:===========================================\n",
|
||
"model.layers.0.post_attention_layernorm\n",
|
||
"19:===========================================\n",
|
||
"model.layers.1\n",
|
||
"20:===========================================\n",
|
||
"model.layers.1.self_attn\n",
|
||
"21:===========================================\n",
|
||
"model.layers.1.self_attn.q_proj\n",
|
||
"22:===========================================\n",
|
||
"model.layers.1.self_attn.k_proj\n",
|
||
"23:===========================================\n",
|
||
"model.layers.1.self_attn.v_proj\n",
|
||
"24:===========================================\n",
|
||
"model.layers.1.self_attn.o_proj\n",
|
||
"25:===========================================\n",
|
||
"model.layers.1.self_attn.q_norm\n",
|
||
"26:===========================================\n",
|
||
"model.layers.1.self_attn.k_norm\n",
|
||
"27:===========================================\n",
|
||
"model.layers.1.mlp\n",
|
||
"28:===========================================\n",
|
||
"model.layers.1.mlp.gate_proj\n",
|
||
"29:===========================================\n",
|
||
"model.layers.1.mlp.up_proj\n",
|
||
"30:===========================================\n",
|
||
"model.layers.1.mlp.down_proj\n",
|
||
"31:===========================================\n",
|
||
"model.layers.1.mlp.act_fn\n",
|
||
"32:===========================================\n",
|
||
"model.layers.1.input_layernorm\n",
|
||
"33:===========================================\n",
|
||
"model.layers.1.post_attention_layernorm\n",
|
||
"34:===========================================\n",
|
||
"model.layers.2\n",
|
||
"35:===========================================\n",
|
||
"model.layers.2.self_attn\n",
|
||
"36:===========================================\n",
|
||
"model.layers.2.self_attn.q_proj\n",
|
||
"37:===========================================\n",
|
||
"model.layers.2.self_attn.k_proj\n",
|
||
"38:===========================================\n",
|
||
"model.layers.2.self_attn.v_proj\n",
|
||
"39:===========================================\n",
|
||
"model.layers.2.self_attn.o_proj\n",
|
||
"40:===========================================\n",
|
||
"model.layers.2.self_attn.q_norm\n",
|
||
"41:===========================================\n",
|
||
"model.layers.2.self_attn.k_norm\n",
|
||
"42:===========================================\n",
|
||
"model.layers.2.mlp\n",
|
||
"43:===========================================\n",
|
||
"model.layers.2.mlp.gate_proj\n",
|
||
"44:===========================================\n",
|
||
"model.layers.2.mlp.up_proj\n",
|
||
"45:===========================================\n",
|
||
"model.layers.2.mlp.down_proj\n",
|
||
"46:===========================================\n",
|
||
"model.layers.2.mlp.act_fn\n",
|
||
"47:===========================================\n",
|
||
"model.layers.2.input_layernorm\n",
|
||
"48:===========================================\n",
|
||
"model.layers.2.post_attention_layernorm\n",
|
||
"49:===========================================\n",
|
||
"model.layers.3\n",
|
||
"50:===========================================\n",
|
||
"model.layers.3.self_attn\n",
|
||
"51:===========================================\n",
|
||
"model.layers.3.self_attn.q_proj\n",
|
||
"52:===========================================\n",
|
||
"model.layers.3.self_attn.k_proj\n",
|
||
"53:===========================================\n",
|
||
"model.layers.3.self_attn.v_proj\n",
|
||
"54:===========================================\n",
|
||
"model.layers.3.self_attn.o_proj\n",
|
||
"55:===========================================\n",
|
||
"model.layers.3.self_attn.q_norm\n",
|
||
"56:===========================================\n",
|
||
"model.layers.3.self_attn.k_norm\n",
|
||
"57:===========================================\n",
|
||
"model.layers.3.mlp\n",
|
||
"58:===========================================\n",
|
||
"model.layers.3.mlp.gate_proj\n",
|
||
"59:===========================================\n",
|
||
"model.layers.3.mlp.up_proj\n",
|
||
"60:===========================================\n",
|
||
"model.layers.3.mlp.down_proj\n",
|
||
"61:===========================================\n",
|
||
"model.layers.3.mlp.act_fn\n",
|
||
"62:===========================================\n",
|
||
"model.layers.3.input_layernorm\n",
|
||
"63:===========================================\n",
|
||
"model.layers.3.post_attention_layernorm\n",
|
||
"64:===========================================\n",
|
||
"model.layers.4\n",
|
||
"65:===========================================\n",
|
||
"model.layers.4.self_attn\n",
|
||
"66:===========================================\n",
|
||
"model.layers.4.self_attn.q_proj\n",
|
||
"67:===========================================\n",
|
||
"model.layers.4.self_attn.k_proj\n",
|
||
"68:===========================================\n",
|
||
"model.layers.4.self_attn.v_proj\n",
|
||
"69:===========================================\n",
|
||
"model.layers.4.self_attn.o_proj\n",
|
||
"70:===========================================\n",
|
||
"model.layers.4.self_attn.q_norm\n",
|
||
"71:===========================================\n",
|
||
"model.layers.4.self_attn.k_norm\n",
|
||
"72:===========================================\n",
|
||
"model.layers.4.mlp\n",
|
||
"73:===========================================\n",
|
||
"model.layers.4.mlp.gate_proj\n",
|
||
"74:===========================================\n",
|
||
"model.layers.4.mlp.up_proj\n",
|
||
"75:===========================================\n",
|
||
"model.layers.4.mlp.down_proj\n",
|
||
"76:===========================================\n",
|
||
"model.layers.4.mlp.act_fn\n",
|
||
"77:===========================================\n",
|
||
"model.layers.4.input_layernorm\n",
|
||
"78:===========================================\n",
|
||
"model.layers.4.post_attention_layernorm\n",
|
||
"79:===========================================\n",
|
||
"model.layers.5\n",
|
||
"80:===========================================\n",
|
||
"model.layers.5.self_attn\n",
|
||
"81:===========================================\n",
|
||
"model.layers.5.self_attn.q_proj\n",
|
||
"82:===========================================\n",
|
||
"model.layers.5.self_attn.k_proj\n",
|
||
"83:===========================================\n",
|
||
"model.layers.5.self_attn.v_proj\n",
|
||
"84:===========================================\n",
|
||
"model.layers.5.self_attn.o_proj\n",
|
||
"85:===========================================\n",
|
||
"model.layers.5.self_attn.q_norm\n",
|
||
"86:===========================================\n",
|
||
"model.layers.5.self_attn.k_norm\n",
|
||
"87:===========================================\n",
|
||
"model.layers.5.mlp\n",
|
||
"88:===========================================\n",
|
||
"model.layers.5.mlp.gate_proj\n",
|
||
"89:===========================================\n",
|
||
"model.layers.5.mlp.up_proj\n",
|
||
"90:===========================================\n",
|
||
"model.layers.5.mlp.down_proj\n",
|
||
"91:===========================================\n",
|
||
"model.layers.5.mlp.act_fn\n",
|
||
"92:===========================================\n",
|
||
"model.layers.5.input_layernorm\n",
|
||
"93:===========================================\n",
|
||
"model.layers.5.post_attention_layernorm\n",
|
||
"94:===========================================\n",
|
||
"model.layers.6\n",
|
||
"95:===========================================\n",
|
||
"model.layers.6.self_attn\n",
|
||
"96:===========================================\n",
|
||
"model.layers.6.self_attn.q_proj\n",
|
||
"97:===========================================\n",
|
||
"model.layers.6.self_attn.k_proj\n",
|
||
"98:===========================================\n",
|
||
"model.layers.6.self_attn.v_proj\n",
|
||
"99:===========================================\n",
|
||
"model.layers.6.self_attn.o_proj\n",
|
||
"100:===========================================\n",
|
||
"model.layers.6.self_attn.q_norm\n",
|
||
"101:===========================================\n",
|
||
"model.layers.6.self_attn.k_norm\n",
|
||
"102:===========================================\n",
|
||
"model.layers.6.mlp\n",
|
||
"103:===========================================\n",
|
||
"model.layers.6.mlp.gate_proj\n",
|
||
"104:===========================================\n",
|
||
"model.layers.6.mlp.up_proj\n",
|
||
"105:===========================================\n",
|
||
"model.layers.6.mlp.down_proj\n",
|
||
"106:===========================================\n",
|
||
"model.layers.6.mlp.act_fn\n",
|
||
"107:===========================================\n",
|
||
"model.layers.6.input_layernorm\n",
|
||
"108:===========================================\n",
|
||
"model.layers.6.post_attention_layernorm\n",
|
||
"109:===========================================\n",
|
||
"model.layers.7\n",
|
||
"110:===========================================\n",
|
||
"model.layers.7.self_attn\n",
|
||
"111:===========================================\n",
|
||
"model.layers.7.self_attn.q_proj\n",
|
||
"112:===========================================\n",
|
||
"model.layers.7.self_attn.k_proj\n",
|
||
"113:===========================================\n",
|
||
"model.layers.7.self_attn.v_proj\n",
|
||
"114:===========================================\n",
|
||
"model.layers.7.self_attn.o_proj\n",
|
||
"115:===========================================\n",
|
||
"model.layers.7.self_attn.q_norm\n",
|
||
"116:===========================================\n",
|
||
"model.layers.7.self_attn.k_norm\n",
|
||
"117:===========================================\n",
|
||
"model.layers.7.mlp\n",
|
||
"118:===========================================\n",
|
||
"model.layers.7.mlp.gate_proj\n",
|
||
"119:===========================================\n",
|
||
"model.layers.7.mlp.up_proj\n",
|
||
"120:===========================================\n",
|
||
"model.layers.7.mlp.down_proj\n",
|
||
"121:===========================================\n",
|
||
"model.layers.7.mlp.act_fn\n",
|
||
"122:===========================================\n",
|
||
"model.layers.7.input_layernorm\n",
|
||
"123:===========================================\n",
|
||
"model.layers.7.post_attention_layernorm\n",
|
||
"124:===========================================\n",
|
||
"model.layers.8\n",
|
||
"125:===========================================\n",
|
||
"model.layers.8.self_attn\n",
|
||
"126:===========================================\n",
|
||
"model.layers.8.self_attn.q_proj\n",
|
||
"127:===========================================\n",
|
||
"model.layers.8.self_attn.k_proj\n",
|
||
"128:===========================================\n",
|
||
"model.layers.8.self_attn.v_proj\n",
|
||
"129:===========================================\n",
|
||
"model.layers.8.self_attn.o_proj\n",
|
||
"130:===========================================\n",
|
||
"model.layers.8.self_attn.q_norm\n",
|
||
"131:===========================================\n",
|
||
"model.layers.8.self_attn.k_norm\n",
|
||
"132:===========================================\n",
|
||
"model.layers.8.mlp\n",
|
||
"133:===========================================\n",
|
||
"model.layers.8.mlp.gate_proj\n",
|
||
"134:===========================================\n",
|
||
"model.layers.8.mlp.up_proj\n",
|
||
"135:===========================================\n",
|
||
"model.layers.8.mlp.down_proj\n",
|
||
"136:===========================================\n",
|
||
"model.layers.8.mlp.act_fn\n",
|
||
"137:===========================================\n",
|
||
"model.layers.8.input_layernorm\n",
|
||
"138:===========================================\n",
|
||
"model.layers.8.post_attention_layernorm\n",
|
||
"139:===========================================\n",
|
||
"model.layers.9\n",
|
||
"140:===========================================\n",
|
||
"model.layers.9.self_attn\n",
|
||
"141:===========================================\n",
|
||
"model.layers.9.self_attn.q_proj\n",
|
||
"142:===========================================\n",
|
||
"model.layers.9.self_attn.k_proj\n",
|
||
"143:===========================================\n",
|
||
"model.layers.9.self_attn.v_proj\n",
|
||
"144:===========================================\n",
|
||
"model.layers.9.self_attn.o_proj\n",
|
||
"145:===========================================\n",
|
||
"model.layers.9.self_attn.q_norm\n",
|
||
"146:===========================================\n",
|
||
"model.layers.9.self_attn.k_norm\n",
|
||
"147:===========================================\n",
|
||
"model.layers.9.mlp\n",
|
||
"148:===========================================\n",
|
||
"model.layers.9.mlp.gate_proj\n",
|
||
"149:===========================================\n",
|
||
"model.layers.9.mlp.up_proj\n",
|
||
"150:===========================================\n",
|
||
"model.layers.9.mlp.down_proj\n",
|
||
"151:===========================================\n",
|
||
"model.layers.9.mlp.act_fn\n",
|
||
"152:===========================================\n",
|
||
"model.layers.9.input_layernorm\n",
|
||
"153:===========================================\n",
|
||
"model.layers.9.post_attention_layernorm\n",
|
||
"154:===========================================\n",
|
||
"model.layers.10\n",
|
||
"155:===========================================\n",
|
||
"model.layers.10.self_attn\n",
|
||
"156:===========================================\n",
|
||
"model.layers.10.self_attn.q_proj\n",
|
||
"157:===========================================\n",
|
||
"model.layers.10.self_attn.k_proj\n",
|
||
"158:===========================================\n",
|
||
"model.layers.10.self_attn.v_proj\n",
|
||
"159:===========================================\n",
|
||
"model.layers.10.self_attn.o_proj\n",
|
||
"160:===========================================\n",
|
||
"model.layers.10.self_attn.q_norm\n",
|
||
"161:===========================================\n",
|
||
"model.layers.10.self_attn.k_norm\n",
|
||
"162:===========================================\n",
|
||
"model.layers.10.mlp\n",
|
||
"163:===========================================\n",
|
||
"model.layers.10.mlp.gate_proj\n",
|
||
"164:===========================================\n",
|
||
"model.layers.10.mlp.up_proj\n",
|
||
"165:===========================================\n",
|
||
"model.layers.10.mlp.down_proj\n",
|
||
"166:===========================================\n",
|
||
"model.layers.10.mlp.act_fn\n",
|
||
"167:===========================================\n",
|
||
"model.layers.10.input_layernorm\n",
|
||
"168:===========================================\n",
|
||
"model.layers.10.post_attention_layernorm\n",
|
||
"169:===========================================\n",
|
||
"model.layers.11\n",
|
||
"170:===========================================\n",
|
||
"model.layers.11.self_attn\n",
|
||
"171:===========================================\n",
|
||
"model.layers.11.self_attn.q_proj\n",
|
||
"172:===========================================\n",
|
||
"model.layers.11.self_attn.k_proj\n",
|
||
"173:===========================================\n",
|
||
"model.layers.11.self_attn.v_proj\n",
|
||
"174:===========================================\n",
|
||
"model.layers.11.self_attn.o_proj\n",
|
||
"175:===========================================\n",
|
||
"model.layers.11.self_attn.q_norm\n",
|
||
"176:===========================================\n",
|
||
"model.layers.11.self_attn.k_norm\n",
|
||
"177:===========================================\n",
|
||
"model.layers.11.mlp\n",
|
||
"178:===========================================\n",
|
||
"model.layers.11.mlp.gate_proj\n",
|
||
"179:===========================================\n",
|
||
"model.layers.11.mlp.up_proj\n",
|
||
"180:===========================================\n",
|
||
"model.layers.11.mlp.down_proj\n",
|
||
"181:===========================================\n",
|
||
"model.layers.11.mlp.act_fn\n",
|
||
"182:===========================================\n",
|
||
"model.layers.11.input_layernorm\n",
|
||
"183:===========================================\n",
|
||
"model.layers.11.post_attention_layernorm\n",
|
||
"184:===========================================\n",
|
||
"model.layers.12\n",
|
||
"185:===========================================\n",
|
||
"model.layers.12.self_attn\n",
|
||
"186:===========================================\n",
|
||
"model.layers.12.self_attn.q_proj\n",
|
||
"187:===========================================\n",
|
||
"model.layers.12.self_attn.k_proj\n",
|
||
"188:===========================================\n",
|
||
"model.layers.12.self_attn.v_proj\n",
|
||
"189:===========================================\n",
|
||
"model.layers.12.self_attn.o_proj\n",
|
||
"190:===========================================\n",
|
||
"model.layers.12.self_attn.q_norm\n",
|
||
"191:===========================================\n",
|
||
"model.layers.12.self_attn.k_norm\n",
|
||
"192:===========================================\n",
|
||
"model.layers.12.mlp\n",
|
||
"193:===========================================\n",
|
||
"model.layers.12.mlp.gate_proj\n",
|
||
"194:===========================================\n",
|
||
"model.layers.12.mlp.up_proj\n",
|
||
"195:===========================================\n",
|
||
"model.layers.12.mlp.down_proj\n",
|
||
"196:===========================================\n",
|
||
"model.layers.12.mlp.act_fn\n",
|
||
"197:===========================================\n",
|
||
"model.layers.12.input_layernorm\n",
|
||
"198:===========================================\n",
|
||
"model.layers.12.post_attention_layernorm\n",
|
||
"199:===========================================\n",
|
||
"model.layers.13\n",
|
||
"200:===========================================\n",
|
||
"model.layers.13.self_attn\n",
|
||
"201:===========================================\n",
|
||
"model.layers.13.self_attn.q_proj\n",
|
||
"202:===========================================\n",
|
||
"model.layers.13.self_attn.k_proj\n",
|
||
"203:===========================================\n",
|
||
"model.layers.13.self_attn.v_proj\n",
|
||
"204:===========================================\n",
|
||
"model.layers.13.self_attn.o_proj\n",
|
||
"205:===========================================\n",
|
||
"model.layers.13.self_attn.q_norm\n",
|
||
"206:===========================================\n",
|
||
"model.layers.13.self_attn.k_norm\n",
|
||
"207:===========================================\n",
|
||
"model.layers.13.mlp\n",
|
||
"208:===========================================\n",
|
||
"model.layers.13.mlp.gate_proj\n",
|
||
"209:===========================================\n",
|
||
"model.layers.13.mlp.up_proj\n",
|
||
"210:===========================================\n",
|
||
"model.layers.13.mlp.down_proj\n",
|
||
"211:===========================================\n",
|
||
"model.layers.13.mlp.act_fn\n",
|
||
"212:===========================================\n",
|
||
"model.layers.13.input_layernorm\n",
|
||
"213:===========================================\n",
|
||
"model.layers.13.post_attention_layernorm\n",
|
||
"214:===========================================\n",
|
||
"model.layers.14\n",
|
||
"215:===========================================\n",
|
||
"model.layers.14.self_attn\n",
|
||
"216:===========================================\n",
|
||
"model.layers.14.self_attn.q_proj\n",
|
||
"217:===========================================\n",
|
||
"model.layers.14.self_attn.k_proj\n",
|
||
"218:===========================================\n",
|
||
"model.layers.14.self_attn.v_proj\n",
|
||
"219:===========================================\n",
|
||
"model.layers.14.self_attn.o_proj\n",
|
||
"220:===========================================\n",
|
||
"model.layers.14.self_attn.q_norm\n",
|
||
"221:===========================================\n",
|
||
"model.layers.14.self_attn.k_norm\n",
|
||
"222:===========================================\n",
|
||
"model.layers.14.mlp\n",
|
||
"223:===========================================\n",
|
||
"model.layers.14.mlp.gate_proj\n",
|
||
"224:===========================================\n",
|
||
"model.layers.14.mlp.up_proj\n",
|
||
"225:===========================================\n",
|
||
"model.layers.14.mlp.down_proj\n",
|
||
"226:===========================================\n",
|
||
"model.layers.14.mlp.act_fn\n",
|
||
"227:===========================================\n",
|
||
"model.layers.14.input_layernorm\n",
|
||
"228:===========================================\n",
|
||
"model.layers.14.post_attention_layernorm\n",
|
||
"229:===========================================\n",
|
||
"model.layers.15\n",
|
||
"230:===========================================\n",
|
||
"model.layers.15.self_attn\n",
|
||
"231:===========================================\n",
|
||
"model.layers.15.self_attn.q_proj\n",
|
||
"232:===========================================\n",
|
||
"model.layers.15.self_attn.k_proj\n",
|
||
"233:===========================================\n",
|
||
"model.layers.15.self_attn.v_proj\n",
|
||
"234:===========================================\n",
|
||
"model.layers.15.self_attn.o_proj\n",
|
||
"235:===========================================\n",
|
||
"model.layers.15.self_attn.q_norm\n",
|
||
"236:===========================================\n",
|
||
"model.layers.15.self_attn.k_norm\n",
|
||
"237:===========================================\n",
|
||
"model.layers.15.mlp\n",
|
||
"238:===========================================\n",
|
||
"model.layers.15.mlp.gate_proj\n",
|
||
"239:===========================================\n",
|
||
"model.layers.15.mlp.up_proj\n",
|
||
"240:===========================================\n",
|
||
"model.layers.15.mlp.down_proj\n",
|
||
"241:===========================================\n",
|
||
"model.layers.15.mlp.act_fn\n",
|
||
"242:===========================================\n",
|
||
"model.layers.15.input_layernorm\n",
|
||
"243:===========================================\n",
|
||
"model.layers.15.post_attention_layernorm\n",
|
||
"244:===========================================\n",
|
||
"model.layers.16\n",
|
||
"245:===========================================\n",
|
||
"model.layers.16.self_attn\n",
|
||
"246:===========================================\n",
|
||
"model.layers.16.self_attn.q_proj\n",
|
||
"247:===========================================\n",
|
||
"model.layers.16.self_attn.k_proj\n",
|
||
"248:===========================================\n",
|
||
"model.layers.16.self_attn.v_proj\n",
|
||
"249:===========================================\n",
|
||
"model.layers.16.self_attn.o_proj\n",
|
||
"250:===========================================\n",
|
||
"model.layers.16.self_attn.q_norm\n",
|
||
"251:===========================================\n",
|
||
"model.layers.16.self_attn.k_norm\n",
|
||
"252:===========================================\n",
|
||
"model.layers.16.mlp\n",
|
||
"253:===========================================\n",
|
||
"model.layers.16.mlp.gate_proj\n",
|
||
"254:===========================================\n",
|
||
"model.layers.16.mlp.up_proj\n",
|
||
"255:===========================================\n",
|
||
"model.layers.16.mlp.down_proj\n",
|
||
"256:===========================================\n",
|
||
"model.layers.16.mlp.act_fn\n",
|
||
"257:===========================================\n",
|
||
"model.layers.16.input_layernorm\n",
|
||
"258:===========================================\n",
|
||
"model.layers.16.post_attention_layernorm\n",
|
||
"259:===========================================\n",
|
||
"model.layers.17\n",
|
||
"260:===========================================\n",
|
||
"model.layers.17.self_attn\n",
|
||
"261:===========================================\n",
|
||
"model.layers.17.self_attn.q_proj\n",
|
||
"262:===========================================\n",
|
||
"model.layers.17.self_attn.k_proj\n",
|
||
"263:===========================================\n",
|
||
"model.layers.17.self_attn.v_proj\n",
|
||
"264:===========================================\n",
|
||
"model.layers.17.self_attn.o_proj\n",
|
||
"265:===========================================\n",
|
||
"model.layers.17.self_attn.q_norm\n",
|
||
"266:===========================================\n",
|
||
"model.layers.17.self_attn.k_norm\n",
|
||
"267:===========================================\n",
|
||
"model.layers.17.mlp\n",
|
||
"268:===========================================\n",
|
||
"model.layers.17.mlp.gate_proj\n",
|
||
"269:===========================================\n",
|
||
"model.layers.17.mlp.up_proj\n",
|
||
"270:===========================================\n",
|
||
"model.layers.17.mlp.down_proj\n",
|
||
"271:===========================================\n",
|
||
"model.layers.17.mlp.act_fn\n",
|
||
"272:===========================================\n",
|
||
"model.layers.17.input_layernorm\n",
|
||
"273:===========================================\n",
|
||
"model.layers.17.post_attention_layernorm\n",
|
||
"274:===========================================\n",
|
||
"model.layers.18\n",
|
||
"275:===========================================\n",
|
||
"model.layers.18.self_attn\n",
|
||
"276:===========================================\n",
|
||
"model.layers.18.self_attn.q_proj\n",
|
||
"277:===========================================\n",
|
||
"model.layers.18.self_attn.k_proj\n",
|
||
"278:===========================================\n",
|
||
"model.layers.18.self_attn.v_proj\n",
|
||
"279:===========================================\n",
|
||
"model.layers.18.self_attn.o_proj\n",
|
||
"280:===========================================\n",
|
||
"model.layers.18.self_attn.q_norm\n",
|
||
"281:===========================================\n",
|
||
"model.layers.18.self_attn.k_norm\n",
|
||
"282:===========================================\n",
|
||
"model.layers.18.mlp\n",
|
||
"283:===========================================\n",
|
||
"model.layers.18.mlp.gate_proj\n",
|
||
"284:===========================================\n",
|
||
"model.layers.18.mlp.up_proj\n",
|
||
"285:===========================================\n",
|
||
"model.layers.18.mlp.down_proj\n",
|
||
"286:===========================================\n",
|
||
"model.layers.18.mlp.act_fn\n",
|
||
"287:===========================================\n",
|
||
"model.layers.18.input_layernorm\n",
|
||
"288:===========================================\n",
|
||
"model.layers.18.post_attention_layernorm\n",
|
||
"289:===========================================\n",
|
||
"model.layers.19\n",
|
||
"290:===========================================\n",
|
||
"model.layers.19.self_attn\n",
|
||
"291:===========================================\n",
|
||
"model.layers.19.self_attn.q_proj\n",
|
||
"292:===========================================\n",
|
||
"model.layers.19.self_attn.k_proj\n",
|
||
"293:===========================================\n",
|
||
"model.layers.19.self_attn.v_proj\n",
|
||
"294:===========================================\n",
|
||
"model.layers.19.self_attn.o_proj\n",
|
||
"295:===========================================\n",
|
||
"model.layers.19.self_attn.q_norm\n",
|
||
"296:===========================================\n",
|
||
"model.layers.19.self_attn.k_norm\n",
|
||
"297:===========================================\n",
|
||
"model.layers.19.mlp\n",
|
||
"298:===========================================\n",
|
||
"model.layers.19.mlp.gate_proj\n",
|
||
"299:===========================================\n",
|
||
"model.layers.19.mlp.up_proj\n",
|
||
"300:===========================================\n",
|
||
"model.layers.19.mlp.down_proj\n",
|
||
"301:===========================================\n",
|
||
"model.layers.19.mlp.act_fn\n",
|
||
"302:===========================================\n",
|
||
"model.layers.19.input_layernorm\n",
|
||
"303:===========================================\n",
|
||
"model.layers.19.post_attention_layernorm\n",
|
||
"304:===========================================\n",
|
||
"model.layers.20\n",
|
||
"305:===========================================\n",
|
||
"model.layers.20.self_attn\n",
|
||
"306:===========================================\n",
|
||
"model.layers.20.self_attn.q_proj\n",
|
||
"307:===========================================\n",
|
||
"model.layers.20.self_attn.k_proj\n",
|
||
"308:===========================================\n",
|
||
"model.layers.20.self_attn.v_proj\n",
|
||
"309:===========================================\n",
|
||
"model.layers.20.self_attn.o_proj\n",
|
||
"310:===========================================\n",
|
||
"model.layers.20.self_attn.q_norm\n",
|
||
"311:===========================================\n",
|
||
"model.layers.20.self_attn.k_norm\n",
|
||
"312:===========================================\n",
|
||
"model.layers.20.mlp\n",
|
||
"313:===========================================\n",
|
||
"model.layers.20.mlp.gate_proj\n",
|
||
"314:===========================================\n",
|
||
"model.layers.20.mlp.up_proj\n",
|
||
"315:===========================================\n",
|
||
"model.layers.20.mlp.down_proj\n",
|
||
"316:===========================================\n",
|
||
"model.layers.20.mlp.act_fn\n",
|
||
"317:===========================================\n",
|
||
"model.layers.20.input_layernorm\n",
|
||
"318:===========================================\n",
|
||
"model.layers.20.post_attention_layernorm\n",
|
||
"319:===========================================\n",
|
||
"model.layers.21\n",
|
||
"320:===========================================\n",
|
||
"model.layers.21.self_attn\n",
|
||
"321:===========================================\n",
|
||
"model.layers.21.self_attn.q_proj\n",
|
||
"322:===========================================\n",
|
||
"model.layers.21.self_attn.k_proj\n",
|
||
"323:===========================================\n",
|
||
"model.layers.21.self_attn.v_proj\n",
|
||
"324:===========================================\n",
|
||
"model.layers.21.self_attn.o_proj\n",
|
||
"325:===========================================\n",
|
||
"model.layers.21.self_attn.q_norm\n",
|
||
"326:===========================================\n",
|
||
"model.layers.21.self_attn.k_norm\n",
|
||
"327:===========================================\n",
|
||
"model.layers.21.mlp\n",
|
||
"328:===========================================\n",
|
||
"model.layers.21.mlp.gate_proj\n",
|
||
"329:===========================================\n",
|
||
"model.layers.21.mlp.up_proj\n",
|
||
"330:===========================================\n",
|
||
"model.layers.21.mlp.down_proj\n",
|
||
"331:===========================================\n",
|
||
"model.layers.21.mlp.act_fn\n",
|
||
"332:===========================================\n",
|
||
"model.layers.21.input_layernorm\n",
|
||
"333:===========================================\n",
|
||
"model.layers.21.post_attention_layernorm\n",
|
||
"334:===========================================\n",
|
||
"model.layers.22\n",
|
||
"335:===========================================\n",
|
||
"model.layers.22.self_attn\n",
|
||
"336:===========================================\n",
|
||
"model.layers.22.self_attn.q_proj\n",
|
||
"337:===========================================\n",
|
||
"model.layers.22.self_attn.k_proj\n",
|
||
"338:===========================================\n",
|
||
"model.layers.22.self_attn.v_proj\n",
|
||
"339:===========================================\n",
|
||
"model.layers.22.self_attn.o_proj\n",
|
||
"340:===========================================\n",
|
||
"model.layers.22.self_attn.q_norm\n",
|
||
"341:===========================================\n",
|
||
"model.layers.22.self_attn.k_norm\n",
|
||
"342:===========================================\n",
|
||
"model.layers.22.mlp\n",
|
||
"343:===========================================\n",
|
||
"model.layers.22.mlp.gate_proj\n",
|
||
"344:===========================================\n",
|
||
"model.layers.22.mlp.up_proj\n",
|
||
"345:===========================================\n",
|
||
"model.layers.22.mlp.down_proj\n",
|
||
"346:===========================================\n",
|
||
"model.layers.22.mlp.act_fn\n",
|
||
"347:===========================================\n",
|
||
"model.layers.22.input_layernorm\n",
|
||
"348:===========================================\n",
|
||
"model.layers.22.post_attention_layernorm\n",
|
||
"349:===========================================\n",
|
||
"model.layers.23\n",
|
||
"350:===========================================\n",
|
||
"model.layers.23.self_attn\n",
|
||
"351:===========================================\n",
|
||
"model.layers.23.self_attn.q_proj\n",
|
||
"352:===========================================\n",
|
||
"model.layers.23.self_attn.k_proj\n",
|
||
"353:===========================================\n",
|
||
"model.layers.23.self_attn.v_proj\n",
|
||
"354:===========================================\n",
|
||
"model.layers.23.self_attn.o_proj\n",
|
||
"355:===========================================\n",
|
||
"model.layers.23.self_attn.q_norm\n",
|
||
"356:===========================================\n",
|
||
"model.layers.23.self_attn.k_norm\n",
|
||
"357:===========================================\n",
|
||
"model.layers.23.mlp\n",
|
||
"358:===========================================\n",
|
||
"model.layers.23.mlp.gate_proj\n",
|
||
"359:===========================================\n",
|
||
"model.layers.23.mlp.up_proj\n",
|
||
"360:===========================================\n",
|
||
"model.layers.23.mlp.down_proj\n",
|
||
"361:===========================================\n",
|
||
"model.layers.23.mlp.act_fn\n",
|
||
"362:===========================================\n",
|
||
"model.layers.23.input_layernorm\n",
|
||
"363:===========================================\n",
|
||
"model.layers.23.post_attention_layernorm\n",
|
||
"364:===========================================\n",
|
||
"model.layers.24\n",
|
||
"365:===========================================\n",
|
||
"model.layers.24.self_attn\n",
|
||
"366:===========================================\n",
|
||
"model.layers.24.self_attn.q_proj\n",
|
||
"367:===========================================\n",
|
||
"model.layers.24.self_attn.k_proj\n",
|
||
"368:===========================================\n",
|
||
"model.layers.24.self_attn.v_proj\n",
|
||
"369:===========================================\n",
|
||
"model.layers.24.self_attn.o_proj\n",
|
||
"370:===========================================\n",
|
||
"model.layers.24.self_attn.q_norm\n",
|
||
"371:===========================================\n",
|
||
"model.layers.24.self_attn.k_norm\n",
|
||
"372:===========================================\n",
|
||
"model.layers.24.mlp\n",
|
||
"373:===========================================\n",
|
||
"model.layers.24.mlp.gate_proj\n",
|
||
"374:===========================================\n",
|
||
"model.layers.24.mlp.up_proj\n",
|
||
"375:===========================================\n",
|
||
"model.layers.24.mlp.down_proj\n",
|
||
"376:===========================================\n",
|
||
"model.layers.24.mlp.act_fn\n",
|
||
"377:===========================================\n",
|
||
"model.layers.24.input_layernorm\n",
|
||
"378:===========================================\n",
|
||
"model.layers.24.post_attention_layernorm\n",
|
||
"379:===========================================\n",
|
||
"model.layers.25\n",
|
||
"380:===========================================\n",
|
||
"model.layers.25.self_attn\n",
|
||
"381:===========================================\n",
|
||
"model.layers.25.self_attn.q_proj\n",
|
||
"382:===========================================\n",
|
||
"model.layers.25.self_attn.k_proj\n",
|
||
"383:===========================================\n",
|
||
"model.layers.25.self_attn.v_proj\n",
|
||
"384:===========================================\n",
|
||
"model.layers.25.self_attn.o_proj\n",
|
||
"385:===========================================\n",
|
||
"model.layers.25.self_attn.q_norm\n",
|
||
"386:===========================================\n",
|
||
"model.layers.25.self_attn.k_norm\n",
|
||
"387:===========================================\n",
|
||
"model.layers.25.mlp\n",
|
||
"388:===========================================\n",
|
||
"model.layers.25.mlp.gate_proj\n",
|
||
"389:===========================================\n",
|
||
"model.layers.25.mlp.up_proj\n",
|
||
"390:===========================================\n",
|
||
"model.layers.25.mlp.down_proj\n",
|
||
"391:===========================================\n",
|
||
"model.layers.25.mlp.act_fn\n",
|
||
"392:===========================================\n",
|
||
"model.layers.25.input_layernorm\n",
|
||
"393:===========================================\n",
|
||
"model.layers.25.post_attention_layernorm\n",
|
||
"394:===========================================\n",
|
||
"model.layers.26\n",
|
||
"395:===========================================\n",
|
||
"model.layers.26.self_attn\n",
|
||
"396:===========================================\n",
|
||
"model.layers.26.self_attn.q_proj\n",
|
||
"397:===========================================\n",
|
||
"model.layers.26.self_attn.k_proj\n",
|
||
"398:===========================================\n",
|
||
"model.layers.26.self_attn.v_proj\n",
|
||
"399:===========================================\n",
|
||
"model.layers.26.self_attn.o_proj\n",
|
||
"400:===========================================\n",
|
||
"model.layers.26.self_attn.q_norm\n",
|
||
"401:===========================================\n",
|
||
"model.layers.26.self_attn.k_norm\n",
|
||
"402:===========================================\n",
|
||
"model.layers.26.mlp\n",
|
||
"403:===========================================\n",
|
||
"model.layers.26.mlp.gate_proj\n",
|
||
"404:===========================================\n",
|
||
"model.layers.26.mlp.up_proj\n",
|
||
"405:===========================================\n",
|
||
"model.layers.26.mlp.down_proj\n",
|
||
"406:===========================================\n",
|
||
"model.layers.26.mlp.act_fn\n",
|
||
"407:===========================================\n",
|
||
"model.layers.26.input_layernorm\n",
|
||
"408:===========================================\n",
|
||
"model.layers.26.post_attention_layernorm\n",
|
||
"409:===========================================\n",
|
||
"model.layers.27\n",
|
||
"410:===========================================\n",
|
||
"model.layers.27.self_attn\n",
|
||
"411:===========================================\n",
|
||
"model.layers.27.self_attn.q_proj\n",
|
||
"412:===========================================\n",
|
||
"model.layers.27.self_attn.k_proj\n",
|
||
"413:===========================================\n",
|
||
"model.layers.27.self_attn.v_proj\n",
|
||
"414:===========================================\n",
|
||
"model.layers.27.self_attn.o_proj\n",
|
||
"415:===========================================\n",
|
||
"model.layers.27.self_attn.q_norm\n",
|
||
"416:===========================================\n",
|
||
"model.layers.27.self_attn.k_norm\n",
|
||
"417:===========================================\n",
|
||
"model.layers.27.mlp\n",
|
||
"418:===========================================\n",
|
||
"model.layers.27.mlp.gate_proj\n",
|
||
"419:===========================================\n",
|
||
"model.layers.27.mlp.up_proj\n",
|
||
"420:===========================================\n",
|
||
"model.layers.27.mlp.down_proj\n",
|
||
"421:===========================================\n",
|
||
"model.layers.27.mlp.act_fn\n",
|
||
"422:===========================================\n",
|
||
"model.layers.27.input_layernorm\n",
|
||
"423:===========================================\n",
|
||
"model.layers.27.post_attention_layernorm\n",
|
||
"424:===========================================\n",
|
||
"model.layers.28\n",
|
||
"425:===========================================\n",
|
||
"model.layers.28.self_attn\n",
|
||
"426:===========================================\n",
|
||
"model.layers.28.self_attn.q_proj\n",
|
||
"427:===========================================\n",
|
||
"model.layers.28.self_attn.k_proj\n",
|
||
"428:===========================================\n",
|
||
"model.layers.28.self_attn.v_proj\n",
|
||
"429:===========================================\n",
|
||
"model.layers.28.self_attn.o_proj\n",
|
||
"430:===========================================\n",
|
||
"model.layers.28.self_attn.q_norm\n",
|
||
"431:===========================================\n",
|
||
"model.layers.28.self_attn.k_norm\n",
|
||
"432:===========================================\n",
|
||
"model.layers.28.mlp\n",
|
||
"433:===========================================\n",
|
||
"model.layers.28.mlp.gate_proj\n",
|
||
"434:===========================================\n",
|
||
"model.layers.28.mlp.up_proj\n",
|
||
"435:===========================================\n",
|
||
"model.layers.28.mlp.down_proj\n",
|
||
"436:===========================================\n",
|
||
"model.layers.28.mlp.act_fn\n",
|
||
"437:===========================================\n",
|
||
"model.layers.28.input_layernorm\n",
|
||
"438:===========================================\n",
|
||
"model.layers.28.post_attention_layernorm\n",
|
||
"439:===========================================\n",
|
||
"model.layers.29\n",
|
||
"440:===========================================\n",
|
||
"model.layers.29.self_attn\n",
|
||
"441:===========================================\n",
|
||
"model.layers.29.self_attn.q_proj\n",
|
||
"442:===========================================\n",
|
||
"model.layers.29.self_attn.k_proj\n",
|
||
"443:===========================================\n",
|
||
"model.layers.29.self_attn.v_proj\n",
|
||
"444:===========================================\n",
|
||
"model.layers.29.self_attn.o_proj\n",
|
||
"445:===========================================\n",
|
||
"model.layers.29.self_attn.q_norm\n",
|
||
"446:===========================================\n",
|
||
"model.layers.29.self_attn.k_norm\n",
|
||
"447:===========================================\n",
|
||
"model.layers.29.mlp\n",
|
||
"448:===========================================\n",
|
||
"model.layers.29.mlp.gate_proj\n",
|
||
"449:===========================================\n",
|
||
"model.layers.29.mlp.up_proj\n",
|
||
"450:===========================================\n",
|
||
"model.layers.29.mlp.down_proj\n",
|
||
"451:===========================================\n",
|
||
"model.layers.29.mlp.act_fn\n",
|
||
"452:===========================================\n",
|
||
"model.layers.29.input_layernorm\n",
|
||
"453:===========================================\n",
|
||
"model.layers.29.post_attention_layernorm\n",
|
||
"454:===========================================\n",
|
||
"model.layers.30\n",
|
||
"455:===========================================\n",
|
||
"model.layers.30.self_attn\n",
|
||
"456:===========================================\n",
|
||
"model.layers.30.self_attn.q_proj\n",
|
||
"457:===========================================\n",
|
||
"model.layers.30.self_attn.k_proj\n",
|
||
"458:===========================================\n",
|
||
"model.layers.30.self_attn.v_proj\n",
|
||
"459:===========================================\n",
|
||
"model.layers.30.self_attn.o_proj\n",
|
||
"460:===========================================\n",
|
||
"model.layers.30.self_attn.q_norm\n",
|
||
"461:===========================================\n",
|
||
"model.layers.30.self_attn.k_norm\n",
|
||
"462:===========================================\n",
|
||
"model.layers.30.mlp\n",
|
||
"463:===========================================\n",
|
||
"model.layers.30.mlp.gate_proj\n",
|
||
"464:===========================================\n",
|
||
"model.layers.30.mlp.up_proj\n",
|
||
"465:===========================================\n",
|
||
"model.layers.30.mlp.down_proj\n",
|
||
"466:===========================================\n",
|
||
"model.layers.30.mlp.act_fn\n",
|
||
"467:===========================================\n",
|
||
"model.layers.30.input_layernorm\n",
|
||
"468:===========================================\n",
|
||
"model.layers.30.post_attention_layernorm\n",
|
||
"469:===========================================\n",
|
||
"model.layers.31\n",
|
||
"470:===========================================\n",
|
||
"model.layers.31.self_attn\n",
|
||
"471:===========================================\n",
|
||
"model.layers.31.self_attn.q_proj\n",
|
||
"472:===========================================\n",
|
||
"model.layers.31.self_attn.k_proj\n",
|
||
"473:===========================================\n",
|
||
"model.layers.31.self_attn.v_proj\n",
|
||
"474:===========================================\n",
|
||
"model.layers.31.self_attn.o_proj\n",
|
||
"475:===========================================\n",
|
||
"model.layers.31.self_attn.q_norm\n",
|
||
"476:===========================================\n",
|
||
"model.layers.31.self_attn.k_norm\n",
|
||
"477:===========================================\n",
|
||
"model.layers.31.mlp\n",
|
||
"478:===========================================\n",
|
||
"model.layers.31.mlp.gate_proj\n",
|
||
"479:===========================================\n",
|
||
"model.layers.31.mlp.up_proj\n",
|
||
"480:===========================================\n",
|
||
"model.layers.31.mlp.down_proj\n",
|
||
"481:===========================================\n",
|
||
"model.layers.31.mlp.act_fn\n",
|
||
"482:===========================================\n",
|
||
"model.layers.31.input_layernorm\n",
|
||
"483:===========================================\n",
|
||
"model.layers.31.post_attention_layernorm\n",
|
||
"484:===========================================\n",
|
||
"model.layers.32\n",
|
||
"485:===========================================\n",
|
||
"model.layers.32.self_attn\n",
|
||
"486:===========================================\n",
|
||
"model.layers.32.self_attn.q_proj\n",
|
||
"487:===========================================\n",
|
||
"model.layers.32.self_attn.k_proj\n",
|
||
"488:===========================================\n",
|
||
"model.layers.32.self_attn.v_proj\n",
|
||
"489:===========================================\n",
|
||
"model.layers.32.self_attn.o_proj\n",
|
||
"490:===========================================\n",
|
||
"model.layers.32.self_attn.q_norm\n",
|
||
"491:===========================================\n",
|
||
"model.layers.32.self_attn.k_norm\n",
|
||
"492:===========================================\n",
|
||
"model.layers.32.mlp\n",
|
||
"493:===========================================\n",
|
||
"model.layers.32.mlp.gate_proj\n",
|
||
"494:===========================================\n",
|
||
"model.layers.32.mlp.up_proj\n",
|
||
"495:===========================================\n",
|
||
"model.layers.32.mlp.down_proj\n",
|
||
"496:===========================================\n",
|
||
"model.layers.32.mlp.act_fn\n",
|
||
"497:===========================================\n",
|
||
"model.layers.32.input_layernorm\n",
|
||
"498:===========================================\n",
|
||
"model.layers.32.post_attention_layernorm\n",
|
||
"499:===========================================\n",
|
||
"model.layers.33\n",
|
||
"500:===========================================\n",
|
||
"model.layers.33.self_attn\n",
|
||
"501:===========================================\n",
|
||
"model.layers.33.self_attn.q_proj\n",
|
||
"502:===========================================\n",
|
||
"model.layers.33.self_attn.k_proj\n",
|
||
"503:===========================================\n",
|
||
"model.layers.33.self_attn.v_proj\n",
|
||
"504:===========================================\n",
|
||
"model.layers.33.self_attn.o_proj\n",
|
||
"505:===========================================\n",
|
||
"model.layers.33.self_attn.q_norm\n",
|
||
"506:===========================================\n",
|
||
"model.layers.33.self_attn.k_norm\n",
|
||
"507:===========================================\n",
|
||
"model.layers.33.mlp\n",
|
||
"508:===========================================\n",
|
||
"model.layers.33.mlp.gate_proj\n",
|
||
"509:===========================================\n",
|
||
"model.layers.33.mlp.up_proj\n",
|
||
"510:===========================================\n",
|
||
"model.layers.33.mlp.down_proj\n",
|
||
"511:===========================================\n",
|
||
"model.layers.33.mlp.act_fn\n",
|
||
"512:===========================================\n",
|
||
"model.layers.33.input_layernorm\n",
|
||
"513:===========================================\n",
|
||
"model.layers.33.post_attention_layernorm\n",
|
||
"514:===========================================\n",
|
||
"model.layers.34\n",
|
||
"515:===========================================\n",
|
||
"model.layers.34.self_attn\n",
|
||
"516:===========================================\n",
|
||
"model.layers.34.self_attn.q_proj\n",
|
||
"517:===========================================\n",
|
||
"model.layers.34.self_attn.k_proj\n",
|
||
"518:===========================================\n",
|
||
"model.layers.34.self_attn.v_proj\n",
|
||
"519:===========================================\n",
|
||
"model.layers.34.self_attn.o_proj\n",
|
||
"520:===========================================\n",
|
||
"model.layers.34.self_attn.q_norm\n",
|
||
"521:===========================================\n",
|
||
"model.layers.34.self_attn.k_norm\n",
|
||
"522:===========================================\n",
|
||
"model.layers.34.mlp\n",
|
||
"523:===========================================\n",
|
||
"model.layers.34.mlp.gate_proj\n",
|
||
"524:===========================================\n",
|
||
"model.layers.34.mlp.up_proj\n",
|
||
"525:===========================================\n",
|
||
"model.layers.34.mlp.down_proj\n",
|
||
"526:===========================================\n",
|
||
"model.layers.34.mlp.act_fn\n",
|
||
"527:===========================================\n",
|
||
"model.layers.34.input_layernorm\n",
|
||
"528:===========================================\n",
|
||
"model.layers.34.post_attention_layernorm\n",
|
||
"529:===========================================\n",
|
||
"model.layers.35\n",
|
||
"530:===========================================\n",
|
||
"model.layers.35.self_attn\n",
|
||
"531:===========================================\n",
|
||
"model.layers.35.self_attn.q_proj\n",
|
||
"532:===========================================\n",
|
||
"model.layers.35.self_attn.k_proj\n",
|
||
"533:===========================================\n",
|
||
"model.layers.35.self_attn.v_proj\n",
|
||
"534:===========================================\n",
|
||
"model.layers.35.self_attn.o_proj\n",
|
||
"535:===========================================\n",
|
||
"model.layers.35.self_attn.q_norm\n",
|
||
"536:===========================================\n",
|
||
"model.layers.35.self_attn.k_norm\n",
|
||
"537:===========================================\n",
|
||
"model.layers.35.mlp\n",
|
||
"538:===========================================\n",
|
||
"model.layers.35.mlp.gate_proj\n",
|
||
"539:===========================================\n",
|
||
"model.layers.35.mlp.up_proj\n",
|
||
"540:===========================================\n",
|
||
"model.layers.35.mlp.down_proj\n",
|
||
"541:===========================================\n",
|
||
"model.layers.35.mlp.act_fn\n",
|
||
"542:===========================================\n",
|
||
"model.layers.35.input_layernorm\n",
|
||
"543:===========================================\n",
|
||
"model.layers.35.post_attention_layernorm\n",
|
||
"544:===========================================\n",
|
||
"model.norm\n",
|
||
"545:===========================================\n",
|
||
"model.rotary_emb\n",
|
||
"546:===========================================\n",
|
||
"lm_head\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"for i,m in enumerate(model.named_modules()):\n",
|
||
" print(str(i) + \":===========================================\")\n",
|
||
" print(m[0])"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "b2250da5-79a6-4a5d-8124-913e3bc3b33a",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "conda_focallora4",
|
||
"language": "python",
|
||
"name": "conda_focallora4"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.12.12"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|