{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "94f47e49-b4d8-4816-90e6-441baa97223a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda/envs/focallora4/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🥷 Injecting Spy Interceptor into Flash Attention 2...\n",
      "✅ Spy Interceptor installed successfully!\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import os\n",
    "import json\n",
    "import csv\n",
    "import argparse\n",
    "import math\n",
    "import random\n",
    "import re\n",
    "from typing import List, Tuple, Dict\n",
    "\n",
    "import torch\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from evallib import quick_eval_mmlu, quick_eval_asr_util\n",
    "from transformers import (\n",
    "    AutoConfig, AutoTokenizer, AutoModelForCausalLM,\n",
    "    BitsAndBytesConfig, get_linear_schedule_with_warmup,\n",
    ")\n",
    "from transformers.models.llama.modeling_llama import ALL_ATTENTION_FUNCTIONS\n",
    "from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training, PeftModel\n",
    "from lib.tokenize_data_mask import apply_chat_tokenize_with_strip_and_mark\n",
    "from lib.printcolor import print_tok_color\n",
    "# -----------------------------------------------------------------------------\n",
    "# 1. Flash Attention Spy Interceptor (核心機制)\n",
    "# -----------------------------------------------------------------------------\n",
    "print(\"🥷 Injecting Spy Interceptor into Flash Attention 2...\")\n",
    "\n",
    "if \"flash_attention_2\" not in ALL_ATTENTION_FUNCTIONS:\n",
    "    raise RuntimeError(\"Current environment does not support Flash Attention 2!\")\n",
    "\n",
    "orig_flash_attn = ALL_ATTENTION_FUNCTIONS[\"flash_attention_2\"]\n",
    "\n",
    "def wrapped_flash_attn(module, query, key, value, attention_mask, scaling, **kwargs):\n",
    "    \"\"\"\n",
    "    攔截 Flash Attention，並在 Breakpoint 驗證手算結果與原始 FA2 結果的一致性。\n",
    "    \"\"\"\n",
    "    # -------------------------------------------------------------------------\n",
    "    # [TASK A] 間諜行動：偷算 Last Token Weight\n",
    "    # -------------------------------------------------------------------------\n",
    "    # 1. 切片 (只看最後一個 Token)\n",
    "    q_last = query[..., -1:, :] \n",
    "    \n",
    "    # 2. 處理 GQA (手動 Expand Key 和 Value)\n",
    "    num_heads = query.shape[1]\n",
    "    num_kv_heads = key.shape[1]\n",
    "    \n",
    "    # 用於後面計算 Manual Output\n",
    "    k_expanded = key\n",
    "    v_expanded = value \n",
    "\n",
    "    if num_heads != num_kv_heads:\n",
    "        n_rep = num_heads // num_kv_heads\n",
    "        k_expanded = key.repeat_interleave(n_rep, dim=1)\n",
    "        v_expanded = value.repeat_interleave(n_rep, dim=1) # Value 也要 Expand 才能乘\n",
    "    \n",
    "    # 3. 手動計算 Tiny Attention Weight [Batch, Heads, 1, Seq]\n",
    "    # Q: [B, H, 1, D], K: [B, H, S, D] -> QK^T: [B, H, 1, S]\n",
    "    attn_weights_tiny = torch.matmul(q_last, k_expanded.transpose(2, 3)) * scaling\n",
    "    \n",
    "    # 4. 處理 Mask\n",
    "    if attention_mask is not None:\n",
    "        if attention_mask.dim() == 4:\n",
    "            if attention_mask.size(2) > 1:\n",
    "                mask_slice = attention_mask[..., -1:, :]\n",
    "                attn_weights_tiny = attn_weights_tiny + mask_slice\n",
    "            else:\n",
    "                attn_weights_tiny = attn_weights_tiny + attention_mask\n",
    "        elif attention_mask.dim() == 2:\n",
    "            mask_expanded = attention_mask[:, None, None, :]\n",
    "            if attention_mask.dtype == torch.bool or (attention_mask.max() <= 1.0 and attention_mask.min() >= 0.0):\n",
    "                min_dtype = torch.finfo(attn_weights_tiny.dtype).min\n",
    "                attn_weights_tiny = attn_weights_tiny.masked_fill(mask_expanded == 0, min_dtype)\n",
    "            else:\n",
    "                attn_weights_tiny = attn_weights_tiny + mask_expanded\n",
    "                \n",
    "    # 5. Softmax (得到機率分佈 P)\n",
    "    # 注意：這裡轉成 float32 做 softmax 以求精確，實際 FA2 內部可能是 fp16/bf16\n",
    "    attn_probs = torch.nn.functional.softmax(attn_weights_tiny, dim=-1, dtype=torch.float32).to(query.dtype)\n",
    "\n",
    "    # 填入 Config (原本的邏輯)\n",
    "    retrieve_map = getattr(module.config, \"retrieve_attn_map\", None)\n",
    "    if retrieve_map is not None and hasattr(module, \"layer_idx\") and module.layer_idx in retrieve_map:\n",
    "        target_heads_dict = retrieve_map[module.layer_idx]\n",
    "        for h_idx in target_heads_dict.keys():\n",
    "            # 存入 config 用於 Loss 計算\n",
    "            target_heads_dict[h_idx] = attn_probs[:, h_idx, :, :]\n",
    "\n",
    "    # -------------------------------------------------------------------------\n",
    "    # [DEBUG] 驗證環節：手算 Output vs Flash Attention Output\n",
    "    # -------------------------------------------------------------------------\n",
    "    \n",
    "    # 1. 執行原始 Flash Attention\n",
    "    orig_result = orig_flash_attn(module, query, key, value, attention_mask, scaling=scaling, **kwargs)\n",
    "    debug=False\n",
    "    if debug:\n",
    "        orig_tensor = orig_result[0]\n",
    "        # 2. 手動計算 Output ( O = Attention_Probs * V )\n",
    "        # attn_probs: [B, H, 1, S]\n",
    "        # v_expanded: [B, H, S, D]\n",
    "        # manual_out: [B, H, 1, D]\n",
    "        manual_out = torch.matmul(attn_probs, v_expanded)\n",
    "        \n",
    "        # 3. 對齊形狀以便比較\n",
    "        # FA2 output 通常是 [Batch, Seq, Heads, Dim]\n",
    "        # 我們取最後一個 token: [Batch, 1, Heads, Dim]\n",
    "        orig_out_last = orig_tensor[:, -1:, :, :]\n",
    "        \n",
    "        # 手算結果原本是 [Batch, Heads, 1, Dim]，轉置成 [Batch, 1, Heads, Dim]\n",
    "        manual_out_check = manual_out.transpose(1, 2)\n",
    "    \n",
    "        # 4. 計算誤差\n",
    "        diff = (orig_out_last - manual_out_check).abs()\n",
    "        max_diff = diff.max().item()\n",
    "        mean_diff = diff.mean().item()\n",
    "        \n",
    "        print(f\"\\n🕵️ [Layer {getattr(module, 'layer_idx', '?')}] Validation:\")\n",
    "        print(f\"   Max Diff: {max_diff:.8f}\")\n",
    "        print(f\"   Mean Diff: {mean_diff:.8f}\")\n",
    "        \n",
    "        # 如果誤差過大 (例如 > 1e-3)，自動暫停檢查\n",
    "        # 注意：BF16 下 FlashAttention 和標準 Matmul 會有一定精度差異是正常的\n",
    "        if max_diff > 1e-2: \n",
    "            print(\"⚠️ Warning: Large difference detected!\")\n",
    "        \n",
    "        # 呼叫 breakpoint 讓你進去檢查\n",
    "        # 在 pdb 輸入: \n",
    "        #   p manual_out_check[0,0,0,:5] \n",
    "        #   p orig_out_last[0,0,0,:5]\n",
    "        # 來對比數值\n",
    "        breakpoint() \n",
    "\n",
    "    return orig_result\n",
    "\n",
    "# 替換全局函數\n",
    "ALL_ATTENTION_FUNCTIONS[\"flash_attention_2\"] = wrapped_flash_attn\n",
    "print(\"✅ Spy Interceptor installed successfully!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7e2904a3-3abf-44b0-9d78-43710a3c4d62",
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED = 42\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "\n",
    "def get_lora_targets(model, layers: List[int]) -> List[str]:\n",
    "    mtype = (getattr(model.config, \"model_type\", \"\") or \"\").lower()\n",
    "    if \"llama\" in mtype or \"mistral\" in mtype:\n",
    "        return [f\"model.layers.{i}.self_attn.{p}\" for i in layers for p in (\"q_proj\", \"k_proj\")]\n",
    "    # Fallback / Other architectures\n",
    "    cand = []\n",
    "    for name, _ in model.named_modules():\n",
    "        if any(f\".{i}.\" in name for i in layers) and name.split(\".\")[-1] in {\"q_proj\", \"k_proj\"}:\n",
    "            cand.append(name)\n",
    "    return cand\n",
    "\n",
    "def load_model(model_path: str):\n",
    "    cfg = AutoConfig.from_pretrained(model_path, trust_remote_code=True)\n",
    "    tok = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=True,padding_side=\"left\")\n",
    "    \n",
    "    if tok.pad_token_id is None:\n",
    "        tok.pad_token = tok.eos_token\n",
    "        tok.pad_token_id = tok.eos_token_id\n",
    "    \n",
    "    bnb_cfg = BitsAndBytesConfig(\n",
    "        load_in_4bit=True,\n",
    "        bnb_4bit_compute_dtype=torch.float16,\n",
    "        bnb_4bit_use_double_quant=True,\n",
    "        bnb_4bit_quant_type=\"nf4\",\n",
    "    )\n",
    "    model = AutoModelForCausalLM.from_pretrained(\n",
    "        model_path,\n",
    "        config=cfg,\n",
    "        quantization_config=bnb_cfg,\n",
    "        device_map=\"auto\",\n",
    "        trust_remote_code=True,\n",
    "        attn_implementation=\"flash_attention_2\",\n",
    "    )\n",
    "    return model, tok, tok # use same tokenizer for simplicity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "28290215-6860-4220-a02b-35e882da579a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_target_structure(heads_file: str, topk_spec: str) -> Dict[int, List[int]]:\n",
    "    \"\"\"Load heads from file and parse into {layer: [heads]} structure.\"\"\"\n",
    "    if not os.path.exists(heads_file):\n",
    "        raise FileNotFoundError(f\"Heads file not found: {heads_file}\")\n",
    "    \n",
    "    with open(heads_file, \"r\") as f:\n",
    "        ihead_list = json.load(f)\n",
    "        if type(ihead_list[0]) == str:\n",
    "            all_heads = [(str(tag), 0.0) for tag in ihead_list]\n",
    "        else:\n",
    "            all_heads = ihead_list\n",
    "    \n",
    "    # Filter Top-K\n",
    "    count = len(all_heads)\n",
    "    if topk_spec.endswith(\"p\"):\n",
    "        count = max(1, math.ceil(float(topk_spec[:-1]) / 100.0 * len(all_heads)))\n",
    "    else:\n",
    "        count = int(topk_spec)\n",
    "    \n",
    "    target_heads = all_heads[:min(count, len(all_heads))]\n",
    "     \n",
    "    # Parse into structure\n",
    "    structure = {}\n",
    "    for tag, _ in target_heads:\n",
    "        try:\n",
    "            # tag format: \"L22_H7\"\n",
    "            parts = tag.split('_')\n",
    "            l = int(parts[0][1:])\n",
    "            h = int(parts[1][1:])\n",
    "            structure.setdefault(l, []).append(h)\n",
    "        except Exception:\n",
    "            continue\n",
    "            \n",
    "    print(f\"🎯 Selected {len(target_heads)} heads from {topk_spec}.\")\n",
    "    return structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "211437db-b128-4d9c-8251-d38b43e3bb35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🎯 Selected 223 heads from 21.75p.\n"
     ]
    }
   ],
   "source": [
    "model_path = \"/data/local/models/Qwen3-8B\"\n",
    "head_path=\"/data/local/hujk/ISH/head_scoring/qwen3-8b_sep/heads_sorted/all_roc_inst_0.1.json\"\n",
    "\n",
    "model_path = \"/data/local/models/Llama-3.1-8B-Instruct\"\n",
    "head_path=\"/data/local/hujk/ISH/head_scoring/llama31-8b_sep/heads_sorted/all_roc_inst_0.1.json\"\n",
    "target_heads = get_target_structure(head_path,\"21.75p\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "44d6094d-74e8-47a1-bac7-fa012b90ede7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{13: [7,\n",
       "  14,\n",
       "  8,\n",
       "  13,\n",
       "  29,\n",
       "  22,\n",
       "  20,\n",
       "  10,\n",
       "  6,\n",
       "  9,\n",
       "  19,\n",
       "  2,\n",
       "  24,\n",
       "  28,\n",
       "  26,\n",
       "  16,\n",
       "  0,\n",
       "  25,\n",
       "  12,\n",
       "  23,\n",
       "  17],\n",
       " 16: [3, 16, 4, 22, 14, 6, 17, 27, 24, 19, 15, 12, 5, 25],\n",
       " 9: [24, 26, 31, 27, 22, 20, 13, 12, 29, 28, 30, 18, 1, 9, 2, 23],\n",
       " 15: [23, 12, 14, 21, 27, 15, 5, 10, 29, 0, 13, 22, 3, 20, 24, 16, 6, 8],\n",
       " 17: [31, 20, 30, 25, 22, 21, 7, 27, 14, 4, 28, 13, 5],\n",
       " 14: [29, 7, 13, 3, 6, 23, 0, 28, 18, 1, 24, 10, 12, 14, 2, 15, 21, 27],\n",
       " 5: [29, 4, 15, 30, 31, 3, 16, 17, 25],\n",
       " 12: [9, 24, 26, 4, 30, 15, 31, 16, 8, 21, 27, 25, 11, 10, 22, 28, 0, 23],\n",
       " 7: [14, 9, 28, 31, 11, 10, 17, 8, 29, 18, 24, 12, 23],\n",
       " 8: [25, 4, 15, 31, 13, 24, 19, 30, 27, 20, 8],\n",
       " 18: [25, 29, 11, 10, 28, 24, 27, 19, 17, 12, 5, 18, 0, 2, 9],\n",
       " 6: [7, 29, 22, 30, 20, 13, 0, 4],\n",
       " 10: [11, 20, 24, 19, 9, 0, 16, 30, 18, 1, 22, 25, 21, 13, 17, 23, 10],\n",
       " 19: [21, 20, 3, 22],\n",
       " 11: [23, 25, 9, 26, 21, 24, 10, 27, 11, 6],\n",
       " 20: [30, 23, 0, 9, 21, 24, 15],\n",
       " 22: [9, 4, 17],\n",
       " 4: [15, 31, 4],\n",
       " 26: [2],\n",
       " 21: [8],\n",
       " 3: [30, 28],\n",
       " 23: [17]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_heads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f1bbf423-65a8-401f-ad5a-8a644bc53952",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading checkpoint shards: 100% 4/4 [00:12<00:00,  3.21s/it]\n"
     ]
    }
   ],
   "source": [
    "model, tok, tok2 = load_model(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "66dda564-e757-4784-9a72-48e2add45151",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lora_targets(model, layers: List[int]) -> List[str]:\n",
    "    mtype = (getattr(model.config, \"model_type\", \"\") or \"\").lower()\n",
    "    if \"llama\" in mtype or \"mistral\" in mtype:\n",
    "        return [f\"model.layers.{i}.self_attn.{p}\" for i in layers for p in (\"q_proj\", \"k_proj\")]\n",
    "    # Fallback / Other architectures\n",
    "    cand = []\n",
    "    for name, _ in model.named_modules():\n",
    "        if any(f\".{i}.\" in name for i in layers) and name.split(\".\")[-1] in {\"q_proj\", \"k_proj\"}:\n",
    "            cand.append(name)\n",
    "    return cand"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f4a9304c-0e5c-4c8d-b270-0cbc4f20e3bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "ls =get_lora_targets(model,target_heads)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "27374e33-9478-489b-a4a6-14a51644a445",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['model.layers.10.self_attn.k_proj',\n",
       " 'model.layers.10.self_attn.q_proj',\n",
       " 'model.layers.11.self_attn.k_proj',\n",
       " 'model.layers.11.self_attn.q_proj',\n",
       " 'model.layers.12.self_attn.k_proj',\n",
       " 'model.layers.12.self_attn.q_proj',\n",
       " 'model.layers.13.self_attn.k_proj',\n",
       " 'model.layers.13.self_attn.q_proj',\n",
       " 'model.layers.14.self_attn.k_proj',\n",
       " 'model.layers.14.self_attn.q_proj',\n",
       " 'model.layers.15.self_attn.k_proj',\n",
       " 'model.layers.15.self_attn.q_proj',\n",
       " 'model.layers.16.self_attn.k_proj',\n",
       " 'model.layers.16.self_attn.q_proj',\n",
       " 'model.layers.17.self_attn.k_proj',\n",
       " 'model.layers.17.self_attn.q_proj',\n",
       " 'model.layers.18.self_attn.k_proj',\n",
       " 'model.layers.18.self_attn.q_proj',\n",
       " 'model.layers.19.self_attn.k_proj',\n",
       " 'model.layers.19.self_attn.q_proj',\n",
       " 'model.layers.20.self_attn.k_proj',\n",
       " 'model.layers.20.self_attn.q_proj',\n",
       " 'model.layers.21.self_attn.k_proj',\n",
       " 'model.layers.21.self_attn.q_proj',\n",
       " 'model.layers.22.self_attn.k_proj',\n",
       " 'model.layers.22.self_attn.q_proj',\n",
       " 'model.layers.23.self_attn.k_proj',\n",
       " 'model.layers.23.self_attn.q_proj',\n",
       " 'model.layers.26.self_attn.k_proj',\n",
       " 'model.layers.26.self_attn.q_proj',\n",
       " 'model.layers.3.self_attn.k_proj',\n",
       " 'model.layers.3.self_attn.q_proj',\n",
       " 'model.layers.4.self_attn.k_proj',\n",
       " 'model.layers.4.self_attn.q_proj',\n",
       " 'model.layers.5.self_attn.k_proj',\n",
       " 'model.layers.5.self_attn.q_proj',\n",
       " 'model.layers.6.self_attn.k_proj',\n",
       " 'model.layers.6.self_attn.q_proj',\n",
       " 'model.layers.7.self_attn.k_proj',\n",
       " 'model.layers.7.self_attn.q_proj',\n",
       " 'model.layers.8.self_attn.k_proj',\n",
       " 'model.layers.8.self_attn.q_proj',\n",
       " 'model.layers.9.self_attn.k_proj',\n",
       " 'model.layers.9.self_attn.q_proj']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "62ce6fe8-1a01-4dd6-a682-830e12eb7c11",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:===========================================\n",
      "\n",
      "1:===========================================\n",
      "model\n",
      "2:===========================================\n",
      "model.embed_tokens\n",
      "3:===========================================\n",
      "model.layers\n",
      "4:===========================================\n",
      "model.layers.0\n",
      "5:===========================================\n",
      "model.layers.0.self_attn\n",
      "6:===========================================\n",
      "model.layers.0.self_attn.q_proj\n",
      "7:===========================================\n",
      "model.layers.0.self_attn.k_proj\n",
      "8:===========================================\n",
      "model.layers.0.self_attn.v_proj\n",
      "9:===========================================\n",
      "model.layers.0.self_attn.o_proj\n",
      "10:===========================================\n",
      "model.layers.0.self_attn.q_norm\n",
      "11:===========================================\n",
      "model.layers.0.self_attn.k_norm\n",
      "12:===========================================\n",
      "model.layers.0.mlp\n",
      "13:===========================================\n",
      "model.layers.0.mlp.gate_proj\n",
      "14:===========================================\n",
      "model.layers.0.mlp.up_proj\n",
      "15:===========================================\n",
      "model.layers.0.mlp.down_proj\n",
      "16:===========================================\n",
      "model.layers.0.mlp.act_fn\n",
      "17:===========================================\n",
      "model.layers.0.input_layernorm\n",
      "18:===========================================\n",
      "model.layers.0.post_attention_layernorm\n",
      "19:===========================================\n",
      "model.layers.1\n",
      "20:===========================================\n",
      "model.layers.1.self_attn\n",
      "21:===========================================\n",
      "model.layers.1.self_attn.q_proj\n",
      "22:===========================================\n",
      "model.layers.1.self_attn.k_proj\n",
      "23:===========================================\n",
      "model.layers.1.self_attn.v_proj\n",
      "24:===========================================\n",
      "model.layers.1.self_attn.o_proj\n",
      "25:===========================================\n",
      "model.layers.1.self_attn.q_norm\n",
      "26:===========================================\n",
      "model.layers.1.self_attn.k_norm\n",
      "27:===========================================\n",
      "model.layers.1.mlp\n",
      "28:===========================================\n",
      "model.layers.1.mlp.gate_proj\n",
      "29:===========================================\n",
      "model.layers.1.mlp.up_proj\n",
      "30:===========================================\n",
      "model.layers.1.mlp.down_proj\n",
      "31:===========================================\n",
      "model.layers.1.mlp.act_fn\n",
      "32:===========================================\n",
      "model.layers.1.input_layernorm\n",
      "33:===========================================\n",
      "model.layers.1.post_attention_layernorm\n",
      "34:===========================================\n",
      "model.layers.2\n",
      "35:===========================================\n",
      "model.layers.2.self_attn\n",
      "36:===========================================\n",
      "model.layers.2.self_attn.q_proj\n",
      "37:===========================================\n",
      "model.layers.2.self_attn.k_proj\n",
      "38:===========================================\n",
      "model.layers.2.self_attn.v_proj\n",
      "39:===========================================\n",
      "model.layers.2.self_attn.o_proj\n",
      "40:===========================================\n",
      "model.layers.2.self_attn.q_norm\n",
      "41:===========================================\n",
      "model.layers.2.self_attn.k_norm\n",
      "42:===========================================\n",
      "model.layers.2.mlp\n",
      "43:===========================================\n",
      "model.layers.2.mlp.gate_proj\n",
      "44:===========================================\n",
      "model.layers.2.mlp.up_proj\n",
      "45:===========================================\n",
      "model.layers.2.mlp.down_proj\n",
      "46:===========================================\n",
      "model.layers.2.mlp.act_fn\n",
      "47:===========================================\n",
      "model.layers.2.input_layernorm\n",
      "48:===========================================\n",
      "model.layers.2.post_attention_layernorm\n",
      "49:===========================================\n",
      "model.layers.3\n",
      "50:===========================================\n",
      "model.layers.3.self_attn\n",
      "51:===========================================\n",
      "model.layers.3.self_attn.q_proj\n",
      "52:===========================================\n",
      "model.layers.3.self_attn.k_proj\n",
      "53:===========================================\n",
      "model.layers.3.self_attn.v_proj\n",
      "54:===========================================\n",
      "model.layers.3.self_attn.o_proj\n",
      "55:===========================================\n",
      "model.layers.3.self_attn.q_norm\n",
      "56:===========================================\n",
      "model.layers.3.self_attn.k_norm\n",
      "57:===========================================\n",
      "model.layers.3.mlp\n",
      "58:===========================================\n",
      "model.layers.3.mlp.gate_proj\n",
      "59:===========================================\n",
      "model.layers.3.mlp.up_proj\n",
      "60:===========================================\n",
      "model.layers.3.mlp.down_proj\n",
      "61:===========================================\n",
      "model.layers.3.mlp.act_fn\n",
      "62:===========================================\n",
      "model.layers.3.input_layernorm\n",
      "63:===========================================\n",
      "model.layers.3.post_attention_layernorm\n",
      "64:===========================================\n",
      "model.layers.4\n",
      "65:===========================================\n",
      "model.layers.4.self_attn\n",
      "66:===========================================\n",
      "model.layers.4.self_attn.q_proj\n",
      "67:===========================================\n",
      "model.layers.4.self_attn.k_proj\n",
      "68:===========================================\n",
      "model.layers.4.self_attn.v_proj\n",
      "69:===========================================\n",
      "model.layers.4.self_attn.o_proj\n",
      "70:===========================================\n",
      "model.layers.4.self_attn.q_norm\n",
      "71:===========================================\n",
      "model.layers.4.self_attn.k_norm\n",
      "72:===========================================\n",
      "model.layers.4.mlp\n",
      "73:===========================================\n",
      "model.layers.4.mlp.gate_proj\n",
      "74:===========================================\n",
      "model.layers.4.mlp.up_proj\n",
      "75:===========================================\n",
      "model.layers.4.mlp.down_proj\n",
      "76:===========================================\n",
      "model.layers.4.mlp.act_fn\n",
      "77:===========================================\n",
      "model.layers.4.input_layernorm\n",
      "78:===========================================\n",
      "model.layers.4.post_attention_layernorm\n",
      "79:===========================================\n",
      "model.layers.5\n",
      "80:===========================================\n",
      "model.layers.5.self_attn\n",
      "81:===========================================\n",
      "model.layers.5.self_attn.q_proj\n",
      "82:===========================================\n",
      "model.layers.5.self_attn.k_proj\n",
      "83:===========================================\n",
      "model.layers.5.self_attn.v_proj\n",
      "84:===========================================\n",
      "model.layers.5.self_attn.o_proj\n",
      "85:===========================================\n",
      "model.layers.5.self_attn.q_norm\n",
      "86:===========================================\n",
      "model.layers.5.self_attn.k_norm\n",
      "87:===========================================\n",
      "model.layers.5.mlp\n",
      "88:===========================================\n",
      "model.layers.5.mlp.gate_proj\n",
      "89:===========================================\n",
      "model.layers.5.mlp.up_proj\n",
      "90:===========================================\n",
      "model.layers.5.mlp.down_proj\n",
      "91:===========================================\n",
      "model.layers.5.mlp.act_fn\n",
      "92:===========================================\n",
      "model.layers.5.input_layernorm\n",
      "93:===========================================\n",
      "model.layers.5.post_attention_layernorm\n",
      "94:===========================================\n",
      "model.layers.6\n",
      "95:===========================================\n",
      "model.layers.6.self_attn\n",
      "96:===========================================\n",
      "model.layers.6.self_attn.q_proj\n",
      "97:===========================================\n",
      "model.layers.6.self_attn.k_proj\n",
      "98:===========================================\n",
      "model.layers.6.self_attn.v_proj\n",
      "99:===========================================\n",
      "model.layers.6.self_attn.o_proj\n",
      "100:===========================================\n",
      "model.layers.6.self_attn.q_norm\n",
      "101:===========================================\n",
      "model.layers.6.self_attn.k_norm\n",
      "102:===========================================\n",
      "model.layers.6.mlp\n",
      "103:===========================================\n",
      "model.layers.6.mlp.gate_proj\n",
      "104:===========================================\n",
      "model.layers.6.mlp.up_proj\n",
      "105:===========================================\n",
      "model.layers.6.mlp.down_proj\n",
      "106:===========================================\n",
      "model.layers.6.mlp.act_fn\n",
      "107:===========================================\n",
      "model.layers.6.input_layernorm\n",
      "108:===========================================\n",
      "model.layers.6.post_attention_layernorm\n",
      "109:===========================================\n",
      "model.layers.7\n",
      "110:===========================================\n",
      "model.layers.7.self_attn\n",
      "111:===========================================\n",
      "model.layers.7.self_attn.q_proj\n",
      "112:===========================================\n",
      "model.layers.7.self_attn.k_proj\n",
      "113:===========================================\n",
      "model.layers.7.self_attn.v_proj\n",
      "114:===========================================\n",
      "model.layers.7.self_attn.o_proj\n",
      "115:===========================================\n",
      "model.layers.7.self_attn.q_norm\n",
      "116:===========================================\n",
      "model.layers.7.self_attn.k_norm\n",
      "117:===========================================\n",
      "model.layers.7.mlp\n",
      "118:===========================================\n",
      "model.layers.7.mlp.gate_proj\n",
      "119:===========================================\n",
      "model.layers.7.mlp.up_proj\n",
      "120:===========================================\n",
      "model.layers.7.mlp.down_proj\n",
      "121:===========================================\n",
      "model.layers.7.mlp.act_fn\n",
      "122:===========================================\n",
      "model.layers.7.input_layernorm\n",
      "123:===========================================\n",
      "model.layers.7.post_attention_layernorm\n",
      "124:===========================================\n",
      "model.layers.8\n",
      "125:===========================================\n",
      "model.layers.8.self_attn\n",
      "126:===========================================\n",
      "model.layers.8.self_attn.q_proj\n",
      "127:===========================================\n",
      "model.layers.8.self_attn.k_proj\n",
      "128:===========================================\n",
      "model.layers.8.self_attn.v_proj\n",
      "129:===========================================\n",
      "model.layers.8.self_attn.o_proj\n",
      "130:===========================================\n",
      "model.layers.8.self_attn.q_norm\n",
      "131:===========================================\n",
      "model.layers.8.self_attn.k_norm\n",
      "132:===========================================\n",
      "model.layers.8.mlp\n",
      "133:===========================================\n",
      "model.layers.8.mlp.gate_proj\n",
      "134:===========================================\n",
      "model.layers.8.mlp.up_proj\n",
      "135:===========================================\n",
      "model.layers.8.mlp.down_proj\n",
      "136:===========================================\n",
      "model.layers.8.mlp.act_fn\n",
      "137:===========================================\n",
      "model.layers.8.input_layernorm\n",
      "138:===========================================\n",
      "model.layers.8.post_attention_layernorm\n",
      "139:===========================================\n",
      "model.layers.9\n",
      "140:===========================================\n",
      "model.layers.9.self_attn\n",
      "141:===========================================\n",
      "model.layers.9.self_attn.q_proj\n",
      "142:===========================================\n",
      "model.layers.9.self_attn.k_proj\n",
      "143:===========================================\n",
      "model.layers.9.self_attn.v_proj\n",
      "144:===========================================\n",
      "model.layers.9.self_attn.o_proj\n",
      "145:===========================================\n",
      "model.layers.9.self_attn.q_norm\n",
      "146:===========================================\n",
      "model.layers.9.self_attn.k_norm\n",
      "147:===========================================\n",
      "model.layers.9.mlp\n",
      "148:===========================================\n",
      "model.layers.9.mlp.gate_proj\n",
      "149:===========================================\n",
      "model.layers.9.mlp.up_proj\n",
      "150:===========================================\n",
      "model.layers.9.mlp.down_proj\n",
      "151:===========================================\n",
      "model.layers.9.mlp.act_fn\n",
      "152:===========================================\n",
      "model.layers.9.input_layernorm\n",
      "153:===========================================\n",
      "model.layers.9.post_attention_layernorm\n",
      "154:===========================================\n",
      "model.layers.10\n",
      "155:===========================================\n",
      "model.layers.10.self_attn\n",
      "156:===========================================\n",
      "model.layers.10.self_attn.q_proj\n",
      "157:===========================================\n",
      "model.layers.10.self_attn.k_proj\n",
      "158:===========================================\n",
      "model.layers.10.self_attn.v_proj\n",
      "159:===========================================\n",
      "model.layers.10.self_attn.o_proj\n",
      "160:===========================================\n",
      "model.layers.10.self_attn.q_norm\n",
      "161:===========================================\n",
      "model.layers.10.self_attn.k_norm\n",
      "162:===========================================\n",
      "model.layers.10.mlp\n",
      "163:===========================================\n",
      "model.layers.10.mlp.gate_proj\n",
      "164:===========================================\n",
      "model.layers.10.mlp.up_proj\n",
      "165:===========================================\n",
      "model.layers.10.mlp.down_proj\n",
      "166:===========================================\n",
      "model.layers.10.mlp.act_fn\n",
      "167:===========================================\n",
      "model.layers.10.input_layernorm\n",
      "168:===========================================\n",
      "model.layers.10.post_attention_layernorm\n",
      "169:===========================================\n",
      "model.layers.11\n",
      "170:===========================================\n",
      "model.layers.11.self_attn\n",
      "171:===========================================\n",
      "model.layers.11.self_attn.q_proj\n",
      "172:===========================================\n",
      "model.layers.11.self_attn.k_proj\n",
      "173:===========================================\n",
      "model.layers.11.self_attn.v_proj\n",
      "174:===========================================\n",
      "model.layers.11.self_attn.o_proj\n",
      "175:===========================================\n",
      "model.layers.11.self_attn.q_norm\n",
      "176:===========================================\n",
      "model.layers.11.self_attn.k_norm\n",
      "177:===========================================\n",
      "model.layers.11.mlp\n",
      "178:===========================================\n",
      "model.layers.11.mlp.gate_proj\n",
      "179:===========================================\n",
      "model.layers.11.mlp.up_proj\n",
      "180:===========================================\n",
      "model.layers.11.mlp.down_proj\n",
      "181:===========================================\n",
      "model.layers.11.mlp.act_fn\n",
      "182:===========================================\n",
      "model.layers.11.input_layernorm\n",
      "183:===========================================\n",
      "model.layers.11.post_attention_layernorm\n",
      "184:===========================================\n",
      "model.layers.12\n",
      "185:===========================================\n",
      "model.layers.12.self_attn\n",
      "186:===========================================\n",
      "model.layers.12.self_attn.q_proj\n",
      "187:===========================================\n",
      "model.layers.12.self_attn.k_proj\n",
      "188:===========================================\n",
      "model.layers.12.self_attn.v_proj\n",
      "189:===========================================\n",
      "model.layers.12.self_attn.o_proj\n",
      "190:===========================================\n",
      "model.layers.12.self_attn.q_norm\n",
      "191:===========================================\n",
      "model.layers.12.self_attn.k_norm\n",
      "192:===========================================\n",
      "model.layers.12.mlp\n",
      "193:===========================================\n",
      "model.layers.12.mlp.gate_proj\n",
      "194:===========================================\n",
      "model.layers.12.mlp.up_proj\n",
      "195:===========================================\n",
      "model.layers.12.mlp.down_proj\n",
      "196:===========================================\n",
      "model.layers.12.mlp.act_fn\n",
      "197:===========================================\n",
      "model.layers.12.input_layernorm\n",
      "198:===========================================\n",
      "model.layers.12.post_attention_layernorm\n",
      "199:===========================================\n",
      "model.layers.13\n",
      "200:===========================================\n",
      "model.layers.13.self_attn\n",
      "201:===========================================\n",
      "model.layers.13.self_attn.q_proj\n",
      "202:===========================================\n",
      "model.layers.13.self_attn.k_proj\n",
      "203:===========================================\n",
      "model.layers.13.self_attn.v_proj\n",
      "204:===========================================\n",
      "model.layers.13.self_attn.o_proj\n",
      "205:===========================================\n",
      "model.layers.13.self_attn.q_norm\n",
      "206:===========================================\n",
      "model.layers.13.self_attn.k_norm\n",
      "207:===========================================\n",
      "model.layers.13.mlp\n",
      "208:===========================================\n",
      "model.layers.13.mlp.gate_proj\n",
      "209:===========================================\n",
      "model.layers.13.mlp.up_proj\n",
      "210:===========================================\n",
      "model.layers.13.mlp.down_proj\n",
      "211:===========================================\n",
      "model.layers.13.mlp.act_fn\n",
      "212:===========================================\n",
      "model.layers.13.input_layernorm\n",
      "213:===========================================\n",
      "model.layers.13.post_attention_layernorm\n",
      "214:===========================================\n",
      "model.layers.14\n",
      "215:===========================================\n",
      "model.layers.14.self_attn\n",
      "216:===========================================\n",
      "model.layers.14.self_attn.q_proj\n",
      "217:===========================================\n",
      "model.layers.14.self_attn.k_proj\n",
      "218:===========================================\n",
      "model.layers.14.self_attn.v_proj\n",
      "219:===========================================\n",
      "model.layers.14.self_attn.o_proj\n",
      "220:===========================================\n",
      "model.layers.14.self_attn.q_norm\n",
      "221:===========================================\n",
      "model.layers.14.self_attn.k_norm\n",
      "222:===========================================\n",
      "model.layers.14.mlp\n",
      "223:===========================================\n",
      "model.layers.14.mlp.gate_proj\n",
      "224:===========================================\n",
      "model.layers.14.mlp.up_proj\n",
      "225:===========================================\n",
      "model.layers.14.mlp.down_proj\n",
      "226:===========================================\n",
      "model.layers.14.mlp.act_fn\n",
      "227:===========================================\n",
      "model.layers.14.input_layernorm\n",
      "228:===========================================\n",
      "model.layers.14.post_attention_layernorm\n",
      "229:===========================================\n",
      "model.layers.15\n",
      "230:===========================================\n",
      "model.layers.15.self_attn\n",
      "231:===========================================\n",
      "model.layers.15.self_attn.q_proj\n",
      "232:===========================================\n",
      "model.layers.15.self_attn.k_proj\n",
      "233:===========================================\n",
      "model.layers.15.self_attn.v_proj\n",
      "234:===========================================\n",
      "model.layers.15.self_attn.o_proj\n",
      "235:===========================================\n",
      "model.layers.15.self_attn.q_norm\n",
      "236:===========================================\n",
      "model.layers.15.self_attn.k_norm\n",
      "237:===========================================\n",
      "model.layers.15.mlp\n",
      "238:===========================================\n",
      "model.layers.15.mlp.gate_proj\n",
      "239:===========================================\n",
      "model.layers.15.mlp.up_proj\n",
      "240:===========================================\n",
      "model.layers.15.mlp.down_proj\n",
      "241:===========================================\n",
      "model.layers.15.mlp.act_fn\n",
      "242:===========================================\n",
      "model.layers.15.input_layernorm\n",
      "243:===========================================\n",
      "model.layers.15.post_attention_layernorm\n",
      "244:===========================================\n",
      "model.layers.16\n",
      "245:===========================================\n",
      "model.layers.16.self_attn\n",
      "246:===========================================\n",
      "model.layers.16.self_attn.q_proj\n",
      "247:===========================================\n",
      "model.layers.16.self_attn.k_proj\n",
      "248:===========================================\n",
      "model.layers.16.self_attn.v_proj\n",
      "249:===========================================\n",
      "model.layers.16.self_attn.o_proj\n",
      "250:===========================================\n",
      "model.layers.16.self_attn.q_norm\n",
      "251:===========================================\n",
      "model.layers.16.self_attn.k_norm\n",
      "252:===========================================\n",
      "model.layers.16.mlp\n",
      "253:===========================================\n",
      "model.layers.16.mlp.gate_proj\n",
      "254:===========================================\n",
      "model.layers.16.mlp.up_proj\n",
      "255:===========================================\n",
      "model.layers.16.mlp.down_proj\n",
      "256:===========================================\n",
      "model.layers.16.mlp.act_fn\n",
      "257:===========================================\n",
      "model.layers.16.input_layernorm\n",
      "258:===========================================\n",
      "model.layers.16.post_attention_layernorm\n",
      "259:===========================================\n",
      "model.layers.17\n",
      "260:===========================================\n",
      "model.layers.17.self_attn\n",
      "261:===========================================\n",
      "model.layers.17.self_attn.q_proj\n",
      "262:===========================================\n",
      "model.layers.17.self_attn.k_proj\n",
      "263:===========================================\n",
      "model.layers.17.self_attn.v_proj\n",
      "264:===========================================\n",
      "model.layers.17.self_attn.o_proj\n",
      "265:===========================================\n",
      "model.layers.17.self_attn.q_norm\n",
      "266:===========================================\n",
      "model.layers.17.self_attn.k_norm\n",
      "267:===========================================\n",
      "model.layers.17.mlp\n",
      "268:===========================================\n",
      "model.layers.17.mlp.gate_proj\n",
      "269:===========================================\n",
      "model.layers.17.mlp.up_proj\n",
      "270:===========================================\n",
      "model.layers.17.mlp.down_proj\n",
      "271:===========================================\n",
      "model.layers.17.mlp.act_fn\n",
      "272:===========================================\n",
      "model.layers.17.input_layernorm\n",
      "273:===========================================\n",
      "model.layers.17.post_attention_layernorm\n",
      "274:===========================================\n",
      "model.layers.18\n",
      "275:===========================================\n",
      "model.layers.18.self_attn\n",
      "276:===========================================\n",
      "model.layers.18.self_attn.q_proj\n",
      "277:===========================================\n",
      "model.layers.18.self_attn.k_proj\n",
      "278:===========================================\n",
      "model.layers.18.self_attn.v_proj\n",
      "279:===========================================\n",
      "model.layers.18.self_attn.o_proj\n",
      "280:===========================================\n",
      "model.layers.18.self_attn.q_norm\n",
      "281:===========================================\n",
      "model.layers.18.self_attn.k_norm\n",
      "282:===========================================\n",
      "model.layers.18.mlp\n",
      "283:===========================================\n",
      "model.layers.18.mlp.gate_proj\n",
      "284:===========================================\n",
      "model.layers.18.mlp.up_proj\n",
      "285:===========================================\n",
      "model.layers.18.mlp.down_proj\n",
      "286:===========================================\n",
      "model.layers.18.mlp.act_fn\n",
      "287:===========================================\n",
      "model.layers.18.input_layernorm\n",
      "288:===========================================\n",
      "model.layers.18.post_attention_layernorm\n",
      "289:===========================================\n",
      "model.layers.19\n",
      "290:===========================================\n",
      "model.layers.19.self_attn\n",
      "291:===========================================\n",
      "model.layers.19.self_attn.q_proj\n",
      "292:===========================================\n",
      "model.layers.19.self_attn.k_proj\n",
      "293:===========================================\n",
      "model.layers.19.self_attn.v_proj\n",
      "294:===========================================\n",
      "model.layers.19.self_attn.o_proj\n",
      "295:===========================================\n",
      "model.layers.19.self_attn.q_norm\n",
      "296:===========================================\n",
      "model.layers.19.self_attn.k_norm\n",
      "297:===========================================\n",
      "model.layers.19.mlp\n",
      "298:===========================================\n",
      "model.layers.19.mlp.gate_proj\n",
      "299:===========================================\n",
      "model.layers.19.mlp.up_proj\n",
      "300:===========================================\n",
      "model.layers.19.mlp.down_proj\n",
      "301:===========================================\n",
      "model.layers.19.mlp.act_fn\n",
      "302:===========================================\n",
      "model.layers.19.input_layernorm\n",
      "303:===========================================\n",
      "model.layers.19.post_attention_layernorm\n",
      "304:===========================================\n",
      "model.layers.20\n",
      "305:===========================================\n",
      "model.layers.20.self_attn\n",
      "306:===========================================\n",
      "model.layers.20.self_attn.q_proj\n",
      "307:===========================================\n",
      "model.layers.20.self_attn.k_proj\n",
      "308:===========================================\n",
      "model.layers.20.self_attn.v_proj\n",
      "309:===========================================\n",
      "model.layers.20.self_attn.o_proj\n",
      "310:===========================================\n",
      "model.layers.20.self_attn.q_norm\n",
      "311:===========================================\n",
      "model.layers.20.self_attn.k_norm\n",
      "312:===========================================\n",
      "model.layers.20.mlp\n",
      "313:===========================================\n",
      "model.layers.20.mlp.gate_proj\n",
      "314:===========================================\n",
      "model.layers.20.mlp.up_proj\n",
      "315:===========================================\n",
      "model.layers.20.mlp.down_proj\n",
      "316:===========================================\n",
      "model.layers.20.mlp.act_fn\n",
      "317:===========================================\n",
      "model.layers.20.input_layernorm\n",
      "318:===========================================\n",
      "model.layers.20.post_attention_layernorm\n",
      "319:===========================================\n",
      "model.layers.21\n",
      "320:===========================================\n",
      "model.layers.21.self_attn\n",
      "321:===========================================\n",
      "model.layers.21.self_attn.q_proj\n",
      "322:===========================================\n",
      "model.layers.21.self_attn.k_proj\n",
      "323:===========================================\n",
      "model.layers.21.self_attn.v_proj\n",
      "324:===========================================\n",
      "model.layers.21.self_attn.o_proj\n",
      "325:===========================================\n",
      "model.layers.21.self_attn.q_norm\n",
      "326:===========================================\n",
      "model.layers.21.self_attn.k_norm\n",
      "327:===========================================\n",
      "model.layers.21.mlp\n",
      "328:===========================================\n",
      "model.layers.21.mlp.gate_proj\n",
      "329:===========================================\n",
      "model.layers.21.mlp.up_proj\n",
      "330:===========================================\n",
      "model.layers.21.mlp.down_proj\n",
      "331:===========================================\n",
      "model.layers.21.mlp.act_fn\n",
      "332:===========================================\n",
      "model.layers.21.input_layernorm\n",
      "333:===========================================\n",
      "model.layers.21.post_attention_layernorm\n",
      "334:===========================================\n",
      "model.layers.22\n",
      "335:===========================================\n",
      "model.layers.22.self_attn\n",
      "336:===========================================\n",
      "model.layers.22.self_attn.q_proj\n",
      "337:===========================================\n",
      "model.layers.22.self_attn.k_proj\n",
      "338:===========================================\n",
      "model.layers.22.self_attn.v_proj\n",
      "339:===========================================\n",
      "model.layers.22.self_attn.o_proj\n",
      "340:===========================================\n",
      "model.layers.22.self_attn.q_norm\n",
      "341:===========================================\n",
      "model.layers.22.self_attn.k_norm\n",
      "342:===========================================\n",
      "model.layers.22.mlp\n",
      "343:===========================================\n",
      "model.layers.22.mlp.gate_proj\n",
      "344:===========================================\n",
      "model.layers.22.mlp.up_proj\n",
      "345:===========================================\n",
      "model.layers.22.mlp.down_proj\n",
      "346:===========================================\n",
      "model.layers.22.mlp.act_fn\n",
      "347:===========================================\n",
      "model.layers.22.input_layernorm\n",
      "348:===========================================\n",
      "model.layers.22.post_attention_layernorm\n",
      "349:===========================================\n",
      "model.layers.23\n",
      "350:===========================================\n",
      "model.layers.23.self_attn\n",
      "351:===========================================\n",
      "model.layers.23.self_attn.q_proj\n",
      "352:===========================================\n",
      "model.layers.23.self_attn.k_proj\n",
      "353:===========================================\n",
      "model.layers.23.self_attn.v_proj\n",
      "354:===========================================\n",
      "model.layers.23.self_attn.o_proj\n",
      "355:===========================================\n",
      "model.layers.23.self_attn.q_norm\n",
      "356:===========================================\n",
      "model.layers.23.self_attn.k_norm\n",
      "357:===========================================\n",
      "model.layers.23.mlp\n",
      "358:===========================================\n",
      "model.layers.23.mlp.gate_proj\n",
      "359:===========================================\n",
      "model.layers.23.mlp.up_proj\n",
      "360:===========================================\n",
      "model.layers.23.mlp.down_proj\n",
      "361:===========================================\n",
      "model.layers.23.mlp.act_fn\n",
      "362:===========================================\n",
      "model.layers.23.input_layernorm\n",
      "363:===========================================\n",
      "model.layers.23.post_attention_layernorm\n",
      "364:===========================================\n",
      "model.layers.24\n",
      "365:===========================================\n",
      "model.layers.24.self_attn\n",
      "366:===========================================\n",
      "model.layers.24.self_attn.q_proj\n",
      "367:===========================================\n",
      "model.layers.24.self_attn.k_proj\n",
      "368:===========================================\n",
      "model.layers.24.self_attn.v_proj\n",
      "369:===========================================\n",
      "model.layers.24.self_attn.o_proj\n",
      "370:===========================================\n",
      "model.layers.24.self_attn.q_norm\n",
      "371:===========================================\n",
      "model.layers.24.self_attn.k_norm\n",
      "372:===========================================\n",
      "model.layers.24.mlp\n",
      "373:===========================================\n",
      "model.layers.24.mlp.gate_proj\n",
      "374:===========================================\n",
      "model.layers.24.mlp.up_proj\n",
      "375:===========================================\n",
      "model.layers.24.mlp.down_proj\n",
      "376:===========================================\n",
      "model.layers.24.mlp.act_fn\n",
      "377:===========================================\n",
      "model.layers.24.input_layernorm\n",
      "378:===========================================\n",
      "model.layers.24.post_attention_layernorm\n",
      "379:===========================================\n",
      "model.layers.25\n",
      "380:===========================================\n",
      "model.layers.25.self_attn\n",
      "381:===========================================\n",
      "model.layers.25.self_attn.q_proj\n",
      "382:===========================================\n",
      "model.layers.25.self_attn.k_proj\n",
      "383:===========================================\n",
      "model.layers.25.self_attn.v_proj\n",
      "384:===========================================\n",
      "model.layers.25.self_attn.o_proj\n",
      "385:===========================================\n",
      "model.layers.25.self_attn.q_norm\n",
      "386:===========================================\n",
      "model.layers.25.self_attn.k_norm\n",
      "387:===========================================\n",
      "model.layers.25.mlp\n",
      "388:===========================================\n",
      "model.layers.25.mlp.gate_proj\n",
      "389:===========================================\n",
      "model.layers.25.mlp.up_proj\n",
      "390:===========================================\n",
      "model.layers.25.mlp.down_proj\n",
      "391:===========================================\n",
      "model.layers.25.mlp.act_fn\n",
      "392:===========================================\n",
      "model.layers.25.input_layernorm\n",
      "393:===========================================\n",
      "model.layers.25.post_attention_layernorm\n",
      "394:===========================================\n",
      "model.layers.26\n",
      "395:===========================================\n",
      "model.layers.26.self_attn\n",
      "396:===========================================\n",
      "model.layers.26.self_attn.q_proj\n",
      "397:===========================================\n",
      "model.layers.26.self_attn.k_proj\n",
      "398:===========================================\n",
      "model.layers.26.self_attn.v_proj\n",
      "399:===========================================\n",
      "model.layers.26.self_attn.o_proj\n",
      "400:===========================================\n",
      "model.layers.26.self_attn.q_norm\n",
      "401:===========================================\n",
      "model.layers.26.self_attn.k_norm\n",
      "402:===========================================\n",
      "model.layers.26.mlp\n",
      "403:===========================================\n",
      "model.layers.26.mlp.gate_proj\n",
      "404:===========================================\n",
      "model.layers.26.mlp.up_proj\n",
      "405:===========================================\n",
      "model.layers.26.mlp.down_proj\n",
      "406:===========================================\n",
      "model.layers.26.mlp.act_fn\n",
      "407:===========================================\n",
      "model.layers.26.input_layernorm\n",
      "408:===========================================\n",
      "model.layers.26.post_attention_layernorm\n",
      "409:===========================================\n",
      "model.layers.27\n",
      "410:===========================================\n",
      "model.layers.27.self_attn\n",
      "411:===========================================\n",
      "model.layers.27.self_attn.q_proj\n",
      "412:===========================================\n",
      "model.layers.27.self_attn.k_proj\n",
      "413:===========================================\n",
      "model.layers.27.self_attn.v_proj\n",
      "414:===========================================\n",
      "model.layers.27.self_attn.o_proj\n",
      "415:===========================================\n",
      "model.layers.27.self_attn.q_norm\n",
      "416:===========================================\n",
      "model.layers.27.self_attn.k_norm\n",
      "417:===========================================\n",
      "model.layers.27.mlp\n",
      "418:===========================================\n",
      "model.layers.27.mlp.gate_proj\n",
      "419:===========================================\n",
      "model.layers.27.mlp.up_proj\n",
      "420:===========================================\n",
      "model.layers.27.mlp.down_proj\n",
      "421:===========================================\n",
      "model.layers.27.mlp.act_fn\n",
      "422:===========================================\n",
      "model.layers.27.input_layernorm\n",
      "423:===========================================\n",
      "model.layers.27.post_attention_layernorm\n",
      "424:===========================================\n",
      "model.layers.28\n",
      "425:===========================================\n",
      "model.layers.28.self_attn\n",
      "426:===========================================\n",
      "model.layers.28.self_attn.q_proj\n",
      "427:===========================================\n",
      "model.layers.28.self_attn.k_proj\n",
      "428:===========================================\n",
      "model.layers.28.self_attn.v_proj\n",
      "429:===========================================\n",
      "model.layers.28.self_attn.o_proj\n",
      "430:===========================================\n",
      "model.layers.28.self_attn.q_norm\n",
      "431:===========================================\n",
      "model.layers.28.self_attn.k_norm\n",
      "432:===========================================\n",
      "model.layers.28.mlp\n",
      "433:===========================================\n",
      "model.layers.28.mlp.gate_proj\n",
      "434:===========================================\n",
      "model.layers.28.mlp.up_proj\n",
      "435:===========================================\n",
      "model.layers.28.mlp.down_proj\n",
      "436:===========================================\n",
      "model.layers.28.mlp.act_fn\n",
      "437:===========================================\n",
      "model.layers.28.input_layernorm\n",
      "438:===========================================\n",
      "model.layers.28.post_attention_layernorm\n",
      "439:===========================================\n",
      "model.layers.29\n",
      "440:===========================================\n",
      "model.layers.29.self_attn\n",
      "441:===========================================\n",
      "model.layers.29.self_attn.q_proj\n",
      "442:===========================================\n",
      "model.layers.29.self_attn.k_proj\n",
      "443:===========================================\n",
      "model.layers.29.self_attn.v_proj\n",
      "444:===========================================\n",
      "model.layers.29.self_attn.o_proj\n",
      "445:===========================================\n",
      "model.layers.29.self_attn.q_norm\n",
      "446:===========================================\n",
      "model.layers.29.self_attn.k_norm\n",
      "447:===========================================\n",
      "model.layers.29.mlp\n",
      "448:===========================================\n",
      "model.layers.29.mlp.gate_proj\n",
      "449:===========================================\n",
      "model.layers.29.mlp.up_proj\n",
      "450:===========================================\n",
      "model.layers.29.mlp.down_proj\n",
      "451:===========================================\n",
      "model.layers.29.mlp.act_fn\n",
      "452:===========================================\n",
      "model.layers.29.input_layernorm\n",
      "453:===========================================\n",
      "model.layers.29.post_attention_layernorm\n",
      "454:===========================================\n",
      "model.layers.30\n",
      "455:===========================================\n",
      "model.layers.30.self_attn\n",
      "456:===========================================\n",
      "model.layers.30.self_attn.q_proj\n",
      "457:===========================================\n",
      "model.layers.30.self_attn.k_proj\n",
      "458:===========================================\n",
      "model.layers.30.self_attn.v_proj\n",
      "459:===========================================\n",
      "model.layers.30.self_attn.o_proj\n",
      "460:===========================================\n",
      "model.layers.30.self_attn.q_norm\n",
      "461:===========================================\n",
      "model.layers.30.self_attn.k_norm\n",
      "462:===========================================\n",
      "model.layers.30.mlp\n",
      "463:===========================================\n",
      "model.layers.30.mlp.gate_proj\n",
      "464:===========================================\n",
      "model.layers.30.mlp.up_proj\n",
      "465:===========================================\n",
      "model.layers.30.mlp.down_proj\n",
      "466:===========================================\n",
      "model.layers.30.mlp.act_fn\n",
      "467:===========================================\n",
      "model.layers.30.input_layernorm\n",
      "468:===========================================\n",
      "model.layers.30.post_attention_layernorm\n",
      "469:===========================================\n",
      "model.layers.31\n",
      "470:===========================================\n",
      "model.layers.31.self_attn\n",
      "471:===========================================\n",
      "model.layers.31.self_attn.q_proj\n",
      "472:===========================================\n",
      "model.layers.31.self_attn.k_proj\n",
      "473:===========================================\n",
      "model.layers.31.self_attn.v_proj\n",
      "474:===========================================\n",
      "model.layers.31.self_attn.o_proj\n",
      "475:===========================================\n",
      "model.layers.31.self_attn.q_norm\n",
      "476:===========================================\n",
      "model.layers.31.self_attn.k_norm\n",
      "477:===========================================\n",
      "model.layers.31.mlp\n",
      "478:===========================================\n",
      "model.layers.31.mlp.gate_proj\n",
      "479:===========================================\n",
      "model.layers.31.mlp.up_proj\n",
      "480:===========================================\n",
      "model.layers.31.mlp.down_proj\n",
      "481:===========================================\n",
      "model.layers.31.mlp.act_fn\n",
      "482:===========================================\n",
      "model.layers.31.input_layernorm\n",
      "483:===========================================\n",
      "model.layers.31.post_attention_layernorm\n",
      "484:===========================================\n",
      "model.layers.32\n",
      "485:===========================================\n",
      "model.layers.32.self_attn\n",
      "486:===========================================\n",
      "model.layers.32.self_attn.q_proj\n",
      "487:===========================================\n",
      "model.layers.32.self_attn.k_proj\n",
      "488:===========================================\n",
      "model.layers.32.self_attn.v_proj\n",
      "489:===========================================\n",
      "model.layers.32.self_attn.o_proj\n",
      "490:===========================================\n",
      "model.layers.32.self_attn.q_norm\n",
      "491:===========================================\n",
      "model.layers.32.self_attn.k_norm\n",
      "492:===========================================\n",
      "model.layers.32.mlp\n",
      "493:===========================================\n",
      "model.layers.32.mlp.gate_proj\n",
      "494:===========================================\n",
      "model.layers.32.mlp.up_proj\n",
      "495:===========================================\n",
      "model.layers.32.mlp.down_proj\n",
      "496:===========================================\n",
      "model.layers.32.mlp.act_fn\n",
      "497:===========================================\n",
      "model.layers.32.input_layernorm\n",
      "498:===========================================\n",
      "model.layers.32.post_attention_layernorm\n",
      "499:===========================================\n",
      "model.layers.33\n",
      "500:===========================================\n",
      "model.layers.33.self_attn\n",
      "501:===========================================\n",
      "model.layers.33.self_attn.q_proj\n",
      "502:===========================================\n",
      "model.layers.33.self_attn.k_proj\n",
      "503:===========================================\n",
      "model.layers.33.self_attn.v_proj\n",
      "504:===========================================\n",
      "model.layers.33.self_attn.o_proj\n",
      "505:===========================================\n",
      "model.layers.33.self_attn.q_norm\n",
      "506:===========================================\n",
      "model.layers.33.self_attn.k_norm\n",
      "507:===========================================\n",
      "model.layers.33.mlp\n",
      "508:===========================================\n",
      "model.layers.33.mlp.gate_proj\n",
      "509:===========================================\n",
      "model.layers.33.mlp.up_proj\n",
      "510:===========================================\n",
      "model.layers.33.mlp.down_proj\n",
      "511:===========================================\n",
      "model.layers.33.mlp.act_fn\n",
      "512:===========================================\n",
      "model.layers.33.input_layernorm\n",
      "513:===========================================\n",
      "model.layers.33.post_attention_layernorm\n",
      "514:===========================================\n",
      "model.layers.34\n",
      "515:===========================================\n",
      "model.layers.34.self_attn\n",
      "516:===========================================\n",
      "model.layers.34.self_attn.q_proj\n",
      "517:===========================================\n",
      "model.layers.34.self_attn.k_proj\n",
      "518:===========================================\n",
      "model.layers.34.self_attn.v_proj\n",
      "519:===========================================\n",
      "model.layers.34.self_attn.o_proj\n",
      "520:===========================================\n",
      "model.layers.34.self_attn.q_norm\n",
      "521:===========================================\n",
      "model.layers.34.self_attn.k_norm\n",
      "522:===========================================\n",
      "model.layers.34.mlp\n",
      "523:===========================================\n",
      "model.layers.34.mlp.gate_proj\n",
      "524:===========================================\n",
      "model.layers.34.mlp.up_proj\n",
      "525:===========================================\n",
      "model.layers.34.mlp.down_proj\n",
      "526:===========================================\n",
      "model.layers.34.mlp.act_fn\n",
      "527:===========================================\n",
      "model.layers.34.input_layernorm\n",
      "528:===========================================\n",
      "model.layers.34.post_attention_layernorm\n",
      "529:===========================================\n",
      "model.layers.35\n",
      "530:===========================================\n",
      "model.layers.35.self_attn\n",
      "531:===========================================\n",
      "model.layers.35.self_attn.q_proj\n",
      "532:===========================================\n",
      "model.layers.35.self_attn.k_proj\n",
      "533:===========================================\n",
      "model.layers.35.self_attn.v_proj\n",
      "534:===========================================\n",
      "model.layers.35.self_attn.o_proj\n",
      "535:===========================================\n",
      "model.layers.35.self_attn.q_norm\n",
      "536:===========================================\n",
      "model.layers.35.self_attn.k_norm\n",
      "537:===========================================\n",
      "model.layers.35.mlp\n",
      "538:===========================================\n",
      "model.layers.35.mlp.gate_proj\n",
      "539:===========================================\n",
      "model.layers.35.mlp.up_proj\n",
      "540:===========================================\n",
      "model.layers.35.mlp.down_proj\n",
      "541:===========================================\n",
      "model.layers.35.mlp.act_fn\n",
      "542:===========================================\n",
      "model.layers.35.input_layernorm\n",
      "543:===========================================\n",
      "model.layers.35.post_attention_layernorm\n",
      "544:===========================================\n",
      "model.norm\n",
      "545:===========================================\n",
      "model.rotary_emb\n",
      "546:===========================================\n",
      "lm_head\n"
     ]
    }
   ],
   "source": [
    "for i,m in enumerate(model.named_modules()):\n",
    "    print(str(i) + \":===========================================\")\n",
    "    print(m[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2250da5-79a6-4a5d-8124-913e3bc3b33a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_focallora4",
   "language": "python",
   "name": "conda_focallora4"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}