{
  "base_tokenizer": "llama",
  "tokenizer_paths": {
    "llama": "/data/local/hujk/models/Llama-3.1-8B-Instruct",
    "qwen3": "/data/local/hujk/models/Qwen3-8B"
  },
  "files": [
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_00512.json",
      "target_total_len": 512,
      "filler_repeat": 18,
      "token_lengths": {
        "llama": 500,
        "qwen3": 482
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_01024.json",
      "target_total_len": 1024,
      "filler_repeat": 41,
      "token_lengths": {
        "llama": 1006,
        "qwen3": 988
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_02048.json",
      "target_total_len": 2048,
      "filler_repeat": 88,
      "token_lengths": {
        "llama": 2040,
        "qwen3": 2022
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_04096.json",
      "target_total_len": 4096,
      "filler_repeat": 181,
      "token_lengths": {
        "llama": 4086,
        "qwen3": 4068
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_08192.json",
      "target_total_len": 8192,
      "filler_repeat": 367,
      "token_lengths": {
        "llama": 8178,
        "qwen3": 8160
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_12288.json",
      "target_total_len": 12288,
      "filler_repeat": 553,
      "token_lengths": {
        "llama": 12270,
        "qwen3": 12252
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_16384.json",
      "target_total_len": 16384,
      "filler_repeat": 740,
      "token_lengths": {
        "llama": 16384,
        "qwen3": 16366
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_24576.json",
      "target_total_len": 24576,
      "filler_repeat": 1112,
      "token_lengths": {
        "llama": 24568,
        "qwen3": 24550
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_32768.json",
      "target_total_len": 32768,
      "filler_repeat": 1484,
      "token_lengths": {
        "llama": 32752,
        "qwen3": 32734
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_40960.json",
      "target_total_len": 40960,
      "filler_repeat": 1857,
      "token_lengths": {
        "llama": 40958,
        "qwen3": 40940
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_49152.json",
      "target_total_len": 49152,
      "filler_repeat": 2229,
      "token_lengths": {
        "llama": 49142,
        "qwen3": 49124
      }
    },
    {
      "path": "/home/hujk/gitrs/Paper2026/SortedCode2/3-1_model_training_data_gen/len_test/len_65536.json",
      "target_total_len": 65536,
      "filler_repeat": 2974,
      "token_lengths": {
        "llama": 65532,
        "qwen3": 65514
      }
    }
  ]
}