#!/usr/bin/env python3 # -*- coding: utf-8 -*- import argparse import json from pathlib import Path from typing import Any, Dict, List import random def user(content: str) -> List[Dict[str, str]]: return [{"role": "user", "content": content}] def tool(content: str) -> List[Dict[str, str]]: # Use "tool" role as requested return [{"role": "tool", "content": content}] def assistant(content: str) -> List[Dict[str, str]]: return [{"role": "assistant", "content": content}] def suffile(parts: List[str]) -> str: # You didn't specify the exact formatting; this is a simple, deterministic join. # Change separator if you need (e.g., "\n\n", special tokens, etc.) return "\n".join(parts) def read_json(path: Path) -> Any: with path.open("r", encoding="utf-8") as f: return json.load(f) def write_jsonl(path: Path, rows: List[Dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as f: for row in rows: f.write(json.dumps(row, ensure_ascii=False) + "\n") def build_candidates(datasets: List[Dict[str, Any]], rng: random.Random) -> List[List[Dict[str, str]]]: """ dataset_message_candidate: a list of message sequences (each sequence is a list of message dicts) Built per your 7-variant recipe. """ n = len(datasets) if n < 2: raise ValueError("Need at least 2 records to sample 'other data randomly' as requested.") dataset_message_candidate: List[List[Dict[str, str]]] = [] for i, data in enumerate(datasets): # from current data, we always take injection (as in your text) inj_cur = data.get("injection", "") # (you also said extract inst/cont/ans from each data, but then you override inst/cont/ans by sampling; # we keep the extraction below for completeness, but we follow your sampling behavior.) _inst_cur = data.get("instruction", "") _cont_cur = data.get("input", "") _ans_cur = data.get("output", "") # Create 7 independent lists (NOT [[]]*7 which aliases) message_list: List[List[Dict[str, str]]] = [[] for _ in range(7)] # --- "2nd run": sample inst/cont/ans from OTHER data (paired) j = rng.randrange(n - 1) if j >= i: j += 1 other1 = datasets[j] inst_o1 = other1.get("instruction", "") cont_o1 = other1.get("input", "") ans_o1 = other1.get("output", "") inst_wrapped = f"{inst_o1}" inj_wrapped = f"{inj_cur}" # message_list[0] += user(inst) + tool(cont) + assistant(ans) message_list[0].extend(user(inst_wrapped)) message_list[0].extend(tool(cont_o1)) message_list[0].extend(assistant(ans_o1)) # message_list[1] += user(inst + cont) + assistant(ans) message_list[1].extend(user(inst_wrapped + cont_o1)) message_list[1].extend(assistant(ans_o1)) # message_list[2] += user(inst + cont + inj) + assistant(ans) message_list[2].extend(user(inst_wrapped + cont_o1 + inj_wrapped)) message_list[2].extend(assistant(ans_o1)) # message_list[3] += user(inst) + tool(cont + inj) + assistant(ans) message_list[3].extend(user(inst_wrapped)) message_list[3].extend(tool(cont_o1 + inj_wrapped)) message_list[3].extend(assistant(ans_o1)) # --- Now sample inst2/cont2 from OTHER data (paired) k = rng.randrange(n - 1) if k >= i: k += 1 other2 = datasets[k] inst2 = other2.get("instruction", "") cont2 = other2.get("input", "") # NOTE: your pseudocode says: # inst , inj= f"{inst2}", f"{cont2}" # i.e. "inj" becomes wrapped cont2 (yes, weird, but we follow it). inst2_wrapped = f"{inst2}" cont2_wrapped = cont2 # You reference cont from earlier; in your pseudocode it's `cont` from the first sampled other1. cont_base = cont_o1 # message_list[4] += user(inst) + tool(suffile([cont, inst2])) + assistant(ans) message_list[4].extend(user(inst2_wrapped)) message_list[4].extend(tool(suffile([cont_base, inst2]))) message_list[4].extend(assistant(ans_o1)) # message_list[5] += user(inst) + tool(suffile([cont, cont2])) + assistant(ans) message_list[5].extend(user(inst2_wrapped)) message_list[5].extend(tool(suffile([cont_base, cont2]))) message_list[5].extend(assistant(ans_o1)) # message_list[6] += user(inst) + tool(suffile([cont, inst2, const2])) + assistant(ans) # Your pseudocode has a typo "const2"; interpret as cont2. message_list[6].extend(user(inst2_wrapped)) message_list[6].extend(tool(suffile([cont_base, inst2, cont2]))) message_list[6].extend(assistant(ans_o1)) dataset_message_candidate.extend(message_list) return dataset_message_candidate def build_output_dataset( datasets: List[Dict[str, Any]], dataset_message_candidate: List[List[Dict[str, str]]], rng: random.Random, ) -> List[Dict[str, Any]]: """ output_dataset: per your pseudocode, for each record in datasets we build one temp_message_list by concatenating random(1,10) candidates; after each concat, remove last assistant(ans). We store each produced conversation as a JSONL row: {"messages": [...]} """ output_rows: List[Dict[str, Any]] = [] for candidate in dataset_message_candidate: temp_message_list: List[Dict[str, str]] = [] # random(1,10) -> interpret as randint(1, 9) because Python range(1,10) yields 1..9 for _i in range(rng.randrange(1, 2)): if _i == 0: cand = candidate else: cand = rng.choice(dataset_message_candidate) temp_message_list.extend(cand) # remove last assistant(ans) if temp_message_list and temp_message_list[-1].get("role") == "assistant": temp_message_list.pop() output_rows.append({"messages": temp_message_list}) return output_rows def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--orig-inj-datapath", required=True, help="Path to original JSON dataset (list of dicts).") ap.add_argument("--output-path", required=True, help="Output JSONL path.") args = ap.parse_args() in_path = Path(args.orig_inj_datapath) out_path = Path(args.output_path) datasets = read_json(in_path) if not isinstance(datasets, list): raise ValueError("Input JSON must be a list of records (dict).") # Extract fields as you requested (even though later sampling uses 'other data') # This also sanity-checks schema early. for idx, d in enumerate(datasets): if not isinstance(d, dict): raise ValueError(f"Record {idx} is not a dict.") for key in ("instruction", "input", "output", "injection"): if key not in d: raise ValueError(f"Record {idx} missing required key: {key}") rng = random.Random(42) dataset_message_candidate = build_candidates(datasets, rng) print(len(dataset_message_candidate)) output_rows = build_output_dataset(datasets, dataset_message_candidate, rng) write_jsonl(out_path, output_rows) print(f"Wrote {len(output_rows)} conversations to {out_path}") if __name__ == "__main__": main()