17 lines
601 B
Python
17 lines
601 B
Python
ORIG_DATASET_PATH="../1_raw_dataset/Should-It-Be-Executed-Or-Processed/datasets/train_dataset.json"
|
|
OUT_PATH="sep_head_ident.jsonl"
|
|
import json
|
|
datas = json.loads(open(ORIG_DATASET_PATH).read())
|
|
|
|
out_file = open(OUT_PATH,"w")
|
|
|
|
def craft_get_data(data):
|
|
inst = data["system_prompt"]
|
|
data = data["data_prompt_clean"]
|
|
if len(data.split(" ")) < 3:
|
|
return
|
|
inst_data = f"<inst>{inst}</inst> <data>{data}</data>"
|
|
inv_inst_data = f"<data>{data}</data> <inst>{inst}</inst>"
|
|
out_file.write(json.dumps([inst_data,inv_inst_data]) + "\n")
|
|
for data in datas:
|
|
craft_get_data(data) |