first commit

Codes/2-1_head_identification_preprocess/.ipynb_checkpoints/convert_sep-checkpoint.py

@@ -0,0 +1,17 @@
+ORIG_DATASET_PATH="../1_raw_dataset/Should-It-Be-Executed-Or-Processed/datasets/train_dataset.json"
+OUT_PATH="sep_head_ident.jsonl"
+import json
+datas = json.loads(open(ORIG_DATASET_PATH).read())
+
+out_file = open(OUT_PATH,"w")
+
+def craft_get_data(data):
+    inst = data["system_prompt"]
+    data = data["data_prompt_clean"]
+    if len(data.split(" ")) < 3:
+        return
+    inst_data = f"<inst>{inst}</inst> <data>{data}</data>"
+    inv_inst_data = f"<data>{data}</data> <inst>{inst}</inst>"
+    out_file.write(json.dumps([inst_data,inv_inst_data]) + "\n")
+for data in datas:
+    craft_get_data(data)