48 lines
1.7 KiB
Python
48 lines
1.7 KiB
Python
|
# 将微调的数据格式转为预训练的格式
|
|||
|
|
import json
|
|||
|
|
|
|||
|
|
|
|||
|
|
def convert(data_path:str, target_path:str):
|
|||
|
|
# 假设原始JSON数据存储在名为'data.json'的文件中
|
|||
|
|
filename = data_path
|
|||
|
|
|
|||
|
|
# 读取文件内容
|
|||
|
|
with open(filename, 'rt', encoding='utf-8') as file:
|
|||
|
|
original_json = file.read()
|
|||
|
|
|
|||
|
|
# 将原始JSON字符串解析为Python对象
|
|||
|
|
data = json.loads(original_json)
|
|||
|
|
|
|||
|
|
# 遍历每个对话
|
|||
|
|
converted_data = []
|
|||
|
|
|
|||
|
|
# 遍历原始数据中的每个对话对象
|
|||
|
|
for conversation_group in data:
|
|||
|
|
# 遍历每个对话
|
|||
|
|
for dialog in conversation_group["conversation"]:
|
|||
|
|
# 创建一个新的对话对象，用于存储转换后的对话
|
|||
|
|
new_conversation_group = {
|
|||
|
|
"conversation": []
|
|||
|
|
}
|
|||
|
|
# 创建一个新的对话，其中输出被替换为"xxx"
|
|||
|
|
new_dialog = {
|
|||
|
|
"input": '',
|
|||
|
|
"output": f'问题:{dialog["input"]}\n答案:{dialog["output"]}',
|
|||
|
|
}
|
|||
|
|
# 将新的对话添加到新对话对象的列表中
|
|||
|
|
new_conversation_group["conversation"].append(new_dialog)
|
|||
|
|
|
|||
|
|
# 将新对话对象添加到转换后的数据列表中
|
|||
|
|
converted_data.append(new_conversation_group)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 将更新后的数据转换回JSON字符串，并格式化输出
|
|||
|
|
updated_json = json.dumps(converted_data, indent=4, ensure_ascii=False)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 将更新后的JSON数据写入到新的文件中
|
|||
|
|
with open(f'{target_path}', 'wt', encoding='utf-8') as file:
|
|||
|
|
file.write(updated_json)
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
convert(data_path='./output2.json', target_path='pt.json')
|