85 lines
2.8 KiB
Python
85 lines
2.8 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
import json
|
|||
|
|
import os
|
|||
|
|
from tqdm import tqdm
|
|||
|
|
import SparkApi
|
|||
|
|
|
|||
|
|
# 输入文件路径
|
|||
|
|
input_file = 'output/train_expanded.jsonl'
|
|||
|
|
# 断点文件路径
|
|||
|
|
checkpoint_file = 'output/expand_checkpoint.txt'
|
|||
|
|
# 临时文件路径
|
|||
|
|
temp_file = 'output/tmp_train_expanded.jsonl'
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 调用API生成回答
|
|||
|
|
def generate_answer_via_api(question):
|
|||
|
|
appid = "48d04aae"
|
|||
|
|
api_secret = "ZDE1ZGZmNTQ1YWYxZjcxYTI5Mjk0NGIz"
|
|||
|
|
api_key = "3ad87d03c4e3a4fb7d7b36a7dfa3be00"
|
|||
|
|
domain = "4.0Ultra"
|
|||
|
|
Spark_url = "wss://spark-api.xf-yun.com/v4.0/chat"
|
|||
|
|
|
|||
|
|
prompt = (
|
|||
|
|
f"你是一位油橄榄栽培领域的专家，需要基于给定内容生成高质量的问答对。"
|
|||
|
|
f"生成的问答对用于油橄榄知识库微调，请确保问答的准确性和相关性。具体要求如下：\n"
|
|||
|
|
f"每个回答应该准确且不超过50字，同时不少于20字，以保证内容的简洁和有用性。\n"
|
|||
|
|
f"问题：{question}\n\n"
|
|||
|
|
f"请生成一个详细回答。"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
question_data = [{"role": "user", "content": prompt}]
|
|||
|
|
SparkApi.answer = ""
|
|||
|
|
SparkApi.main(appid, api_key, api_secret, Spark_url, domain, question_data)
|
|||
|
|
return SparkApi.answer.strip()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 加载断点进度
|
|||
|
|
def load_checkpoint():
|
|||
|
|
if os.path.exists(checkpoint_file):
|
|||
|
|
with open(checkpoint_file, 'r') as f:
|
|||
|
|
return int(f.read().strip()) # 返回已处理的行索引
|
|||
|
|
return 0 # 没有断点则从0开始
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 保存断点进度
|
|||
|
|
def save_checkpoint(index):
|
|||
|
|
with open(checkpoint_file, 'w') as f:
|
|||
|
|
f.write(str(index))
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
# 加载断点进度
|
|||
|
|
start_index = load_checkpoint()
|
|||
|
|
|
|||
|
|
with open(input_file, 'r', encoding='utf-8') as f, open(temp_file, 'w', encoding='utf-8') as temp_f:
|
|||
|
|
for i, line in enumerate(tqdm(f)):
|
|||
|
|
item = json.loads(line)
|
|||
|
|
|
|||
|
|
# 从断点开始处理
|
|||
|
|
if i >= start_index:
|
|||
|
|
input_content = item['input']
|
|||
|
|
output_content = item['output']
|
|||
|
|
|
|||
|
|
# # 检查是否是未提供回答的问答对
|
|||
|
|
# if "未给" in output_content:
|
|||
|
|
# # 使用API生成新的回答
|
|||
|
|
# new_answer = generate_answer_via_api(input_content)
|
|||
|
|
# item['output'] = new_answer
|
|||
|
|
|
|||
|
|
if len(output_content)<11:
|
|||
|
|
# 使用API生成新的回答
|
|||
|
|
new_answer = generate_answer_via_api(input_content)
|
|||
|
|
item['output'] = new_answer
|
|||
|
|
|
|||
|
|
# 保存当前的进度索引
|
|||
|
|
save_checkpoint(i)
|
|||
|
|
|
|||
|
|
# 写入更新内容到临时文件
|
|||
|
|
json.dump(item, temp_f, ensure_ascii=False)
|
|||
|
|
temp_f.write('\n')
|
|||
|
|
|
|||
|
|
# 替换原始文件
|
|||
|
|
os.replace(temp_file, input_file)
|
|||
|
|
print(f"已更新 {input_file} 文件，包含重新生成的回答。")
|