From 07527619fdf9f8cf4e22849ac418f8d91c77a8d2 Mon Sep 17 00:00:00 2001 From: Mxode Date: Fri, 8 Mar 2024 19:00:45 +0800 Subject: [PATCH] Calling parameters from config.py --- scripts/qa_generation/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/qa_generation/main.py b/scripts/qa_generation/main.py index 7807434..4324e3e 100644 --- a/scripts/qa_generation/main.py +++ b/scripts/qa_generation/main.py @@ -3,7 +3,7 @@ import json from tqdm import tqdm from datetime import datetime -from config.config import result_dir +from config.config import result_dir, storage_interval, window_size, overlap_size from model.qwen import call_qwen_single_turn from util.logger import get_logger from util.data_loader import get_file_list, get_txt_content, capture_qa @@ -35,7 +35,7 @@ def generate_qa( storage_counter = 0 storage_list = [] for file_path in file_list: - contents = get_txt_content(file_path) + contents = get_txt_content(file_path, window_size=window_size, overlap_size=overlap_size) storage_list = [] _, file_name = os.path.split(file_path) @@ -72,4 +72,4 @@ if __name__ == '__main__': # 创建generated文件夹 os.makedirs('./data/generated', exist_ok=True) - generate_qa() + generate_qa(interval=storage_interval)