63 lines
2.1 KiB
Python
63 lines
2.1 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import json
|
|
import os
|
|
|
|
|
|
def save_merge_json(data_lis, file_path):
|
|
with open(file_path, 'wt', encoding='utf-8') as file:
|
|
json.dump(data_lis, file, ensure_ascii=False, separators=(',\n',':'))
|
|
|
|
|
|
def get_all_file_paths(folder_path, file_type='.jsonl'):
|
|
# 确保传入的是一个目录
|
|
if not os.path.isdir(folder_path):
|
|
raise ValueError(f"{folder_path} is not a valid directory")
|
|
|
|
# 获取文件夹下所有文件的路径
|
|
file_paths = [os.path.join(folder_path, file) for file in os.listdir(
|
|
folder_path) if os.path.isfile(os.path.join(folder_path, file)) and (file_type in file)]
|
|
return file_paths
|
|
|
|
|
|
if __name__ == '__main__':
|
|
conversion_lis = []
|
|
|
|
folder_path = r'./' # python merge_jsonl.py > curr.txt
|
|
|
|
merge_path = folder_path.split('/')[-1]
|
|
try:
|
|
merge_last_path = folder_path.split('/')[-2] if folder_path.split('/')[-2]!='.' else ''
|
|
except:
|
|
merge_last_path = ''
|
|
print(f'merge_path={merge_path},merge_last_path={merge_last_path}')
|
|
|
|
|
|
for path in get_all_file_paths(folder_path):
|
|
print(path.encode("utf-8"))
|
|
|
|
with open(path, 'rt', encoding='utf-8') as file:
|
|
for line in file:
|
|
# # 移除行尾的换行符
|
|
# if line == '\n':
|
|
# line = line.rstrip('\n')
|
|
line = line.rstrip('\n')
|
|
# 解析JSON
|
|
try:
|
|
data = json.loads(line)
|
|
conversion_lis.append(data)
|
|
# conversion_lis.append('\n')
|
|
except json.JSONDecodeError as e:
|
|
print(f"Error decoding JSON: {e}")
|
|
|
|
if merge_last_path!='':
|
|
save_merge_json_path = rf'./{merge_last_path}/{merge_path}_merge.json'
|
|
elif merge_path!='':
|
|
save_merge_json_path = rf'./{merge_path}_merge.json'
|
|
else:
|
|
save_merge_json_path = rf'./curr_merge.json'
|
|
|
|
save_merge_json(data_lis=conversion_lis,
|
|
file_path=save_merge_json_path)
|
|
print(len(conversion_lis),save_merge_json_path)
|