update three merge_json*.py files and corresponding tutorial in CN and EN

update three merge_json*.py files and corresponding tutorial in CN and EN

.gitignore

@@ -7,6 +7,8 @@ pdf/
 .idea/
 
 *.jsonl
+*.json
+*.txt
 # ./generate_data/*.josnl
 # ./generate_data/*/*/*.josnl
 

generate_data/merge_json.py

@@ -0,0 +1,40 @@
+import json
+import os
+
+
+def save_merge_json(data_lis, file_path):
+    import json
+
+    with open(file_path, 'wt', encoding='utf-8') as file:
+        json.dump(data_lis, file, ensure_ascii=False)
+
+
+def get_all_file_paths(folder_path):
+    # 确保传入的是一个目录
+    if not os.path.isdir(folder_path):
+        raise ValueError(f"{folder_path} is not a valid directory")
+
+    # 获取文件夹下所有文件的路径
+    file_paths = [os.path.join(folder_path, file) for file in os.listdir(
+        folder_path) if os.path.isfile(os.path.join(folder_path, file))]
+    return file_paths
+
+
+if __name__ == '__main__':
+    conversion_lis = []
+
+    for path in get_all_file_paths(r'data\res-aiwei'):
+        print(path)
+
+        with open(path, 'rt', encoding='utf-8') as file:
+            for line in file:
+                # 移除行尾的换行符
+                line = line.rstrip('\n')
+                # 解析JSON
+                try:
+                    data = json.loads(line)
+                    conversion_lis.append(data)
+                except json.JSONDecodeError as e:
+                    print(f"Error decoding JSON: {e}")
+        save_merge_json(data_lis=conversion_lis,
+                        file_path=r'.\merge.json')

generate_data/merge_jsonl.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-  
+
+import json
+import os
+
+
+def save_merge_json(data_lis, file_path):
+    with open(file_path, 'wt', encoding='utf-8') as file:
+        json.dump(data_lis, file, ensure_ascii=False, separators=(',\n',':'))
+
+
+def get_all_file_paths(folder_path, file_type='.jsonl'):
+    # 确保传入的是一个目录
+    if not os.path.isdir(folder_path):
+        raise ValueError(f"{folder_path} is not a valid directory")
+
+    # 获取文件夹下所有文件的路径
+    file_paths = [os.path.join(folder_path, file) for file in os.listdir(
+        folder_path) if os.path.isfile(os.path.join(folder_path, file)) and (file_type in file)]
+    return file_paths
+
+
+if __name__ == '__main__':
+    conversion_lis = []
+    
+    folder_path = r'./'  # python merge_jsonl.py > curr.txt
+    
+    merge_path = folder_path.split('/')[-1]
+    try:
+        merge_last_path = folder_path.split('/')[-2] if folder_path.split('/')[-2]!='.' else ''
+    except:
+        merge_last_path = '' 
+    print(f'merge_path={merge_path},merge_last_path={merge_last_path}')
+        
+
+    for path in get_all_file_paths(folder_path):
+        print(path.encode("utf-8"))
+
+        with open(path, 'rt', encoding='utf-8') as file:
+            for line in file:
+                # # 移除行尾的换行符
+                # if line == '\n':
+                #     line = line.rstrip('\n')
+                line = line.rstrip('\n')
+                # 解析JSON
+                try:
+                    data = json.loads(line)
+                    conversion_lis.append(data)
+                    # conversion_lis.append('\n')
+                except json.JSONDecodeError as e:
+                    print(f"Error decoding JSON: {e}")
+                    
+    if merge_last_path!='':
+        save_merge_json_path = rf'./{merge_last_path}/{merge_path}_merge.json'  
+    elif merge_path!='':
+        save_merge_json_path = rf'./{merge_path}_merge.json'
+    else:
+        save_merge_json_path = rf'./curr_merge.json'
+                    
+    save_merge_json(data_lis=conversion_lis,
+                    file_path=save_merge_json_path)
+    print(len(conversion_lis),save_merge_json_path)

generate_data/merge_jsonl_r.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-  
+
+import json
+import os
+
+
+def save_merge_json(data_lis, file_path):
+    with open(file_path, 'wt', encoding='utf-8') as file:
+        json.dump(data_lis, file, ensure_ascii=False, separators=(',\n',':'))
+
+
+def get_all_file_paths(folder_path, file_type='.jsonl'):
+    # 确保传入的是一个目录
+    if not os.path.isdir(folder_path):
+        raise ValueError(f"{folder_path} is not a valid directory")
+
+    # 获取文件夹下所有文件的路径
+    file_paths = [os.path.join(folder_path, file) for file in os.listdir(
+        folder_path) if os.path.isfile(os.path.join(folder_path, file)) and (file_type in file)]
+    return file_paths
+
+
+if __name__ == '__main__':
+    
+    data_ai = 'qwen'  # python merge_jsonl_r.py > qwen.txt
+    # data_ai = 'zhipuai'  # python merge_jsonl_r.py > zhipuai.txt
+    root_dir  = rf'./{data_ai}/'
+    
+    save_final_merge_json_path = f'{data_ai}_final_merge.json'
+
+    subfolders = [os.path.join(root_dir, d) for d in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, d))]
+
+    final_list = []
+    for folder_path in subfolders:
+        conversion_lis = []
+        merge_path = folder_path.split('/')[-1]
+        try:
+            merge_last_path = folder_path.split('/')[-2] if folder_path.split('/')[-2]!='.' else ''
+        except:
+            merge_last_path = '' 
+        print(f'merge_path={merge_path},merge_last_path={merge_last_path}'.encode("utf-8"))
+            
+
+        for path in get_all_file_paths(folder_path):
+            print(path.encode("utf-8"))
+
+            with open(path, 'rt', encoding='utf-8') as file:
+                for line in file:
+                    # # 移除行尾的换行符
+                    # if line == '\n':
+                    #     line = line.rstrip('\n')
+                    line = line.rstrip('\n')
+                    # 解析JSON
+                    try:
+                        data = json.loads(line)
+                        conversion_lis.append(data)
+                        # conversion_lis.append('\n')
+                    except json.JSONDecodeError as e:
+                        print(f"Error decoding JSON: {e}")
+                        
+            if merge_last_path!='':
+                save_merge_json_path = rf'./{merge_last_path}/{merge_path}_merge.json'  
+            elif merge_path!='':
+                save_merge_json_path = rf'./{merge_path}_merge.json'
+            else:
+                save_merge_json_path = rf'./curr_merge.json'
+                            
+            save_merge_json(data_lis=conversion_lis,
+                            file_path=save_merge_json_path)
+        
+        final_list = final_list+conversion_lis
+        print(f'{len(conversion_lis)},{len(final_list)},{save_merge_json_path}'.encode("utf-8"))
+        
+    save_merge_json(data_lis=final_list,file_path=save_final_merge_json_path)
+    print(len(conversion_lis),save_final_merge_json_path.encode("utf-8"))
+        
+        

generate_data/tutorial.md

@@ -22,7 +22,7 @@
 
 ## **三、实践步骤**
 
-1. **初始化**
+### 1. **初始化**
 
 * 安装所需的软件和库
 
@@ -34,49 +34,62 @@
 
   可参见 `config.yml`均有注释
 
-2. **模型选择与配置**
+### 2. **模型选择与配置**
 
 * 根据需求选择适合的模型
   为了使大家都能够玩上大模型，我们选用InterLLM2-7B作为我们的基线模型（消费级显卡也可部署微调的哦）
 * 对模型进行必要的配置和调整
   根据我们的数据集以及配置策略，使用XTuner进行微调
 
-3. **数据生成**
+### 3. **数据生成**
+
+#### **三种改进前的数据生成方法**
 
 * 使用通义千问大模型进行数据生成
   
-  ```bash
+```bash
   # 终端运行
   bash run_qwen.bash
-
+```
-  # 或者不使用终端运行
-  python qwen_gen_data_NoBash.py
-  ```
 
 * 使用百度文心大模型进行数据生成
 
-  ```bash
+```bash
   # 终端运行
   python ernie_gen_data.py
-  ```
+```
-
-* 使用智谱GLM大模型进行数据生成
-  
-  ```bash
-  # 终端运行
-  python zhipuai_gen_data.py
-  ```
 
 * 使用讯飞星火大模型进行数据生成
   
-  ```bash
+```bash
   # 终端运行
   python ./xinghuo/gen_data.py
-  ```
+```
 
-1. **自我认知数据集的整合**
+#### **改进的两种数据生成方法**
+
+采用改进的数据生成方法生成多轮对话时，首先需要定义`ai_tool`变量，该变量表示LLM模型的名称（`qwen`或`zhipuai`）。根据`ai_tool`变量的值，创建一个`{ai_tool}`文件夹。
+
+然后，遍历所有的`area`值，接着根据不同的`emotion`值生成多轮对话。生成的对话会每隔`save_interval`次迭代写入到`./{ai_tool}/{area}/{emotion}.jsonl`文件中。这个过程会重复执行`total_num_each_emo_area`次。
+
+* 使用**改进的**通义千问大模型数据生成方法
+  
+```bash
+   # 或者不使用bash，直接运行
+  python qwen_gen_data_NoBash.py
+```
+
+* 使用**改进的**智谱GLM大模型数据生成方法
+  
+```bash
+  # 终端运行
+  python zhipuai_gen_data.py
+```
+
+### 4. **自我认知数据集的整合**
 
 * 自我认知数据集需要按照格式手动生成，如下格式即可
+
   ```json
   [
       {
@@ -98,19 +111,27 @@
   ]
   ```
 
-5. **数据集整合**
+### 5. **数据集整合**
 
-   在进行数据集整合之前，我们要检查生成的数据是否存在格式错误，类型不符合等情况。
+#### Case 1: 使用`python ernie_gen_data.py`、`bash run_qwen.bash`或者`python ./xinghuo/gen_data.py`
 
-* 首先使用`check.py`进行数据检查。
+* 首先使用`check.py`进行数据检查。在进行数据集整合之前，我们要检查生成的数据是否存在格式错误，类型不符合等情况。
-* 然后使用`merge_json.py`将所有的json整合为一个总的json文件。
+* 然后使用`merge_json.py`将所有的json（或者使用`merge_jsonl.py`将所有的jsonl）文件整合为一个总的json文件。
 
-6. **评估与优化**
+#### Case 2: 使用`python qwen_gen_data_NoBash.py`或者`python zhipuai_gen_data.py`
+
+在这种情况下，我们需要在使用两种改进的生成方法生成多轮对话后，将`{data_ai}`文件夹下所有`{area}`子文件夹中的所有`{emotion}.jsonl`文件合并为`{data_ai}_final_merge.json`文件。
+
+* 由于采用了改进的数据生成方法和不同的存储生成对话结构，因此我们可以免除对数据集的检查。
+* 然后使用`merge_jsonl_r.py`将`qwen`或者`zhipuai`定义为`data_ai`变量，并将其文件夹下所有领域（`area`）下所有的jsonl文件整合为一个总的json文件并取名为`{area}_merge.json`,最终在`{data_ai}`文件夹下生成`{data_ai}_final_merge.json`。
+* 然后我们可以手动合成`qwen_final_merge.json`和`zhipuai_final_merge.json`为`qwen_zhipuai_final_merge.json`文件了， 注意合并后的json文件夹中，最外面只有一对`[]`，中间是`{}`包裹的多轮对话。
+
+### 6. **评估与优化**
 
 * 使用适当的评估指标对生成的数据集进行评估
 * 根据评估结果进行必要的优化和调整
 
-7. **测试与部署**
+### 7. **测试与部署**
 
 * 使用独立测试集对训练好的模型进行评估
 * 根据测试结果进行必要的调整和优化

generate_data/tutorial_EN.md

@@ -22,7 +22,7 @@ In order to have a better representation of our large mental models, we must hav
 
 ## **III. Practical steps**
 
-1. **Initialize**
+### 1. **Initialize**
 
 * Install the required software and libraries
 
@@ -34,7 +34,7 @@ In order to have a better representation of our large mental models, we must hav
 
   See `config.yml` for annotations
 
-2. **Model selection and configuration**
+### 2. **Model selection and configuration**
 
 * Select the right model for your needs
   In order to enable everyone to play with the large model, we chose the InterLLM2-7B as our baseline model (consumer graphics cards can also be deployed fine-tuned oh).
@@ -42,40 +42,52 @@ In order to have a better representation of our large mental models, we must hav
 * Make necessary configurations and adjustments to the model
   Use XTuner for fine-tuning based on our dataset and configuration strategy.
 
-3. **Data generation**
+### 3. **Data generation**
 
-* Data generation using Tongyi Qianwen
+#### **Three original methods for data generation**
 
-  ```bash
+* 1.Data generation using Tongyi Qianwen 
+  
+```bash
   # Terminal operation
   bash run_qwen.bash
+```
 
-  # Or just use python without bash
+* 2.Data generation using Wenxin Yiyan
-  python qwen_gen_data_NoBash.py
-  ```
   
-* Data generation using Wenxin Yiyan
+```bash
-  
-  ```bash
   # Terminal operation
   python ernie_gen_data.py
-  ```
+```
 
-* Data generation using Zhipu GLM
+* 3.Data generation using IFlystar Fire
   
-  ```bash
+```bash
-  # Terminal operation
-  python zhipuai_gen_data.py
-  ```
-
-* Data generation using IFlystar Fire
-  
-  ```bash
   # Terminal operation
   python ./xinghuo/gen_data.py
-  ```
+```
 
-4. **Integration of self-cognition datasets**
+#### **Two improved methods for data generation**
+
+When generating multi-turn dialogues with these two improved methods, the first step is to define the value of the `ai_tool` variable, which represents the LLM model name (`qwen` or `zhipuai`). Based on the value of this `ai_tool` variable, a `{ai_tool}` folder is created. 
+
+Then, all `area` values are traversed, followed by different `emotion` values for generating multi-turn dialogues. The generated dialogues are written to the `./{ai_tool}/{area}/{emotion}.jsonl` file every `save_interval` iterations. This process is repeated `total_num_each_emo_area` times.
+
+* 1.Using the **improved** method for generating data with the Qwen model:
+  
+```bash
+  # Alternatively, you can run it directly without using bash
+  python qwen_gen_data_NoBash.py
+```
+
+* 2.Using the **improved** method for generating data with the Zhipuai GLM-4 model:
+
+```bash
+  # Alternatively, you can run it directly without using bash
+  python zhipuai_gen_data.py
+```
+
+### 4. **Integration of self-cognition datasets**
 
 * Self-cognition dataset this needs to be manually generated in accordance with the format, the following format can be
   
@@ -100,16 +112,27 @@ In order to have a better representation of our large mental models, we must hav
   ]
   ```
 
-5. **dataset integration**
+### 5. **Dataset Integration**
 
-Before dataset integration, we need to check whether the generated data has formatting errors, type mismatches, etc. We need check.py to check the data. Finally, merge_json.py is used to combine all the json into one overall json file.
+#### **Case 1**: Using `python ernie_gen_data.py`, `bash run_qwen.bash`, or `python ./xinghuo/gen_data.py`
 
-6. **Evaluation and optimization**
+* First, use `check.py` to check the data. Before integrating the dataset, we need to check whether the generated data has format errors or type mismatches.
+* Then, use `merge_json.py` to consolidate all json files (or use `merge_jsonl.py` to consolidate all jsonl files) into one overall json file.
+
+#### **Case 2**: Using `python qwen_gen_data_NoBash.py` or `python zhipuai_gen_data.py`
+
+In this case, we need to merge all `{emotion}.jsonl` files in all `{area}` subfolders under the `{data_ai}` folder into `{data_ai}_final_merge.json` after we use two improved generation methods to generate multi-round conversations.
+
+* As we have adopted improved data generation methods and different storage generation dialog structures, we can avoid checking the dataset.
+* Then, use `merge_jsonl_r.py` to define `qwen` or `zhipuai` as the `data_ai` variable, and consolidate all jsonl files in all areas (`area`) into one overall json file named `{area}_merge.json`. Finally, generate `{data_ai}_final_merge.json` in the `{data_ai}` folder.
+* We can then manually merge `qwen_final_merge.json` and `zhipuai_final_merge.json` into `qwen_zhipuai_final_merge.json`. Note that in the merged json file, there is only one pair of `[]` on the outside, and the multi-round dialogues are wrapped in `{}`.
+
+### 6. **Evaluation and optimization**
 
 * Evaluate the generated dataset using appropriate evaluation metrics
 * Make necessary optimizations and adjustments based on the evaluation results
 
-7. **Testing and deployment**
+### 7. **Testing and deployment**
 
 * Evaluate the trained model using an independent test set
 * Make necessary adjustments and optimizations based on test results