From dac4b201928773c8fdf281f583fc9f6876da2304 Mon Sep 17 00:00:00 2001 From: MING_X Date: Sat, 4 May 2024 18:08:54 +0800 Subject: [PATCH 1/3] Update README.md --- datasets/README.md | 4 +++ datasets/README_EN.md | 4 +++ rag/pdf2md/README.md | 84 +------------------------------------------ 3 files changed, 9 insertions(+), 83 deletions(-) diff --git a/datasets/README.md b/datasets/README.md index eebb02b..f9390f4 100644 --- a/datasets/README.md +++ b/datasets/README.md @@ -24,6 +24,8 @@ | *General* | multi_turn_dataset_2 | Conversation | 27,000+ | | *General* | single_turn_dataset_1 | QA | 14,000+ | | *General* | single_turn_dataset_2 | QA | 18,300+ | +| *General* | self_cognition_EmoLLM | QA | 85+ | +| *General* | ruozhiba_raw | QA | 240+ | | *Role-play* | aiwei | Conversation | 4000+ | | *Role-play* | SoulStar | QA | 11,200+ | | *Role-play* | tiangou | Conversation | 3900+ | @@ -41,6 +43,8 @@ * 数据集 `multi_turn_dataset_2` 来源 [CPsyCounD](https://github.com/CAS-SIAT-XinHai/CPsyCoun) * 数据集 `single_turn_dataset_1` 来自本项目 * 数据集 `single_turn_dataset_2` 来自本项目 +* 数据集 `self_cognition_EmoLLM` 来自本项目 +* 数据集 `ruozhiba_raw` 来源[COIG-CQIA](https://huggingface.co/datasets/m-a-p/COIG-CQIA/viewer/ruozhiba) ### **Role-play** diff --git a/datasets/README_EN.md b/datasets/README_EN.md index d683561..7180468 100644 --- a/datasets/README_EN.md +++ b/datasets/README_EN.md @@ -22,6 +22,8 @@ | *General* | multi_turn_dataset_2 | Conversation | 27,000+ | | *General* | single_turn_dataset_1 | QA | 14,000+ | | *General* | single_turn_dataset_2 | QA | 18,300+ | +| *General* | self_cognition_EmoLLM | QA | 85+ | +| *General* | ruozhiba_raw | QA | 240+ | | *Role-play* | aiwei | Conversation | 4000+ | | *Role-play* | SoulStar | QA | 11,200+ | | *Role-play* | tiangou | Conversation | 3900+ | @@ -38,6 +40,8 @@ * dataset `multi_turn_dataset_2` from [CPsyCounD](https://github.com/CAS-SIAT-XinHai/CPsyCoun) * dataset `single_turn_dataset_1` from this repo * dataset `single_turn_dataset_2` from this repo +* dataset `self_cognition_EmoLLM` from this repo +* dataset `ruozhiba_raw` from [COIG-CQIA](https://huggingface.co/datasets/m-a-p/COIG-CQIA/viewer/ruozhiba) **Role-play**： * dataset `aiwei` from this repo diff --git a/rag/pdf2md/README.md b/rag/pdf2md/README.md index 6f669b3..b52bab5 100644 --- a/rag/pdf2md/README.md +++ b/rag/pdf2md/README.md @@ -4,90 +4,8 @@ 通过使用doc2x的库，实现将pdf文件转换为结构化md文档。 -通过代码调用(需要提供api_key)： +通过代码调用(需要提供api_key)，详见代码`pdf2md.py` -~~~python -import requests as rq -import json -import os -import zipfile - -class PDF2MD: - def __init__(self, api_key): - self.api_key = api_key - self.url = "https://api.doc2x.noedgeai.com/api/v1/pdf" - self.export_url = "https://api.doc2x.noedgeai.com/api/export" - - def convert(self, filepath, to="md"): - filename = os.path.splitext(os.path.basename(filepath))[0] - - res = rq.post(self.url, files={"file": open(filepath, "rb")}, headers={"Authorization": "Bearer " + self.api_key}, stream=True) - - if res.status_code == 200: - txt_path = filename + ".txt" - with open(txt_path, "w", encoding="utf-8") as f: - for line in res.iter_lines(): - if len(line) > 0: - decoded_line = line.decode("utf-8") - f.write(decoded_line + "\n") - print(decoded_line) - - uuid = json.loads(decoded_line.replace("data: ", ''))['uuid'] - print(uuid) - - if to == "md" or to == 'latex': - path = filename + '.zip' - elif to == 'docx': - path = filename + '.docx' - - export_url = self.export_url + "?request_id=" + uuid + "&to=" + to - res = rq.get(export_url, headers={"Authorization": "Bearer " + self.api_key}) - - if res.status_code == 200: - with open(path, "wb") as f: - f.write(res.content) - print("下载成功,存入:", path) - - if to == "md" or to == 'latex': - zip_file = zipfile.ZipFile(path) - - # 创建以原始文件名命名的文件夹 - if not os.path.exists(filename): - os.mkdir(filename) - - # 解压到该文件夹内 - for names in zip_file.namelist(): - zip_file.extract(names, filename) - zip_file.close() - - # 找到解压后的md文件 - for file in os.listdir(filename): - if file.endswith(".md"): - extracted_md = os.path.join(filename, file) - break - - # 重命名md文件 - new_md_name = os.path.join(filename, filename+'.md') - os.rename(extracted_md, new_md_name) - print("解压并重命名md文件为:", new_md_name) - - else: - print(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text))) - else: - print(format("[ERROR] status code: %d, body: %s" % (res.status_code, res.text))) - - - -def main(): - api_key = "sk-xxx" - filepath = r"test.pdf" - converter = PDF2MD(api_key) - converter.convert(filepath, to="md") - - -if __name__ == "__main__": - main() -~~~ ## 通过网页使用在线PDF2MD服务： From de94bcb499b5bf68fed2f7c76492c59ab3dd8290 Mon Sep 17 00:00:00 2001 From: MING_X Date: Sat, 4 May 2024 18:27:18 +0800 Subject: [PATCH 2/3] Update README.md --- README.md | 10 +++------- README_EN.md | 5 +++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c6625b5..ffd9e15 100644 --- a/README.md +++ b/README.md @@ -122,12 +122,6 @@ 模型下载量

-- 【2024.2.5】 项目荣获公众号**NLP工程化**推文宣传[推文链接](https://mp.weixin.qq.com/s/78lrRl2tlXEKUfElnkVx4A)，为博主推广一波，欢迎大家关注！！🥳🥳 - -

- 公众号二维码 -

- - 【2024.2.3】 [项目宣传视频](https://www.bilibili.com/video/BV1N7421N76X/)完成 😊 - 【2024.1.27】 完善数据构建文档、微调指南、部署指南、Readme等相关文档 👏 - 【2024.1.25】 EmoLLM V1.0 已部署上线 https://openxlab.org.cn/apps/detail/jujimeizuo/EmoLLM 😀 @@ -143,7 +137,9 @@ 浦语挑战赛创新创意奖

-- 项目荣获公众号**NLP工程化**[推文宣传](https://mp.weixin.qq.com/s/78lrRl2tlXEKUfElnkVx4A) +- 🎉感谢以下媒体及公众号朋友对本项目的报道和支持(以下排名不分先后! 若有遗漏、十分抱歉, 一并感激! 欢迎补充!): [NLP工程化](https://mp.weixin.qq.com/s/78lrRl2tlXEKUfElnkVx4A), [机智流](https://mp.weixin.qq.com/s/_wMCmssRMGd0Oz5OVVkjAA), [爱可可爱生活](https://mp.weixin.qq.com/s/4WaCg4OpkCWXEuWHuV4r3w), [阿郎小哥](https://mp.weixin.qq.com/s/_MSMeL1XHP0v5lDi3YaPVw), [大模型日知路](https://mp.weixin.qq.com/s/FYYibsCXtfU6FFM9TuKILA), [AI Code](https://mp.weixin.qq.com/s/yDWGY3S4CwCi6U_irsFmqA) 等! + +- 项目宣传视频 [EmoLLM](https://www.bilibili.com/video/BV1N7421N76X/) 已发布，欢迎大家围观 😀 ### 🎯路线图 diff --git a/README_EN.md b/README_EN.md index 2ccccf3..9e6b833 100644 --- a/README_EN.md +++ b/README_EN.md @@ -149,9 +149,10 @@ The Model aims to fully understand and promote the mental health of individuals, Challenge Innovation and Creativity Award

+- 🎉 Thanks to the following media and friends for their coverage and support of our project(Listed below in no particular order! Sorry for any omissions, we appreciate it! Feel free to add!): [NLP工程化](https://mp.weixin.qq.com/s/78lrRl2tlXEKUfElnkVx4A), [机智流](https://mp.weixin.qq.com/s/_wMCmssRMGd0Oz5OVVkjAA), [爱可可爱生活](https://mp.weixin.qq.com/s/4WaCg4OpkCWXEuWHuV4r3w), [阿郎小哥](https://mp.weixin.qq.com/s/_MSMeL1XHP0v5lDi3YaPVw), [大模型日知路](https://mp.weixin.qq.com/s/FYYibsCXtfU6FFM9TuKILA), [AI Code](https://mp.weixin.qq.com/s/yDWGY3S4CwCi6U_irsFmqA), etc! -- The project has been promoted by the official WeChat account **NLP Engineering**. Here's the [link](https://mp.weixin.qq.com/s/78lrRl2tlXEKUfElnkVx4A). - +- Project Vedio [EmoLLM](https://www.bilibili.com/video/BV1N7421N76X/) has been released for viewing! 😀 + ### Roadmap

From c9791814cf64303f0287c4adcb6c0c0b3d8090a4 Mon Sep 17 00:00:00 2001 From: aJupyter Date: Tue, 7 May 2024 23:11:37 +0800 Subject: [PATCH 3/3] add Incremental Pre-training Guide --- README.md | 1 + README_EN.md | 1 + xtuner_config/pt/README.md | 36 ++++ xtuner_config/pt/ft2pt.py | 48 +++++ .../pt/internlm2_chat_1_8b_qlora_e3_pt.py | 202 ++++++++++++++++++ 5 files changed, 288 insertions(+) create mode 100644 xtuner_config/pt/README.md create mode 100644 xtuner_config/pt/ft2pt.py create mode 100644 xtuner_config/pt/internlm2_chat_1_8b_qlora_e3_pt.py diff --git a/README.md b/README.md index ffd9e15..24cbfed 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ ### 🎇最近更新 +- 【2024.5.7】[增量预训练指南](xtuner_config/pt/README.md) - 【2024.4.20】[LLAMA3微调指南](xtuner_config/README_llama3_8b_instruct_qlora_alpaca_e3_M.md)及基于[LLaMA3_8b_instruct的艾薇](https://openxlab.org.cn/models/detail/ajupyter/EmoLLM-LLaMA3_8b_instruct_aiwei)开源 - 【2023.4.14】新增[快速开始](docs/quick_start.md)和保姆级教程[BabyEmoLLM](Baby_EmoLLM.ipynb) - 【2024.4.2】在 Huggingface 上传[老母亲心理咨询师](https://huggingface.co/brycewang2018/EmoLLM-mother/tree/main) diff --git a/README_EN.md b/README_EN.md index 9e6b833..f2875a1 100644 --- a/README_EN.md +++ b/README_EN.md @@ -101,6 +101,7 @@ The Model aims to fully understand and promote the mental health of individuals, ### Recent Updates +- [2024.5.7][Incremental Pre-training Guide](xtuner_config/pt/README.md) - [2024.4.20] [LLAMA3 fine-tuning guide](xtuner_config/README_llama3_8b_instruct_qlora_alpaca_e3_M.md) and based on [LLaMA3_8b_instruct's aiwei](https://openxlab.org.cn/models/detail/ajupyter/EmoLLM-LLaMA3_8b_instruct_aiwei) open source - [2023.4.14] Added [Quick Start](docs/quick_start_EN.md) and Nanny level tutorial [BabyEmoLLM](Baby_EmoLLM.ipynb) - [2024.4.2] Uploaded at Huggingface [Old Mother Counsellor](https://huggingface.co/brycewang2018/EmoLLM-mother/tree/main) diff --git a/xtuner_config/pt/README.md b/xtuner_config/pt/README.md new file mode 100644 index 0000000..ee2b0e9 --- /dev/null +++ b/xtuner_config/pt/README.md @@ -0,0 +1,36 @@ +# 增量预训练教程 + +# 增量预训练简介 +增量预训练旨在提升模型在特定领域或任务的能力。 + + +# 预训练流程 +- Step1 处理数据 +- Step2 配置config（全量、Lora、Qlora） +- Step3 启动训练（单卡、多卡、是否使用deepspeed） +- Step4 模型合成 +- Step5 模型测试 +- Step6 模型上传 + +# EmoLLM增量预训练教程 +基于微调中的数据集[datasets](../../datasets)修改而来 + +- Step1 修改`ft2pt.py`中的文件路径 +这里以[output2.json](../../datasets/processed/output2.json)为例，运行脚本生成[pt.json](../../datasets/pt/pt.json) + +- Step2 [config](./internlm2_chat_1_8b_qlora_e3_pt.py) +注意：本config采用了**变长注意力 (Variable Length Attention)** +需要安装flash_attn +`MAX_JOBS=4 pip install flash-attn --no-build-isolation` + + +- Step3 训练： +``` +# On a single GPU +xtuner train internlm2_chat_1_8b_qlora_e3_pt.py --deepspeed deepspeed_zero2 +# On multiple GPUs +(DIST) NPROC_PER_NODE=${GPU_NUM} xtuner train internlm2_chat_1_8b_qlora_e3_pt.py --deepspeed deepspeed_zero2 +(SLURM) srun ${SRUN_ARGS} xtuner train internlm2_chat_1_8b_qlora_e3_pt.py --launcher slurm --deepspeed deepspeed_zero2 +``` + +- 其余流程请参考[微调教程](../../xtuner_config/README.md) \ No newline at end of file diff --git a/xtuner_config/pt/ft2pt.py b/xtuner_config/pt/ft2pt.py new file mode 100644 index 0000000..4151f5c --- /dev/null +++ b/xtuner_config/pt/ft2pt.py @@ -0,0 +1,48 @@ +# 将微调的数据格式转为预训练的格式 +import json + + +def convert(data_path:str, target_path:str): + # 假设原始JSON数据存储在名为'data.json'的文件中 + filename = data_path + + # 读取文件内容 + with open(filename, 'rt', encoding='utf-8') as file: + original_json = file.read() + + # 将原始JSON字符串解析为Python对象 + data = json.loads(original_json) + + # 遍历每个对话 + converted_data = [] + + # 遍历原始数据中的每个对话对象 + for conversation_group in data: + # 遍历每个对话 + for dialog in conversation_group["conversation"]: + # 创建一个新的对话对象，用于存储转换后的对话 + new_conversation_group = { + "conversation": [] + } + # 创建一个新的对话，其中输出被替换为"xxx" + new_dialog = { + "input": '', + "output": f'问题:{dialog["input"]}\n答案:{dialog["output"]}', + } + # 将新的对话添加到新对话对象的列表中 + new_conversation_group["conversation"].append(new_dialog) + + # 将新对话对象添加到转换后的数据列表中 + converted_data.append(new_conversation_group) + + + # 将更新后的数据转换回JSON字符串，并格式化输出 + updated_json = json.dumps(converted_data, indent=4, ensure_ascii=False) + + + # 将更新后的JSON数据写入到新的文件中 + with open(f'{target_path}', 'wt', encoding='utf-8') as file: + file.write(updated_json) + +if __name__ == '__main__': + convert(data_path='./output2.json', target_path='pt.json') \ No newline at end of file diff --git a/xtuner_config/pt/internlm2_chat_1_8b_qlora_e3_pt.py b/xtuner_config/pt/internlm2_chat_1_8b_qlora_e3_pt.py new file mode 100644 index 0000000..d37b476 --- /dev/null +++ b/xtuner_config/pt/internlm2_chat_1_8b_qlora_e3_pt.py @@ -0,0 +1,202 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from datasets import load_dataset +from mmengine.dataset import DefaultSampler +from mmengine.hooks import (CheckpointHook, DistSamplerSeedHook, IterTimerHook, + LoggerHook, ParamSchedulerHook) +from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR, LinearLR +from peft import LoraConfig +from torch.optim import AdamW +from transformers import (AutoModelForCausalLM, AutoTokenizer, + BitsAndBytesConfig) + +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import alpaca_map_fn, template_map_fn_factory +from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook, + VarlenAttnArgsToMessageHubHook) +from xtuner.engine.runner import TrainLoop +from xtuner.model import SupervisedFinetune +from xtuner.parallel.sequence import SequenceParallelSampler +from xtuner.utils import PROMPT_TEMPLATE, SYSTEM_TEMPLATE + +####################################################################### +# PART 1 Settings # +####################################################################### +# Model +pretrained_model_name_or_path = '/root/share/new_models/Shanghai_AI_Laboratory/internlm2-chat-1_8b' +use_varlen_attn = True # True + +# Data +data_path = '/root/wxz/work/pt/pt.json' +prompt_template = PROMPT_TEMPLATE.internlm2_chat +max_length = 2048 +pack_to_max_length = True + +# parallel +sequence_parallel_size = 1 + +# Scheduler & Optimizer +batch_size = 1 # per_device +accumulative_counts = 1 +accumulative_counts *= sequence_parallel_size +dataloader_num_workers = 0 +max_epochs = 3 +optim_type = AdamW +lr = 2e-4 +betas = (0.9, 0.999) +weight_decay = 0 +max_norm = 1 # grad clip +warmup_ratio = 0.03 + +# Save +save_steps = 500 +save_total_limit = 2 # Maximum checkpoints to keep (-1 means unlimited) + +# Evaluate the generation performance during the training +evaluation_freq = 500 +SYSTEM = SYSTEM_TEMPLATE.alpaca +evaluation_inputs = [ + '我们经常因为一些小事争吵，他总是忽略我的感受。我感到很孤独，' +] + +####################################################################### +# PART 2 Model & Tokenizer # +####################################################################### +tokenizer = dict( + type=AutoTokenizer.from_pretrained, + pretrained_model_name_or_path=pretrained_model_name_or_path, + trust_remote_code=True, + padding_side='right') + +model = dict( + type=SupervisedFinetune, + use_varlen_attn=use_varlen_attn, + llm=dict( + type=AutoModelForCausalLM.from_pretrained, + pretrained_model_name_or_path=pretrained_model_name_or_path, + trust_remote_code=True, + torch_dtype=torch.float16,), + lora=dict( + type=LoraConfig, + r=64, + lora_alpha=16, + lora_dropout=0.1, + bias='none', + task_type='CAUSAL_LM')) + +####################################################################### +# PART 3 Dataset & Dataloader # +####################################################################### +alpaca_en = dict( + type=process_hf_dataset, + dataset=dict(type=load_dataset, path='json', data_files=dict(train=data_path)), + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=None, + template_map_fn=None, + remove_unused_columns=True, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length, + use_varlen_attn=use_varlen_attn) + +sampler = SequenceParallelSampler \ + if sequence_parallel_size > 1 else DefaultSampler +train_dataloader = dict( + batch_size=batch_size, + num_workers=dataloader_num_workers, + dataset=alpaca_en, + sampler=dict(type=sampler, shuffle=True), + collate_fn=dict(type=default_collate_fn, use_varlen_attn=use_varlen_attn)) + +####################################################################### +# PART 4 Scheduler & Optimizer # +####################################################################### +# optimizer +optim_wrapper = dict( + type=AmpOptimWrapper, + optimizer=dict( + type=optim_type, lr=lr, betas=betas, weight_decay=weight_decay), + clip_grad=dict(max_norm=max_norm, error_if_nonfinite=False), + accumulative_counts=accumulative_counts, + loss_scale='dynamic', + dtype='float16') + +# learning policy +# More information: https://github.com/open-mmlab/mmengine/blob/main/docs/en/tutorials/param_scheduler.md # noqa: E501 +param_scheduler = [ + dict( + type=LinearLR, + start_factor=1e-5, + by_epoch=True, + begin=0, + end=warmup_ratio * max_epochs, + convert_to_iter_based=True), + dict( + type=CosineAnnealingLR, + eta_min=0.0, + by_epoch=True, + begin=warmup_ratio * max_epochs, + end=max_epochs, + convert_to_iter_based=True) +] + +# train, val, test setting +train_cfg = dict(type=TrainLoop, max_epochs=max_epochs) + +####################################################################### +# PART 5 Runtime # +####################################################################### +# Log the dialogue periodically during the training process, optional +custom_hooks = [ + dict(type=DatasetInfoHook, tokenizer=tokenizer), +] + +if use_varlen_attn: + custom_hooks += [dict(type=VarlenAttnArgsToMessageHubHook)] + +# configure default hooks +default_hooks = dict( + # record the time of every iteration. + timer=dict(type=IterTimerHook), + # print log every 10 iterations. + logger=dict(type=LoggerHook, log_metric_by_epoch=False, interval=10), + # enable the parameter scheduler. + param_scheduler=dict(type=ParamSchedulerHook), + # save checkpoint per `save_steps`. + checkpoint=dict( + type=CheckpointHook, + by_epoch=False, + interval=save_steps, + max_keep_ckpts=save_total_limit), + # set sampler seed in distributed evrionment. + sampler_seed=dict(type=DistSamplerSeedHook), +) + +# configure environment +env_cfg = dict( + # whether to enable cudnn benchmark + cudnn_benchmark=False, + # set multi process parameters + mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), + # set distributed parameters + dist_cfg=dict(backend='nccl'), +) + +# set visualizer +visualizer = None + +# set log level +log_level = 'INFO' + +# load from which checkpoint +load_from = None + +# whether to resume training from the loaded checkpoint +resume = False + +# Defaults to use random seed and disable `deterministic` +randomness = dict(seed=None, deterministic=False) + +# set log processor +log_processor = dict(by_epoch=False)