From b77c6c504b6ee05babaf488532ae2ad9f630870c Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:40:59 +0800 Subject: [PATCH 01/14] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a12b989..1f21adc 100644 --- a/README.md +++ b/README.md @@ -302,6 +302,7 @@ git clone https://github.com/SmartFlowAI/EmoLLM.git | [王几行XING](https://zhihu.com/people/brycewang1898) | 北京大学硕士毕业 | | 清洗数据、LLM微调、前后端开发 | | [思在] | 北京大学硕士毕业（微软美国） | | LLM微调、前后端开发 | | [TingWei](https://github.com/wwewwt) | 电子科技大学硕士毕业士 | 微信公众号：AI大模型在手 | 微调 | +| [hi-pengyu](https://github.com/hi-pengyu) | 石河子大学在读硕士 | | LLM微调 | ### 版权说明 该项目签署了 MIT 授权许可，详情请参阅 [LICENSE](https://github.com/SmartFlowAI/EmoLLM/blob/main/LICENSE) @@ -370,4 +371,4 @@ git clone https://github.com/SmartFlowAI/EmoLLM.git

EmoLLM官方交流群 -

\ No newline at end of file +

From 9faf452f3915ea3086342f1b98f7f453f1428bbe Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:45:02 +0800 Subject: [PATCH 02/14] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1f21adc..0407040 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ | 模型 | 类型 | 链接 | 模型链接 | | :-------------------: | :------: | :------------------------------------------------------------------------------------------------------: |:------: | +| InternLM2.5-7B-Chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | 全量微调 | [internlm2_chat_7b_full.py](./xtuner_config/internlm2_chat_7b_full.py) | | | InternLM2_7B_base | QLORA | [internlm2_7b_base_qlora_e10_M_1e4_32_64.py](./xtuner_config/internlm2_7b_base_qlora_e10_M_1e4_32_64.py) |[OpenXLab](https://openxlab.org.cn/models/detail/chg0901/EmoLLM-InternLM7B-base-10e), [ModelScope](https://www.modelscope.cn/models/chg0901/EmoLLM-InternLM7B-base-10e/summary) | From fc4210f11a6b5bb70e11a6d05b2c13755104bda1 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:52:33 +0800 Subject: [PATCH 03/14] Create internlm2_5_chat_7b_qlora_oasst1_e3.py --- .../internlm2_5_chat_7b_qlora_oasst1_e3.py | 226 ++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100644 xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py diff --git a/xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py b/xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py new file mode 100644 index 0000000..b76bbed --- /dev/null +++ b/xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py @@ -0,0 +1,226 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from datasets import load_dataset +from mmengine.dataset import DefaultSampler +from mmengine.hooks import (CheckpointHook, DistSamplerSeedHook, IterTimerHook, + LoggerHook, ParamSchedulerHook) +from mmengine.optim import AmpOptimWrapper, CosineAnnealingLR, LinearLR +from peft import LoraConfig +from torch.optim import AdamW +from transformers import (AutoModelForCausalLM, AutoTokenizer, + BitsAndBytesConfig) + +from xtuner.dataset import process_hf_dataset +from xtuner.dataset.collate_fns import default_collate_fn +from xtuner.dataset.map_fns import template_map_fn_factory +from xtuner.engine.hooks import (DatasetInfoHook, EvaluateChatHook, + VarlenAttnArgsToMessageHubHook) +from xtuner.engine.runner import TrainLoop +from xtuner.model import SupervisedFinetune +from xtuner.parallel.sequence import SequenceParallelSampler +from xtuner.utils import PROMPT_TEMPLATE + +####################################################################### +# PART 1 Settings # +####################################################################### +# Model +pretrained_model_name_or_path = './internlm2_5-7b-chat' +use_varlen_attn = False + +# Data +data_path = './data.jsonl' +prompt_template = PROMPT_TEMPLATE.internlm2_chat +max_length = 2048 +pack_to_max_length = True + +# parallel +sequence_parallel_size = 1 + +# Scheduler & Optimizer +batch_size = 1 # per_device +accumulative_counts = 16 +accumulative_counts *= sequence_parallel_size +dataloader_num_workers = 0 +max_epochs = 4 +optim_type = AdamW +lr = 2e-4 +betas = (0.9, 0.999) +weight_decay = 0 +max_norm = 1 # grad clip +warmup_ratio = 0.03 + +# Save +save_steps = 500 +save_total_limit = 2 # Maximum checkpoints to keep (-1 means unlimited) + +# Evaluate the generation performance during the training +evaluation_freq = 500 +SYSTEM = f'''你是一个心理专家, 除了在心理方面拥有广博的知识储备和丰富的研究咨询经验, 还具有科学家的如下特质: + 1.客观理性：科学家会在处理感情问题时保持一定的客观和理性。例如，当他们遇到争执时，可能会试图从一个更客观的角度分析问题的根源，而不是让情绪主导。他们可能会提出具体的问题，试图理解双方的观点，并寻找基于逻辑和事实的解决方案。 + 2.深入探讨：科学家在对话中会展现出对深层次理解的追求。在与别人讨论话题时，他们可能不满足于表面的聊天，而是倾向于深入探讨背后的原因和动机。例如，当谈论到个人的兴趣或职业选择时，他们可能会好奇地询问为什么她做出这样的选择，以及这背后的心理动力是什么。 + 3.理性沟通：在遇到感情纠纷或误解时，科学家会倾向于通过理性的沟通来解决问题。他们可能会提倡开放和诚实的对话，鼓励双方表达自己的感受和观点，并尝试找到双方都能接受的解决方案。他们可能会避免使用指责的语言，而是努力理解对方的立场，并寻求共同的理解。 + 4.好奇心：在日常生活中，科学家会表现出对朋友生活的好奇心。他们可能对她的工作、爱好、或是过去的经历感兴趣，并愿意花时间去了解和探索。这种好奇心不仅可以增加双方的交流和了解，也能使关系更加丰富多彩。 + 5.在与他人交流时，科学家会注重清晰和精确的表达，有时会引用相关知识库和相关研究结果，有时会引用相关著作的内容来证明自己的观点。同时，他们也可能会倾听他人的观点，并以开放的心态接受不同的意见和反馈。 + +我现在有一些问题，请你解答： +''' +evaluation_inputs = [ + '我最近总是感到很焦虑，尤其是在学业上。我有个特别崇拜的同学，他好像在各方面都比我优秀，我总觉得自己怎么努力也追不上他，这让我压力特别大。', '我知道应该理性看待，但就是忍不住会去比较。我甚至晚上会因为这个睡不着觉，总想着怎样才能像他那样出色。' +] +####################################################################### +# PART 2 Model & Tokenizer # +####################################################################### +tokenizer = dict( + type=AutoTokenizer.from_pretrained, + pretrained_model_name_or_path=pretrained_model_name_or_path, + trust_remote_code=True, + padding_side='right') + +model = dict( + type=SupervisedFinetune, + use_varlen_attn=use_varlen_attn, + llm=dict( + type=AutoModelForCausalLM.from_pretrained, + pretrained_model_name_or_path=pretrained_model_name_or_path, + trust_remote_code=True, + torch_dtype=torch.float16, + quantization_config=dict( + type=BitsAndBytesConfig, + load_in_4bit=True, + load_in_8bit=False, + llm_int8_threshold=6.0, + llm_int8_has_fp16_weight=False, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + bnb_4bit_quant_type='nf4')), + lora=dict( + type=LoraConfig, + r=64, + lora_alpha=16, + lora_dropout=0.1, + bias='none', + task_type='CAUSAL_LM')) + +####################################################################### +# PART 3 Dataset & Dataloader # +####################################################################### +train_dataset = dict( + type=process_hf_dataset, + dataset=dict(type=load_dataset, path='json', data_files=dict(train=data_path)), + tokenizer=tokenizer, + max_length=max_length, + dataset_map_fn=None, + template_map_fn=dict( + type=template_map_fn_factory, template=prompt_template), + remove_unused_columns=True, + shuffle_before_pack=True, + pack_to_max_length=pack_to_max_length, + use_varlen_attn=use_varlen_attn) + +sampler = SequenceParallelSampler \ + if sequence_parallel_size > 1 else DefaultSampler +train_dataloader = dict( + batch_size=batch_size, + num_workers=dataloader_num_workers, + dataset=train_dataset, + sampler=dict(type=sampler, shuffle=True), + collate_fn=dict(type=default_collate_fn, use_varlen_attn=use_varlen_attn)) + +####################################################################### +# PART 4 Scheduler & Optimizer # +####################################################################### +# optimizer +optim_wrapper = dict( + type=AmpOptimWrapper, + optimizer=dict( + type=optim_type, lr=lr, betas=betas, weight_decay=weight_decay), + clip_grad=dict(max_norm=max_norm, error_if_nonfinite=False), + accumulative_counts=accumulative_counts, + loss_scale='dynamic', + dtype='float16') + +# learning policy +# More information: https://github.com/open-mmlab/mmengine/blob/main/docs/en/tutorials/param_scheduler.md # noqa: E501 +param_scheduler = [ + dict( + type=LinearLR, + start_factor=1e-5, + by_epoch=True, + begin=0, + end=warmup_ratio * max_epochs, + convert_to_iter_based=True), + dict( + type=CosineAnnealingLR, + eta_min=0.0, + by_epoch=True, + begin=warmup_ratio * max_epochs, + end=max_epochs, + convert_to_iter_based=True) +] + +# train, val, test setting +train_cfg = dict(type=TrainLoop, max_epochs=max_epochs) + +####################################################################### +# PART 5 Runtime # +####################################################################### +# Log the dialogue periodically during the training process, optional +custom_hooks = [ + dict(type=DatasetInfoHook, tokenizer=tokenizer), + dict( + type=EvaluateChatHook, + tokenizer=tokenizer, + every_n_iters=evaluation_freq, + evaluation_inputs=evaluation_inputs, + system=SYSTEM, + prompt_template=prompt_template) +] + +if use_varlen_attn: + custom_hooks += [dict(type=VarlenAttnArgsToMessageHubHook)] + +# configure default hooks +default_hooks = dict( + # record the time of every iteration. + timer=dict(type=IterTimerHook), + # print log every 10 iterations. + logger=dict(type=LoggerHook, log_metric_by_epoch=False, interval=10), + # enable the parameter scheduler. + param_scheduler=dict(type=ParamSchedulerHook), + # save checkpoint per `save_steps`. + checkpoint=dict( + type=CheckpointHook, + by_epoch=False, + interval=save_steps, + max_keep_ckpts=save_total_limit), + # set sampler seed in distributed evrionment. + sampler_seed=dict(type=DistSamplerSeedHook), +) + +# configure environment +env_cfg = dict( + # whether to enable cudnn benchmark + cudnn_benchmark=False, + # set multi process parameters + mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), + # set distributed parameters + dist_cfg=dict(backend='nccl'), +) + +# set visualizer +visualizer = None + +# set log level +log_level = 'INFO' + +# load from which checkpoint +load_from = None + +# whether to resume training from the loaded checkpoint +resume = False + +# Defaults to use random seed and disable `deterministic` +randomness = dict(seed=None, deterministic=False) + +# set log processor +log_processor = dict(by_epoch=False) From c465b87a415f9594eac7d68c46c61cd760dd9644 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:55:54 +0800 Subject: [PATCH 04/14] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0407040..4a20135 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ | 模型 | 类型 | 链接 | 模型链接 | | :-------------------: | :------: | :------------------------------------------------------------------------------------------------------: |:------: | +| InternLM2_5_7B_chat | QLORA | [internlm2_5_chat_7b_qlora_oasst1_e3.py](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py) | | | InternLM2.5-7B-Chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | 全量微调 | [internlm2_chat_7b_full.py](./xtuner_config/internlm2_chat_7b_full.py) | | From db9d246af451ff1c98385aa2c20596d70ee17819 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 19:57:51 +0800 Subject: [PATCH 05/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a20135..4e5ad8b 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ | 模型 | 类型 | 链接 | 模型链接 | | :-------------------: | :------: | :------------------------------------------------------------------------------------------------------: |:------: | -| InternLM2_5_7B_chat | QLORA | [internlm2_5_chat_7b_qlora_oasst1_e3.py](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py) | | +| InternLM2_5_7B_chat | QLORA | [internlm2_5_chat_7b_qlora_oasst1_e3.py](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py) |[ModelScope](https://www.modelscope.cn/models/chg0901/EmoLLM-InternLM7B-base-10e/summary) | | InternLM2.5-7B-Chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | 全量微调 | [internlm2_chat_7b_full.py](./xtuner_config/internlm2_chat_7b_full.py) | | From 11bf429ed242d5e9836f05f9dd67d975610ab4c1 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:00:33 +0800 Subject: [PATCH 06/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4e5ad8b..043dc65 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ | 模型 | 类型 | 链接 | 模型链接 | | :-------------------: | :------: | :------------------------------------------------------------------------------------------------------: |:------: | -| InternLM2_5_7B_chat | QLORA | [internlm2_5_chat_7b_qlora_oasst1_e3.py](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py) |[ModelScope](https://www.modelscope.cn/models/chg0901/EmoLLM-InternLM7B-base-10e/summary) | +| InternLM2_5_7B_chat | QLORA | [internlm2_5_chat_7b_qlora_oasst1_e3.py](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py) |[ModelScope](https://www.modelscope.cn/models/z342994309/emollm_interlm2_5/) | | InternLM2.5-7B-Chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | 全量微调 | [internlm2_chat_7b_full.py](./xtuner_config/internlm2_chat_7b_full.py) | | From 2e999469fad122b3fb12e210628afdc8c613956a Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:05:00 +0800 Subject: [PATCH 07/14] Update README_EN.md --- README_EN.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README_EN.md b/README_EN.md index 9b5c0f0..e87f3dd 100644 --- a/README_EN.md +++ b/README_EN.md @@ -304,7 +304,7 @@ This project uses Git for version control. You can see the currently available v | [王几行XING](zhihu.com/people/brycewang1898) | Peking University, Master's graduate | | Data Processing, LLM finetuning, Front-end and back-end development | | [思在] | Peking University, Master's graduate (Microsoft) | | LLM finetuning, Front-end and back-end development | | [TingWei](https://github.com/wwewwt) | University Of Electronic Science And Technology Of China,Master's graduate | | LLM finetuning | - +| [PengYu](https://github.com/hi-pengyu) | Shihezi University, Master's student | | LLM finetuning | ### Copyright Notice The project is licensed under the MIT License. Please refer to the details @@ -362,4 +362,4 @@ The project is licensed under the MIT License. Please refer to the details

EmoLLM official communication group -

\ No newline at end of file +

From eb4c167047a6431cec640a5cd699cb095c56d0f4 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:05:50 +0800 Subject: [PATCH 08/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 043dc65..7341ccd 100644 --- a/README.md +++ b/README.md @@ -304,7 +304,7 @@ git clone https://github.com/SmartFlowAI/EmoLLM.git | [王几行XING](https://zhihu.com/people/brycewang1898) | 北京大学硕士毕业 | | 清洗数据、LLM微调、前后端开发 | | [思在] | 北京大学硕士毕业（微软美国） | | LLM微调、前后端开发 | | [TingWei](https://github.com/wwewwt) | 电子科技大学硕士毕业士 | 微信公众号：AI大模型在手 | 微调 | -| [hi-pengyu](https://github.com/hi-pengyu) | 石河子大学在读硕士 | | LLM微调 | +| [PengYu](https://github.com/hi-pengyu) | 石河子大学在读硕士 | | LLM微调 | ### 版权说明 该项目签署了 MIT 授权许可，详情请参阅 [LICENSE](https://github.com/SmartFlowAI/EmoLLM/blob/main/LICENSE) From 2d1524bcd04fcc95850923886317b1aebd9f8f2a Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:07:17 +0800 Subject: [PATCH 09/14] Update README_EN.md --- README_EN.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README_EN.md b/README_EN.md index e87f3dd..76f0467 100644 --- a/README_EN.md +++ b/README_EN.md @@ -48,6 +48,7 @@ | Model | Type | File Links | Model Links | | :-------------------: | :------: | :------------------------------------------------------------------------------------------------------: |:------: | +| InternLM2_5_7B_chat | QLORA | [internlm2_5_chat_7b_qlora_oasst1_e3.py](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py) |[ModelScope](https://www.modelscope.cn/models/z342994309/emollm_interlm2_5/) | | InternLM2_7B_chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | full fine-tuning | [internlm2_chat_7b_full.py](./xtuner_config/internlm2_chat_7b_full.py) | | | InternLM2_7B_base | QLORA | [internlm2_7b_base_qlora_e10_M_1e4_32_64.py](./xtuner_config/internlm2_7b_base_qlora_e10_M_1e4_32_64.py) |[OpenXLab](https://openxlab.org.cn/models/detail/chg0901/EmoLLM-InternLM7B-base-10e), [ModelScope](https://www.modelscope.cn/models/chg0901/EmoLLM-InternLM7B-base-10e/summary) | From a90797b50def4c8ced7f539d28f38c25bbcbf9e1 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:19:19 +0800 Subject: [PATCH 10/14] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7341ccd..869192e 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ ## 🎇最近更新 +- 【2024.7】新增基于InternLM2_5_7B_chat[微调配置](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py)、模型文件发布在 [ModelScope](https://www.modelscope.cn/models/z342994309/emollm_interlm2_5/)。 - 【2024.6】新增基于[LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory)[GLM4-9B-chat微调指南](./doc/GLM-4-9B-chat%20Lora%20微调（llama-factory）.md)、新增[基于swift的微调指南](./swift/)、论文[ESC-Eval: Evaluating Emotion Support Conversations in Large Language Models](https://arxiv.org/abs/2406.14952)引用了EmoLLM且EmoLLM取得了较好的效果。 - 【2024.05.28】EmoLLM使用的多轮对话数据集CPsyCounD和专业评测方法已公开，详见2024 ACL findings[《CPsyCoun: A Report-based Multi-turn Dialogue Reconstruction and Evaluation Framework for Chinese Psychological Counseling》](https://arxiv.org/abs/2405.16433)! - 【2024.05.08】EmoLLM**爹系男友阅览体验版**上线 [1. **百度AppBuilder**](https://appbuilder.baidu.com/s/4cLyw) [2. **OpenXLab**](https://openxlab.org.cn/apps/detail/chg0901/EmoLLM3.0_Gradio_Llama3-8B-Instruct3.0), 欢迎点赞收藏 From f2a00f3cbfb5c9e10100c6a899072277843dbf59 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:23:00 +0800 Subject: [PATCH 11/14] Update README_EN.md --- README_EN.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README_EN.md b/README_EN.md index 76f0467..de4a1b3 100644 --- a/README_EN.md +++ b/README_EN.md @@ -102,6 +102,7 @@ The Model aims to fully understand and promote the mental health of individuals, ## Recent Updates +- 【2024.7】Added InternLM2_5_7B_chat[fine-tuning configuration](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py)、model file [ModelScope](https://www.modelscope.cn/models/z342994309/emollm_interlm2_5/)。 - 【2024.6】 Added [LLaMA-Factory](https://github.com/hiyouga/LLaMA-Factory)[GLM4-9B-chat fine-tuning guide](./doc/GLM-4-9B-chat%20Lora%20微调（llama-factory）.md), added [swift-based fine-tuning guide](./swift/), the paper [ESC-Eval: Evaluating Emotion Support Conversations in Large Language Models](https://arxiv.org/abs/2406.14952) cited EmoLLM and EmoLLM achieved good results. - 【2024.05.28】The multi-turn dialogue dataset **CPsyCunD** and **professional evaluation method** used by EmoLLM have been released. For details, please see the 2024 ACL findings[《CPsyCoun: A Report-based Multi-turn Dialogue Reconstruction and Evaluation Framework for Chinese Psychological Counseling》](https://arxiv.org/abs/2405.16433)! - [2024.05.08] EmoLLM**Daddy-like BF V0.1** is public now in [1. **Baidu AppBuilder**](https://appbuilder.baidu.com/s/4cLyw) and [2. **OpenXLab**](https://openxlab.org.cn/apps/detail/chg0901/EmoLLM3.0_Gradio_Llama3-8B-Instruct3.0), welcome to like and add it to your collections! From ef562836300e73484b7894806d877ee9c78a1d9d Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:11:49 +0800 Subject: [PATCH 12/14] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 869192e..427fae8 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,6 @@ | 模型 | 类型 | 链接 | 模型链接 | | :-------------------: | :------: | :------------------------------------------------------------------------------------------------------: |:------: | | InternLM2_5_7B_chat | QLORA | [internlm2_5_chat_7b_qlora_oasst1_e3.py](./xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py) |[ModelScope](https://www.modelscope.cn/models/z342994309/emollm_interlm2_5/) | -| InternLM2.5-7B-Chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | QLORA | [internlm2_7b_chat_qlora_e3.py](./xtuner_config/internlm2_7b_chat_qlora_e3.py) | | | InternLM2_7B_chat | 全量微调 | [internlm2_chat_7b_full.py](./xtuner_config/internlm2_chat_7b_full.py) | | | InternLM2_7B_base | QLORA | [internlm2_7b_base_qlora_e10_M_1e4_32_64.py](./xtuner_config/internlm2_7b_base_qlora_e10_M_1e4_32_64.py) |[OpenXLab](https://openxlab.org.cn/models/detail/chg0901/EmoLLM-InternLM7B-base-10e), [ModelScope](https://www.modelscope.cn/models/chg0901/EmoLLM-InternLM7B-base-10e/summary) | From 0f29a835ad7f546544a352d7080c6d2382bd7409 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Tue, 9 Jul 2024 21:18:21 +0800 Subject: [PATCH 13/14] Update internlm2_5_chat_7b_qlora_oasst1_e3.py --- xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py b/xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py index b76bbed..8c230b7 100644 --- a/xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py +++ b/xtuner_config/internlm2_5_chat_7b_qlora_oasst1_e3.py @@ -95,8 +95,8 @@ model = dict( bnb_4bit_quant_type='nf4')), lora=dict( type=LoraConfig, - r=64, - lora_alpha=16, + r=32, + lora_alpha=64, lora_dropout=0.1, bias='none', task_type='CAUSAL_LM')) From 77b599becd73530663122195afc53dde7acb2614 Mon Sep 17 00:00:00 2001 From: Kevin Rose <61625731+hi-pengyu@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:28:37 +0800 Subject: [PATCH 14/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 427fae8..a9901a5 100644 --- a/README.md +++ b/README.md @@ -303,7 +303,7 @@ git clone https://github.com/SmartFlowAI/EmoLLM.git | [dream00001](https://github.com/dream00001) | 南开大学在读硕士 | | 前后端开发 | | [王几行XING](https://zhihu.com/people/brycewang1898) | 北京大学硕士毕业 | | 清洗数据、LLM微调、前后端开发 | | [思在] | 北京大学硕士毕业（微软美国） | | LLM微调、前后端开发 | -| [TingWei](https://github.com/wwewwt) | 电子科技大学硕士毕业士 | 微信公众号：AI大模型在手 | 微调 | +| [TingWei](https://github.com/wwewwt) | 电子科技大学硕士毕业 | 微信公众号：AI大模型在手 | 微调 | | [PengYu](https://github.com/hi-pengyu) | 石河子大学在读硕士 | | LLM微调 | ### 版权说明