72 lines
2.1 KiB
Python
72 lines
2.1 KiB
Python
# -*- coding: utf-8 -*-
|
||
# @Time : 2024/10/24 11:10
|
||
# @Author : 黄子寒
|
||
# @Email : 1064071566@qq.com
|
||
# @File : LDArec.py
|
||
# @Project : EmoLLM
|
||
import json
|
||
import jieba
|
||
from gensim import corpora
|
||
from gensim.models.ldamodel import LdaModel
|
||
from collections import defaultdict
|
||
|
||
|
||
# 加载问答对数据
|
||
def load_qa_data(file_path):
|
||
qa_pairs = []
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
for line in f:
|
||
qa_pairs.append(json.loads(line.strip()))
|
||
return qa_pairs
|
||
|
||
|
||
# 加载中文停用词
|
||
def load_stopwords(file_path):
|
||
with open(file_path, 'r', encoding='utf-8') as f:
|
||
return set([line.strip() for line in f])
|
||
|
||
|
||
# 使用jieba对中文文本进行分词，并去除停用词
|
||
def preprocess_text(text, stopwords):
|
||
words = jieba.lcut(text) # 使用jieba进行中文分词
|
||
words = [word for word in words if word not in stopwords and len(word) > 1] # 去除停用词和长度为1的词
|
||
return words
|
||
|
||
|
||
# 生成LDA主题模型
|
||
def build_lda_model(qa_pairs, stopwords, num_topics=5):
|
||
# 处理所有问题文本
|
||
questions = [qa['input'] for qa in qa_pairs]
|
||
processed_questions = [preprocess_text(question, stopwords) for question in questions]
|
||
|
||
# 创建字典和词袋模型
|
||
dictionary = corpora.Dictionary(processed_questions)
|
||
corpus = [dictionary.doc2bow(text) for text in processed_questions]
|
||
|
||
# 训练LDA模型
|
||
lda_model = LdaModel(corpus, num_topics=num_topics, id2word=dictionary, passes=15)
|
||
return lda_model, dictionary, corpus
|
||
|
||
|
||
# 打印每个主题的关键词
|
||
def print_topics(lda_model, num_words=10):
|
||
for idx, topic in lda_model.print_topics(num_words=num_words):
|
||
print(f"主题 {idx}: {topic}")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
qa_file = "output/train_optimized_multiple.jsonl" # 问答对文件
|
||
stopwords_file = "chinese_stopwords.txt" # 停用词文件
|
||
|
||
# 加载问答对
|
||
qa_pairs = load_qa_data(qa_file)
|
||
|
||
# 加载停用词
|
||
stopwords = load_stopwords(stopwords_file)
|
||
|
||
# 构建LDA主题模型
|
||
lda_model, dictionary, corpus = build_lda_model(qa_pairs, stopwords, num_topics=20)
|
||
|
||
# 打印主题及其关键词
|
||
print_topics(lda_model)
|