2. データのクリーニング

• 正規表現を使用して非漢字文字を削除
• ストップワードのフィルタリング
• 結巴分词を使用して中国語のテキストを分割

import re
import jieba.analyse
from wordcloud import WordCloud, STOPWORDS

pattern = re.compile(r'[^ws]')
cleaned_comments = pattern.sub('', comments)

stopwords = set(STOPWORDS)
with open('./StopWords.txt', encoding="utf-8") as f:
    stopwords.update(word.strip() for word in f)
keywords = {word: score for word, score in keywords.items() if word not in stopwords}

result = jieba.analyse.textrank(cleaned_comments, topK=150, withWeight=True)

3. WordCloudでの表示

wordcloud = WordCloud(font_path="simhei.ttf", mask=np.array(bg_pic), background_color="white", max_font_size=80, stopwords=stopwords).generate_from_frequencies(keywords)

plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()

print('表示成功!')

実装コード

import warnings
import jieba
import jieba.analyse
import re
import matplotlib.pyplot as plt
import requests
from bs4 import BeautifulSoup as bs
from wordcloud import WordCloud, STOPWORDS

warnings.filterwarnings("ignore")
plt.rcParams['figure.figsize'] = (10.0, 5.0)

def getNowPlayingMovieList():
    url = 'https://movie.douban.com/nowplaying/guangzhou'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0'
    }
    try:
        resp = requests.get(url, headers=headers)
        resp.raise_for_status()
        html = resp.text
    except requests.exceptions.HTTPError as errh:
        print(f"HTTPエラー: {errh}")
        return []
    except requests.exceptions.RequestException as err:
        print(f"リクエストエラー: {err}")
        return []
    soup = bs(html, 'html.parser')
    nowplaying_movie = soup.find('div', id='nowplaying')
    if not nowplaying_movie:
        return []
    nowplaying_movie_list = nowplaying_movie.find_all('li', class_='list-item')
    nowplaying_list = []
    for item in nowplaying_movie_list:
        nowplaying_dict = {}
        nowplaying_dict['id'] = item['data-subject']
        nowplaying_dict['name'] = item.find('img')['alt']
        nowplaying_list.append(nowplaying_dict)
    return nowplaying_list

def getCommentsById(movieId, pageNum):
    eachCommentList = []
    if pageNum <= 0:
        return eachCommentList
    start = (pageNum - 1) * 20
    url = f'https://movie.douban.com/subject/{movieId}/comments?start={start}&limit=20'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.0'
    }
    try:
        resp = requests.get(url, headers=headers)
        resp.raise_for_status()
        html = resp.text
    except requests.exceptions.HTTPError as errh:
        print(f"HTTPエラー: {errh}")
        return []
    except requests.exceptions.RequestException as err:
        print(f"リクエストエラー: {err}")
        return []
    soup = bs(html, 'html.parser')
    comment_div_lits = soup.find_all('div', class_='comment')
    for item in comment_div_lits:
        if item.find('p'):
            eachCommentList.append(item.find('p').text.strip())
    return eachCommentList

def main():
    NowPlayingMovie_list = getNowPlayingMovieList()
    if not NowPlayingMovie_list:
        print("映画リストが取得できませんでした")
        return

    commentList = []
    for i in range(1, 11):
        comments_temp = getCommentsById(NowPlayingMovie_list[0]['id'], i)
        commentList.extend(comments_temp)

    comments = " ".join(commentList)
    pattern = re.compile(r'[^ws]')
    cleaned_comments = pattern.sub('', comments)

    result = jieba.analyse.textrank(cleaned_comments, topK=150, withWeight=True)
    keywords = {word: weight for word, weight in result}

    stopwords = set(STOPWORDS)
    with open('./StopWords.txt', encoding="utf-8") as f:
        stopwords.update(word.strip() for word in f)

    keywords = {word: score for word, score in keywords.items() if word not in stopwords}

    wordcloud = WordCloud(font_path="simhei.ttf", background_color="white", max_font_size=80, stopwords=stopwords).generate_from_frequencies(keywords)
    plt.imshow(wordcloud, interpolation='bilinear')
    plt.axis("off")
    plt.show()
    print('表示成功!')

if __name__ == "__main__":
    main()