Pythonで現代の季語を調べてみた

import requests
from bs4 import BeautifulSoup
import re
import neologdn
from janome.tokenizer import Tokenizer
import collections

#urlを登録
url_spring = 'http://www.rankingbook.com/category/music/spring/favorite/'
url_summer = 'http://www.rankingbook.com/category/music/summer/favorite/'
url_autumn = 'http://www.rankingbook.com/category/music/autumn/favorite/'
url_winter = 'http://www.rankingbook.com/category/music/winter/favorite/'

#曲名と歌手を取得
def getmusic(url):
    r = requests.get(url)
    soup = BeautifulSoup(r.content, "html.parser")
    music = soup.find_all("a", attrs={"target": "_top"})
    music.extend(soup.find_all("td", attrs={"class": "b","align":"LEFT"}))
　#取得した情報を曲名と歌手名に分割
    for i in range(len(music)):
        music[i] = list(music[i].strings)[0][:-1]
        music[i] = re.split("[(（／＜…『</]",music[i])
    return music

#歌手情報が無い曲などを削除
def delerror(music):
    delindex = []
    for i in range(len(music)):
        if(len(music[i])!=2):
            delindex.append(i)
    delindex.reverse()
    for i in delindex:
        del music[i]
    
    
music_spring = getmusic(url_spring)
music_summer= getmusic(url_summer)
music_autumn = getmusic(url_autumn)
music_winter = getmusic(url_winter)

delerror(music_spring)
delerror(music_summer)
delerror(music_autumn)
delerror(music_winter)

#歌詞を取得
def getkashi(musics):
    kashi = ""
    for music in musics:
        url = "https://www.uta-net.com/search/?Aselect=2&Keyword="+music[0]+"&Bselect=4&x=24&y=15"
        r = requests.get(url)
        soup = BeautifulSoup(r.content, "html.parser")
        #曲名検索の結果、ヒットがあるかを判定
        if(len(soup.find_all("td", attrs={"class": "side td1"})) != 0):
            
            #ヒットした曲の中に歌手名が一致するものがあるかを判定
            singers = soup.find_all("td", attrs={"class": "td2"})
            singer = -1
            for i in range(len(singers)):
                if(list(singers[i].strings)[0] == music[1]):
                    singer = i
            if(singer != -1):
                
                #歌詞を取得
                href = soup.find_all("td", attrs={"class": "side td1"})[singer].contents[0].get("href")
        
                r = requests.get("https://www.uta-net.com"+href)
                soup = BeautifulSoup(r.content, "html.parser")
            
                kashi_list = list(soup.find_all("div", attrs={"id": "kashi_area"})[0].strings)
                for kashi_ in kashi_list:
                    kashi += kashi_
    return kashi


#ファイルへ保存
#春
kashi_spring = getkashi(music_spring)
with open('spring.txt', 'w') as f:
    f.write(kashi_spring)
#夏
kashi_summer = getkashi(music_summer)
with open('summer.txt', 'w') as f:
    f.write(kashi_summer)
#秋
kashi_autumn = getkashi(music_autumn)
with open('autumn.txt', 'w') as f:
    f.write(kashi_autumn)
#冬
kashi_winter = getkashi(music_winter)
with open('winter.txt', 'w') as f:
    f.write(kashi_winter)

#データの整理
def organization(text):
    # 英数字,記号,空白,改行の削除
    text = re.sub("[a-zA-Z0-9_]","",text)
    text = re.sub("[!-/:[email protected][-`{-~…！「」“”。？、・～♪‥‘’]","",text)
    text = re.sub("\s","",text)
    text = re.sub("\n","",text)
    #正規化
    text = neologdn.normalize(text)   
    return text

kashi_spring = organization(kashi_spring)
kashi_summer = organization(kashi_summer)
kashi_autumn = organization(kashi_autumn)
kashi_winter = organization(kashi_winter)

明日 名詞,副詞可能,*,*,*,*,明日,アシタ,アシタ
は 助詞,係助詞,*,*,*,*,は,ハ,ワ
晴れる 動詞,自立,*,*,一段,基本形,晴れる,ハレル,ハレル
だろ 助動詞,*,*,*,特殊・ダ,未然形,だ,ダロ,ダロ
う 助動詞,*,*,*,不変化型,基本形,う,ウ,ウ
か 助詞,副助詞／並立助詞／終助詞,*,*,*,*,か,カ,カ
。 記号,句点,*,*,*,*,。,。,。

#形態素解析を行い、名詞を取り出す
def getnoun(text):
    t = Tokenizer()
    tokens = t.tokenize(text)
    noun = []
    for token in tokens:
        partOfSpeech = token.part_of_speech.split(",")[0]
        if partOfSpeech == "名詞":
            noun.append(token.surface)
    return noun

#名詞取り出し
noun_spring = getnoun(kashi_spring)
noun_summer = getnoun(kashi_summer)
noun_autumn = getnoun(kashi_autumn)
noun_winter = getnoun(kashi_winter)

#出現数カウント
count_dict_spring = collections.Counter(noun_spring)
count_dict_summer = collections.Counter(noun_summer)
count_dict_autumn = collections.Counter(noun_autumn)
count_dict_winter = collections.Counter(noun_winter)

#上位30単語をリストで取得
word_spring = count_dict_spring.most_common(30)
word_summer = count_dict_summer.most_common(30)
word_autumn = count_dict_autumn.most_common(30)
word_winter = count_dict_winter.most_common(30)

#指定した文字列をリストから削除
def dellist(text,del_text):
    del_index = []
    for i in range(len(text)):
        for delt in del_text:
            if(text[i][0]==delt):
                del_index.append(i)
    del_index.reverse()
    for deli in del_index:
        del text[deli]

#"春","夏","秋","冬"をストップワードとして登録
stopword = ["春","夏","秋","冬"]

#ストップワードをリストから削除
dellist(word_spring,stopword)
dellist(word_summer,stopword)
dellist(word_autumn,stopword)
dellist(word_winter,stopword)
重複している単語削除のソースコードはこんな感じです。

#共通する要素のインデックス番号を返す
def delcommon(text1,text2):
    del_index1 = []
    del_index2 = []
    for i in range(len(text1)):
        for j in range(len(text2)):
            if(text1[i][0]==text2[j][0]):
                del_index1.append(i)
                del_index2.append(j)
    return del_index1,del_index2

#指定したインデックス番号の要素を削除
def delindex(text,del_index):
    del_index = list(set(del_index))
    del_index.sort(reverse=True)
    for deli in del_index:
        del text[deli]

#共通する要素のインデックスを取得
del_index_spring,del_index_summer = delcommon(word_spring,word_summer)
del_index_sp1,del_index_autumn = delcommon(word_spring,word_autumn)
del_index_sp2,del_index_winter = delcommon(word_spring,word_winter)
del_index_su1,del_index_au1 = delcommon(word_summer,word_autumn)
del_index_su2,del_index_wi1 = delcommon(word_summer,word_winter)
del_index_au2,del_index_wi2 = delcommon(word_autumn,word_winter)

#季節ごとにインデックス番号を結合
del_index_spring.extend(del_index_sp1)
del_index_spring.extend(del_index_sp2)
del_index_summer.extend(del_index_su1)
del_index_summer.extend(del_index_su2)
del_index_autumn.extend(del_index_au1)
del_index_autumn.extend(del_index_au2)
del_index_winter.extend(del_index_wi1)
del_index_winter.extend(del_index_wi2)

#指定したインデックス番号の要素を削除
delindex(word_spring,del_index_spring)
delindex(word_summer,del_index_summer)
delindex(word_autumn,del_index_autumn)
delindex(word_winter,del_index_winter)

春
[('花', 60), ('さくら', 42), ('それ', 31), ('桜', 27), ('声', 26), ('前', 25)]
夏
[('海', 99), ('太陽', 74), ('波', 64), ('目', 52)]
秋
[('涙', 41), ('好き', 21), ('そば', 20), ('ひとつ', 19), ('どこ', 18), ('雨', 16)]
冬
[('雪', 138), ('街', 79), ('夜', 78), ('言葉', 48)]