人工知能に「ニュアンス」を伝えたい！

# 必要なモジュールをインポートする
import random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from keras.utils.np_utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from keras.layers.recurrent import SimpleRNN
from keras.layers.normalization import BatchNormalization
from keras.callbacks import EarlyStopping
from keras.optimizers import Adam
from sklearn.model_selection import train_test_split
from bs4 import BeautifulSoup
import requests
import re
import numpy as np

#スクレイピング先のurl
url = "http://name.m3q.jp/ranking?g=1" #男性
#url = "http://name.m3q.jp/ranking?g=2" #女性

name_list = []
for i in range(100):
    res = requests.get(url)
    res.encoding = res.apparent_encoding
    soup = BeautifulSoup(res.text,"html.parser")
    
    names = soup.find("div",class_="tags-list").find_all("a")

    for name in names:
        name_list.append(name.text + "\n")
    
    #次ページのurlを取得
    next_url = soup.find("a",class_="flat-button page-nav-next")
    url = next_url.get("href") 
    
#google driveにテキストファイルとして保存
with open("./boy_name.txt", mode='w') as f:
    f.writelines(name_list)

#ファイルを一行読み込み
str_list = []
with open("./girl_name.txt") as f:
    for s_line in f:
        str_list.append(s_line) 

for i, mystr in enumerate(str_list):
    #数字の削除
    str_list[i] = re.sub("[0-9]","",str_list[i])
    #空白の削除
    str_list[i] = re.sub(" ","",str_list[i])
 
#改行のみの要素，被っている要素を削除
str_list = [i for i in str_list if i != "\n"]
str_list = [x for x in str_list if x] 

#すべてのひらがなを含む文字列を生成
all_letters = "あいうえおかきくけこさしすせそたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをんがぎぐげござじずづぜぞだぢでどはびぶべぼゃゅょ"
n_letters = len(all_letters)

category = ["男性","女性"]
n_category = len(category)

#データの読み込み
girl_data = pd.read_csv("girl_name.txt", sep="\n", header=None)
boy_data = pd.read_csv("boy_name.txt", sep="\n", header=None)

name_list = []
label_list = []
max_len=0

for name in girl_data[0]:
    name_list.append(name)
    label_list.append(0)
    if len(name) > max_len:
        max_len = len(name)
    
for name in boy_data[0]:
    name_list.append(name)
    label_list.append(1)
    if len(name) > max_len:
        max_len = len(name)

# カテゴリ予測なので、正解ラベルを0,1のバイナリデータの配列に変換する
label_list = to_categorical(label_list, n_category)

#one-hot表現に変換
# 文字のインデックスを返す　"あ"= 0
def letterToIndex(letter):
    return all_letters.find(letter)

# 名前をone-hot表現でベクトルに変換
def lineToTensor(line):
    tensor = np.zeros([max_len,n_letters],dtype=int)
    for li, letter in enumerate(line):
        tensor[li][letterToIndex(letter)] = 1
    return tensor

#名前を変換
data_list=[]
for name in name_list:
    data_list.append(lineToTensor(name))
data_list = np.array(data_list)

def pred_activity_rnn(input_dim,
                       activate_method='softmax',  # 活性化関数
                       loss_method='categorical_crossentropy',  # 損失関数
                       optimizer_method=Adam(lr=0.00005, beta_1=0.9, beta_2=0.999),  # パラメータの更新方法
                       kernel_init_method='glorot_normal',  # 重みの初期化方法
                       batch_normalization=False,  # バッチ正規化
                       dropout_rate=0.5  # ドロップアウト率
                       ):
    # レイヤーを定義
    model = Sequential()
    model.add(
        SimpleRNN(
            input_shape=(input_dim[0], input_dim[1]),
            units=60,
            kernel_initializer=kernel_init_method,
            return_sequences=True  # さらにRNNレイヤーを重ねるのであればTrueにする
        ))

    # バッチごとに正規化を行う
    if batch_normalization:
        model.add(BatchNormalization())

    # ドロップアウトにより、ニューロンをランダムに削除する
    # pred_activity_rnn(dropout_rate=0.5)ならドロップレートを50%で設定する
    if dropout_rate:
        model.add(Dropout(dropout_rate))

    model.add(
        SimpleRNN(
            units=30,
            kernel_initializer=kernel_init_method,
            return_sequences=False  # この後にRNNレイヤーはないのでFalseにする
        ))

    # バッチごとに正規化を行う
    if batch_normalization:
        model.add(BatchNormalization())

    # ドロップアウトにより、ニューロンをランダムに削除する
    if dropout_rate:
        model.add(Dropout(dropout_rate))

    model.add(Dense(units=n_category, activation=activate_method))
    model.compile(loss=loss_method, optimizer=optimizer_method,
                  metrics=['accuracy'])

    return model

# モデルの作成
input_size = [data_list.shape[1], data_list.shape[2]]
turned_model = pred_activity_rnn(
    input_dim=input_size,
    activate_method='softmax',
    loss_method='categorical_crossentropy',
    optimizer_method='adamax',
    kernel_init_method='glorot_normal',
    batch_normalization=False
)

X_train,X_test,y_train,y_test =
 train_test_split(data_list, label_list, test_size=0.3,random_state=0)

# early_stoppingの定義
early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=1)

# 学習スタート
history = turned_model.fit(
    X_train,
    y_train,
    batch_size=100,
    epochs=20,
    validation_split=0.3,
    callbacks=None,
    verbose=2
)

# 精度の推移図を出力
plt.figure(figsize=(8, 5))
plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.xticks(np.arange(0, 100 + 1, 10))
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()

# 損失関数の推移図を出力
plt.figure(figsize=(8, 5))
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.xticks(np.arange(0, 100 + 1, 10))
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
y_pred = turned_model.predict(X_test, batch_size=100, verbose=0)
score = turned_model.evaluate(X_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])

Test loss: 0.4411894009242079
Test accuracy: 0.8050989601005047

category = ["女性","男性"]
test_name = ["あいこ","かな","りか","めい","たつき","りょう","たかお","るん","あくわ","ちふす","れう"]
test_label = ["女性","女性","女性","女性","男性","男性","男性","女性","男性","女性","女性"]
test_vec = []

for name in test_name:
    test_vec.append(lineToTensor(name))
test_vec = np.array(test_vec)
    
result = pd.DataFrame()
result["名前"] = test_name 
y_pred = turned_model.predict(test_vec)
y_pred = [category[np.argmax(np.array(x))] for x in y_pred]
result["判定"] = y_pred
result["正解"] = test_label
result

#すべてのalphabetを含む文字列を生成
all_letters = "abcdefghijklmnopqrstuvwxyz"

#ひらがな→アルファベットに変換
from pykakasi import kakasi

name_list = []
label_list = []

kakasi = kakasi()
kakasi.setMode("H", "a")

conv = kakasi.getConverter()
max_len=0

for name in girl_data[0]:
    alpha = conv.do(name)
    name_list.append(alpha)
    label_list.append(0)  
    if len(alpha) > max_len:
        max_len = len(alpha)
       
for name in boy_data[0]:
    alpha = conv.do(name)
    name_list.append(alpha) 
    label_list.append(1)
    if len(alpha) > max_len:
        max_len = len(alpha)

!pip install pykakasi

Test loss: 0.484358627357886
Test accuracy: 0.7742368333174133