（備忘録-python）自然言語処理超入門：(やっと)Transformerの仕組みを学び・使う（準初心者向け）

from tensorflow.keras.datasets import imdb

# imdb = imdb
(train_data, train_labels), (test_data, test_labels) = imdb.load_data()

#単語が整数にマッピングされた辞書を取得
word_index = imdb.get_word_index()

# 最初の要素を予約（単語を登録）
word_index = {k:(v+3) for k,v in word_index.items()} 
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # 不明な単語
word_index["<UNUSED>"] = 3

# 整数を単語にマッピングする辞書を作成
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])

def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])

import pandas as pd
train_df=pd.DataFrame(train_data)[0].map(decode_review).reset_index(drop=True)
test_df=pd.DataFrame(test_data)[0].map(decode_review).reset_index(drop=True)

# all_df=pd.concat([train_df,test_df])[::10].reset_index(drop=True)
# all_df

import re
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
#nltk.download('wordnet')
#nltk.download('omw-1.4')

from nltk.stem.porter import PorterStemmer 
stemmer = PorterStemmer()

import nltk
from nltk.corpus import stopwords
# nltk.download('stopwords')

def clean_text(x):
    #ノイズ除去
    soup = BeautifulSoup(x, 'html.parser')
    text= soup.get_text()
    
    #アルファベット以外をスペースに置き換え
    text_ = re.sub(r'[^a-zA-Z]', ' ', text)
    
    #単語長が短いものものは削除(中身による)+その後の処理のために分割
    text_ = [word for word in text_.split() if len(word) > 2]
    
    #形態素=>動詞
    text_ = [lemmatizer.lemmatize(word.lower(), pos="v") for word in text_]
    
    #ステミング
#     text_ = [stemmer.stem(word) for word in text_]
    
    #stopword除去
    A = [word for word in text_ if word not in stopwords.words('english')]
    
    #単語同士をスペースでつなぎ, 文章に戻す
    #その後の処理で戻す必要ない場合はコメントアウト
    clean_text = ' '.join(A)
    return clean_text


#軽くするために10個飛ばし
clean_text_df=train_df[::10].map(clean_text)
clean_text_df

clean_text_test_df=test_df[::10].map(clean_text)
clean_text_test_df

clean_text_df=train_df[::10].map(clean_text)
clean_text_df

clean_text_test_df=test_df[::10].map(clean_text)
clean_text_test_df

from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

all_text=pd.concat([train_texts,test_texts]).reset_index(drop=True)

sentences = []
for text in all_text:
    text_list = text.split(' ')
    sentences.append(text_list)

tokenizer = Tokenizer()
tokenizer.fit_on_texts(sentences)


sequences_tk = tokenizer.texts_to_sequences(sentences)
#pd.DataFrame(sequences_tk)

MAX_SEQUENCE_LENGTH =int(pd.DataFrame(sequences_tk).shape[1])
MAX_SEQUENCE_LENGTH

#要素の合わない配列に対して、0 で埋めるなどして配列のサイズを一致させる。
X=pad_sequences(sequences_tk, maxlen=MAX_SEQUENCE_LENGTH, truncating='post')


X_train=X[:train_texts.shape[0]]
X_test=X[train_texts.shape[0]:]


word_index = tokenizer.word_index
num_words = len(word_index)
print(num_words)

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras import optimizers
from tensorflow.keras import regularizers

class TransformerBlock(layers.Layer):
    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):
        super(TransformerBlock, self).__init__()
        self.att = layers.MultiHeadAttention(num_heads=num_heads, key_dim=embed_dim)
        self.ffn = keras.Sequential(
            [layers.Dense(ff_dim, activation="relu"), layers.Dense(embed_dim),]
        )
        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)
        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)
        self.dropout1 = layers.Dropout(rate)
        self.dropout2 = layers.Dropout(rate)

    def call(self, inputs, training):
        attn_output = self.att(inputs, inputs)
        attn_output = self.dropout1(attn_output, training=training)
        out1 = self.layernorm1(inputs + attn_output)
        ffn_output = self.ffn(out1)
        ffn_output = self.dropout2(ffn_output, training=training)
        return self.layernorm2(out1 + ffn_output)


class TokenAndPositionEmbedding(layers.Layer):
    def __init__(self, maxlen, vocab_size, embed_dim):
        super(TokenAndPositionEmbedding, self).__init__()
        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)
        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)

    def call(self, x):
        maxlen = tf.shape(x)[-1]
        positions = tf.range(start=0, limit=maxlen, delta=1)
        positions = self.pos_emb(positions)
        x = self.token_emb(x)
        return x + positions


embed_dim = 128  # Embedding size for each token
num_heads = 8  # Number of attention heads
ff_dim = 128  # Hidden layer size in feed forward network inside transformer

inputs = layers.Input(shape=(maxlen,))
embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)
x = embedding_layer(inputs)
transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)
x = transformer_block(x)
x = layers.GlobalAveragePooling1D()(x)
x = layers.Dropout(0.1)(x)

x = layers.Dense(64, kernel_regularizer=regularizers.l1_l2(l1=0.01, l2=0.01), activation="relu")(x)
x = layers.Dropout(0.1)(x)
outputs = layers.Dense(2, activation="softmax")(x)

model = keras.Model(inputs=inputs, outputs=outputs)



# Preparing callbacks.
adam=optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model.compile(optimizer=adam,
              loss='sparse_categorical_crossentropy',
              metrics=['acc'])

callbacks = [
    EarlyStopping(patience=10)
]

seed_everything(42)

# Train the model.
history=model.fit(x=X_train,
                  y=train_label,
                  batch_size=64,
                  epochs=100,
                  validation_split=0.2,
                  callbacks=callbacks,
                  shuffle=True)


hist_df = pd.DataFrame(history.history)
# 可視化
plt.figure()
hist_df[['acc', 'val_acc']].plot()
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.show()

plt.figure()
hist_df[['loss', 'val_loss']].plot()
plt.ylabel('loss')
plt.xlabel('epoch')
plt.show()