（備忘録-python）自然言語処理超入門：(やっと)BERTの仕組みを学び・使う(英文)（準初心者向け）

from bs4 import BeautifulSoup    # importする

from tensorflow.keras.datasets import imdb

# imdb = imdb
(train_data, train_labels), (test_data, test_labels) = imdb.load_data()

#単語が整数にマッピングされた辞書を取得
word_index = imdb.get_word_index()

# 最初の要素を予約（単語を登録）
word_index = {k:(v+3) for k,v in word_index.items()} 
word_index["<PAD>"] = 0
word_index["<START>"] = 1
word_index["<UNK>"] = 2  # 不明な単語
word_index["<UNUSED>"] = 3

# 整数を単語にマッピングする辞書を作成
reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])

def decode_review(text):
    return ' '.join([reverse_word_index.get(i, '?') for i in text])

import pandas as pd
train_df=pd.DataFrame(train_data)[0].map(decode_review).reset_index(drop=True)#データを軽くするため
test_df=pd.DataFrame(test_data)[0].map(decode_review).reset_index(drop=True)#データを軽くするため



#前処理
import re
from nltk.stem import WordNetLemmatizer
lemmatizer = WordNetLemmatizer()
#nltk.download('wordnet')
#nltk.download('omw-1.4')

from nltk.stem.porter import PorterStemmer 
stemmer = PorterStemmer()

import nltk
from nltk.corpus import stopwords
# nltk.download('stopwords')

def clean_text(x):
    #ノイズ除去
    soup = BeautifulSoup(x, 'html.parser')
    text= soup.get_text()
    
    #アルファベット以外をスペースに置き換え
    text_ = re.sub(r'[^a-zA-Z]', ' ', text)
    
    #単語長が短いものものは削除(中身による)+その後の処理のために分割
    text_ = [word for word in text_.split()]# if len(word) > 2]
    
    #形態素=>動詞
    text_ = [lemmatizer.lemmatize(word.lower(), pos="v") for word in text_]
    
    #ステミング
#     text_ = [stemmer.stem(word) for word in text_]
    
    #stopword除去
#     A = [word for word in text_ if word not in stopwords.words('english')]
    
    #単語同士をスペースでつなぎ, 文章に戻す
    #その後の処理で戻す必要ない場合はコメントアウト
#     clean_text = ' '.join(A)
    clean_text = ' '.join(text_)
    return clean_text



#データ数減らして処理を軽くしたい...
clean_text_df=train_df[::5].map(clean_text)
clean_text_df

clean_text_test_df=test_df[::10].map(clean_text)
clean_text_test_df


train_texts=clean_text_df.reset_index(drop=True)
train_label=train_labels[::5]

test_texts=clean_text_test_df.reset_index(drop=True)
test_label=test_labels[::10]

np.shape(train_texts),np.shape(train_label),np.shape(test_texts),np.shape(test_label)

import tqdm as notebook_tqdm

#from transformers import AutoTokenizer,glue_convert_examples_to_features,BertTokenizer,DistilBertTokenizer
#from tensorflow.keras.preprocessing.text import Tokenizer
#from tensorflow.keras.preprocessing.sequence import pad_sequences

tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")


# テキストのリストをtransformers用の入力データに変換(BERTの、他のBERTシリーズでは使えない可能性あり)

def to_features(texts, max_length):
    shape = (len(texts), max_length)
    # input_idsやattention_mask, token_type_idsの説明はglossaryに記載(cf. https://huggingface.co/transformers/glossary.html)
    input_ids = np.zeros(shape, dtype="int32")
    attention_mask = np.zeros(shape, dtype="int32")
    token_type_ids = np.zeros(shape, dtype="int32")
    for i, text in enumerate(texts):
        encoded_dict = tokenizer.encode_plus(text, max_length=max_length, pad_to_max_length=True,truncation=True)
        input_ids[i] = encoded_dict["input_ids"]
        attention_mask[i] = encoded_dict["attention_mask"]
        token_type_ids[i] = encoded_dict["token_type_ids"]
    return [tf.cast(input_ids, tf.int32), tf.cast(attention_mask, tf.int32), tf.cast(token_type_ids, tf.int32)]


max_length=500

x_train = to_features(train_texts, max_length)
# y_train = tf.keras.utils.to_categorical(train_labels, num_classes=4)
y_train = tf.cast(train_label, tf.int32)

x_valid = to_features(test_texts, max_length)
# y_valid = tf.keras.utils.to_categorical(valid_labels, num_classes=4)
y_valid = tf.cast(test_label, tf.int32)

from transformers import TFBertModel
bert = TFBertModel.from_pretrained('bert-base-uncased')

# 層をfreeze(学習させないように)する
# bert.trainable= not True

from tensorflow import keras
from tensorflow.keras import optimizers, losses, metrics
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Dense, Dropout
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint

def make_model(bert, num_classes, max_length, bert_frozen=True):
    # bertモデルはリストになっているので、取り出す

    # 層をfreeze(学習させないように)する、消せばfine-tune
    #bert.layers[0].trainable = not bert_frozen

    # input
    input_ids = Input(shape=(max_length, ), dtype='int32', name='input_ids')
    attention_mask = Input(shape=(max_length, ), dtype='int32', name='attention_mask')
    token_type_ids = Input(shape=(max_length, ), dtype='int32', name='token_type_ids')
    inputs = [input_ids, attention_mask,token_type_ids]

    # bert
    x = bert.layers[0](inputs)
    # x: sequence_output, pooled_output
    # 2種類の出力がある。

    # TFBertForSequenceClassificationにならってpooled_outputのみ使用
    out = x[1]

    # fc layer(add layers for transfer learning)
    #out = Dropout(0.25)(out)
    out = Dense(128, activation='relu')(out)
    out = Dropout(0.4)(out)
    out = Dense(num_classes, activation='softmax')(out)
    return Model(inputs=inputs, outputs=out)



seed_everything(42)
adam=optimizers.Adam(learning_rate=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)

epochs = 10
max_length = 500
batch_size = 64
num_classes = 2

model = make_model(bert, num_classes, max_length)

model.compile(optimizer=adam,
              loss='sparse_categorical_crossentropy',
              metrics=['acc'])

# Train the model.
seed_everything(42)

callbacks = [
    EarlyStopping(patience=3,restore_best_weights=True)
]

result=model.fit(x=x_train,
              y=y_train,
              batch_size=batch_size,
              epochs=epochs,
              validation_data=(x_valid, y_valid),#遅いから、いったん学習のみで
              callbacks=callbacks,
              use_multiprocessing=True,
              workers=-1,
              )