开源radam结合wwm-bert分类

Discussion

Home Discussion Board BAAI & ICT - False news Detection Task 1 开源radam结合wwm-bert分类

空空

开源radam结合wwm-bert分类

posted in BAAI & ICT - False news Detection Task 1

2019-09-17 15:24

import numpy as np
import pandas as pd
from keras_bert import load_trained_model_from_checkpoint, Tokenizer
import codecs
import codecs

import numpy as np
import pandas as pd

from keras_bert import load_trained_model_from_checkpoint, Tokenizer

max_seq_len = 100
config_path = 'wwm/bert_config.json'
checkpoint_path = 'wwm/bert_model.ckpt'
dict_path = 'wwm/vocab.txt'

token_dict = {}

with codecs.open(dict_path, 'r', 'utf8') as reader:
    for line in reader:
        token = line.strip()
        token_dict[token] = len(token_dict)

class OurTokenizer(Tokenizer):
    def _tokenize(self, text):
        R = []
        for c in text:
            if c in self._token_dict:
                R.append(c)
            elif self._is_space(c):
                R.append('[unused1]')  # space类用未经训练的[unused1]表示
            else:
                R.append('[UNK]')  # 剩余的字符是[UNK]
        return R

tokenizer = OurTokenizer(token_dict)

train_set = pd.read_csv('data/train.csv').values.tolist()

data = []

for i in train_set:
    data.append((i[1], i[2]))

# 按照9:1的比例划分训练集和验证集
random_order = list(range(len(data)))
np.random.shuffle(random_order)
train_data = [data[j] for i, j in enumerate(random_order) if i % 10 != 0]
valid_data = [data[j] for i, j in enumerate(random_order) if i % 10 == 0]

def seq_padding(X, padding=0):
 L = [len(x) for x in X]
 ML = max(L)
 return np.array([
 np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X
 ])

class data_generator:
    def __init__(self, data, batch_size=16):
        self.data = data
        self.batch_size = batch_size
        self.steps = len(self.data) // self.batch_size
        if len(self.data) % self.batch_size != 0:
            self.steps += 1

def __len__(self):
        return self.steps

def __iter__(self):
        while True:
            idxs = list(range(len(self.data)))
            np.random.shuffle(idxs)
            X1, X2, Y = [], [], []
            for i in idxs:
                d = self.data[i]
                text = d[0][:max_seq_len]
                x1, x2 = tokenizer.encode(first=text)
                y = d[1]
                X1.append(x1)
                X2.append(x2)
                Y.append([y])
                if len(X1) == self.batch_size or i == idxs[-1]:
                    X1 = seq_padding(X1)
                    X2 = seq_padding(X2)
                    Y = seq_padding(Y)
                    yield [X1, X2], Y
                    [X1, X2, Y] = [], [], []

from keras.layers import *
from keras.models import Model
from keras.optimizers import Adam
from keras_radam import RAdam

bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None)

for l in bert_model.layers:
    l.trainable = True

x1_in = Input(shape=(None,))
x2_in = Input(shape=(None,))

x = bert_model([x1_in, x2_in])
x = Lambda(lambda x: x[:, 0])(x)
p = Dense(1, activation='sigmoid')(x)

model = Model([x1_in, x2_in], p)
model.compile(
    loss='binary_crossentropy',
    optimizer=RAdam(1e-5),  # 用足够小的学习率
    metrics=['accuracy']
)
model.summary()

train_D = data_generator(train_data)
valid_D = data_generator(valid_data)

model.fit_generator(
    train_D.__iter__(),
    steps_per_epoch=len(train_D),
    epochs=20,
    validation_data=valid_D.__iter__(),
    validation_steps=len(valid_D)
)
model.save("bert_radam_model")
test_set = pd.read_csv('data/test_stage1.csv').values.tolist()
test_data = []
for i in test_set:
    test_data.append((i[1], i[2]))
test_D=data_generator(test_data)

23 comments

markeloff

2019-09-18 03:13

空空 reply markeloff

2019-09-18 06:26

Reply

0

1个epoch到83分。再跑32epoch应该可以过九十五

空空 reply markeloff

2019-09-27 01:40

Reply

0

抱歉没比赛经验，最后成绩是86分对不起，又吹牛逼了

hua666 reply 空空

2019-09-27 07:02

Reply

0

在验证集上的得分好像与提交上去的测试集的得分完全不对等啊

空空

2019-10-09 15:23