– 因此，傳統的語言模型演算法大多透過詞袋模型(Bag-of-words model)的方法進行特徵萃取，而這個方法在像是垃圾郵件分類等任務中也取得非常好的成績：
– 這種向量化的方式非常類似於類別變項的處理，也有另外一個稱呼叫做獨熱編碼(one hot encoding)。
my.eval.metric.loss <- mx.metric.custom(
name = "cross-entropy",
function(label, pred) {
return(as.array(pred))
}
)
my_model <- mx.model.FeedForward.create(symbol = ce_loss, X = my_iter, optimizer = my_optimizer,
array.batch.size = 20, ctx = mx.gpu(), num.round = 30,
eval.metric = my.eval.metric.loss,
batch.end.callback = mx.callback.log.speedometer(frequency = 50, batch.size = 20))library(pROC)
roc_result <- roc(Test.Y, pred_y)
plot(roc_result, col = "red", main = paste0('AUC = ', formatC(roc_result$auc, 3, format = 'f')))
我們不可能有足夠完整的字典做同義字的整合及片語的蒐集。那這樣你應該能想像當我們使用詞袋模型時，必須對每個文章都產生極長的向量，這不但浪費並且容易過度擬合。
如果在未來應用時存在訓練樣本中未曾出現的單字，那分類器將「無法」利用這個資訊。
– 如果我們想要把這個突破應用到其他領域上，那勢必要想個辦法連結圖片與該領域的關係，而目前看起來詞袋模型似乎並不合用。
– 雖然卷積網路雖然最初雖然是設計來做識別影像的，但只要我們能把「文字描述」轉為「圖片」格式，我們就一樣能利用卷積神經網路進行分類。
– 有一種想法是先把文字轉為向量，並且盡可能讓相似詞的向量足夠相近，這個方法叫做詞嵌入(word embedding)
– 因此，只要我們建構一個用上下文預測中間的字的模型，那當你輸入「chronic」以及「disease」並要求網路做出預測時，他會沒有辦法判斷該輸出「kidney」或者是「renal」，我們就能由此判斷這兩個字擁有相似的字意。
– 為了較快實現後面的課程，這裡已經準備好了一個由英文Wiki上所有的文章中所訓練而成的詞嵌入矩陣，請點擊這裡下載！
library(magrittr)
library(dplyr)
library(plyr)
library(data.table)
word.data <- fread('data/wiki.txt', header = FALSE, showProgress = FALSE)
words.ref <- word.data %>% select(V1) %>% setDF %>% .[,1] %>% as.character
words.ref <- c("", words.ref)
words.index <- 1:length(words.ref)
names(words.index) <- words.ref
words.matrix <- word.data %>% select(-V1) %>% setDF %>% as.matrix
words.matrix <- rbind(rep(0, 50), words.matrix)load('data/Discharge_Notes.RData')
words_pos <- words.index[Processed_Notes[[1]]]
words_pos[is.na(words_pos)] <- 1
text_img.array <- t(words.matrix[words_pos,])
dim(text_img.array)## [1] 50 301library(imager)
par(mai = rep(0, 4))
img <- text_img.array
img[img>2] <- 2
img[img<-2] <- -2
plot(as.cimg(t(img)))
my.eval.metric.loss <- mx.metric.custom(
name = "cross-entropy",
function(label, pred) {
return(as.array(pred))
}
)
my_model <- mx.model.FeedForward.create(symbol = ce_loss, X = my_iter, optimizer = my_optimizer,
array.batch.size = 20, ctx = mx.gpu(), num.round = 30,
eval.metric = my.eval.metric.loss,
batch.end.callback = mx.callback.log.speedometer(frequency = 50, batch.size = 20))library(pROC)
roc_result <- roc(Test.Y, pred_y)
plot(roc_result, col = "red", main = paste0('AUC = ', formatC(roc_result$auc, 3, format = 'f')))
– 另外，這樣的模型有更少的參數量，因此也會較難發生過度擬合的問題！
– 讓我們參考一下Chin Lin等人發表的：Artificial Intelligence Learning Semantics via External Resources for Classifying Diagnosis Codes in Discharge Notes，看看人家是怎樣設計Model architecture的：
data <- mx.symbol.Variable(name = 'data')
label <- mx.symbol.Variable(name = 'label')
concat_lst <- NULL
filter_sizes <- 1:5
num_filter <- c(40, 30, 15, 10, 5)
for (i in 1:length(filter_sizes)) {
convi <- mx.symbol.Convolution(data = data, kernel = c(50, filter_sizes[i]), num_filter = num_filter[i], name = paste0('conv', i))
relui <- mx.symbol.Activation(data = convi, act_type = "relu", name = paste0('relu', i))
pooli <- mx.symbol.max(data = relui, axis = 2:3, keepdims = FALSE, name = paste0('pool', i))
concat_lst <- append(concat_lst, pooli)
}
h_pool <- mx.symbol.concat(data = concat_lst, num.args = length(filter_sizes), dim = 1, name = 'h_pool')
h_drop <- mx.symbol.Dropout(data = h_pool, p = 0.5, name = 'h_drop')
fc1 <- mx.symbol.FullyConnected(data = h_drop, num.hidden = 1, name = 'fc1')
logistic_pred <- mx.symbol.sigmoid(data = fc1, name = 'logistic_pred')
eps <- 1e-8
ce_loss_pos <- mx.symbol.broadcast_mul(mx.symbol.log(logistic_pred + eps), label)
ce_loss_neg <- mx.symbol.broadcast_mul(mx.symbol.log(1 - logistic_pred + eps), 1 - label)
ce_loss_mean <- 0 - mx.symbol.mean(ce_loss_pos + ce_loss_neg)
ce_loss <- mx.symbol.MakeLoss(ce_loss_mean, name = 'ce_loss')my.eval.metric.loss <- mx.metric.custom(
name = "cross-entropy",
function(label, pred) {
return(as.array(pred))
}
)
my_model <- mx.model.FeedForward.create(symbol = ce_loss, X = my_iter, optimizer = my_optimizer,
array.batch.size = 20, ctx = mx.gpu(), num.round = 30,
eval.metric = my.eval.metric.loss,
batch.end.callback = mx.callback.log.speedometer(frequency = 50, batch.size = 20))my_model$symbol <- logistic_pred
pred_y <- predict(my_model, Test.img_array)
roc_result <- roc(Test.Y, pred_y)
plot(roc_result, col = "red", main = paste0('AUC = ', formatC(roc_result$auc, 3, format = 'f')))
new_arg <- mxnet:::mx.model.init.params(symbol = logistic_pred,
input.shape = list(data = c(494, 1, 20)),
output.shape = NULL, initializer = mxnet:::mx.init.Xavier(rnd_type = "uniform", magnitude = 2.24),
ctx = mx.gpu())
new_arg$arg.params$embed1_weight <- mx.nd.array(t(words.matrix))
my.eval.metric.loss <- mx.metric.custom(
name = "cross-entropy",
function(label, pred) {
return(as.array(pred))
}
)
my_model <- mx.model.FeedForward.create(symbol = ce_loss, X = my_iter, optimizer = my_optimizer,
array.batch.size = 20, ctx = mx.gpu(), num.round = 30,
eval.metric = my.eval.metric.loss,
arg.params = new_arg$arg.params, fixed.param = 'embed1_weight',
batch.end.callback = mx.callback.log.speedometer(frequency = 50, batch.size = 20))my_model$symbol <- logistic_pred
my_model$arg.params <- append(my_model$arg.params, new_arg$arg.params['embed1_weight'])
pred_y <- predict(my_model, Test.seq_array)library(pROC)
roc_result <- roc(Test.Y, pred_y)
plot(roc_result, col = "red", main = paste0('AUC = ', formatC(roc_result$auc, 3, format = 'f')))
– 但是wiki共有34萬多個單字，而這個月的病歷中僅僅存在5千多個單字，這代表的是假定允許模型訓練時更新詞嵌入矩陣的權重，那也僅僅有那5千多個單字的權重會改變
– 更嚴重的是，假設病歷中原先有H5N1這個單字而沒有H1N1，在最開始的時候他們兩個的向量本來是很接近的，會不會訓練一陣子之後就離開了?
data <- mx.symbol.Variable(name = 'data')
label <- mx.symbol.Variable(name = 'label')
embed1 <- mx.symbol.Embedding(data = data, input.dim = length(words.ref), output.dim = 50, name = 'embed1')
embed2 <- mx.symbol.Convolution(data = embed1, kernel = c(50, 1), num_filter = 50, no.bias = TRUE, name = 'embed2')
concat_lst <- NULL
filter_sizes <- 1:5
num_filter <- c(40, 30, 15, 10, 5)
for (i in 1:length(filter_sizes)) {
convi <- mx.symbol.Convolution(data = embed2, kernel = c(1, filter_sizes[i]), num_filter = num_filter[i], name = paste0('conv', i))
relui <- mx.symbol.Activation(data = convi, act_type = "relu", name = paste0('relu', i))
pooli <- mx.symbol.max(data = relui, axis = 2:3, keepdims = FALSE, name = paste0('pool', i))
concat_lst <- append(concat_lst, pooli)
}
h_pool <- mx.symbol.concat(data = concat_lst, num.args = length(filter_sizes), dim = 1, name = 'h_pool')
h_drop <- mx.symbol.Dropout(data = h_pool, p = 0.5, name = 'h_drop')
fc1 <- mx.symbol.FullyConnected(data = h_drop, num.hidden = 1, name = 'fc1')
logistic_pred <- mx.symbol.sigmoid(data = fc1, name = 'logistic_pred')
eps <- 1e-8
ce_loss_pos <- mx.symbol.broadcast_mul(mx.symbol.log(logistic_pred + eps), label)
ce_loss_neg <- mx.symbol.broadcast_mul(mx.symbol.log(1 - logistic_pred + eps), 1 - label)
ce_loss_mean <- 0 - mx.symbol.mean(ce_loss_pos + ce_loss_neg)
ce_loss <- mx.symbol.MakeLoss(ce_loss_mean, name = 'ce_loss')– 注意一下維度的改變：
## $embed1_output
## [1] 50 494 1 20## $embed2_output
## [1] 1 494 50 20