在NLP中结合文本和数字特征进行机器学习

importnumpyasnpfromsklearn.feature_extraction.textimportTfidfVectorizerfromsklearn.pipelineimportPipeline, FeatureUnionfromsklearn.ensembleimportRandomForestClassifierfromsklearn.preprocessingimportFunctionTransformerfromsklearn.model_selectionimportGridSearchCV, StratifiedKFold#CreateFunctionTransformertouseFeatureUniondefget_numeric_data(x):
return [record[:-2].astype(float) forrecordinx]
defget_text_data(x):
return [record[-1] forrecordinx]
transfomer_numeric=FunctionTransformer(get_numeric_data)
transformer_text=FunctionTransformer(get_text_data)
#CreateapipelinetoconcatenateTfidfVectorandNumericdata#UseRandomForestClassifierasanexamplepipeline=Pipeline([
    ('features', FeatureUnion([
            ('numeric_features', Pipeline([
                ('selector', transfomer_numeric)
            ])),
              ('text_features', Pipeline([
                ('selector', transformer_text),
                ('vec', TfidfVectorizer(analyzer='word'))
            ]))
          ])),
    ('clf', RandomForestClassifier())
])
#GridSearchParametersforRandomForestparam_grid= {'clf__n_estimators': np.linspace(1, 100, 10, dtype=int),
'clf__min_samples_split': [3, 10],
'clf__min_samples_leaf': [3],
'clf__max_features': [7],
'clf__max_depth': [None],
'clf__criterion': ['gini'],
'clf__bootstrap': [False]}
#Trainingconfigkfold=StratifiedKFold(n_splits=7)
scoring= {'Accuracy': 'accuracy', 'F1': 'f1_macro'}
refit='F1'#PerformGridSearchrf_model=GridSearchCV(pipeline, param_grid=param_grid, cv=kfold, scoring=scoring,
refit=refit, n_jobs=-1, return_train_score=True, verbose=1)
rf_model.fit(X_train, Y_train)
rf_best=rf_model.best_estimator_

classLSTMTextClassifier(nn.Module):
def__init__(self, vocab_size, embed_size, lstm_size, dense_size, numeric_feature_size, output_size, lstm_layers=1, dropout=0.1):
super().__init__()
self.vocab_size=vocab_sizeself.embed_size=embed_sizeself.lstm_size=lstm_sizeself.output_size=output_sizeself.lstm_layers=lstm_layersself.dropout=dropoutself.embedding=nn.Embedding(vocab_size, embed_size)
self.lstm=nn.LSTM(embed_size, lstm_size, lstm_layers, dropout=dropout, batch_first=False)
self.dropout=nn.Dropout(0.2)
self.fc1=nn.Linear(lstm_size, dense_size)
self.fc2=nn.Linear(dense_size+numeric_feature_size, output_size)
self.softmax=nn.LogSoftmax(dim=1)
definit_hidden(self, batch_size):
weight=next(self.parameters()).datahidden= (weight.new(self.lstm_layers, batch_size, self.lstm_size).zero_(),
weight.new(self.lstm_layers, batch_size, self.lstm_size).zero_())
returnhiddendefforward(self, nn_input_text, nn_input_meta, hidden_state):
batch_size=nn_input_text.size(0)
nn_input_text=nn_input_text.long()
embeds=self.embedding(nn_input_text)
lstm_out, hidden_state=self.lstm(embeds, hidden_state)
lstm_out=lstm_out[-1,:,:]
lstm_out=self.dropout(lstm_out)
dense_out=self.fc1(lstm_out)
concat_layer=torch.cat((dense_out, nn_input_meta.float()), 1)
out=self.fc2(concat_layer)
logps=self.softmax(out)
returnlogps, hidden_stateclassBertTextClassifier(nn.Module):
def__init__(self, hidden_size, dense_size, numeric_feature_size, output_size, dropout=0.1):
super().__init__()
self.output_size=output_sizeself.dropout=dropout#Usepre-trainedBERTmodelself.bert=BertModel.from_pretrained('bert-base-uncased', output_hidden_states=True, output_attentions=True)
forparaminself.bert.parameters():
param.requires_grad=Trueself.weights=nn.Parameter(torch.rand(13, 1))
self.dropout=nn.Dropout(dropout)
self.fc1=nn.Linear(hidden_size, dense_size)
self.fc2=nn.Linear(dense_size+numeric_feature_size, output_size)
self.softmax=nn.LogSoftmax(dim=1)
defforward(self, input_ids, nn_input_meta):        
all_hidden_states, all_attentions=self.bert(input_ids)[-2:]
batch_size=input_ids.shape[0]
ht_cls=torch.cat(all_hidden_states)[:, :1, :].view(13, batch_size, 1, 768)
atten=torch.sum(ht_cls*self.weights.view(13, 1, 1, 1), dim=[1, 3])
atten=F.softmax(atten.view(-1), dim=0)
feature=torch.sum(ht_cls*atten.view(13, 1, 1, 1), dim=[0, 2])
dense_out=self.fc1(self.dropout(feature))
concat_layer=torch.cat((dense_out, nn_input_meta.float()), 1)
out=self.fc2(concat_layer)
logps=self.softmax(out)
returnlogps