传统的序列模型CRF与HMM区别

# 导入所需的库

import nltk

import sklearn_crfsuite

from sklearn_crfsuite import metrics

# 加载NER数据集

nltk.download('conll2002')

train_sents = list(nltk.corpus.conll2002.iob_sents('esp.train'))

test_sents = list(nltk.corpus.conll2002.iob_sents('esp.testb'))

# 特征提取函数

def word2features(sent, i):

word = sent[i][0]

features = {

'bias': 1.0,

'word.lower()': word.lower(),

'word[-3:]': word[-3:],

'word[-2:]': word[-2:],

'word.isupper()': word.isupper(),

'word.istitle()': word.istitle(),

'word.isdigit()': word.isdigit()

}

if i > 0:

word1 = sent[i-1][0]

features.update({

'-1:word.lower()': word1.lower(),

'-1:word.istitle()': word1.istitle(),

'-1:word.isupper()': word1.isupper()

})

else:

features['BOS'] = True

if i < len(sent)-1:

word1 = sent[i+1][0]

features.update({

'+1:word.lower()': word1.lower(),

'+1:word.istitle()': word1.istitle(),

'+1:word.isupper()': word1.isupper()

})

else:

features['EOS'] = True

return features

def sent2features(sent):

return [word2features(sent, i) for i in range(len(sent))]

def sent2labels(sent):

return [label for token, postag, label in sent]

def sent2tokens(sent):

return [token for token, postag, label in sent]

# 特征提取

X_train = [sent2features(s) for s in train_sents]

y_train = [sent2labels(s) for s in train_sents]

X_test = [sent2features(s) for s in test_sents]

y_test = [sent2labels(s) for s in test_sents]

# 训练CRF模型

crf = sklearn_crfsuite.CRF(

algorithm='lbfgs',

c1=0.1,

c2=0.1,

max_iterations=100,

all_possible_transitions=True

)

crf.fit(X_train, y_train)

# 在测试集上进行预测

y_pred = crf.predict(X_test)

# 输出模型评估结果

print(metrics.flat_classification_report(y_test, y_pred, labels=crf.classes_))