【建议收藏】|信息抽取与经济学文本量化分析（下）

model.load_weights('best_model.weights')
def evaluate_data(data):
    """评估函数，计算f1、precision、recall
    """
    X, Y, Z = 1e-10, 1e-10, 1e-10
    f = open('dev_pred.jsonl', 'w', encoding='utf-8')
    pbar = tqdm()
    for d in data:
        R = set([SPO(spo) for spo in extract_spoes(d)])
        if len(R) < 1:
            continue
        T = set()
        X += len(R & T)
        Y += len(R)
        Z += len(T)
        f1, precision, recall = 2 * X / (Y + Z), X / Y, X / Z
        pbar.update()
        pbar.set_description(
            'f1: %.5f, precision: %.5f, recall: %.5f' % (f1, precision, recall)
        )
        s = json.dumps({
            'text': d,
            'spo_list': list(R),
        },
                       ensure_ascii=False)
        f.write(s + '\n')
    pbar.close()
    f.close()
    return f1, precision, recall
evaluate_data(word_list_10000)

{"text": "公司与员工签订劳动合同，规范劳动关系管理工作。公司制定了合理的薪酬管理制度，对公司 员工薪酬设计和管理原则、福利与年假、定级和调薪等方面进行了明确规定。公司高度重视对员工的培训 工作，提升员工素质，实现员工与企业共同成长。",
 "spo_list": [["提升员工素质", "Influence", "实现员工与企业共同成长"]]}

import json
dev_pred = open("dev_pred_50000.jsonl").readlines()
all_data = []
for dev_pred_one in dev_pred:
    all_data.append(json.loads(dev_pred_one))

import networkx as nx
import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams['font.sans-serif'] = ['FZSongYi-Z13S'] # 指定默认字体
from matplotlib import font_manager
# fname中选择一个你本机查询出来的字体 若没有中文字体则需要你本人手动安装
font = font_manager.FontProperties(fname="./simhei.ttf")
plt.figure(figsize=(30, 30))
G = nx.DiGraph()
all_nodes = []
all_relation = []
for one_data in all_data:
    for spo_one in one_data["spo_list"]:
        all_nodes.append(spo_one[0].replace(" ",""))
        all_nodes.append(spo_one[2].replace(" ",""))
        all_relation.append((spo_one[0].replace(" ",""),spo_one[2].replace(" ","")))
all_set_nodes = list(set(all_nodes))
# G.add_nodes_from(all_set_nodes[:200])   # 添加节点 1 2 3
G.add_edges_from(all_relation)   # 添加多条边
pagerank_list = nx.pagerank(G, alpha=1)

a = sorted(pagerank_list.items(),key = lambda x:x[1],reverse = True)
a[:30]

[('对公司的经营业绩产生不利影响', 0.0008828635052164171),
 ('增加公司的经营风险', 0.000840754589394436),
 ('对公司经营业绩产生不利影响', 0.000629184609583069),
 ('提高生产效率', 0.0005525447685727118),
 ('公司的盈利能力', 0.0005452975937731509),
 ('降低生产成本', 0.0005247477884018558),
 ('投资者在证券交易中遭受损失', 0.0005004412738005438),
 ('利润', 0.000495572338004653),
 ('差异', 0.0004899555161635981),
 ('业绩同比较大增长', 0.0004629239604155248),
 ('公司的经营业绩', 0.00039319747216364984),
 ('提高客户的满意度和忠诚度', 0.0003917398023592222),
 ('研发费用变动', 0.0003686641308238447),
 ('促进公司的新能源汽车业务持续发展', 0.00036793980326920686),
 ('对公司的经营业绩造成不利影响', 0.00036793529592163085),
 ('促进企业与员工和谐共赢、共同发展', 0.00036501995631277545),
 ('影响公司盈利能力', 0.00036501995631277545),
 ('医药行业在国民经济的比重日益扩大', 0.00036501995631277545),
 ('促进公司经营业绩的提升', 0.00036501995631277545),
 ('筹资活动产生的现金流量净额变动', 0.0003423087022285049),
 ('债务人信用风险的预期变动', 0.0003412154498751841),
 ('现金及现金等价物净增加额同比减少', 0.0003397577800707564),
 ('提升企业整体盈利水平', 0.00033609035859508547),
 ('提高企业市场竞争力', 0.0003178754563592766),
 ('机器人高精密减速机业务获得高速发展', 0.00031595327363316507),
 ('行业高质量发展水平持续提升', 0.00031595327363316507),
 ('夯实公司的行业领先地位', 0.00031595327363316507),
 ('提升经济效', 0.00031595327363316507),
 ('增强公司竞争力', 0.00031595327363316507),
 ('良好的基本面没有发生改变', 0.00031595327363316507)]

plt.figure(figsize=(90, 90))
nx.draw(G, pos=layout, node_size=10,node_color='r', with_labels=True)
plt.savefig('./2021企业年报因果抽取-pagerank.jpg', dpi=200, bbox_inches = 'tight')

from pyvis.network import Network
import os
import pandas as pd
import networkx as nx
G = nx.Graph()
nt = Network('1500px', '1500px', notebook=True, neighborhood_highlight=False, select_menu=True, filter_menu=False, )
for one_data in all_relation_data:
    try:
        if relation_counts[one_data[0].strip()] > 1:
            G.add_edge(one_data[0].strip(),one_data[2].strip(), label=one_data[1])
    except:
        continue
nt.from_nx(G)
nt.show_buttons(filter_=['physics'])
nt.show("2021企业事件因果可视化.html")

import json
all_relation = open("dev_pred_50000.jsonl").readlines()
all_relation_data = []
entities_id_mapping = {}
relation_count = []
for one in all_relation:
    relations = json.loads(one)["spo_list"]
    for relation in relations:
        all_relation_data.append((relation[0].replace(" ",""),
                                  relation[1],
                                  relation[2].replace(" ","")))
    relation_count.append(relation[0].replace(" ",""))
all_relation_data = list(set(all_relation_data))
len(all_relation_data)

from collections import Counter
count = Counter(relation_count)  # 统计词频
relation_counts = dict(count)

from pyvis.network import Network
import os
import pandas as pd
import networkx as nx
G = nx.Graph()
nt = Network('1500px', '1500px', notebook=True)
for one_data in all_relation_data:
    try:
        if relation_counts[one_data[0].strip()] > 5:
            G.add_edge(one_data[0].strip(),one_data[2].strip(), label="导致")
    except:
        continue
nt.from_nx(G)
# nt.show_buttons(filter_=['physics'])
nt.show("2021企业事件因果可视化.view.top.5.html")