dataset_path = r'@_数据集_原始/Reuters-21578/reutersNLTK.xlsx'
df = pd.read_excel(dataset_path, encoding='utf-8', sep=',')
result_single = {} // 统计每个标签出现的次数
result_multi = {}
a = [] // 存放每个文本的标签,可重复,一维数组
b = [] // 存放每个文本的标签数量,一维数组
# 提取标签
for ls in df['categories']:
    b.append(len(re.compile(r"'(.*?)'").findall(ls)))
    for l in re.compile(r"'(.*?)'").findall(ls):
        a.append(l)
# 计算标签次数
for i in set(a):
    result_single[i] = a.count(i)
# 倒叙排列后输出
label_list = sorted([_ for _ in result_single.items()], key=lambda x: x[1], reverse=True)
print(label_list)

label_key = [word_count[0] for idx, word_count in enumerate(label_list)]
label_val = [word_count[1] for idx, word_count in enumerate(label_list)]
print("标签个数:", len(label_key))
print("文本标签数目:", set(b))
print("每个文本平均有标签:", sum(label_val) / len(df['categories']))

k = {}
# 计算共现标签
for i in set(b):
    k[i] = b.count(i)
print(k)

import random
import pandas as pd
dataset_path =r'@_数据集_原始/Reuters-21578/reutersNLTK.xlsx'
df = pd.read_excel(dataset_path, encoding='utf-8', sep=',')
""" 多标签数据集划分 Reuter-21578
    label_name :    标签列名
    content_name :  内容列名
    export_name :   导出的文件名
    proportion :    训练-验证-测试比例, 数组, 加和为10, eg: [7,1.5,1.5] 
"""
def divideDataSet(label_name, content_name, export_name, proportion):
    contents = df[content_name]
    labels = df[label_name]
    ids = df['ids']
    train, val, test = [], [], []
    for i, id in enumerate(ids):
        if id[:3] == 'tra':
            train.append({'label': labels[i], 'content': contents[i]})
        else:
            val.append({'label': labels[i], 'content': contents[i]})
    random.shuffle(train)
    random.shuffle(val)
    test = val[:1500]
    val = val[1500:]
    print(len(train), len(val), len(test))
    random.shuffle(train)
    random.shuffle(val)
    random.shuffle(test)
    train.insert(0, {'label': 'label', 'content': 'content'})
    val.insert(0, {'label': 'label', 'content': 'content'})
    test.insert(0, {'label': 'label', 'content': 'content'})
    with open(export_name + r'train.csv', 'a', newline='', encoding='utf-8') as f:
        xieru = csv.DictWriter(f, ['label', 'content'], delimiter=',')
        xieru.writerows(train)  # writerows方法是一下子写入多行内容
    with open(export_name + r'dev.csv', 'a', newline='', encoding='utf-8') as f:
        xieru = csv.DictWriter(f, ['label', 'content'], delimiter=',')
        xieru.writerows(val)  # writerows方法是一下子写入多行内容
    with open(export_name + r'test.csv', 'a', newline='', encoding='utf-8') as f:
        xieru = csv.DictWriter(f, ['label', 'content'], delimiter=',')
        xieru.writerows(test)  # writerows方法是一下子写入多行内容
divideDataSet('categories', 'text', root_path + r'@_数据集_已处理/Reuters-21578/data/', [])