查阅网上很多博客关于其他文本格式转化成utf-8，会出现中文乱码的情况，自己也遇到了，看到网上很多乱七八糟的博客，都没有什么实质性的解决方案。为此专门写了这篇博客，希望能对你们有帮助。
导入模块
# -*- coding: utf-8 -*- # @Time : 2021/8/10 12:03 # @Author : xiehou # @File : test.py # @Software: PyCharm import pandas as pd from pandas import DataFrame import os from chardet import detect
对所有文本文件进行读取，并重写
def reformat_txt(txt_dir_path): txt_names = os.listdir(txt_dir_path) for txt_name in txt_names: with open(txt_dir_path + '/' + txt_name, 'rb') as file: content = file.read() # 获取文件的编码格式 coding = detect(content)['encoding'] # 以该文件的格式进行读取 with open(txt_dir_path + '/' + txt_name, 'r', encoding=coding) as file: content = file.read() # 对之前读取的文件进行重写到result文件夹中，并以utf-8的格式进行写入 # 方便在对其文件内容进行读取 with open('./result/' + txt_name, 'w', encoding='utf-8') as f: f.write(content)
对重写文件以utf-8的格式进行读取
def read_txt(txt_dir_path): all_information = [] txt_names = os.listdir(txt_dir_path) for txt_name in txt_names: with open(txt_dir_path + '/' + txt_name, 'r', encoding='utf-8') as file: content = file.readlines() all_information.append((content, txt_name)) return all_information
将文本内容写入到csv文件中
def write_csv(all_information, tag, csv_path): tags = [] titles = [] contents = [] for information in all_information: title = information[1] content = information[0] titles.append(title) contents.append(content) tags.append(tag) infor_dict = { "标题": titles, "内容": contents, "标签": tags } data = DataFrame(infor_dict) data.to_csv(csv_path)
主函数
if __name__ == '__main__': # 原始文件夹,该文件夹包含utf-8格式和ansi格式等各种文本格式, # 需要更改文件夹名称 txt_dir_path = './file' reformat_txt(txt_dir_path) # 中间文件夹,不需要改动(需要自行新建该文件夹，要不然会报错) txt_dir_path = './result' all_information = read_txt(txt_dir_path) # 目标csv的标签 tag = '涉恐事件2.0' csv_path = './' + tag + '.csv' write_csv(all_information, tag, csv_path)
完整代码
# -*- coding: utf-8 -*- # @Time : 2021/8/10 12:03 # @Author : xiehou # @File : test.py # @Software: PyCharm import pandas as pd from pandas import DataFrame import os from chardet import detect def reformat_txt(txt_dir_path): txt_names = os.listdir(txt_dir_path) for txt_name in txt_names: with open(txt_dir_path + '/' + txt_name, 'rb') as file: content = file.read() # 获取文件的编码格式 coding = detect(content)['encoding'] # 以该文件的格式进行读取 with open(txt_dir_path + '/' + txt_name, 'r', encoding=coding) as file: content = file.read() # 对之前读取的文件进行重写到result文件夹中，并以utf-8的格式进行写入 # 方便在对其文件内容进行读取 with open('./result/' + txt_name, 'w', encoding='utf-8') as f: f.write(content) def read_txt(txt_dir_path): all_information = [] txt_names = os.listdir(txt_dir_path) for txt_name in txt_names: with open(txt_dir_path + '/' + txt_name, 'r', encoding='utf-8') as file: content = file.readlines() all_information.append((content, txt_name)) return all_information def write_csv(all_information, tag, csv_path): tags = [] titles = [] contents = [] for information in all_information: title = information[1] content = information[0] titles.append(title) contents.append(content) tags.append(tag) infor_dict = { "标题": titles, "内容": contents, "标签": tags } data = DataFrame(infor_dict) data.to_csv(csv_path) if __name__ == '__main__': # 原始文件夹,该文件夹包含utf-8格式和ansi格式等各种文本格式, # 需要更改文件夹名称 txt_dir_path = './file' reformat_txt(txt_dir_path) # 中间文件夹,不需要改动(需要自行新建该文件夹，要不然会报错) txt_dir_path = './result' all_information = read_txt(txt_dir_path) # 目标csv的标签 tag = '涉恐事件2.0' csv_path = './' + tag + '.csv' write_csv(all_information, tag, csv_path)
如果有问题可以在评论区私信我，如果有帮助请帮忙点个赞👍
