中文csv文本编码转utf8那些事 - python实现

简介: 中文csv文本编码转utf8那些事 - python实现

中文csv文本编码转utf8那些事 - python

请参考以下代码:

# -*- coding: utf-8 -*-  
##!/usr/bin/python3
# @Author : Jack Lee
# @Email : 291148484@163.com
import os
import time
import codecs
import chardet
class CodeError(ValueError):pass
def get_time() -> str:
    return str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
def which_codes(filepath):
    # c = codecs.open(filepath,"r")
    # print(c)
    # return c.encoding
    with open(filepath,'rb') as f:
        content = f.read()
    return chardet.detect(content).get('encoding').lower()
def open_gbk_as_str(filepath)-> str:
    with open(filepath,'r',encoding='gbk') as f:
        content = f.read()
    return content
def open_big5_as_str(filepath)-> str:
    with open(filepath,'r',encoding='big5') as f:
        content = f.read()
    return content
def open_big5hkscs_as_str(filepath)-> str:
    with open(filepath,'r',encoding='big5hkscs') as f:
        content = f.read()
    return content
def open_cp950_as_str(filepath)-> str:
    with open(filepath,'r',encoding='cp950') as f:
        content = f.read()
    return content
def open_gb2312_as_str(filepath)-> str:
    with open(filepath,'r',encoding='gb2312') as f:
        content = f.read()
    return content
def open_hz_as_str(filepath)-> str:
    with open(filepath,'r',encoding='hz') as f:
        content = f.read()
    return content
def open_ascii_as_str(filepath)-> str:
    with open(filepath,'r',encoding='ascii') as f:
        content = f.read()
    return content
def open_utf8_as_str(filepath)-> str:
    with open(filepath,'r',encoding='utf-8') as f:
        content = f.read()
    return content
def open_utf16_as_str(filepath)-> str:
    with open(filepath,'rb') as f:
        content = f.read()
    return str(content,encoding='utf-16')
def open_utf32_as_str(filepath)-> str:
    with open(filepath,'rb') as f:
        content = f.read()
    return str(content,encoding='utf-32')
def open_as_str(filepath):
    codes = which_codes(filepath)
    # gbk encode
    if codes == "936" or codes == "cp936" or codes == "ms936" or codes == "gbk":
        print("[INFO] "+get_time()+" GBK, codes =",codes)
        return open_gbk_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # big5 encode
    if codes == "big5" or codes == "big5-tw" or codes == "csbig5":
        print("[INFO] "+get_time()+" big5, codes =",codes)
        return open_big5_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # big5hkscs encode
    if codes == "big5hkscs" or codes == "big5-hkscs" or codes == "hkscs":
        print("[INFO] "+get_time()+" big5hkscs, codes =",codes)
        return open_big5hkscs_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # cp950 encode
    if codes == "cp950" or codes == "950" or codes == "ms950":
        print("[INFO] "+get_time()+" cp950, codes =",codes)
        return open_cp950_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # gb2312 encode
    if codes == "gb2312" or codes == "chinese" or codes == "csiso58gb231280" or codes == "euc-cn" or codes == "euccn" or codes == "eucgb2312-cn" or codes == "gb2312-1980" or codes == "gb2312-80" or codes == "iso-ir-58":
        print("[INFO] "+get_time()+" gb2312, codes =",codes)
        return open_gb2312_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # hz encode
    if codes == "hz" or codes == "hzgb" or codes == "hz-gb" or codes == "hz-gb-2312":
        print("[INFO] "+get_time()+" hz, codes =",codes)
        return open_cp950_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # ascii encode
    elif codes == 'ascii':
        print("[INFO] "+get_time()+" encoding = ascii")
        return open_ascii_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # utf-16 encode
    elif codes == 'utf-16' or codes == 'U16' or codes == 'utf16' or codes == 'utf_16':
        print("[INFO] "+get_time()+" utf-16, codes =",codes)
        return open_utf16_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # utf-32 encode
    elif codes == 'utf-32' or codes == 'U32' or codes == 'utf32' or codes == 'utf_32':
        print("[INFO] "+get_time()+" utf-32, codes =",codes)
        return open_utf32_as_str(filepath).encode(encoding='utf-8').decode(encoding='utf-8')
    # utf-8 encode
    elif codes == "utf-8" or codes == "U8" or codes == "cp65001" or codes == "utf8" or codes == "UTF":
        print("[INFO] "+get_time()+" utf-8, codes =",codes)
        return open_utf8_as_str(filepath)
    # other unrecognized codes:
    else:
        print('[CRITICAL] '+get_time()+' The current encoding used is:"'+codes+'", the program failed to process this encoding.')
        raise CodeError('Text file:"'+filepath+'" which encoding method cannot be read.')
# Used to replace the specified csv with the corresponding utf8 encoded csv.
def replace_by_utf8_csv(filepath):
    s = ""
    l = open_as_str(filepath).split('\n\r')
    for i in l:
        if i!="":
            s=s+i
    with open(filepath, 'w', encoding='utf-8') as f:
        f.write(s)
    print('[INFO] '+get_time()+' Translated file '+ filepath +' into utf-8.')
# print(open_as_str(r'C:\Users\a2911\Desktop\script\sources\aaa.csv'))

说明:

  1. 调用 open_as_str 函数用于打开一个文本文件,得到相应的 utf-8 字符串;
  2. 调用 replace_by_utf8_csv 用于,将一个非 utf-8 的 csv 替换为同名 utf-8 的 csv。
目录
相关文章
|
1月前
|
存储 Python
用python将csv转excel (.xls和.xlsx)的几种方式
用python将csv转excel (.xls和.xlsx)的几种方式
100 4
|
30天前
|
存储 移动开发 算法
Python怎么将图片转换成base64编码
Python怎么将图片转换成base64编码
24 0
|
29天前
|
存储 数据可视化 数据处理
利用python将Excel文件拆分为多个CSV
利用python将Excel文件拆分为多个CSV
26 0
|
29天前
|
存储 数据处理 Python
Python中读写CSV文件的深入探讨
Python中读写CSV文件的深入探讨
17 0
|
30天前
|
安全 UED Python
Python中使用Tkinter和Difflib模块实现文本比对功能
Python中使用Tkinter和Difflib模块实现文本比对功能
18 0
|
1月前
|
存储 IDE Shell
Python生成CSV文件
Python生成CSV文件
24 0
|
2月前
|
Python
20个改善编码的Python异常处理技巧,让你的代码更高效
异常处理是写好代码的一个重要的方面,虽然许多开发人员都熟悉基本的try-except块,但是有很多更深入的知识可以使异常处理更高效、更可读和更python化。所以本文将介绍关于Python异常的20个可以显著改善编码的Python异常处理技巧,这些技巧可以让你熟练的掌握Python的异常处理。
215 4
|
2月前
|
Python
血常规(自动生成报告)(配套Python工具正在编码中)
血常规(自动生成报告)(配套Python工具正在编码中)
46 0
|
2月前
|
Python
在Python中,序数编码(Ordinal Encoding)
在Python中,序数编码(Ordinal Encoding)
37 6
|
2月前
|
机器学习/深度学习 算法 Python
在Python中,独热编码(One-Hot Encoding)
在Python中,独热编码(One-Hot Encoding)
63 8