Python代码如下
import pandas as pd
# 读取数据
data = pd.read_csv('data_row.csv')
# 检查异常值
def detect_outliers(data):
outliers = []
for col in data.columns:
q1 = data[col].quantile(0.25)
q3 = data[col].quantile(0.75)
iqr = q3 - q1
lower_bound = q1 - 1.5 * iqr
upper_bound = q3 + 1.5 * iqr
outliers.extend(data[(data[col] < lower_bound) | (data[col] > upper_bound)].index)
return list(set(outliers))
outliers = detect_outliers(data)
print("异常数据数量：", len(outliers))
# 处理异常值
data.drop(outliers, inplace=True)
# 保存清洗后的数据
data.to_csv('clean_data_row.csv', index=False)
下面我们修改成C#代码
创建控制台程序，Nuget安装 CsvHelper 和 pythonnet
public class Program
{
const string PathToPythonDir = "D:\\Python311";
const string DllOfPython = "python311.dll";
static void Main(string[] args)
{
// 数据清洗
CleanData();
}
/// <summary>
/// 数据清洗
/// </summary>
static void CleanData()
{
var originDatas = ReadCsvWithCsvHelper("data_row.csv");
var outliers = DetectOutliers(originDatas);
var outlierHashset = new HashSet<int>(outliers);
// 清洗过后的数据
var cleanDatas = originDatas.Where((r, index) => !outlierHashset.Contains(index)).ToList();
try
{
Runtime.PythonDLL = Path.Combine(PathToPythonDir, DllOfPython);
PythonEngine.Initialize();
using (Py.GIL())
{
dynamic pd = Py.Import("pandas");
dynamic np = Py.Import("numpy");
dynamic plt = Py.Import("matplotlib.pyplot");
dynamic fft = Py.Import("scipy.fftpack");
dynamic oData = np.array(originDatas.ToArray());
int oDataLength = oData.__len__();
dynamic data = np.array(cleanDatas.ToArray());
int dataLength = data.__len__();
// 绘制原始数据图和清洗后数据图
plt.figure(figsize: new dynamic[] { 12, 6 });
// 原始数据图
plt.subplot(1, 2, 1);
plt.plot(np.arange(oDataLength), oData);
plt.title("Original Datas");
// 清洗后数据图
plt.subplot(1, 2, 2);
plt.plot(np.arange(dataLength), data);
plt.title("Clean Datas");
// 布局调整，防止重叠
plt.tight_layout();
// 显示图表
plt.show();
}
}
catch (Exception e)
{
Console.WriteLine("报错了：" + e.Message + "\r\n" + e.StackTrace);
}
}
/// <summary>
/// 检测异快讯常值
/// </summary>
/// <param name="datas">原始数据集合</param>
/// <returns>返回异常值在集合中的索引</returns>
static List<int> DetectOutliers(List<double[]> datas)
{
List<int> outliers = new List<int>();
var first = datas.First();
for (int i = 0; i < first.Length; i++)
{
var values = datas.AsEnumerable().Select((row, index) => Tuple.Create(row[i], index)).ToArray();
double q1 = Enumerable.OrderBy(values, x => x.Item1).ElementAt((int)(values.Length * 0.25)).Item1;
double q3 = Enumerable.OrderBy(values, x => x.Item1).ElementAt((int)(values.Length * 0.75)).Item1;
double iqr = q3 - q1;
double lowerBound = q1 - 1.5 * iqr;
double upperBound = q3 + 1.5 * iqr;
outliers.AddRange(values.AsEnumerable()
.Where(row => row.Item1 < lowerBound || row.Item1 > upperBound)
.Select(row => row.Item2));
}
return outliers.Distinct().ToList();
}
/// <summary>
/// 读取CSV数据
/// </summary>
/// <param name="filePath">文件路径</param>
/// <returns>文件中数据集合，都是double类型</returns>
static List<double[]> ReadCsvWithCsvHelper(string filePath)
{
using (var reader = new StreamReader(filePath))
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
var result = new List<double[]>();
// 如果你的CSV文件有标题行，可以调用ReadHeader来读取它们
csv.Read();
csv.ReadHeader();
while (csv.Read())
{
result.Add(new double[] {
csv.GetField<double>(0),
csv.GetField<double>(1),
csv.GetField<double>(2),
});
}
return result;
}
}
}
以下是运行后结果，左边是原始数据折线图，右边是清洗后数据折线图
