《Pandas 简易速速上手小册》第8章：Pandas 高级数据分析技巧（2024 最新版）

import pandas as pd
# 示例客户数据
customer_data = {
    'name': ['Alice Smith', 'bob Johnson', 'CHARLIE BROWN'],
    'age': ['25', 'thirty-five', '40'],
    'email': ['alice@example.com', 'BOB@example.com', 'charlie@example.net']
}
customer_df = pd.DataFrame(customer_data)

# 标准化姓名格式：首字母大写，其他字母小写
customer_df['name'] = customer_df['name'].apply(lambda x: x.title())
# 转换年龄为数值类型：将非数字的年龄转换为 NaN，然后填充平均年龄
customer_df['age'] = pd.to_numeric(customer_df['age'], errors='coerce').fillna(customer_df['age'].dropna().astype(int).mean())
# 邮箱地址转小写
customer_df['email'] = customer_df['email'].map(lambda x: x.lower())

# 示例产品评分数据
product_scores = {
    'product_id': [1, 2, 3],
    'score': [4, 3.5, 5]
}
scores_df = pd.DataFrame(product_scores)

# 对评分加权调整：评分低于5的乘以1.1，但不超过5
scores_df['adjusted_score'] = scores_df['score'].apply(lambda x: min(x * 1.1, 5))

# 示例客户地址数据
address_data = {
    'customer_id': [1, 2, 3],
    'address': ['123 Elm St.', ' 456 Maple Ave', '789 Oak Dr, ']
}
address_df = pd.DataFrame(address_data)

# 去除地址中的标点和多余空格
import re
address_df['address'] = address_df['address'].apply(lambda x: re.sub(r'[^\w\s]', '', x.strip()))

import pandas as pd
import numpy as np
# 生成大型销售数据示例
np.random.seed(0)
sales_data = {
    'product_id': np.random.randint(1, 100, 1000000),
    'sales_amount': np.random.rand(1000000) * 100
}
sales_df = pd.DataFrame(sales_data)

# 向量化计算总销售额
total_sales = sales_df.groupby('product_id')['sales_amount'].sum()

# 生成大型客户数据示例
customer_data = {
    'customer_id': np.arange(1, 1000001),
    'age': np.random.randint(18, 80, 1000000),
    'email_count': np.random.randint(1, 10, 1000000)
}
customer_df = pd.DataFrame(customer_data)

# 转换数据类型
customer_df['customer_id'] = customer_df['customer_id'].astype('int32')
customer_df['age'] = customer_df['age'].astype('int8')
customer_df['email_count'] = customer_df['email_count'].astype('int8')

# 生成大型文本数据示例
text_data = ['This is a SAMPLE text.' * 10 for _ in range(100000)]
text_df = pd.DataFrame(text_data, columns=['text'])

import pandas as pd
import numpy as np
# 生成示例销售数据并保存到 CSV 文件
chunk_size = 10000
num_chunks = 500
for i in range(num_chunks):
    df = pd.DataFrame({
        'SaleID': range(i * chunk_size, (i + 1) * chunk_size),
        'ProductID': np.random.randint(1, 100, chunk_size),
        'SaleAmount': np.random.rand(chunk_size) * 100
    })
    df.to_csv('/mnt/data/sales_large.csv', mode='a', index=False, header=(i == 0))

chunk_iter = pd.read_csv('/mnt/data/sales_large.csv', chunksize=chunk_size)
total_sales = 0
for chunk in chunk_iter:
    total_sales += chunk['SaleAmount'].sum()
print(f"Total sales amount: {total_sales}")

# 假设 df 是一个大型 DataFrame
df.to_parquet('/mnt/data/sales_large.parquet')
# 读取 Parquet 文件
df_parquet = pd.read_parquet('/mnt/data/sales_large.parquet')

# 注意：此代码示例需要在支持 Dask 的环境中运行
from dask import dataframe as dd
# 读取数据
dask_df = dd.read_csv('/mnt/data/sales_large.csv')
# 使用 Dask 进行计算
total_sales_dask = dask_df['SaleAmount'].sum().compute()
print(f"Total sales amount with Dask: {total_sales_dask}")