通过Ti-One机器学习平台玩转2020腾讯广告算法大赛：数据预处理（二）

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024 ** 2
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
    end_mem = df.memory_usage().sum() / 1024 ** 2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (
                start_mem - end_mem) / start_mem))
    return df

train_dir = "train_preliminary/"
test_dir = "test/"
click_train = pd.read_csv(train_dir + "click_log.csv")
click_train = reduce_mem_usage(click_train)
ad_train = pd.read_csv(train_dir + "ad.csv")
ad_train = reduce_mem_usage(ad_train)
click_log = click_train.merge(ad_train, how="left", on="creative_id", )
click_log["type"] = "train"
click_test = pd.read_csv(test_dir + "click_log.csv")
ad_test = pd.read_csv(test_dir + "ad.csv")
click_test = reduce_mem_usage(click_test)
ad_test = reduce_mem_usage(ad_test)
click_log_test = click_test.merge(ad_test, how="left", on="creative_id", )
click_log_test['type'] = "test"

click_all = click_log.append(click_log_test)

click_all.to_pickle("all-raw.pkl")