胃肠道癌症图像分割数据分析

!pip install wandb >log.log

WARNING: You are using pip version 22.0.4; however, version 22.1.2 is available.
You should consider upgrading via the '/opt/conda/envs/python35-paddle120-env/bin/python -m pip install --upgrade pip' command.


from pathlib import Path
import random
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from glob import glob
from PIL import Image
from IPython.display import IFrame
import imageio
import os

!unzip -qoa data/data149096/uw-madison-gi-tract-image-segmentation.zip -d data

root_dir = Path('./data/')
df_train = pd.read_csv(root_dir / 'train.csv')
df_train.head()
.dataframe tbody tr th:only-of-type {         vertical-align: middle;     } .dataframe tbody tr th {     vertical-align: top; } .dataframe thead th {     text-align: right; }

dic = {}
for data in df_train.iterrows():
    p_key = data[1]['id']
    if not dic.get(p_key):
        dic[p_key] = {}
        dic[p_key]['large_bowel_segmentation'] = None
        dic[p_key]['small_bowel_segmentation'] = None
        dic[p_key]['stomach_segmentation'] = None
    if data[1]['class'] == 'large_bowel':
        key = 'large_bowel_segmentation'
    elif data[1]['class'] == 'small_bowel':
        key = 'small_bowel_segmentation'
    elif data[1]['class'] == 'stomach':
        key = 'stomach_segmentation'
    value = data[1]['segmentation']
    dic[p_key][key] = value
df_segmentation = pd.DataFrame.from_dict(dic, orient='index')
df_segmentation.head()
.dataframe tbody tr th:only-of-type {         vertical-align: middle;     } .dataframe tbody tr th {     vertical-align: top; } .dataframe thead th {     text-align: right; }

# 生成图像文件列表
images_list = glob('./data/train/*/*/scans/*.png')
# path转dataframe
df_images = pd.DataFrame({'Path':images_list})
# 从文件路径分割
path_split = df_images['Path'].str.split('/',n=6,expand=True)
# 分割 [4] and [6]数据
df_images['CaseNum_Day'] = path_split[4]
df_images['SliceNum'] = path_split[6]
# 获取 day, slice, height and width
case_split = df_images['CaseNum_Day'].str.split('_',n=2, expand=True)
df_images['Case'] = case_split[0].str[4:].astype(int)
df_images['Day'] = case_split[1].str[3:].astype(int)
# 获取 slice, height and width
fileName_split = df_images['SliceNum'].str.split('_',n=6, expand=True)
df_images['Slice'] = fileName_split[1].astype(int)
df_images['Height'] = fileName_split[2].astype(int)
df_images['Width'] = fileName_split[3].astype(int)
#'id' in segmentation
df_images['id'] = df_images['CaseNum_Day'] + '_slice_' + fileName_split[1]
df_images.head()
.dataframe tbody tr th:only-of-type {         vertical-align: middle;     } .dataframe tbody tr th {     vertical-align: top; } .dataframe thead th {     text-align: right; }

# 数据信息统计
print('Unique case numbers ',len(df_images['Case'].unique()))
print('Unique Days ',len(df_images['Day'].unique()))
print('Unique Heights ',df_images['Height'].unique())
print('Unique Widths ',df_images['Width'].unique())

Unique case numbers  85
Unique Days  35
Unique Heights  [360 276 266 234]
Unique Widths  [310 276 266 234]

# Unique CaseNum_Day
unique_case_day = df_images['CaseNum_Day'].unique()
print(len(unique_case_day))

274

# 获取像素位置的函数
def get_pixel_loc(random_segmentation, img_shape):
    loc = []
    for rle_string in random_segmentation:
        p_loc = []
        if isinstance(rle_string, str): # Handle NaN!
            rle = [int(i) for i in rle_string.split(' ')]
            pairs = list(zip(rle[0::2],rle[1::2]))    
            for start, length in pairs:
                for p_pos in range(start, start + length):
                    p_loc.append([p_pos // img_shape[1], p_pos % img_shape[1], len(loc)])
        loc.append(p_loc)
    return loc
# 获取 mask
def get_mask(rle_group, img_shape):
    masks = np.zeros(img_shape)
    masks = np.repeat(masks[:, :, np.newaxis], 3, axis=2)
    for rle_single in rle_group:
        for loc in rle_single:
            masks[tuple(loc)] = 1
    return masks
# 应用 mask
def apply_mask(image, loc_segm, img_shape):
    image = image / image.max()
    masks = get_mask(loc_segm, img_shape)    
    image = np.dstack((image + masks[..., 0], 
                       image + masks[..., 1],
                       image + masks[..., 2]))
    image = image / image.max()
    return image

# 从数据集打印随机mask
# mask数据获取
mask_encoding = df_segmentation.loc[(df_segmentation['large_bowel_segmentation'].notnull()) |
                                    (df_segmentation['small_bowel_segmentation'].notnull()) |
                                    (df_segmentation['stomach_segmentation'].notnull())]
# 列表id转换
mask_id = list(mask_encoding.index)
for _ in range(5):   
    random_id = mask_id[np.random.randint(0,len(mask_id) - 1)]
    random_segmentation = mask_encoding.loc[random_id]
    splits = random_id.split('_')
    x = df_images[(df_images['Case']==int(splits[0][4:]))
                       &(df_images['Day']==int(splits[1][3:]))
                       &(df_images['Slice']==int(splits[3]))]
    image = np.array(Image.open(x['Path'].values[0]))
    k = image.shape
    p_loc = get_pixel_loc(random_segmentation, k)
    fig, ax = plt.subplots(1,3, figsize=(12,16))
    ax[0].set_title('Image')
    ax[0].imshow(image, cmap='gray')
    ax[1].set_title('Mask')
    ax[1].imshow(get_mask(p_loc, k))
    ax[2].imshow(apply_mask(image, p_loc, k))
plt.show()

def save_gif(random_case_day):
    gif_name = str(random_case_day + '.gif')
    filenames = []
    df_selected_images = df_images[df_images['CaseNum_Day'] == random_case_day].sort_values('Slice').reset_index(drop=True)
    for i, row in df_selected_images.iterrows():
        image = np.array(Image.open(row['Path']))
        k = image.shape    
        segmentation = df_segmentation.loc[row['id']]
        pixel_loc = get_pixel_loc(segmentation, k)
        masked_image = apply_mask(image, pixel_loc, k)
        filename = f'{i}.png'
        filenames.append(filename)
        plt.imsave(filename, masked_image)
    with imageio.get_writer(gif_name, mode='I') as writer:
        for filename in filenames:
            image = imageio.imread(filename)
            writer.append_data(image)
    for filename in set(filenames):
        os.remove(filename)
    shape = (df_selected_images.iloc[0]['Width'], 
             df_selected_images.iloc[0]['Height'])
    return gif_name, shape

random_case_day = unique_case_day[np.random.randint(0,len(unique_case_day) - 1)]
out = save_gif(random_case_day)
IFrame(out[0], 
       out[1][0], 
       out[1][1])