【大模型】智能驾驶汽车虚拟仿真视频数据理解-学习打卡

!pip install paddleclip

import paddle
from PIL import Image
from clip import tokenize, load_model
import glob, json, os
import cv2
from PIL import Image
from tqdm import tqdm_notebook
import numpy as np
from sklearn.preprocessing import normalize
import matplotlib.pyplot as plt
# 加载预训练的 ViT_B_32 模型
# load_model: 是一个自定义函数，用于加载模型和数据转换器（transforms）
# pretrained=True: 表示加载预训练的权重
# 返回值包括：
# model: ViT_B_32 模型对象
# transforms: 数据转换器对象，用于将输入数据转换为适合模型处理的形式
model, transforms = load_model('ViT_B_32', pretrained=True)

# 设置图片路径和标签
img_path = "kobe.jpeg"
labels = ['kobe', 'james', 'Jordan']
# 准备输入数据
img = Image.open(img_path)
display(img)
image = transforms(Image.open(img_path)).unsqueeze(0)
text = tokenize(labels)
# 计算特征
with paddle.no_grad():
    logits_per_image, logits_per_text = model(image, text)
    probs = paddle.nn.functional.softmax(logits_per_image, axis=-1)
# 打印结果
for label, prob in zip(labels, probs.squeeze()):
    print('该图片为 %s 的概率是：%.02f%%' % (label, prob*100.))

该图片为 kobe 的概率是：98.68%
该图片为 james 的概率是：0.57%
该图片为 Jordan 的概率是：0.75%

cn_match_words = {
    "工况描述": ["高速/城市快速路", "城区", "郊区", "隧道", "停车场", "加油站/充电站", "未知"],
    "天气": ["晴天", "雨天", "多云", "雾天", "下雪", "未知"],
    "时间": ["白天", "夜晚", "拂晓/日暮", "未知"],
    "道路结构": ["十字路口", "丁字路口", "上下匝道", "车道汇入", "进出停车场", "环岛", "正常车道", "未知"],
    "一般障碍物": ["雉桶", "水马", "碎石/石块", "井盖", "减速带", "没有"],
    "道路异常情况": ["油污/水渍", "积水", "龟裂", "起伏不平", "没有", "未知"],
    "自车行为": ["直行", "左转", "右转", "停止", "掉头", "加速", "减速", "变道", "其它"],
    "最近的交通参与者": ["行人", "小型汽车", "卡车", "交警", "没有", "未知", "其它"],
    "最近的交通参与者行为": ["直行", "左转", "右转", "停止", "掉头", "加速", "减速", "变道", "其它"],
}
en_match_words = {
"scerario" : ["suburbs","city street","expressway","tunnel","parking-lot","gas or charging stations","unknown"],
"weather" : ["clear","cloudy","raining","foggy","snowy","unknown"],
"period" : ["daytime","dawn or dusk","night","unknown"],
"road_structure" : ["normal","crossroads","T-junction","ramp","lane merging","parking lot entrance","round about","unknown"],
"general_obstacle" : ["nothing","speed bumper","traffic cone","water horse","stone","manhole cover","nothing","unknown"],
"abnormal_condition" : ["uneven","oil or water stain","standing water","cracked","nothing","unknown"],
"ego_car_behavior" : ["slow down","go straight","turn right","turn left","stop","U-turn","speed up","lane change","others"],
"closest_participants_type" : ["passenger car","bus","truck","pedestrain","policeman","nothing","others","unknown"],
"closest_participants_behavior" : ["slow down","go straight","turn right","turn left","stop","U-turn","speed up","lane change","others"],
}

cap = cv2.VideoCapture('./初赛测试视频/41.avi')
img = cap.read()[1]
image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
image = Image.fromarray(image)
image.resize((600, 300))

# 定义一个JSON对象，用于提交结果。
submit_json = {
    "author" : "abc" ,
    "time" : "231011",
    "model" : "model_name",
    "test_results" : []
}
# 使用glob模块获取当前目录下'初赛测试视频'文件夹中所有的文件路径，并按照字典序进行排序 
paths = glob.glob('./初赛测试视频/*')
paths.sort()
# 遍历获取到的所有文件路径  
for video_path in paths:  
    print(video_path)  # 打印当前处理的文件路径  
    # 通过文件路径获取视频的剪辑ID（clip_id），这里假设每个文件名都包含剪辑ID  
    clip_id = video_path.split('/')[-1]  
    # 使用OpenCV打开视频文件，并尝试读取第一帧（帧率可能不同，所以使用[1]获取第一帧）  
    cap = cv2.VideoCapture(video_path)  
    img = cap.read()[1]   
    # 将图像从BGR格式转换为RGB格式，然后将其转换为PIL图像对象，以便后续处理  
    image = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  
    image = Image.fromarray(image)  
    image = transforms(image).unsqueeze(0)  # transforms可能是一个函数，用于对图像进行预处理，例如调整大小、裁剪等操作，然后使用unsqueeze(0)增加一个维度以适应模型需要  
    # 定义一个字典，用于存储当前视频的一些基本信息，这些信息可能会被用于后续的文本描述  
    single_video_result = {  
        "clip_id": clip_id,  
        "scerario" : "cityroad",  # 这可能是一个拼写错误，应该是 "scenario"  
        "weather":"unknown",  # 天气条件  
        "period":"night",  # 时间段（白天、晚上等）  
        "road_structure":"ramp",  # 道路结构（直路、弯路等）  
        "general_obstacle":"nothing",  # 一般的障碍物（如车辆、行人等）  
        "abnormal_condition":"nothing",  # 异常情况（如事故、交通拥堵等）  
        "ego_car_behavior":"turning right",  # 自车行为（如直行、转弯等）  
        "closest_participants_type":"passenger car",  # 最接近的参与者类型（如行人、其他车辆等）  
        "closest_participants_behavior":"braking"  # 最接近的参与者行为（如刹车、加速等）  
    }  
    # 对于英文匹配词汇字典中的每一个关键字（除了"weather"和"road_structure"），执行以下操作：  
    for keyword in en_match_words.keys():  
        if keyword not in ["weather", "road_structure"]:  # 如果关键字不是"weather"或"road_structure"则继续执行下面的代码块  
            continue  # 跳过当前循环，进入下一次循环，下一个关键字的处理  
        texts = np.array(en_match_words[keyword])  # 获取该关键字对应的文本列表（假设en_match_words是一个字典，其中键是关键字，值是对应的文本列表）  
        with paddle.no_grad():  # 在这个代码块中关闭梯度计算，以加速计算过程并节省内存资源  
            logits_per_image, logits_per_text = model(image, tokenize(en_match_words[keyword]))  # 使用模型对图片和文本进行预测，得到每个文本的预测logits值  
            probs = paddle.nn.functional.softmax(logits_per_image, axis=-1)  # 对预测的logits值进行softmax处理，得到每个文本的概率值  
        probs = probs.numpy()  # 将概率值转换为numpy数组格式，以便后续处理  
        single_video_result[keyword] = texts[probs[0].argsort()[::-1][0]]  # 根据概率值排序文本列表，并取出第一个文本作为最终结果，存储到single

with open('clip_result.json', 'w', encoding='utf-8') as up:
    json.dump(submit_json, up, ensure_ascii=False)

if keyword not in ["weather", "road_structure"]:

if keyword not in ["weather", "road_structure", "scerario", "road_structure"]:

if keyword not in ["weather", "road_structure", "scerario", "period"]:

if keyword not in ["weather", "road_structure", "scerario", "period", "closest_participants_behavior"]: