【机器学习】GLM4-9B-Chat大模型/GLM-4V-9B多模态大模型概述、原理及推理实战

class GLMBlock(torch.nn.Module):
    """A single transformer layer.
    Transformer layer takes input with size [s, b, h] and returns an
    output of the same size.
    """
 
    def __init__(self, config: ChatGLMConfig, layer_number, device=None):
        super(GLMBlock, self).__init__()
        self.layer_number = layer_number
 
        self.apply_residual_connection_post_layernorm = config.apply_residual_connection_post_layernorm
 
        self.fp32_residual_connection = config.fp32_residual_connection
 
        LayerNormFunc = RMSNorm if config.rmsnorm else LayerNorm
        # Layernorm on the input data.
        self.input_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device,
                                             dtype=config.torch_dtype)
 
        # Self attention.
        self.self_attention = SelfAttention(config, layer_number, device=device)
        self.hidden_dropout = config.hidden_dropout
 
        # Layernorm on the attention output
        self.post_attention_layernorm = LayerNormFunc(config.hidden_size, eps=config.layernorm_epsilon, device=device,
                                                      dtype=config.torch_dtype)
 
        # MLP
        self.mlp = MLP(config, device=device)
 
    def forward(
            self, hidden_states, attention_mask, rotary_pos_emb, kv_cache=None, use_cache=True,
    ):
        # hidden_states: [s, b, h]
 
        # Layer norm at the beginning of the transformer layer.
        layernorm_output = self.input_layernorm(hidden_states)
        # Self attention.
        attention_output, kv_cache = self.self_attention(
            layernorm_output,
            attention_mask,
            rotary_pos_emb,
            kv_cache=kv_cache,
            use_cache=use_cache
        )
 
        # Residual connection.
        if self.apply_residual_connection_post_layernorm:
            residual = layernorm_output
        else:
            residual = hidden_states
 
        layernorm_input = torch.nn.functional.dropout(attention_output, p=self.hidden_dropout, training=self.training)
        layernorm_input = residual + layernorm_input
 
        # Layer norm post the self attention.
        layernorm_output = self.post_attention_layernorm(layernorm_input)
 
        # MLP.
        mlp_output = self.mlp(layernorm_output)
 
        # Second residual connection.
        if self.apply_residual_connection_post_layernorm:
            residual = layernorm_output
        else:
            residual = layernorm_input
 
        output = torch.nn.functional.dropout(mlp_output, p=self.hidden_dropout, training=self.training)
        output = residual + output
 
        return output, kv_cache

pip install torch==2.3.0 -i https://mirrors.cloud.tencent.com/pypi/simple

from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM
model_dir = snapshot_download('ZhipuAI/glm-4-9b-chat')
import torch
 
device = "cuda:2" # the device to load the model onto
 
tokenizer = AutoTokenizer.from_pretrained(model_dir,trust_remote_code=True)
 
prompt = "介绍一下大语言模型"
messages = [
    {"role": "system", "content": "你是一个智能助理."},
    {"role": "user", "content": prompt}
]
text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True
)
model_inputs = tokenizer([text], return_tensors="pt").to(device)
 
model = AutoModelForCausalLM.from_pretrained(
    model_dir,
    device_map="cuda:2",
    trust_remote_code=True
)
 
gen_kwargs = {"max_length": 512, "do_sample": True, "top_k": 1}
with torch.no_grad():
    outputs = model.generate(**model_inputs, **gen_kwargs)
    outputs = outputs[:, model_inputs['input_ids'].shape[1]:]
    print(tokenizer.decode(outputs[0], skip_special_tokens=True))
 
"""
generated_ids = model.generate(
    model_inputs.input_ids,
    max_new_tokens=512
)
generated_ids = [
    output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
]
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(response)
"""

from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM
model_dir = snapshot_download('ZhipuAI/glm-4-9b-chat')
 
 
#from modelscope import AutoModelForCausalLM, AutoTokenizer
#from modelscope import GenerationConfig
 
tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="cuda:2", trust_remote_code=True, torch_dtype=torch.bfloat16).eval()
#model.generation_config = GenerationConfig.from_pretrained("ZhipuAI/glm-4-9b-chat", trust_remote_code=True) # 可指定不同的生成长度、top_p等相关超参
 
response, history = model.chat(tokenizer, "你好", history=None)
print(response)
response, history = model.chat(tokenizer, "浙江的省会在哪里？", history=history) 
print(response)
response, history = model.chat(tokenizer, "它有什么好玩的景点", history=history)
print(response)

from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM
model_dir = snapshot_download('ZhipuAI/glm-4v-9b')
#model_dir = snapshot_download('qwen/Qwen-VL-Chat') #本来想一套代码支持qwen-vl和glm-4v,发现两个模型的chat函数传参不同,需要特殊处理适配
import torch
from PIL import Image
 
device = "auto" # the device to load the model onto
 
tokenizer = AutoTokenizer.from_pretrained(model_dir,trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_dir, device_map=device, trust_remote_code=True,torch_dtype=torch.float16).eval()
image = Image.open("./demo.jpeg").convert("RGB")
response, history = model.chat(tokenizer,image=image, query="这是什么?", history=None)
print(response)
response, history = model.chat(tokenizer, query='再说一遍都有什么', history=history)
print(response)

from modelscope import snapshot_download
from transformers import AutoTokenizer, AutoModelForCausalLM
#model_dir = snapshot_download('ZhipuAI/glm-4v-9b')
model_dir = snapshot_download('qwen/Qwen-VL-Chat')
import torch
from PIL import Image
 
device = "auto" # the device to load the model onto
 
tokenizer = AutoTokenizer.from_pretrained(model_dir,trust_remote_code=True)
model = AutoModelForCausalLM.from_pretrained(model_dir, device_map=device, trust_remote_code=True,fp16=True).eval()
 
query = tokenizer.from_list_format([
    {'image': './test.png'}, # Either a local path or an url
    {'text': '这是什么?'},
])
 
response, history = model.chat(tokenizer, query=query,history=None)
print(response)
 
response, history = model.chat(tokenizer, query='框出图中的小狗', history=history)
print(response)
#<ref>小狗</ref><box>(366,115),(840,995)</box>
image = tokenizer.draw_bbox_on_latest_picture(response, history)
if image:
  image.save('1.jpg')
else:
  print("no box")