魔搭社区LLM模型部署实践 —— 以ChatGLM3为例

pip install xinference[ggml]>=0.4.3

xinference -p 9997

from xinference.client import Client
client = Client("http://localhost:9997")
model_uid = client.launch_model(
    model_name="chatglm3",
    model_format="ggmlv3", 
    model_size_in_billions=6,
    quantization="q4_0",
    )
model = client.get_model(model_uid)
chat_history = []
prompt = "最大的动物是什么？"
model.chat(
    prompt,
    chat_history,
    generate_config={"max_tokens": 1024}
)

git clone --recursive https://github.com/li-plus/chatglm.cpp.git && cd chatglm.cpp
python3 -m pip install torch tabulate tqdm transformers accelerate sentencepiece
cmake -B build
cmake --build build -j --config Release

# 先将文件夹cd到swift根目录中
python tools/merge_lora_weights_to_model.py --model_id_or_path /dir/to/your/base/model --model_revision master --ckpt_dir /dir/to/your/lora/model

# 先将文件夹cd到chatglm.cpp根目录中
python3 chatglm_cpp/convert.py -i {ckpt_dir}-merged -t q4_0 -o chatglm-ggml.bin

./build/bin/main -m chatglm-ggml.bin -i
# 以下对话为使用agent数据集训练后的效果
# Prompt   > how are you?
# ChatGLM3 > <|startofthink|>```JSON
# {"api_name": "greeting", "apimongo_instance": "ddb1e34-0406-42a3-a547a220a2", "parameters": {"text": "how are # you?"}}}
# ```<|endofthink|>
#
# I'm an AI assistant and I can only respond to text input. I don't have the ability to respond to audio or # video input.

xinference -p 9997

from xinference.client import Client
client = Client("http://localhost:9997")
model_uid = client.launch_model(model_name="custom-chatglm")
model = client.get_model(model_uid)
chat_history = []
prompt = "What is the largest animal?"
model.chat(
    prompt,
    chat_history,
    generate_config={"max_tokens": 1024}
)
# {'id': 'chatcmpl-df3c2c28-f8bc-4e79-9c99-2ae3950fd459', 'object': 'chat.completion', 'created': 1699367362, 'model': '021c2b74-7d7a-11ee-b1aa-ead073d837c1', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': "According to records kept by the Guinness World Records, the largest animal in the world is the Blue Whale, specifically, the Right and Left Whales, which were both caught off the coast of Newfoundland. The two whales measured a length of 105.63 meters, or approximately 346 feet long, and had a corresponding body weight of 203,980 pounds, or approximately 101 tons. It's important to note that this was an extremely rare event and the whales that size don't commonly occur."}, 'finish_reason': None}], 'usage': {'prompt_tokens': -1, 'completion_tokens': -1, 'total_tokens': -1}}