from modelscope import snapshot_download
model_dir = snapshot_download("qwen/Qwen1.5-MoE-A2.7B-Chat")

from modelscope import AutoModelForCausalLM, AutoTokenizer
device = "cuda" # the device to load the model onto
model = AutoModelForCausalLM.from_pretrained(
    "qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4",
    torch_dtype="auto",
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained("qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4")
prompt = "Give me a short introduction to large language model."
messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": prompt}
]
text = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True
)
model_inputs = tokenizer([text], return_tensors="pt").to(device)
generated_ids = model.generate(
    model_inputs.input_ids,
    max_new_tokens=512
)
generated_ids = [
    output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
]
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(response)

git clone https://github.com/wenyujin333/vllm.git
cd vllm
git checkout add_qwen_moe
pip install -e .

export VLLM_USE_MODELSCOPE=True

python -m vllm.entrypoints.openai.api_server --model qwen/Qwen1.5-MoE-A2.7B-Chat

curl http://localhost:8000/v1/chat/completions \
    -H "Content-Type: application/json" \
    -d '{
    "model": "qwen/Qwen1.5-MoE-A2.7B-Chat",
    "messages": [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "Tell me something about large language models."}
    ]
    }'

git clone https://github.com/modelscope/swift.git
cd swift
pip install .[llm]

# https://github.com/modelscope/swift/blob/main/examples/pytorch/llm/scripts/qwen1half-moe-a2_7b-chat/lora/sft.sh
# Experimental environment: A100
# 42GB GPU memory
PYTHONPATH=../../.. \
CUDA_VISIBLE_DEVICES=0 \
python llm_sft.py \
    --model_type qwen1half-moe-a2_7b-chat \
    --sft_type lora \
    --tuner_backend swift \
    --dtype AUTO \
    --output_dir output \
    --dataset dureader-robust-zh \
    --train_dataset_sample 10000 \
    --num_train_epochs 1 \
    --max_length 1024 \
    --check_dataset_strategy warning \
    --lora_rank 8 \
    --lora_alpha 32 \
    --lora_dropout_p 0.05 \
    --lora_target_modules ALL \
    --gradient_checkpointing true \
    --batch_size 1 \
    --weight_decay 0.1 \
    --learning_rate 1e-4 \
    --gradient_accumulation_steps 16 \
    --max_grad_norm 0.5 \
    --warmup_ratio 0.03 \
    --eval_steps 100 \
    --save_steps 100 \
    --save_total_limit 2 \
    --logging_steps 10 \
    --use_flash_attn true \
    --self_cognition_sample 1000 \
    --model_name 卡卡罗特 \
    --model_author 陶白白 \

--custom_train_dataset_path xxx.jsonl \
--custom_val_dataset_path yyy.jsonl \

# Experimental environment: A100
# 3GB GPU memory
PYTHONPATH=../../.. \
CUDA_VISIBLE_DEVICES=0 \
python llm_infer.py \
    --ckpt_dir "output/qwen1half-moe-a2_7b-chat/vx-xxx/checkpoint-xxx" \
    --load_dataset_config true \
    --use_flash_attn true \
    --max_new_tokens 2048 \
    --temperature 0.1 \
    --top_p 0.7 \
    --repetition_penalty 1. \
    --do_sample true \
    --merge_lora false \

[PROMPT]<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
学校需新添28套课桌椅，共付款1820元，椅子每把17元，课桌每张多少元？<|im_end|>
<|im_start|>assistant
[OUTPUT]设课桌每张x元。
根据题意，椅子每把17元，课桌每张x元。
则椅子的总费用为17元/把 × 28把 = 476元。
课桌的总费用为x元/张 × 28张 = 28x元。
根据题意，课桌和椅子的总费用为1820元。
所以，28x + 476 = 1820。
解方程得：28x = 1820 - 476 = 1344。
x = 1344 / 28 = 48。
所以，课桌每张48元。
Answer: 48<|im_end|>
[LABELS]假设课桌的价格为x元每张。
根据题意可得方程：
28(17+x) = 1820
化简得：
476 + 28x = 1820
移项得：
28x = 1820 - 476
化简得：
28x = 1344
将等式两边同时除以28，得：
x = 1344 / 28
化简得：
x = 48
所以，每张课桌的价格为48元。
Answer: 48
--------------------------------------------------
[PROMPT]<|im_start|>system
You are a helpful assistant.<|im_end|>
<|im_start|>user
甲数是18，乙数比甲数的2倍少5，乙数=．<|im_end|>
<|im_start|>assistant
[OUTPUT]设乙数为x，则根据题意可得：
x = 2 * 18 - 5
x = 36 - 5
x = 31
所以乙数为31。
Answer: 31<|im_end|>
[LABELS]首先，让我们用变量来表示甲数和乙数。
甲数 = 18
乙数 = 2 * 甲数 - 5
将甲数的值代入乙数的表达式中，我们得到：
乙数 = 2 * 18 - 5
      = 36 - 5
      = 31
所以，乙数=31。
Answer: 31