from modelscope import snapshot_download
model_dir = snapshot_download("LLM-Research/Phi-3-mini-128k-instruct")

import torch
from modelscope import snapshot_download
from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
torch.random.manual_seed(0)
model_dir = snapshot_download("LLM-Research/Phi-3-mini-128k-instruct")
model = AutoModelForCausalLM.from_pretrained(
    model_dir, 
    device_map="cuda", 
    torch_dtype="auto", 
    trust_remote_code=True, 
)
tokenizer = AutoTokenizer.from_pretrained(model_dir)
messages = [
    {"role": "system", "content": "You are a helpful digital assistant. Please provide safe, ethical and accurate information to the user."},
    {"role": "user", "content": "Can you provide ways to eat combinations of bananas and dragonfruits?"},
    {"role": "assistant", "content": "Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey."},
    {"role": "user", "content": "What about solving an 2x + 3 = 7 equation?"},
]
pipe = pipeline(
    "text-generation",
    model=model,
    tokenizer=tokenizer,
)
generation_args = {
    "max_new_tokens": 500,
    "return_full_text": False,
    "temperature": 0.0,
    "do_sample": False,
}
output = pipe(messages, **generation_args)
print(output[0]['generated_text'])

git clone https://www.modelscope.cn/LLM-Research/Phi-3-mini-4k-instruct-onnx.git

pip install --pre onnxruntime-genaicurl https://raw.githubusercontent.com/microsoft/onnxruntime-genai/main/examples/python/model-qa.py -o model-qa.py

python model-qa.py -m Phi-3-mini-4k-instruct-onnx/cpu_and_mobile/cpu-int4-rtn-block-32 -l 2048

git clone https://github.com/modelscope/swift.git
cd swift
pip install -e .[all]

CUDA_VISIBLE_DEVICES=0,1,2,3
NPROC_PER_NODE=4 \
swift sft \
  --model_type phi3-4b-4k-instruct \
  --dataset ms-agent-for-agentfabric-default alpaca-en ms-bench ms-agent-for-agentfabric-addition coig-cqia-ruozhiba coig-cqia-zhihu coig-cqia-exam coig-cqia-chinese-traditional coig-cqia-logi-qa coig-cqia-segmentfault coig-cqia-wiki \
  --batch_size 2 \
  --max_length 2048 \
  --use_loss_scale true \
  --gradient_accumulation_steps 16 \
  --learning_rate 5e-5 \
  --use_flash_attn true \
  --eval_steps 500 \
  --save_steps 500 \
  --train_dataset_sample -1 \
  --dataset_test_ratio 0.1 \
  --val_dataset_sample 10000 \
  --num_train_epochs 2 \
  --check_dataset_strategy none \
  --gradient_checkpointing true \
  --weight_decay 0.01 \
  --warmup_ratio 0.03 \
  --save_total_limit 2 \
  --logging_steps 10 \
  --sft_type lora \
  --lora_target_modules ALL \
  --lora_rank 8 \
  --lora_alpha 32

--custom_train_dataset_path xxx.jsonl \
--custom_val_dataset_path yyy.jsonl \

# Experimental environment: A100
CUDA_VISIBLE_DEVICES=0 \
swift infer \
    --ckpt_dir "/path/to/output/phi3-4b-4k-instruct/vx-xxx/checkpoint-xxx" \
    --load_dataset_config true \
    --max_new_tokens 2048 \
    --temperature 0.1 \
    --top_p 0.7 \
    --repetition_penalty 1. \
    --do_sample true \
    --merge_lora false \