容器计算服务ACS单张GPU即可快速搭建QwQ-32B推理模型

git lfs install
GIT_LFS_SKIP_SMUDGE=1 git clone https://www.modelscope.cn/Qwen/QwQ-32B.git
cd QwQ-32B
git lfs pull

ossutil mkdir oss://<your-bucket-name>/models/QwQ-32B
ossutil cp -r ./QwQ-32B oss://<your-bucket-name>/models/QwQ-32B

apiVersion: v1
kind: Secret
metadata:
  name: oss-secret
stringData:
  akId: <your-oss-ak> # 配置用于访问OSS的AccessKey ID
  akSecret: <your-oss-sk> # 配置用于访问OSS的AccessKey Secret
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: llm-model
  labels:
    alicloud-pvname: llm-model
spec:
  capacity:
    storage: 30Gi 
  accessModes:
    - ReadOnlyMany
  persistentVolumeReclaimPolicy: Retain
  csi:
    driver: ossplugin.csi.alibabacloud.com
    volumeHandle: llm-model
    nodePublishSecretRef:
      name: oss-secret
      namespace: default
    volumeAttributes:
      bucket: <your-bucket-name> # bucket名称
      url: <your-bucket-endpoint> # Endpoint信息，如oss-cn-hangzhou-internal.aliyuncs.com
      otherOpts: "-o umask=022 -o max_stat_cache_size=0 -o allow_other"
      path: <your-model-path> # 本示例中为/models/QwQ-32B/
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: llm-model
spec:
  accessModes:
    - ReadOnlyMany
  resources:
    requests:
      storage: 30Gi
  selector:
    matchLabels:
      alicloud-pvname: llm-model

kubectl apply -f- <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: qwq-32b
  name: qwq-32b
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: qwq-32b
  template:
    metadata:
      labels:
        app: qwq-32b
        alibabacloud.com/compute-class: gpu
        alibabacloud.com/gpu-model-series: <example-model>
    spec:
      volumes:
        - name: model
          persistentVolumeClaim:
            claimName: llm-model
        - name: dshm
          emptyDir:
            medium: Memory
            sizeLimit: 30Gi
      containers:
      - command:
        - sh
        - -c
        - vllm serve /models/QwQ-32B --port 8000 --trust-remote-code --served-model-name qwq-32b --max-model-len 32768 --gpu-memory-utilization 0.95 --enforce-eager
        image: registry-cn-wulanchabu-vpc.ack.aliyuncs.com/ack-demo/vllm:v0.7.2
        name: vllm
        ports:
        - containerPort: 8000
        readinessProbe:
          tcpSocket:
            port: 8000
          initialDelaySeconds: 30
          periodSeconds: 30
        resources:
          limits:
            nvidia.com/gpu: "1"
            cpu: "16"
            memory: 128G
        volumeMounts:
          - mountPath: /models/QwQ-32B
            name: model
          - mountPath: /dev/shm
            name: dshm
---
apiVersion: v1
kind: Service
metadata:
  name: qwq-32b-v1
spec:
  type: ClusterIP
  ports:
  - port: 8000
    protocol: TCP
    targetPort: 8000
  selector:
    app: qwq-32b
EOF

kubectl apply -f- << EOF 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: openwebui
spec:
  replicas: 1
  selector:
    matchLabels:
      app: openwebui
  template:
    metadata:
      labels:
        app: openwebui
    spec:
      containers:
      - env:
        - name: ENABLE_OPENAI_API
          value: "True"
        - name: ENABLE_OLLAMA_API
          value: "False"
        - name: OPENAI_API_BASE_URL
          value: http://qwq-32b-v1:8000/v1
        - name: ENABLE_AUTOCOMPLETE_GENERATION
          value: "False"
        - name: ENABLE_TAGS_GENERATION
          value: "False"
        image: kube-ai-registry.cn-shanghai.cr.aliyuncs.com/kube-ai/open-webui:main
        name: openwebui
        ports:
        - containerPort: 8080
          protocol: TCP
        volumeMounts:
        - mountPath: /app/backend/data
          name: data-volume
      volumes:
      - emptyDir: {}
        name: data-volume
---
apiVersion: v1
kind: Service
metadata:
  name: openwebui
  labels:
    app: openwebui
spec:
  type: ClusterIP
  ports:
  - port: 8080
    protocol: TCP
    targetPort: 8080
  selector:
    app: openwebui
EOF

kubectl port-forward svc/openwebui 8080:8080

kubectl apply -f- <<EOF 
apiVersion: apps/v1 
kind: Deployment
metadata:
  name: vllm-benchmark
  labels:
    app: vllm-benchmark
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm-benchmark
  template:
    metadata:
      labels:
        app: vllm-benchmark
    spec:
      volumes:
      - name: llm-model
        persistentVolumeClaim:
          claimName: llm-model
      containers:
      - name: vllm-benchmark
        image: kube-ai-registry.cn-shanghai.cr.aliyuncs.com/kube-ai/vllm-benchmark:v1
        command:
        - "sh"
        - "-c"
        - "sleep inf"
        volumeMounts:
        - mountPath: /models/QwQ-32B
          name: llm-model
EOF

# 执行以下命令进入Benchmark Pod
PODNAME=$(kubectl get po -o custom-columns=":metadata.name"|grep "vllm-benchmark")
kubectl exec -it $PODNAME -- bash
# 下载压测数据集
pip3 install modelscope
modelscope download --dataset gliang1001/ShareGPT_V3_unfiltered_cleaned_split ShareGPT_V3_unfiltered_cleaned_split.json --local_dir /root/

# 执行压测 input_length=4096,tp=4,output_lenght=512,concurrency=8,num_prompts=80
python3 /root/vllm/benchmarks/benchmark_serving.py \
--backend vllm \
--model /models/QwQ-32B \
--served-model-name qwq-32b \
--trust-remote-code \
--dataset-name random \
--dataset-path /root/ShareGPT_V3_unfiltered_cleaned_split.json \
--random-input-len 4096 \
--random-output-len 512 \
--random-range-ratio 1 \
--num-prompts 80 \
--max-concurrency 8 \
--host qwq-32b-v1 \
--port 8000 \
--endpoint /v1/completions \
--save-result \
2>&1 | tee benchmark_serving.txt

Starting initial single prompt test run...
Initial test run completed. Starting main benchmark run...
Traffic request rate: inf
Burstiness factor: 1.0 (Poisson process)
Maximum request concurrency: 8
100%|██████████| 80/80 [07:44<00:00,  5.81s/it]
============ Serving Benchmark Result ============
Successful requests:                     80        
Benchmark duration (s):                  464.74    
Total input tokens:                      327680    
Total generated tokens:                  39554     
Request throughput (req/s):              0.17      
Output token throughput (tok/s):         85.11     
Total Token throughput (tok/s):          790.18    
---------------Time to First Token----------------
Mean TTFT (ms):                          10315.97  
Median TTFT (ms):                        12470.54  
P99 TTFT (ms):                           17580.34  
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          71.03     
Median TPOT (ms):                        66.24     
P99 TPOT (ms):                           95.95     
---------------Inter-token Latency----------------
Mean ITL (ms):                           71.02     
Median ITL (ms):                         58.12     
P99 ITL (ms):                            60.26     
==================================================