ACK Gateway with AI Extension：面向Kubernetes大模型推理的智能路由实践

GIT_LFS_SKIP_SMUDGE=1 git clone https://www.modelscope.cn/Qwen/QwQ-32B.git
cd QwQ-32B
git lfs pull

ossutil mkdir oss://<Your-Bucket-Name>/QwQ-32B
ossutil cp -r ./QwQ-32B oss://<Your-Bucket-Name>/QwQ-32B

kubectl apply -f- <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: qwq-32b
  name: qwq-32b
  namespace: default
spec:
  replicas: 5 # 根据GPU节点数量进行调整
  selector:
    matchLabels:
      app: qwq-32b
  template:
    metadata:
      labels:
        app: qwq-32b
      annotations:
        prometheus.io/path: /metrics
        prometheus.io/port: "8000"
        prometheus.io/scrape: "true"
    spec:
      volumes:
        - name: model
          persistentVolumeClaim:
            claimName: llm-model
        - name: dshm
          emptyDir:
            medium: Memory
            sizeLimit: 30Gi
      containers:
      - command:
        - sh
        - -c
        - vllm serve /models/QwQ-32B --port 8000 --trust-remote-code --served-model-name qwq-32b --tensor-parallel=4 --max-model-len 8192 --gpu-memory-utilization 0.95 --enforce-eager
        image: registry-cn-hangzhou.ack.aliyuncs.com/dev/vllm:v0.7.2
        name: vllm
        ports:
        - containerPort: 8000
        readinessProbe:
          tcpSocket:
            port: 8000
          initialDelaySeconds: 30
          periodSeconds: 30
        resources:
          limits:
            nvidia.com/gpu: "4"
        volumeMounts:
          - mountPath: /models/QwQ-32B
            name: model
          - mountPath: /dev/shm
            name: dshm
---
apiVersion: v1
kind: Service
metadata:
  name: qwq-32b-v1
spec:
  type: ClusterIP
  ports:
  - port: 8000
    protocol: TCP
    targetPort: 8000
  selector:
    app: qwq-32b
EOF

kubectl apply -f- <<EOF
apiVersion: gateway.networking.k8s.io/v1
kind: GatewayClass
metadata:
  name: inference-gateway
spec:
  controllerName: gateway.envoyproxy.io/gatewayclass-controller
---
apiVersion: gateway.networking.k8s.io/v1
kind: Gateway
metadata:
  name: inference-gateway
spec:
  gatewayClassName: inference-gateway
  listeners:
    - name: http
      protocol: HTTP
      port: 8080
    - name: llm-gw
      protocol: HTTP
      port: 8081
---
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
  name: backend
spec:
  parentRefs:
    - name: inference-gateway
      sectionName: http
  rules:
    - backendRefs:
        - group: ""
          kind: Service
          name: vllm-llama2-7b-pool
          port: 8000
      matches:
        - path:
            type: PathPrefix
            value: /
      timeouts:
        request: "24h"
        backendRequest: "24h"
EOF

kubectl apply -f- <<EOF
apiVersion: inference.networking.x-k8s.io/v1alpha1
kind: InferencePool
metadata:
  annotations:
    inference.networking.x-k8s.io/attach-to: |
      name: inference-gateway
      port: 8081
  name: reasoning-pool
spec:
  targetPortNumber: 8000
  selector:
    app: qwq-32b
EOF

kubectl apply -f- <<EOF
apiVersion: inference.networking.x-k8s.io/v1alpha1
kind: InferenceModel
metadata:
  name: inferencemodel-sample
spec:
  criticality: Critical
  modelName: qwq
  poolRef:
    group: inference.networking.x-k8s.io
    kind: InferencePool
    name: reasoning-pool
  targetModels:
  - name: qwq-32b
    weight: 100
EOF

GATEWAY_IP=$(kubectl get gateway/inference-gateway -o jsonpath='{.status.addresses[0].value}')
curl -X POST ${GATEWAY_IP}/v1/chat/completions \
-H 'Content-Type: application/json' \
-d '{
    "model": "qwq",
    "messages": [
      {
        "role": "user",
        "content": "你是谁？" 
      }
    ]
}' -v

kubectl apply -f- <<EOF 
apiVersion: apps/v1 
kind: Deployment
metadata:
  name: vllm-benchmark
  labels:
    app: vllm-benchmark
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm-benchmark
  template:
    metadata:
      labels:
        app: vllm-benchmark
    spec:
      volumes:
      - name: llm-model
        persistentVolumeClaim:
          claimName: llm-model
      containers:
      - name: vllm-benchmark
        image: kube-ai-registry.cn-shanghai.cr.aliyuncs.com/kube-ai/vllm-benchmark:v1
        command:
        - "sh"
        - "-c"
        - "sleep inf"
        volumeMounts:
        - mountPath: /models/QwQ-32B
          name: llm-model
EOF

# 执行以下命令进入Benchmark Pod
PODNAME=$(kubectl get po -o custom-columns=":metadata.name"|grep "vllm-benchmark")
kubectl exec -it $PODNAME -- bash
# 下载压测数据集
pip3 install modelscope
modelscope download --dataset gliang1001/ShareGPT_V3_unfiltered_cleaned_split ShareGPT_V3_unfiltered_cleaned_split.json --local_dir /root/

python3 /root/vllm/benchmarks/benchmark_serving.py \
--backend vllm \
--model /models/QwQ-32B \
--served-model-name qwq-32b \
--trust-remote-code \
--dataset-name random \
--dataset-path /root/ShareGPT_V3_unfiltered_cleaned_split.json \
--random-prefix-len 1000 \
--random-input-len 4000 \
--random-output-len 3000 \
--random-range-ratio 0.2 \
--num-prompts 3000 \
--max-concurrency 60 \
--host 172.16.12.92 \ # ACK Gateway 网关内网ip
--port 8080 \ # 请求8080端口，网关通过默认方式对推理服务进行路由
--endpoint /v1/completions \
--save-result \
2>&1 | tee benchmark_serving.txt
============ Serving Benchmark Result ============
Successful requests:                     2990      
Benchmark duration (s):                  5822.60   
Total input tokens:                      10071917  
Total generated tokens:                  5334789   
Request throughput (req/s):              0.51      
Output token throughput (tok/s):         916.22    
Total Token throughput (tok/s):          2646.02   
---------------Time to First Token----------------
Mean TTFT (ms):                          2456.44   
Median TTFT (ms):                        1366.07   
P99 TTFT (ms):                           23509.43  
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          63.50     
Median TPOT (ms):                        63.33     
P99 TPOT (ms):                           75.22     
---------------Inter-token Latency----------------
Mean ITL (ms):                           63.47     
Median ITL (ms):                         55.49     
P99 ITL (ms):                            59.16     
==================================================

python3 /root/vllm/benchmarks/benchmark_serving.py \
--backend vllm \
--model /models/QwQ-32B \
--served-model-name qwq-32b \
--trust-remote-code \
--dataset-name random \
--dataset-path /root/ShareGPT_V3_unfiltered_cleaned_split.json \
--random-prefix-len 1000 \
--random-input-len 4000 \
--random-output-len 3000 \
--random-range-ratio 0.2 \
--num-prompts 3000 \
--max-concurrency 60 \
--host 172.16.12.92 \
--port 8081 \ # 请求8081端口，网关基于推理扩展对推理服务进行路由
--endpoint /v1/completions \
--save-result \
2>&1 | tee benchmark_serving_ext.txt
============ Serving Benchmark Result ============
Successful requests:                     2990      
Benchmark duration (s):                  5755.77   
Total input tokens:                      10071917  
Total generated tokens:                  5332890   
Request throughput (req/s):              0.52      
Output token throughput (tok/s):         926.53    
Total Token throughput (tok/s):          2676.41   
---------------Time to First Token----------------
Mean TTFT (ms):                          1796.93   
Median TTFT (ms):                        1470.97   
P99 TTFT (ms):                           8857.07   
-----Time per Output Token (excl. 1st token)------
Mean TPOT (ms):                          63.10     
Median TPOT (ms):                        63.03     
P99 TPOT (ms):                           70.68     
---------------Inter-token Latency----------------
Mean ITL (ms):                           63.06     
Median ITL (ms):                         55.37     
P99 ITL (ms):                            59.17     
==================================================