Hy4-preview (实验性)¶

npu-smi info

docker pull quay.io/ascend/vllm-ascend:hy4-a3

docker images | grep hy4-a3

export IMAGE=quay.io/ascend/vllm-ascend:hy4-a3
export CONTAINER_NAME=vllm_ascend_hy4

# Generate NPU device mount parameters (Atlas 800I A3: /dev/davinci[0-15])
DEVICES=""
for i in $(seq 0 15); do
    DEVICES="${DEVICES} --device /dev/davinci${i}"
done

docker run -itd \
    --name ${CONTAINER_NAME} \
    --shm-size=1000g \
    ${DEVICES} \
    --device /dev/davinci_manager \
    --device /dev/devmm_svm \
    --device /dev/hisi_hdc \
    -v /usr/local/dcmi:/usr/local/dcmi \
    -v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
    -v /mnt/weight:/mnt/weight \
    ${IMAGE} bash

docker exec -it ${CONTAINER_NAME} bash

export HCCL_BUFFSIZE=128
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True
export VLLM_LOGGING_LEVEL=INFO

# Ensure the model path matches the directory recorded during download
vllm serve /path/to/Hy4-preview-w8a8 \
  --host 127.0.0.1 --port 8000 \
  --tensor-parallel-size 16 \
  --served-model-name hy4 \
  --max-num-seqs 8 \
  --max-model-len 512 \
  --max-num-batched-tokens 512 \
  --enable-expert-parallel \
  --trust-remote-code \
  --quantization ascend \
  --gpu-memory-utilization 0.92 \
  --seed 1024 \
  --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}' \
  --speculative-config '{"method": "mtp","num_speculative_tokens": 3}' \
  --additional-config '{"enable_mlapo": true, "enable_fused_mc2": 1, "ascend_compilation_config": {"enable_npugraph_ex": true}}'

#!/bin/bash
set -u

# === Node network configuration (adjust the NIC name and IP according to the actual environment) ===
NODE1_IP="192.168.1.1"       # Node 1 IP, change to the actual machine IP
NODE2_IP="192.168.1.2"       # Node 2 IP, change to the actual machine IP
NIC="eth0"                   # Change to the NIC name corresponding to NODE1_IP, viewable via the `ifconfig` command
DP_RPC_PORT=12890

# === Service configuration ===
MODEL="/path/to/Hy4-preview-w8a8"  # Change to the actual weight path
PORT=8000

# === Environment variables ===
export HCCL_BUFFSIZE=128
export HCCL_IF_IP=${NODE1_IP}
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=${NIC}
export GLOO_SOCKET_IFNAME=${NIC}
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True

# Ensure the model path matches the directory recorded during download
vllm serve ${MODEL} \
  --host 0.0.0.0 \
  --port ${PORT} \
  --data-parallel-size 2 \
  --data-parallel-size-local 1 \
  --data-parallel-address ${NODE1_IP} \
  --data-parallel-rpc-port ${DP_RPC_PORT} \
  --tensor-parallel-size 16 \
  --served-model-name hy4 \
  --enable-expert-parallel \
  --max-num-seqs 256 \
  --max-model-len 96000 \
  --max-num-batched-tokens 1024 \
  --trust-remote-code \
  --quantization ascend \
  --gpu-memory-utilization 0.85 \
  --enable-prefix-caching \
  --seed 1024 \
  --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}' \
  --speculative-config '{"method": "mtp","num_speculative_tokens": 3}' \
  --additional-config '{"enable_mlapo": true, "enable_fused_mc2": 1, "ascend_compilation_config": {"enable_npugraph_ex": true}}'

#!/bin/bash
set -u

# === Node network configuration ===
NODE1_IP="192.168.1.1"       # Node 1 IP, change to the actual machine IP
NODE2_IP="192.168.1.2"       # Node 2 IP, change to the actual machine IP
NIC="eth0"                   # Change to the NIC name corresponding to NODE2_IP, viewable via the `ifconfig` command
DP_RPC_PORT=12890

# === Service configuration (consistent with Node 1) ===
MODEL="/path/to/Hy4-preview-w8a8"  # Change to the actual weight path

# === Environment variables (consistent with Node 1) ===
export HCCL_BUFFSIZE=128
export HCCL_IF_IP=${NODE2_IP}
export HCCL_OP_EXPANSION_MODE=AIV
export HCCL_SOCKET_IFNAME=${NIC}
export GLOO_SOCKET_IFNAME=${NIC}
export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True

# Ensure the model path matches the directory recorded during download
vllm serve ${MODEL} \
  --headless \
  --data-parallel-start-rank 1 \
  --data-parallel-size 2 \
  --data-parallel-size-local 1 \
  --data-parallel-address ${NODE1_IP} \
  --data-parallel-rpc-port ${DP_RPC_PORT} \
  --tensor-parallel-size 16 \
  --enable-expert-parallel \
  --max-num-seqs 256 \
  --max-model-len 96000 \
  --max-num-batched-tokens 1024 \
  --trust-remote-code \
  --quantization ascend \
  --gpu-memory-utilization 0.85 \
  --enable-prefix-caching \
  --seed 1024 \
  --compilation-config '{"cudagraph_mode": "FULL_DECODE_ONLY"}' \
  --speculative-config '{"method": "mtp","num_speculative_tokens": 3}' \
  --additional-config '{"enable_mlapo": true, "enable_fused_mc2": 1, "ascend_compilation_config": {"enable_npugraph_ex": true}}'

INFO:     Started server process [PID]
INFO:     Waiting for application startup.
INFO:     Application startup complete.

curl -sS -X POST http://127.0.0.1:8000/v1/chat/completions -H 'Content-Type: application/json' -d '{
    "model":"hy4",
    "messages":[{
        "role":"user",
        "content":"Who are you?"
    }],
    "max_tokens":256,
    "temperature":0
}'

INFO:     [IP]:[PORT] - "POST /v1/chat/completions HTTP/1.1" 200 OK

{"id":"chatcmpl-84c57a059e1d510c","object":"chat.completion","created":1787890448,"model":"hy4","choices":[{"index":0,"message":{"role":"assistant","content":"Okay, I'm the user asking \"Who are you?\" and I need to respond according to my identity and behavioral guidelines. First, my identity is Hunyuan, a large model developed by Tencent. The user's question directly asks about my identity, so I should clearly state who I am without extra information. Based on the core constraints, when the question involves identity confirmation, I need to clearly explain my identity and developer. Also, the user didn't mention any other functions or context, so keeping it concise is fine. Check if there's anything that needs clarification, like whether the user wants to know about features, but the current question is simply an identity inquiry, so a direct answer is enough. Make sure the response meets the format requirements, no markdown, and keep it natural and conversational.</think:opensource>I am Hunyuan, a large model developed by Tencent.","refusal":null,"annotations":null,"audio":null,"function_call":null,"reasoning":null},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null,"routed_experts":null}],"service_tier":null,"system_fingerprint":"vllm-0.23.0-tp16-dp2-ep-36628cea","usage":{"prompt_tokens":25,"total_tokens":201,"completion_tokens":176,"prompt_tokens_details":null,"completion_tokens_details":null},"prompt_logprobs":null,"prompt_token_ids":null,"prompt_text":null,"kv_transfer_params":null}