SigLIP2¶

docker pull quay.io/ascend/vllm-ascend:v0.23.0-310p

# Set the vLLM Ascend image name.
export IMAGE=quay.io/ascend/vllm-ascend:v0.23.0-310p
export NAME=siglip2-dev

# Start the container with the variables defined above.
# Atlas 300I DUO uses a single NPU device (/dev/davinci0).
docker run --rm \
--name $NAME \
--net=host \
--shm-size=1g \
--privileged=true \
--device /dev/davinci0 \
--device /dev/davinci_manager \
--device /dev/devmm_svm \
--device /dev/hisi_hdc \
-v /usr/local/dcmi:/usr/local/dcmi \
-v /usr/local/Ascend/driver/tools/hccn_tool:/usr/local/Ascend/driver/tools/hccn_tool \
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-v /etc/ascend_install.info:/etc/ascend_install.info \
-v /root/.cache:/root/.cache \
-it $IMAGE bash

docker ps | grep $NAME

pip show vllm-ascend

#!/bin/sh
# Replace <YOUR_MODEL_PATH> with the path recorded in Section 3.1.
export MODEL_PATH=<YOUR_MODEL_PATH>

vllm serve $MODEL_PATH \
    --served-model-name $MODEL_PATH \
    --runner pooling \
    --chat-template template_basic.jinja \
    --limit-mm-per-prompt '{"image": 1}' \
    --compilation-config '{"cudagraph_capture_sizes": [64,32]}' \
    --additional-config '{"ascend_compilation_config": {"enable_npugraph_ex": false}}' \
    --dtype float16 \
    --port 8000 \
    --max-model-len 64

export MODEL_PATH=<YOUR_MODEL_PATH>  # match --served-model-name from Section 5.1

curl -X POST http://127.0.0.1:8000/v1/embeddings \
  -H "Content-Type: application/json" \
  -d "{
    \"model\": \"${MODEL_PATH}\",
    \"input\": [\"This is a photo of a dog.\"]
  }"

export MODEL_PATH=<YOUR_MODEL_PATH>  # match --served-model-name from Section 5.1
IMG_B64=$(base64 -w 0 /path/to/image.jpg)

curl -X POST http://127.0.0.1:8000/v1/embeddings \
  -H "Content-Type: application/json" \
  -d "{
    \"model\": \"${MODEL_PATH}\",
    \"encoding_format\": \"float\",
    \"messages\": [{
      \"role\": \"user\",
      \"content\": [{
        \"type\": \"image_url\",
        \"image_url\": {\"url\": \"data:image/jpeg;base64,${IMG_B64}\"}
      }]
    }]
  }"

from vllm import LLM

MODEL_PATH = "<YOUR_MODEL_PATH>"  # Replace with the path recorded in Section 3.1

llm = LLM(
    model=MODEL_PATH,
    runner="pooling",
    limit_mm_per_prompt={"image": 1},
    max_model_len=64,
)

# Text
text_out = llm.embed(
    ["This is a photo of a dog."],
    tokenization_kwargs={"padding": "max_length", "max_length": 64},
)
print(len(text_out[0].outputs.embedding))

# Image (empty prompt; field name must be multi_modal_data)
from PIL import Image

img = Image.open("/path/to/image.jpg").convert("RGB")
img_out = llm.embed(
    {"prompt": "", "multi_modal_data": {"image": img}},
)
print(len(img_out[0].outputs.embedding))

import ast
import numpy as np
from PIL import Image
from vllm import LLM

TEXT_TEMPLATE = "This is a photo of {}."
TOKEN_KWARGS = {"padding": "max_length", "max_length": 64}

def load_classnames(path):
    with open(path, encoding="utf-8") as f:
        d = ast.literal_eval(f.read())
    return [d[i] for i in range(1000)]

def load_val_label(path):
    gt = {}
    with open(path, encoding="utf-8") as f:
        for line in f:
            parts = line.strip().split()
            if len(parts) >= 2:
                gt[parts[0].split(".")[0]] = int(parts[1])
    return gt

MODEL_PATH = "<YOUR_MODEL_PATH>"  # Replace with the path recorded in Section 3.1

llm = LLM(
    model=MODEL_PATH,
    runner="pooling",
    limit_mm_per_prompt={"image": 1},
    max_model_len=64,
)

classnames = load_classnames("imagenet1000_clsidx_to_labels.txt")
prompts = [TEXT_TEMPLATE.format(c) for c in classnames]
text_feats = np.asarray(
    [o.outputs.embedding for o in llm.embed(prompts, tokenization_kwargs=TOKEN_KWARGS)],
    dtype=np.float32,
)
text_feats /= np.linalg.norm(text_feats, axis=1, keepdims=True)

gt = load_val_label("val_label.txt")
correct = 0
total = 0
for stem, label in gt.items():
    path = f"ImageNet/val/{stem}.JPEG"  # or .jpeg
    img = Image.open(path).convert("RGB")
    feat = np.asarray(
        llm.embed({"prompt": "", "multi_modal_data": {"image": img}})[0]
        .outputs.embedding,
        dtype=np.float32,
    )
    feat /= np.linalg.norm(feat)
    if np.argmax(feat @ text_feats.T) == label:
        correct += 1
    total += 1

print(f"Top-1 accuracy: {100.0 * correct / total:.2f}%")

"""Benchmark SigLIP2 /v1/embeddings serving over HTTP."""

import base64
import io
import json
import statistics
import time
import urllib.error
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed

import numpy as np
from PIL import Image

BASE_URL = "http://127.0.0.1:8000"
MODEL = "<YOUR_MODEL_PATH>"  # match --served-model-name from Section 5.1
NUM_REQUESTS = 200
CONCURRENCY = 8
WARMUP = 10
MODE = "both"  # "text", "image", or "both"
TEXT = "This is a photo of a dog."
IMAGE_SIZE = (224, 224)
TIMEOUT_S = 120.0


def post_json(url: str, payload: dict) -> tuple[bool, float, str]:
    body = json.dumps(payload).encode("utf-8")
    req = urllib.request.Request(
        url,
        data=body,
        headers={"Content-Type": "application/json"},
        method="POST",
    )
    t0 = time.perf_counter()
    try:
        with urllib.request.urlopen(req, timeout=TIMEOUT_S) as resp:
            resp.read()
            ok = resp.status == 200
    except urllib.error.HTTPError as e:
        detail = e.read().decode("utf-8", errors="replace")[:200]
        return False, time.perf_counter() - t0, f"HTTP {e.code}: {detail}"
    except Exception as e:
        return False, time.perf_counter() - t0, str(e)
    return ok, time.perf_counter() - t0, ""


def random_jpeg_b64(width: int, height: int, seed: int) -> str:
    arr = np.random.default_rng(seed).integers(
        0, 256, size=(height, width, 3), dtype=np.uint8
    )
    buf = io.BytesIO()
    Image.fromarray(arr, mode="RGB").save(buf, format="JPEG")
    return base64.b64encode(buf.getvalue()).decode("ascii")


def build_payload(mode: str, seed: int) -> dict:
    if mode == "text":
        return {
            "model": MODEL,
            "input": [TEXT],
            "encoding_format": "float",
        }
    w, h = IMAGE_SIZE
    b64 = random_jpeg_b64(w, h, seed)
    return {
        "model": MODEL,
        "encoding_format": "float",
        "messages": [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": f"data:image/jpeg;base64,{b64}",
                        },
                    }
                ],
            }
        ],
    }


def percentile(values: list[float], percent: float) -> float:
    sorted_values = sorted(values)
    rank = (len(sorted_values) - 1) * (percent / 100.0)
    lower = int(rank)
    upper = min(lower + 1, len(sorted_values) - 1)
    if lower == upper:
        return sorted_values[lower]
    return sorted_values[lower] + (sorted_values[upper] - sorted_values[lower]) * (
        rank - lower
    )


def run_benchmark(mode: str) -> None:
    url = f"{BASE_URL.rstrip('/')}/v1/embeddings"
    total = NUM_REQUESTS + WARMUP
    payloads = [build_payload(mode, i) for i in range(total)]

    def send_one(index: int) -> tuple[bool, float]:
        ok, latency_s, _ = post_json(url, payloads[index])
        return ok, latency_s

    if WARMUP:
        with ThreadPoolExecutor(max_workers=min(CONCURRENCY, WARMUP)) as pool:
            list(pool.map(send_one, range(WARMUP)))

    t0 = time.perf_counter()
    with ThreadPoolExecutor(max_workers=CONCURRENCY) as pool:
        futures = [pool.submit(send_one, WARMUP + i) for i in range(NUM_REQUESTS)]
        results = [fut.result() for fut in as_completed(futures)]
    wall_s = time.perf_counter() - t0

    ok_lat_ms = [lat * 1000 for ok, lat in results if ok]
    failed = sum(1 for ok, _ in results if not ok)
    print(f"\n=== {mode} ===")
    print(f"  successful: {len(ok_lat_ms)}/{NUM_REQUESTS}")
    print(f"  failed:     {failed}")
    print(f"  duration:   {wall_s:.2f}s")
    print(f"  throughput: {len(ok_lat_ms) / wall_s:.2f} req/s")
    if ok_lat_ms:
        print(f"  mean E2EL:  {statistics.mean(ok_lat_ms):.2f} ms")
        print(f"  median E2EL:{statistics.median(ok_lat_ms):.2f} ms")
        print(f"  p99 E2EL:   {percentile(ok_lat_ms, 99):.2f} ms")


if __name__ == "__main__":
    if MODE in ("text", "both"):
        run_benchmark("text")
    if MODE in ("image", "both"):
        run_benchmark("image")

vllm serve $MODEL_PATH \
    ... \
    --max-num-seqs 32 \
    --max-num-batched-tokens 512 \
    --max-model-len 64