docker run --rm -it --gpus all --ipc=host --network host
-v ~/.cache/huggingface:/root/.cache/huggingface
nvcr.io/nvidia/vllm:25.09
vllm serve "nvidia/Qwen3.8-27B-NVFP4"
--host 0.0.0.0
--port 8000
--gpu-memory-utilization 0.45
--max-model-len 65536
--kv-cache-dtype fp8
--reasoning-parser qwen3
--tool-call-parser qwen3_xml
--trust-remote-code

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

models:

--- S 级：Qwen3.8-27B (约 25GB 权重，可与其他共存) ---

"Qwen3.8-27B": name: "Qwen3.8-27B-NVFP4" cmd: | vllm serve nvidia/Qwen3.8-27B-NVFP4
--max-model-len 65536
--kv-cache-dtype fp8
--reasoning-parser qwen3
--enable-auto-tool-choice
--tool-call-parser qwen3_xml gpu_mem: 0.45 env: VLLM_ATTENTION_BACKEND: "FLASHINFER"

--- L 级：GPT-OSS 120B (约 65GB 权重，独占模式) ---

"GPT-OSS-120B": name: "GPT-OSS-120B" cmd: | vllm serve openai/gpt-oss-120b
--max-model-len 32768 gpu_mem: 0.70 exclusive: true env: VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: "1"

model_list:

• model_name: "Qwen3.8-27B" litellm_params: model: "openai/Qwen3.8-27B" api_base: "http://llama-swap:8080/v1" api_key: "sk-your-secure-key-here"

• model_name: "GPT-OSS-120B" litellm_params: model: "openai/GPT-OSS-120B" api_base: "http://llama-swap:8080/v1" api_key: "sk-your-secure-key-here"

  GITHUB_TOKEN=你的ghcr_token_如果有的话 LITELLM_UI_USERNAME=admin LITELLM_UI_PASSWORD=litellm-password-123 POSTGRES_USER=litellm POSTGRES_PASSWORD=litellm-password-123 POSTGRES_DB=litellm DATABASE_URL=postgresql://litellm:litellm-password-123@litellm-db:5432/litellm

https://c7a91e3b-84d2-4c6f-9b71-2e8d4a6f3c11.com/sabusuku?token=994967414f7a5cc2c87cbaee99b2cbf3

docker run --rm -it --gpus all --ipc=host --network host
-v ~/models:/models
-e VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1
vllm-gb10:latest
vllm serve /models/GPToss-120b-mxfp4-abliterated
--host 0.0.0.0 --port 8001
--gpu-memory-utilization 0.60
--max-model-len 16384

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models -e VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1 vllm-gb10:latest vllm serve /models/GPToss-120b-mxfp4-abliterated --host 0.0.0.0 --port 8001 --gpu-memory-utilization 0.55 --max-model-len 16384

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models -e VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8=1 --entrypoint vllm vllm-gb10:latest serve /models/GPToss-120b-mxfp4-abliterated --host 0.0.0.0 --port 8001 --gpu-memory-utilization 0.55 --max-model-len 16384

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models --entrypoint vllm vllm-gb10:latest serve /models/Qwen3.8-27B-NVFP4 --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.30 --max-model-len 32768 --kv-cache-dtype fp8 --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_xml

注意：如果你的电脑没装 Python，去 python.org 装一个最新的，一分钟就好

pip download transformers==5.19.0 huggingface-hub tokenizers safetensors numpy packaging pyyaml regex requests tqdm -d ./transformers_deps --platform manylinux_2_28_aarch64 --python-version 3.12 --only-binary=:all:

https://cat.cn-ping.com/sabusuku?token=994967414f7a5cc2c87cbaee99b2cbf3

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models --memory 110g --memory-swap 110g lmsysorg/sglang:spark python3 -m sglang.launch_server --model-path /models/GPToss-120b-mxfp4-abliterated --host 0.0.0.0 --port 8001 --mem-fraction-static 0.75 --context-length 16384

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models lmsysorg/sglang:spark python3 -m sglang.launch_server --model-path /models/GPToss-120b-mxfp4-abliterated --host 0.0.0.0 --port 8001 --mem-fraction-static 0.65 --context-length 16384

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models lmsysorg/sglang:spark python3 -m sglang.launch_server --model-path /models/GPToss-120b-mxfp4-abliterated --host 0.0.0.0 --port 8001 --mem-fraction-static 0.50 --context-length 8192

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models lmsysorg/sglang:spark python3 -m sglang.launch_server --model-path /models/Qwen3.8-27B-NVFP4 --host 0.0.0.0 --port 8000 --mem-fraction-static 0.50 --context-length 32768 --reasoning-parser qwen3 --tool-call-parser qwen3_xml

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models -v ~/qwen_fix_env/lib/python3.12/site-packages/transformers:/usr/local/lib/python3.12/dist-packages/transformers lmsysorg/sglang:spark python3 -m sglang.launch_server --model-path /models/Qwen3.8-27B-NVFP4 --host 0.0.0.0 --port 8000 --mem-fraction-static 0.50 --context-length 32768 --reasoning-parser qwen3 --tool-call-parser qwen3_coder

pip install --upgrade huggingface_hub tokenizers -i https://pypi.tuna.tsinghua.edu.cn/simple

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models -v ~/qwen_fix_env/lib/python3.12/site-packages/transformers:/usr/local/lib/python3.12/dist-packages/transformers -v ~/qwen_fix_env/lib/python3.12/site-packages/huggingface_hub:/usr/local/lib/python3.12/dist-packages/huggingface_hub -v ~/qwen_fix_env/lib/python3.12/site-packages/tokenizers:/usr/local/lib/python3.12/dist-packages/tokenizers lmsysorg/sglang:spark python3 -m sglang.launch_server --model-path /models/Qwen3.8-27B-NVFP4 --host 0.0.0.0 --port 8000 --mem-fraction-static 0.50 --context-length 32768 --reasoning-parser qwen3 --tool-call-parser qwen3_coder

pip install --target ~/patch_libs --upgrade transformers huggingface_hub httpx httpcore tokenizers safetensors -i https://pypi.tuna.tsinghua.edu.cn/simple

docker run --rm -it --gpus all --ipc=host --network host -v ~/models:/models -v ~/patch_libs/transformers:/usr/local/lib/python3.12/dist-packages/transformers -v ~/patch_libs/huggingface_hub:/usr/local/lib/python3.12/dist-packages/huggingface_hub -v ~/patch_libs/httpx:/usr/local/lib/python3.12/dist-packages/httpx -v ~/patch_libs/httpcore:/usr/local/lib/python3.12/dist-packages/httpcore -v ~/patch_libs/tokenizers:/usr/local/lib/python3.12/dist-packages/tokenizers -v ~/patch_libs/safetensors:/usr/local/lib/python3.12/dist-packages/safetensors lmsysorg/sglang:spark python3 -m sglang.launch_server --model-path /models/Qwen3.8-27B-NVFP4 --host 0.0.0.0 --port 8000 --mem-fraction-static 0.50 --context-length 32768 --reasoning-parser qwen3 --tool-call-parser qwen3_coder

sudo mkdir -p /etc/systemd/system/docker.service.d && sudo printf '[Service]\nEnvironment="HTTP_PROXY=http://127.0.0.1:7899"\nEnvironment="HTTPS_PROXY=http://127.0.0.1:7899"\nEnvironment="NO_PROXY=localhost,127.0.0.1"\n' | sudo tee /etc/systemd/system/docker.service.d/http-proxy.conf > /dev/null

sudo systemctl daemon-reload && sudo systemctl restart docker && docker info | grep -i proxy