DeepSeek Open Weights: Self-Hosting & Fine-Tuning Guide

pip install vllm

# Serve V4 Flash (AWQ quantized)
vllm serve deepseek-ai/DeepSeek-V4-Flash-AWQ \
    --max-model-len 131072 \
    --gpu-memory-utilization 0.95

# Serve V4 Pro (FP16 on 2× A100)
vllm serve deepseek-ai/DeepSeek-V4-Pro \
    --tensor-parallel-size 2 \
    --max-model-len 131072

# Optimize for throughput
--max-num-seqs 256           # Concurrent sequences
--max-model-len 131072       # Limit context to manage VRAM
--gpu-memory-utilization 0.95 # Use nearly all GPU memory
--enable-prefix-caching       # Enable prefix caching (similar to API)
--enable-chunked-prefill      # Better throughput for long prompts

# Optimize for latency
--max-num-seqs 8             # Fewer concurrent sequences
--max-num-batched-tokens 8192 # Smaller batches

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = "deepseek-ai/DeepSeek-V4-Flash"
quant_path = "DeepSeek-V4-Flash-AWQ"

model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)

model.quantize(tokenizer, quant_config={"zero_point": True, "q_group_size": 128})
model.save_quantized(quant_path)

# Convert to GGUF
python convert-hf-to-gguf.py DeepSeek-V4-Flash --outtype q4_k_m

# Run with llama.cpp
./llama-cli -m DeepSeek-V4-Flash-Q4_K_M.gguf \
    -p "Explain quantum computing" \
    -n 512 \
    --ctx-size 32768

from auto_gptq import AutoGPTQForCausalLM

model = AutoGPTQForCausalLM.from_pretrained(
    "deepseek-ai/DeepSeek-V4-Flash",
    quantize_config={"bits": 4, "group_size": 128}
)

from peft import LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM, TrainingArguments, Trainer

model = AutoModelForCausalLM.from_pretrained(
    "deepseek-ai/DeepSeek-V4-Flash",
    load_in_8bit=True,  # Quantize for memory efficiency
    device_map="auto"
)

lora_config = LoraConfig(
    r=16,                   # LoRA rank
    lora_alpha=32,          # Scaling factor
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM"
)

model = get_peft_model(model, lora_config)

training_args = TrainingArguments(
    output_dir="./deepseek-lora",
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    num_train_epochs=3,
    learning_rate=2e-4,
    fp16=True,
    logging_steps=10,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
)
trainer.train()
model.save_pretrained("./deepseek-lora-final")

from transformers import BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True
)

model = AutoModelForCausalLM.from_pretrained(
    "deepseek-ai/DeepSeek-V4-Flash",
    quantization_config=bnb_config,
    device_map="auto"
)
# Then apply LoRA as above