GPU 为什么会挨饿：数据墙、投机解码与设计空间探索Why Does the GPU Starve: The Data Wall, Speculative Decoding, and Design-Space Exploration

model = mlsysim.Models.Vision.ResNet50
hardware = mlsysim.Hardware.Cloud.A100
solver = SingleNodeModel()
profile = solver.solve(model=model, hardware=hardware, batch_size=256, precision="fp16")

import mlsysim
from mlsysim.solvers import SingleNodeModel, DataModel
from mlsysim.solvers import TransformationModel

from mlsysim.solvers import SingleNodeModel
from mlsysim.core.units import Q_
from mlsysim.show import table, info

model = mlsysim.Models.Vision.ResNet50
hardware = mlsysim.Hardware.Cloud.A100
solver = SingleNodeModel()

# Baseline: ResNet-50 on A100, batch 256, FP16
profile = solver.solve(model=model, hardware=hardware, batch_size=256, precision="fp16")

info("GPU Compute Baseline",
     Model=model.name,
     Hardware=hardware.name,
     Batch_size=256,
     Step_latency=profile.latency.to('ms'),
     Throughput=f"{profile.throughput:.0f} img/s",
     Bottleneck=profile.bottleneck)

from mlsysim.solvers import DataModel

sample_size = Q_("500 KB")  # Average ImageNet JPEG
batch_size = 256

# Data demand = batch_size x sample_size / step_time
step_time_s = profile.latency.to("s").magnitude
data_per_step = (batch_size * sample_size.to("GB")).magnitude
demand_rate = Q_(data_per_step / step_time_s, "GB/s")

data_solver = DataModel()
data_result = data_solver.solve(workload_data_rate=demand_rate, hardware=hardware)

info("Storage I/O Check",
     Data_demand=f"{demand_rate:.3f}",
     Storage_supply=f"{data_result.supply_bw:.2f}",
     Utilization=f"{data_result.utilization:.1%}",
     Is_stalled=data_result.is_stalled)

from mlsysim.solvers import TransformationModel

transform_solver = TransformationModel()
cpu_throughput = Q_("2 GB/s")  # 8 workers x 250 MB/s each

t = transform_solver.solve(
    batch_size=256,
    sample_size_bytes=sample_size,
    cpu_throughput=cpu_throughput,
    accelerator_step_time=profile.latency
)

info("CPU vs GPU Pipeline",
     CPU_transform_time=t.transform_time,
     GPU_step_time=t.accelerator_step_time,
     CPU_is_bottleneck=t.is_bottleneck,
     GPU_utilization=f"{t.accelerator_utilization:.1%}",
     Slowdown_factor=f"{t.slowdown_factor:.2f}x")

rows = []
for bs in [16, 32, 64, 128, 256, 512, 1024]:
    p = solver.solve(model=model, hardware=hardware, batch_size=bs, precision="fp16")

    t = transform_solver.solve(
        batch_size=bs,
        sample_size_bytes=sample_size,
        cpu_throughput=cpu_throughput,
        accelerator_step_time=p.latency
    )

    binding = "Transformation" if t.is_bottleneck else p.bottleneck
    rows.append([
        bs,
        f"{p.latency.to('ms').magnitude:.2f} ms",
        f"{t.transform_time.to('ms').magnitude:.2f} ms",
        binding,
        f"{t.accelerator_utilization:.1%}"
    ])

table(["Batch", "GPU Step", "CPU Xform", "Binding", "GPU Util"], rows)

rows = []
for n_workers in [8, 16, 32,64, 128]:
    cpu_tp = Q_(f"{n_workers * 250} MB/s")

    p = solver.solve(model=model, hardware=hardware, batch_size=512, precision="fp16")

    t = transform_solver.solve(
        batch_size=512,
        sample_size_bytes=sample_size,
        cpu_throughput=cpu_tp,
        accelerator_step_time=p.latency
    )

    rows.append([n_workers, cpu_tp.to('GB/s'), f"{t.accelerator_utilization:.1%}"])

table(["Workers", "Throughput", "GPU Util @ bs=512"], rows)

import mlsysim
from mlsysim.engine.dse import DSE
from mlsysim.solvers import DistributedModel, EconomicsModel
from mlsysim.engine.pipeline import Pipeline

# Our fixed baseline constants
model = mlsysim.Models.Language.Llama3_70B
fleet = mlsysim.Systems.Clusters.Frontier_8K

# The evaluation function accepts a dictionary of parameters and returns a structured object
def evaluate_config(params):
    pipe = Pipeline([DistributedModel(), EconomicsModel()])

    return pipe.run(
        model=model,
        fleet=fleet,
        batch_size=params["batch_size"],
        tp_size=params["tp"],
        pp_size=params["pp"],
        precision="fp16",
        efficiency=0.45,
        duration_days=30
    )

# 1. Define the dimensions of your search grid
space = {
    "batch_size": [1024, 2048, 4096, 8192],
    "tp": [1, 2, 4, 8],
    "pp": [1, 2, 4]
}

# 2. Initialize the Design Space Explorer
dse = DSE(
    space=space,
    objective="maximize: DistributedModel.effective_throughput"
)

# 3. Search! (Tests all 4 * 4 * 3 = 48 combinations analytically)
print("Starting search...")

result = dse.search(evaluate_config)

# 4. Analyze the best configuration
best_params = result["best_params"]
best_throughput = result["best_objective"]

print(f"\nBest Configuration: TP={best_params['tp']}, PP={best_params['pp']}, Batch={best_params['batch_size']}")
print(f"Max Throughput: {best_throughput:,.0f} tokens/sec")

print(f"\nTop 3 Configurations:")
print(f"{'TP':>4} | {'PP':>4} | {'Batch':>6} | {'Throughput':>12} | {'Latency':>10}")
print("-" * 45)

for candidate in result["top_candidates"][:3]:
    p = candidate["params"]
    dist_res = candidate["result"]["DistributedModel"]

    throughput = dist_res.effective_throughput.m_as("1/s")
    latency = dist_res.step_latency_total.m_as("ms")

    print(f"{p['tp']:>4} | {p['pp']:>4} | {p['batch_size']:>6} | {throughput:>12,.0f} | {latency:>10.1f}ms")

import mlsysim
from mlsysim.solvers import ServingModel, InferenceScalingModel

hw = mlsysim.Hardware.Cloud.H100
target = mlsysim.Models.Language.Llama3_70B
draft8b = mlsysim.Models.Language.Llama3_8B
draft_gpt2 = mlsysim.Models.Language.GPT2
solver = ServingModel()

# E4: 基线 vs 投机解码(8B draft, alpha=0.75)
base = solver.solve(target, hw, seq_len=2048, batch_size=1)
spec = solver.solve(target, hw, seq_len=2048, batch_size=1,
                    draft_model=draft8b, draft_acceptance_rate=0.75)
print(f"baseline ITL: {base.itl.to('ms').magnitude:.2f} ms")
print(f"spec ITL:     {spec.itl.to('ms').magnitude:.2f} ms")
print(f"speedup:      {base.itl/spec.itl:.2f}x")

# O1a: 接受率扫描
for a in [0.5, 0.75, 0.9]:
    r = solver.solve(target, hw, seq_len=2048, batch_size=1,
                     draft_model=draft8b, draft_acceptance_rate=a)
    print(f"alpha={a}: ITL {r.itl.to('ms').magnitude:7.2f} ms  speedup {base.itl/r.itl:.2f}x")

# O1b: draft 模型大小扫描
for name, dm in [("GPT2(1.5B)", draft_gpt2), ("Llama3-8B", draft8b), ("Llama3-70B", target)]:
    r = solver.solve(target, hw, seq_len=2048, batch_size=1,
                     draft_model=dm, draft_acceptance_rate=0.75)
    print(f"draft={name}: ITL {r.itl.to('ms').magnitude:7.2f} ms  speedup {base.itl/r.itl:.2f}x")

# O3: 推理时计算(K=0/8/32)
isolver = InferenceScalingModel()
for k in [0, 8, 32]:
    r = isolver.solve(mlsysim.Models.Language.Llama3_8B, hw, reasoning_steps=k)
    print(f"K={k}: total {r.total_reasoning_time.to('s').magnitude:.3f} s  "
          f"energy {r.energy_per_query.to('kJ').magnitude:.3f} kJ")