NVIDIA H20 (8x) - qwen3-coder-30b-a3b-instruct

October 24, 2025 at 08:18 PM

Dataset: reference (v1.0)

NVIDIA NVIDIA H20 qwen qwen3-coder-30b-a3b-instruct

Best Performance

Click a metric to highlight the best run in the table below

Best Output TPS

8,987.38

Peak generation speed

Best Input TPS

30,699.04

Peak prefill speed

Best Energy Efficiency

0.02 kWh/MT

Energy cost per 1M tokens

Best TTFT (P95)

58.88 ms

Lowest latency

Best E2E (P95)

988.69 ms

Lowest latency

Test Matrix Results

Performance across different input/output token combinations and concurrency levels

Input Tokens	Output Tokens	Concurrency	Output TPS	Input TPS	Energy Cost (kWh/MT)	TTFT Mean	TTFT P95	E2E P95	Success Rate
Best Run for Output TPS
1,024	1,024	1024x	8,987.38★	9,678.22	0.04	13,549.26	21,404.54	58,999.03	100.0%
128	128	1x	100.95	96.21	1.41	357.83	357.83	1,267.42	100.0%
128	128	2x	196.77	192.16	0.75	265.99	378.74	1,287.84	100.0%
128	128	4x	245.68	243.76	0.60	577.58	968.70	2,080.17	100.0%
128	128	8x	416.77	417.99	0.39	890.12	1,277.39	2,451.56	100.0%
128	128	16x	638.24	644.49	0.28	963.23	1,978.12	3,188.94	100.0%
128	128	32x	1,254.47	1,265.27	0.16	1,494.13	1,828.26	3,224.97	100.0%
128	128	64x	1,471.04	1,471.04	0.16	2,970.58	3,643.49	5,519.00	100.0%
128	128	128x	1,776.90	1,803.66	0.14	5,325.22	6,479.79	8,721.50	100.0%
128	128	256x	1,512.99	1,556.08	0.13	16,522.15	18,214.51	20,671.62	100.0%
128	512	1x	144.80	34.50	1.68	64.10	64.10	3,534.83	100.0%
128	512	2x	227.75	57.80	1.03	122.35	126.98	4,310.54	100.0%
128	512	4x	452.48	116.62	0.59	215.59	245.17	4,337.80	100.0%
128	512	8x	889.14	233.78	0.39	282.35	303.86	4,386.70	100.0%
128	512	16x	1,431.43	375.71	0.29	449.31	499.32	5,484.00	100.0%
128	512	32x	2,121.82	554.62	0.22	860.71	918.97	7,371.21	100.0%
128	512	64x	3,241.76	829.00	0.16	1,744.06	2,090.91	9,756.43	100.0%
128	512	128x	4,358.01	1,119.70	0.13	4,921.43	5,829.60	14,402.60	100.0%
128	512	256x	4,419.11	1,158.49	0.12	15,761.99	17,502.16	27,582.93	100.0%
128	512	512x	5,225.33	1,362.49	0.09	25,257.10	29,517.44	41,742.47	100.0%
128	512	1024x	6,777.19	1,785.32	0.09	30,828.39	43,830.41	56,140.29	100.0%
128	1,024	1x	145.08	17.29	1.89	78.44	78.44	7,056.73	100.0%
128	1,024	2x	205.82	35.29	1.32	123.61	125.03	6,889.54	100.0%
128	1,024	4x	435.09	63.84	0.72	163.54	175.72	7,828.76	100.0%
128	1,024	8x	759.73	121.50	0.48	237.82	248.38	8,446.06	100.0%
128	1,024	16x	1,357.39	202.83	0.33	457.93	496.69	10,079.49	100.0%
128	1,024	32x	2,167.79	320.87	0.24	837.11	862.86	12,765.08	100.0%
128	1,024	64x	3,507.73	492.15	0.18	1,525.10	1,883.46	16,497.78	100.0%
128	1,024	128x	4,997.05	719.99	0.13	3,941.79	4,952.57	22,283.27	100.0%
128	1,024	256x	6,294.38	918.60	0.11	12,991.51	14,580.03	34,715.29	100.0%
128	1,024	512x	6,466.53	957.67	0.09	25,426.55	29,760.97	59,956.53	100.0%
128	1,024	1024x	7,321.76	1,106.56	0.08	49,795.40	75,677.92	102,843.78	100.0%
128	2,048	1x	144.55	9.63	2.03	66.90	66.90	12,665.91	100.0%
128	2,048	2x	175.01	17.69	1.63	93.02	93.57	13,584.86	100.0%
128	2,048	4x	355.18	35.60	0.91	224.75	239.69	13,707.01	100.0%
128	2,048	8x	468.22	55.32	0.71	286.34	329.08	17,624.29	100.0%
128	2,048	16x	1,026.32	127.18	0.40	469.87	491.90	13,703.95	100.0%
128	2,048	32x	1,623.51	195.39	0.29	837.13	897.06	18,941.65	100.0%
128	2,048	64x	2,604.93	283.25	0.23	1,650.04	1,973.31	26,774.97	100.0%
128	2,048	128x	3,968.42	435.81	0.16	4,026.85	4,953.73	33,833.84	100.0%
128	2,048	256x	5,769.65	649.68	0.12	12,932.52	14,410.78	47,957.70	100.0%
128	2,048	512x	6,135.59	690.66	0.10	27,069.57	29,897.41	82,594.73	100.0%
128	2,048	1024x	8,870.32	1,014.91	0.07	12,239.45	24,975.96	79,239.08	100.0%
512	128	1x	126.48	490.12	0.46	96.28	96.28	1,011.83	100.0%
512	128	2x	258.85	1,006.07	0.24	120.15	122.19	988.69	100.0%
512	128	4x	400.00	1,545.31	0.18	202.81	240.45	1,276.81	100.0%
512	128	8x	619.11	2,388.15	0.12	312.14	371.37	1,650.91	100.0%
512	128	16x	902.60	3,497.58	0.09	603.97	1,088.79	2,260.57	100.0%
512	128	32x	1,282.58	5,000.31	0.07	1,035.78	1,586.03	3,177.34	100.0%
512	128	64x	1,981.26	7,758.03	0.06	1,747.70	2,283.07	4,079.58	100.0%
512	128	128x	2,037.23	7,953.02	0.05	4,173.56	5,677.89	7,891.70	100.0%
512	128	256x	1,683.61	6,690.91	0.05	14,010.02	15,425.45	18,471.51	100.0%
512	512	1x	143.18	139.52	1.13	58.88	58.88	3,554.71	100.0%
512	512	2x	267.15	259.59	0.62	110.25	129.39	3,818.87	100.0%
512	512	4x	531.48	514.57	0.36	181.43	191.96	3,841.51	100.0%
512	512	8x	897.19	867.75	0.24	285.21	326.91	4,545.70	100.0%
512	512	16x	1,490.60	1,462.59	0.18	456.84	510.38	5,416.54	100.0%
512	512	32x	1,801.03	1,787.15	0.15	917.87	1,030.92	8,910.64	100.0%
512	512	64x	3,224.56	3,182.23	0.11	1,779.75	2,147.56	9,902.88	100.0%
512	512	128x	4,536.84	4,454.27	0.08	4,009.99	4,927.05	14,207.16	100.0%
512	512	256x	4,793.07	4,759.53	0.07	13,710.91	15,211.68	26,199.92	100.0%
512	512	512x	4,950.74	4,918.40	0.06	23,814.58	29,779.46	43,340.15	100.0%
512	512	1024x	3,551.45	3,599.55	0.08	51,912.71	61,944.13	76,988.12	61.2%
512	1,024	1x	136.35	83.39	1.39	61.62	61.62	5,939.74	100.0%
512	1,024	2x	266.85	133.63	0.83	78.88	80.99	7,429.38	100.0%
512	1,024	4x	443.11	263.21	0.51	158.60	178.60	7,460.75	100.0%
512	1,024	8x	668.63	400.28	0.37	264.82	281.85	9,746.30	100.0%
512	1,024	16x	1,095.53	671.52	0.27	454.82	505.60	11,807.22	100.0%
512	1,024	32x	2,084.31	1,150.09	0.18	977.65	1,060.97	13,851.74	100.0%
512	1,024	64x	3,363.23	1,815.02	0.14	1,556.76	1,939.92	17,271.95	100.0%
512	1,024	128x	5,133.17	2,834.19	0.10	4,150.30	5,093.80	21,847.95	100.0%
512	1,024	256x	5,942.20	3,264.39	0.08	14,806.97	16,445.96	37,510.74	100.0%
512	1,024	512x	7,881.26	4,348.08	0.06	6,894.79	14,102.15	47,299.46	100.0%
512	1,024	1024x	6,752.88	3,792.25	0.06	62,981.83	88,879.02	116,021.97	97.3%
512	2,048	1x	136.45	84.28	1.41	97.28	97.28	5,877.02	100.0%
512	2,048	2x	230.21	90.86	1.04	114.41	127.91	10,761.82	100.0%
512	2,048	4x	276.05	125.47	0.82	194.61	234.01	14,850.37	100.0%
512	2,048	8x	480.60	258.46	0.51	303.32	347.75	14,007.55	100.0%
512	2,048	16x	1,026.05	535.49	0.30	485.76	530.63	12,998.07	100.0%
512	2,048	32x	1,527.40	663.55	0.24	894.79	968.14	20,960.94	100.0%
512	2,048	64x	2,646.11	1,119.46	0.17	1,595.68	1,982.21	25,318.38	100.0%
512	2,048	128x	3,761.00	1,531.67	0.13	4,073.16	5,157.92	39,091.19	100.0%
512	2,048	256x	5,495.34	2,349.42	0.10	15,633.74	17,043.38	51,384.96	100.0%
512	2,048	512x	6,657.92	2,727.58	0.08	28,838.84	33,077.74	88,549.94	100.0%
512	2,048	1024x	5,744.55	2,435.67	0.08	56,422.53	76,924.96	121,377.86	67.6%
1,024	128	1x	120.41	917.22	0.29	147.73	147.73	1,062.05	100.0%
1,024	128	2x	235.51	1,797.61	0.15	171.42	173.82	1,083.88	100.0%
1,024	128	4x	440.62	3,364.03	0.10	241.87	247.35	1,158.38	100.0%
1,024	128	8x	620.15	4,753.03	0.07	352.11	415.17	1,644.04	100.0%
1,024	128	16x	1,064.45	8,165.80	0.05	596.16	667.06	1,919.21	100.0%
1,024	128	32x	1,251.00	9,631.32	0.05	1,025.03	1,378.31	3,247.01	100.0%
1,024	128	64x	1,966.62	15,216.26	0.03	1,636.98	2,047.48	3,960.91	100.0%
1,024	128	128x	2,125.74	16,497.18	0.03	4,288.40	5,295.08	7,370.80	100.0%
1,024	128	256x	1,558.91	12,227.15	0.03	15,708.79	17,452.30	20,224.54	100.0%
1,024	512	1x	134.07	255.30	0.84	137.40	137.40	3,818.64	100.0%
1,024	512	2x	270.97	517.07	0.41	103.91	106.74	3,768.28	100.0%
1,024	512	4x	529.88	1,011.38	0.25	178.04	179.64	3,860.35	100.0%
1,024	512	8x	720.45	1,379.78	0.19	371.64	420.71	5,670.60	100.0%
1,024	512	16x	1,253.44	2,426.41	0.14	529.45	596.97	6,461.38	100.0%
1,024	512	32x	2,169.00	4,195.30	0.09	925.21	1,123.77	7,470.39	100.0%
1,024	512	64x	3,157.12	6,220.11	0.07	1,589.38	1,962.11	10,021.42	100.0%
1,024	512	128x	4,171.09	8,185.27	0.06	4,384.64	5,389.68	15,171.94	100.0%
1,024	512	256x	4,530.67	8,888.04	0.05	14,517.09	16,391.26	27,651.47	100.0%
1,024	512	512x	4,798.65	9,451.39	0.04	27,802.10	31,532.03	46,435.57	100.0%
1,024	512	1024x	4,165.11	8,326.96	0.05	55,657.95	65,893.23	80,264.89	78.7%
1,024	1,024	1x	142.88	136.04	1.11	135.54	135.54	7,166.83	100.0%
1,024	1,024	2x	205.29	258.29	0.69	162.82	164.78	7,383.72	100.0%
1,024	1,024	4x	524.27	511.45	0.38	235.81	238.77	7,581.12	100.0%
1,024	1,024	8x	809.08	928.85	0.25	344.19	405.21	8,389.74	100.0%
1,024	1,024	16x	1,052.96	1,097.75	0.22	567.71	719.25	14,301.49	100.0%
1,024	1,024	32x	2,051.10	2,149.74	0.14	928.06	1,092.74	14,596.95	100.0%
1,024	1,024	64x	3,329.70	3,502.06	0.11	1,671.52	2,112.69	17,802.89	100.0%
1,024	1,024	128x	4,659.12	4,919.23	0.08	4,096.33	5,201.22	25,327.47	100.0%
1,024	1,024	256x	5,915.93	6,140.88	0.06	13,927.69	15,846.02	40,285.21	100.0%
1,024	1,024	512x	6,705.48	6,983.05	0.05	25,274.97	30,634.48	65,183.42	100.0%
1,024	2,048	1x	143.21	103.43	1.29	114.30	114.30	9,418.16	100.0%
1,024	2,048	2x	277.42	210.36	0.70	152.25	152.90	9,277.09	100.0%
1,024	2,048	4x	451.01	274.95	0.53	204.23	206.09	14,164.14	100.0%
1,024	2,048	8x	825.70	638.28	0.31	337.98	409.19	12,180.72	100.0%
1,024	2,048	16x	1,072.31	761.93	0.26	562.80	661.79	20,606.89	100.0%
1,024	2,048	32x	1,615.04	1,196.17	0.19	1,039.98	1,293.23	26,203.05	100.0%
1,024	2,048	64x	2,862.24	2,159.94	0.14	1,618.26	2,024.20	27,331.83	100.0%
1,024	2,048	128x	4,342.44	3,372.21	0.10	4,378.97	5,483.98	34,930.26	100.0%
1,024	2,048	256x	5,533.53	4,302.69	0.08	15,978.01	17,706.67	54,805.90	100.0%
1,024	2,048	512x	6,554.26	5,138.70	0.06	30,800.51	36,045.82	90,565.54	100.0%
1,024	2,048	1024x	7,038.70	5,563.11	0.06	68,518.77	104,970.44	149,591.78	92.8%
2,048	128	1x	116.15	1,785.84	0.16	176.53	176.53	1,101.46	100.0%
2,048	128	2x	227.35	3,480.46	0.09	202.69	202.72	1,121.77	100.0%
2,048	128	4x	428.81	6,565.33	0.05	262.66	264.68	1,190.11	100.0%
2,048	128	8x	677.25	10,351.85	0.04	385.84	467.03	1,504.84	100.0%
2,048	128	16x	986.96	15,111.59	0.03	603.06	768.13	2,058.60	100.0%
2,048	128	32x	1,171.05	17,898.17	0.03	1,253.93	1,614.51	3,485.03	100.0%
2,048	128	64x	1,897.07	29,077.83	0.02	1,676.88	2,118.52	4,227.04	100.0%
2,048	128	128x	1,990.43	30,699.04	0.02	4,278.82	5,568.20	8,010.61	100.0%
2,048	128	256x	1,478.22	23,130.70	0.02	14,712.43	17,271.20	20,999.90	100.0%
2,048	512	1x	137.41	528.18	0.46	170.47	170.47	3,724.57	100.0%
2,048	512	2x	268.48	1,027.53	0.25	149.77	200.03	3,808.80	100.0%
2,048	512	4x	504.44	1,934.60	0.16	240.78	270.28	4,040.70	100.0%
2,048	512	8x	875.14	3,410.76	0.10	276.51	289.60	4,582.74	100.0%
2,048	512	16x	1,341.71	5,538.42	0.07	483.74	575.13	5,636.16	100.0%
2,048	512	32x	2,031.57	8,062.36	0.06	996.23	1,166.91	7,742.34	100.0%
2,048	512	64x	2,973.94	11,648.45	0.05	1,724.61	2,279.53	10,664.51	100.0%
2,048	512	128x	3,773.37	14,697.04	0.04	5,125.66	6,118.25	16,809.62	100.0%
2,048	512	256x	4,269.79	16,892.61	0.03	14,551.38	16,682.42	29,144.83	100.0%
2,048	512	512x	4,284.82	16,885.13	0.03	28,654.95	33,831.52	50,832.77	100.0%
2,048	512	1024x	5,010.86	19,754.35	0.03	51,550.45	66,020.76	83,404.04	99.3%
2,048	1,024	1x	132.36	350.12	0.64	171.29	171.29	5,612.67	100.0%
2,048	1,024	2x	201.29	485.14	0.46	286.93	339.32	7,929.48	100.0%
2,048	1,024	4x	365.07	993.66	0.26	227.12	271.46	7,870.78	100.0%
2,048	1,024	8x	607.91	1,854.06	0.17	410.84	507.62	7,997.20	100.0%
2,048	1,024	16x	969.53	2,655.21	0.13	684.48	887.78	11,574.09	100.0%
2,048	1,024	32x	1,838.98	4,203.41	0.09	1,235.60	1,569.60	14,859.29	100.0%
2,048	1,024	64x	2,635.55	5,895.28	0.07	1,805.72	2,676.20	21,160.98	100.0%
2,048	1,024	128x	4,317.60	9,196.18	0.06	4,588.54	5,753.50	26,931.35	100.0%
2,048	1,024	256x	5,412.73	11,663.65	0.04	14,586.03	16,318.73	41,571.09	100.0%
2,048	1,024	512x	6,106.24	13,150.44	0.04	28,453.73	32,802.86	69,524.37	100.0%
2,048	1,024	1024x	6,979.14	15,088.30	0.04	51,309.95	77,584.28	111,507.41	100.0%
2,048	2,048	1x	131.34	391.64	0.60	174.16	174.16	5,016.19	100.0%
2,048	2,048	2x	249.50	878.50	0.30	208.02	209.28	4,422.84	100.0%
2,048	2,048	4x	412.35	1,071.05	0.26	294.36	368.50	7,300.46	100.0%
2,048	2,048	8x	616.33	1,753.33	0.18	436.72	592.46	8,534.30	100.0%
2,048	2,048	16x	1,044.74	2,451.10	0.14	620.01	771.81	12,028.82	100.0%
2,048	2,048	32x	1,631.37	3,033.79	0.12	1,201.28	1,395.16	18,825.83	100.0%
2,048	2,048	64x	2,371.71	4,126.99	0.10	1,797.57	2,371.49	26,999.84	100.0%
2,048	2,048	128x	3,694.75	6,039.81	0.07	4,382.02	5,457.36	36,439.70	100.0%
2,048	2,048	256x	5,160.21	8,538.17	0.06	14,468.52	16,590.19	56,393.14	100.0%
2,048	2,048	512x	6,284.51	10,354.29	0.05	28,222.37	32,373.63	90,943.42	100.0%
2,048	2,048	1024x	6,549.34	10,651.70	0.04	67,147.69	103,330.82	146,263.04	89.2%

Hardware Configuration

GPU ManufacturerNVIDIA

GPU ModelNVIDIA H20

GPU Count8

GPU Memory (Total)760 GB

GPU Driver570.195.03

CUDA VersionUnknown

Compute Capability9.0

Power Limit (per GPU)500 W

CPU ModelIntel(R) Xeon(R) Platinum 8469C

RAM1,007 GB

Software Configuration

Inference FrameworkvLLM

Framework Versionv0.11.0

OSUbuntu

OS Version24.04.3 LTS (Noble Numbat)

Kernel Version6.8.0-79-generic

Python Version3.12.3

Model Configuration

Providerqwen

Model Nameqwen3-coder-30b-a3b-instruct

QuantizationBF16

Inference Configuration

Runtime parameters used across all benchmark runs

Max Model Length253600

Tensor Parallel Size1

Pipeline Parallel Size1

GPU Memory Utilization85.00

Temperature0.70

Top-P1.00

Top-K-1