NVIDIA H20 (8x) - llama-3.3-70b-instruct (High Throughput)

October 25, 2025 at 04:41 AM

Dataset: reference (v1.0)

NVIDIA NVIDIA H20 meta-llama llama-3.3-70b-instruct

Best Performance

Click a metric to highlight the best run in the table below

Best Output TPS

5,091.04

Peak generation speed

Best Input TPS

7,327.23

Peak prefill speed

Best Energy Efficiency

0.10 kWh/MT

Energy cost per 1M tokens

Best TTFT (P95)

39.57 ms

Lowest latency

Best E2E (P95)

3,596.85 ms

Lowest latency

Test Matrix Results

Performance across different input/output token combinations and concurrency levels

Input Tokens	Output Tokens	Concurrency	Output TPS	Input TPS	Energy Cost (kWh/MT)	TTFT Mean	TTFT P95	E2E P95	Success Rate
Best Run for Output TPS
128	512	512x	5,091.04★	1,352.77	0.14	3,242.22	4,908.41	48,317.33	100.0%
128	128	1x	31.96	30.46	6.29	655.38	655.38	4,005.14	100.0%
128	128	2x	63.43	61.94	4.33	680.25	681.16	4,035.16	100.0%
128	128	4x	141.51	140.41	2.10	148.59	220.11	3,615.85	100.0%
128	128	8x	255.74	256.49	1.66	206.02	515.26	3,879.05	100.0%
128	128	16x	487.04	490.61	0.90	401.70	639.13	4,199.69	100.0%
128	128	32x	950.48	958.19	0.47	492.39	729.45	4,268.38	100.0%
128	128	64x	1,609.75	1,602.28	0.28	721.07	1,286.97	5,053.52	100.0%
128	128	128x	2,640.10	2,648.18	0.16	1,285.49	1,981.23	6,113.96	100.0%
128	128	256x	3,337.71	3,369.64	0.12	2,363.06	3,352.89	9,646.29	100.0%
128	128	512x	3,626.31	3,660.22	0.11	4,344.18	6,514.30	17,760.21	100.0%
128	128	1024x	3,673.46	3,690.12	0.10	6,814.12	13,008.42	35,082.17	100.0%
128	512	1x	37.88	9.03	10.09	78.82	78.82	13,517.33	100.0%
128	512	2x	75.68	18.48	7.02	39.40	39.57	13,529.08	100.0%
128	512	4x	150.14	37.24	3.56	56.79	69.76	13,636.70	100.0%
128	512	8x	287.53	74.47	2.90	67.17	97.27	13,787.70	100.0%
128	512	16x	576.38	147.40	1.46	75.75	93.60	13,989.55	100.0%
128	512	32x	1,071.49	278.48	0.79	138.01	207.79	14,712.26	100.0%
128	512	64x	2,059.97	540.29	0.40	224.41	363.00	15,061.78	100.0%
128	512	128x	3,623.37	937.23	0.24	401.93	778.23	17,379.96	100.0%
128	512	256x	4,449.86	1,181.12	0.17	886.95	1,744.82	27,674.45	100.0%
128	512	1024x	4,641.72	1,223.69	0.13	6,873.22	12,796.13	104,149.47	100.0%
128	1,024	1x	37.48	7.65	10.44	84.79	84.79	15,930.78	100.0%
128	1,024	2x	74.76	13.49	7.52	66.60	91.60	18,486.58	100.0%
128	1,024	4x	115.85	20.90	4.51	129.21	169.86	23,386.28	100.0%
128	1,024	8x	225.46	42.27	3.10	146.10	198.60	23,627.26	100.0%
128	1,024	16x	487.76	86.31	1.79	265.34	383.84	23,342.79	100.0%
128	1,024	32x	824.56	144.59	1.03	369.20	502.28	26,817.27	100.0%
128	1,024	64x	1,540.91	265.09	0.57	746.74	959.64	30,616.22	100.0%
128	1,024	128x	2,662.90	452.46	0.33	1,363.42	2,140.51	34,556.07	100.0%
128	1,024	256x	4,123.53	716.81	0.21	2,006.78	3,442.23	43,859.01	100.0%
128	1,024	512x	4,619.05	805.08	0.17	4,224.95	6,922.77	80,164.53	100.0%
128	1,024	1024x	4,314.14	749.14	0.16	7,637.08	12,483.24	164,435.11	100.0%
128	2,048	1x	37.54	5.66	10.92	86.82	86.82	21,526.74	100.0%
128	2,048	2x	70.07	13.77	7.63	92.03	94.98	18,000.53	100.0%
128	2,048	4x	110.25	18.66	5.54	75.16	120.41	26,080.57	100.0%
128	2,048	8x	250.60	45.14	2.74	211.54	311.16	22,134.71	100.0%
128	2,048	16x	409.72	69.99	1.93	244.51	331.00	25,948.19	100.0%
128	2,048	32x	804.80	138.27	1.10	313.15	527.56	28,036.41	100.0%
128	2,048	64x	1,491.48	258.28	0.60	575.26	885.63	29,023.51	100.0%
128	2,048	128x	1,994.51	332.84	0.39	1,096.86	1,606.45	34,067.74	100.0%
128	2,048	256x	2,809.56	474.12	0.28	2,365.60	4,004.02	50,959.87	100.0%
128	2,048	512x	4,009.61	690.87	0.19	4,123.50	5,967.71	78,749.04	100.0%
128	2,048	1024x	4,152.95	711.41	0.17	7,573.76	12,603.72	163,617.76	100.0%
512	128	1x	35.35	136.98	2.53	293.97	293.97	3,620.82	100.0%
512	128	2x	70.91	275.62	1.69	156.03	260.19	3,596.85	100.0%
512	128	4x	129.23	499.24	0.93	428.80	569.78	3,960.32	100.0%
512	128	8x	229.24	934.47	0.77	489.09	818.18	4,224.46	100.0%
512	128	16x	418.85	1,661.99	0.41	745.81	1,322.63	4,767.35	100.0%
512	128	32x	661.47	2,617.84	0.24	1,428.74	2,389.85	6,077.71	100.0%
512	128	64x	893.54	3,476.22	0.18	2,831.54	4,808.55	9,119.60	100.0%
512	128	128x	1,364.32	5,351.04	0.12	4,901.63	7,420.42	11,821.29	100.0%
512	128	256x	1,564.32	6,099.17	0.10	7,259.02	12,814.74	20,693.58	100.0%
512	128	512x	1,572.51	6,156.15	0.10	13,884.66	28,137.18	40,946.44	100.0%
512	128	1024x	1,440.66	5,636.92	0.11	23,410.36	49,656.27	86,114.71	100.0%
512	512	1x	37.13	35.97	6.54	286.51	286.51	13,788.74	100.0%
512	512	2x	64.30	72.54	4.45	162.51	274.16	13,518.11	100.0%
512	512	4x	139.59	138.40	2.38	373.23	576.49	14,290.04	100.0%
512	512	8x	258.12	277.66	1.89	306.07	576.77	14,222.31	100.0%
512	512	16x	494.52	514.56	0.98	935.36	1,535.14	15,413.83	100.0%
512	512	32x	878.36	920.15	0.55	1,538.04	2,527.96	17,328.80	100.0%
512	512	64x	1,498.89	1,571.58	0.31	2,930.66	4,496.83	20,228.50	100.0%
512	512	128x	2,388.12	2,489.77	0.19	4,415.39	7,516.45	25,485.54	100.0%
512	512	256x	2,890.76	2,968.52	0.15	7,428.54	13,301.30	42,722.05	100.0%
512	512	512x	3,180.80	3,253.67	0.13	13,171.73	25,322.57	77,867.71	100.0%
512	512	1024x	2,506.45	2,553.11	0.16	28,165.04	54,070.63	192,947.53	100.0%
512	1,024	1x	36.92	44.12	5.82	285.92	285.92	11,215.67	100.0%
512	1,024	2x	58.13	46.72	5.59	287.50	290.10	20,800.53	100.0%
512	1,024	4x	123.73	90.75	3.67	357.04	525.89	21,144.34	100.0%
512	1,024	8x	228.07	160.76	2.55	329.49	548.23	23,916.08	100.0%
512	1,024	16x	377.62	316.41	1.39	1,279.98	2,075.04	24,375.76	100.0%
512	1,024	32x	689.84	512.47	0.82	1,643.32	2,578.93	29,582.76	100.0%
512	1,024	64x	1,345.07	930.95	0.45	2,743.34	3,605.31	32,939.21	100.0%
512	1,024	128x	2,275.69	1,540.90	0.26	4,442.15	7,089.42	38,586.73	100.0%
512	1,024	256x	3,086.75	2,099.51	0.18	7,451.46	13,244.25	58,879.22	100.0%
512	1,024	512x	3,335.07	2,278.74	0.15	12,490.05	25,172.83	108,502.25	100.0%
512	1,024	1024x	2,879.88	1,958.77	0.18	26,836.48	53,000.30	228,172.63	91.8%
512	2,048	1x	36.96	37.88	6.28	289.44	289.44	13,067.99	100.0%
512	2,048	2x	62.38	52.07	4.08	436.15	559.32	18,810.11	100.0%
512	2,048	4x	125.81	90.33	2.91	503.21	834.04	21,515.70	100.0%
512	2,048	8x	225.08	173.82	2.46	649.58	1,073.45	22,360.50	100.0%
512	2,048	16x	408.73	323.59	1.34	768.43	1,095.91	22,676.33	100.0%
512	2,048	32x	628.02	464.54	0.85	1,450.53	2,081.97	28,378.63	100.0%
512	2,048	64x	990.05	655.07	0.57	2,565.14	4,140.69	37,018.87	100.0%
512	2,048	128x	1,857.68	1,245.47	0.32	4,980.35	8,220.98	42,526.96	100.0%
512	2,048	256x	2,898.77	1,945.30	0.20	7,682.35	13,720.56	57,380.99	100.0%
512	2,048	512x	3,061.74	2,049.44	0.17	13,702.84	28,078.69	107,297.82	100.0%
512	2,048	1024x	2,815.44	1,906.34	0.18	28,047.96	53,739.10	230,007.99	93.0%
1,024	128	1x	32.74	249.36	1.60	542.71	542.71	3,910.35	100.0%
1,024	128	2x	65.76	501.93	1.02	282.15	498.14	3,866.52	100.0%
1,024	128	4x	115.21	879.61	0.73	670.54	989.13	4,437.61	100.0%
1,024	128	8x	185.84	1,421.60	0.46	1,238.74	2,063.19	5,505.41	100.0%
1,024	128	16x	268.27	2,058.03	0.31	2,170.81	4,094.48	7,628.32	100.0%
1,024	128	32x	461.89	3,543.87	0.19	3,049.03	4,791.31	8,838.84	100.0%
1,024	128	64x	710.05	5,475.63	0.13	4,477.93	7,548.60	11,447.39	100.0%
1,024	128	128x	800.33	6,223.37	0.11	7,766.37	14,180.42	20,107.50	100.0%
1,024	128	256x	893.08	6,881.62	0.10	14,007.65	27,174.16	36,345.72	100.0%
1,024	128	512x	824.76	6,349.84	0.10	23,632.88	50,470.32	75,547.97	100.0%
1,024	512	1x	37.68	71.76	4.29	40.56	40.56	13,587.23	100.0%
1,024	512	2x	72.69	138.70	3.14	536.68	546.15	14,084.38	100.0%
1,024	512	4x	139.12	265.54	2.04	555.23	1,004.19	14,714.71	100.0%
1,024	512	8x	278.15	531.92	1.27	499.02	904.84	14,712.07	100.0%
1,024	512	16x	480.74	925.16	0.57	1,636.23	2,566.09	16,976.51	100.0%
1,024	512	32x	789.79	1,561.75	0.39	3,099.84	4,733.44	20,089.62	100.0%
1,024	512	64x	1,305.77	2,616.20	0.24	4,515.93	8,012.44	24,005.21	100.0%
1,024	512	128x	1,761.77	3,487.55	0.16	8,108.65	17,396.28	35,975.69	100.0%
1,024	512	256x	2,328.96	4,699.47	0.13	13,282.48	26,046.76	53,327.14	100.0%
1,024	512	512x	1,788.44	3,561.24	0.15	24,492.27	50,562.93	130,532.06	100.0%
1,024	1,024	1x	36.85	47.52	5.66	548.44	548.44	20,491.23	100.0%
1,024	1,024	2x	66.50	90.23	4.05	534.08	540.37	21,421.09	100.0%
1,024	1,024	4x	117.17	139.47	2.95	677.13	1,002.85	27,010.82	100.0%
1,024	1,024	8x	222.93	273.92	1.82	626.63	1,083.05	27,602.96	100.0%
1,024	1,024	16x	408.04	516.45	1.07	1,689.28	2,599.37	28,819.99	100.0%
1,024	1,024	32x	698.25	951.73	0.58	3,160.99	5,081.88	30,780.86	100.0%
1,024	1,024	64x	1,040.64	1,476.13	0.36	4,773.31	9,940.71	36,402.73	100.0%
1,024	1,024	128x	1,763.66	2,460.29	0.22	8,016.87	15,749.09	47,380.27	100.0%
1,024	1,024	256x	2,389.05	3,311.01	0.16	13,359.51	27,079.09	72,277.92	100.0%
1,024	1,024	512x	1,948.45	2,672.76	0.17	27,056.67	54,715.33	172,337.42	99.4%
1,024	2,048	1x	37.57	44.78	5.78	43.87	43.87	21,745.95	100.0%
1,024	2,048	2x	69.27	80.42	3.08	809.77	1,039.21	24,182.59	100.0%
1,024	2,048	4x	120.39	129.33	3.16	553.25	1,006.86	29,565.27	100.0%
1,024	2,048	8x	192.97	245.86	1.70	1,134.50	1,584.65	29,708.61	100.0%
1,024	2,048	16x	427.80	553.18	0.98	1,082.86	2,093.34	27,684.94	100.0%
1,024	2,048	32x	735.47	926.29	0.59	3,239.53	4,758.68	31,547.43	100.0%
1,024	2,048	64x	881.92	1,193.99	0.40	4,786.88	9,441.73	36,386.14	100.0%
1,024	2,048	128x	1,638.60	2,236.08	0.24	7,405.04	13,569.22	49,507.26	100.0%
1,024	2,048	256x	1,988.19	2,684.38	0.20	13,300.84	25,924.81	76,151.15	100.0%
1,024	2,048	512x	1,965.59	2,647.54	0.18	27,077.17	53,839.95	172,093.52	100.0%
2,048	128	1x	28.93	444.85	0.88	1,053.80	1,053.80	4,423.98	100.0%
2,048	128	2x	57.67	882.86	0.63	556.45	1,020.32	4,387.65	100.0%
2,048	128	4x	93.16	1,426.31	0.47	1,324.95	1,959.17	5,490.84	100.0%
2,048	128	8x	157.76	2,411.34	0.31	1,562.58	3,060.50	6,487.84	100.0%
2,048	128	16x	208.72	3,188.04	0.20	3,023.97	5,523.85	9,786.82	100.0%
2,048	128	32x	288.33	4,425.18	0.15	5,191.05	9,548.23	14,093.78	100.0%
2,048	128	64x	413.50	6,317.89	0.11	7,354.53	14,837.74	19,723.12	100.0%
2,048	128	128x	475.69	7,327.23	0.10	14,010.51	26,377.06	34,039.10	100.0%
2,048	128	256x	413.16	6,335.13	0.10	25,180.91	52,822.09	75,279.78	100.0%
2,048	512	1x	35.02	134.62	2.72	1,050.57	1,050.57	14,618.82	100.0%
2,048	512	2x	70.05	268.08	1.93	551.17	1,010.70	14,567.20	100.0%
2,048	512	4x	120.67	461.88	1.07	2,084.89	3,114.64	16,970.04	100.0%
2,048	512	8x	228.01	955.50	0.82	1,581.73	2,111.10	16,379.23	100.0%
2,048	512	16x	412.41	1,725.37	0.44	1,816.63	4,038.06	18,117.24	100.0%
2,048	512	32x	618.93	2,573.60	0.28	4,109.45	9,073.31	24,289.08	100.0%
2,048	512	64x	857.89	3,465.54	0.19	7,506.41	17,590.37	36,059.16	100.0%
2,048	512	128x	1,180.81	4,718.03	0.14	14,188.77	29,733.34	52,919.76	100.0%
2,048	512	256x	1,038.05	4,136.62	0.14	25,549.22	56,287.16	106,358.84	100.0%
2,048	1,024	1x	37.67	110.15	3.21	43.06	43.06	17,840.48	100.0%
2,048	1,024	2x	70.29	250.21	2.03	1,062.09	1,065.29	15,633.99	100.0%
2,048	1,024	4x	110.97	367.80	1.59	301.69	919.19	20,611.84	100.0%
2,048	1,024	8x	202.23	634.84	0.89	2,061.01	3,093.26	23,909.93	100.0%
2,048	1,024	16x	340.14	1,041.69	0.69	2,549.58	4,041.84	28,692.34	100.0%
2,048	1,024	32x	601.09	1,830.50	0.40	4,820.85	9,498.68	32,703.51	100.0%
2,048	1,024	64x	842.61	2,441.16	0.27	8,585.67	17,927.10	46,991.12	100.0%
2,048	1,024	128x	1,395.01	3,878.52	0.17	11,421.69	26,542.36	61,362.97	100.0%
2,048	1,024	256x	1,567.25	4,345.31	0.15	25,528.26	51,688.54	111,218.90	100.0%
2,048	1,024	512x	1,387.57	3,845.88	0.17	39,537.89	113,625.21	146,779.67	59.4%
2,048	2,048	1x	35.83	86.10	3.85	1,052.66	1,052.66	22,829.81	100.0%
2,048	2,048	2x	63.74	177.40	1.95	1,581.05	2,045.12	21,946.25	100.0%
2,048	2,048	4x	95.43	283.90	1.83	1,067.78	1,942.16	25,996.98	100.0%
2,048	2,048	8x	211.02	637.50	0.95	1,947.80	3,074.51	24,456.17	100.0%
2,048	2,048	16x	183.63	512.69	1.00	3,007.91	5,062.20	37,167.61	100.0%

Hardware Configuration

GPU ManufacturerNVIDIA

GPU ModelNVIDIA H20

GPU Count8

GPU Memory (Total)760 GB

GPU Driver570.195.03

CUDA VersionUnknown

Compute Capability9.0

Power Limit (per GPU)500 W

CPU ModelIntel(R) Xeon(R) Platinum 8469C

RAM1,007 GB

Software Configuration

Inference FrameworkvLLM

Framework Version0.9.0

OSUbuntu

OS Version24.04.3 LTS (Noble Numbat)

Kernel Version6.8.0-79-generic

Python Version3.12.3

Model Configuration

Providermeta-llama

Model Namellama-3.3-70b-instruct

QuantizationBF16

Inference Configuration

Runtime parameters used across all benchmark runs

Max Model Length8192

Tensor Parallel Size1

Pipeline Parallel Size1

GPU Memory Utilization0.95

Temperature0.70

Top-P1.00

Top-K-1