NVIDIA A100 80GB PCIe (2x) - gemma-3-27b-it

November 2, 2025 at 09:37 PM

Dataset: reference (v1.0)

NVIDIA NVIDIA A100 80GB PCIe google gemma-3-27b-it

Best Performance

Click a metric to highlight the best run in the table below

Best Output TPS

1,834.30

Peak generation speed

Best Input TPS

4,909.53

Peak prefill speed

Best Energy Efficiency

0.03 kWh/MT

Energy cost per 1M tokens

Best TTFT (P95)

80.32 ms

Lowest latency

Best E2E (P95)

5,447.75 ms

Lowest latency

Test Matrix Results

Performance across different input/output token combinations and concurrency levels

Input Tokens	Output Tokens	Concurrency	Output TPS	Input TPS	Energy Cost (kWh/MT)	TTFT Mean	TTFT P95	E2E P95	Success Rate
Best Run for Output TPS
128	128	256x	1,834.30★	1,837.61	0.04	4,777.46	9,628.89	17,018.42	100.0%
128	128	1x	13.62	12.98	2.49	4,152.62	4,152.62	9,396.89	100.0%
128	128	2x	26.86	26.23	1.38	4,320.55	4,354.14	9,528.95	100.0%
128	128	4x	92.74	92.01	0.79	308.88	344.66	5,513.05	100.0%
128	128	8x	184.54	185.08	0.39	250.83	309.54	5,545.32	100.0%
128	128	16x	342.93	345.45	0.20	400.06	516.97	5,957.65	100.0%
128	128	32x	586.23	587.09	0.12	672.08	731.57	6,449.38	100.0%
128	128	64x	1,045.43	1,040.58	0.07	883.79	1,202.04	7,684.86	100.0%
128	128	128x	1,164.33	1,164.19	0.06	1,273.82	2,055.39	9,145.84	100.0%
128	128	512x	1,577.36	1,579.82	0.05	13,534.38	26,816.81	34,711.40	100.0%
128	512	1x	24.10	5.74	3.41	297.84	297.84	21,245.41	100.0%
128	512	2x	47.33	11.56	2.67	474.73	637.56	21,617.18	100.0%
128	512	4x	94.62	23.47	1.35	330.53	382.47	21,637.97	100.0%
128	512	8x	178.36	46.87	0.71	320.94	390.94	21,900.00	100.0%
128	512	16x	347.33	89.59	0.37	400.96	486.85	22,980.55	100.0%
128	512	32x	664.91	167.75	0.19	548.04	649.37	24,434.30	100.0%
128	512	64x	1,192.09	298.29	0.11	812.66	1,020.05	27,259.75	100.0%
128	512	128x	1,674.19	418.42	0.08	1,141.31	1,711.12	37,835.43	100.0%
128	512	256x	1,426.00	357.81	0.09	18,357.92	36,390.40	69,700.54	98.8%
128	1,024	1x	23.93	2.85	3.75	299.56	299.56	42,786.41	100.0%
128	1,024	2x	47.89	5.85	1.90	301.09	326.00	42,756.49	100.0%
128	1,024	4x	94.18	11.68	1.50	263.52	274.18	43,486.01	100.0%
128	1,024	8x	170.63	23.25	0.83	206.66	254.05	44,163.08	100.0%
128	1,024	16x	345.59	45.11	0.42	277.85	317.41	45,714.38	100.0%
128	1,024	32x	629.70	81.61	0.22	436.29	594.72	50,201.52	100.0%
128	1,024	64x	1,114.99	141.91	0.13	596.76	901.81	57,353.71	100.0%
128	1,024	128x	1,057.38	133.75	0.13	1,969.37	2,075.37	103,104.07	100.0%
128	2,048	1x	24.02	2.72	3.81	80.32	80.32	44,800.66	100.0%
128	2,048	2x	42.00	4.11	2.22	148.05	181.66	60,052.09	100.0%
128	2,048	4x	85.90	7.24	1.73	144.02	180.23	69,989.56	100.0%
128	2,048	8x	130.30	11.61	1.04	200.22	237.81	82,964.39	100.0%
128	2,048	16x	289.48	25.38	0.50	332.71	426.89	80,950.15	100.0%
128	2,048	32x	503.00	43.17	0.30	405.57	587.87	90,677.70	100.0%
128	2,048	64x	870.55	73.95	0.17	583.43	802.92	105,871.88	100.0%
128	2,048	128x	971.14	80.39	0.15	2,108.37	1,884.70	186,177.52	100.0%
128	2,048	256x	976.11	81.74	0.15	10,353.29	120,575.72	203,997.26	54.7%
512	128	1x	23.49	91.04	0.84	176.10	176.10	5,447.75	100.0%
512	128	2x	45.85	178.19	0.43	243.29	307.70	5,579.84	100.0%
512	128	4x	91.44	353.28	0.33	178.65	230.44	5,597.27	100.0%
512	128	8x	174.45	672.91	0.18	385.87	462.19	5,865.29	100.0%
512	128	16x	302.29	1,171.37	0.10	658.67	1,037.68	6,760.84	100.0%
512	128	32x	543.38	2,117.41	0.06	775.69	1,526.57	7,467.25	100.0%
512	128	64x	795.80	3,095.98	0.04	1,531.87	2,890.21	10,059.09	100.0%
512	128	128x	814.23	3,163.50	0.04	2,190.30	4,982.79	16,272.00	100.0%
512	128	256x	858.70	3,336.16	0.04	11,657.46	22,272.12	32,641.36	100.0%
512	128	512x	880.52	3,420.79	0.04	29,592.03	56,120.52	67,103.07	100.0%
512	128	1024x	925.95	3,595.35	0.04	59,996.64	117,297.72	128,748.46	95.9%
512	512	1x	23.86	23.12	2.16	168.88	168.88	21,457.16	100.0%
512	512	2x	47.66	46.31	1.66	167.36	167.74	21,483.96	100.0%
512	512	4x	92.48	89.32	0.85	298.04	452.46	22,139.55	100.0%
512	512	8x	183.30	176.77	0.45	290.36	449.62	22,340.59	100.0%
512	512	16x	334.03	332.02	0.24	572.37	675.39	23,892.49	100.0%
512	512	32x	597.50	592.38	0.13	843.43	1,486.96	26,855.45	100.0%
512	512	64x	1,026.14	1,003.75	0.08	1,371.07	2,736.96	31,549.84	100.0%
512	512	128x	935.23	912.51	0.09	2,830.78	5,351.42	65,564.65	100.0%
512	1,024	1x	23.76	11.51	2.92	170.54	170.54	43,102.69	100.0%
512	1,024	2x	47.31	22.99	1.43	173.78	237.96	43,282.33	100.0%
512	1,024	4x	94.76	45.76	1.11	236.76	314.67	43,216.00	100.0%
512	1,024	8x	179.70	87.48	0.61	403.71	481.84	45,147.94	100.0%
512	1,024	16x	329.22	165.21	0.32	547.92	877.88	47,987.78	100.0%
512	1,024	32x	571.31	294.00	0.18	1,074.72	1,746.99	54,188.76	100.0%
512	1,024	64x	870.59	431.96	0.12	1,334.03	2,813.22	65,613.62	100.0%
512	1,024	128x	934.32	463.19	0.12	5,335.74	7,645.09	133,033.10	100.0%
512	1,024	256x	921.12	459.16	0.12	25,354.67	81,778.51	185,696.85	69.1%
512	2,048	1x	23.69	7.53	3.26	185.72	185.72	65,793.64	100.0%
512	2,048	2x	44.65	13.39	2.62	128.52	165.95	73,836.81	100.0%
512	2,048	4x	81.40	24.70	1.42	271.22	348.10	78,430.80	100.0%
512	2,048	8x	165.92	53.63	0.71	432.50	594.92	73,406.66	100.0%
512	2,048	16x	267.86	102.42	0.44	717.33	974.16	76,198.84	100.0%
512	2,048	32x	487.80	170.87	0.24	706.81	1,286.12	88,675.57	100.0%
512	2,048	64x	812.15	272.68	0.15	1,503.61	2,944.34	108,729.82	100.0%
512	2,048	128x	864.44	290.59	0.14	4,263.04	7,552.00	202,964.31	100.0%
512	2,048	256x	898.40	302.18	0.14	4,281.60	7,631.59	202,180.03	50.0%
1,024	128	1x	22.58	172.02	0.50	315.86	315.86	5,665.77	100.0%
1,024	128	2x	45.09	344.20	0.37	203.01	300.89	5,663.92	100.0%
1,024	128	4x	86.05	656.97	0.20	450.45	588.55	5,948.06	100.0%
1,024	128	8x	161.06	1,231.99	0.10	621.17	836.34	6,353.42	100.0%
1,024	128	16x	267.71	2,053.73	0.06	861.28	1,602.45	7,598.02	100.0%
1,024	128	32x	420.23	3,224.27	0.04	1,295.67	2,772.01	9,602.70	100.0%
1,024	128	64x	629.52	4,833.94	0.03	2,081.42	4,853.40	12,804.67	100.0%
1,024	128	128x	513.54	3,943.83	0.04	11,855.37	23,113.43	31,341.12	100.0%
1,024	512	1x	23.44	44.64	1.50	308.17	308.17	21,839.53	100.0%
1,024	512	2x	46.91	89.51	1.14	294.35	296.40	21,828.25	100.0%
1,024	512	4x	92.67	176.89	0.58	328.16	533.96	22,090.17	100.0%
1,024	512	8x	174.53	333.76	0.31	595.66	852.80	23,447.66	100.0%
1,024	512	16x	324.44	622.22	0.17	853.21	1,588.12	25,216.93	100.0%
1,024	512	32x	559.05	1,083.73	0.10	1,367.74	2,734.65	28,880.21	100.0%
1,024	512	64x	729.31	1,400.04	0.07	2,346.53	5,184.35	38,222.52	100.0%
1,024	512	128x	807.84	1,572.45	0.07	23,111.18	48,167.98	79,419.21	100.0%
1,024	512	256x	708.50	1,367.12	0.08	44,510.30	88,790.50	123,295.98	78.5%
1,024	1,024	1x	23.53	22.40	2.20	291.82	291.82	43,520.17	100.0%
1,024	1,024	2x	46.98	44.82	1.11	330.81	351.66	43,589.89	100.0%
1,024	1,024	4x	93.41	89.15	0.86	436.34	571.71	43,834.64	100.0%
1,024	1,024	8x	180.42	172.52	0.45	639.99	870.24	45,401.79	100.0%
1,024	1,024	16x	327.79	322.33	0.25	830.58	1,235.10	48,711.97	100.0%
1,024	1,024	32x	573.71	565.40	0.14	1,469.48	2,801.18	55,474.69	100.0%
1,024	1,024	64x	727.69	720.48	0.10	2,757.07	5,901.90	69,010.16	100.0%
1,024	1,024	128x	753.78	750.76	0.10	37,507.42	79,812.52	145,994.24	100.0%
1,024	1,024	256x	723.89	719.74	0.11	39,612.82	81,305.35	147,599.46	52.3%
1,024	2,048	1x	23.54	13.44	2.77	299.43	299.43	72,521.45	100.0%
1,024	2,048	2x	43.19	26.96	2.13	286.70	289.51	71,812.95	100.0%
1,024	2,048	4x	86.91	50.51	1.12	395.48	586.62	77,264.13	100.0%
1,024	2,048	8x	142.22	86.59	0.69	682.59	1,092.63	87,586.86	100.0%
1,024	2,048	16x	275.83	179.62	0.35	873.09	1,673.97	82,485.92	100.0%
1,024	2,048	32x	470.11	305.88	0.21	1,369.04	2,774.19	101,004.87	100.0%
1,024	2,048	64x	730.31	496.25	0.13	4,429.61	7,985.87	121,306.76	100.0%
1,024	2,048	128x	744.37	508.42	0.13	41,455.41	114,732.23	203,274.04	85.9%
2,048	128	1x	21.19	325.72	0.27	571.48	571.48	6,042.43	100.0%
2,048	128	2x	43.21	661.55	0.20	321.52	519.45	5,905.05	100.0%
2,048	128	4x	72.46	1,109.40	0.11	930.76	1,479.19	7,055.29	100.0%
2,048	128	8x	125.12	1,912.51	0.07	1,224.39	2,410.81	8,159.45	100.0%
2,048	128	16x	240.91	3,679.68	0.04	1,271.11	2,282.46	8,450.72	100.0%
2,048	128	32x	321.38	4,909.53	0.03	2,664.85	5,749.00	12,581.70	100.0%
2,048	128	64x	306.48	4,682.22	0.03	7,446.95	19,557.91	25,754.00	100.0%
2,048	512	1x	23.02	88.50	0.93	539.77	539.77	22,234.93	100.0%
2,048	512	2x	45.33	173.48	0.46	799.16	1,027.07	22,587.95	100.0%
2,048	512	4x	90.34	345.77	0.36	590.32	1,003.98	22,661.48	100.0%
2,048	512	8x	160.83	626.81	0.19	1,113.80	1,931.38	24,949.99	100.0%
2,048	512	16x	283.72	1,133.90	0.11	1,644.42	3,126.38	27,553.98	100.0%
2,048	512	32x	485.30	1,936.61	0.07	2,680.54	5,600.67	32,087.26	100.0%
2,048	512	64x	473.82	1,845.56	0.07	10,150.89	42,856.09	66,706.49	100.0%
2,048	1,024	1x	23.31	44.79	1.51	545.80	545.80	43,934.24	100.0%
2,048	1,024	2x	46.71	89.38	1.16	538.24	546.20	43,840.09	100.0%
2,048	1,024	4x	80.10	178.74	0.57	702.97	1,018.16	43,848.00	100.0%
2,048	1,024	8x	144.96	341.13	0.33	910.54	1,404.02	45,856.48	100.0%
2,048	1,024	16x	265.75	637.11	0.18	1,075.65	2,277.95	49,021.03	100.0%
2,048	1,024	32x	505.58	1,103.95	0.10	1,684.10	4,409.18	56,491.28	100.0%
2,048	1,024	64x	534.98	1,130.36	0.09	11,317.89	44,270.66	97,610.50	100.0%
2,048	1,024	128x	666.51	1,370.22	0.08	44,801.77	90,058.54	153,034.36	93.0%
2,048	1,024	256x	661.72	1,348.64	0.08	45,463.57	89,444.37	152,945.21	46.5%
2,048	2,048	1x	23.37	29.88	1.91	559.10	559.10	65,812.09	100.0%
2,048	2,048	2x	41.91	59.83	1.45	544.45	552.85	64,769.45	100.0%
2,048	2,048	4x	69.81	133.43	0.79	708.07	1,082.78	58,450.36	100.0%
2,048	2,048	8x	110.21	212.48	0.46	1,095.70	1,894.30	69,456.32	100.0%
2,048	2,048	16x	203.71	368.77	0.27	1,337.30	2,820.31	81,592.39	100.0%
2,048	2,048	32x	360.93	576.84	0.16	3,500.17	6,652.96	105,053.43	100.0%
2,048	2,048	64x	560.51	832.70	0.11	11,440.95	41,217.92	129,621.59	100.0%
2,048	2,048	128x	608.21	900.75	0.11	42,563.21	115,326.01	195,998.54	77.3%

Hardware Configuration

GPU ManufacturerNVIDIA

GPU ModelNVIDIA A100 80GB PCIe

GPU Count2

GPU Memory (Total)160 GB

GPU Driver570.195.03

CUDA VersionUnknown

Compute Capability8.0

Power Limit (per GPU)300 W

CPU ModelIntel Xeon Processor (Icelake)

RAM31 GB

Software Configuration

Inference FrameworkvLLM

Framework Version0.11.0

OSUbuntu

OS Version22.04.5 LTS (Jammy Jellyfish)

Kernel Version5.15.0-88-generic

Python Version3.10.12

Model Configuration

Providergoogle

Model Namegemma-3-27b-it

QuantizationBF16

Inference Configuration

Runtime parameters used across all benchmark runs

Max Model Length8192

Tensor Parallel Size1

Pipeline Parallel Size1

GPU Memory Utilization0.90

Temperature0.70

Top-P1.00

Top-K-1