\documentclass[varwidth=598pt]{standalone}

\usepackage{booktabs} % professional-quality tables
\usepackage{multirow} % multi-row cells in tables
\usepackage{colortbl} % color for tables (\cellcolor, \rowcolor)
\usepackage[table]{xcolor} % enhanced colors for tables
\usepackage{array} % more flexible column formats (often used)
\usepackage{tabularx} % for tables with auto-stretch columns (if used)
\usepackage{graphicx} % for images in tables or figure/table floats
\usepackage{amssymb} % math symbols
\usepackage{amsmath} % math environments
\usepackage{soul} % highlighting (used for colored text/cells)
\usepackage[normalem]{ulem} % underlining, strikethroughs
\usepackage[T1]{fontenc} % font encoding
\usepackage[utf8]{inputenc}% input encoding (legacy, fine for pdflatex)
\usepackage{microtype} % better text appearance
\usepackage{hyperref} % hyperlinks
\usepackage{textcomp} % for extra symbols
\usepackage{enumitem} % for compact lists (if used in table notes)
\usepackage{adjustbox}
\usepackage{tabu}
\usepackage{pifont} % http://ctan.org/pkg/pifont
\usepackage{bbding} % \XSolidBrush
\usepackage{makecell}

\begingroup
\makeatletter
\renewcommand{\fnum@table}{}%
\long\def\@makecaption#1#2{%
\vskip\abovecaptionskip
\centering #2\par
\vskip\belowcaptionskip
}
\makeatother

\begin{document}
\begin{table}
\centering
\begin{adjustbox}{max width=\textwidth}
\begin{tabular}{p{2.2cm}<{}|p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}|p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}|p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{0.75cm}<{\centering}p{1.75cm}<{\centering}}
\toprule[1.5pt]
\multicolumn{2}{c|}{}&\multicolumn{7}{c|}{\textbf{Natural}}&\multicolumn{4}{c|}{\textbf{Specialized}}&\multicolumn{5}{c|}{\textbf{Structured}}&\\
&\multicolumn{1}{c|}{{\rotatebox[origin=c]{90}{\;Memory (MB)}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Caltech101}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Cifar100}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{DTD}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Flower102}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Pets}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{SVHN}}}
&\multicolumn{1}{c|}{{\rotatebox[origin=c]{90}{Sun397}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Camelyon}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{EuroSAT}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Resisc45}}}
&\multicolumn{1}{c|}{{\rotatebox[origin=c]{90}{Retinopathy}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Clevr-Count}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{DMLab}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{KITTI-Dist}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{sNORB-Azim}}}
&\multicolumn{1}{c|}{{\rotatebox[origin=c]{90}{sNORB-Ele}}}
&\multicolumn{1}{c}{{\rotatebox[origin=c]{90}{Average}}}\\
\hline
\specialrule{0em}{1pt}{1pt}
\multicolumn{19}{l}{\emph{Method}}\\
\hline
\specialrule{0em}{1pt}{1pt}
Full tuning & 4.25 & 89.4 & 53.3 & 66.1 & 97.3 & 87.3 & 90.7 & 39.2 & 83.2 & 95.3 & 86.1 & 75.4 & 62.8 & 47.2 & 77.5 & 31.2 & 32.8 & 69.7 \\
\wours & 4.02 & 89.9 & 55.9 & 67.8 & 97.2 & 88.4 & 90.4 & 38.9 & 85.8 & 95.8 & 86.7 & 75.7 & 74.7 & 50.2 & 77.9 & 31.8 & 31.6 & 71.2 ($\uparrow$ 1.5) \\
Linear probing & 1.84 & 41.6 & 86.4 & 65.9 & 97.6 & 87.2 & 36.8 & 51.1 & 79.0 & 88.4 & 72.9 & 74.0 & 34.1 & 34.8 & 59.6 & 13.2 & 22.9 & 59.1 \\
\midrule
SSF & 4.13 & 89.4 & 74.0 & 72.9 & 99.2 & 91.1 & 80.7 & 56.0 & 83.3 & 94.8 & 85.3 & 75.6 & 78.5 & 45.0 & 76.9 & 23.0 & 36.9 & 72.7 \\
\wours & 3.46 & 89.1 & 74.1 & 73.0 & 99.1 & 91.3 & 80.8 & 56.3 & 82.8 & 94.9 & 85.4 & 74.8 & 78.6 & 44.7 & 75.5 & 24.6 & 36.5 & 72.6 ($\downarrow$ 0.1) \\
Hydra & 3.10 & 91.3 & 72.6 & 70.9 & 99.2 & 91.3 & 88.6 & 55.7 & 82.3 & 95.2 & 85.1 & 76.1 & 81.9 & 51.7 & 78.9 & 34.5 & 40.5 & 74.7 \\
\wours & 2.88 & 91.0 & 72.8 & 70.6 & 99.2 & 91.4 & 88.2 & 56.0 & 83.2 & 94.9 & 84.3 & 75.9 & 82.7 & 51.6 & 79.9 & 34.2 & 41.4 & 74.8 ($\uparrow$ 0.1) \\
LoRA & 2.86 & 89.3 & 64.7 & 68.8 & 99.1 & 90.0 & 82.3 & 52.6 & 81.7 & 95.3 & 83.7 & 74.4 & 80.4 & 47.3 & 77.9 & 28.0 & 38.1 & 72.1 \\
\wours & 2.74 & 88.9 & 67.3 & 69.6 & 99.1 & 90.7 & 83.5 & 53.3 & 81.9 & 95.2 & 83.4 & 74.3 & 79.8 & 47.1 & 78.9 & 29.7 & 40.3 & 72.7 ($\uparrow$ 0.6) \\
\bottomrule
\end{tabular}%
\end{adjustbox}
\caption{Table 1 : Results on a subset of the VTAB-1k benchmark. All methods use a ViT-Base-224/16 model pre-trained on ImageNet-21k. The batch sizes and ranks are the same across all tasks.}
\end{table}
\end{document}