\documentclass[tikz,border=5pt]{standalone}
\usepackage{xcolor}
\usepackage{amsmath}
\usepackage{tikz}
\usetikzlibrary{shapes,arrows.meta,fit}
\definecolor{darkblue}{RGB}{39,64,139}
\definecolor{lightgreen}{RGB}{144,238,144}
\tikzstyle{rect} = [rectangle, draw=black, thick, rounded corners, minimum width=1.5cm, minimum height=0.5cm, text centered, fill=darkblue, text=white, align=center]
\tikzstyle{rect2} = [rectangle, draw=black, thick, rounded corners, minimum width=1.5cm, minimum height=0.5cm, text centered, fill=lightgreen, text=black, align=center]
\tikzstyle{rectd} = [rectangle, draw=black, dashed, thick, rounded corners, minimum width=1.5cm, minimum height=0.5cm, text centered, inner sep=0.1cm]
\tikzstyle{circ} = [circle, draw=black, thick, minimum size=0.8cm, text centered]
\begin{document}
\begin{tikzpicture}[>=stealth, node distance=1.5cm]
\definecolor{darkblue}{RGB}{39,64,139}
\definecolor{lightgreen}{RGB}{144,238,144}
\tikzstyle{rect} = [rectangle, draw=black, thick, rounded corners, minimum width=1.5cm, minimum height=0.5cm, text centered, fill=darkblue, text=white, align=center]
\tikzstyle{rect2} = [rectangle, draw=black, thick, rounded corners, minimum width=1.5cm, minimum height=0.5cm, text centered, fill=lightgreen, text=black, align=center]
\tikzstyle{rectd} = [rectangle, draw=black, dashed, thick, rounded corners, minimum width=1.5cm, minimum height=0.5cm, text centered, inner sep=0.1cm]
\tikzstyle{circ} = [circle, draw=black, thick, minimum size=0.8cm, text centered]
\node (a1) [rect] {Layer-norm};
\node (a2) [rect, right of=a1, xshift=0.7cm] {Window\\Shifting};
\node (a3) [circ, right of=a2] {$Q$};
\node (ida) [at=(a1.west), yshift=0.7cm, xshift=1.0cm, align=right] {Fused Op \#1};
\node (rect1) [rectd, fit=(a1)(a2)(a3)(ida)] {};
\node (b) [rect2, right of=a3, xshift=0.4cm] {QKV\\Gemm};
\node (c1) [circ, right of = b, xshift=0.4cm] {$dQ$};
\node (c2) [rect, right of=c1] {QKV\\Bias};
\node (c3) [circ, right of=c2] {$Q$};
\node (idc) [at=(c1.west), yshift=0.7cm, xshift=1.0cm, align=left] {Fused Op \#2};
\node (rect2) [rectd, fit=(c1)(c2)(c3)(idc)] {};
\node (d1) [rect2, below of=a1, xshift=-2cm, yshift=-0.6cm] {$Q \cdot K$\\Gemm};
\node (d2) [circ, right of=d1] {$dQ$};
\node (d3) [rect, right of=d2] {Softmax \&\\Pos. Bias};
\node (d4) [circ, right of=d3] {$Q$};
\node (idd) [at=(d2.west), yshift=0.7cm, xshift=1.0cm, align=left] {Fused Op \#3};
\node (rect3) [rectd, fit=(d2)(d3)(d4)(idd)] {};
\node (d5) [rect2, right of=d4, xshift = -0.1cm] {$V \cdot \textrm{att}$\\Gemm};
\node (e1) [rect2, right of=d5, xshift=0.3cm] {Proj\\Gemm};
\node (f1) [circ, right of=e1] {$dQ$};
\node (f2) [rect, right of=f1] {Proj\\Bias};
\node (f3) [rect, right of=f2, xshift=0.5cm] {Add\\(Residual)};
\node (f4) [circ, right of=f3, xshift=0.2cm] {$Q$};
\node (idf) [at=(f1.west), yshift=0.7cm, xshift=1.0cm, align=left] {Fused Op \#4};
\node (rect4) [rectd, fit=(f1)(f2)(f3)(f4)(idf)] {};
\node (f) [rect2, below of=d1, xshift=1cm, yshift=-0.6cm] {FC1\\Gemm};
\node (g1) [circ, right of = f, xshift=0.4cm] {$dQ$};
\node (g2) [rect, right of=g1, xshift=0.5cm] {FC1 Bias \& \\GELU activation};
\node (g3) [circ, right of=g2, xshift=0.5cm] {$Q$};
\node (idg) [at=(g1.west), yshift=0.7cm, xshift=1.0cm, align=left] {Fused Op \#5};
\node (rect5) [rectd, draw=red, fit=(g1)(g2)(g3)(idg)] {};
\node (h) [rect2, right of=g3, xshift=0.3cm] {FC2\\Gemm};
\node (i1) [circ, right of=h, xshift=0.4cm] {$dQ$};
\node (i2) [rect, right of=i1, xshift=0cm] {FC2 Bias};
\node (i3) [rect, right of=i2, xshift=0.4cm] {Add \&\\Layer-norm};
\node (idi) [at=(i1.west), yshift=0.7cm, xshift=1.0cm, align=left] {Fused Op \#6};
\node (rect6) [rectd, fit=(i1)(i2)(i3)(idi)] {};
\draw[->, thick] (a1) -- (a2);
\draw[->, thick] (a2) -- (a3);
\draw[->, thick] (a3) -- (b);
\draw[->, thick] (b) -- (c1);
\draw[->, thick] (c1) -- (c2);
\draw[->, thick] (c2) -- (c3);
\draw[->, thick] (d1) -- (d2);
\draw[->, thick] (d2) -- (d3);
\draw[->, thick] (d3) -- (d4);
\draw[->, thick] (d4) -- (d5);
\draw[->, thick] (d5) -- (e1);
\draw[->, thick] (e1) -- (f1);
\draw[->, thick] (f1) -- (f2);
\draw[->, thick] (f2) -- (f3);
\draw[->, thick] (f3) -- (f4);
\draw[->, thick] (f) -- (g1);
\draw[->, thick] (g1) -- (g2);
\draw[->, thick] (g2) -- (g3);
\draw[->, thick] (g3) -- (h);
\draw[->, thick] (h) -- (i1);
\draw[->, thick] (i1) -- (i2);
\draw[->, thick] (i2) -- (i3);
\draw[thick,->] (c3) -- ++(1,0) -- ++(0,-0.85) -- ++(-13.5,0) -- (d1.north);
\draw[thick,->] (f4) -- ++(1,0) -- ++(0,-0.85) -- ++(-14.4,0) -- (f.north);
\draw[->, thick] ([xshift=-1cm]a1.west) -- ++(1,0);
\draw[->, thick] (i3.east) -- ++(1,0);
\end{tikzpicture}
\end{document}