\documentclass[manuscript]{acmart}

\usepackage{subfigure}
\AtBeginDocument{%
  \providecommand\BibTeX{{%
    \normalfont B\kern-0.5em{\scshape i\kern-0.25em b}\kern-0.8em\TeX}}}
\copyrightyear{2022}
\acmYear{2022}
\setcopyright{rightsretained}
\acmConference[ICMI '22]{INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION}{November 7--11, 2022}{Bengaluru, India}
\acmBooktitle{INTERNATIONAL CONFERENCE ON MULTIMODAL INTERACTION (ICMI '22), November 7--11, 2022, Bengaluru, India}\acmDOI{10.1145/3536221.3558066}
\acmISBN{978-1-4503-9390-4/22/11}
\begin{document}

\title{The ReprGesture entry to the GENEA Challenge 2022}
\author{Sicheng Yang}
\email{yangsc21@mails.tsinghua.edu.cn}
\affiliation{%
  \institution{Tsinghua University}
  \city{Shenzhen}
  \country{China}
}

\author{Zhiyong Wu}
\authornote{Corresponding authors}
\affiliation{%
  \institution{Tsinghua University}
  \city{Shenzhen}
  \country{China}
}
\affiliation{%
  \institution{The Chinese University of Hong Kong}
  \city{Hong Kong SAR}
  \country{China}
}
\email{zywu@sz.tsinghua.edu.cn}
\orcid{0000-0001-8533-0524}

\author{Minglei Li}
\authornotemark[1]
\email{liminglei29@huawei.com}
\affiliation{%
  \institution{Huawei Cloud Computing Technologies Co., Ltd}
  \city{Shenzhen}
  \country{China}
}

\author{Mengchen Zhao}
\email{zhaomengchen@huawei.com}
\affiliation{%
  \institution{Huawei Noah's Ark Lab}
  \city{Shenzhen}
  \country{China}
}

\author{Jiuxin Lin}
\email{linjx21@mails.tsinghua.edu.cn}
\author{Liyang Chen}
\email{cly21@mails.tsinghua.edu.cn}
\author{Weihong Bao}
\email{bwh21@mails.tsinghua.edu.cn}
\affiliation{%
  \institution{Tsinghua University}
  \city{Shenzhen}
  \country{China}
}

\renewcommand{\shortauthors}{Sicheng Yang et al.}

\begin{abstract}
  This paper describes the ReprGesture entry to the Generation and Evaluation of Non-verbal Behaviour for Embodied Agents (GENEA) challenge 2022.
  The GENEA challenge provides the processed datasets and performs crowdsourced evaluations to compare the performance of different gesture generation systems.
  In this paper, we explore an automatic gesture generation system based on multimodal representation learning. 
  We use WavLM features for audio, FastText features for text and position and rotation matrix features for gesture.
  Each modality is projected to two distinct subspaces: modality-invariant and modality-specific.
  To learn inter-modality-invariant commonalities and capture the characters of modality-specific representations, gradient reversal layer based adversarial classifier and modality reconstruction decoders are used during training.
  The gesture decoder generates proper gestures using all representations and features related to the rhythm in the audio.
  Our code, pre-trained models and demo are available at \url{https://github.com/YoungSeng/ReprGesture}.
\end{abstract}

\begin{CCSXML}
<ccs2012>
   <concept>
       <concept_id>10010147.10010178</concept_id>
       <concept_desc>Computing methodologies~Artificial intelligence</concept_desc>
       <concept_significance>500</concept_significance>
       </concept>
   <concept>
       <concept_id>10003120.10003121</concept_id>
       <concept_desc>Human-centered computing~Human computer interaction (HCI)</concept_desc>
       <concept_significance>500</concept_significance>
       </concept>
   <concept>
       <concept_id>10010147.10010178.10010179</concept_id>
       <concept_desc>Computing methodologies~Natural language processing</concept_desc>
       <concept_significance>500</concept_significance>
       </concept>
 </ccs2012>
\end{CCSXML}

\ccsdesc[500]{Computing methodologies~Artificial intelligence}
\ccsdesc[500]{Human-centered computing~Human computer interaction (HCI)}
\ccsdesc[500]{Computing methodologies~Natural language processing}

\keywords{gesture generation, data-driven animation, modality-invaiant, modality-specific, representation learning, deep learning}
\maketitle
\section{Introduction}

Nonverbal behavior plays a key role in conveying messages in human communication \cite{10.1145/3397481.3450692}, including facial expressions, hand gestures and body gestures.
Co-speech gestures introduce better self-expression.
In the virtual world, it helps to present a rather realistic digital avatar.
Gesture generation studies how to generate human-like, natural, speech-oriented gestures.
There are many different techniques for gesture generation.
In this paper, we focus on the task of speech-driven gesture generation.
Representative speech-driven gesture generation are either rule-based or data-driven \cite{10.1145/3414685.3417838}.

Many data-driven works for gesture generation are based on multimodal fusion and representation learning.
Taras et al. map speech acoustic and semantic features into continuous 3D gestures \cite{10.1145/3382507.3418815}.
Youngwoo et al. propose an end-to-end model to generate co-speech gestures using text, audio, and speaker identity \cite{10.1145/3414685.3417838}.
Jing et al. sample gesture in a variational autoencoder (VAE) latent space and infer rhythmic motion from speech prosody to address the non-deterministic mapping from speech to gesture \cite{Xu2022FreeformBM}.
Taras et al. propose a speech-driven gesture-production method based on representation learning \cite{doi:10.1080/10447318.2021.1883883}. 
Xian et al. propose the hierarchical audio features extractor and pose inferrer to learn discriminative representations \cite{liu2022learning}. 
Jing et al. present a co-speech gesture generation model whose latent space is split into shared code and motion-specific code \cite{9710107}.

However, gesture generation is a challenging task because of cross-modality learning issue and the weak correlation between speech and gestures.
The inherent heterogeneity of the representations creates a gap among different modalities.
It is necessary to address the weak correlation among different modalities and provide a holistic view of the multimodal data during gesture generation.

Inspired by \cite{10.1145/3414685.3417838} and \cite{10.1145/3394171.3413678}, we propose a gesture generation system based on multimodal representation learning. 
In particular, we first extract features of audio, text and gestures.
Then, a system consisting of four components is proposed:
(1) Each modality is projected to two distinct representations: modality-invariant and modality-specific.
(2) A gradient reversal layer based adversarial classifier is used to reduce the discrepancy between the modality-invariant representations of each modality.
(3) Modality decoders are used to reconstruct each modality, allowing modality-specific representations to capture the details of their respective modality.
(4) The gesture decoder takes six modality representations (two per modality) and rhythm-related features in audio as its input and generates proper gestures.

The main contributions of our work are:
(1) A multimodal representation learning approach is proposed for gesture generation, which ensures comprehensive decoupling of multimodal data.
(2) To solve the problem of heterogeneity of different modalities in feature fusion, each modality is projected to two subspaces (modality-invariant and modality-specific) to get multimodal representations using domain learning and modality reconstruction.
(3) Ablation studies demonstrate the role of different components in the system.

The task of the GENEA 2022 challenge is to generate corresponding gestures from the given audio and text.
A complete task description can be accessed in \cite{yoon2022genea}.
We submitted our system to the GENEA 2022 challenge to be evaluated with other gesture generation systems in a large user study. 
\section{Method}
\begin{figure}[h]
  \centering
  \includegraphics[width=0.95\linewidth]{fig/1_3.pdf}
  \caption{Gesture generation through modality -invariant and -specific subspaces.}
  \Description{Gesture generation through modality -invariant and -specific subspaces.}
  \label{Architecture}
\end{figure}

\subsection{The architecture of the proposed system}

As shown in Figure \ref{Architecture}, the system generates a sequence of human gestures from a sequence of $\mathbf{u}_{m} (m \in \{t,a,g\})$ that contain the features of text, audio and seed gestures.
The architecture of the proposed model consists of five modules: feature extraction, modality representation, modality reconstruction, domain learning and gesture generation.
The following describes each of these modules in detail.

\subsubsection{Feature extraction}
~\\
For each of the modality, the pipeline of extracting features is as follows:
\begin{itemize}
    \item Text: We first use FastText \cite{10.1162/tacl_a_00051} to get the word embeddings. Padding tokens are inserted to make the words temporally match the gestures by following \cite{10.1145/3414685.3417838}.
    One-dimensional (1D) convolutional layers are then adopted to generate 32-D text feature sequence $\mathbf{U}_{t}$ (`$t$' for `text') from the 300-D word embeddings.
    \item Audio: All audio recordings are downsampled to 16kHz, and features are generated from the pre-trained models of WavLM Large \cite{DBLP:journals/corr/abs-2110-13900}. 
    We further adjust sizes, strides and padding in the 1D convolutional layers
    to reduce the dimension of features from 1024 to 128 forming the final audio feature sequence $\mathbf{U}_{a}$ (`$a$' for `audio').
    \item Gesture: Due to the poor quality of hand motion-capture, we only use 18 joints corresponding to the upper body without hands or fingers. Root normalization is used to make objects face the same direction. 
    We apply standard normalization (zero mean and unit variant) to all joints.
    Seed gestures for the first few frames are utilized for better continuity between consecutive syntheses, as in \cite{10.1145/3414685.3417838}.
    On top of these, position and 3 × 3 rotation matrix features are computed, and the size of final gesture sequence $\mathbf{U}_{g}$ (`$g$' for `gesture') feature is 216.
\end{itemize}

\subsubsection{Modality representation}
~\\
First, for each modality $m \in \{t,a,g\}$, we use a linear layer with leaky ReLU activation and layer normalization to map its feature sequence $\mathbf{U}_{m}$ into a new feature sequence $\mathbf{u}_{m} \in \mathbb{R}^{T \times d_{h}}$ with the same feature dimension $d_{h}$.
Then, we project each sequence $\mathbf{u}_{m}$ to two distinct representations: modality-invariant $\mathbf{h}_{m}^{c}$ and modality-specific $\mathbf{h}_{m}^{p}$.
Afterwards, $\mathbf{h}_{m}^{c}$ learns a shared representation in a common subspace with distributional similarity constraints \cite{8715409}.
$\mathbf{h}_{m}^{p}$ captures the unique characteristics of that modality.
We derive the representations using the simple feed-forward neural encoding functions:
\begin{equation}
\mathbf{h}_{m}^{c}=E_{c}\left(\mathbf{u}_{m} ; \theta^{c}\right), \quad \mathbf{h}_{m}^{p}=E_{p}\left(\mathbf{u}_{m} ; \theta_{m}^{p}\right)
\end{equation}
Encoder $E_{c}$ shares the parameters $\theta^{c}$ across all three modalities, whereas $E_{p}$ assigns separate parameters $\theta_{m}^{p}$ for each modality.

\subsubsection{Representation learning}
~\\
Domain learning can improve a model’s ability to extract domain-invariant features \cite{NIPS2016_45fbc6d3}.
We use an adversarial classifier to minimize domain loss that reduces the discrepancy among shared representations of each modality.

The domain loss can be formulated as:
\begin{equation}
\mathcal{L}_{domain}=-\sum_{i=1}^{3} \mathbb{E}[ \log \left(D_{repr}(d_m)\right)]
\end{equation}
where $D_{repr}$ represents feed-forward neural discriminator, $d_m$ represents the result after gradient reversal of $\mathbf{h}_{m}^{p}$.

The modality reconstruction loss $\mathcal{L}_{\text {recon}}$ is computed on the reconstructed modality and the original input $\mathbf{u}_{m}$.
The $\mathcal{L}_{\text {recon}}$ is used to ensure the hidden representations to capture the details of their respective modality. 

Specifically, a modality decoder $D$ is proposed to reconstruct $\mathbf{u}_{m}$:
\begin{equation}
\hat{\mathbf{u}}_{m}=D\left(\mathbf{h}_{m}^{c}+\mathbf{h}_{m}^{p} ; \theta^{d}\right)
\end{equation}
where $\theta^{d}$ are the parameters of the modality decoder. The modality reconstruction loss can then be computed as:
\begin{equation}
\mathcal{L}_{\text {recon}}=\frac{1}{3}\left(\sum_{m \in\{t, a, g\}} \frac{\left\|\mathbf{u}_{m}-\hat{\mathbf{u}}_{m}\right\|_{2}^{2}}{d_{h}}\right)
\end{equation}
where $\|\cdot\|_{2}^{2}$ is the squared $L_2$-norm. 

\subsubsection{Gesture generation}
~\\

\begin{figure}[h]
  \centering
  \includegraphics[width=0.82\linewidth]{fig/2_.pdf}
  \caption{Architecture of the gesture generation module.}
  \Description{Architecture of the gesture generation module.}
  \label{generation}
\end{figure}

We use generative adversarial network (GAN) based gesture decoder for generating gestures. Gestures are directly related to rhythm and beat, thus we concatenate audio rhythm related features (pitch, energy and volume) and the output of six stacked modality representations together and send them to Transformer encoders with multi-head self-attention as the generator, as shown in Figure \ref{generation}.
The generator part is trained using  $\mathcal{L}_{gesture}$ consisting of the Huber loss  
and the MSE loss, and the discriminator part is trained with $\mathcal{L}_{GAN}$.

\begin{equation}
\mathcal{L}_{gesture}=\alpha \cdot \mathbb{E}\left[\frac{1}{t} \sum_{i=1}^{t} \operatorname{HuberLoss}\left(g_{i}, \hat{g}_{i}\right)\right] + \beta \cdot \mathbb{E}\left[\frac{1}{t} \sum_{i=1}^{t} \|\left(g_{i}, \hat{g}_{i}\right)\|_{2}^{2}\right]
\label{Lgesture}
\end{equation}
\begin{equation}
\mathcal{L}_{GAN}=-\mathbb{E}[\log (D_{gesture}(g))]-\mathbb{E}[\log (1-D_{gesture}(\hat{g}))]
\end{equation}
where $D_{gesture}$ represents gesture discriminator using multilayered bidirectional gated recurrent unit (GRU) \cite{KyunghyunCho2014LearningPR} that outputs binary output for each time step, $t$ is the length of the gesture sequence, $g_i$ represents the $i$th human gesture, $\hat{g_i}$ represents the $i$th generated gesture.

The loss of the proposed system can be computed as:
\begin{equation}
    \mathcal{L}_{total} = \mathcal{L}_{gesture} + \gamma \cdot \mathcal{L}_{GAN} + \delta \cdot \mathcal{L}_{domain} + 
    \epsilon \cdot \mathcal{L}_{recon}
    \label{total}
\end{equation}

\subsection{Data processing and experiment setup}

\subsubsection{Data and data processing}
~\\
In the challenge, the Talking With Hands 16.2M \cite{9010909} is used as the standard dataset. 
Each video is separated into two independent sides with one speaker each.
The audio and text in the dataset have been aligned.
For more details please refer to the challenge paper \cite{yoon2022genea}. 
We note that the data in the training, validation and test sets are extremely unbalanced, so we only use the data from the speaker with identity "1" for training. And we believe that if speech and gesture data are trained on the same person, the gesture behavior would match the speech.

\subsubsection{Experiment setup}
~\\
The proposed system is trained on training data only, using the ADAM \cite{2014Adam} optimizer (learning rate is e-4, $\beta_1$ = 0.5, $\beta_2$ = 0.98) with a batch size of 128 for 100 steps. 
We set $\alpha=300$, $\beta=50$ for Equation (\ref{Lgesture}) and $\gamma=5, \delta=0.1, \epsilon=0.1$ (we noticed in our experiments that too large $\delta$ and $\epsilon$ will lead to non-convergence) for Equation (\ref{total}). There is a warm-up period of 10 epochs in which the $\mathcal{L}_{GAN}$ is not used ($\gamma$ = 0).
The feature dimension $d_h$ of sequence $\textbf{u}_m$ is 48.
During training, each training sample having 100 frames is sampled with a stride of 10 from the valid motion sections; the initial 10 frames are used as seed gesture poses and the model is trained to generate the remaining 90 poses (3 seconds). 

\section{Evaluation}

\subsection{Evaluation setup}

The GENEA Challenge 2022 evaluation is divided into two tiers, and we participated in the upper-body motion tier.
The challenge organizers conducted a detailed evaluation comparing all submitted systems\cite{yoon2022genea}.
The challenge evaluates human-likeness to assess motion quality, and appropriateness to assess how well the gestures match the speech.
The evaluation is based on the HEMVIP methodology \cite{10.1145/3462244.3479957} and Mean Opinion Score (MOS) \cite{1996Methods}.
There are in total 11 systems participated in the upper-body tier. The following abbreviations are used to represent each model in the evaluation:
\begin{itemize}
    \item UNA: Ground truth (`U' for the upper-body tier, `NA' for `natural').
    \item UBT: The official text-based baseline \cite{8793720}, which takes transcribed speech text with word-level timing information as the input modality (`B' for `baseline', `T' for `text').
    \item UBA: The official audio-based baseline \cite{10.1145/3308532.3329472},  which takes speech audio into account when generating output (`A' for `audio').
    \item USJ–USQ: 8 participants’ submissions to the upper-body tier (ours is USN).
\end{itemize}

For more details about the evaluation studies, please refer to the challenge paper \cite{yoon2022genea}.
\subsection{Subjective evaluation results and discussion}

\subsubsection{Human-likeness Evaluation}
~\\
\begin{figure}[h]
  \centering
  \subfigure[Box visualizing the ratings distribution in Upper-body study. ]{
  \label{Fig.sub.1}
  \includegraphics[width=0.43\linewidth]{fig/upper-body_human-likeness_boxplot.pdf}}
  \quad
  \subfigure[Significance of pairwise differences between conditions.]{
  \label{Fig.sub.2}
  \includegraphics[width=0.43\linewidth]{fig/upper-body_human-likeness_median_pref.pdf}}
  \caption{(a) Red bars are the median ratings (each with a 0.05 confidence interval); yellow diamonds are mean ratings (also with a 0.05 confidence interval). Box edges are at 25 and 75 percentiles, while whiskers cover 95\% of all ratings for each condition. 
  (b) White means that the condition listed on the ${y}$-axis rated significantly above the condition on the $x$-axis, black means the opposite ($y$ rated below $x$), and grey means no statistically significant difference at the level $\alpha$ = 0.05 after Holm-Bonferroni correction.}
  \Description{Box plots visualizing the ratings distribution in Upper-body study.}
  \label{Upper_result}
\end{figure}

In this evaluations, study participants are asked to rate ``How human-like does the gesture motion appear?'' on a scale from 0 (worst) to 100 (best).
Bar plots and significance comparisons are shown in Figure \ref{Upper_result}.
Our system (USN) receives a median score of 44 and a mean score of 44.2, and is ranked fourth among the participating systems. 

\subsubsection{Appropriateness evaluation}
~\\
\begin{figure}[h]
  \centering
  \includegraphics[width=0.45\linewidth]{fig/upper-body_appropriateness_matched_pref.pdf}
  \caption{Bar plots visualizing the response distribution in the appropriateness studies. The blue bar (bottom) represents responses where subjects preferred the matched motion, the light grey bar (middle) represents tied (``They are equal'') responses, and the red bar (top) represents responses preferring mismatched motion, with the height of each bar being proportional to the fraction of responses in each category. The black horizontal line bisecting the light grey bar shows the proportion of matched responses after splitting ties, each with a 0.05 confidence interval. The dashed black line indicates chance-level performance.}
  \Description{Box plots visualizing the ratings distribution in the upper-body study.}
  \label{appropriateness}
\end{figure}

In this evaluation, participants are asked to choose the character on the left, on the right, or indicate that the two are equally well matched to response ``Please indicate which character’s motion best matches the speech, both in terms of rhythm and intonation and in terms of meaning.''
Bar plots are shown in Figure \ref{appropriateness}.
Our system (USN) receives a ``Percent matched'' 54.6, which identifies how often participants preferred matched over mismatched motion in terms of appropriateness. 
Our system is rated seventh in appropriateness among the participants’ submissions.
It should be noted that the difference of our system to the five higher-ranked systems (USL, UBA, USO, USK and USJ) is not significant.
Furthermore, if we only consider the ratio of matched motion, i.e., the blue bar
in Figure \ref{appropriateness},
our system is ranked fifth among the participating systems.
\subsection{Ablation studies}

\begin{table}[]
\caption{Ablation studies results. 
`w/o' is short for `without'.
Bold indicates the best metric, i.e. the one closest to the ground truth.}
  \label{tab:Ablation}
\resizebox{\textwidth}{!}
{
 \begin{tabular}{cccccccc}
\toprule
Name            & Average jerk        & \begin{tabular}[c]{@{}c@{}}Average \\ acceleration\end{tabular} & \begin{tabular}[c]{@{}c@{}}Global \\ CCA\end{tabular} & \begin{tabular}[c]{@{}c@{}}CCA for \\ each sequence\end{tabular} & \begin{tabular}[c]{@{}c@{}}Hellinger\\ distance average\end{tabular} $\downarrow$ & \begin{tabular}[c]{@{}c@{}}FGD on \\ feature space\end{tabular} $\downarrow$ & \begin{tabular}[c]{@{}c@{}}FGD on raw \\ data space\end{tabular} $\downarrow$ \\
\midrule
Ground Truth (GT)              & 18149.74 $\pm$ 2252.61 & 401.24 $\pm$ 67.57                                                 & 1.000                                                 & 1.00 $\pm$ 0.00                                                     & 0.0                                                                & 0.0                                                                & 0.0                                                                 \\
ReprGesture     & 2647.59 $\pm$ 1200.05  & 146.90 $\pm$ 46.09                                                 & 0.726                                                 & \textbf{0.95 $\pm$ 0.02}                                                     & \textbf{0.155}                                                                & 0.86                                                            & \textbf{184.753}                                                          \\
w/o WavLM       & 1775.09 $\pm$ 512.08   & 77.53 $\pm$ 21.92                                                  & \textbf{0.761}                                                 & 0.94 $\pm$ 0.03                                                     & 0.353                                                                & 3.054                                                           & 321.383                                                          \\
w/o $\mathcal{L}_{GAN}$    & \textbf{9731.54 $\pm$ 3636.06}  & \textbf{242.15 $\pm$ 81.81}                                                 & 0.664                                                 & 0.93 $\pm$ 0.03                                                     & 0.342                                                                & 2.053                                                           & 277.539                                                          \\
w/o $\mathcal{L}_{recon}$ & 533.95 $\pm$ 193.18 & 39.49 $\pm$ 12.23                                                 & 0.710                                                & 0.93 $\pm$ 0.03                                                     & 0.283                                                                & 0.731                                                           & 659.150                                                          \\
w/o $\mathcal{L}_{domain}$ & 2794.79 $\pm$ 1153.75  & 135.62 $\pm$ 25.13                                                 & 0.707                                                 & 0.94 $\pm$ 0.03                                                     & 0.267                                                                & \textbf{0.653}                                                           & 874.209                                                          \\
w/o Repr        & 2534.34 $\pm$ 1151.38  & 123.02 $\pm$ 40.90                                                 & 0.723                                                 & 0.94 $\pm$ 0.04                                                     & 0.298                                                                & 0.829                                                           & 514.706  \\         \bottomrule                                              
\end{tabular}
}
\end{table}

Moreover, we conduct ablation studies to address the performance effects from different components in the system.
The GENEA challenge computes some objective metrics of motion quality
by 
GENEA numerical evaluations\footnote{\url{https://github.com/genea-workshop/genea_numerical_evaluations}}.
For calculation and meaning of these objective evaluation metrics, please refer to the challenge paper \cite{yoon2022genea}.
A perfect natural system should have average jerk and acceleration very similar to natural motion.
The closer the Canonical correlation analysis (CCA) to 1, the better.
Lower Hellinger distance and Fr\'{e}chet gesture distance (FGD) are better.
To compute the FGD, we train an autoencoder using the training set of the challenge.

The results of our ablations studes are summarized in Table \ref{tab:Ablation}.
Supported by the results, when we do not use WavLM to extract audio features, but use 1D convolution instead, the Hellinger distance average and FGD on feature space present the worst performance.
When the model is trained without the GAN loss, the average jerk and average acceleration are better, but the global CCA and CCA for each sequence are decreased.
When the reconstruction loss is removed, the average jerk and average acceleration are worst. The generated gesture movements are few and of small range.
When the model is trained using Central Moment Discrepancy (CMD) loss \cite{10.1145/3394171.3413678} instead of domain loss, the best FGD on feature space and the worst FGD on raw data space are obtained.
When the modality representations are removed (w/o Repr), we feed the modality sequence $\mathbf{u}_t, \mathbf{u}_a$ and $\mathbf{u}_g$ directly to the gesture decoder and only use the $\mathcal{L}_{task}$ loss, the performances of all metrics have deteriorated except for FGD on
feature space.

\section{Conclusions and discussion}

In this paper, we propose a gesture generation system based on multimodal representation learning, where the considered modalities include text, audio and gesture. 
Each modality is projected into two different subspaces: modality-invariant and modality-specific.
To learn the commonality among different modalities, an adversarial classifier based on gradient reversal layer is used.
To capture the features of modality-specific representations, we adopt a modality reconstruction decoder.
The gesture decoder utilizes all representations and audio rhythmic features to generate appropriate gestures.
In subjective evaluation, our system is ranked fourth among the participating systems in human-likeness evaluation, and ranked seventh in appropriateness evaluation. Whereas, for appropriateness, the differences between our system and the five higher-ranked systems are not significant.

For appropriateness evaluation, whether there is a relationship between subjective evaluation and segmentation duration deserves to be investigated.
The segments are around 8 to 10 seconds during evaluation\cite{yoon2022genea}.
We believe that a longer period of time (e.g. 20-30 seconds) might produce more pronounced and convincing appropriateness results.

There is room for improvement in this research.
First, we only use data from one person to learn gesture due to unbalanced dataset issue.
Such one-to-one mapping could produce boring and homogeneous gestures during inference.
Second, 
the finger motions are not considered
because of the low motion-capture quality. 
Such finger motions could be involved in the future if some data cleanup procedures could be conducted.
Third, besides text and audio, more modalities (e.g. emotions, facial expressions and semantic meaning of gestures \cite{Liu2022BEATAL}) could be taken into consideration to generate more appropriate gestures.

\begin{acks}
This work is supported by Shenzhen Science and Technology Innovation Committee (WDZC20200818121348001), National Natural Science Foundation of China (62076144) and Shenzhen Key Laboratory of next generation interactive media innovative technology (ZDSYS20210623092001004).
\end{acks}
\bibliographystyle{ACM-Reference-Format}
\bibliography{my}
\end{document}


Peer reviews:

content: confidence: 4: The reviewer is confident but not absolutely certain that the evaluation is correct
rating: 6: Marginally above acceptance threshold
review: This paper presents a speech-driven gesture generation method based on multimodal representation learning. Each modality is projected to two distinct subspaces: modality-invariant and modality-specific. Gradient reversal layer based adversarial classifier and modality reconstruction decoders are used during training.
Average human-likeness ratings below 50% have been achieved, indicating that the proposed approach was not enough to generate natural gesture motions.

Ablation studies should be included to show the effects of modality-invariant only, or modality-specific only.
The video sample in the anonymous webpage shows an example of the generated motion.
I'm not aware of how the other dialogue segments used in the evaluation are, but in the sample, the target speaker is basically in listening mode, so that the instants the co-speech gestures are generated are very short.

title: The proposed method is somehow novel, but it lacks of ablation studies to validate the effects of using single modalities and cross-modality subspaces.

content: confidence: 4: The reviewer is confident but not absolutely certain that the evaluation is correct
rating: 9: Top 15% of accepted papers, strong accept
review: The paper proposed a gesture generation method by incorporating representation learning into previously proposed network architectures. Six representations for audio, text, gesture (two per modality) are used. The method takes audio, text, and seed gestures as input and output is a sequence of gesture.

The paper is well organized and written but it is easier to read if put new lines after sub-sub-section titles. 

The technical descriptions are well written and the experiments would be reproducible.

Some research, such as [10], have already addressed the importance of the audio or human-pose representations for gesture generation. So, the contribution is incremental, but having modality-mixed representations is somewhat novel.

The evaluation and sub-materials show good results. Having ablation study, also, help us to better understand the effects of proposed method. However, the effects of proposed representations were not very significant.

title: The evaluation and sub-materials show good results.

content: confidence: 4: The reviewer is confident but not absolutely certain that the evaluation is correct
rating: 7: Good paper, accept
review: The paper proposes a novel deterministic speech to gesture generation model by learning a modal-invariant and modal-specific features for different modalities used as inputs, i.e., audio, text, and seed poses. Ablation study showed that this method could improve the accuracy of the generated gestures.

Strengths:
1. Although the paper was not the best among proposed systems, the ablation study showed that the separation for modal-invariant feature and modal-specific feature were helpful to the accuracy of the generated gestures, i.e., a lower Hellinger distance and FGD on raw data space. This indicates that the proposed method is more effective to determine the gesture shape than purely mixing different modality features.
2. The using of WavLM, as a pre-trained neural network for audio feature extraction, improves the jerk and acceleration of the generated gestures, even though the pitch, energy and volume were used at the same time. This shows that prosodic features of audio, i.e., pitch and energy, are not enough for predicting the rhythmic movements in the gestures.

Potential issue:
The paper uses an autoencoder-like training scheme to help learn the hidden feature space. However, the reconstruction error was computed on the reconstructed results and the output of one of the hidden layers, whose parameters are being updated when training. This could lead to an undesired behavior that the encoder and decoder agree with each other while not considering the original input at all. A more common approach for training an autoencoder is to compute the reconstruction error on the original input or a transformation of the original input.

Weaknesses:
1. More details are necessary for understanding the domain learning. While the authors propose to extract modal-invariant features from modalities as in [5], they did not use the original similarity loss for these features. Instead, they propose to use domain learning to reach this goal. Although this could be one of the originalities of this paper, the authors did not thoroughly explain how domain learning works and how it can achieve a similar effect as central moment discrepancy (CMD) loss, or its potential advantages and disadvantages compared with CMD.
2. The authors claims that the reconstruction loss is used to ensure the hidden representations to capture the details. However, no ablation was provided for this. Thus, it is unclear that how much or if this loss is useful.
3. The authors did not use traditional feature extraction method on the audio such as mel-spectrogram, which is more common in the literature. The difference between using WavLM and basic features is unknown.

Questions:
1. The scales of coefficients for different loss terms ranges widely, e.g., alpha is 300 and epsilon is 0.1 in equation (5). How did the authors adjust the hyper-parameters and reach such a different scale?
2. Why did the author choose 3 seconds as the length for generation? Any reference for this?

title: Novel and interesting paper with a little unclarity.