Upload folder using huggingface_hub

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml
+# Editor-based HTTP Client requests
+/httpRequests/

.idea/ai-avatar-backend.iml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.9 (avatar-ai)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="pytest" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,78 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="79" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="45">
+            <item index="0" class="java.lang.String" itemvalue="beautifulsoup4" />
+            <item index="1" class="java.lang.String" itemvalue="bs4" />
+            <item index="2" class="java.lang.String" itemvalue="mecab-python3" />
+            <item index="3" class="java.lang.String" itemvalue="pydantic" />
+            <item index="4" class="java.lang.String" itemvalue="joblib" />
+            <item index="5" class="java.lang.String" itemvalue="scikit-learn" />
+            <item index="6" class="java.lang.String" itemvalue="torch" />
+            <item index="7" class="java.lang.String" itemvalue="numpy" />
+            <item index="8" class="java.lang.String" itemvalue="requests" />
+            <item index="9" class="java.lang.String" itemvalue="unidic-lite" />
+            <item index="10" class="java.lang.String" itemvalue="mysql-connector-python" />
+            <item index="11" class="java.lang.String" itemvalue="sqlalchemy" />
+            <item index="12" class="java.lang.String" itemvalue="pycaret" />
+            <item index="13" class="java.lang.String" itemvalue="tensorflow-text" />
+            <item index="14" class="java.lang.String" itemvalue="keybert" />
+            <item index="15" class="java.lang.String" itemvalue="pandas" />
+            <item index="16" class="java.lang.String" itemvalue="tqdm" />
+            <item index="17" class="java.lang.String" itemvalue="fastapi" />
+            <item index="18" class="java.lang.String" itemvalue="spacy" />
+            <item index="19" class="java.lang.String" itemvalue="regex" />
+            <item index="20" class="java.lang.String" itemvalue="tensorflow-hub" />
+            <item index="21" class="java.lang.String" itemvalue="uvicorn" />
+            <item index="22" class="java.lang.String" itemvalue="xgboost" />
+            <item index="23" class="java.lang.String" itemvalue="onnxruntime" />
+            <item index="24" class="java.lang.String" itemvalue="av" />
+            <item index="25" class="java.lang.String" itemvalue="onnx" />
+            <item index="26" class="java.lang.String" itemvalue="opencv-python-headless" />
+            <item index="27" class="java.lang.String" itemvalue="streamlit_webrtc" />
+            <item index="28" class="java.lang.String" itemvalue="torchvision" />
+            <item index="29" class="java.lang.String" itemvalue="cdifflib" />
+            <item index="30" class="java.lang.String" itemvalue="protobuf" />
+            <item index="31" class="java.lang.String" itemvalue="python-stdnum" />
+            <item index="32" class="java.lang.String" itemvalue="transformers" />
+            <item index="33" class="java.lang.String" itemvalue="nltk" />
+            <item index="34" class="java.lang.String" itemvalue="sentence-transformers" />
+            <item index="35" class="java.lang.String" itemvalue="datasets" />
+            <item index="36" class="java.lang.String" itemvalue="fsspec" />
+            <item index="37" class="java.lang.String" itemvalue="neuralcoref" />
+            <item index="38" class="java.lang.String" itemvalue="dateparse" />
+            <item index="39" class="java.lang.String" itemvalue="faker" />
+            <item index="40" class="java.lang.String" itemvalue="sentencepiece" />
+            <item index="41" class="java.lang.String" itemvalue="langid" />
+            <item index="42" class="java.lang.String" itemvalue="boto3" />
+            <item index="43" class="java.lang.String" itemvalue="redis" />
+            <item index="44" class="java.lang.String" itemvalue="celery" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N801" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyTypedDictInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="wbc.assets.templates.template_script.process.pattern_name" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.9 (avatar-ai)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/ai-avatar-backend.iml" filepath="$PROJECT_DIR$/.idea/ai-avatar-backend.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="74049918-f54d-42f5-ad7d-c4697cbf6820" name="Default Changelist" comment="">
+      <change afterPath="$PROJECT_DIR$/Dockerfile" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
+    </list>
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="ProjectId" id="2WCeCg5aWbUXqGVwe1ThKSdE6Li" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent">
+    <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
+    <property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
+    <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+    <property name="nodejs_package_manager_path" value="npm" />
+    <property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="74049918-f54d-42f5-ad7d-c4697cbf6820" name="Default Changelist" comment="" />
+      <created>1696240862794</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1696240862794</updated>
+      <workItem from="1696240864567" duration="150000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+  <component name="Vcs.Log.Tabs.Properties">
+    <option name="TAB_STATES">
+      <map>
+        <entry key="MAIN">
+          <value>
+            <State />
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+</project>

Dockerfile

@@ -0,0 +1,19 @@
+FROM node:16
+
+# Set the working directory inside the container
+WORKDIR /usr/src/app
+
+# Copy package.json and package-lock.json (if available) to the working directory
+COPY package*.json ./
+
+# Install application dependencies
+RUN npm install
+
+# Copy the rest of the application to the working directory
+COPY . .
+
+# Expose the port the app runs on
+EXPOSE 5000
+
+# Command to run the application
+CMD [ "npm", "start" ]

README.md

@@ -1,11 +1,19 @@
----
-title: Ai Avatar Backend
-emoji: 🏃
-colorFrom: yellow
-colorTo: blue
-sdk: docker
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Talking Avatar backend
+The text to speech and blendShapes converter for https://github.com/bornfree/talking_avatar.
+Uses the [Azure APIs](https://learn.microsoft.com/en-us/azure/cognitive-services/speech-service/how-to-speech-synthesis-viseme) to get stuff done.
+
+This is a simple ExpressJS app.
+
+### Get keys from Azure and create a `.env` in the root directory with
+```
+AZURE_KEY=
+AZURE_REGION=
+```
+
+
+### To run
+```
+$ npm install
+$ npm start
+```
+

app.js

@@ -0,0 +1,45 @@
+var createError = require('http-errors');
+var express = require('express');
+var path = require('path');
+var cookieParser = require('cookie-parser');
+var logger = require('morgan');
+var cors = require('cors');
+
+var indexRouter = require('./routes/index');
+
+var app = express();
+
+
+// view engine setup
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'pug');
+
+var corsOptions = {
+  origin: '*'
+};
+app.use(cors(corsOptions));
+app.use(logger('dev'));
+app.use(express.json());
+app.use(express.urlencoded({ extended: false }));
+app.use(cookieParser());
+app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', indexRouter);
+
+// catch 404 and forward to error handler
+app.use(function(req, res, next) {
+  next(createError(404));
+});
+
+// error handler
+app.use(function(err, req, res, next) {
+  // set locals, only providing error in development
+  res.locals.message = err.message;
+  res.locals.error = req.app.get('env') === 'development' ? err : {};
+
+  // render the error page
+  res.status(err.status || 500);
+  res.render('error');
+});
+
+module.exports = app;

bin/www

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+/**
+ * Module dependencies.
+ */
+
+var app = require('../app');
+var debug = require('debug')('talking-avatar-backend:server');
+var http = require('http');
+
+/**
+ * Get port from environment and store in Express.
+ */
+
+var port = normalizePort(process.env.PORT || '5000');
+app.set('port', port);
+
+/**
+ * Create HTTP server.
+ */
+
+var server = http.createServer(app);
+
+/**
+ * Listen on provided port, on all network interfaces.
+ */
+
+server.listen(port);
+server.on('error', onError);
+server.on('listening', onListening);
+
+/**
+ * Normalize a port into a number, string, or false.
+ */
+
+function normalizePort(val) {
+  var port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+/**
+ * Event listener for HTTP server "error" event.
+ */
+
+function onError(error) {
+  if (error.syscall !== 'listen') {
+    throw error;
+  }
+
+  var bind = typeof port === 'string'
+    ? 'Pipe ' + port
+    : 'Port ' + port;
+
+  // handle specific listen errors with friendly messages
+  switch (error.code) {
+    case 'EACCES':
+      console.error(bind + ' requires elevated privileges');
+      process.exit(1);
+      break;
+    case 'EADDRINUSE':
+      console.error(bind + ' is already in use');
+      process.exit(1);
+      break;
+    default:
+      throw error;
+  }
+}
+
+/**
+ * Event listener for HTTP server "listening" event.
+ */
+
+function onListening() {
+  var addr = server.address();
+  var bind = typeof addr === 'string'
+    ? 'pipe ' + addr
+    : 'port ' + addr.port;
+  debug('Listening on ' + bind);
+}

helpers/blendshapeNames.js

@@ -0,0 +1,55 @@
+module.exports = ["eyeBlinkLeft",
+"eyeLookDownLeft",
+"eyeLookInLeft",
+"eyeLookOutLeft",
+"eyeLookUpLeft",
+"eyeSquintLeft",
+"eyeWideLeft",
+"eyeBlinkRight",
+"eyeLookDownRight",
+"eyeLookInRight",
+"eyeLookOutRight",
+"eyeLookUpRight",
+"eyeSquintRight",
+"eyeWideRight",
+"jawForward",
+"jawLeft",
+"jawRight",
+"jawOpen",
+"mouthClose",
+"mouthFunnel",
+"mouthPucker",
+"mouthLeft",
+"mouthRight",
+"mouthSmileLeft",
+"mouthSmileRight",
+"mouthFrownLeft",
+"mouthFrownRight",
+"mouthDimpleLeft",
+"mouthDimpleRight",
+"mouthStretchLeft",
+"mouthStretchRight",
+"mouthRollLower",
+"mouthRollUpper",
+"mouthShrugLower",
+"mouthShrugUpper",
+"mouthPressLeft",
+"mouthPressRight",
+"mouthLowerDownLeft",
+"mouthLowerDownRight",
+"mouthUpperUpLeft",
+"mouthUpperUpRight",
+"browDownLeft",
+"browDownRight",
+"browInnerUp",
+"browOuterUpLeft",
+"browOuterUpRight",
+"cheekPuff",
+"cheekSquintLeft",
+"cheekSquintRight",
+"noseSneerLeft",
+"noseSneerRight",
+"tongueOut",
+"headRoll",
+"leftEyeRoll",
+"rightEyeRoll"]

helpers/callOpenAI.js

@@ -0,0 +1,20 @@
+const OpenAI = require('openai');
+
+const openai = new OpenAI({
+    apiKey: process.env.OPENAI_API_KEY, // defaults to process.env["OPENAI_API_KEY"]
+    dangerouslyAllowBrowser: true
+});
+
+async function queryOpenAIAndSave(userContent) {
+    const response = await openai.chat.completions.create({
+        model: "gpt-3.5-turbo",
+        messages: [
+            {"role": "system", "content": "You are a helpful assistant made by Detomo company. Please answer the following questions with maximum 2 sentence. Answer by the language of the question."},
+            {"role": "user", "content": userContent},
+        ],
+    });
+
+    return response.choices[0].message.content;
+}
+
+module.exports = queryOpenAIAndSave;

helpers/tts.js

@@ -0,0 +1,92 @@
+// azure-cognitiveservices-speech.js
+require('dotenv').config()
+const sdk = require('microsoft-cognitiveservices-speech-sdk');
+const blendShapeNames = require('./blendshapeNames');
+const _ = require('lodash');
+const voicesMap = {
+    'en-US': 'en-US-AmberNeural',
+    'ja-JP': 'ja-JP-MayuNeural',
+    'vi-VN': 'vi-VN-NamMinhNeural',
+};
+
+let SSML = `<speak version="1.0" xmlns="http://www.w3.org/2001/10/synthesis" xmlns:mstts="http://www.w3.org/2001/mstts" xml:lang="en-US">
+<voice name="en-US-JennyNeural">
+  <mstts:viseme type="FacialExpression"/>
+  __TEXT__
+</voice>
+</speak>`;
+
+const key = process.env.AZURE_KEY;
+const region = process.env.AZURE_REGION;
+        
+/**
+ * Node.js server code to convert text to speech
+ * @returns stream
+ * @param {*} text text to convert to audio/speech
+ * @param language
+ */
+const textToSpeech = async (text, language)=> {
+    
+    // convert callback function to promise
+    return new Promise((resolve, reject) => {
+
+        const voice = voicesMap[language];
+        let ssml = SSML.replace("__TEXT__", text).replace("<voice name=\"en-US-JennyNeural\">", `<voice name="${voice}">`);
+
+        
+        const speechConfig = sdk.SpeechConfig.fromSubscription(key, region);
+        speechConfig.speechSynthesisOutputFormat = 5; // mp3
+        
+        let audioConfig = null;
+        
+        // if (filename) {
+        let randomString = Math.random().toString(36).slice(2, 7);
+        let filename = `./public/speech-${randomString}.mp3`;
+        audioConfig = sdk.AudioConfig.fromAudioFileOutput(filename);
+        // }
+
+        let blendData = [];
+        let timeStep = 1/60;
+        let timeStamp = 0;
+
+        const synthesizer = new sdk.SpeechSynthesizer(speechConfig, audioConfig);
+
+        // Subscribes to viseme received event
+        synthesizer.visemeReceived = function (s, e) {
+
+            // `Animation` is an xml string for SVG or a json string for blend shapes
+            var animation = JSON.parse(e.animation);
+
+            _.each(animation.BlendShapes, blendArray => {
+
+                let blend = {};
+                _.each(blendShapeNames, (shapeName, i) => {
+                    blend[shapeName] = blendArray[i];
+                });
+        
+                blendData.push({
+                    time: timeStamp,
+                    blendshapes: blend
+                });
+                timeStamp += timeStep;
+            });
+
+        }
+
+
+        synthesizer.speakSsmlAsync(
+            ssml,
+            result => {
+                
+                synthesizer.close();
+                resolve({blendData, filename: `/speech-${randomString}.mp3`});
+
+            },
+            error => {
+                synthesizer.close();
+                reject(error);
+            }); 
+    });
+};
+
+module.exports = textToSpeech;

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "talking-avatar-backend",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "start": "nodemon ./bin/www"
+  },
+  "dependencies": {
+    "axios": "^1.5.1",
+    "cookie-parser": "~1.4.4",
+    "cors": "^2.8.5",
+    "debug": "~2.6.9",
+    "dotenv": "^16.0.3",
+    "express": "^4.18.2",
+    "fs": "^0.0.1-security",
+    "http-errors": "~1.6.3",
+    "lodash": "^4.17.21",
+    "microsoft-cognitiveservices-speech-sdk": "^1.23.0",
+    "morgan": "~1.9.1",
+    "openai": "^4.11.0",
+    "pug": "^3.0.2"
+  }
+}

public/stylesheets/style.css

@@ -0,0 +1,8 @@
+body {
+  padding: 50px;
+  font: 14px "Lucida Grande", Helvetica, Arial, sans-serif;
+}
+
+a {
+  color: #00B7FF;
+}

routes/index.js

@@ -0,0 +1,27 @@
+var express = require('express');
+var router = express.Router();
+var textToSpeech = require('../helpers/tts');
+var callOpenAI = require('../helpers/callOpenAI');  // Import the helper function
+
+router.post('/talk', async function(req, res, next) {
+  try {
+    const ttsResult = await textToSpeech(req.body.text, req.body.language);
+    res.json(ttsResult);
+
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+router.post('/chat', async function(req, res, next) {
+  try {
+    const userContent = req.body.text;
+    const openAIResponse = await callOpenAI(userContent);
+
+    res.json({ response: openAIResponse });
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+module.exports = router;

views/error.pug

@@ -0,0 +1,6 @@
+extends layout
+
+block content
+  h1= message
+  h2= error.status
+  pre #{error.stack}

views/index.pug

@@ -0,0 +1,5 @@
+extends layout
+
+block content
+  h1= title
+  p Welcome to #{title}

views/layout.pug

@@ -0,0 +1,7 @@
+doctype html
+html
+  head
+    title= title
+    link(rel='stylesheet', href='/stylesheets/style.css')
+  body
+    block content