Using TUIO with javascript
Tags: javascript, tuio, touchscreen, node
Published: 2024-06-18 07:50:32

I recently developed an application for a museum running on a [TILE](https://www.displax.com/tile) display from Displax.
In order to get this up and running I needed to create a node.js server and websocket connection. Recent examples of this are hard to find online, so I want to share my code for anyone in the same situation, hopefully this saves you some time :-)

First you must have installed a server and node.js

Then install the packages needed through terminal/command prompt:

```
npm install osc express socket.io bufferutil utf-8-validate --no-audit
```


In your html file include this:
https://cdn.socket.io/4.7.5/socket.io.min.js

Now you need to create a server file, let's call it "server.js":

```
const bufferUtil = require('bufferutil');//maybe not needed, but maybe it speeds things up!
var osc = require('osc');
const express = require('express');
const { createServer } = require('node:http');
const { Server } = require('socket.io');
const app = express();
const server = createServer(app);
var socket;

const io = require("socket.io")(server, {cors:{origin: "*",methods: ["GET", "POST"]}});

//Listen to the TUIO data
const udpPort = new osc.UDPPort({
  localAddress: "127.0.0.1",
  localPort: 3333,
  metadata: true
});

//Listen/send on port 3000 or 5000
server.listen(3000, () => {
  console.log('Server running at http://localhost:3000');
});

io.on('connection', function (_socket) {
  socket = _socket;
    socket.on('config', function (obj) {
      console.log('config', obj);
    });
});

// Listen for incoming OSC bundles.
udpPort.on("bundle", function (oscBundle){
  if(socket && oscBundle.packets.length > 2) socket.emit('message', oscBundle);//only send TUIO v1.1
});

// Open the socket.
udpPort.open();
```


Now for connecting in the front-end, we need this script:

```
function WSConnection(_address){
	const socket = io(_address);//, {withCredentials: true}
	socket.on('error', function(){
		console.log("Error connecting!");
	});
	socket.on('connect', function(){
		console.log("Server connected");
		socket.emit('config', {server:{port: 3333,host: '127.0.0.1'},client: {port: 3334,host: '127.0.0.1'}});
	});
	socket.on('message', function(oscBundle){
	});
}
//Init
WSConnection("http://127.0.0.1:3000");//or port 5000
```

To start the server open terminal and cd into the server.js folder and enter:
`node server.js`

Now your have a running node.js server and a front-end that listens to any TUIO objects being sent through the server.

In order to test you can download the "TUIOSimulator.app" from here:
https://github.com/gregharding/TUIOSimulator


When above works you are facing the next challenge. Depending on the hardware and software you are using, TUIO objects can often be a little unreliable. Sometimes events are fired too slowly, so you think an object is removed from the display. So I made some custom work arounds for these scenarios. In my example I'm using both touch and object recognition, so I have to use both "tuio/2Dcur" and "tuio/2Dobj".

This is my "message" function:

```
var _alive = [], _aliveTags = [], _aliveCursors = [], _cursors = [];
var _l = 0, _id = 0, _numAlive = 0, _cursorId = 0;
var _v = "";

socket.on('message', function(oscBundle){
	_l = oscBundle.packets.length;
	for(var i=0;i<_l;i++){
		if(oscBundle.packets[i].address == "/tuio/2Dcur"){
			_v = oscBundle.packets[i].args[0].value;
			if(_v == "alive"){
				//Find alive cursors
				_numAlive = oscBundle.packets[i].args.length - 1;
				_alive = [];
				for(var l=0;l<_numAlive;l++){
					_cursorId = oscBundle.packets[i].args[l+1].value%100;
					_alive.push(_cursorId);
					if(_aliveCursors.indexOf(_cursorId) == -1){
						console.log("Add cursor",_cursorId);
						_aliveCursors.push(_cursorId);
						_cursors[_cursorId] = new TuioObj(_cursorId,false);
					}
					else _cursors[_cursorId]._removed = false;//keep alive (if set to be removed on next render)
				}
				//Find old cursors not alive anymore
				_numAlive = _aliveCursors.length;
				for(var l=0;l<_numAlive;l++){
					if(_alive.indexOf(_aliveCursors[l]) == -1){
						_id = _aliveCursors[l];
						if(_cursors[_id]._removed){
							console.log("Destroy cursor", _id);
							_cursors[_id].destroy();
							delete _cursors[_id];
							_aliveCursors.splice(l,1);
							_numAlive--;
							l = 0;
						}
						else{
							//console.log("Remove cursor", _id);
							_cursors[_id]._removed = true;
						}
					}
				}
			}
			else if(_v == "set"){
				_cursorId = oscBundle.packets[i].args[1].value%100;
				if(_aliveCursors.indexOf(_cursorId) != -1) _cursors[_cursorId].setXY(oscBundle.packets[i].args[2].value * _appW,oscBundle.packets[i].args[3].value * _appH);
				else console.log("Cursor not found!", _cursorId);
			}
		}
		else if(oscBundle.packets[i].address == "/tuio/2Dobj"){
			_v = oscBundle.packets[i].args[0].value;
			if(_v == "alive"){
				//Find alive cursors
				_numAlive = oscBundle.packets[i].args.length - 1;
				_alive = [];
				for(var l=0;l<_numAlive;l++){
					_cursorId = oscBundle.packets[i].args[l+1].value%100;
					_alive.push(_cursorId);
					if(_aliveTags.indexOf(_cursorId) == -1){
						console.log("Add tag",_cursorId);
						_aliveTags.push(_cursorId);
						_tags[_cursorId] = new TuioObj(_cursorId,true);
					}
					else _tags[_cursorId]._removed = false;//keep alive (if set to be removed on next render)
				}
				//Find old cursors not alive anymore
				_numAlive = _aliveTags.length;
				for(var l=0;l<_numAlive;l++){
					if(_alive.indexOf(_aliveTags[l]) == -1){
						_id = _aliveTags[l];
						if(_tags[_id]._removed){
							console.log("Destroy tag", _id);
							_tags[_id].destroy();
							delete _tags[_id];
							_aliveTags.splice(l,1);
							_numAlive--;
							l = 0;
						}
						else{
							//console.log("Remove tag", _id);
							_tags[_id]._removed = true;
						}
					}
				}
			}
			else if(_v == "set"){
				//console.log("Tag static id:", oscBundle.packets[i].args[2].value)
				_cursorId = oscBundle.packets[i].args[1].value%100;
				if(_aliveTags.indexOf(_cursorId) != -1) _tags[_cursorId].setXYR(oscBundle.packets[i].args[3].value * _appW,oscBundle.packets[i].args[4].value * _appH,oscBundle.packets[i].args[5].value);
				else console.log("Tag not found!", _cursorId);
			}
		}
	}
});
```

For each TUIO object, either a Tag or Touch, I am using this TuioObj, here's a simplified version of mine:

```
function TuioObj(_id,_isTag){
	var _this = this;
	_this._removed = false;//TUIO "alive" events are not reliable and often remove an object only to add it instantly again!
	
	//Touch (x and y coordinate)
	_this.setXY = function(x,y){}
	
	//Tag (x and y coordinate and rotation value)
	_this.setXYR = function(x,y,r){}

	//Destroy (remove DOM elements, event listeners etc.)
	_this.destroy = function(){}
}
```

Of course many more features, like idle handling, smooth movement (check my [post on lerp](https://dev.to/iliketoplay/using-lerp-and-damp-in-javascript-3f7p)) and distance measurement (for click handling etc.) can be added, but now you should have a template to get you started.