gaspard7/wiki_gpac huggingface dataset:
Accepting input connections

Once your capabilities are setup, you can get notifications of new inputs through the `filter.configure_pid` callback. This allows you to keep track of PID re-configurations, for later processing.

```

...
filter.pids=[];
...

filter.configure_pid=function(pid)
{
	if (this.pids.indexOf(pid)<0) {
		//this is a new PID, do something
	}
	//check input properties
	let st = pid.get_props('StreamType');
	if (st=='Visual') {
		let width = pid.get_props('Width');
	} 
	else if (st=='Audio') {
		let sr = pid.get_props('SampleRate');
	} else {
		//return GF_Err is allowed
		return GF_NOT_SUPPORTED; 
	}
	//no return value means no error
}
```

The above code allows monitoring PID configuration and performs simple PID property check.

__Discussion__  

Filters properties are mapped to their native type, e.g. unsigned int, boolean, string, float and double, or to objects for vector, arrays, and fractions. There are however a few exceptions here:

- the `StreamType` property is converted to a string (see `gpac -h props` and [Properties](filters_properties))
- the `PixelFormat` property is converted to a string (see `gpac -h props` and [Properties](filters_properties))
- the `AudioFormat` property is converted to a string (see `gpac -h props` and [Properties](filters_properties))
- the `CodecID` property is converted to a string corresponding to the first short name of the codec (see `gpac -h codecs`  and [Properties](filters_properties))
Packet Query

Once you have an input PID in place in your filter, you can start fetching packets from this PID in the `filter.process` callback. The packet access API follows the same principles as non-JS filters:

- packets are always delivered in processing order
- only the first packet of an input PID packet queue can be fetched, and must be explicitly removed
- packets can be reference counted for later reuse
- packets properties can be reference counted for later reuse, while still discarding the associated packet data, thereby unblocking the filter chain
- If you need access to the second packet in the PID queue, you need to reference the first packet and drop the input PID packet queue.
 
```
filter.process = function()
{
	let pck_in = filter.in_pid.get_packet();
	if (pck_in==null) return;
	//any further call to filter.in_pid.get_packet() will return the same pck_in object, until drop_packet is called
	print('Packet DTS is ' + pck_in.dts);
	let data = pck_in.data;
	//inspect data

	//drop the packet - at this stage, if the packet has not been referenced, its associated data is potentially destroyed and shall no longer be used
	filter.in_pid.drop_packet(); 
}
```

The above code will print the DTS of each packet in the input PID.
A simple packet inspector

The following shows a simple packet inspector script

```
//prepare our input PID array
filter.pids=[];
//accept any stream types except files, forcing a demultiplexer
filter.set_cap({id: "StreamType", value: "File", excluded: true} );
//accept only framed inputs (no ADTS or TS packets or the like)
filter.set_cap({id: "Unframed", value: "true", excluded: true} );
//indicate we accept any number of input PIDs.
filter.max_pids=-1;

//configure callback
filter.configure_pid=function(pid)
{
	if (this.pids.indexOf(pid)<0) {
		print('New pid ' + pid.get_prop('ID'));
		//send a PLAY event on the PID (we are a sink filter)
		let evt = new FilterEvent(GF_FEVT_PLAY);
		evt.start_range = 0.0;
		pid.send_event(evt);
		//remember our PID
		this.pids.push(pid);
		//only fetch full coded frames (Access Units)
		pid.framing_mode=true;
		pid.nb_pck = 0;
	} else {
		print('Reconfigure pid ' + pid.get_prop('ID'));	
	}
	//print input properties
	print('Properties:');

	let i=0;
	while (1) {
		let prop = pid.enum_properties(i);
		if (!prop) break;
		i++;
		//print in GPAC JS can use a log level as first argument
		print(GF_LOG_INFO, "Prop " + prop.name + " (type " + prop.type + " ): " + JSON.stringify(prop.value) );
	}
	//no return value means no error
}

//process callback
filter.process=function()
{
	this.pids.forEach(function(pid) {
		//dump any packet ready on the PID
		while (1) {
			//get packet
			let pck = pid.get_packet();
			if (!pck) {
				if (pid.eos) print(GF_LOG_INFO, "pid is in end of stream");
				break;
			}
			//print packet info
			print(GF_LOG_INFO, "PID" + pid.name + " PCK" + pid.nb_pck + " DTS " + pck.dts + " CTS " + pck.cts + " SAP " + pck.sap + " size " + pck.size);
			//dump all packet properties (usually none)
			let i=0;
			while (1) {
				let prop = pck.enum_properties(i);
				if (!prop) break;
				i++;
				print(GF_LOG_INFO, "Prop " + prop.name + " (type " + prop.type + " ): " + prop.value);
			}
			//access packet data, and dump first 4 bytes
			let data = pck.data;
			if (data) {
				let view = new Uint8Array(data);
				print('data buffer, size ' + view.length + ' first 4 bytes ' + view[0].toString(16) + '' + view[1].toString(16) + '' + view[2].toString(16) + '' + view[3].toString(16));
			}

			//drop packet
			pid.drop_packet();
		}
	}
	);
}
```

To run the inspector script on a source, simply execute:
```
gpac -i source_file script.js
```
Declaring output PIDs

In GPAC, output PIDs can be declared pretty much at any time in the life cycle of the filter, except upon destruction. You must however declare your output PIDs capabilities:

```
filter.set_cap({id: "StreamType", value: "Video", output: true});
filter.set_cap({id: "CodecID", value: "raw", output: true});
```

The above script will declare an output capability bundle of RAW (uncompressed) video. Note that we use here 'Video' instead of 'Visual', the two names are equivalent.

An output PID can only be created for the current filter. This will setup a new output PID dispatching raw video in rgb format:
```
filter.opid = filter.new_pid();
filter.opid.set_prop("StreamType", "Video");
filter.opid.set_prop("CodecID", "raw");
filter.opid.set_prop("PixelFormat", "rgb");
```

You can also assign user-defined properties to the PID as follows:
```
filter.opid.set_prop("MyTestProperty", "My Current State", true);
```

PID properties are automatically attached to packets sent on the PID. Any modification to the PID properties will mark the next packet sent as a reconfiguration point, which will trigger a `configure_pid` on consuming filters. If you wish to attach information on a PID without triggering reconfiguration (for example, PID state not related to packet processing, such as current download rate), you can use PID information. Any modification to the PID information set will mark the next packet sent as an info update point, which will trigger a call to `process_event`  on consuming filters.

 ```
 filter.opid.set_info("MyTestInfo", "My Current Info", true);
 ```
 
When a PID is used to forward (potentially after processing) packets from an input PID, it is recommended to copy properties of the source PID to the destination PID as follows:

```
filter.configure_pid = function(pid)
{
	if (typeof pid.opid == 'undefined') {
		pid.opid = filter.new_pid();
	}
	pid.opid.copy_props(pid);
	//then modify opid properties as desired
}
```

__Discussion__  
The property copy is recommended especially when SourceIDs, URL templating or filter chain templating are used, as explained in [general filter concepts](filters_general). In these cases, the graph resolution or a destination filter may use a property of the input PID, which would be lost if not copied.

As a general rule, you should always copy source PID properties to output PID, and then rewrite or remove any needed properties. Removing a property is done by using a `null`value:
```
pid.opid.set_props(pid, "MIMEType", null);
```
A simple packet generator

The following is a simple packet generator creating 100 packets of subtitles

```
//we produce one text stream, using codec "simple text" ('subs')
filter.set_cap({id: "StreamType", value: "Text", output: true});
filter.set_cap({id: "CodecID", value: "subs", output: true});
filter.nb_pck = 0;

filter.initialize = function() {
	this.opid = this.new_pid();
	this.opid.set_prop("StreamType", "Text");
	this.opid.set_prop("CodecID", "subs");
	this.opid.set_prop("Timescale", "1000");
	//you can set a decoder config for text streams, usually a file header - not that this is optional
	this.opid.set_prop("DecoderConfig", "My Super Config");
}

filter.process = function()
{
	if (!this.opid)
		return GF_EOS;

	if (this.nb_pck>=100) {
 		this.opid.eos = true;
 		return GF_EOS;
	}
	this.nb_pck++;
	
	pck = this.opid.new_packet("Packet number " + this.nb_pck);
	pck.cts = 1000*this.nb_pck;
	pck.dts = 1000*this.nb_pck;
	pck.dur = 1000;
	pck.sap = GF_FILTER_SAP_1;
	pck.send();
}
```
Loading filters from a JS filter

JS filters can load other filters to create complex processing chain. This can be done in the following ways:

* load a source filter: this allows loading a demultiplexing chain loading data from a given URL
```
let src_f = filter.add_source("myfile.ts");
let src_f = filter.add_source("http://host/source.mp4");

```
Any URL supported by GPAC for source filter loading can be used.

* load a destination (sink) filter: this allows loading a multiplexing chain targeting a given URL
```
let src_f = filter.add_destination("myfile.ts");
let src_f = filter.add_destination("pipe://mymux.gsf");

```

Any URL supported by GPAC  for destination/sink filter loading can be used.

* load a generic filter: this allows loading any filter supported by GPAC
```
let vout_f = filter.add_filter("vout");

```

This will load the video output filter with no specific arguments. 

For each of these methods, the filter name or URL used  can specify filter options, as usual within GPAC filter chains. For example:
```
let vout_f = filter.add_filter("vout:vsync=no");

```

This will load the video output filter with vsync disabled. 


A filter loaded by a JS filter does not expose the JSFilter API, but the [FilterInstance API](https://doxygen.gpac.io/interface_filter_instance.html). In other words, you cannot manipulate the loaded filter by adding or removing PIDs or processing/sending packets in place of this filter. If you build a complex filter chain and need to indicate which of your filter outputs may connect to which loaded destination, you must set the FIDs and SourceID options of your filters, or use the `set_source()` function.

Example using filter IDs:
```
let a_filter = filter.add_filter("vflip:FID=MyFlip");
...
a_filter.set_source(filter);

let vout_f = filter.add_filter("vout:vsync=no");
//vout will only accept PID coming from the filter with ID "MyFlip"
vout_f.set_source(a_filter, "SourceID=MyFlip");
```


Example using PID properties:
```
let an_output = filter.new_pid();
...
an_output.set_prop("MuxSrc", "myPid");

let vout_f = filter.add_filter("vout:vsync=no");
//vout will only accept PID coming from this filter and with property "MuxSrc" set to "myPid" 
vout_f.set_source(filter, "MuxSrc=myPid");
```
Including filters in your distribution {:data-level=beginner"}

JS files located in GPAC distribution or in the directories indicated using [-js-dirs](core_options#js-dirs) option can describe filters usable by the filter engine based on their name (file without extension or directory name).

Single file case:
```
myfilters/
myfilters/foo.js
```

The filter can be loaded using `foo`, e.g. `gpac -js-dirs=myfilters -i src foo`.

The JS sources for a given filter can also be gathered in a single folder; in that case, the main JS file shall be called `init.js`. For example:
```
myfilters/
myfilters/foojs/
myfilters/foojs/init.js
```

The filter can be loaded using `foojs`, e.g. `gpac -js-dirs=myfilters -i src foojs`.
Standalone script

The standalone mode works by specifying a JS file to the dashin filter using its [-algo](dashin#algo) option:

```
gpac -i source.mpd --algo=mydash.js ...
```

The JS script is loaded with a global object called `dashin` with 4 callback functions:

-  period_reset: indicates start or end of a period (optional)
-  new_group: indicates setup of a new adaptation group (optional), i.e. DASH AdaptationSet or HLS Variant Stream. For HEVC tiling, each tile will be declared as a group, as well as the base tile track
-  rate_adaptation: performs rate adaptation for a group (mandatory). Return value is the new quality index, or -1 to keep as current, -2 to discard (debug, segments won't be fetched/decoded)
-  download_monitor: performs download monitoring, potentially resulting in download abort request (optional)

The following is a basic JS example performing dummy adaptation (always play lowest quality) :

```
import { Sys as sys } from 'gpaccore'

print("Hello DASH custom algo !");

let groups = [];

dashin.period_reset = function (reset_type)
{
	print("Period reset type " + reset_type);
	if (!reset_type)
		groups = [];
}

dashin.new_group = function (group)
{
	print("New group: " + JSON.stringify(group));
	//remember the group (adaptation set in dash)
	groups.push(group);
}

dashin.rate_adaptation = function (group_idx, base_group_idx, force_lower_complexity, stats)
{
	print(`Getting called in custom algo ! group ${group_idx} base_group ${base_group_idx} force_lower_complexity ${force_lower_complexity}`);
	print('Stats: ' + JSON.stringify(stats));

	//always use lowest quality
	//you could check group.SRD to perform spatial adaptation
	return 0;
}

dashin.download_monitor = function (group_idx, stats)
{
	print("Download info " + JSON.stringify(stats));
	return -1;
}

```
Attaching algo from a JS session

The first step in your JS is to create a DASH object implementing the callbacks previously indicated:


```
//custom rate adaptation object
let dashalgo = {};
dashalgo.groups = [];

dashalgo.rate_adaptation = function (group, base_group, force_lower_complexity, stats)
{
	return 0;
}

dashalgo.new_group = function (group)
{
	this.groups.push(group);
}

dashalgo.period_reset = function (type)
{
	if (!type)
		this.groups = [];
}

```


You will then need to setup a JS session monitoring filter creation process:

```
session.set_new_filter_fun( (f) => {
		print("new filter " + f.name);

		//bind our custom rate adaptation logic
		if (f.name == "dashin") {
			f.bind(dashalgo);
		}
} ); 

```

And you're good to try new rate adaptation algorithm !
Overview {:data-level="all"}

We discuss here how to use [gpac](gpac_general) or the [JavaScript Filter](jsf) to query and control from JavaScript the filter session in GPAC.  
The [JS scripts](https://github.com/gpac/testsuite/tree/filters/media/jsf) in the gpac test suite are also a good source of examples.

The [JS FilterSession](https://doxygen.gpac.io/group__fsess__grp.html) provides JS bindings to the GPAC filter session object. It is recommended to also check the documentation of the [JS APIs](https://doxygen.gpac.io/group__jsf__grp.html) for more details. 


The FilterSession JS bindings can be loaded in three different ways in GPAC.

- A global controller script can be specified as an argument to gpac using [-js](gpac_general#js) option:
```gpac -js=myscript.js` [OPTIONAL ARGS]```

- available by default in [JavaScript Filters](https://doxygen.gpac.io/group__jsf__grp.html) API

- available by default in [JavaScript Compositor](https://doxygen.gpac.io/group__scenejs__grp.html) API

The filter session object is exposed as a global variable named `session`.

__WARNING__  
The filter session API can only be loaded once per session. The implies that using both `-js` and the compositor, or `-js` and a JSFilter using this API will fail.

__Discussion__  
Since the session API is available in a JSFilter, you can load a script directly using `gpac script.js`. This will however create a JSFilter inside the session, but this filter will be automatically disabled (not used in the graph resolution, leaving it not connected) if the following conditions are met after initialization:

- filter did not assign any capabilities
- filter did not create any output PID
- filter did not post any task using filter.post_task
Listing filters in a session

__WARNING__  
The [filter object](https://doxygen.gpac.io/interface_j_s_f_s_filter.html) exposed by this API different from the [JavaScript Filter](https://doxygen.gpac.io/group__jsf__grp.html) one: it cannot be used to create packets, PIDs and so on on a given filter.


Due to the dynamic nature of a filter session (multi-threading, dynamic filter chain configuration), getting a listing of filters should be done in exclusive mode:
```
session.lock_filters(true);
let nb_filters = session.nb_filters;

for (let i=0; i<session.nb_filters; i++) {
	let filter = session.get_filter();
}
session.lock_filters(false);
```

Each filter JS object is valid for the lifetime of the underlying filter.

The filter session can be configured to check for filter creation and deletion through callbacks:

```
let all_filters = [];

session.set_new_filter_fun( (f) => {
	print("new filter " + f.name);
	f.iname = "JS"+f.name;
	all_filters.push(f);
} ); 

session.set_del_filter_fun( (f) => {
	print("delete filter " + f.iname);
	let idx = all_filters.indexOf(f);
	if (idx>=0)
		all_filters.splice (idx, 1);
}); 

```

You can also check if a filter has not been destroyed by using `is_destroyed()` function. 


The specific property `iname` is a string identifier reserved for JS, and used to query a filter by name in the session:

```
let f = session.get_filter('my cool filter');
```

Note that in this case, there is no need to lock the session from javascript.
The property `iname` is also shared with the [JavaScript Filter](https://doxygen.gpac.io/group__jsf__grp.html).
Getting notified during session execution

The filter session is running through an internal task scheduler. You can post tasks to this scheduler for your script:

```
session.post_task( ()=> {
	if (session.last_task) {
		print("we are done ");
		//the task will no longer be called 
		return false;
	}
	all_filters.forEach( (f) => { print('Name: '+f.name)});
	//call back in 1sec
	return 1000;
});

```
Creating filters

You can load any filter during the session, connecting it from any existing filter if needed. 
- source filters must be loaded as `src=URL:opts`
- destination filters must be loaded as `dst=URL:opts`

__Note__  
Injecting a filter in the middle of a connected chain (i.e. going from A->B to A->newF->B) is currently not supported.


```
//load a source
let src = session.add_filter("video.mp4");
//load an inspect filter, getting its input only from src
let f = session.add_filter("inspect", src);
```

All filter options valid in [GPAC](filters_general) can be used. This means that you can specify complex filter graphs using SID and FID syntax:

```
//load 3 sources
let src1 = session.add_filter("video.mp4:FID=1");
let src2 = session.add_filter("video.mp4:FID=2");
let src3 = session.add_filter("video.mp4:FID=1");
//load an inspect filter, getting its input only from src1 and src2
let f = session.add_filter("inspect:SID=1,2");

//load a ISOBMF mux filter, getting its input only from src2 and src3
let f = session.add_filter("dst=mux.mp4:SID=2,3");
```

To insert a filter before or after a specific filter:
```
let f = session.get_filter('my cool filter');
//insert a TS mux after a given filter
f.insert("dst=mux.ts");
```

To remove a given filter:
```
let f = session.get_filter('my cool filter');
f.remove();
```

To update a filter option:
```
let f = session.get_filter('my cool filter');
f.update("opt_name", "opt_val");
```
Querying filters

All properties of a filter object are enumerable:

```
print("Filter properties:");
for(let propertyName in f) {
	print("f." + propertyName + " : " + f[propertyName]);
}
```

You can query the number of input and output PIDs of a filter, and enumerate their properties:

```
//input PID
print("Filter num input PIDs: " + f.nb_ipid);
for (i=0; i<f.nb_ipid; i++) {
	//enum props
	f.ipid_props(i, (name, type, val) => { print('input pid prop ' + name + ' type ' + type + ' val ' + val);})

	//or direct query
	let st = f.ipid_props(i, 'StreamType');
}

//output PID
print("Filter num output PIDs: " + f.nb_opid);
for (i=0; i<f.nb_opid; i++) {
	f.opid_props(i, (name, type, val) => { print('output pid prop ' + name + ' type ' + type + ' val ' + val);})
}
```

Reminder: The list of available properties is [`gpac -h props`](filters_properties).
 

You can query the source filter of a given input PID:

```
print("Filter sources: ");
for (i=0; i<f.nb_ipid; i++) {
	let s = f.ipid_source(i);
	print("PID"+i+" source: " + s.name);
}
```

You can query the filters connected to an output PID:

```
print("Filter destinations: ");
for (i=0; i<f.nb_opid; i++) {
	let sinks = f.opid_sinks(i);
	print("PID"+i+" destinations:");
	for (j=0; j<sinks.length; j++) {
		print(" "+sinks[i].name);
	}
}
```

You can query the arguments defined on a filter:

```
let args = f.all_args();
print("" + args.length + " arguments: " + JSON.stringify(args) );
```
Handling events

The filter session can receive UI-related events global to the session.

```
session.set_event_fun( (evt) => {
print("evt " + evt.name);
if (evt.type != GF_FEVT_USER) return;

if (evt.ui_type == GF_EVENT_SIZE) {
	print('display size is ' + evt.width + 'x' + evt.height);
	return false;
}

});

```

The filter session can also be used to fire events events on filters accepting UI events. 

```
let f_evt = new FilterEvent(GF_FEVT_USER);
f_evt.ui_type = GF_EVENT_SET_CAPTION;
f_evt.caption = "forced caption";
session.fire_event(f_evt);
```

The filter session can also be used to fire non-UI related events on filters. You must be extra careful when using this, as this might trigger unwanted behavior in the chain. Typically:

- upstream events (towards sink) should only be fired on source filters (nb_ipid = 0)
- downstream events (towards source) should only be fired on sink filters (nb_opid = 0)

```
let f_evt = new FilterEvent(GF_EVENT_STOP);
session.fire_event(f_evt, target_filter);
```
Remote interaction

GPAC is by default compiled with [Remotery](https://github.com/Celtoys/Remotery) support, and can use the underlying websocket server of remotery to communicate with a web browser.

You will need for this:

- to launch GPAC with remotery activate by specifying [-rmt](core_options#rmt)
- set a handler function to listen to messages from the web client using `session.set_rmt_fun`
- send messsages to the web client using `session.rmt_send`

The following is an example of using remotery in JS:

```
session.set_rmt_fun( (text)=> {
	print("rmt message " + text);
	//do something
	
	//reply, either at once or later
	session.rmt_send("yep !");
});

```

Since the remotery code in GPAC is not modified, only text messages can currently be sent. We recommend exchanging data through JSON.

By default, sampling times collecting is enabled in remotery. You can enable or disable at runtime this:

```
if (session.rmt_enabled)
	print("disabling rmt sampling!");
session.rmt_enabled = false;
});

```
Creating custom filters

You can create your own custom filters in a JS session using `new_filter`. The returned object will be a  [JavaScript Filter](jsfilter) with the following limitations:

- no custom arguments for the filter can be set
- the `initialize` function is not called
- the filter cannot be cloned
- the filter cannot be used as source of filters loading a source filter graph dynamically, such as the dashin filter.
- the filter cannot be used as destination of filters loading a destination filter graph dynamically, such as the dasher filter.

```

let my_filter = session.new_filter("MyFilter");

//let the filter accept any input of type video
my_filter.set_cap({id: "StreamType", value: "Visual"});

//check input connections
my_filter.configure_pid = function(pid)
{
}

//process packets
my_filter.process = function()
{
}

```