SlowGuess/NeurIPS_2023_OCR huggingface dataset:
class AnimalBehaviorAnalysis:   
""   
Methods   
 $--$    
get_object_names() -> List[str] get names of all objects   
""   
def animals_state_events(self, state_type, comparison, bodyparts  $=$  ['all'],   
):   
""   
Parameters   
state_type: str Must be one of 'speed', 'acceleration' comparison: str Must be a comparison operator followed by a number like  $<  50$  Examples   
>>> # when is the animal moving faster than 3 pixels across frames? >>> def task_program(): >>> behavior_analysis  $\equiv$  AnimalBehaviorAnalysis() >>> animal_faster Than_3_events  $=$ $\hookrightarrow$  behavior_analysis.animals_state_events('speed','>3') >>> return animal_faster Than_3_events   
""   
return animals_state_events(state_type, comparison)
def superanimal(video_inference(self) -> None:
    ""
Examples
>>> # extract keypoints (aka pose) from the video file
>>> def task_program():
    >>> behavior_analysis = AnimalBehaviorAnalysis()
>>> behavior_analysis(superanimal(video_inference())
" return superanimal(video_inference()
def animals_object_events(self,
object_name: str, relation_query, comparison = None, negate = False, bodyparts: List[str] = ['all'], min_window = 0, ) -> EventDict: object_name: str. Name of the object relation_query: str. Must be one of 'to_left', 'to_right', 'tobelow', to 'to_above', 'overlap', 'distance', 'angle', 'orientation' comparison: str, Must be a comparison operator followed by a number like  $<50$ , optional bodyparts: List[str], optional min_window: min length of the event to include max_window: max length of the event to include negate: bool, default false whether to negate the spatial events. For example, if negate is set True, inside roi would be outside roi Returns: EventDict Examples >>> # find where the animal is to the left of object 6 >>> def taskprogram(): >>> behavior_analysis = AnimalBehaviorAnalysis() >>> left_to_object6_events = behavior_analysis.animals_object_events('6', to 'left', bodyparts = ['all']) >>> return left_to_object6_events >>> # how much time the animal spends in closed arm? >>> def taskprogram(): >>> behavior_analysis = AnimalBehaviorAnalysis() >>> in Closed_arm_events = behavior_analysis.animals_object_events('closed arm', 'overlap', bodyparts = ['all'], negate = False) >>> return in Closed_arm_events >>> # get events where animals's distance to animal0 is less than 30 >>> def taskprogram(): >>> behavior_analysis = AnimalBehaviorAnalysis() >>> distance_social_events = behavior_analysis.animals_object_events('animal0', 'disntace', comparison ='<30', bodyparts = ['nose']) return animals_object_events(object_name, relation_query)
def plot_trjectory(self, bodyparts: List[str], events: Union[Dict[str, dict], dict] =
$\hookrightarrow$  None, \*\*karges):
bodyparts : List[str]
events: Union[Dict[str, dict], dict], optional
The type must be either dict or dictionary of dict
kwargs : parameters for plt.scan
Returns:
Tuple[plt].[Figure, plt.Axes]
tuple of figure and axes
>>> # plot trajectory of the animal.
>>> def task_program():
>>> behavior_analysis = AnimalBehaviorAnalysis()
>>> fig, ax = behavior_analysis.plot_trjectory(['all'])  
>>> return fig, ax  
>>> # plot trajectory of the nose of the animal with the event that animal  
 $\rightarrow$  overlaps with object 6  
>>> def task_program():
>>> behavior_analysis = AnimalBehaviorAnalysis()
>>> overlap_6_events = behavior_analysis.animals_object_events('6',  
 $\rightarrow$  'overlap', bodyparts = ['all']
>>> fig, ax = behavior_analysis.plot_trjectory(['"nose"', events =  
 $\rightarrow$  overlap_6_events)
>>> return fig, ax
"..." return plot_trjectory(bodyparts, events)
do not call animals_social_events if there is no multiple animals or social events
def animals_social_events(self,
					 relation_query_list,
					comparison_list,
					animal_state關係_query_list = [],
					animal_state_comparison_list = [],
					 bodyparts = ['all'],
					otheranimal_bodyparts = ['all'],
					 min_window = 11,
					pixels_per_cm = 8,
					smooth_window_size = 5):
				"The function is specifically for capturing multiple animals social events
Parameters
			- - -
			relation_query_list: List[str]
			1ist of relation query for animals-animals relationship. Must be chosen from
				['to_left', 'to_right', 'tobelow', 'to_overlap', 'distance', 'orientation',
				'closest_distance', 'angle']. 
					comparison_list: List[str]
					1ist of comparison operator such as ':=' ,'<','>',<=','>='. 
					animal_state關係_query_list: List[str]
					1ist of relation query for state of the animal that interacts with other animals.
			'Must be chosen from ['speed', 'acceleration', 'confidence']. 
					animal_state_comparison_list: List[str]: 
					1ist of comparison operator such as ':=' ,'<','>',<=','>='. 
					 bodyparts: List[str] 
					1ist of bodyparts for the main animal
					otheranimal_bodyparts: list[str] 
					1ist of bodyparts for the other animal
					min_window: int, optional
					Only include events that are longer than min_window
					pixels_per_cm: int, optional
					how many pixels for 1 centimeter
					smooth_window_size: int, optional
					smooth window size for smoothing the events.
					Examples
						>>> # Define 
 as a social behavior where distance between animals are
 $\rightarrow$  less than 100, one animal is in front of the other animal and the chasing animal has
 $\rightarrow$  speed faster than 3. Get chases.
						>>> def task_program():
						>>> chase_events = behavior_analysis.animals_social_events(['distance'],
						'orientation'],
						>>> [f'< 100', f'= {Orientation.FRONT}]',
						>>> animal_state關係_query_list = ['speed'],
>>> animal_state比較comparison_list=['>=3'],   
>>> bodyparts  $=$  ['all'],   
>>> otheranimal_bodyparts  $=$  ['all'])   
>>> return chase_events   
return animals_social_events(relation_query_list, comparison_list)
class Orientation(IntEnum):
Attributes
---->
FRONT
BACK
LEFT
RIGHT
" "
class Event:
    @classmethod
def add_simultaneous_events(cls, *events_list: List[dict]) -> dict:
        Parameters
        ...
        events_list: List[dict]
        Returns
        ...
        EventDict
        Examples
        ...
        >> get events for the animal's nose in the roi and the animal's tail base not in
    the roi
        >> def task_program():
            >>> behavior_analysis = AnimalBehaviorAnalysis()
            >>> nose_in_roi_events = behavior_analysisanimals_object_events('ROI',
        'overlap', bodyparts = ['nose'], negate = False)
            >>> tail_base_not_in_roi_events = 
    behavior_analysisanimals_object_events('ROI', 'overlap', bodyparts = ['tail base'],
    negate = True)
            >>> return Event.add_simultaneous_events(nose_in_roi_events,
    tail_base_not_in_roi_events)
        return add_simultaneous_events(event_list)
    @classmethod
def add Sequential_events(cls,
                    *events_list: List[dict],
                    continuous = False) -> dict:
        ..." 
Keywords such as "then", "later" might suggest it is a sequential event
Parameters
        ...
        events_list: List[dict]
        Returns
        ...
        EventDict
        Examples
        ...
        >> # Get events where the animal enters object 6 from the bottom.
        >> def task_program():
            >>> behavior_analysis = AnimalBehaviorAnalysis()
            >>> bottom_to_object_6_events = 
    behavior_analysisanimals_object_events('6', 'tobelow', bodyparts = ['all']
    >>> enter_object_6_events = behavior_analysis-enter_object('6', bodyparts = 
    ['all']) 
    >>> enter_object_6_from_bottom_events = 
    Event.addSequential_events(bottom_to_object_6_events, enter_object_6_events)
>>> return enter_object_6_from_bottom_events
>>> # Find events where animal leaves object 6
>>> def task_program():
>>> behavior_analysis = AnimalBehaviorAnalysis()
>>> leave_object_6_events = behavior_analysis.leave_object('6', bodyparts = ['all'])  
>>> return leave_object_6_events
>>> # Find events where the animal leaves object 6 and then enters object 3
>>> def task.program():
>>> behavior_analysis = AnimalBehaviorAnalysis()
>>> leave_object_6_events = behavior_analysis.leave_object('6', bodyparts = ['all'])  
>>> enter_object_3_events = behavior_analysisentered_object('3', bodyparts = ['all'])  
>>> leave_object_6_and-enter_object_3_events =  
Event.addsequential_events(leave_object_6_events, enter_object_3_events)
>>> return leave_object_6_and-enter_object_3_events
>>> # find events where the animal moves from left of object 6 to right of object  
to 6.
>>> def task PROGRAM():
>>> behavior_analysis = AnimalBehaviorAnalysis()
>>> left_to_object_6_events =  
behavior_analysisanimals_object_events('6','to_left', bodyparts = ['all'])  
>>> right_to_object_6_events =  
behavior_analysisanimals_object_events('6','to_right', bodyparts = ['all'])  
>>> left_to_right_events =  
Event.addsequential_events(left_to_object_6_events, right_to_object_6_events)
>>> return left_to_right_events
""
return addsequential_events(eventlist)  
class EventDict_dict:
def sort(self):
    Sort the eventdict
Return
-----
EventDict
return sort()
Integration of behavioral modules (non-exhaustive)
As described in the Main Section 3.5 and Main Figure 2, AmadeusGPT can use integration modules for task-specific behavior analysis, and implements dynamic loading by converting API documents into text embedding vectors and caching them in RAM or disk, allowing users to retrieve the most relevant modules by cosine similarity using their prompt as a query. Namely, the user could use these modules to integrate functionality. Or, they can write simple modules and contribute them to the AmadeusGPT codebase for other users to leverage as well. Below we provide examples of integration with the newly introduced dimensionality reduction tool called cebra [35], and UMAP [34]. In short, to integrate into the codebase the user should write a minimal working example in the style of the core API docs. Then, after dynamic loading, queries relating to UMAP or cebra (as shown here) would generate the following code (below) and related output plots (Figure 6).
CEBRA
CEBRA is a nonlinear dimensionality reduction tool for joint modeling of neural data and behavioral labels [35], which might be of interest to users of AmadeusGPT. In Appendix Figure 6 we show a 3D CEBRA embedding of mouse poses (aligned) colored by when it overlaps with different objects. Note that the CEBRA embedding in Figure 6 is on a 3D n-sphere that is equivalent to a 2D Euclidean
Annotated MausHaus masks
Figure 6: Integration examples. Left to right: MausHaus masks with human annotated names, examples of UMAP and cebra integration modules, plots colored as in annotations. See below for user prompts to produce the plots.
space [35]. Below the user query is shown below to produce images in Figure 6, along with the related API docs.
Query: "get 3D embeddings using cebra and plot the embedding."
def compute_embedding_with_cebra(n_dimention  $= 3$  max_iterations  $\coloneqq 100$  ..   
"" Examples   
>>> # create a 3 dimensional embedding with cebra   
>>> def task_program(): behavior_analysis  $=$  AnimalBehaviorAnalysis() embedding  $=$  behavior_analysis.compute_embedding_with_cebra(n_dimention  $= 3$    
>>> return embedding   
"" return compute_embedding_with_cebra(n_dimention  $=$  n_dimention,   
 $\leftrightarrow$  max_iterations  $\equiv$  max_iterations)
UMAP
UMAP [34] is a popular nonlinear dimensionality reduction tool that is also core to several behavioral analysis tools such as B-SOID [46]. In Appendix Figure 6 we show 2D UMAP embedding of poses (aligned) when it overlaps with different objects.
Query: "Get 2D embeddings using umap and plot the embedding"
def compute_embedding_with_umap(n_dimention  $= 3$  ..   
"   
Examples   
>>> # create a 3 dimensional embedding umap   
>>> def task_program features):   
>>> behavior_analysis  $\equiv$  AnimalBehaviorAnalysis()   
>>> embedding  $\equiv$  behavior_analysis.compute_embedding_with_umap(n_dimention  $= 3$    
>>> return embedding   
""   
return compute_embedding_with_umap(n_dimention  $\equiv$  n_dimention)
Global spatio-temporal behavioral analyses
Many core behavioral analysis tools rely on computing metrics based on kinematic or movement analysis. For example, time spent locomoting, time in an ROI, or computing the distance traveled within a specific area is highly popular in behavioral neuroscience research. Thus, inspired by DLC2Kinematics [47], we show API docs to integrate two such common integration modules:
def calculate_distance_travelled(self, events):
    Parameters
events: EventDict  
Return  
int  
Examples  
>>> # what is the distance travelled in closed arm?  
>>> def task_program():  
>>> behavior_analysis = AnimalBehaviorAnalysis()  
>>> roi_events = behavior_analysisanimals_object_events('closed arm','overlap',  
 $\rightarrow$  bodyparts  $=$  ['all'])  
>>> distance_in_roi = behavior_analysis.calculate_distance_travelled(roi_events)  
>>> return distance_in_roi  
return calculate_distance_travelled(events)  
def plot_object_ethogram(self, object_names):  
Parameters  
object_names: List[str]  
list of object names  
Return:  
Tuple[plt.Figure,plt.Axes] tuple of figure and axes  
Examples  
>>> # plot ethogram for animal overlapping all objects  
>>> def task_program():  
>>> behavior_analysis = AnimalBehaviorAnalysis()  
>>> object_names = behavior_analysis.get_object_names()  
>>> fig, ax = behavior_analysis.plot_object_ethogram(object_names)  
>>> return fig, ax  
"..."  
return plot_object_ethogram(object_names)
Additional Experiments: behavioral analysis examples
Here, as related to Main Section 4, we provide more example queries and responses from AmadeusGPT to support Main Figure 5a (EPM), to Main Figure 5b (MausHaus), and to Main Figure 5c (MABe) and results in Table 1.
Extended Results on EPM with AmadeusGPT
We show additional user queries and generated code from AmadeusGPT related to the EPM data presented in the Main Figure 5a.
Extended Results on Haus with AmadeusGPT
We show the user queries and generated code from AmadeusGPT that produce the images shown in the Main Figure 5b.
Query: "Give me the trajectories of the animal entering object 18 from right"
AmadeusGPT Code Output:
def task_program():
    behavior_analysis = AnimalBehaviorAnalysis()
    right_to_object_18_events = behavior_analysis.animals_object_events('18', 'to_right',
                   $\leftrightarrow$  bodyparts = ['all'])  
    enter_object_18_events = behavior_analysis-enter_object('18', bodyparts = ['all'])  
    enter_object_18_from_right_events =  
         $\leftrightarrow$  Event.addsequential_events(right_to_object_18_events, enter_object_18_events)
"Define  $< |$  head dips  $|>$  as a behavior where the animal's bodyparts mouse_center and at the same time neck in the roi and its head_midpoint outside roi. Get head_dips."
Amadeus Code Outputs:
def task_program(   ):
	behavior_analysis  $=$  AnimalBehaviorAnalysis(   )
	mouse_center_neck_in_rio_events  $=$  behavior_analysis.animals_object_events( \
	'ROI', 'overlap', bodyparts=['mouse_center', 'neck'], negate=True)
-head_midpoint_not_in_rio_events  $=$  behavior_analysis.animals_object_events( \
	'ROI', 'overlap', bodyparts=['head_midpoint'], negate=True)
-head_dips_events  $=$  Event.add_simultaneous_events( \
	mouse_center_neck_in_rio_events, head_midpoint_not_in_rio_events)
return head_dips_events
Amadeus Chat Outputs (Execution):
"When is the animal moving faster than 8 pixels per frame?"
def task_program():
    behavior_analysis = AnimalBehaviorAnalysis()
    animal_faster Than_8_events = behavior_analysis.animals_state_events( \
        'speed', '>>8')
    return animal_faster Than_8_events
"Define  $< |$  freezing  $\mid >$  as a behavior where the animal is slower than 0.5. When is the mouse freezing?"
def task_program():
    behavior_analysis = AnimalBehaviorAnalysis()
    freezing_events = behavior_analysis.animals_state_events('speed', '<0.5')
    return freezing_events
Figure 7: Extended Results on classical behavioral task (EPM) with AmadeusGPT. (a) Using the EPM dataset [13], we show additional queries and responses from AmadeusGPT. The raster plot is an ethogram where there is a tick for every related event in time, i.e., the mouse is in the closed arm or in the open arm.
fig, ax = behavior_analysis.plot_trjectory(['all'], events =  $\leftrightarrow$  enter_object_18_from_right_events)
return fig, ax
Query: "plot trajectories of the animal for the shortest event where it enters object 18 and then its neck enters object 17"
def task_program():
behavior_analysis = AnimalBehaviorAnalysis()
enter_object_18_events = behavior_analysis-enter_object('18', bodyparts=['all'])  
neck-enter_object_17_events = behavior_analysis-enter_object('17',  
 $\leftrightarrow$  bodyparts=['neck'])  
enter_object_18_and Neck-enter_object_17_events =  
 $\leftrightarrow$  Event.addsequential_events(enter_object_18_events, neck-enter_object_17_events)  
fig, ax = behavior_analysis.plot_trjectory(bodyparts=['all'],  
 $\leftrightarrow$  events=enter_object_18_and Neck-enter_object_17_events)  
return fig, ax
MABe Mouse Triplet Benchmark, extended details
In the MABe 2022 Mouse Triplet Benchmark, participants are given a dataset of tracking data of triplets of socially interacting mice and asked to submit a frame-by-frame representation of the dataset. Here are the following user prompts and AmadeusGPT code outputs of the sub-tasks we considered and the generated code by AmadeusGPT.
Query: "Define 
 as a social behavior where the closest distance between animals is less than 40 pixels and the chased animal is in front of the chasing animal and the chasing animal has to travel faster than 3.4 pixels per frame. The smooth window size is 25 and min window is 30. Get chases."
def task_program():
    behavior_analysis = AnimalBehaviorAnalysis()
    chase_events = behavior_analysis.animals_social_events(['closest_distance', 'orientation'],
            ['<40', ':=' + str(Orientation.FRONT)],
            animal_state_relation_query_list=['speed'],
            animal_state比較comparison_list=['>3.4'],
            bodyparts=['all'],
            otheranimal_bodyparts=['all'],
            smooth_window_size=25,
            min_window=30)
    return chase_events
Query: "Define <oral genital contact|> as a social behavior where closest distance between this animal's "nose" and other animal's bodyparts "tail base" is less than 15, get the events for oral genital contact."
def task_program():
    behavior_analysis = AnimalBehaviorAnalysis()
    oral_genital_contact_events =  $\rightarrow$  behavior_analysisanimals_social_events(['closest_distance'],
[< 15'],
bodyparts = ['nose'],
otheranimal_bodyparts = ['tail base'])
Query: " Define <loral oral contact!> as a social behavior where closest distance between this animal's "nose" and other animal's "nose" is less than 15, get events for oral oral contact."
def task_program():
    behavior_analysis = AnimalBehaviorAnalysis()
    oral_oral_contact_events =  $\leftrightarrow$  behavior_analysisanimals_social_events(['closest_distance'],
['< 15'],
bodyparts = ['nose'],
otheranimal_bodyparts = ['nose'])  
return oral_oral_contact_events
Query: "Define <approaches> as a social behavior where the mice move from at least 40 pixels apart sequentially to less than 8 pixels apart. Get approaches events."
def taskProgram(): behavior_analysis  $=$  AnimalBehaviorAnalysis() distance_events  $=$  behavior_analysis.animals_social_events(['distance'], ['>40']) close_distance_events  $=$  behavior_analysis.animals_social_events(['distance'], ['<8']) approaches_events  $=$  Event.addsequential_events(distance_events,  $\leftrightarrow$  close_distance_events) return approaches_events
Query: "Define <huddles> as a social behavior where the distance between animals is less than 50, and the relative speed between two animals is less than 4 pixels per frame. The smooth window size is 61 and min window is 75. Get huddles."
def taskProgram(): behavior_analysis  $=$  AnimalBehaviorAnalysis() huddle_events  $=$  behavior_analysis.animals_social_events(['distance', 'speed'], ['<50','<4'], bodyparts=['all'], min_window=75, smooth_window_size=61) return huddle_events
Query: "Define <contact> as a social behavior where the closest distance between animals is less than 12. The smooth window size is 11 and min window is 5. Get contact."
def taskProgram(): behavior_analysis  $=$  AnimalBehaviorAnalysis() contact_events  $=$  behavior_analysis.animals_social_events(['closest_distance'],  $\leftrightarrow$  ['<12'], smooth_window_size=11,min_window=5) return contact_events
Query: "Define  $<\mid$ close  $\mid>$  as a social behavior where the closest distance between animals is less than 24. The min window is 5. Get close."
def taskProgram(): behavior_analysis  $=$  AnimalBehaviorAnalysis() close_events  $=$  behavior_analysisanimals_social_events(['closest_distance'], ['<24'],  $\leftrightarrow$  min_window=5) return close_events
Query: "Define  $<$ watching|> as a social behavior where distance between animals is less than 260 and larger than 50 and angle between animals is less than 15. The min window is 100 and smooth window is 15. Get watching."
def taskProgram(): behavior_analysis  $=$  AnimalBehaviorAnalysis() watching_events  $=$  behavior_analysis.animals_social_events(['distance','distance',  $\leftrightarrow$  'angle'], ['<260','>50','<15'], bodyparts=['all'],
min_window  $= 100$  smooth_window_size  $\equiv 15$  return watching_events
Query: "Define <loral ear contact|> as a social behavior where the closest distance between animal's nose and other animals' left ear and right ear is less than 10. The smooth window size is 5 and min window is 15. Get oral ear contact."
def task_program():
    behavior_analysis = AnimalBehaviorAnalysis()
    oral_ear_contact_events =
         $\leftrightarrow$  behavior_analysis.animals_social_events(['closest_distance'],
                      $\leftrightarrow$  'closest_distance'],
                     ['<10', '<10'],
                     bodyparts=['nose'],
                     otheranimal_bodyparts=['left ear', 'right ear'],
                     min_window=15,
                     smooth_window_size=5)
    return oral_ear_contact_events
Prompt robustness testing
Related to Main Section 4, we show the full base questions and generated permutations we tested to gauge robustness.
The input, base out of distribution (OOD) questions
(1) define  $< |head \, dips| >$  as a behavior where the animal's bodypart mouse_center bodypart  $\rightarrow$  neck in the roi and its bodypart head_midpoint outside roi
(2) When is the animal moving faster than 8?
(3) define  $< |freezing|>$  as a behavior where the animal is slower than 0.5. When is the mouse freezing?
(4) How much time the animal spends in the closed arm?
(5) How much time the animal spends outside the closed arm?
The 25 generated OOD questions