Repository: allenai/ir_datasets (language: python)

Issue:
TREC 2023 Tip-of-the-Tongue
**Dataset Information:**

The training and dev data of the TREC 2023 Tip-of-the-Tongue track are now available: https://trec-tot.github.io/guidelines

Description from the [website](https://trec-tot.github.io/guidelines):

```
Tip of the tongue: The phenomenon of failing to retrieve something from memory, combined with partial recall and the feeling that retrieval is imminent
```

```
In terms of input and output, the movie identification task is relatively straightforward—given an input TOT request, output a ranked list of movies. Each movie must be identified by its Wikipedia page id and the correct movie should be ranked as high as possible. For each query, runs should return a ranked list of 1000 Wikipedia page ids. Runs will be evaluated using IR metrics that are appropriate for IR tasks with one relevant document, such as discounted cumulative gain, reciprocal rank, and success@k.
```

**Dataset ID(s) & supported entities:**

- `tip-of-the-tongue/train`
- `tip-of-the-tongue/dev`
- `tip-of-the-tongue/test` (not yet released)

**Checklist**

Mark each task once completed. All should be checked prior to merging a new dataset.
  
 - [x] Dataset definition (in `ir_datasets/datasets/[topid].py`)
 - [x] Tests (in `tests/integration/[topid].py`)
 - [x] Metadata generated (using `ir_datasets generate_metadata` command, should appear in `ir_datasets/etc/metadata.json`)
 - [x] Documentation (in `ir_datasets/etc/[topid].yaml`)
   - [ ] Documentation generated in https://github.com/seanmacavaney/ir-datasets.com/
 - [x] Downloadable content (in `ir_datasets/etc/downloads.json`)
   - [ ] Download verification action (in `.github/workflows/verify_downloads.yml`). Only one needed per `topid`.
   - [x] Any small public files from NIST (or other potentially troublesome files) mirrored in https://github.com/seanmacavaney/irds-mirror/. Mirrored status properly reflected in `downloads.json`.
  
**Additional comments/concerns/ideas/etc.**



Discussion / hints:
I would like to implement this ticket.
cc @samarthbhargav 

Fix patch:
diff --git a/ir_datasets/datasets/__init__.py b/ir_datasets/datasets/__init__.py
index dc961bb8..3d00ba8c 100644
--- a/ir_datasets/datasets/__init__.py
+++ b/ir_datasets/datasets/__init__.py
@@ -41,6 +41,7 @@
 from . import trec_mandarin
 from . import trec_spanish
 from . import trec_robust04
+from . import trec_tot
 from . import tripclick
 from . import tweets2013_ia
 from . import vaswani
@@ -51,4 +52,4 @@
 from . import trec_cast # must be after wapo,car,msmarco_passage
 from . import hc4
 from . import neuclir # must be after hc4
-from . import sara
\ No newline at end of file
+from . import sara
diff --git a/ir_datasets/datasets/trec_tot.py b/ir_datasets/datasets/trec_tot.py
new file mode 100644
index 00000000..f7c8af67
--- /dev/null
+++ b/ir_datasets/datasets/trec_tot.py
@@ -0,0 +1,71 @@
+import ir_datasets
+from ir_datasets.util import ZipExtract, Cache, Lazy, DownloadConfig
+from ir_datasets.formats import TrecQrels, JsonlQueries, JsonlDocs, TrecQrels
+from ir_datasets.datasets.base import Dataset, FilteredQueries, FilteredQrels, YamlDocumentation, Deprecated
+from typing import NamedTuple, List, Dict
+
+NAME = 'trec-tot'
+
+
+class TipOfTheTongueDoc(NamedTuple):
+    doc_id: str
+    page_title: str
+    wikidata_id: str
+    wikidata_classes: List[str]
+    text: str
+    sections: Dict[str, str]
+    infoboxes: List[Dict[str, str]]
+
+    def default_text(self):
+        """
+        We use the title and text of the TipOfTheTongueQuery as default_text because that is everything available for users who want to respond to such an information need.
+        """
+        return self.page_title + ' ' + self.text
+
+
+class TipOfTheTongueQuery(NamedTuple):
+    query_id: str
+    url: str
+    domain: str
+    title: str
+    text: str
+    sentence_annotations: List[Dict[str, str]]
+
+    def default_text(self):
+        return self.title + ' ' + self.text
+
+
+QUERY_MAP = {'query_id': 'id', 'url': 'url', 'domain': 'domain', 'title': 'title', 'text': 'text', 'sentence_annotations': 'sentence_annotations'}
+
+
+def _init():
+    documentation = YamlDocumentation(f'docs/{NAME}.yaml')
+    base_path = ir_datasets.util.home_path()/NAME
+    dlc = DownloadConfig.context(NAME, base_path)
+    subsets = {}
+
+    main_dlc = dlc['main']
+    base = Dataset(
+        documentation('_'),
+    )
+    ir_datasets.registry.register(NAME, base)
+
+    docs_2023_handler = JsonlDocs(Cache(ZipExtract(main_dlc, 'TREC-TOT/corpus.jsonl'), base_path/'2023/corpus.jsonl'), doc_cls=TipOfTheTongueDoc, lang='en')
+    subsets['2023'] = Dataset(
+        docs_2023_handler,
+        documentation('2023'),
+    )
+    ir_datasets.registry.register(f'{NAME}/2023', subsets['2023'])
+    for s in ['train', 'dev']:
+        subsets[f'2023/{s}'] = Dataset(
+            docs_2023_handler,
+            JsonlQueries(Cache(ZipExtract(main_dlc, f'TREC-TOT/{s}/queries.jsonl'), base_path/f'2023/{s}/queries.jsonl'), query_cls=TipOfTheTongueQuery, mapping=QUERY_MAP, lang='en'),
+            TrecQrels(Cache(ZipExtract(main_dlc, f'TREC-TOT/{s}/qrel.txt'), base_path/f'2023/{s}/qrel.txt'), {0: 'Not Relevant', 1: 'Relevant'}),
+            documentation(f'2023/{s}'),
+        )
+        ir_datasets.registry.register(f'{NAME}/2023/{s}', subsets[f'2023/{s}'])
+
+    return base, subsets
+
+
+base, subsets = _init()
diff --git a/ir_datasets/docs/sara.yaml b/ir_datasets/docs/sara.yaml
index dce6200e..3c5a8cb4 100644
--- a/ir_datasets/docs/sara.yaml
+++ b/ir_datasets/docs/sara.yaml
@@ -7,4 +7,5 @@ A set of sensitivity-aware relevance assessments. More information is avaliable
 
 <ul>
 <li><a href="https://github.com/JackMcKechnie/SARA-A-Collection-of-Sensitivity-Aware-Relevance-Assessments">SARA</a></li>
-</ul>
\ No newline at end of file
+</ul>
+'
diff --git a/ir_datasets/docs/trec-tot.yaml b/ir_datasets/docs/trec-tot.yaml
new file mode 100644
index 00000000..1633c546
--- /dev/null
+++ b/ir_datasets/docs/trec-tot.yaml
@@ -0,0 +1,28 @@
+_:
+  pretty_name: 'TREC Tip-of-the-Tongue'
+  desc: '
+<p>
+Tip of the tongue: The phenomenon of failing to retrieve something from memory, combined with partial recall and the feeling that retrieval is imminent. More details <a href="https://trec-tot.github.io/guidelines">are available on the official page for the TREC Tip-of-the-Tongue (ToT) Track</a>.
+</p>
+'
+
+2023:
+  desc: '
+<p>
+Corpus for the TREC 2023 tip-of-the-tongue search track.
+</p>
+'
+
+2023/train:
+  desc: '
+<p>
+Train query set for TREC 2023 tip-of-the-tongue search track.
+</p>
+'
+
+2023/dev:
+  desc: '
+<p>
+Dev query set for TREC 2023 tip-of-the-tongue search track.
+</p>
+'
diff --git a/ir_datasets/etc/downloads.json b/ir_datasets/etc/downloads.json
index 88f57a1b..7ff58060 100644
--- a/ir_datasets/etc/downloads.json
+++ b/ir_datasets/etc/downloads.json
@@ -3845,6 +3845,14 @@
       "cache_path": "trec4/qrels.gz"
     }
   },
+
+  "trec-tot": {
+    "main": {
+      "url": "https://surfdrive.surf.nl/files/index.php/s/FaEK4xc6Xp2JcAJ/download",
+      "expected_md5": "f84fe82cb80e3ee1072576c8d6c4a417",
+      "cache_path": "trec-tot.zip"
+    }
+  },
   
   "tripclick": {
     "benchmark": {
diff --git a/ir_datasets/etc/metadata.json b/ir_datasets/etc/metadata.json
index 23d8aee1..722866bd 100644
--- a/ir_datasets/etc/metadata.json
+++ b/ir_datasets/etc/metadata.json
@@ -560,6 +560,9 @@
   "trec-spanish": {"docs": {"count": 120605, "fields": {"doc_id": {"max_len": 13, "common_prefix": ""}}}},
   "trec-spanish/trec3": {"docs": {"_ref": "trec-spanish"}, "queries": {"count": 25}, "qrels": {"count": 19005, "fields": {"relevance": {"counts_by_value": {"1": 4766, "0": 14239}}}}},
   "trec-spanish/trec4": {"docs": {"_ref": "trec-spanish"}, "queries": {"count": 25}, "qrels": {"count": 13109, "fields": {"relevance": {"counts_by_value": {"1": 2202, "0": 10907}}}}},
+  "trec-tot/2023": {"docs": {"count": 231852, "fields": {"doc_id": {"max_len": 8, "common_prefix": ""}}}},
+  "trec-tot/2023/dev": {"docs": {"count": 231852, "fields": {"doc_id": {"max_len": 8, "common_prefix": ""}}}, "queries": {"count": 150}, "qrels": {"count": 150, "fields": {"relevance": {"counts_by_value": {"1": 150}}}}},
+  "trec-tot/2023/train": {"docs": {"count": 231852, "fields": {"doc_id": {"max_len": 8, "common_prefix": ""}}}, "queries": {"count": 150}, "qrels": {"count": 150, "fields": {"relevance": {"counts_by_value": {"1": 150}}}}},
   "tripclick": {"docs": {"count": 1523878, "fields": {"doc_id": {"max_len": 8, "common_prefix": ""}}}},
   "tripclick/logs": {"docs": {"count": 5196956, "fields": {"doc_id": {"max_len": 8, "common_prefix": ""}}}, "qlogs": {"count": 5317350}},
   "tripclick/test": {"docs": {"_ref": "tripclick"}, "queries": {"count": 3525}, "scoreddocs": {"count": 3486402}},


Test patch:
diff --git a/test/integration/trec_tot.py b/test/integration/trec_tot.py
new file mode 100644
index 00000000..7301df30
--- /dev/null
+++ b/test/integration/trec_tot.py
@@ -0,0 +1,49 @@
+import re
+import unittest
+from ir_datasets.formats import TrecQrel
+from ir_datasets.datasets.trec_tot import TipOfTheTongueDoc, TipOfTheTongueQuery
+from .base import DatasetIntegrationTest
+import ir_datasets
+
+
+class TestTipOfTheTongue(DatasetIntegrationTest):
+    def test_tip_of_the_tongue_docs(self):
+         self._test_docs('trec-tot/2023', count=231852, items={
+              0: TipOfTheTongueDoc('330', 'Actrius', 'Q2823770', [['Q11424', 'film']], re.compile('^Actresses \\(Catalan: Actrius\\) is a 1997 Catalan language Spanish drama film produced and directed by .{1648}ical work is true to the original, but does not stray far from a theatrical rendering of the story\\."$', flags=48), {'abstract': 'Actresses (Catalan: Actrius) is a 1997 Catalan language Spanish drama film produced and directed by Ventura Pons and based on the award-winning stage play "E.R." by Josep Maria Benet i Jornet. The film has no male actors, with all roles played by females. The film was produced in 1996. ', 'synopsis': "Synopsis.\nIn order to prepare herself to play a role commemorating the life of legendary actress Empar Ribera, young actress (Mercè Pons) interviews three established actresses who had been the Ribera's pupils: the international diva Glòria Marc (Núria Espert), the television star Assumpta Roca (Rosa Maria Sardà), and dubbing director Maria Caminal (Anna Lizaran).", 'recognition': 'Recognition.\nScreenings.\n"Actrius" screened in 2001 at the Grauman\'s Egyptian Theatre in an American Cinematheque retrospective of the works of its director. The film had first screened at the same location in 1998. It was also shown at the 1997 Stockholm International Film Festival.\nReception.\nIn "Movie - Film - Review", Christopher Tookey wrote that though the actresses were "competent in roles that may have some reference to their own careers", the film "is visually unimaginative, never escapes its stage origins, and is almost totally lacking in revelation or surprising incident". Noting that there were "occasional, refreshing moments of intergenerational bitchiness", they did not "justify comparisons to "All About Eve"", and were "insufficiently different to deserve critical parallels with "Rashomon"". He also wrote that "The Guardian" called the film a "slow, stuffy chamber-piece", and that "The Evening Standard" stated the film\'s "best moments exhibit the bitchy tantrums seething beneath the threesome\'s composed veneers". MRQE wrote "This cinematic adaptation of a theatrical work is true to the original, but does not stray far from a theatrical rendering of the story."', 'screenings': 'Screenings.\n"Actrius" screened in 2001 at the Grauman\'s Egyptian Theatre in an American Cinematheque retrospective of the works of its director. The film had first screened at the same location in 1998. It was also shown at the 1997 Stockholm International Film Festival.', 'reception': 'Reception.\nIn "Movie - Film - Review", Christopher Tookey wrote that though the actresses were "competent in roles that may have some reference to their own careers", the film "is visually unimaginative, never escapes its stage origins, and is almost totally lacking in revelation or surprising incident". Noting that there were "occasional, refreshing moments of intergenerational bitchiness", they did not "justify comparisons to "All About Eve"", and were "insufficiently different to deserve critical parallels with "Rashomon"". He also wrote that "The Guardian" called the film a "slow, stuffy chamber-piece", and that "The Evening Standard" stated the film\'s "best moments exhibit the bitchy tantrums seething beneath the threesome\'s composed veneers". MRQE wrote "This cinematic adaptation of a theatrical work is true to the original, but does not stray far from a theatrical rendering of the story."'}, [{'name': 'film', 'params': {'name': 'Actresses', 'image': 'Actrius film poster.jpg', 'caption': 'Catalan language film poster', 'native_name': "([[Catalan language|Catalan]]: '''''Actrius''''')", 'director': '[[Ventura Pons]]', 'producer': 'Ventura Pons', 'writer': '[[Josep Maria Benet i Jornet]]', 'screenplay': 'Ventura Pons', 'based_on': "{{based on|(stage play) ''E.R.''|Josep Maria Benet i Jornet}}", 'starring': '{{ubl|[[Núria Espert]]|[[Rosa Maria Sardà]]|[[Anna Lizaran]]|[[Mercè Pons]]}}', 'narrator': '<!-- or: |narrators = -->', 'music': 'Carles Cases', 'cinematography': 'Tomàs Pladevall', 'editing': 'Pere Abadal', 'production_companies': '{{ubl|[[Canal+|Canal+ España]]|Els Films de la Rambla S.A.|[[Generalitat de Catalunya|Generalitat de Catalunya - Departament de Cultura]]|[[Televisión Española]]}}', 'distributor': '[[Buena Vista International]]', 'released': '{{film date|df=yes|1997|1|17|[[Spain]]}}', 'runtime': '100 minutes', 'country': 'Spain', 'language': 'Catalan', 'gross': '<!--(please use condensed and rounded values, e.g. "£11.6 million" not "£11,586,221")-->'}}]),
+              9: TipOfTheTongueDoc('3837', 'Blazing Saddles', 'Q957323', [['Q11424', 'film']], re.compile('^Blazing Saddles is a 1974 American satirical western black comedy film directed by Mel Brooks, who a.{16434}tion DVD in 2004 and a Blu\\-ray version in 2006\\. A 40th anniversary Blu\\-ray set was released in 2014\\.$', flags=48), {'abstract': 'Blazing Saddles is a 1974 American satirical western black comedy film directed by Mel Brooks, who also wrote the screenplay with Andrew Bergman, Richard Pryor, Norman Steinberg, and Alan Uger. The film stars Cleavon Little and Gene Wilder. The film received generally positive reviews from critics and audiences, was nominated for three Academy Awards and is ranked No. 6 on the American Film Institute\'s "100 Years...100 Laughs" list.\nBrooks appears in three supporting roles, Governor William J. Le Petomane, a Yiddish-speaking Native American chief and "a director" in line to help invade Rock Ridge (a nod to Hitchcock); he also dubs lines for one of Lili Von Shtupp\'s backing troupe. The supporting cast includes Slim Pickens, Alex Karras, and David Huddleston, as well as Brooks regulars Dom DeLuise, Madeline Kahn, and Harvey Korman. Bandleader Count Basie has a cameo as himself, appearing with his orchestra.\nThe film is full of deliberate anachronisms, from the Count Basie Orchestra playing "April in Paris" in the Wild West, to Slim Pickens referring to the "Wide World of Sports".\nIn 2006, "Blazing Saddles" was deemed "culturally, historically, or aesthetically significant" by the Library of Congress and was selected for preservation in the National Film Registry.', 'plot': 'Plot.\nOn the American frontier of 1874, a new railroad under construction will have to be rerouted through the town of Rock Ridge in order to avoid quicksand. Realizing this will make Rock Ridge worth millions, territorial attorney general Hedley Lamarr plans to force Rock Ridge\'s residents out of the town and sends a gang of thugs, led by his flunky Taggart, to shoot the sheriff and trash the town. The townspeople demand that Governor William J. Le Petomane appoint a new sheriff to protect them. Lamarr persuades dim-witted Le Petomane to appoint Bart, a black railroad worker about to be executed for assaulting Taggart. A black sheriff, Lamarr reasons, will offend the townspeople, create chaos, and leave Rock Ridge at his mercy.\nAfter an initial hostile reception (Bart takes himself "hostage" to escape), he relies on his quick wits and the assistance of Jim, an alcoholic gunslinger known as the "Waco Kid", to overcome the townspeople\'s hostility. Bart subdues Mongo, an immensely strong and dim-witted yet philosophical henchman sent to kill him, then outwits German seductress-for-hire Lili Von Shtupp at her own game, with Lili falling in love with him. Upon release, Mongo vaguely informs Bart of Lamarr\'s connection to the railroad, so Bart and Jim visit the railroad worksite and discover from Bart\'s best friend Charlie that the railway is planned to go through Rock Ridge. Taggart and his men arrive to kill Bart, but Jim outshoots them and forces their retreat. Lamarr, furious that his schemes have backfired, recruits an army of thugs, including common criminals, motorcycle gangsters, Ku Klux Klansmen, Nazis, and Methodists.\nEast of Rock Ridge, Bart introduces the white townspeople to the black, Chinese, and Irish railroad workers who have agreed to help in exchange for acceptance by the community, and explains his plan to defeat Lamarr\'s army. They labor all night to build a perfect copy of the town as a diversion. When Bart realizes it will not fool the villains, the townsfolk construct copies of themselves. Bart, Jim, and Mongo buy time by constructing the "Gov. William J. Le Petomane Thruway", forcing the raiding party to send for change to pay the toll. Once through the tollbooth, the raiders attack the fake town and its population of dummies, which have been booby-trapped with dynamite. After Jim detonates the bombs with his sharpshooting, launching bad guys and horses skyward, the Rock Ridgers attack the villains.\nThe resulting brawl between townsfolk, railroad workers, and Lamarr\'s thugs literally breaks the fourth wall and bursts onto a neighboring movie set where director Buddy Bizarre is filming a Busby Berkeley-style top-hat-and-tails musical number; into the studio commissary for a food fight; and spilling out of the Warner Bros. film lot onto the streets of Burbank. Lamarr, realizing he has been beaten, hails a taxi and orders the cabbie to "drive me off this picture". He ducks into Mann\'s Chinese Theatre, which is showing the premiere of "Blazing Saddles". As he settles into his seat, he sees onscreen Bart arriving on horseback outside the theatre. Bart blocks Lamarr\'s escape and shoots him in the groin. Bart and Jim then enter the theater to watch the end of the film, in which Bart announces to the townspeople that he is moving on because his work is done (and he is bored). Riding out of town, he finds Jim, still eating his popcorn, and invites him along to "nowhere special". The two friends briefly ride into the desert before dismounting and boarding a limousine, which drives off into the sunset.', 'cast': 'Cast.\nCast notes:', 'production': 'Production.\nThe idea for the film came from a story outline written by Andrew Bergman that he originally intended to develop and produce himself. "I wrote a first draft called "Tex-X"" (a play on Malcolm X\'s name), he said. "Alan Arkin was hired to direct and James Earl Jones was going to play the sheriff. That fell apart, as things often do." Brooks was taken with the story, which he described as "hip talk—1974 talk and expressions—happening in 1874 in the Old West", and purchased the film rights from Bergman. Though he had not worked with a writing team since "Your Show of Shows", he hired a group of writers (including Bergman) to expand the outline, and posted a large sign: "Please do not write a polite script." Brooks described the writing process as chaotic: ""Blazing Saddles" was more or less written in the middle of a drunken fistfight. There were five of us all yelling loudly for our ideas to be put into the movie. Not only was I the loudest, but luckily I also had the right as director to decide what was in or out." Bergman remembers the room being just as chaotic, telling "Creative Screenwriting", "In the beginning, we had five people. One guy left after a couple of weeks. Then, it was basically me, Mel, Richie Pryor and Norman Steinberg. Richie left after the first draft and then Norman, Mel and I wrote the next three or four drafts. It was a riot. It was a rioter’s room!"\nThe original title, "Tex X", was rejected to avoid it being mistaken for an X-rated film, as were "Black Bart" – a reference to Black Bart, a white highwayman of the 19th century – and "Purple Sage". Brooks said he finally conceived "Blazing Saddles" one morning while taking a shower.\nThe casting was problematic. Richard Pryor was Brooks\' original choice to play Sheriff Bart, but the studio, claiming his history of drug arrests made him uninsurable, refused to approve financing with Pryor as the star. The role of Sheriff Bart went to Cleavon Little, and Pryor remained as a screenwriter instead. Brooks offered the other leading role, the Waco Kid, to John Wayne; he declined, deeming the film "too blue" for his family-oriented image, but assured Brooks that "he would be the first one in line to see it." Gig Young was cast, but he collapsed during his first scene from what was later determined to be alcohol withdrawal syndrome, and Gene Wilder was flown in to replace him. Johnny Carson and Wilder both turned down the Hedley Lamarr role before Harvey Korman was cast. Madeline Kahn objected when Brooks asked to see her legs during her audition. "She said, \'So it\'s THAT kind of an audition? Brooks recalled. "I explained that I was a happily married man and that I needed someone who could straddle a chair with her legs like Marlene Dietrich in "Destry Rides Again." So she lifted her skirt and said, \'No touching.\nBrooks had numerous conflicts over content with Warner Bros. executives, including frequent use of the word "nigger", Lili Von Shtupp\'s seduction scene, the cacophony of flatulence around the campfire, and Mongo punching out a horse. Brooks, whose contract gave him final content control, declined to make any substantive changes, with the exception of cutting Bart\'s final line during Lili\'s seduction: "I hate to disappoint you, ma\'am, but you\'re sucking my arm." When asked later about the many "nigger" references, Brooks said he received consistent support from Pryor and Little. He added, "If they did a remake of "Blazing Saddles" today [2012], they would leave out the N-word. And then, you\'ve got no movie." Brooks said he received many letters of complaint after the film\'s release.\nThe film was almost not released. "When we screened it for executives, there were few laughs," said Brooks. "The head of distribution said, \'Let\'s dump it and take a loss.\' But [studio president John] Calley insisted they open it in New York, Los Angeles and Chicago as a test. It became the studio\'s top moneymaker that summer." The world premiere took place on February 7, 1974, at the Pickwick Drive-In Theater in Burbank; 250 invited guests—including Little and Wilder—watched the film on horseback.', 'songs and music': 'Songs and music.\nMel Brooks wrote the music and lyrics for three of "Blazing Saddles" songs, "The Ballad of Rock Ridge", "I\'m Tired", and "The French Mistake". Brooks also wrote the lyrics to the title song, with music by John Morris, the composer of the film\'s score. To sing the title song, Brooks advertised in the trade papers for a "Frankie Laine–type" singer; to his surprise, Laine himself offered his services. "Frankie sang his heart out ... and we didn\'t have the heart to tell him it was a spoof. He never heard the whip cracks; we put those in later. We got so lucky with his serious interpretation of the song."\nThe choreographer for "I\'m Tired" and "The French Mistake" was Alan Johnson. "I\'m Tired" is a homage to and parody of Marlene Dietrich\'s singing of Cole Porter\'s song "I\'m the Laziest Gal in Town" in Alfred Hitchcock\'s 1950 film "Stage Fright", as well as "Falling in Love Again (Can\'t Help It)" from "The Blue Angel".\nThe orchestrations were by Morris and Jonathan Tunick.\nThe first studio-licensed release of the full music soundtrack to "Blazing Saddles" was on La-La Land Records on August 26, 2008. Remastered from original studio vault elements, the limited-edition CD – a run of 3,000 – features the songs from the film as well as Morris\'s score. Instrumental versions of all the songs are bonus tracks on the disc. The disc features liner notes featuring comments from Mel Brooks and John Morris.', 'reception': 'Reception.\nWhile "Blazing Saddles" is now considered a classic comedy, critical reaction was mixed when the film was released. Vincent Canby wrote:\nRoger Ebert gave the film four stars out of four, calling it a "crazed grab bag of a movie that does everything to keep us laughing except hit us over the head with a rubber chicken. Mostly, it succeeds. It\'s an audience picture; it doesn\'t have a lot of classy polish and its structure is a total mess. But of course! What does that matter while Alex Karras is knocking a horse cold with a right cross to the jaw?" Gene Siskel awarded three stars out of four and called it "bound to rank with the funniest of the year," adding, "Whenever the laughs begin to run dry, Brooks and his quartet of gag writers splash about in a pool of obscenities that score belly laughs if your ears aren\'t sensitive and if you\'re hip to western movie conventions being parodied." "Variety" wrote, "If comedies are measured solely by the number of yocks they generate from audiences, then \'Blazing Saddles\' must be counted a success ... Few viewers will have time between laughs to complain that pic is essentially a raunchy, protracted version of a television comedy skit."\nCharles Champlin of the "Los Angeles Times" called the film "irreverent, outrageous, improbable, often as blithely tasteless as a stag night at the Friar\'s Club and almost continuously funny." Gary Arnold of "The Washington Post" was negative, writing that "Mel Brooks squanders a snappy title on a stockpile of stale jokes. To say that this slapdash Western spoof lacks freshness and spontaneity and originality is putting it mildly. \'Blazing Saddles\' is at once a messy and antiquated gag machine." Jan Dawson of "The Monthly Film Bulletin" wrote, "Perhaps it is pedantic to complain that the whole is not up to the sum of its parts when, for the curate\'s egg that it is, "Blazing Saddles" contains so many good parts and memorable performances." John Simon wrote a negative review of "Blazing Saddles", saying, "All kinds of gags—chiefly anachronisms, irrelevancies, reverse ethnic jokes, and out and out vulgarities—are thrown together pell-mell, batted about insanely in all directions, and usually beaten into the ground."\nOn review aggregator Rotten Tomatoes, the film has an 89% approval rating based on 61 reviews, with a average rating of 8.20/10. The site\'s consensus reads: "Daring, provocative, and laugh-out-loud funny, "Blazing Saddles" is a gleefully vulgar spoof of Westerns that marks a high point in Mel Brooks\' storied career." On Metacritic it has a score of 73% based on reviews from 12 critics, indicating "generally favorable reviews".\nDuring production for the film, retired longtime film star Hedy Lamarr sued Warner Bros. for $100,000, charging that the film\'s running parody of her name infringed on her right to privacy. Brooks said that he was flattered and chose to not fight it in court; the studio settled out of court for a small sum and an apology for "almost using her name." Brooks said that Lamarr "never got the joke." This lawsuit would be referenced by an in-film joke where Brooks\' character, the Governor, tells Hedley Lamarr that "This is 1874; you\'ll be able to sue HER."\nIshmael Reed\'s 1969 novel "Yellow Back Radio Broke-Down" has been cited as an important precursor or influence for "Blazing Saddles", a connection that Reed himself has made.\nBox office.\nThe film earned theatrical rentals of $26.7 million in its initial release in the United States and Canada. In its 1976 reissue, it earned a further $10.5 million and another $8 million in 1979. Its total rentals in the United States and Canada totalled $47.8 million from a gross of $119.5 million, becoming only the tenth film up to that time to pass the $100 million mark.', 'box office': 'Box office.\nThe film earned theatrical rentals of $26.7 million in its initial release in the United States and Canada. In its 1976 reissue, it earned a further $10.5 million and another $8 million in 1979. Its total rentals in the United States and Canada totalled $47.8 million from a gross of $119.5 million, becoming only the tenth film up to that time to pass the $100 million mark.', 'awards and accolades': 'Awards and accolades.\nWhile addressing his group of bad guys, Harvey Korman\'s character reminds them that, although they are risking their lives, he is "risking an almost certain Academy Award nomination for Best Supporting Actor!" Korman did not receive an Oscar bid, but the film did get three nominations at the 47th Academy Awards, including Best Supporting Actress for Madeline Kahn.\nIn 2006, "Blazing Saddles" was deemed "culturally, historically, or aesthetically significant" by the Library of Congress and was selected for preservation in the National Film Registry.\nUpon the release of the 30th anniversary special edition in 2004, "Today" said that the movie "skewer[ed] just about every aspect of racial prejudice while keeping the laughs coming" and that it was "at the top of a very short list" of comedies still funny after 30 years. In 2014, NPR wrote that four decades after the movie was made it was "still as biting a satire" on racism as ever, although its treatment of gays and women was "not self-aware at all".\nThe film is recognized by American Film Institute in these lists:', 'adaptations': 'Adaptations.\nTV series.\nA television pilot titled "Black Bart" was produced for CBS based on Bergman\'s original story. It featured Louis Gossett Jr. as Bart and Steve Landesberg as his drunkard sidekick, a former Confederate officer named "Reb Jordan". Other cast members included Millie Slavin and Noble Willingham. Bergman is listed as the sole creator. CBS aired the pilot once on April 4, 1975. The pilot episode of "Black Bart" was later included as a bonus feature on the "Blazing Saddles" 30th Anniversary DVD and the Blu-ray disc.\nPossible stage production.\nIn September 2017, Mel Brooks indicated his desire to do a stage play version of "Blazing Saddles" in the future.', 'tv series': 'TV series.\nA television pilot titled "Black Bart" was produced for CBS based on Bergman\'s original story. It featured Louis Gossett Jr. as Bart and Steve Landesberg as his drunkard sidekick, a former Confederate officer named "Reb Jordan". Other cast members included Millie Slavin and Noble Willingham. Bergman is listed as the sole creator. CBS aired the pilot once on April 4, 1975. The pilot episode of "Black Bart" was later included as a bonus feature on the "Blazing Saddles" 30th Anniversary DVD and the Blu-ray disc.', 'possible stage production': 'Possible stage production.\nIn September 2017, Mel Brooks indicated his desire to do a stage play version of "Blazing Saddles" in the future.', 'in popular culture': 'In popular culture.\nThe Rock Ridge standard for CD and DVD media is named after the town in "Blazing Saddles".\nThe 2022 animated film "", starring Michael Cera, Samuel L. Jackson, Michelle Yeoh, and Ricky Gervais, was originally titled "Blazing Samurai" and its creators called it "equally inspired by and an homage to "Blazing Saddles"." Brooks served as an executive producer for the production, and voiced one of the characters.', 'home media': 'Home media.\nThe film was first released on DVD in 1997, followed by a 30th Anniversary Special Edition DVD in 2004 and a Blu-ray version in 2006. A 40th anniversary Blu-ray set was released in 2014.'}, [{'name': 'film', 'params': {'name': 'Blazing Saddles', 'image': 'blazing_saddles_movie_poster.jpg', 'caption': 'Theatrical release poster by [[John Alvin]]<ref name="lat">{{cite news|first=Jocelyn|last=Stewart|title=John Alvin, 59; created movie posters for such films as \'Blazing Saddles\' and \'E.T.\'|url=https://www.latimes.com/news/printedition/california/la-me-alvin10feb10,1,5113268.story|newspaper=[[Los Angeles Times]]|date=February 10, 2008|access-date=February 10, 2008|archive-date=March 14, 2010|archive-url=https://web.archive.org/web/20100314184353/http://articles.latimes.com/2008/feb/10/local/me-alvin10|url-status=live}}</ref>', 'director': '[[Mel Brooks]]', 'producer': '[[Michael Hertzberg]]', 'screenplay': '{{Plainlist|\n* Mel Brooks\n* [[Norman Steinberg]]\n* [[Andrew Bergman]]\n* [[Richard Pryor]]\n* [[Alan Uger]]\n}}', 'story': 'Andrew Bergman', 'starring': '{{Plainlist|\n* [[Cleavon Little]]\n* [[Gene Wilder]]\n* [[Slim Pickens]]\n* [[Alex Karras]]\n* Mel Brooks\n* [[Harvey Korman]]\n* [[Madeline Kahn]]\n}}', 'music': '[[John Morris (composer)|John Morris]]', 'cinematography': '[[Joseph Biroc]]', 'editing': '{{Plainlist|\n* [[Danford B. Greene|Danford Greene]]\n* [[John C. Howard]]\n}}', 'studio': 'Crossbow Productions', 'distributor': '[[Warner Bros.]]', 'released': '{{Film date|1974|2|7}}', 'runtime': '93 minutes<ref name=afi>{{AFI film|54998}}</ref>', 'country': 'United States', 'budget': '$2.6 million', 'gross': '$119.6 million (US and Canada)<ref>{{cite web|url=https://www.boxofficemojo.com/release/rl4064773633/rankings/|title=Blazing Saddles (1974)|access-date=January 17, 2012|website=[[Box Office Mojo]]|archive-date=July 14, 2019|archive-url=https://web.archive.org/web/20190714112524/https://www.boxofficemojo.com/movies/?id=blazingsaddles.htm|url-status=live}}</ref>'}}]),
+              231851: TipOfTheTongueDoc('72634502', 'Skulle det dukke opp flere lik er det bare å ringe', 'Q17415598', [['Q11424', 'film']], re.compile('^Skulle det dukke opp flere lik er det bare å ringe \\(If Any More Bodies Should Turn Up, Just Call\\) is.{192} the film\\. The stage play was a translation of Jack Popplewell\'s British play "Busybody", from 1964\\.$', flags=48), {'abstract': 'Skulle det dukke opp flere lik er det bare å ringe (If Any More Bodies Should Turn Up, Just Call) is a Norwegian crime comedy from 1970 directed by Knut Bohwim. The film stars Arve Opsahl and Aud Schønemann. The film is based on a popular theater production from 1968 with the same cast as in the film. The stage play was a translation of Jack Popplewell\'s British play "Busybody", from 1964.'}, [{'name': 'film', 'params': {'name': 'Skulle det dukke opp flere lik, er det bare å ringe', 'director': '[[Knut Bohwim]]', 'producer': 'Anne Vennerød', 'writer': '[[Jon Lennart Mjøen]]', 'based_on': "[[Jack Popplewell]]'s play ''Busybody''", 'starring': '[[Aud Schønemann]]<br/>[[Arve Opsahl]]<br/>[[Arne Aas (actor)|Arne Aas]]<br/>[[Carsten Byhring]]<br/>[[Thorleif Reiss]]<br/>[[Jorunn Kjellsby]]', 'music': '[[Egil Monn-Iversen]]', 'cinematography': '[[Mattis Mathiesen]]', 'editing': '[[Reidar Lund]]<br/>[[Olav Engebretsen]]', 'distributor': 'EMI Produksjon', 'released': 'March 2, 1970', 'runtime': '98 minutes', 'country': 'Norway', 'language': 'Norwegian'}}]),
+         })
+
+    def test_test_tip_of_the_tongue_queries_train(self):
+        self._test_queries('trec-tot/2023/train', count=150, items={
+            0: TipOfTheTongueQuery('763', 'https://irememberthismovie.com/super-rare-surreal-dystopian-masterpiece/', 'movie', 'Super Rare Surreal Dystopian Masterpiece', 'Very rare movie that is scifi/dystopian/experimental/surreal. It’s like Stalker meets el Topo meets Holy Mountain meets Alphaville meets Delicatessen meets Hard to be a God, like Kurosawa, Tarkovsky, and Lynch had a kid together. It was color, possibly Russian, and I don’t really remember the decade but want to say 60s or 70s, though could easily be more recent. It is VERY rare, there is only one crappy partial print of it, and that is what the youtube version is from. Lot of wide shots in a surreal wilderness, winter settings, strange bleeding saturation in some shots. Crazy costumes. Seriously one of the strangest films I’ve ever seen and my favorite films are strange/weird ones. If you’ve ever seen what you’re thinking of on a “best weird movies” or “you’ve never seen this!” list, that’s NOT it. I don’t think this film even has a cult following of ten people. It’s an actual rare gem. Have been looking through selections at 366 Weird Movies and not found it yet (btw the way most of those titles are exactly the kind of not-actually-rare movies this film is definitely not).', [{'id': 1, 'text': 'Very rare movie that is scifi/dystopian/experimental/surreal.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': True, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 2, 'text': 'It’s like Stalker meets el Topo meets Holy Mountain meets Alphaville meets Delicatessen meets Hard to be a God, like Kurosawa, Tarkovsky, and Lynch had a kid together.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': True, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': True, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 3, 'text': 'It was color, possibly Russian, and I don’t really remember the decade but want to say 60s or 70s, though could easily be more recent.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': True, 'origin_language': False, 'category': False, 'character': False, 'production_visual': True, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': True, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 4, 'text': 'It is VERY rare, there is only one crappy partial print of it, and that is what the youtube version is from.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 5, 'text': 'Lot of wide shots in a surreal wilderness, winter settings, strange bleeding saturation in some shots.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': True, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 6, 'text': 'Crazy costumes.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 7, 'text': 'Seriously one of the strangest films I’ve ever seen and my favorite films are strange/weird ones.', 'labels': {'opinion': True, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': True, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 8, 'text': 'If you’ve ever seen what you’re thinking of on a “best weird movies” or “you’ve never seen this!”', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': True, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 9, 'text': 'list, that’s NOT it.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 10, 'text': 'I don’t think this film even has a cult following of ten people.', 'labels': {'opinion': True, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 11, 'text': 'It’s an actual rare gem.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 12, 'text': 'Have been looking through selections at 366 Weird Movies and not found it yet (btw the way most of those titles are exactly the kind of not-actually-rare movies this film is definitely not).', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': True, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}]),
+            9: TipOfTheTongueQuery('293', 'https://irememberthismovie.com/date-movie-from-awhile-ago/', 'movie', 'Date Movie from awhile ago', 'This was a movie from at least 10 years ago.\xa0 The lead actor and actress are on a first date and eating at a burger stand or shop of some sort and the guy mentions how hard dates are because the guy worries the whole time about whether he will get a kiss or not.\xa0 So the female says lets just kiss now and get it out of the way. I dont recall how long ago it was, any of the actors or what else happened in the movie.\xa0 i think it was a romantic comedy of sorts.', [{'id': 1, 'text': 'This was a movie from at least 10 years ago.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': True, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 2, 'text': 'The lead actor and actress are on a first date and eating at a burger stand or shop of some sort and the guy mentions how hard dates are because the guy worries the whole time about whether he will get a kiss or not.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 3, 'text': 'So the female says lets just kiss now and get it out of the way.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': True, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 4, 'text': 'I dont recall how long ago it was, any of the actors or what else happened in the movie.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': True, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 5, 'text': 'i think it was a romantic comedy of sorts.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': True, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}]),
+            149: TipOfTheTongueQuery('828', 'https://irememberthismovie.com/animated-movie-consisted-of-several-short-movies-that-was-once-on-netflix/', 'movie', 'Animated Movie (consisted of several short movies) That Was Once On Netflix', 'I remember watching an animated movie in the years 2009-2015. I don’t remember the name or any of the actors of the movie, but I remember a lot about it. First off, it was animated. It wasn’t super high quality, the people/things looked like silhouettes (you couldn’t see their faces or exact clothing details, but you could see the shadows of them). Second, it was about a boy and a girl acting out these “plays” in an old theater owned by an old man, who may have been their grandfather. There was also an owl in the theater, and it looked at the camera and hooted whenever they changed from one “play” to another. Third, to change from one costume to another, they stood on a sort of platform thing. It “scanned” them, and the silhouettes of the clothes they described appeared on their bodies. The plays/plots I remember are as follows : Boy gets put in prison (wrongly accused of stealing a necklace, I think), but gets pitied by a princess and she takes him food/clothing. However, the good princess’s evil sister tries to marry the boy when he gets out of prison. When he refuses/realizes she’s the evil princess, she curses him to be a wolf. Boy then eventually gets freed by the good princess. Boy finds himself in a city of gold, but an evil dragon rules the city. The dragon gives the people gold, but only if they sacrifice the most beautiful maiden to the dragon every month/year. Boy eventually slays dragon and marries the girl, who was the most beautiful at that time. Boy is in Africa/similar place and likes to play the drum, but is shunned for doing so. The girl’s father is the clan/tribe chief and is dying from illness. Boy gets thrown out of clan/tribe, but saves an old medicine man from a leopard/big cat. Old medicine man teaches boy how to play drum, which ends up healing the sick clan/tribe chief. The drum also makes the clan/tribe unbeatable in battle. Boy is an adventurer in a strange land where he crawls through a tunnel and gets put in prison. He has to choose/marry one of the king’s three daughters, but he doesn’t know which one. A bee flies around the head of the correct daughter because he fed the bee earlier in the tunnel. He also gets help from other monsters he met in the tunnel/cave. There may have been more, but it’s been years since I saw the movie. It was', [{'id': 1, 'text': 'I remember watching an animated movie in the years 2009-2015.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': True, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': True, 'situational_evidence': False}}}, {'id': 2, 'text': 'I don’t remember the name or any of the actors of the movie, but I remember a lot about it.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 3, 'text': 'First off, it was animated.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': True, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 4, 'text': 'It wasn’t super high quality, the people/things looked like silhouettes (you couldn’t see their faces or exact clothing details, but you could see the shadows of them).', 'labels': {'opinion': True, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': True, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 5, 'text': 'Second, it was about a boy and a girl acting out these “plays” in an old theater owned by an old man, who may have been their grandfather.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 6, 'text': 'There was also an owl in the theater, and it looked at the camera and hooted whenever they changed from one “play” to another.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': True, 'origin_actor': False, 'location_type': True, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 7, 'text': 'Third, to change from one costume to another, they stood on a sort of platform thing.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 8, 'text': 'It “scanned” them, and the silhouettes of the clothes they described appeared on their bodies.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 9, 'text': 'The plays/plots I remember are as follows : Boy gets put in prison (wrongly accused of stealing a necklace, I think), but gets pitied by a princess and she takes him food/clothing.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 10, 'text': 'However, the good princess’s evil sister tries to marry the boy when he gets out of prison.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 11, 'text': 'When he refuses/realizes she’s the evil princess, she curses him to be a wolf.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 12, 'text': 'Boy then eventually gets freed by the good princess.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 13, 'text': 'Boy finds himself in a city of gold, but an evil dragon rules the city.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 14, 'text': 'The dragon gives the people gold, but only if they sacrifice the most beautiful maiden to the dragon every month/year.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 15, 'text': 'Boy eventually slays dragon and marries the girl, who was the most beautiful at that time.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 16, 'text': 'Boy is in Africa/similar place and likes to play the drum, but is shunned for doing so.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': True, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 17, 'text': 'The girl’s father is the clan/tribe chief and is dying from illness.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 18, 'text': 'Boy gets thrown out of clan/tribe, but saves an old medicine man from a leopard/big cat.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 19, 'text': 'Old medicine man teaches boy how to play drum, which ends up healing the sick clan/tribe chief.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 20, 'text': 'The drum also makes the clan/tribe unbeatable in battle.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 21, 'text': 'Boy is an adventurer in a strange land where he crawls through a tunnel and gets put in prison.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 22, 'text': 'He has to choose/marry one of the king’s three daughters, but he doesn’t know which one.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 23, 'text': 'A bee flies around the head of the correct daughter because he fed the bee earlier in the tunnel.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 24, 'text': 'He also gets help from other monsters he met in the tunnel/cave.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 25, 'text': 'There may have been more, but it’s been years since I saw the movie.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': True, 'situational_evidence': False}}}, {'id': 26, 'text': 'It was', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}]),
+        })
+
+    def test_test_tip_of_the_tongue_queries_dev(self):
+        self._test_queries('trec-tot/2023/dev', count=150, items={
+            0: TipOfTheTongueQuery('152', 'https://irememberthismovie.com/foriegn-film-about-3-strangers-in-an-apartment/', 'movie', 'Foriegn Film about 3 Strangers in an Apartment', 'Movie from  the early 2000s I believe about three people living in an apartment but never running into each other. One woman and two men are in the apartment. The woman is the realtor or owner of the apartment and at least one of the guys is a squator/homeless. It is a Korean or Chinese film I think. Art house flick… I think it won a few awards from film festivals like Cannes. Help if you can! ?', [{'id': 1, 'text': 'Movie from the early 2000s I believe about three people living in an apartment but never running into each other.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': True, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 2, 'text': 'One woman and two men are in the apartment.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 3, 'text': 'The woman is the realtor or owner of the apartment and at least one of the guys is a squator/homeless.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 4, 'text': 'It is a Korean or Chinese film I think.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': True, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 5, 'text': 'Art house flick.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': True, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 6, 'text': 'I think it won a few awards from film festivals like Cannes.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 7, 'text': 'Help if you can!', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': True, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}]),
+            9: TipOfTheTongueQuery('813', 'https://irememberthismovie.com/i-remember-this-movie-3/', 'movie', 'I remember this movie', 'Looking for movie I saw when I was a kid. It may have been an episode of a TV show. Anyway, this newly married couple bought an old lighthouse or at least a house that was on a cliff overlooking the sea.The woman found a bunch of mirrors in the attic of the house that mesmerized her. She would dance in front of them. The story was a former female occupant had died in the house under suspicious circumstances, maybe she committed suicide or something. But the mirrors were haunted and the ghost arose and fought with the woman. Her husband came to help and tried to beat the ghost with a metal rod, a fire poker. Of course, the solid object went through the ghost and he actually beat his wife to death. Despondent , he thought that he saw his wife dancing in front of a window beckoning to him and he jumped through the window to his death. Another couple witnessed this whereupon the woman fainted and her husband scooped her up and rushed from the house. It was a black and white movie and I think I saw it sometime in the early ’60s on television. I have looked at The Outer Limits Episodes but could find nothing. Would like to see it again. Any help appreciated.', [{'id': 1, 'text': 'Looking for movie I saw when I was a kid.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': True, 'situational_evidence': False}}}, {'id': 2, 'text': 'It may have been an episode of a TV show.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 3, 'text': 'Anyway, this newly married couple bought an old lighthouse or at least a house that was on a cliff overlooking the sea.The woman found a bunch of mirrors in the attic of the house that mesmerized her.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 4, 'text': 'She would dance in front of them.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 5, 'text': 'The story was a former female occupant had died in the house under suspicious circumstances, maybe she committed suicide or something.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 6, 'text': 'But the mirrors were haunted and the ghost arose and fought with the woman.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 7, 'text': 'Her husband came to help and tried to beat the ghost with a metal rod, a fire poker.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 8, 'text': 'Of course, the solid object went through the ghost and he actually beat his wife to death.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 9, 'text': 'Despondent , he thought that he saw his wife dancing in front of a window beckoning to him and he jumped through the window to his death.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 10, 'text': 'Another couple witnessed this whereupon the woman fainted and her husband scooped her up and rushed from the house.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 11, 'text': 'It was a black and white movie and I think I saw it sometime in the early ’60s on television.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': True, 'character': False, 'production_visual': True, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': True, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': True, 'situational_evidence': False}}}, {'id': 12, 'text': 'I have looked at The Outer Limits Episodes but could find nothing.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': True, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 13, 'text': 'Would like to see it again.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': True, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 14, 'text': 'Any help appreciated.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': True, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}]),
+            149: TipOfTheTongueQuery('521', 'https://irememberthismovie.com/ghost-story/', 'movie', 'Ghost story', 'There’s an old one I got to see just once as a kid. Dude takes a bet to survive one nite in a mansion. So, he’s there thinking he’s alone and all these people show up. Remember two real well. A girl he falls I love with and some scientist. Scientist talking at one point shows experiment where he’s talking about how long something can live, cuts the head off a snake, tells dude to try to pick up snake head and dude almost gets bit. Another scene he’s with the girl in bed laying his head in her chest and jumps up worried cause he can’t hear her heartbeat. End of show he walks out stands by the mansion gates and it slams on him with the spikes stabbing him in the back. He loses the bet but gets the girl', [{'id': 1, 'text': 'There’s an old one I got to see just once as a kid.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': True, 'situational_evidence': False}}}, {'id': 2, 'text': 'Dude takes a bet to survive one nite in a mansion.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 3, 'text': 'So, he’s there thinking he’s alone and all these people show up.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 4, 'text': 'Remember two real well.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 5, 'text': 'A girl he falls I love with and some scientist.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 6, 'text': 'Scientist talking at one point shows experiment where he’s talking about how long something can live, cuts the head off a snake, tells dude to try to pick up snake head and dude almost gets bit.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 7, 'text': 'Another scene he’s with the girl in bed laying his head in her chest and jumps up worried cause he can’t hear her heartbeat.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 8, 'text': 'End of show he walks out stands by the mansion gates and it slams on him with the spikes stabbing him in the back.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 9, 'text': 'He loses the bet but gets the girl', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}]),
+        })
+
+    def test_test_tip_of_the_tongue_qrels_train(self):
+        self._test_qrels('trec-tot/2023/train', count=150, items={
+            0: TrecQrel('763', '16742289', 1, '0'),
+            9: TrecQrel('293', '142456', 1, '0'),
+            149: TrecQrel('828', '30672517', 1, '0'),
+        })
+
+
+    def test_test_tip_of_the_tongue_qrels_dev(self):
+        self._test_qrels('trec-tot/2023/dev', count=150, items={
+            0: TrecQrel('152', '1940119', 1, '0'),
+            9: TrecQrel('813', '2310134', 1, '0'),
+            149: TrecQrel('521', '2911505', 1, '0'),
+        })
+
+
+if __name__ == '__main__':
+    unittest.main()
+