Repository: nltk/nltk

Issue:
Duplicate merged synsets and lemmas with OMW
The OMW data packages include thousands of duplicate lemmas (https://github.com/nltk/nltk_data/issues/194#issue-1614884960). That problem may be fixed in future data releases, but even then, NLTK will still need to perform a check for duplicate synsets and lemmas at load time.

The reason is that when using OMW together with newer Wordnet versions, the situation changes because a few synsets have been merged. When merging two synsets that both contain a sense of the same word, these two senses are collapsed into one. But without a check for duplicates at load time, the same sense can be appended several times to the list of  lemmas or synsets.

So, for example, each of the following calls outputs duplicates:

```
from nltk.corpus import wordnet2021 as wn

finssl = wn.synsets('baseball', lang="fin")
print(finssl)
```
[Synset('baseball.n.01'), Synset('baseball.n.01'), Synset('hardball.n.02'), Synset('two-hitter.n.01')]

```
ss = finssl[0]
print(ss.lemmas(lang="fin"))
```
[Lemma('baseball.n.01.baseball'), Lemma('baseball.n.01.baseball-peli'), Lemma('baseball.n.01.baseball')]

`print(ss.lemma_names(lang="fin"))`
['baseball', 'baseball-peli', 'baseball']

`print(wn.lemmas('baseball', lang="fin"))`
 [Lemma('baseball.n.01.baseball'), Lemma('baseball.n.01.baseball'), Lemma('baseball.n.01.baseball'), Lemma('baseball.n.01.baseball'), Lemma('hardball.n.02.baseball'), Lemma('two-hitter.n.01.baseball')]

Fortunately, this problem is easily fixed with a simple check at load time.



Discussion:

Resolving patch:
diff --git a/nltk/corpus/reader/wordnet.py b/nltk/corpus/reader/wordnet.py
--- a/nltk/corpus/reader/wordnet.py
+++ b/nltk/corpus/reader/wordnet.py
@@ -1216,7 +1216,7 @@ def __init__(self, root, omw_reader):
         self.map30 = self.map_wn()
 
         # Language data attributes
-        self.lg_attrs = ["lemma", "none", "def", "exe"]
+        self.lg_attrs = ["lemma", "of", "def", "exe"]
 
     def index_sense(self, version=None):
         """Read sense key to synset id mapping from index.sense file in corpus directory"""
@@ -1251,7 +1251,7 @@ def map_to_many(self, version="wordnet"):
         return synset_to_many
 
     def map_to_one(self, version="wordnet"):
-        self.nomap[version] = []
+        self.nomap[version] = set()
         self.splits[version] = {}
         synset_to_many = self.map_to_many(version)
         synset_to_one = {}
@@ -1273,7 +1273,7 @@ def map_to_one(self, version="wordnet"):
                     # where only Lithuanian and Slovak use the "s" ss_type.
                     synset_to_one[f"{source[:-1]}a"] = target
             else:
-                self.nomap[version].append(source)
+                self.nomap[version].add(source)
         return synset_to_one
 
     def map_wn(self, version="wordnet"):
@@ -1294,7 +1294,9 @@ def merged_synsets(self, version="wordnet"):
             for source, targets in self.map_to_many(version).items():
                 for target in targets:
                     merge[target].add(source)
-            self.merges[version] = {s: t for s, t in merge.items() if len(t) > 1}
+            self.merges[version] = {
+                trg: src for trg, src in merge.items() if len(src) > 1
+            }
         return self.merges[version]
 
     # Open Multilingual WordNet functions, contributed by
@@ -2230,8 +2232,9 @@ def custom_lemmas(self, tab_file, lang):
                     else:
                         # Some OMW offsets were never in Wordnet:
                         if (
-                            offset_pos not in self.nomap
-                            and offset_pos.replace("a", "s") not in self.nomap
+                            offset_pos not in self.nomap["wordnet"]
+                            and offset_pos.replace("a", "s")
+                            not in self.nomap["wordnet"]
                         ):
                             warnings.warn(
                                 f"{lang}: invalid offset {offset_pos} in '{line}'"
@@ -2247,11 +2250,15 @@ def custom_lemmas(self, tab_file, lang):
                 if len(pair) == 1 or pair[0] == lg:
                     if attr == "lemma":
                         val = val.strip().replace(" ", "_")
-                        self._lang_data[lang][1][val.lower()].append(offset_pos)
+                        lang_offsets = self._lang_data[lang][1][val.lower()]
+                        if offset_pos not in lang_offsets:
+                            lang_offsets.append(offset_pos)
                     if attr in self.lg_attrs:
-                        self._lang_data[lang][self.lg_attrs.index(attr)][
+                        lang_lemmas = self._lang_data[lang][self.lg_attrs.index(attr)][
                             offset_pos
-                        ].append(val)
+                        ]
+                        if val not in lang_lemmas:
+                            lang_lemmas.append(val)
 
     def disable_custom_lemmas(self, lang):
         """prevent synsets from being mistakenly added"""


Test patch: