Repository: KnowledgeCaptureAndDiscovery/somef (language: python)

Issue:
Fixed issue 150

Acknowledgments is not detected
For example, in https://github.com/oeg-upm/wot-hive the header analysis is not returned.
If image has the tool name, we should consider it as logo
For example: https://github.com/urllib3/urllib3/blob/main/README.md
Parser not working well for level 3 and level 4 headers
See for example https://github.com/oeg-upm/AI4EU_raidologist 

Command `somef describe -r https://github.com/oeg-upm/AI4EU_raidologist -o test.json`

Results: 
![Screenshot from 2022-03-12 12-20-55](https://user-images.githubusercontent.com/1520666/158016060-79377158-bbea-4b23-b212-fbcb6a13f7e8.png)

This includes three headers, which does not correspond to the 2 classification results returned. The parser is not working well here.
Somef misses some Dockerfiles
In https://github.com/oeg-upm/wot-hive/tree/main/docker/auroral-hive

somef does not seem to detect any dockerfiles, but there is one.


Discussion / hints:

Fix patch:
diff --git a/src/somef/cli.py b/src/somef/cli.py
index ee4a0486..e95f608a 100644
--- a/src/somef/cli.py
+++ b/src/somef/cli.py
@@ -150,7 +150,7 @@ def load_repository_metadata(repository_url, header, ignore_github_metadata=Fals
     Returns the readme text and required metadata
     """
     if repository_url.rfind("gitlab.com") > 0:
-        return load_repository_metadata_gitlab(repository_url, header, ignore_github_metadata)
+        return load_repository_metadata_gitlab(repository_url, header)
 
     print(f"Loading Repository {repository_url} Information....")
     ## load general response of the repository
@@ -279,7 +279,8 @@ def get_path(obj, path):
         stargazers_info['count'] = filtered_resp['stargazers_count']
         stargazers_info['date'] = date
         del filtered_resp['stargazers_count']
-    filtered_resp['stargazersCount'] = stargazers_info
+    if len(stargazers_info.keys()) > 0:
+        filtered_resp['stargazersCount'] = stargazers_info
 
     # get social features: forks_count
     forks_info = {}
@@ -287,7 +288,8 @@ def get_path(obj, path):
         forks_info['count'] = filtered_resp['forks_count']
         forks_info['date'] = date
         del filtered_resp['forks_count']
-    filtered_resp['forksCount'] = forks_info
+    if len(forks_info.keys()) > 0:
+        filtered_resp['forksCount'] = forks_info
 
     ## get languages
     if not ignore_github_metadata:
@@ -817,15 +819,6 @@ def load_local_repository_metadata(local_repo):
     scriptfiles = []
     text = ""
     repo_dir = os.path.abspath(local_repo)
-    if os.path.exists(os.path.join(repo_dir, "README")):
-        with open(os.path.join(os.path.join(repo_dir, "README")), "r", encoding='utf-8') as data_file:
-            text = data_file.read()
-    elif os.path.exists(os.path.join(repo_dir, "README.MD")):
-        with open(os.path.join(os.path.join(repo_dir, "README.MD")), "r", encoding='utf-8') as data_file:
-            text = data_file.read()
-    elif os.path.exists(os.path.join(repo_dir, "README.md")):
-        with open(os.path.join(os.path.join(repo_dir, "README.md")), "r", encoding='utf-8') as data_file:
-            text = data_file.read()
     for dirpath, dirnames, filenames in os.walk(repo_dir):
         repo_relative_path = os.path.relpath(dirpath, repo_dir)
         for filename in filenames:
@@ -833,6 +826,14 @@ def load_local_repository_metadata(local_repo):
                 dockerfiles.append(os.path.join(repo_dir, repo_relative_path, filename))
             if filename.lower().endswith(".ipynb"):
                 notebooks.append(os.path.join(repo_dir, repo_relative_path, filename))
+            if "README" == filename.upper() or "README.MD" == filename.upper():
+                if (repo_relative_path == "."):
+                    try:
+                        with open(os.path.join(dirpath, filename), "rb") as data_file:
+                            data_file_text = data_file.read()
+                            text = data_file_text.decode("utf-8")
+                    except:
+                        print("README Error: error while reading file content")
             if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
                 try:
                     with open(os.path.join(dirpath, filename), "rb") as data_file:
@@ -944,6 +945,32 @@ def remove_bibtex(string_list):
     return string_list
 
 
+def remove_links_images(text):
+    # process images
+    images = re.findall(r"!\[(.*?)?\]\((.*?)?\)", text)
+    for image in images:
+        link_text = image[1]
+        pos = text.find(link_text)
+        if pos != -1:
+            init = text[:pos].rindex("![")
+            end = text[pos:].index(")")
+            image_text = text[init:pos+end+1]
+            text = text.replace(image_text,"")
+
+    #process links
+    links = re.findall(r"\[(.*?)?\]\(([^)]+)\)", text)
+    for link in links:
+        link_text = link[1]
+        pos = text.find(link_text)
+        if pos != -1:
+            init = text[:pos].rindex("[")
+            end = text[pos:].index(")")
+            link_text = text[init:pos + end + 1]
+            text = text.replace(link_text, "")
+
+    # remove blank spaces and \n
+    return text.strip()
+
 ## Function takes readme text as input and divides it into excerpts
 ## Returns the extracted excerpts
 def create_excerpts(string_list):
@@ -952,7 +979,13 @@ def create_excerpts(string_list):
     # divisions = createExcerpts.split_into_excerpts(string_list)
     divisions = parser_somef.extract_blocks_excerpts(string_list)
     print("Text Successfully split. \n")
-    return divisions
+    # issue 337
+    output = []
+    for division in divisions:
+        division = remove_links_images(division)
+        if len(division) > 0:
+            output.append(division)
+    return output
 
 
 def run_classifiers(excerpts, file_paths):
@@ -1294,8 +1327,106 @@ def extract_logo(unfiltered_text, repo_url):
     return logo
 
 
-# TO DO: join with logo detection
 def extract_images(unfiltered_text, repo_url):
+    logo = ""
+    has_logo = False
+    images = []
+    repo = False
+    repo_name = ""
+    if repo_url != None and repo_url != "":
+        url = urlparse(repo_url)
+        path_components = url.path.split('/')
+        repo_name = path_components[2]
+        repo = True
+
+    html_text = markdown.markdown(unfiltered_text)
+    img_md = re.findall(r"!\[[^\]]*\]\((.*?)?\)", html_text)
+    result = [_.start() for _ in re.finditer("<img ", html_text)]
+    for img in img_md:
+        if not has_logo and repo:
+            start = img.rindex("/")
+            if img.find(repo_name, start) > 0:
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            elif get_alt_text_md(html_text, img) == repo_name or get_alt_text_md(html_text, img).upper() == "LOGO":
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            else:
+                start = img.rindex("/")
+                if img.upper().find("LOGO", start) > 0:
+                    logo = rename_github_image(img, repo_url)
+                    has_logo = True
+                else:
+                    images.append(rename_github_image(img, repo_url))
+        else:
+            images.append(rename_github_image(img, repo_url))
+    for index_img in result:
+        init = html_text.find("src=\"", index_img)
+        end = html_text.find("\"", init + 5)
+        img = html_text[init + 5:end]
+        if not has_logo and repo:
+            start = img.rindex("/")
+            image_name = img[start:]
+            if image_name.find(repo_name) > 0 or image_name.upper().find("LOGO") > 0:
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            elif get_alt_text_img(html_text, index_img) == repo_name or get_alt_text_img(html_text, index_img).upper() == "LOGO":
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            else:
+                images.append(rename_github_image(img, repo_url))
+        else:
+            start = img.rindex("/")
+            if img.upper().find("LOGO", start) > 0:
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            else:
+                images.append(rename_github_image(img, repo_url))
+
+    return logo, images
+
+
+def rename_github_image(img, repo_url):
+    if not img.startswith("http") and repo_url is not None and repo_url != "":
+        if repo_url.find("/tree/") > 0:
+            repo_url = repo_url.replace("/tree/", "/")
+        else:
+            repo_url = repo_url + "/master/"
+        repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
+        if not repo_url.endswith("/"):
+            repo_url = repo_url + "/"
+        img = repo_url + img
+    return img
+
+
+def get_alt_text_md(text, image):
+    stop = text.find(image) - 2
+    start = text[:stop].rindex("![") + 2
+    return text[start:stop]
+
+
+def get_alt_text_img(html_text, index):
+    end = html_text.find(">", index)
+    output = ""
+    if html_text.find("alt=", index, end) > 0:
+        texto = html_text[index:end]
+        init = texto.find("alt=\"") + 5
+        end = texto.index("\"", init)
+        output = texto[init:end]
+    return output
+
+
+def get_alt_text_html(text, image):
+    stop = text.find(image)
+    start = text[:stop].rindex("<img")
+    if text[start:stop].find("alt=") != -1:
+        start = text.find("alt=",start) + 4
+        stop = text.find("\"",start)-1
+        return text[start:stop]
+    return ""
+
+# TO DO: join with logo detection
+def extract_images_old(unfiltered_text, repo_url):
     logo = ""
     images = []
     html_text = markdown.markdown(unfiltered_text)
@@ -1329,6 +1460,20 @@ def extract_images(unfiltered_text, repo_url):
                     img = repo_url + img
                 logo = img
 
+    if logo == "" and repo_url != "" and repo_url != None:
+        print(repo_url)
+        url = urlparse(repo_url)
+        path_components = url.path.split('/')
+        repo_name = path_components[2]
+
+        for image in images:
+            start = image.rindex("/")
+            if image.find(repo_name,start) > 0:
+                logo = image
+                images.remove(image)
+                break
+
+
     return logo, images
 
 
@@ -1348,9 +1493,19 @@ def extract_support(unfiltered_text):
 
     return results
 
+def extract_package_distributions(unfiltered_text):
+    output = ""
+    index_package_distribution = unfiltered_text.find("[![PyPI]")
+    if index_package_distribution > 0:
+        init = unfiltered_text.find(")](", index_package_distribution)
+        end = unfiltered_text.find(")", init + 3)
+        package_distribution = unfiltered_text[init + 3:end]
+        output = requests.get(package_distribution).url
+
+    return output
 
 def merge(header_predictions, predictions, citations, citation_file_text, dois, binder_links, long_title,
-          readthedocs_links, repo_status, arxiv_links, logo, images, support_channels):
+          readthedocs_links, repo_status, arxiv_links, logo, images, support_channels, package_distribution):
     """
     Function that takes the predictions using header information, classifier and bibtex/doi parser
     Parameters
@@ -1415,6 +1570,10 @@ def merge(header_predictions, predictions, citations, citation_file_text, dois,
         predictions['supportChannels'] = {'excerpt': support_channels, 'confidence': [1.0],
                                           'technique': 'Regular expression'}
 
+    if len(package_distribution) != 0:
+        predictions['packageDistribution'] = {'excerpt': package_distribution, 'confidence': [1.0],
+                                          'technique': 'Regular expression'}
+
     for i in range(len(readthedocs_links)):
         if 'documentation' not in predictions.keys():
             predictions['documentation'] = []
@@ -1595,66 +1754,16 @@ def average_confidence(x):
                 "@id": "https://github.com/" + author_name
             }
         ]
-    if "acknowledgement" in repo_data:
-        codemeta_output["acknowledgement"] = data_path(["acknowledgement", "excerpt"])
-    if "support" in repo_data:
-        codemeta_output["support"] = data_path(["support", "excerpt"])
     if "citation" in repo_data:
         codemeta_output["citation"] = data_path(["citation", "excerpt"])
-    if "citationFile" in repo_data:
-        codemeta_output["citationFile"] = data_path(["citationFile", "excerpt"])
-    if "codeOfConduct" in repo_data:
-        codemeta_output["codeOfConduct"] = data_path(["codeOfConduct", "excerpt"])
-    if "forks_count" in repo_data:
-        codemeta_output["forks_count"] = data_path(["forks_count", "excerpt"])
-    if "forks_url" in repo_data:
-        codemeta_output["forks_url"] = data_path(["forks_url", "excerpt"])
-    if "fullName" in repo_data:
-        codemeta_output["fullName"] = data_path(["fullName", "excerpt"])
-    if "hasBuildFile" in repo_data:
-        codemeta_output["hasBuildFile"] = data_path(["hasBuildFile", "excerpt"])
-    if "hasDocumentation" in repo_data:
-        codemeta_output["hasDocumentation"] = data_path(["hasDocumentation", "excerpt"])
-    if "hasExecutableNotebook" in repo_data:
-        codemeta_output["hasExecutableNotebook"] = data_path(["hasExecutableNotebook", "excerpt"])
     if "identifier" in repo_data:
         codemeta_output["identifier"] = data_path(["identifier", "excerpt"])
-    if "invocation" in repo_data:
-        codemeta_output["invocation"] = data_path(["invocation", "excerpt"])
     if "issueTracker" in repo_data:
         codemeta_output["issueTracker"] = data_path(["issueTracker", "excerpt"])
-    if "name" in repo_data:
-        codemeta_output["name"] = data_path(["name", "excerpt"])
-    if "ownerType" in repo_data:
-        codemeta_output["ownerType"] = data_path(["ownerType", "excerpt"])
     if "readme_url" in repo_data:
-        codemeta_output["readme_url"] = data_path(["readme_url", "excerpt"])
-    if "stargazers_count" in repo_data:
-        codemeta_output["stargazers_count"] = data_path(["stargazers_count", "excerpt"])
-    if "arxivLinks" in repo_data:
-        codemeta_output["arxivLinks"] = data_path(["arxivLinks", "excerpt"])
-    if "codeOfConduct" in repo_data:
-        codemeta_output["codeOfConduct"] = data_path(["codeOfConduct", "excerpt"])
-    if "contributingGuidelines" in repo_data:
-        codemeta_output["contributingGuidelines"] = data_path(["contributingGuidelines", "excerpt"])
-    if "contributingGuidelinesFile" in repo_data:
-        codemeta_output["contributingGuidelinesFile"] = data_path(["contributingGuidelinesFile", "excerpt"])
-    if "licenseFile" in repo_data:
-        codemeta_output["licenseFile"] = data_path(["licenseFile", "excerpt"])
-    if "licenseText" in repo_data:
-        codemeta_output["licenseText"] = data_path(["licenseText", "excerpt"])
-    if "acknowledgments" in repo_data:
-        codemeta_output["acknowledgments"] = data_path(["acknowledgments", "excerpt"])
-    if "acknowledgmentsFile" in repo_data:
-        codemeta_output["acknowledgmentsFile"] = data_path(["acknowledgmentsFile", "excerpt"])
+        codemeta_output["readme"] = data_path(["readme_url", "excerpt"])
     if "contributors" in repo_data:
-        codemeta_output["contributors"] = data_path(["contributors", "excerpt"])
-    if "contributorsFile" in repo_data:
-        codemeta_output["contributorsFile"] = data_path(["contributorsFile", "excerpt"])
-    if "hasScriptFile" in repo_data:
-        codemeta_output["hasScriptFile"] = data_path(["hasScriptFile", "excerpt"])
-    if "executableExample" in repo_data:
-        codemeta_output["executableExample"] = data_path(["executableExample", "excerpt"])
+        codemeta_output["contributor"] = data_path(["contributors", "excerpt"])
     if descriptions_text:
         codemeta_output["description"] = descriptions_text
     if published_date != "":
@@ -1738,7 +1847,7 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc
         assert (doc_src is not None)
         if not path.exists(doc_src):
             sys.exit("Error: Document does not exist at given path")
-        with open(doc_src, 'r') as doc_fh:
+        with open(doc_src, 'r', encoding="UTF-8") as doc_fh:
             text = doc_fh.read()
         github_data = {}
 
@@ -1768,6 +1877,7 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc
         #logo = extract_logo(unfiltered_text, repo_url)
         logo, images = extract_images(unfiltered_text, repo_url)
         support_channels = extract_support_channels(unfiltered_text)
+        package_distribution = extract_package_distributions(unfiltered_text)
     else:
         citations = []
         citation_file_text = ""
@@ -1780,9 +1890,10 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc
         logo = ""
         images = []
         support_channels = []
+        package_distribution = ""
 
     predictions = merge(header_predictions, predictions, citations, citation_file_text, dois, binder_links, title,
-                        readthedocs_links, repo_status, arxiv_links, logo, images, support_channels)
+                        readthedocs_links, repo_status, arxiv_links, logo, images, support_channels, package_distribution)
     gitlab_url = False
     if repo_url is not None:
         if repo_url.rfind("gitlab.com") > 0:
diff --git a/src/somef/schema/software_schema.py b/src/somef/schema/software_schema.py
index 4406fabd..2765190e 100644
--- a/src/somef/schema/software_schema.py
+++ b/src/somef/schema/software_schema.py
@@ -125,7 +125,7 @@
         "@type": "xsd:string",
     },
     "sd:hasExecutableNotebook": {
-        "@path": ["executable_example", "excerpt"],
+        "@path": ["executableExample", "excerpt"],
         "@type": "xsd:anyURI"
     },
     "sd:hasCodeOfConduct": {
@@ -226,7 +226,7 @@
         "@type": "xsd:dateTime"
     },
     "sd:hasExample": {
-        "@path": "executable_example",
+        "@path": "executableExample",
         "@type": "xsd:string"
     },
     "sd:hasSupportScriptLocation": {


Test patch:
diff --git a/src/somef/test/test_cli.py b/src/somef/test/test_cli.py
index e9f28894..e0772652 100644
--- a/src/somef/test/test_cli.py
+++ b/src/somef/test/test_cli.py
@@ -358,4 +358,107 @@ def test_no_repository_metadata(self):
             header['Authorization'] = file_paths['Authorization']
         header['accept'] = 'application/vnd.github.v3+json'
         text, github_data = load_repository_metadata("https://github.com/RDFLib/rdflib/tree/6.0.2", header, True)
-        assert ('releases' not in github_data) == True
\ No newline at end of file
+        assert ('releases' not in github_data) == True
+
+    def test_issue_150(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url=None,
+                doc_src=test_data_path + "README-mapshaper.md",
+                local_repo=None,
+                in_file=None,
+                output=None,
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=test_data_path + "test-150.json-ld",
+                pretty=True,
+                missing=False)
+        text_file = open(test_data_path + "test-150.json-ld", "r")
+        data = text_file.read()
+        text_file.close()
+        assert data.find("acknowledgement") == -1
+        os.remove(test_data_path + "test-150.json-ld")
+
+    def test_issue_346(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url=None,
+                doc_src=test_data_path + "README-wothive.md",
+                in_file=None,
+                output=test_data_path + "test-346.json",
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=None,
+                pretty=True,
+                missing=True)
+        text_file = open(test_data_path + "test-346.json", "r")
+        data = text_file.read()
+        text_file.close()
+        assert data.find("acknowledgement") > 0
+        os.remove(test_data_path + "test-346.json")
+
+    def test_issue_241(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url=None,
+                doc_src=test_data_path + "README-tensorflow-2.6.0.md",
+                in_file=None,
+                output=test_data_path + "test-241.json",
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=None,
+                pretty=True,
+                missing=True)
+        text_file = open(test_data_path + "test-241.json", "r")
+        data = text_file.read()
+        text_file.close()
+        assert data.find("packageDistribution") > 0
+        os.remove(test_data_path + "test-241.json")
+
+    def test_issue_320(self):
+        with open(test_data_path + "README-urllib3.md", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = extract_images(test_text, "https://github.com/urllib3/urllib3")
+            assert (not logo == "")
+
+    def test_issue_361(self):
+        text, github_data = load_local_repository_metadata(test_data_repositories + "rdflib-6.0.2")
+        assert len(text) > 0
+
+    def test_issue_380(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url=None,
+                doc_src=test_data_path + "README-raidologist.md",
+                in_file=None,
+                output=test_data_path + "test-380.json",
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=None,
+                pretty=True,
+                missing=True)
+        text_file = open(test_data_path + "test-380.json", "r")
+        data = text_file.read()
+        text_file.close()
+        assert data.find("installation") > 0
+        os.remove(test_data_path + "test-380.json")
+
+    def test_issue_379(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url=None,
+                doc_src=test_data_path + "README-raidologist.md",
+                in_file=None,
+                output=test_data_path + "test-379.json",
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=None,
+                pretty=True,
+                missing=True)
+        text_file = open(test_data_path + "test-379.json", "r")
+        data = text_file.read()
+        text_file.close()
+        json_content = json.loads(data)
+        description = json_content['description']
+        assert len(description) == 1
+        os.remove(test_data_path + "test-379.json")
\ No newline at end of file
diff --git a/src/somef/test/test_data/README-mapshaper.md b/src/somef/test/test_data/README-mapshaper.md
new file mode 100644
index 00000000..68d84685
--- /dev/null
+++ b/src/somef/test/test_data/README-mapshaper.md
@@ -0,0 +1,101 @@
+# Mapshaper
+
+## Introduction
+
+Mapshaper is software for editing Shapefile, GeoJSON, [TopoJSON](https://github.com/mbostock/topojson/wiki), CSV and several other data formats, written in JavaScript.
+
+Mapshaper supports essential map making tasks like simplifying shapes, editing attribute data, clipping, erasing, dissolving, filtering and more.
+
+See the [project wiki](https://github.com/mbloch/mapshaper/wiki) for documentation on how to use mapshaper.
+
+To suggest improvements, add an [issue](https://github.com/mbloch/mapshaper/issues).
+
+
+## Command line tools
+
+Mapshaper includes several command line programs, which can be run under Mac OS X, Linux and Windows.
+
+* `mapshaper` Runs mapshaper commands.
+* `mapshaper-xl` Works the same as `mapshaper`, but runs with more RAM to support larger files.
+* `mapshaper-gui` Runs the mapshaper Web interface locally.
+
+The project wiki has an [introduction](https://github.com/mbloch/mapshaper/wiki/Introduction-to-the-Command-Line-Tool) to using the command line tool that includes many simple examples.
+
+For a detailed reference, see the [Command Reference](https://github.com/mbloch/mapshaper/wiki/Command-Reference).
+
+
+## Interactive web interface
+
+Visit the public website at [www.mapshaper.org](http://www.mapshaper.org) or use the web UI locally via the `mapshaper-gui` script. 
+
+All processing is done in the browser, so your data stays private, even when using the public website.
+
+The web UI works in recent desktop versions of Chrome, Firefox, Safari and Internet Explorer. Safari before v10.1 and IE before v10 are not supported.
+
+## User-contributed resources
+
+[rmapshaper](https://github.com/ateucher/rmapshaper) is an R package written by Andy Teucher that gives R users access to many of mapshaper's editing commands.
+
+[Here](https://hub.docker.com/r/freifunkhamm/mapshaper) are resources for using mapshaper with Docker, provided by Christian Weiss.
+
+You can find a number of mapshaper tutorials online, including a [two](https://moriartynaps.org/command-carto-part-one/) [part](https://moriartynaps.org/command-carto-part-two/) guide to command line cartography by Dylan Moriarty and [this introduction](https://datavizforall.org/convert-edit-join-and-dissolve-with-mapshaper-org.html) by Jack Dougherty.
+
+
+## Large file support
+
+**Web interface**
+
+Firefox is able to load Shapefiles and GeoJSON files larger than 1GB. Chrome has improved in recent versions, but is still prone to out-of-memory errors when importing files larger than several hundred megabytes.
+
+**Command line interface**
+
+There are hard limits for reading and writing most file types. The maximum output size of a single file of any type is 2GB. Some file types (GeoJSON, CSV, .shp) are read incrementally, so much larger files can be imported.
+
+When working with very large files, mapshaper may become unresponsive or crash with the message "JavaScript heap out of memory."
+
+One option is to run `mapshaper-xl`, which allocates more memory than the standard `mapshaper` program (8GB by default). Starting with version 0.4.131, you can specify the amount of memory to allocate like this: `mapshaper-xl 20gb [commands]`.
+
+Another solution is to run Node directly with the `--max-old-space-size` option. The following example (Mac or Linux) allocates 16GB of memory:
+```bash
+$ node  --max-old-space-size=16000 `which mapshaper` <mapshaper commands>
+```
+
+## Installation
+
+Mapshaper requires [Node.js](http://nodejs.org).
+
+With Node installed, you can install the latest release version of mapshaper using npm. Install with the "-g" flag to make the executable scripts available systemwide.
+
+```bash
+npm install -g mapshaper
+```
+
+To install and run the latest development code from github:
+
+```bash
+git clone git@github.com:mbloch/mapshaper.git
+cd mapshaper
+npm install       # install dependencies
+npm run build     # bundle source code files
+npm link          # (optional) add global symlinks so scripts are available systemwide
+```
+
+## Building and testing
+
+From the project directory, run `npm run build` to build both the cli and web UI modules.
+
+Run `npm test` to run mapshaper's tests.
+
+## License
+
+This software is licensed under [MPL 2.0](http://www.mozilla.org/MPL/2.0/).
+
+According to Mozilla's [FAQ](http://www.mozilla.org/MPL/2.0/FAQ.html), "The MPL's ‘file-level’ copyleft is designed to encourage contributors to share modifications they make to your code, while still allowing them to combine your code with code under other licenses (open or proprietary) with minimal restrictions."
+
+
+
+## Acknowledgements
+
+My colleagues at The New York Times, for countless suggestions, bug reports and general helpfulness.
+
+Mark Harrower, for collaborating on the original "MapShaper" program at the University of Wisconsin&ndash;Madison.
diff --git a/src/somef/test/test_data/README-raidologist.md b/src/somef/test/test_data/README-raidologist.md
new file mode 100644
index 00000000..67ce934d
--- /dev/null
+++ b/src/somef/test/test_data/README-raidologist.md
@@ -0,0 +1,67 @@
+# r.AID.ologist 🖊️
+
+### 🔍**Description of the system**
+
+r.AID.ologist is a framework designed for radiologists to provide assistance on the generation of medical reports. 
+It is devised as a Case-Based Reasoning model, powered by several deep learning models which have been included to improve its performance, as well as providing additional features.  
+
+While CBR is integrated as a continuous cycle, its subparts are designed to be fully modular, thus can be easily substituted to fit new data. The following diagram provides an overview on the system. Modular elements are represented by bricks, implying that they can be replaced by different models as long as they meet the same objective. 
+
+<img src="https://i.ibb.co/7Y4ZyhR/Global-Diagram.png" width=50% height=50%>
+
+### 🧱**Module specification**
+
+The implementations of the different modules are contained in the *functions.py* file from the internal_functions folder. By default, the following models are employed for each of the denoted modules:
+
+#### Retrieve Modules
+
+ 1. ***Image feature extraction***: A combination of a keypoint-based image feature algorithm (KAZE) and a pretrained Convolutional Neural Network (ResNet18). 
+ 2. ***Document embedding***:  An English pretrained model from SciSpacy is used (en_core_sci_md)
+ 3. ***Named Entity Recognition***: An external resource, CliNER, is employed for this task. The employed pretrained model works for English textual data, and distinguishes between three types of named entities: problems, treatments and tests. These types **MUST** remain unchangeable in case of substitution. 
+ 4. ***Noise filtering***: SciSpacy is used to detect existing abbreviations. The ratio of identified abbreviations is obtained as *#_(detected_abbreviations ^ existing_tokens)/ #_detected abbreviations*
+
+#### Reuse Modules
+
+ 1. ***Sectioning Model***: A bidirectional LSTM, implemented via Pytorch, is used to section the input report. The employed sectioning model, alongside its functions, is implemented in the *section_model.py* file contained in the internal_functions folder. In this default model, four sections are considered: **Findings, Comparison, Indication and Impression**
+ 2. ***Scoring Model***: A combination of the aforementioned SciSpacy embedding model with a Random Forest Classifier is employed  to score each report as valid (1) or rejected (0).
+ 3. ***Abbreviation Disambiguation***: The aforementioned SciSpacy English pretrained model is used to extract existing abbreviations. SNOMED's query service is then used to obtain potential disambiguations for each of the detected abbreviations.
+
+### 🖱️ **Installation**
+
+A ready-to-use version of the framework is available as a Docker image in **elviish/raidologist:latest**. Once downloaded, the image can be ran with the command:
+
+    docker run --name raidologist -p *selected_port*:5000 elviish/raidologist:latest
+  
+The framework has been developed using Python 3.7. To locally run the framework,  or o modify it, these are the steps to follow:
+
+ 1. Clone the repository
+ `git clone https://github.com/oeg-upm/AI4EU_raidologist.git`
+ 
+ 2. Install the required dependencies
+ `pip install -r requirements.txt`
+ 
+ 3. Install the spacy model
+ `python -m spacy download en_core_web_sm`
+ 
+ 4. Download the NER model and place it the directory *externals/i2b2*
+`curl -O https://drive.upm.es/s/gOXyNxXgnrDBIEj/download`
+
+ 5. Execute the *main.py* file 
+`python main.py`
+
+ 6. Go to the direction localhost:5000 in your web browser to start using the framework
+
+Issues can be reported at https://github.com/oeg-upm/AI4EU_raidologist/issues
+
+### ❓ **About**
+This framework has been developed as part of the AI4EU project. 
+
+It is licensed under APACHE 2.0.
+
+   
+    
+
+
+ 
+
+
diff --git a/src/somef/test/test_data/README-urllib3.md b/src/somef/test/test_data/README-urllib3.md
new file mode 100644
index 00000000..a72b785f
--- /dev/null
+++ b/src/somef/test/test_data/README-urllib3.md
@@ -0,0 +1,110 @@
+<h1 align="center">
+
+![urllib3](https://github.com/urllib3/urllib3/raw/main/docs/_static/banner_github.svg)
+
+</h1>
+
+<p align="center">
+  <a href="https://pypi.org/project/urllib3"><img alt="PyPI Version" src="https://img.shields.io/pypi/v/urllib3.svg?maxAge=86400" /></a>
+  <a href="https://pypi.org/project/urllib3"><img alt="Python Versions" src="https://img.shields.io/pypi/pyversions/urllib3.svg?maxAge=86400" /></a>
+  <a href="https://discord.gg/urllib3"><img alt="Join our Discord" src="https://img.shields.io/discord/756342717725933608?color=%237289da&label=discord" /></a>
+  <a href="https://github.com/urllib3/urllib3/actions?query=workflow%3ACI"><img alt="Coverage Status" src="https://img.shields.io/badge/coverage-100%25-success" /></a>
+  <a href="https://github.com/urllib3/urllib3/actions?query=workflow%3ACI"><img alt="Build Status on GitHub" src="https://github.com/urllib3/urllib3/workflows/CI/badge.svg" /></a>
+  <a href="https://urllib3.readthedocs.io"><img alt="Documentation Status" src="https://readthedocs.org/projects/urllib3/badge/?version=latest" /></a>
+</p>
+
+urllib3 is a powerful, *user-friendly* HTTP client for Python. Much of the
+Python ecosystem already uses urllib3 and you should too.
+urllib3 brings many critical features that are missing from the Python
+standard libraries:
+
+- Thread safety.
+- Connection pooling.
+- Client-side SSL/TLS verification.
+- File uploads with multipart encoding.
+- Helpers for retrying requests and dealing with HTTP redirects.
+- Support for gzip, deflate, and brotli encoding.
+- Proxy support for HTTP and SOCKS.
+- 100% test coverage.
+
+urllib3 is powerful and easy to use:
+
+```python3
+>>> import urllib3
+>>> http = urllib3.PoolManager()
+>>> resp = http.request("GET", "http://httpbin.org/robots.txt")
+>>> resp.status
+200
+>>> resp.data
+b"User-agent: *\nDisallow: /deny\n"
+```
+
+## Installing
+
+urllib3 can be installed with [pip](https://pip.pypa.io):
+
+```bash
+$ python -m pip install urllib3
+```
+
+Alternatively, you can grab the latest source code from [GitHub](https://github.com/urllib3/urllib3):
+
+```bash
+$ git clone git://github.com/urllib3/urllib3.git
+$ pip install .
+```
+
+
+## Documentation
+
+urllib3 has usage and reference documentation at [urllib3.readthedocs.io](https://urllib3.readthedocs.io).
+
+
+## Community
+
+urllib3 has a [community Discord channel](https://discord.gg/urllib3) for asking questions and
+collaborating with other contributors. Drop by and say hello 👋
+
+
+## Contributing
+
+urllib3 happily accepts contributions. Please see our
+[contributing documentation](https://urllib3.readthedocs.io/en/latest/contributing.html)
+for some tips on getting started.
+
+
+## Security Disclosures
+
+To report a security vulnerability, please use the
+[Tidelift security contact](https://tidelift.com/security).
+Tidelift will coordinate the fix and disclosure with maintainers.
+
+
+## Maintainers
+
+- [@sethmlarson](https://github.com/sethmlarson) (Seth M. Larson)
+- [@pquentin](https://github.com/pquentin) (Quentin Pradet)
+- [@theacodes](https://github.com/theacodes) (Thea Flowers)
+- [@haikuginger](https://github.com/haikuginger) (Jess Shapiro)
+- [@lukasa](https://github.com/lukasa) (Cory Benfield)
+- [@sigmavirus24](https://github.com/sigmavirus24) (Ian Stapleton Cordasco)
+- [@shazow](https://github.com/shazow) (Andrey Petrov)
+
+👋
+
+
+## Sponsorship
+
+If your company benefits from this library, please consider [sponsoring its
+development](https://urllib3.readthedocs.io/en/latest/sponsors.html).
+
+
+## For Enterprise
+
+Professional support for urllib3 is available as part of the [Tidelift
+Subscription][1].  Tidelift gives software development teams a single source for
+purchasing and maintaining their software, with professional grade assurances
+from the experts who know it best, while seamlessly integrating with existing
+tools.
+
+[1]: https://tidelift.com/subscription/pkg/pypi-urllib3?utm_source=pypi-urllib3&utm_medium=referral&utm_campaign=readme