Repository: KnowledgeCaptureAndDiscovery/somef (language: python)

Issue:
Detect ontology files
Similar to what we do with Dockerfiles and notebooks, I would like somef to detect ontology files.

These are files that are ttl, jsonld, .owl, etc. and that have an ontology within. We can use `rdflib` to load them and extract title, prefix and URL (to get started).

We need this to support new functionality in SCC


Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 1ac1aa8d..1367a3c2 100644
--- a/README.md
+++ b/README.md
@@ -164,8 +164,7 @@ If you move any files produced by somef into `/out`, then you will be able to se
 
 ## Usage
 
-### Configure
-Before running SOMEF, you must configure it appropriately. Run
+Before running SOMEF for the first time, you must **configure** it appropriately (you only need to do this once). Run
 
 ```bash
 somef configure
@@ -199,6 +198,9 @@ Options:
   -h, --help  Show this message and exit.
 ```
 
+### Updating SOMEF
+If you update SOMEF to a newer version, we recommend you `configure` again the library (by running `somef configure`). The rationale is that different versions may rely on classifiers which may be stored in a different path. 
+
 ### Run SOMEF
 
 ```bash
@@ -251,9 +253,6 @@ somef describe -r https://github.com/dgarijo/Widoco/ -o test.json -t 0.8
 
 Try SOMEF in Binder with our sample notebook: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/KnowledgeCaptureAndDiscovery/somef/HEAD?filepath=notebook%2FSOMEF%20Usage%20Example.ipynb)
 
-### Add/Remove a Category:
-
-To run a classifier with an additional category or remove an existing one, a corresponding path entry in the config.json should be provided and the category type should be added/removed in the category variable in `cli.py`
 
 ## Contribute:
 
@@ -262,3 +261,7 @@ If you want to contribute with a pull request, please do so by submitting it to
 
 ## Next features:
 To see upcoming features, please have a look at our [open issues](https://github.com/KnowledgeCaptureAndDiscovery/somef/issues) and [milestones](https://github.com/KnowledgeCaptureAndDiscovery/somef/milestones)
+
+## Extending SOMEF categories:
+
+To run a classifier with an additional category or remove an existing one, a corresponding path entry in the config.json should be provided and the category type should be added/removed in the category variable in `cli.py`.
diff --git a/src/__init__.py b/src/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/somef/cli.py b/src/somef/cli.py
index 95a6a191..19a67ab7 100644
--- a/src/somef/cli.py
+++ b/src/somef/cli.py
@@ -1,62 +1,17 @@
-# creatJSON.py
-# parameters:
-## input file: either: url to github repository OR markdown documentation file path
-## output file: json with each excerpt marked with all four classification scores
-
 import argparse
-import base64
 import json
 import os
 import pickle
-import re
 import sys
-import tempfile
-import time
-import urllib
-from datetime import datetime
-
-import markdown
 import validators
-import zipfile
-from io import StringIO
 from os import path
 from pathlib import Path
-from urllib.parse import urlparse
-
-import requests
 from dateutil import parser as date_parser
-from markdown import Markdown
 
-from somef.data_to_graph import DataGraph
-from . import createExcerpts
+from .data_to_graph import DataGraph
 from . import header_analysis
 
-from . import parser_somef
-
-
-## Markdown to plain text conversion: begin ##
-# code snippet from https://stackoverflow.com/a/54923798
-def unmark_element(element, stream=None):
-    if stream is None:
-        stream = StringIO()
-    if element.text:
-        stream.write(element.text)
-    for sub in element:
-        unmark_element(sub, stream)
-    if element.tail:
-        stream.write(element.tail)
-    return stream.getvalue()
-
-
-# patching Markdown
-Markdown.output_formats["plain"] = unmark_element
-__md = Markdown(output_format="plain")
-__md.stripTopLevelTags = False
-
-
-def unmark(text):
-    return __md.convert(text)
-
+from . import parser_somef, regular_expressions, process_repository, markdown_utils, constants
 
 ## Markdown to plain text conversion: end ##
 
@@ -67,921 +22,6 @@ def restricted_float(x):
     return x
 
 
-categories = ['description', 'citation', 'installation', 'invocation']
-# keep_keys = ('description', 'name', 'owner', 'license', 'languages_url', 'forks_url')
-# instead of keep keys, we have this table
-# it says that we want the key "codeRepository", and that we'll get it from the path "html_url" within the result object
-github_crosswalk_table = {
-    "codeRepository": "html_url",
-    "languages_url": "languages_url",
-    "owner": ["owner", "login"],
-    "ownerType": ["owner", "type"],  # used to determine if owner is User or Organization
-    "dateCreated": "created_at",
-    "dateModified": "updated_at",
-    "license": "license",
-    "description": "description",
-    "name": "name",
-    "fullName": "full_name",
-    "issueTracker": "issues_url",
-    "forksUrl": "forks_url",
-    "stargazers_count": "stargazers_count",
-    "forks_count": "forks_count"
-}
-
-release_crosswalk_table = {
-    'tagName': 'tag_name',
-    'name': 'name',
-    'authorName': ['author', 'login'],
-    'authorType': ['author', 'type'],
-    'body': 'body',
-    'tarballUrl': 'tarball_url',
-    'zipballUrl': 'zipball_url',
-    'htmlUrl': 'html_url',
-    'url': 'url',
-    'dateCreated': 'created_at',
-    'datePublished': "published_at",
-}
-
-
-# the same as requests.get(args).json(), but protects against rate limiting
-def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, **kwargs):
-    rate_limited = True
-    response = {}
-    date = ""
-    while rate_limited:
-        response = requests.get(*args, **kwargs)
-        data = response
-        date = data.headers["date"]
-        rate_limit_remaining = data.headers["x-ratelimit-remaining"]
-        epochtime = int(data.headers["x-ratelimit-reset"])
-        date_reset = datetime.fromtimestamp(epochtime)
-        print("Remaining GitHub API requests: " + rate_limit_remaining + " ### Next rate limit reset at: " + str(date_reset))
-        response = response.json()
-        if 'message' in response and 'API rate limit exceeded' in response['message']:
-            rate_limited = True
-            print(f"rate limited. Backing off for {initial_backoff} seconds")
-            time.sleep(initial_backoff)
-
-            # increase the backoff for next time
-            initial_backoff *= backoff_rate
-        else:
-            rate_limited = False
-
-    return response, date
-
-
-# error when github url is wrong
-class GithubUrlError(Exception):
-    # print("The URL provided seems to be incorrect")
-    pass
-
-
-def load_repository_metadata(repository_url, header, ignore_github_metadata=False, readme_only=False):
-    """
-    Function uses the repository_url provided to load required information from github.
-    Information kept from the repository is written in keep_keys.
-    Parameters
-    ----------
-    repository_url
-    header
-
-    Returns
-    -------
-    Returns the readme text and required metadata
-    """
-    if repository_url.rfind("gitlab.com") > 0:
-        return load_repository_metadata_gitlab(repository_url, header, readme_only)
-
-    print(f"Loading Repository {repository_url} Information....")
-    ## load general response of the repository
-    if repository_url[-1] == '/':
-        repository_url = repository_url[:-1]
-    url = urlparse(repository_url)
-    if url.netloc != 'github.com':
-        print("Error: repository must come from github")
-        return " ", {}
-
-    path_components = url.path.split('/')
-
-    if len(path_components) < 3:
-        print("Repository link is not correct. \nThe correct format is https://github.com/{owner}/{repo_name}.")
-        return " ", {}
-
-    owner = path_components[1]
-    repo_name = path_components[2]
-
-    repo_api_base_url = f"https://api.github.com/repos/{owner}/{repo_name}"
-
-    repo_ref = None
-    ref_param = None
-
-    if len(path_components) >= 5:
-        if not path_components[3] == "tree":
-            print(
-                "Github link is not correct. \nThe correct format is https://github.com/{owner}/{repo_name}/tree/{ref}.")
-
-            return " ", {}
-
-        # we must join all after 4, as sometimes tags have "/" in them.
-        repo_ref = "/".join(path_components[4:])
-        ref_param = {"ref": repo_ref}
-
-    print(repo_api_base_url)
-
-    general_resp = {}
-    date = ""
-    if not ignore_github_metadata or readme_only:
-        general_resp, date = rate_limit_get(repo_api_base_url, headers=header)
-
-    if 'message' in general_resp:
-        if general_resp['message'] == "Not Found":
-            print("Error: Repository name is private or incorrect")
-        else:
-            message = general_resp['message']
-            print("Error: " + message)
-
-        raise GithubUrlError
-
-    if ignore_github_metadata:
-        repo_ref = 'master'
-    elif repo_ref is None:
-        repo_ref = general_resp['default_branch']
-
-    if readme_only:
-        repo_archive_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/{repo_ref}/README.md"
-        print(f"Downloading {repo_archive_url}")
-        repo_download = requests.get(repo_archive_url)
-        if repo_download.status_code == 404:
-            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
-            repo_archive_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/master/README.md"
-            print(f"Trying to download {repo_archive_url}")
-            repo_download = requests.get(repo_archive_url)
-
-        if repo_download.status_code != 200:
-            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
-        repo_zip = repo_download.content
-
-        text = repo_zip.decode('utf-8')
-        return text, {}
-
-    ## get only the fields that we want
-    def do_crosswalk(data, crosswalk_table):
-        def get_path(obj, path):
-            if isinstance(path, list) or isinstance(path, tuple):
-                if len(path) == 1:
-                    path = path[0]
-                else:
-                    return get_path(obj[path[0]], path[1:])
-
-            if obj is not None and path in obj:
-                return obj[path]
-            else:
-                return None
-
-        output = {}
-        for codemeta_key, path in crosswalk_table.items():
-            value = get_path(data, path)
-            if value is not None:
-                output[codemeta_key] = value
-            else:
-                print(f"Error: key {path} not present in github repository")
-        return output
-
-    filtered_resp = {}
-    if not ignore_github_metadata:
-        filtered_resp = do_crosswalk(general_resp, github_crosswalk_table)
-        if "issueTracker" in filtered_resp:
-            issueTracker = filtered_resp["issueTracker"]
-            issueTracker = issueTracker.replace("{/number}","")
-            filtered_resp["issueTracker"] = issueTracker
-
-    # add download URL
-    filtered_resp["downloadUrl"] = f"https://github.com/{owner}/{repo_name}/releases"
-
-    ## condense license information
-    license_info = {}
-    if 'license' in filtered_resp:
-        for k in ('name', 'url'):
-            if k in filtered_resp['license']:
-                license_info[k] = filtered_resp['license'][k]
-
-    ## If we didn't find it, look for the license
-    if 'url' not in license_info or license_info['url'] is None:
-
-        possible_license_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/{repo_ref}/LICENSE"
-        license_text_resp = requests.get(possible_license_url)
-
-        # todo: It's possible that this request will get rate limited. Figure out how to detect that.
-        if license_text_resp.status_code == 200:
-            # license_text = license_text_resp.text
-            license_info['url'] = possible_license_url
-
-    if len(license_info) > 0:
-        filtered_resp['license'] = license_info
-
-    topics_headers = header
-    # get keywords / topics
-    if 'topics' in general_resp.keys():
-        filtered_resp['topics'] = general_resp['topics']
-    #else:
-    #    topics_headers = header
-    #    topics_headers['accept'] = 'application/vnd.github.mercy-preview+json'
-    #    topics_resp, date = rate_limit_get(repo_api_base_url + "/topics",
-    #                                       headers=topics_headers)
-
-    #    if 'message' in topics_resp.keys():
-    #        print("Topics Error: " + topics_resp['message'])
-    #    elif topics_resp and 'names' in topics_resp.keys():
-    #        filtered_resp['topics'] = topics_resp['names']
-
-    # get social features: stargazers_count
-    stargazers_info = {}
-    if 'stargazers_count' in filtered_resp:
-        stargazers_info['count'] = filtered_resp['stargazers_count']
-        stargazers_info['date'] = date
-        del filtered_resp['stargazers_count']
-    if len(stargazers_info.keys()) > 0:
-        filtered_resp['stargazersCount'] = stargazers_info
-
-    # get social features: forks_count
-    forks_info = {}
-    if 'forks_count' in filtered_resp:
-        forks_info['count'] = filtered_resp['forks_count']
-        forks_info['date'] = date
-        del filtered_resp['forks_count']
-    if len(forks_info.keys()) > 0:
-        filtered_resp['forksCount'] = forks_info
-
-    ## get languages
-    if not ignore_github_metadata:
-        languages, date = rate_limit_get(filtered_resp['languages_url'], headers=header)
-        if "message" in languages:
-            print("Languages Error: " + languages["message"])
-        else:
-            filtered_resp['languages'] = list(languages.keys())
-
-        del filtered_resp['languages_url']
-
-    # get default README
-    #                                   headers=topics_headers,
-    #readme_info, date = rate_limit_get(repo_api_base_url + "/readme",
-    #                                   headers=topics_headers,
-    #                                   params=ref_param)
-    #if 'message' in readme_info.keys():
-    #    print("README Error: " + readme_info['message'])
-    #    text = ""
-    #else:
-    #    readme = base64.b64decode(readme_info['content']).decode("utf-8")
-    #    text = readme
-    #    filtered_resp['readmeUrl'] = readme_info['html_url']
-
-    # get full git repository
-    # todo: maybe it should be optional, as this could take some time?
-
-    text = ""
-    # create a temporary directory
-    with tempfile.TemporaryDirectory() as temp_dir:
-
-        # download the repo at the selected branch with the link
-        repo_archive_url = f"https://github.com/{owner}/{repo_name}/archive/{repo_ref}.zip"
-        print(f"Downloading {repo_archive_url}")
-        repo_download = requests.get(repo_archive_url)
-        if repo_download.status_code == 404:
-            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
-            repo_archive_url = f"https://github.com/{owner}/{repo_name}/archive/main.zip"
-            print(f"Trying to download {repo_archive_url}")
-            repo_download = requests.get(repo_archive_url)
-
-        if repo_download.status_code != 200:
-            sys.exit(f"Error: Archive request failed with HTTP {repo_download.status_code}")
-        repo_zip = repo_download.content
-
-        repo_zip_file = os.path.join(temp_dir, "repo.zip")
-        repo_extract_dir = os.path.join(temp_dir, "repo")
-
-        with open(repo_zip_file, "wb") as f:
-            f.write(repo_zip)
-
-        with zipfile.ZipFile(repo_zip_file, "r") as zip_ref:
-            zip_ref.extractall(repo_extract_dir)
-
-        repo_folders = os.listdir(repo_extract_dir)
-        assert (len(repo_folders) == 1)
-
-        repo_dir = os.path.join(repo_extract_dir, repo_folders[0])
-
-        notebooks = []
-        dockerfiles = []
-        docs = []
-        scriptfiles = []
-
-        for dirpath, dirnames, filenames in os.walk(repo_dir):
-            repo_relative_path = os.path.relpath(dirpath, repo_dir)
-            for filename in filenames:
-                if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
-                    dockerfiles.append(os.path.join(repo_relative_path, filename))
-                    #dockerfiles.append(repo_relative_path + "/" + filename)
-                if filename.lower().endswith(".ipynb"):
-                    notebooks.append(os.path.join(repo_relative_path, filename))
-                if "README" == filename.upper() or "README.MD" == filename.upper():
-                    if (repo_relative_path == "."):
-                        try:
-                            with open(os.path.join(dirpath, filename), "rb") as data_file:
-                                data_file_text = data_file.read()
-                                text = data_file_text.decode("utf-8")
-                                filtered_resp['readmeUrl'] = convert_to_raw_usercontent(filename, owner, repo_name,
-                                                                                        repo_ref)
-                        except:
-                            print("README Error: error while reading file content")
-
-                if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "rb") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["licenseText"] = unmark(file_text)
-                    except:
-                        # TO DO: try different encodings
-                        filtered_resp["licenseFile"] = convert_to_raw_usercontent(filename, owner, repo_name, repo_ref)
-
-                if "CODE_OF_CONDUCT" == filename.upper() or "CODE_OF_CONDUCT.MD" == filename.upper():
-                    filtered_resp["codeOfConduct"] = convert_to_raw_usercontent(filename, owner, repo_name, repo_ref)
-                if "CONTRIBUTING" == filename.upper() or "CONTRIBUTING.MD" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["contributingGuidelines"] = unmark(file_text)
-                    except:
-                        filtered_resp["contributingGuidelinesFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                                 repo_name,
-                                                                                                 repo_ref)
-                if "ACKNOWLEDGMENT" in filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["acknowledgments"] = unmark(file_text)
-                    except:
-                        filtered_resp["acknowledgmentsFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                          repo_name,
-                                                                                          repo_ref)
-                if "CONTRIBUTORS" == filename.upper() or "CONTRIBUTORS.MD" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["contributors"] = unmark(file_text)
-                    except:
-                        filtered_resp["contributorsFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                       repo_name,
-                                                                                       repo_ref)
-                if "CITATION" == filename.upper() or "CITATION.CFF" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["citation"] = unmark(file_text)
-                    except:
-                        filtered_resp["citationFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                   repo_name,
-                                                                                   repo_ref)
-                if filename.endswith(".sh"):
-                    # scriptfiles.append(os.path.join(repo_relative_path, filename))
-                    scriptfiles.append(repo_relative_path + "/" + filename)
-
-            for dirname in dirnames:
-                if dirname.lower() == "docs":
-                    if repo_relative_path == ".":
-                        docs_path = dirname
-                    else:
-                        if repo_relative_path.find("\\") >= 0:
-                            new_repo_relative_path = repo_relative_path.replace("\\", "/")
-                            docs_path = new_repo_relative_path + "/" + dirname
-                        else:
-                            docs_path = repo_relative_path + "/" + dirname
-
-                    names = os.listdir(os.path.join(repo_dir, docs_path))
-                    for name in names:
-                        if name.lower().endswith(".pdf") or name.lower().endswith(".md") or name.lower().endswith(".html") or name.lower().endswith(".htm"):
-                            docs.append(
-                                f"https://github.com/{owner}/{repo_name}/tree/{urllib.parse.quote(repo_ref)}/{docs_path}")
-                            break;
-                    # print(docs)
-
-        # print("NOTEBOOKS:")
-        # print(notebooks)
-
-        # print("DOCKERFILES:")
-        # print(dockerfiles)
-
-    if len(notebooks) > 0:
-        filtered_resp["hasExecutableNotebook"] = [convert_to_raw_usercontent(x, owner, repo_name, repo_ref) for x in
-                                                  notebooks]
-    if len(dockerfiles) > 0:
-        filtered_resp["hasBuildFile"] = [convert_to_raw_usercontent(x, owner, repo_name, repo_ref) for x in dockerfiles]
-    if len(docs) > 0:
-        filtered_resp["hasDocumentation"] = docs
-
-    if len(scriptfiles) > 0:
-        filtered_resp["hasScriptFile"] = [convert_to_raw_usercontent(x, owner, repo_name, repo_ref) for x in
-                                          scriptfiles]
-
-    # get releases
-    if not ignore_github_metadata:
-        releases_list, date = rate_limit_get(repo_api_base_url + "/releases",
-                                             headers=header)
-
-        if isinstance(releases_list, dict) and 'message' in releases_list.keys():
-            print("Releases Error: " + releases_list['message'])
-        else:
-            filtered_resp['releases'] = [do_crosswalk(release, release_crosswalk_table) for release in releases_list]
-
-    print("Repository Information Successfully Loaded. \n")
-    return text, filtered_resp
-
-
-def load_repository_metadata_gitlab(repository_url, header, readme_only=False):
-    """
-    Function uses the repository_url provided to load required information from github.
-    Information kept from the repository is written in keep_keys.
-    Parameters
-    ----------
-    repository_url
-    header
-
-    Returns
-    -------
-    Returns the readme text and required metadata
-    """
-    print(f"Loading Repository {repository_url} Information....")
-    ## load general response of the repository
-    if repository_url[-1] == '/':
-        repository_url = repository_url[:-1]
-    url = urlparse(repository_url)
-    if url.netloc != 'gitlab.com':
-        print("Error: repository must come from github")
-        return " ", {}
-
-    path_components = url.path.split('/')
-
-    if len(path_components) < 3:
-        print("Gitlab link is not correct. \nThe correct format is https://github.com/{owner}/{repo_name}.")
-        return " ", {}
-
-    owner = path_components[1]
-    repo_name = path_components[2]
-    if len(path_components) == 4:
-        repo_name = repo_name + '/' + path_components[3]
-
-    project_id = get_project_id(repository_url)
-    project_api_url = f"https://gitlab.com/api/v4/projects/{project_id}"
-    print(f"Downloading {project_api_url}")
-    details = requests.get(project_api_url)
-    project_details = details.json()
-    date = details.headers["date"]
-
-    repo_api_base_url = f"{repository_url}"
-
-    repo_ref = None
-    ref_param = None
-
-    if len(path_components) >= 5:
-        if not path_components[4] == "tree":
-            print(
-                "GitLab link is not correct. \nThe correct format is https://gitlab.com/{owner}/{repo_name}.")
-
-            return " ", {}
-
-        # we must join all after 4, as sometimes tags have "/" in them.
-        repo_ref = "/".join(path_components[5:])
-        ref_param = {"ref": repo_ref}
-
-    print(repo_api_base_url)
-    if 'defaultBranch' in project_details.keys():
-        general_resp = {'defaultBranch': project_details['defaultBranch']}
-    elif 'default_branch' in project_details.keys():
-        general_resp = {'defaultBranch': project_details['default_branch']}
-
-    if 'message' in general_resp:
-        if general_resp['message'] == "Not Found":
-            print("Error: repository name is incorrect")
-        else:
-            message = general_resp['message']
-            print("Error: " + message)
-
-        raise GithubUrlError
-
-    if repo_ref is None:
-        repo_ref = general_resp['defaultBranch']
-
-    if readme_only:
-        repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/raw/{repo_ref}/README.md"
-        print(f"Downloading {repo_archive_url}")
-        repo_download = requests.get(repo_archive_url)
-        if repo_download.status_code == 404:
-            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
-            repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/raw/master/README.md"
-            print(f"Trying to download {repo_archive_url}")
-            repo_download = requests.get(repo_archive_url)
-
-        if repo_download.status_code != 200:
-            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
-        repo_zip = repo_download.content
-        print(repo_zip)
-        text = repo_zip.decode('utf-8')
-        return text, {}
-
-    ## get only the fields that we want
-    def do_crosswalk(data, crosswalk_table):
-        def get_path(obj, path):
-            if isinstance(path, list) or isinstance(path, tuple):
-                if len(path) == 1:
-                    path = path[0]
-                else:
-                    return get_path(obj[path[0]], path[1:])
-
-            if obj is not None and path in obj:
-                return obj[path]
-            else:
-                return None
-
-        output = {}
-        for codemeta_key, path in crosswalk_table.items():
-            value = get_path(data, path)
-            if value is not None:
-                output[codemeta_key] = value
-            else:
-                print(f"Error: key {path} not present in gitlab repository")
-        return output
-
-    # filtered_resp = do_crosswalk(general_resp, github_crosswalk_table)
-    filtered_resp = {}
-    # add download URL
-    # filtered_resp["downloadUrl"] = f"https://github.com/{owner}/{repo_name}/releases"
-    filtered_resp["downloadUrl"] = f"https://gitlab.com/{owner}/{repo_name}/-/branches"
-
-    ## condense license information
-    license_info = {}
-    if 'license' in filtered_resp:
-        for k in ('name', 'url'):
-            if k in filtered_resp['license']:
-                license_info[k] = filtered_resp['license'][k]
-
-    ## If we didn't find it, look for the license
-    if 'url' not in license_info or license_info['url'] is None:
-
-        possible_license_url = f"{repository_url}/-/blob/master/LICENSE"
-        license_text_resp = requests.get(possible_license_url)
-
-        # todo: It's possible that this request will get rate limited. Figure out how to detect that.
-        if license_text_resp.status_code == 200:
-            # license_text = license_text_resp.text
-            license_info['url'] = possible_license_url
-
-    if license_info != '':
-        filtered_resp['license'] = license_info
-
-    # get keywords / topics
-    topics_headers = header
-    topics_headers['accept'] = 'application/vnd.github.mercy-preview+json'
-    # topics_resp, date = rate_limit_get(repo_api_base_url + "/topics",
-    #                                   headers=topics_headers)
-    topics_resp = {}
-
-    if 'message' in topics_resp.keys():
-        print("Topics Error: " + topics_resp['message'])
-    elif topics_resp and 'names' in topics_resp.keys():
-        filtered_resp['topics'] = topics_resp['names']
-
-    if project_details['topics'] is not None:
-        filtered_resp['topics'] = project_details['topics']
-
-    # get social features: stargazers_count
-    stargazers_info = {}
-    # if 'stargazers_count' in filtered_resp:
-    if project_details['star_count'] is not None:
-        stargazers_info['count'] = project_details['star_count']
-        stargazers_info['date'] = date
-        if 'stargazers_count' in filtered_resp.keys():
-            del filtered_resp['stargazers_count']
-    filtered_resp['stargazersCount'] = stargazers_info
-
-    # get social features: forks_count
-    forks_info = {}
-    # if 'forks_count' in filtered_resp:
-    if project_details['forks_count'] is not None:
-        forks_info['count'] = project_details['forks_count']
-        forks_info['date'] = date
-        if 'forks_count' in filtered_resp.keys():
-            del filtered_resp['forks_count']
-    filtered_resp['forksCount'] = forks_info
-
-    ## get languages
-    # languages, date = rate_limit_get(filtered_resp['languages_url'])
-    languages = {}
-    filtered_resp['languages_url'] = "languages_url"
-    if "message" in languages:
-        print("Languages Error: " + languages["message"])
-    else:
-        filtered_resp['languages'] = list(languages.keys())
-
-    del filtered_resp['languages_url']
-
-    # get default README
-    # repo_api_base_url https://api.github.com/dgarijo/Widoco/readme
-    # readme_info, date = rate_limit_get(repo_api_base_url + "/readme",
-    #                                   headers=topics_headers,
-    #                                   params=ref_param)
-    readme_info = {}
-    if 'message' in readme_info.keys():
-        print("README Error: " + readme_info['message'])
-        text = ""
-    elif 'content' in readme_info:
-        readme = base64.b64decode(readme_info['content']).decode("utf-8")
-        text = readme
-        filtered_resp['readmeUrl'] = readme_info['html_url']
-
-    if 'readme_url' in project_details:
-        text = get_readme_content(project_details['readme_url'])
-        filtered_resp['readmeUrl'] = project_details['readme_url']
-
-    # get full git repository
-    # todo: maybe it should be optional, as this could take some time?
-
-    # create a temporary directory
-    with tempfile.TemporaryDirectory() as temp_dir:
-
-        # download the repo at the selected branch with the link
-        # https://gitlab.com/unboundedsystems/adapt/-/archive/master/adapt-master.zip
-        repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/archive/{repo_ref}/{repo_name}-{repo_ref}.zip"
-        if len(path_components) == 4:
-            repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/archive/{repo_ref}/{path_components[3]}.zip"
-        print(f"Downloading {repo_archive_url}")
-        repo_download = requests.get(repo_archive_url)
-        repo_zip = repo_download.content
-
-        repo_zip_file = os.path.join(temp_dir, "repo.zip")
-        repo_extract_dir = os.path.join(temp_dir, "repo")
-
-        with open(repo_zip_file, "wb") as f:
-            f.write(repo_zip)
-
-        with zipfile.ZipFile(repo_zip_file, "r") as zip_ref:
-            zip_ref.extractall(repo_extract_dir)
-
-        repo_folders = os.listdir(repo_extract_dir)
-        assert (len(repo_folders) == 1)
-
-        repo_dir = os.path.join(repo_extract_dir, repo_folders[0])
-
-        notebooks = []
-        dockerfiles = []
-        docs = []
-        scriptfiles = []
-
-        for dirpath, dirnames, filenames in os.walk(repo_dir):
-            repo_relative_path = os.path.relpath(dirpath, repo_dir)
-            for filename in filenames:
-                if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
-                    # dockerfiles.append(os.path.join(repo_relative_path, filename))
-                    dockerfiles.append(repo_relative_path + "/" + filename)
-                if filename.lower().endswith(".ipynb"):
-                    notebooks.append(os.path.join(repo_relative_path, filename))
-                if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "rb") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["licenseText"] = unmark(file_text)
-                    except:
-                        # TO DO: try different encodings
-                        filtered_resp["licenseFile"] = convert_to_raw_usercontent(filename, owner, repo_name, repo_ref)
-
-                if "CODE_OF_CONDUCT" == filename.upper() or "CODE_OF_CONDUCT.MD" == filename.upper():
-                    filtered_resp["codeOfConduct"] = convert_to_raw_usercontent(filename, owner, repo_name, repo_ref)
-                if "CONTRIBUTING" == filename.upper() or "CONTRIBUTING.MD" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["contributingGuidelines"] = unmark(file_text)
-                    except:
-                        filtered_resp["contributingGuidelinesFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                                 repo_name,
-                                                                                                 repo_ref)
-                if "ACKNOWLEDGMENT" in filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["acknowledgments"] = unmark(file_text)
-                    except:
-                        filtered_resp["acknowledgmentsFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                          repo_name,
-                                                                                          repo_ref)
-                if "CONTRIBUTORS" == filename.upper() or "CONTRIBUTORS.MD" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["contributors"] = unmark(file_text)
-                    except:
-                        filtered_resp["contributorsFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                       repo_name,
-                                                                                       repo_ref)
-                if "CITATION" == filename.upper() or "CITATION.CFF" == filename.upper():
-                    try:
-                        with open(os.path.join(dirpath, filename), "r") as data_file:
-                            file_text = data_file.read()
-                            filtered_resp["citation"] = unmark(file_text)
-                    except:
-                        filtered_resp["citationFile"] = convert_to_raw_usercontent(filename, owner,
-                                                                                   repo_name,
-                                                                                   repo_ref)
-                if filename.endswith(".sh"):
-                    # scriptfiles.append(os.path.join(repo_relative_path, filename))
-                    scriptfiles.append(repo_relative_path + "/" + filename)
-
-            for dirname in dirnames:
-                if dirname.lower() == "docs":
-                    if repo_relative_path == ".":
-                        docs_path = dirname
-                    else:
-                        docs_path = repo_relative_path + "/" + dirname
-                    names = os.listdir(os.path.join(repo_dir, docs_path))
-                    for name in names:
-                        if name.lower().endswith(".pdf") or name.lower().endswith(".md") or name.lower().endswith(
-                                ".html") or name.lower().endswith(".htm"):
-                            docs.append(
-                                f"https://gitlab.com/{owner}/{repo_name}/-/tree/{urllib.parse.quote(repo_ref)}/{docs_path}")
-                            break;
-
-                    # print(docs)
-
-        # print("NOTEBOOKS:")
-        # print(notebooks)
-
-        # print("DOCKERFILES:")
-        # print(dockerfiles)
-
-    if len(notebooks) > 0:
-        filtered_resp["hasExecutableNotebook"] = [convert_to_raw_usercontent_gitlab(x, owner, repo_name, repo_ref) for x
-                                                  in
-                                                  notebooks]
-    if len(dockerfiles) > 0:
-        filtered_resp["hasBuildFile"] = [convert_to_raw_usercontent_gitlab(x, owner, repo_name, repo_ref) for x in
-                                         dockerfiles]
-    if len(docs) > 0:
-        filtered_resp["hasDocumentation"] = docs
-
-    if len(scriptfiles) > 0:
-        filtered_resp["hasScriptFile"] = [convert_to_raw_usercontent_gitlab(x, owner, repo_name, repo_ref) for x in
-                                          scriptfiles]
-
-    # get releases
-    # releases_list, date = rate_limit_get(repo_api_base_url + "/-/releases",
-    #                                     headers=header)
-
-    releases_list = {}
-    if isinstance(releases_list, dict) and 'message' in releases_list.keys():
-        print("Releases Error: " + releases_list['message'])
-    else:
-        filtered_resp['releases'] = [do_crosswalk(release, release_crosswalk_table) for release in releases_list]
-
-    print("Repository Information Successfully Loaded. \n")
-    return text, filtered_resp
-
-
-def get_project_id(repository_url):
-    print(f"Downloading {repository_url}")
-    response = requests.get(repository_url)
-    response_str = str(response.content.decode('utf-8'))
-    init = response_str.find('\"project_id\":')
-    project_id = "-1"
-    start = init + len("\"project_id\":")
-    if init >= 0:
-        end = 0
-        end_bracket = response_str.find("}", start)
-        comma = response_str.find(",", start)
-        if comma != -1 and comma < end_bracket:
-            end = comma
-        else:
-            end = end_bracket
-        if end >= 0:
-            project_id = response_str[start:end]
-    return project_id
-
-
-def load_local_repository_metadata(local_repo):
-    filtered_resp = {}
-    notebooks = []
-    dockerfiles = []
-    docs = []
-    scriptfiles = []
-    text = ""
-    repo_dir = os.path.abspath(local_repo)
-    for dirpath, dirnames, filenames in os.walk(repo_dir):
-        repo_relative_path = os.path.relpath(dirpath, repo_dir)
-        for filename in filenames:
-            if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
-                dockerfiles.append(os.path.join(repo_dir, repo_relative_path, filename))
-            if filename.lower().endswith(".ipynb"):
-                notebooks.append(os.path.join(repo_dir, repo_relative_path, filename))
-            if "README" == filename.upper() or "README.MD" == filename.upper():
-                if (repo_relative_path == "."):
-                    try:
-                        with open(os.path.join(dirpath, filename), "rb") as data_file:
-                            data_file_text = data_file.read()
-                            text = data_file_text.decode("utf-8")
-                    except:
-                        print("README Error: error while reading file content")
-            if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
-                try:
-                    with open(os.path.join(dirpath, filename), "rb") as data_file:
-                        file_text = data_file.read()
-                        filtered_resp["licenseText"] = unmark(file_text)
-                except:
-                    filtered_resp["licenseFile"] = os.path.join(repo_dir, repo_relative_path, filename)
-            if "CODE_OF_CONDUCT" == filename.upper() or "CODE_OF_CONDUCT.MD" == filename.upper():
-                filtered_resp["codeOfConduct"] = os.path.join(repo_dir, repo_relative_path, filename)
-            if "CONTRIBUTING" == filename.upper() or "CONTRIBUTING.MD" == filename.upper():
-                try:
-                    with open(os.path.join(dirpath, filename), "r") as data_file:
-                        file_text = data_file.read()
-                        filtered_resp["contributingGuidelines"] = unmark(file_text)
-                except:
-                    filtered_resp["contributingGuidelinesFile"] = os.path.join(repo_dir, repo_relative_path, filename)
-            if "ACKNOWLEDGMENT" in filename.upper():
-                try:
-                    with open(os.path.join(dirpath, filename), "r") as data_file:
-                        file_text = data_file.read()
-                        filtered_resp["acknowledgments"] = unmark(file_text)
-                except:
-                    filtered_resp["acknowledgmentsFile"] = os.path.join(repo_dir, repo_relative_path, filename)
-            if "CONTRIBUTORS" == filename.upper() or "CONTRIBUTORS.MD" == filename.upper():
-                try:
-                    with open(os.path.join(dirpath, filename), "r") as data_file:
-                        file_text = data_file.read()
-                        filtered_resp["contributors"] = unmark(file_text)
-                except:
-                    filtered_resp["contributorsFile"] = os.path.join(repo_dir, repo_relative_path, filename)
-            if "CITATION" == filename.upper() or "CITATION.CFF" == filename.upper():
-                try:
-                    with open(os.path.join(dirpath, filename), "r") as data_file:
-                        file_text = data_file.read()
-                        filtered_resp["citation"] = unmark(file_text)
-                except:
-                    filtered_resp["citationFile"] = os.path.join(repo_dir, repo_relative_path, filename)
-            if filename.endswith(".sh"):
-                scriptfiles.append(os.path.join(repo_dir, repo_relative_path, filename))
-
-        for dirname in dirnames:
-            if dirname.lower() == "docs":
-                if repo_relative_path == ".":
-                    docs_path = dirname
-                else:
-                    docs_path = os.path.join(repo_relative_path, dirname)
-
-                names = os.listdir(os.path.join(repo_dir,docs_path))
-                for name in names:
-                    if name.lower().endswith(".pdf") or name.lower().endswith(".md") or name.lower().endswith(
-                            ".html") or name.lower().endswith(".htm"):
-                        docs.append(os.path.join(repo_dir,docs_path))
-
-
-    if len(notebooks) > 0:
-        filtered_resp["hasExecutableNotebook"] = notebooks
-    if len(dockerfiles) > 0:
-        filtered_resp["hasBuildFile"] = dockerfiles
-    if len(docs) > 0:
-        filtered_resp["hasDocumentation"] = docs
-
-    if len(scriptfiles) > 0:
-        filtered_resp["hasScriptFile"] = scriptfiles
-
-    print("Local Repository Information Successfully Loaded. \n")
-    return text, filtered_resp
-
-
-
-def get_readme_content(readme_url):
-    readme_url = readme_url.replace("/blob/", "/raw/")
-    readme = requests.get(readme_url)
-    readme_text = readme.content.decode('utf-8')
-    return readme_text
-
-
-def convert_to_raw_usercontent(partial, owner, repo_name, repo_ref):
-    if partial.startswith("./"):
-        partial = partial.replace("./", "")
-    if partial.startswith(".\\"):
-        partial = partial.replace(".\\", "")
-    if partial.find("\\") >= 0:
-        partial = re.sub("\\\\", "/", partial)
-
-    return f"https://raw.githubusercontent.com/{owner}/{repo_name}/{repo_ref}/{urllib.parse.quote(partial)}"
-
-
-def convert_to_raw_usercontent_gitlab(partial, owner, repo_name, repo_ref):
-    if partial.startswith("./"):
-        partial = partial.replace("./", "")
-    if partial.startswith(".\\"):
-        partial = partial.replace(".\\", "")
-    return f"https://gitlab.com/{owner}/{repo_name}/-/blob/{repo_ref}/{urllib.parse.quote(partial)}"
-
-
 def remove_bibtex(string_list):
     """
         Function that takes the string list and removes all bibtex blocks of it
@@ -994,7 +34,7 @@ def remove_bibtex(string_list):
         The strings list without bibtex blocks
         """
     for x, element in enumerate(string_list):
-        bib_references = extract_bibtex(element)
+        bib_references = regular_expressions.extract_bibtex(element)
         if len(bib_references) > 0:
             top = element.find(bib_references[0])
             init = element.rfind("```", 0, top)
@@ -1005,56 +45,25 @@ def remove_bibtex(string_list):
     return string_list
 
 
-def remove_links_images(text):
-    # process images
-    images = re.findall(r"!\[(.*?)?\]\((.*?)?\)", text)
-    for image in images:
-        link_text = image[1]
-        pos = text.find(link_text)
-        if pos != -1:
-            init = text[:pos].rindex("![")
-            end = text[pos:].index(")")
-            image_text = text[init:pos+end+1]
-            text = text.replace(image_text,"")
-
-    #process links
-    links = re.findall(r"\[(.*?)?\]\(([^)]+)\)", text)
-    for link in links:
-        link_text = link[1]
-        pos = text.find(link_text)
-        if pos != -1:
-            init = text[:pos].rindex("[")
-            end = text[pos:].index(")")
-            link_text = text[init:pos + end + 1]
-            text = text.replace(link_text, "")
-
-    # remove blank spaces and \n
-    return text.strip()
-
-
-def extract_colab_links(text):
-    output = []
-    links = re.findall(r"\[(.*?)?\]\(([^)]+)\)", text)
-    for link in links:
-        link_url = link[1]
-        if link_url.startswith("https://colab.research.google.com/drive"):
-            output.append(link_url)
-
-    return output
-
-## Function takes readme text as input and divides it into excerpts
-## Returns the extracted excerpts
 def create_excerpts(string_list):
+    """
+    Function takes readme text as input and divides it into excerpts
+    Parameters
+    ----------
+    string_list: Markdown text passed as input
+    Returns
+    -------
+    Returns the extracted excerpts
+    """
     print("Splitting text into valid excerpts for classification")
     string_list = remove_bibtex(string_list)
     # divisions = createExcerpts.split_into_excerpts(string_list)
     divisions = parser_somef.extract_blocks_excerpts(string_list)
     print("Text Successfully split. \n")
-    # issue 337
     output = {}
     for division in divisions:
         original = division
-        division = remove_links_images(division)
+        division = regular_expressions.remove_links_images(division)
         if len(division) > 0:
             output[division] = original
     return output
@@ -1080,7 +89,7 @@ def run_classifiers(excerpts, file_paths):
         for key in excerpts.keys():
             text_to_classifier.append(key)
             text_to_results.append(excerpts[key])
-        for category in categories:
+        for category in constants.categories:
             if category not in file_paths.keys():
                 sys.exit("Error: Category " + category + " file path not present in config.json")
             file_name = file_paths[category]
@@ -1114,7 +123,7 @@ def remove_unimportant_excerpts(excerpt_element):
                          'originalHeader': ""}
     else:
         final_excerpt = {'excerpt': "", 'confidence': [], 'technique': 'Supervised classification'}
-    final_excerpt['excerpt'] += excerpt_info;
+    final_excerpt['excerpt'] += excerpt_info
     final_excerpt['confidence'].append(excerpt_confidence)
     if 'originalHeader' in excerpt_element:
         final_excerpt['originalHeader'] += excerpt_element['originalHeader']
@@ -1132,6 +141,7 @@ def is_in_excerpts_headers(text, set_excerpts):
 
     return False, None
 
+
 ## Function takes scores dictionary and a threshold as input
 ## Returns predictions containing excerpts with a confidence above the given threshold.
 def classify(scores, threshold, excerpts_headers, header_parents):
@@ -1218,427 +228,9 @@ def extract_categories_using_header(repo_data):
         return {}, [repo_data]
 
 
-def extract_bibtex(readme_text) -> object:
-    """
-    Function takes readme text as input (cleaned from markdown notation) and runs a regex expression on top of it.
-    Returns list of bibtex citations
-    """
-    regex = r'\@[a-zA-Z]+\{[.\n\S\s]+?[author|title][.\n\S\s]+?[author|title][.\n\S\s]+?\n\}'
-    citations = re.findall(regex, readme_text)
-    return citations
-
-
-def extract_dois(readme_text) -> object:
-    """
-    Function that takes the text of a readme file and searches if there are any DOIs badges.
-    Parameters
-    ----------
-    readme_text Text of the readme
-
-    Returns
-    -------
-    DOIs/identifiers associated with this software component
-    """
-    # regex = r'\[\!\[DOI\]([^\]]+)\]\(([^)]+)\)'
-    # regex = r'\[\!\[DOI\]\(.+\)\]\(([^)]+)\)'
-    regex = r'\[\!\[DOI\]([^\]]+)\]\(([^)]+)\)'
-    dois = re.findall(regex, readme_text)
-    print("Extraction of DOIS from readme completed.\n")
-    return dois
-
-
-def extract_binder_links(readme_text) -> object:
-    """
-    Function that does a regex to extract binder links used as reference in the readme.
-    There could be multiple binder links for one reprository
-    Parameters
-    ----------
-    readme_text
-
-    Returns
-    -------
-    Links with binder notebooks/scripts that are ready to be executed.
-    """
-    regex = r'\[\!\[Binder\]([^\]]+)\]\(([^)]+)\)'
-    binder_links = re.findall(regex, readme_text)
-    print("Extraction of Binder links from readme completed.\n")
-    # remove duplicates
-    collabs = extract_colab_links(readme_text)
-    binder_links += collabs
-    return list(dict.fromkeys(binder_links))
-
-
-def extract_title(unfiltered_text):
-    html_text = markdown.markdown(unfiltered_text)
-    splitted = html_text.split("\n")
-    index = 0
-    limit = len(splitted)
-    output = ""
-    regex = r'<[^<>]+>'
-    while index < limit:
-        line = splitted[index]
-        if line.startswith("<h"):
-            if line.startswith("<h1>"):
-                output = re.sub(regex, '', line)
-            break
-        index += 1
-    return output
-
-
-def extract_title_old(unfiltered_text):
-    """
-    Function to extract a title based on the first header in the readme file
-    Parameters
-    ----------
-    unfiltered_text unfiltered text of the title
-
-    Returns
-    -------
-    Full title of the repo (if found)
-    """
-    underline_header = re.findall('.+[\n]={3,}[\n]', unfiltered_text)
-    # header declared with ====
-    title = ""
-    if len(underline_header) != 0:
-        title = re.split('.+[=]+[\n]+', unfiltered_text)[0].strip()
-    else:
-        # The first occurrence is assumed to be the title.
-        title = re.findall(r'# .+', unfiltered_text)[0]
-        # Remove initial #
-        if title is not None and len(title) > 0:
-            title = title[1:].strip()
-    # Remove other markup (links, etc.)
-    if "[!" in title:
-        title = re.split('\[\!', title)[0].strip()
-    # title = re.sub(r'/\[\!([^\[\]]*)\]\((.*?)\)', '',title)
-    return title
-
-
-def extract_readthedocs(readme_text) -> object:
-    """
-    Function to extract readthedocs links from text
-    Parameters
-    ----------
-    unfiltered_text text readme
-
-    Returns
-    -------
-    Links to the readthedocs documentation
-    """
-    #regex = r'http[s]?://[\w]+.readthedocs.io/'
-    regex = r'http[s]?://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]+.readthedocs.io/'
-    readthedocs_links = re.findall(regex, readme_text)
-    print("Extraction of readthedocs links from readme completed.\n")
-    # remove duplicates (links like [readthedocs](readthedocs) are found twice
-    return list(dict.fromkeys(readthedocs_links))
-
-
-def extract_support_channels(readme_text):
-    """
-    Function to extract Gitter Chat, Reddit and Discord links from text
-    Parameters
-    ----------
-    readme_text text readme
-
-    Returns
-    -------
-    Link to the Gitter Chat
-    """
-    results = []
-
-    index_gitter_chat = readme_text.find("[![Gitter chat]")
-    if index_gitter_chat > 0:
-        init = readme_text.find(")](", index_gitter_chat)
-        end = readme_text.find(")", init + 3)
-        gitter_chat = readme_text[init + 3:end]
-        results.append(gitter_chat)
-
-    init = readme_text.find("(https://www.reddit.com/r/")
-    if init > 0:
-        end = readme_text.find(")", init)
-        repo_status = readme_text[init + 1:end]
-        results.append(repo_status)
-
-    init = readme_text.find("(https://discord.com/invite/")
-    if init > 0:
-        end = readme_text.find(")", init)
-        repo_status = readme_text[init + 1:end]
-        results.append(repo_status)
-
-    return results
-
-
-def extract_repo_status(unfiltered_text):
-    repo_status = ""
-    init = unfiltered_text.find("[![Project Status:")
-    if init > 0:
-        end = unfiltered_text.find("](", init)
-        repo_status = unfiltered_text[init + 3:end]
-        repo_status = repo_status.replace("Project Status: ", "")
-    return repo_status
-
-
-def extract_arxiv_links(unfiltered_text):
-    result_links = [m.start() for m in re.finditer('https://arxiv.org/', unfiltered_text)]
-    result_refs = [m.start() for m in re.finditer('arXiv:', unfiltered_text)]
-    results = []
-    for position in result_links:
-        end = unfiltered_text.find(')', position)
-        link = unfiltered_text[position:end]
-        results.append(link)
-    for position in result_refs:
-        end = unfiltered_text.find('}', position)
-        link = unfiltered_text[position:end]
-        results.append(link.replace('arXiv:', 'https://arxiv.org/abs/'))
-
-    return results
-
-
-def extract_wiki_links(unfiltered_text, repo_url):
-    links = re.findall(r"\[[^\]]*\]\((.*?)?\)", unfiltered_text)
-    output = []
-    ends = 0
-    for link in links:
-        if validators.url(link):
-            if link.endswith("/wiki"):
-                output.append(link)
-            else:
-                ends = unfiltered_text.find("("+link+")") - 1
-                if unfiltered_text[:ends].find("[") != -1:
-                    position = unfiltered_text[:ends].rindex("[") + 1
-                    if unfiltered_text[position:ends].find("wiki") >= 0:
-                        output.append(link)
-            unfiltered_text = unfiltered_text[ends + len(link):]
-
-    #to check if a wiki url is available in the repository
-    if repo_url != "" and repo_url != None and validators.url(repo_url):
-        if repo_url.endswith("/"):
-            repo_url = repo_url + "wiki"
-        else:
-            repo_url = repo_url + "/wiki"
-        wiki = requests.get(repo_url, allow_redirects=False)
-        if wiki.status_code == 200:
-            output.append(repo_url)
-
-    return list(dict.fromkeys(output))
-
-# TO DO: join with image detection in a single method
-def extract_logo(unfiltered_text, repo_url):
-    logo = ""
-    index_logo = unfiltered_text.lower().find("![logo]")
-    if index_logo >= 0:
-        init = unfiltered_text.find("(", index_logo)
-        end = unfiltered_text.find(")", init)
-        logo = unfiltered_text[init + 1:end]
-    else:
-        # This is problematic if alt label is used
-        # TO DO
-        result = [_.start() for _ in re.finditer("<img src=", unfiltered_text)]
-        if len(result) > 0:
-            for index_img in result:
-                init = unfiltered_text.find("\"", index_img)
-                end = unfiltered_text.find("\"", init + 1)
-                img = unfiltered_text[init + 1:end]
-                if img.find("logo") > 0:
-                    logo = img
-
-    if logo != "" and not logo.startswith("http") and repo_url is not None:
-        if repo_url.find("github.com") > 0:
-            if repo_url.find("/tree/") > 0:
-                repo_url = repo_url.replace("/tree/", "/")
-            else:
-                repo_url = repo_url + "/master/"
-            repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
-            if not repo_url.endswith("/"):
-                repo_url = repo_url + "/"
-            logo = repo_url + logo
-        else:
-            if not repo_url.endswith("/"):
-                repo_url = repo_url + "/"
-            logo = repo_url + logo
-    print(logo)
-    return logo
-
-
-def extract_images(unfiltered_text, repo_url):
-    logo = ""
-    has_logo = False
-    images = []
-    repo = False
-    repo_name = ""
-    if repo_url != None and repo_url != "":
-        url = urlparse(repo_url)
-        path_components = url.path.split('/')
-        repo_name = path_components[2]
-        repo = True
-
-    html_text = markdown.markdown(unfiltered_text)
-    img_md = re.findall(r"!\[[^\]]*\]\((.*?)?\)", html_text)
-    result = [_.start() for _ in re.finditer("<img ", html_text)]
-    for img in img_md:
-        if not has_logo and repo:
-            start = img.rindex("/")
-            if img.find(repo_name, start) > 0:
-                logo = rename_github_image(img, repo_url)
-                has_logo = True
-            elif get_alt_text_md(html_text, img) == repo_name or get_alt_text_md(html_text, img).upper() == "LOGO":
-                logo = rename_github_image(img, repo_url)
-                has_logo = True
-            else:
-                start = img.rindex("/")
-                if img.upper().find("LOGO", start) > 0:
-                    logo = rename_github_image(img, repo_url)
-                    has_logo = True
-                else:
-                    images.append(rename_github_image(img, repo_url))
-        else:
-            images.append(rename_github_image(img, repo_url))
-    for index_img in result:
-        init = html_text.find("src=\"", index_img)
-        end = html_text.find("\"", init + 5)
-        img = html_text[init + 5:end]
-        if not has_logo and repo:
-            start = img.rindex("/")
-            image_name = img[start:]
-            if image_name.find(repo_name) > 0 or image_name.upper().find("LOGO") > 0:
-                logo = rename_github_image(img, repo_url)
-                has_logo = True
-            elif get_alt_text_img(html_text, index_img) == repo_name or get_alt_text_img(html_text, index_img).upper() == "LOGO":
-                logo = rename_github_image(img, repo_url)
-                has_logo = True
-            else:
-                images.append(rename_github_image(img, repo_url))
-        else:
-            start = img.rindex("/")
-            if img.upper().find("LOGO", start) > 0:
-                logo = rename_github_image(img, repo_url)
-                has_logo = True
-            else:
-                images.append(rename_github_image(img, repo_url))
-
-    return logo, images
-
-
-def rename_github_image(img, repo_url):
-    if not img.startswith("http") and repo_url is not None and repo_url != "":
-        if repo_url.find("/tree/") > 0:
-            repo_url = repo_url.replace("/tree/", "/")
-        else:
-            repo_url = repo_url + "/master/"
-        repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
-        if not repo_url.endswith("/"):
-            repo_url = repo_url + "/"
-        img = repo_url + img
-    return img
-
-
-def get_alt_text_md(text, image):
-    stop = text.find(image) - 2
-    start = text[:stop].rindex("![") + 2
-    return text[start:stop]
-
-
-def get_alt_text_img(html_text, index):
-    end = html_text.find(">", index)
-    output = ""
-    if html_text.find("alt=", index, end) > 0:
-        texto = html_text[index:end]
-        init = texto.find("alt=\"") + 5
-        end = texto.index("\"", init)
-        output = texto[init:end]
-    return output
-
-
-def get_alt_text_html(text, image):
-    stop = text.find(image)
-    start = text[:stop].rindex("<img")
-    if text[start:stop].find("alt=") != -1:
-        start = text.find("alt=",start) + 4
-        stop = text.find("\"",start)-1
-        return text[start:stop]
-    return ""
-
-# TO DO: join with logo detection
-def extract_images_old(unfiltered_text, repo_url):
-    logo = ""
-    images = []
-    html_text = markdown.markdown(unfiltered_text)
-    # print(html_text)
-    result = [_.start() for _ in re.finditer("<img ", html_text)]
-    if len(result) > 0:
-        for index_img in result:
-            init = html_text.find("src=\"", index_img)
-            end = html_text.find("\"", init + 5)
-            img = html_text[init + 5:end]
-            if img.find("logo") < 0:
-                if not img.startswith("http") and repo_url is not None:
-                    if repo_url.find("/tree/") > 0:
-                        repo_url = repo_url.replace("/tree/", "/")
-                    else:
-                        repo_url = repo_url + "/master/"
-                    repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
-                    if not repo_url.endswith("/"):
-                        repo_url = repo_url + "/"
-                    img = repo_url + img
-                images.append(img)
-            else:
-                if not img.startswith("http") and repo_url is not None:
-                    if repo_url.find("/tree/") > 0:
-                        repo_url = repo_url.replace("/tree/", "/")
-                    else:
-                        repo_url = repo_url + "/master/"
-                    repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
-                    if not repo_url.endswith("/"):
-                        repo_url = repo_url + "/"
-                    img = repo_url + img
-                logo = img
-
-    if logo == "" and repo_url != "" and repo_url != None:
-        print(repo_url)
-        url = urlparse(repo_url)
-        path_components = url.path.split('/')
-        repo_name = path_components[2]
-
-        for image in images:
-            start = image.rindex("/")
-            if image.find(repo_name,start) > 0:
-                logo = image
-                images.remove(image)
-                break
-
-
-    return logo, images
-
-
-def extract_support(unfiltered_text):
-    results = []
-    init = unfiltered_text.find("(https://www.reddit.com/r/")
-    if init > 0:
-        end = unfiltered_text.find(")", init)
-        repo_status = unfiltered_text[init + 1:end]
-        results.append(repo_status)
-
-    init = unfiltered_text.find("(https://discord.com/invite/")
-    if init > 0:
-        end = unfiltered_text.find(")", init)
-        repo_status = unfiltered_text[init + 1:end]
-        results.append(repo_status)
-
-    return results
-
-def extract_package_distributions(unfiltered_text):
-    output = ""
-    index_package_distribution = unfiltered_text.find("[![PyPI]")
-    if index_package_distribution > 0:
-        init = unfiltered_text.find(")](", index_package_distribution)
-        end = unfiltered_text.find(")", init + 3)
-        package_distribution = unfiltered_text[init + 3:end]
-        output = requests.get(package_distribution).url
-
-    return output
-
 def merge(header_predictions, predictions, citations, citation_file_text, dois, binder_links, long_title,
-          readthedocs_links, repo_status, arxiv_links, logo, images, support_channels, package_distribution, wiki_links):
+          readthedocs_links, repo_status, arxiv_links, logo, images, support_channels, package_distribution,
+          wiki_links):
     """
     Function that takes the predictions using header information, classifier and bibtex/doi parser
     Parameters
@@ -1690,7 +282,7 @@ def merge(header_predictions, predictions, citations, citation_file_text, dois,
                                                  'technique': 'Regular expression'})
     if len(binder_links) != 0:
         predictions['executableExample'] = {'excerpt': binder_links, 'confidence': [1.0],
-                                     'technique': 'Regular expression'}
+                                            'technique': 'Regular expression'}
     if len(repo_status) != 0:
         predictions['repoStatus'] = {
             'excerpt': "https://www.repostatus.org/#" + repo_status[0:repo_status.find(" ")].lower(),
@@ -1725,7 +317,7 @@ def merge(header_predictions, predictions, citations, citation_file_text, dois,
 
     if len(package_distribution) != 0:
         predictions['packageDistribution'] = {'excerpt': package_distribution, 'confidence': [1.0],
-                                          'technique': 'Regular expression'}
+                                              'technique': 'Regular expression'}
 
     for i in range(len(readthedocs_links)):
         if 'documentation' not in predictions.keys():
@@ -1762,22 +354,20 @@ def format_output(git_data, repo_data, gitlab_url=False):
     -------
     json representation of the categories found in file
     """
-    text_technigue = 'GitHub API'
+    text_technique = 'GitHub API'
     if gitlab_url:
-        text_technigue = 'GitLab API'
+        text_technique = 'GitLab API'
     print("formatting output")
     file_exploration = ['hasExecutableNotebook', 'hasBuildFile', 'hasDocumentation', 'codeOfConduct',
                         'contributingGuidelines', 'licenseFile', 'licenseText', 'acknowledgments', 'contributors',
-                        'hasScriptFile']
+                        'hasScriptFile', 'ontologies']
     for i in git_data.keys():
-        # print(i)
-        # print(git_data[i])
         if i == 'description':
             if 'description' not in repo_data.keys():
                 repo_data['description'] = []
             if git_data[i] != "":
                 repo_data['description'].append(
-                    {'excerpt': git_data[i], 'confidence': [1.0], 'technique': text_technigue})
+                    {'excerpt': git_data[i], 'confidence': [1.0], 'technique': text_technique})
         else:
             if i in file_exploration:
                 if i == 'hasExecutableNotebook':
@@ -1793,16 +383,17 @@ def format_output(git_data, repo_data, gitlab_url=False):
                             docker_files.append(data)
                     repo_data[i] = []
                     if len(docker_files) > 0:
-                        repo_data[i].insert(0, {'excerpt': docker_files, 'confidence': [1.0], 'technique': 'File Exploration',
-                                    'format': 'Docker file'})
+                        repo_data[i].insert(0, {'excerpt': docker_files, 'confidence': [1.0],
+                                                'technique': 'File Exploration',
+                                                'format': 'Docker file'})
                     if len(docker_compose) > 0:
-                        repo_data[i].insert(0, {'excerpt': docker_compose, 'confidence': [1.0], 'technique': 'File Exploration',
-                                    'format': 'Docker compose file'})
-                    #TODO docker
+                        repo_data[i].insert(0, {'excerpt': docker_compose, 'confidence': [1.0],
+                                                'technique': 'File Exploration',
+                                                'format': 'Docker compose file'})
                 else:
                     repo_data[i] = {'excerpt': git_data[i], 'confidence': [1.0], 'technique': 'File Exploration'}
             elif git_data[i] != "" and git_data[i] != []:
-                repo_data[i] = {'excerpt': git_data[i], 'confidence': [1.0], 'technique': text_technigue}
+                repo_data[i] = {'excerpt': git_data[i], 'confidence': [1.0], 'technique': text_technique}
     # remove empty categories from json
     return remove_empty_elements(repo_data)
 
@@ -1996,7 +587,8 @@ def create_missing_fields_report(repo_data, out_path):
     save_json_output(out, export_path, False)
 
 
-def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, local_repo=None, ignore_github_metadata=False, readme_only=False):
+def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, local_repo=None,
+                 ignore_github_metadata=False, readme_only=False):
     credentials_file = Path(
         os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
     ).expanduser()
@@ -2012,18 +604,18 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc
     if repo_url is not None:
         assert (doc_src is None)
         try:
-            text, github_data = load_repository_metadata(repo_url, header, ignore_github_metadata, readme_only)
+            text, github_data = process_repository.load_github_repository_metadata(repo_url, header, ignore_github_metadata, readme_only)
             if text == "":
                 print("Warning: README document does not exist in the repository")
-        except GithubUrlError:
+        except process_repository.GithubUrlError:
             return None
     elif local_repo is not None:
-        assert (local_repo is not None)
+        # assert (local_repo is not None)
         try:
-            text, github_data = load_local_repository_metadata(local_repo)
+            text, github_data = process_repository.load_local_repository_metadata(local_repo)
             if text == "":
                 print("Warning: README document does not exist in the local repository")
-        except GithubUrlError:
+        except process_repository.GithubUrlError:
             return None
     else:
         assert (doc_src is not None)
@@ -2035,7 +627,7 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc
 
     unfiltered_text = text
     header_predictions, string_list = extract_categories_using_header(unfiltered_text)
-    text = unmark(text)
+    text = markdown_utils.unmark(text)
     excerpts = create_excerpts(string_list)
     if ignore_classifiers or unfiltered_text == '':
         predictions = {}
@@ -2045,22 +637,22 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc
         score_dict = run_classifiers(excerpts, file_paths)
         predictions = classify(score_dict, threshold, excerpts_headers, header_parents)
     if text != '':
-        citations = extract_bibtex(text)
+        citations = regular_expressions.extract_bibtex(text)
         citation_file_text = ""
         if 'citation' in github_data.keys():
             citation_file_text = github_data['citation']
             del github_data['citation']
-        dois = extract_dois(unfiltered_text)
-        binder_links = extract_binder_links(unfiltered_text)
-        title = extract_title(unfiltered_text)
-        readthedocs_links = extract_readthedocs(unfiltered_text)
-        repo_status = extract_repo_status(unfiltered_text)
-        arxiv_links = extract_arxiv_links(unfiltered_text)
-        wiki_links = extract_wiki_links(unfiltered_text,repo_url)
-        #logo = extract_logo(unfiltered_text, repo_url)
-        logo, images = extract_images(unfiltered_text, repo_url)
-        support_channels = extract_support_channels(unfiltered_text)
-        package_distribution = extract_package_distributions(unfiltered_text)
+        dois = regular_expressions.extract_dois(unfiltered_text)
+        binder_links = regular_expressions.extract_binder_links(unfiltered_text)
+        title = regular_expressions.extract_title(unfiltered_text)
+        readthedocs_links = regular_expressions.extract_readthedocs(unfiltered_text)
+        repo_status = regular_expressions.extract_repo_status(unfiltered_text)
+        arxiv_links = regular_expressions.extract_arxiv_links(unfiltered_text)
+        wiki_links = regular_expressions.extract_wiki_links(unfiltered_text, repo_url)
+        # logo = extract_logo(unfiltered_text, repo_url)
+        logo, images = regular_expressions.extract_images(unfiltered_text, repo_url)
+        support_channels = regular_expressions.extract_support_channels(unfiltered_text)
+        package_distribution = regular_expressions.extract_package_distributions(unfiltered_text)
     else:
         citations = []
         citation_file_text = ""
@@ -2077,7 +669,8 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc
         package_distribution = ""
 
     predictions = merge(header_predictions, predictions, citations, citation_file_text, dois, binder_links, title,
-                        readthedocs_links, repo_status, arxiv_links, logo, images, support_channels, package_distribution, wiki_links)
+                        readthedocs_links, repo_status, arxiv_links, logo, images, support_channels,
+                        package_distribution, wiki_links)
     gitlab_url = False
     if repo_url is not None:
         if repo_url.rfind("gitlab.com") > 0:
@@ -2137,7 +730,8 @@ def run_cli(*,
 
     else:
         if repo_url:
-            repo_data = cli_get_data(threshold, ignore_classifiers, repo_url=repo_url, ignore_github_metadata=ignore_github_metadata, readme_only=readme_only)
+            repo_data = cli_get_data(threshold, ignore_classifiers, repo_url=repo_url,
+                                     ignore_github_metadata=ignore_github_metadata, readme_only=readme_only)
         elif local_repo:
             repo_data = cli_get_data(threshold, ignore_classifiers, local_repo=local_repo)
         else:
diff --git a/src/somef/constants.py b/src/somef/constants.py
new file mode 100644
index 00000000..24360503
--- /dev/null
+++ b/src/somef/constants.py
@@ -0,0 +1,43 @@
+from enum import Enum
+
+categories = ['description', 'citation', 'installation', 'invocation']
+# keep_keys = ('description', 'name', 'owner', 'license', 'languages_url', 'forks_url')
+# instead of keep keys, we have this table
+# it says that we want the key "codeRepository", and that we'll get it from the path "html_url" within the result object
+
+github_crosswalk_table = {
+    "codeRepository": "html_url",
+    "languages_url": "languages_url",
+    "owner": ["owner", "login"],
+    "ownerType": ["owner", "type"],  # used to determine if owner is User or Organization
+    "dateCreated": "created_at",
+    "dateModified": "updated_at",
+    "license": "license",
+    "description": "description",
+    "name": "name",
+    "fullName": "full_name",
+    "issueTracker": "issues_url",
+    "forksUrl": "forks_url",
+    "stargazers_count": "stargazers_count",
+    "forks_count": "forks_count"
+}
+
+release_crosswalk_table = {
+    'tagName': 'tag_name',
+    'name': 'name',
+    'authorName': ['author', 'login'],
+    'authorType': ['author', 'type'],
+    'body': 'body',
+    'tarballUrl': 'tarball_url',
+    'zipballUrl': 'zipball_url',
+    'htmlUrl': 'html_url',
+    'url': 'url',
+    'dateCreated': 'created_at',
+    'datePublished': "published_at",
+}
+
+
+class RepositoryType(Enum):
+    GITHUB = 1
+    GITLAB = 2
+    LOCAL = 3
diff --git a/src/somef/createExcerpts.py b/src/somef/createExcerpts.py
index 2e70e13d..b012bf06 100644
--- a/src/somef/createExcerpts.py
+++ b/src/somef/createExcerpts.py
@@ -1,19 +1,9 @@
-import argparse
-import json
-import base64
-from urllib.parse import urlparse
-import sys
-import os
-from os import path
-import requests
 from markdown import Markdown
-from bs4 import BeautifulSoup
 from io import StringIO
-import pickle
-import pprint
-import pandas as pd
-import numpy as np
-import re
+from io import StringIO
+
+from markdown import Markdown
+
 
 ## Markdown to plain text conversion: begin ##
 # code snippet from https://stackoverflow.com/a/54923798
diff --git a/src/somef/extract_ontologies.py b/src/somef/extract_ontologies.py
new file mode 100644
index 00000000..81e754f1
--- /dev/null
+++ b/src/somef/extract_ontologies.py
@@ -0,0 +1,33 @@
+from rdflib import Graph
+from rdflib.plugins.sparql import prepareQuery
+
+
+def is_file_ontology(file_path):
+    """
+    Method that, given a file, returns its URI.
+    This method is in a separate file in case we want to extract additional metadata if required
+    Parameters
+    ----------
+    file_path path of the candidate ontology
+
+    Returns the URI of the target ontology (if there is one)
+    -------
+    """
+    # load in rdf lib
+    try:
+        g = Graph()
+        g.parse(file_path)
+        q1 = prepareQuery('''
+          SELECT ?onto
+          WHERE { 
+            ?onto a <http://www.w3.org/2002/07/owl#Ontology>. 
+          }
+          ''')
+        # TO DO: extract title, preferred ns.
+        # there should be only one ontology per file
+        for r in g.query(q1):
+            print("Found that %s is an ontology" % file_path)
+            return r.onto
+    except:
+        # If the candidate file could not be read, pass
+        pass
diff --git a/src/somef/header_analysis.py b/src/somef/header_analysis.py
index 2d21d37f..64c8ae60 100644
--- a/src/somef/header_analysis.py
+++ b/src/somef/header_analysis.py
@@ -1,15 +1,14 @@
 ## Main function: header_analysis(text)
 ## input file: readme files text data
 ## output file: json files with categories extracted using header analysis; other text data cannot be extracted
+import re
 import string
 
-import markdown
 import numpy as np
 import pandas as pd
-import re
 from textblob import Word
+
 from . import parser_somef
-import json
 
 pd.options.mode.chained_assignment = None  # default='warn'
 
@@ -79,6 +78,7 @@
            Word("report").synsets[6]]
 group.update({"support": support})
 
+
 def extract_bash_code(text):
     splitted = text.split("```")
     output = []
@@ -88,6 +88,7 @@ def extract_bash_code(text):
                 output.append(splitted[index])
     return output
 
+
 def extract_header_content(text):  # extract the header and content of text to dataframe
     print('Extracting headers and content.')
     header = []
diff --git a/src/somef/markdown_utils.py b/src/somef/markdown_utils.py
new file mode 100644
index 00000000..14c17bd2
--- /dev/null
+++ b/src/somef/markdown_utils.py
@@ -0,0 +1,25 @@
+from io import StringIO
+from markdown import Markdown
+
+## Markdown to plain text conversion: begin ##
+# code snippet from https://stackoverflow.com/a/54923798
+def unmark_element(element, stream=None):
+    if stream is None:
+        stream = StringIO()
+    if element.text:
+        stream.write(element.text)
+    for sub in element:
+        unmark_element(sub, stream)
+    if element.tail:
+        stream.write(element.tail)
+    return stream.getvalue()
+
+
+# patching Markdown
+Markdown.output_formats["plain"] = unmark_element
+__md = Markdown(output_format="plain")
+__md.stripTopLevelTags = False
+
+
+def unmark(text):
+    return __md.convert(text)
\ No newline at end of file
diff --git a/src/somef/process_repository.py b/src/somef/process_repository.py
new file mode 100644
index 00000000..3cf32ace
--- /dev/null
+++ b/src/somef/process_repository.py
@@ -0,0 +1,1118 @@
+import base64
+import urllib
+import os
+import tempfile
+import re
+import zipfile
+import time
+import requests
+import sys
+
+from datetime import datetime
+from urllib.parse import urlparse
+from . import markdown_utils, extract_ontologies, constants
+
+
+# the same as requests.get(args).json(), but protects against rate limiting
+def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, **kwargs):
+    rate_limited = True
+    response = {}
+    date = ""
+    while rate_limited:
+        response = requests.get(*args, **kwargs)
+        data = response
+        date = data.headers["date"]
+        rate_limit_remaining = data.headers["x-ratelimit-remaining"]
+        epochtime = int(data.headers["x-ratelimit-reset"])
+        date_reset = datetime.fromtimestamp(epochtime)
+        print("Remaining GitHub API requests: " + rate_limit_remaining + " ### Next rate limit reset at: " + str(
+            date_reset))
+        response = response.json()
+        if 'message' in response and 'API rate limit exceeded' in response['message']:
+            rate_limited = True
+            print(f"rate limited. Backing off for {initial_backoff} seconds")
+            time.sleep(initial_backoff)
+            # increase the backoff for next time
+            initial_backoff *= backoff_rate
+        else:
+            rate_limited = False
+
+    return response, date
+
+
+def load_gitlab_repository_metadata(repository_url, header, readme_only=False):
+    """
+    Function uses the repository_url provided to load required information from gitlab.
+    Information kept from the repository is written in keep_keys.
+    Parameters
+    ----------
+    repository_url: URL of the Gitlab repository to analyze
+    header: headers of the repository
+    readme_only: flag to indicate whether to process the full repo or just the readme
+
+    Returns
+    -------
+    Readme text and required metadata
+    """
+    print(f"Loading Repository {repository_url} Information....")
+    # load general response of the repository
+    if repository_url[-1] == '/':
+        repository_url = repository_url[:-1]
+    url = urlparse(repository_url)
+    if url.netloc != 'gitlab.com':
+        print("Error: repository must come from github")
+        return " ", {}
+
+    path_components = url.path.split('/')
+
+    if len(path_components) < 3:
+        print("Gitlab link is not correct. \nThe correct format is https://github.com/{owner}/{repo_name}.")
+        return " ", {}
+
+    owner = path_components[1]
+    repo_name = path_components[2]
+    if len(path_components) == 4:
+        repo_name = repo_name + '/' + path_components[3]
+
+    project_id = get_project_id(repository_url)
+    project_api_url = f"https://gitlab.com/api/v4/projects/{project_id}"
+    print(f"Downloading {project_api_url}")
+    details = requests.get(project_api_url)
+    project_details = details.json()
+    date = details.headers["date"]
+
+    repo_api_base_url = f"{repository_url}"
+
+    repo_ref = None
+
+    if len(path_components) >= 5:
+        if not path_components[4] == "tree":
+            print(
+                "GitLab link is not correct. \nThe correct format is https://gitlab.com/{owner}/{repo_name}.")
+
+            return " ", {}
+
+        # we must join all after 4, as sometimes tags have "/" in them.
+        repo_ref = "/".join(path_components[5:])
+        ref_param = {"ref": repo_ref}
+
+    print(repo_api_base_url)
+    if 'defaultBranch' in project_details.keys():
+        general_resp = {'defaultBranch': project_details['defaultBranch']}
+    elif 'default_branch' in project_details.keys():
+        general_resp = {'defaultBranch': project_details['default_branch']}
+
+    if 'message' in general_resp:
+        if general_resp['message'] == "Not Found":
+            print("Error: repository name is incorrect")
+        else:
+            message = general_resp['message']
+            print("Error: " + message)
+
+        raise GithubUrlError
+
+    if repo_ref is None:
+        repo_ref = general_resp['defaultBranch']
+
+    if readme_only:
+        repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/raw/{repo_ref}/README.md"
+        print(f"Downloading {repo_archive_url}")
+        repo_download = requests.get(repo_archive_url)
+        if repo_download.status_code == 404:
+            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
+            repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/raw/master/README.md"
+            print(f"Trying to download {repo_archive_url}")
+            repo_download = requests.get(repo_archive_url)
+
+        if repo_download.status_code != 200:
+            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
+        repo_zip = repo_download.content
+        print(repo_zip)
+        text = repo_zip.decode('utf-8')
+        return text, {}
+
+    ## get only the fields that we want
+    def do_crosswalk(data, crosswalk_table):
+        def get_path(obj, path):
+            if isinstance(path, list) or isinstance(path, tuple):
+                if len(path) == 1:
+                    path = path[0]
+                else:
+                    return get_path(obj[path[0]], path[1:])
+
+            if obj is not None and path in obj:
+                return obj[path]
+            else:
+                return None
+
+        output = {}
+        for codemeta_key, path in crosswalk_table.items():
+            value = get_path(data, path)
+            if value is not None:
+                output[codemeta_key] = value
+            else:
+                print(f"Error: key {path} not present in gitlab repository")
+        return output
+
+    # filtered_resp = do_crosswalk(general_resp, github_crosswalk_table)
+    filtered_resp = {"downloadUrl": f"https://gitlab.com/{owner}/{repo_name}/-/branches"}
+
+    # condense license information
+    license_info = {}
+    if 'license' in filtered_resp:
+        for k in ('name', 'url'):
+            if k in filtered_resp['license']:
+                license_info[k] = filtered_resp['license'][k]
+
+    # If we didn't find it, look for the license
+    if 'url' not in license_info or license_info['url'] is None:
+
+        possible_license_url = f"{repository_url}/-/blob/master/LICENSE"
+        license_text_resp = requests.get(possible_license_url)
+
+        # todo: It's possible that this request will get rate limited. Figure out how to detect that.
+        if license_text_resp.status_code == 200:
+            # license_text = license_text_resp.text
+            license_info['url'] = possible_license_url
+
+    if license_info != '':
+        filtered_resp['license'] = license_info
+
+    # get keywords / topics
+    topics_headers = header
+    topics_headers['accept'] = 'application/vnd.github.mercy-preview+json'
+    # topics_resp, date = rate_limit_get(repo_api_base_url + "/topics",
+    #                                   headers=topics_headers)
+    topics_resp = {}
+
+    if 'message' in topics_resp.keys():
+        print("Topics Error: " + topics_resp['message'])
+    elif topics_resp and 'names' in topics_resp.keys():
+        filtered_resp['topics'] = topics_resp['names']
+
+    if project_details['topics'] is not None:
+        filtered_resp['topics'] = project_details['topics']
+
+    # get social features: stargazers_count
+    stargazers_info = {}
+    # if 'stargazers_count' in filtered_resp:
+    if project_details['star_count'] is not None:
+        stargazers_info['count'] = project_details['star_count']
+        stargazers_info['date'] = date
+        if 'stargazers_count' in filtered_resp.keys():
+            del filtered_resp['stargazers_count']
+    filtered_resp['stargazersCount'] = stargazers_info
+
+    # get social features: forks_count
+    forks_info = {}
+    # if 'forks_count' in filtered_resp:
+    if project_details['forks_count'] is not None:
+        forks_info['count'] = project_details['forks_count']
+        forks_info['date'] = date
+        if 'forks_count' in filtered_resp.keys():
+            del filtered_resp['forks_count']
+    filtered_resp['forksCount'] = forks_info
+
+    ## get languages
+    # languages, date = rate_limit_get(filtered_resp['languages_url'])
+    languages = {}
+    filtered_resp['languages_url'] = "languages_url"
+    if "message" in languages:
+        print("Languages Error: " + languages["message"])
+    else:
+        filtered_resp['languages'] = list(languages.keys())
+
+    del filtered_resp['languages_url']
+
+    # get default README
+    # repo_api_base_url https://api.github.com/dgarijo/Widoco/readme
+    # readme_info, date = rate_limit_get(repo_api_base_url + "/readme",
+    #                                   headers=topics_headers,
+    #                                   params=ref_param)
+    readme_info = {}
+    if 'message' in readme_info.keys():
+        print("README Error: " + readme_info['message'])
+        text = ""
+    elif 'content' in readme_info:
+        readme = base64.b64decode(readme_info['content']).decode("utf-8")
+        text = readme
+        filtered_resp['readmeUrl'] = readme_info['html_url']
+
+    if 'readme_url' in project_details:
+        text = get_readme_content(project_details['readme_url'])
+        filtered_resp['readmeUrl'] = project_details['readme_url']
+
+    # create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+
+        # download the repo at the selected branch with the link
+        # https://gitlab.com/unboundedsystems/adapt/-/archive/master/adapt-master.zip
+        repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/archive/{repo_ref}/{repo_name}-{repo_ref}.zip"
+        if len(path_components) == 4:
+            repo_archive_url = f"https://gitlab.com/{owner}/{repo_name}/-/archive/{repo_ref}/{path_components[3]}.zip"
+        print(f"Downloading {repo_archive_url}")
+        repo_download = requests.get(repo_archive_url)
+        repo_zip = repo_download.content
+
+        repo_zip_file = os.path.join(temp_dir, "repo.zip")
+        repo_extract_dir = os.path.join(temp_dir, "repo")
+
+        with open(repo_zip_file, "wb") as f:
+            f.write(repo_zip)
+
+        with zipfile.ZipFile(repo_zip_file, "r") as zip_ref:
+            zip_ref.extractall(repo_extract_dir)
+
+        repo_folders = os.listdir(repo_extract_dir)
+        assert (len(repo_folders) == 1)
+
+        repo_dir = os.path.join(repo_extract_dir, repo_folders[0])
+
+        text, filtered_resp = process_repository_files(repo_dir, filtered_resp, constants.RepositoryType.GITLAB,
+                                                       owner, repo_name, repo_ref)
+    #
+    #     notebooks = []
+    #     dockerfiles = []
+    #     docs = []
+    #     scriptfiles = []
+    #
+    #     for dirpath, dirnames, filenames in os.walk(repo_dir):
+    #         repo_relative_path = os.path.relpath(dirpath, repo_dir)
+    #         for filename in filenames:
+    #             if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
+    #                 # dockerfiles.append(os.path.join(repo_relative_path, filename))
+    #                 dockerfiles.append(repo_relative_path + "/" + filename)
+    #             if filename.lower().endswith(".ipynb"):
+    #                 notebooks.append(os.path.join(repo_relative_path, filename))
+    #             if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "rb") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["licenseText"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     # TO DO: try different encodings
+    #                     filtered_resp["licenseFile"] = convert_to_raw_user_content_gitlab(filename, owner, repo_name,
+    #                                                                                       repo_ref)
+    #
+    #             if "CODE_OF_CONDUCT" == filename.upper() or "CODE_OF_CONDUCT.MD" == filename.upper():
+    #                 filtered_resp["codeOfConduct"] = convert_to_raw_user_content_gitlab(filename, owner, repo_name,
+    #                                                                                     repo_ref)
+    #             if "CONTRIBUTING" == filename.upper() or "CONTRIBUTING.MD" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["contributingGuidelines"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["contributingGuidelinesFile"] = convert_to_raw_user_content_github(filename,
+    #                                                                                                      owner,
+    #                                                                                                      repo_name,
+    #                                                                                                      repo_ref)
+    #             if "ACKNOWLEDGMENT" in filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["acknowledgments"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["acknowledgmentsFile"] = convert_to_raw_user_content_gitlab(filename, owner,
+    #                                                                                               repo_name,
+    #                                                                                               repo_ref)
+    #             if "CONTRIBUTORS" == filename.upper() or "CONTRIBUTORS.MD" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["contributors"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["contributorsFile"] = convert_to_raw_user_content_gitlab(filename, owner,
+    #                                                                                            repo_name,
+    #                                                                                            repo_ref)
+    #             if "CITATION" == filename.upper() or "CITATION.CFF" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["citation"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["citationFile"] = convert_to_raw_user_content_gitlab(filename, owner,
+    #                                                                                        repo_name,
+    #                                                                                        repo_ref)
+    #             if filename.endswith(".sh"):
+    #                 # scriptfiles.append(os.path.join(repo_relative_path, filename))
+    #                 scriptfiles.append(repo_relative_path + "/" + filename)
+    #
+    #         for dirname in dirnames:
+    #             if dirname.lower() == "docs":
+    #                 if repo_relative_path == ".":
+    #                     docs_path = dirname
+    #                 else:
+    #                     docs_path = repo_relative_path + "/" + dirname
+    #                 names = os.listdir(os.path.join(repo_dir, docs_path))
+    #                 for name in names:
+    #                     if name.lower().endswith(".pdf") or name.lower().endswith(".md") or name.lower().endswith(
+    #                             ".html") or name.lower().endswith(".htm"):
+    #                         docs.append(
+    #                             f"https://gitlab.com/{owner}/{repo_name}/-/tree/{urllib.parse.quote(repo_ref)}/{docs_path}")
+    #                         break
+    #
+    # if len(notebooks) > 0:
+    #     filtered_resp["hasExecutableNotebook"] = [convert_to_raw_user_content_gitlab(x, owner, repo_name, repo_ref) for
+    #                                               x
+    #                                               in
+    #                                               notebooks]
+    # if len(dockerfiles) > 0:
+    #     filtered_resp["hasBuildFile"] = [convert_to_raw_user_content_gitlab(x, owner, repo_name, repo_ref) for x in
+    #                                      dockerfiles]
+    # if len(docs) > 0:
+    #     filtered_resp["hasDocumentation"] = docs
+    #
+    # if len(scriptfiles) > 0:
+    #     filtered_resp["hasScriptFile"] = [convert_to_raw_user_content_gitlab(x, owner, repo_name, repo_ref) for x in
+    #                                       scriptfiles]
+
+    # get releases
+    # releases_list, date = rate_limit_get(repo_api_base_url + "/-/releases",
+    #                                     headers=header)
+
+    releases_list = {}
+    if isinstance(releases_list, dict) and 'message' in releases_list.keys():
+        print("Releases Error: " + releases_list['message'])
+    else:
+        filtered_resp['releases'] = [do_crosswalk(release, constants.release_crosswalk_table) for release in
+                                     releases_list]
+
+    print("Repository Information Successfully Loaded. \n")
+    return text, filtered_resp
+
+
+def load_github_repository_metadata(repository_url, header, ignore_github_metadata=False, readme_only=False):
+    """
+    Function uses the repository_url provided to load required information from Github.
+    Information kept from the repository is written in keep_keys.
+    Parameters
+    ----------
+    repository_url
+    header
+
+    Returns
+    -------
+    Returns the readme text and required metadata
+    """
+    if repository_url.rfind("gitlab.com") > 0:
+        return load_gitlab_repository_metadata(repository_url, header, readme_only)
+
+    print(f"Loading Repository {repository_url} Information....")
+    ## load general response of the repository
+    if repository_url[-1] == '/':
+        repository_url = repository_url[:-1]
+    url = urlparse(repository_url)
+    if url.netloc != 'github.com':
+        print("Error: repository must come from github")
+        return " ", {}
+
+    path_components = url.path.split('/')
+
+    if len(path_components) < 3:
+        print("Repository link is not correct. \nThe correct format is https://github.com/{owner}/{repo_name}.")
+        return " ", {}
+
+    owner = path_components[1]
+    repo_name = path_components[2]
+
+    repo_api_base_url = f"https://api.github.com/repos/{owner}/{repo_name}"
+
+    repo_ref = None
+    ref_param = None
+
+    if len(path_components) >= 5:
+        if not path_components[3] == "tree":
+            print(
+                "Github link is not correct. \nThe correct format is https://github.com/{owner}/{repo_name}/tree/{ref}.")
+
+            return " ", {}
+
+        # we must join all after 4, as sometimes tags have "/" in them.
+        repo_ref = "/".join(path_components[4:])
+        ref_param = {"ref": repo_ref}
+
+    print(repo_api_base_url)
+
+    general_resp = {}
+    date = ""
+    if not ignore_github_metadata or readme_only:
+        general_resp, date = rate_limit_get(repo_api_base_url, headers=header)
+
+    if 'message' in general_resp:
+        if general_resp['message'] == "Not Found":
+            print("Error: Repository name is private or incorrect")
+        else:
+            message = general_resp['message']
+            print("Error: " + message)
+
+        raise GithubUrlError
+
+    if ignore_github_metadata:
+        repo_ref = 'master'
+    elif repo_ref is None:
+        repo_ref = general_resp['default_branch']
+
+    if readme_only:
+        repo_archive_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/{repo_ref}/README.md"
+        print(f"Downloading {repo_archive_url}")
+        repo_download = requests.get(repo_archive_url)
+        if repo_download.status_code == 404:
+            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
+            repo_archive_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/master/README.md"
+            print(f"Trying to download {repo_archive_url}")
+            repo_download = requests.get(repo_archive_url)
+
+        if repo_download.status_code != 200:
+            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
+        repo_zip = repo_download.content
+
+        text = repo_zip.decode('utf-8')
+        return text, {}
+
+    ## get only the fields that we want
+    def do_crosswalk(data, crosswalk_table):
+        def get_path(obj, path):
+            if isinstance(path, list) or isinstance(path, tuple):
+                if len(path) == 1:
+                    path = path[0]
+                else:
+                    return get_path(obj[path[0]], path[1:])
+
+            if obj is not None and path in obj:
+                return obj[path]
+            else:
+                return None
+
+        output = {}
+        for codemeta_key, path in crosswalk_table.items():
+            value = get_path(data, path)
+            if value is not None:
+                output[codemeta_key] = value
+            else:
+                print(f"Error: key {path} not present in github repository")
+        return output
+
+    filtered_resp = {}
+    if not ignore_github_metadata:
+        filtered_resp = do_crosswalk(general_resp, constants.github_crosswalk_table)
+        if "issueTracker" in filtered_resp:
+            issue_tracker = filtered_resp["issueTracker"]
+            issue_tracker = issue_tracker.replace("{/number}", "")
+            filtered_resp["issueTracker"] = issue_tracker
+
+    # add download URL
+    filtered_resp["downloadUrl"] = f"https://github.com/{owner}/{repo_name}/releases"
+
+    ## condense license information
+    license_info = {}
+    if 'license' in filtered_resp:
+        for k in ('name', 'url'):
+            if k in filtered_resp['license']:
+                license_info[k] = filtered_resp['license'][k]
+
+    ## If we didn't find it, look for the license
+    if 'url' not in license_info or license_info['url'] is None:
+
+        possible_license_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/{repo_ref}/LICENSE"
+        license_text_resp = requests.get(possible_license_url)
+
+        # todo: It's possible that this request will get rate limited. Figure out how to detect that.
+        if license_text_resp.status_code == 200:
+            # license_text = license_text_resp.text
+            license_info['url'] = possible_license_url
+
+    if len(license_info) > 0:
+        filtered_resp['license'] = license_info
+
+    topics_headers = header
+    # get keywords / topics
+    if 'topics' in general_resp.keys():
+        filtered_resp['topics'] = general_resp['topics']
+    # else:
+    #    topics_headers = header
+    #    topics_headers['accept'] = 'application/vnd.github.mercy-preview+json'
+    #    topics_resp, date = rate_limit_get(repo_api_base_url + "/topics",
+    #                                       headers=topics_headers)
+
+    #    if 'message' in topics_resp.keys():
+    #        print("Topics Error: " + topics_resp['message'])
+    #    elif topics_resp and 'names' in topics_resp.keys():
+    #        filtered_resp['topics'] = topics_resp['names']
+
+    # get social features: stargazers_count
+    stargazers_info = {}
+    if 'stargazers_count' in filtered_resp:
+        stargazers_info['count'] = filtered_resp['stargazers_count']
+        stargazers_info['date'] = date
+        del filtered_resp['stargazers_count']
+    if len(stargazers_info.keys()) > 0:
+        filtered_resp['stargazersCount'] = stargazers_info
+
+    # get social features: forks_count
+    forks_info = {}
+    if 'forks_count' in filtered_resp:
+        forks_info['count'] = filtered_resp['forks_count']
+        forks_info['date'] = date
+        del filtered_resp['forks_count']
+    if len(forks_info.keys()) > 0:
+        filtered_resp['forksCount'] = forks_info
+
+    ## get languages
+    if not ignore_github_metadata:
+        languages, date = rate_limit_get(filtered_resp['languages_url'], headers=header)
+        if "message" in languages:
+            print("Languages Error: " + languages["message"])
+        else:
+            filtered_resp['languages'] = list(languages.keys())
+
+        del filtered_resp['languages_url']
+
+    # get default README
+    #                                   headers=topics_headers,
+    # readme_info, date = rate_limit_get(repo_api_base_url + "/readme",
+    #                                   headers=topics_headers,
+    #                                   params=ref_param)
+    # if 'message' in readme_info.keys():
+    #    print("README Error: " + readme_info['message'])
+    #    text = ""
+    # else:
+    #    readme = base64.b64decode(readme_info['content']).decode("utf-8")
+    #    text = readme
+    #    filtered_resp['readmeUrl'] = readme_info['html_url']
+
+    # get full git repository
+    # todo: maybe it should be optional, as this could take some time?
+
+    text = ""
+    # create a temporary directory
+    with tempfile.TemporaryDirectory() as temp_dir:
+
+        # download the repo at the selected branch with the link
+        repo_archive_url = f"https://github.com/{owner}/{repo_name}/archive/{repo_ref}.zip"
+        print(f"Downloading {repo_archive_url}")
+        repo_download = requests.get(repo_archive_url)
+        if repo_download.status_code == 404:
+            print(f"Error: Archive request failed with HTTP {repo_download.status_code}")
+            repo_archive_url = f"https://github.com/{owner}/{repo_name}/archive/main.zip"
+            print(f"Trying to download {repo_archive_url}")
+            repo_download = requests.get(repo_archive_url)
+
+        if repo_download.status_code != 200:
+            sys.exit(f"Error: Archive request failed with HTTP {repo_download.status_code}")
+        repo_zip = repo_download.content
+
+        repo_zip_file = os.path.join(temp_dir, "repo.zip")
+        repo_extract_dir = os.path.join(temp_dir, "repo")
+
+        with open(repo_zip_file, "wb") as f:
+            f.write(repo_zip)
+
+        with zipfile.ZipFile(repo_zip_file, "r") as zip_ref:
+            zip_ref.extractall(repo_extract_dir)
+
+        repo_folders = os.listdir(repo_extract_dir)
+        assert (len(repo_folders) == 1)
+
+        repo_dir = os.path.join(repo_extract_dir, repo_folders[0])
+
+        text, filtered_resp = process_repository_files(repo_dir, filtered_resp, constants.RepositoryType.GITHUB,
+                                                 owner, repo_name, repo_ref)
+
+    #     notebooks = []
+    #     dockerfiles = []
+    #     docs = []
+    #     script_files = []
+    #     ontologies = []
+    #
+    #     for dirpath, dirnames, filenames in os.walk(repo_dir):
+    #         repo_relative_path = os.path.relpath(dirpath, repo_dir)
+    #         for filename in filenames:
+    #             if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
+    #                 dockerfiles.append(os.path.join(repo_relative_path, filename))
+    #                 # dockerfiles.append(repo_relative_path + "/" + filename)
+    #             if filename.lower().endswith(".ipynb"):
+    #                 notebooks.append(os.path.join(repo_relative_path, filename))
+    #             if "README" == filename.upper() or "README.MD" == filename.upper():
+    #                 if repo_relative_path == ".":
+    #                     try:
+    #                         with open(os.path.join(dirpath, filename), "rb") as data_file:
+    #                             data_file_text = data_file.read()
+    #                             text = data_file_text.decode("utf-8")
+    #                             filtered_resp['readmeUrl'] = convert_to_raw_user_content_github(filename, owner,
+    #                                                                                             repo_name,
+    #                                                                                             repo_ref)
+    #                     except:
+    #                         print("README Error: error while reading file content")
+    #
+    #             if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "rb") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["licenseText"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     # TO DO: try different encodings
+    #                     filtered_resp["licenseFile"] = convert_to_raw_user_content_github(filename, owner, repo_name,
+    #                                                                                       repo_ref)
+    #
+    #             if "CODE_OF_CONDUCT" == filename.upper() or "CODE_OF_CONDUCT.MD" == filename.upper():
+    #                 filtered_resp["codeOfConduct"] = convert_to_raw_user_content_github(filename, owner, repo_name,
+    #                                                                                     repo_ref)
+    #             if "CONTRIBUTING" == filename.upper() or "CONTRIBUTING.MD" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["contributingGuidelines"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["contributingGuidelinesFile"] = convert_to_raw_user_content_github(filename,
+    #                                                                                                      owner,
+    #                                                                                                      repo_name,
+    #                                                                                                      repo_ref)
+    #             if "ACKNOWLEDGMENT" in filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["acknowledgments"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["acknowledgmentsFile"] = convert_to_raw_user_content_github(filename, owner,
+    #                                                                                               repo_name,
+    #                                                                                               repo_ref)
+    #             if "CONTRIBUTORS" == filename.upper() or "CONTRIBUTORS.MD" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["contributors"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["contributorsFile"] = convert_to_raw_user_content_github(filename, owner,
+    #                                                                                            repo_name,
+    #                                                                                            repo_ref)
+    #             if "CITATION" == filename.upper() or "CITATION.CFF" == filename.upper():
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                         file_text = data_file.read()
+    #                         filtered_resp["citation"] = markdown_utils.unmark(file_text)
+    #                 except:
+    #                     filtered_resp["citationFile"] = convert_to_raw_user_content_github(filename, owner,
+    #                                                                                        repo_name,
+    #                                                                                        repo_ref)
+    #             if filename.endswith(".sh"):
+    #                 # scriptfiles.append(os.path.join(repo_relative_path, filename))
+    #                 script_files.append(repo_relative_path + "/" + filename)
+    #             if filename.endswith(".ttl") or filename.endswith(".owl") or filename.endswith(".nt") or filename. \
+    #                     endswith(".xml"):
+    #                 uri = extract_ontologies.is_file_ontology(filename)
+    #                 # Add onto
+    #                 if uri is not None:
+    #                     # and not any(o['uri'] == uri for o in ontologies): This checks if the onto is not already
+    #                     # there, but we return all ontologies we find right now. Filtering is up to users
+    #                     onto = {
+    #                         "uri": uri,
+    #                         "file_url": repo_relative_path + "/" + filename
+    #                     }
+    #                     ontologies.append(onto)
+    #
+    #         for dirname in dirnames:
+    #             if dirname.lower() == "docs":
+    #                 if repo_relative_path == ".":
+    #                     docs_path = dirname
+    #                 else:
+    #                     if repo_relative_path.find("\\") >= 0:
+    #                         new_repo_relative_path = repo_relative_path.replace("\\", "/")
+    #                         docs_path = new_repo_relative_path + "/" + dirname
+    #                     else:
+    #                         docs_path = repo_relative_path + "/" + dirname
+    #
+    #                 names = os.listdir(os.path.join(repo_dir, docs_path))
+    #                 for name in names:
+    #                     if name.lower().endswith(".pdf") or name.lower().endswith(".md") or name.lower().endswith(
+    #                             ".html") or name.lower().endswith(".htm"):
+    #                         docs.append(
+    #                             f"https://github.com/{owner}/{repo_name}/tree/{urllib.parse.quote(repo_ref)}/{docs_path}")
+    #                         break;
+    #
+    # if len(notebooks) > 0:
+    #     filtered_resp["hasExecutableNotebook"] = [convert_to_raw_user_content_github(x, owner, repo_name, repo_ref) for
+    #                                               x in
+    #                                               notebooks]
+    # if len(dockerfiles) > 0:
+    #     filtered_resp["hasBuildFile"] = [convert_to_raw_user_content_github(x, owner, repo_name, repo_ref) for x in
+    #                                      dockerfiles]
+    # if len(docs) > 0:
+    #     filtered_resp["hasDocumentation"] = docs
+    #
+    # if len(script_files) > 0:
+    #     filtered_resp["hasScriptFile"] = [convert_to_raw_user_content_github(x, owner, repo_name, repo_ref) for x in
+    #                                       script_files]
+    # if len(ontologies) > 0:
+    #     for onto in ontologies:
+    #         onto["file_url"] = convert_to_raw_user_content_github(onto["file_url"], owner, repo_name, repo_ref)
+    #     filtered_resp["ontologies"] = ontologies
+
+    # get releases
+    if not ignore_github_metadata:
+        releases_list, date = rate_limit_get(repo_api_base_url + "/releases",
+                                             headers=header)
+
+        if isinstance(releases_list, dict) and 'message' in releases_list.keys():
+            print("Releases Error: " + releases_list['message'])
+        else:
+            filtered_resp['releases'] = [do_crosswalk(release, constants.release_crosswalk_table) for release in
+                                         releases_list]
+
+    print("Repository information successfully loaded.\n")
+    return text, filtered_resp
+
+
+def load_local_repository_metadata(local_repo):
+    filtered_resp = {}
+    # notebooks = []
+    # dockerfiles = []
+    # docs = []
+    # script_files = []
+    text = ""
+    repo_dir = os.path.abspath(local_repo)
+    text, filtered_resp = process_repository_files(repo_dir, filtered_resp, constants.RepositoryType.LOCAL)
+    # for dirpath, dirnames, filenames in os.walk(repo_dir):
+    #     repo_relative_path = os.path.relpath(dirpath, repo_dir)
+    #     for filename in filenames:
+    #         if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
+    #             dockerfiles.append(os.path.join(repo_dir, repo_relative_path, filename))
+    #         if filename.lower().endswith(".ipynb"):
+    #             notebooks.append(os.path.join(repo_dir, repo_relative_path, filename))
+    #         if "README" == filename.upper() or "README.MD" == filename.upper():
+    #             if repo_relative_path == ".":
+    #                 try:
+    #                     with open(os.path.join(dirpath, filename), "rb") as data_file:
+    #                         data_file_text = data_file.read()
+    #                         text = data_file_text.decode("utf-8")
+    #                 except:
+    #                     print("README Error: error while reading file content")
+    #         if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
+    #             try:
+    #                 with open(os.path.join(dirpath, filename), "rb") as data_file:
+    #                     file_text = data_file.read()
+    #                     filtered_resp["licenseText"] = markdown_utils.unmark(file_text)
+    #             except:
+    #                 filtered_resp["licenseFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+    #         if "CODE_OF_CONDUCT" == filename.upper() or "CODE_OF_CONDUCT.MD" == filename.upper():
+    #             filtered_resp["codeOfConduct"] = os.path.join(repo_dir, repo_relative_path, filename)
+    #         if "CONTRIBUTING" == filename.upper() or "CONTRIBUTING.MD" == filename.upper():
+    #             try:
+    #                 with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                     file_text = data_file.read()
+    #                     filtered_resp["contributingGuidelines"] = markdown_utils.unmark(file_text)
+    #             except:
+    #                 filtered_resp["contributingGuidelinesFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+    #         if "ACKNOWLEDGMENT" in filename.upper():
+    #             try:
+    #                 with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                     file_text = data_file.read()
+    #                     filtered_resp["acknowledgments"] = markdown_utils.unmark(file_text)
+    #             except:
+    #                 filtered_resp["acknowledgmentsFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+    #         if "CONTRIBUTORS" == filename.upper() or "CONTRIBUTORS.MD" == filename.upper():
+    #             try:
+    #                 with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                     file_text = data_file.read()
+    #                     filtered_resp["contributors"] = markdown_utils.unmark(file_text)
+    #             except:
+    #                 filtered_resp["contributorsFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+    #         if "CITATION" == filename.upper() or "CITATION.CFF" == filename.upper():
+    #             try:
+    #                 with open(os.path.join(dirpath, filename), "r") as data_file:
+    #                     file_text = data_file.read()
+    #                     filtered_resp["citation"] = markdown_utils.unmark(file_text)
+    #             except:
+    #                 filtered_resp["citationFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+    #         if filename.endswith(".sh"):
+    #             script_files.append(os.path.join(repo_dir, repo_relative_path, filename))
+    #
+    #     for dirname in dirnames:
+    #         if dirname.lower() == "docs":
+    #             if repo_relative_path == ".":
+    #                 docs_path = dirname
+    #             else:
+    #                 docs_path = os.path.join(repo_relative_path, dirname)
+    #             names = os.listdir(os.path.join(repo_dir, docs_path))
+    #             for name in names:
+    #                 if name.lower().endswith(".pdf") or name.lower().endswith(".md") or name.lower().endswith(
+    #                         ".html") or name.lower().endswith(".htm"):
+    #                     docs.append(os.path.join(repo_dir, docs_path))
+    #
+    # if len(notebooks) > 0:
+    #     filtered_resp["hasExecutableNotebook"] = notebooks
+    # if len(dockerfiles) > 0:
+    #     filtered_resp["hasBuildFile"] = dockerfiles
+    # if len(docs) > 0:
+    #     filtered_resp["hasDocumentation"] = docs
+    # if len(script_files) > 0:
+    #     filtered_resp["hasScriptFile"] = script_files
+
+    print("Local Repository Information Successfully Loaded. \n")
+    return text, filtered_resp
+
+
+def get_project_id(repository_url):
+    print(f"Downloading {repository_url}")
+    response = requests.get(repository_url)
+    response_str = str(response.content.decode('utf-8'))
+    init = response_str.find('\"project_id\":')
+    project_id = "-1"
+    start = init + len("\"project_id\":")
+    if init >= 0:
+        end = 0
+        end_bracket = response_str.find("}", start)
+        comma = response_str.find(",", start)
+        if comma != -1 and comma < end_bracket:
+            end = comma
+        else:
+            end = end_bracket
+        if end >= 0:
+            project_id = response_str[start:end]
+    return project_id
+
+
+# error when github url is wrong
+class GithubUrlError(Exception):
+    # print("The URL provided seems to be incorrect")
+    pass
+
+
+def get_readme_content(readme_url):
+    readme_url = readme_url.replace("/blob/", "/raw/")
+    readme = requests.get(readme_url)
+    readme_text = readme.content.decode('utf-8')
+    return readme_text
+
+
+def convert_to_raw_user_content_github(partial, owner, repo_name, repo_ref):
+    if partial.startswith("./"):
+        partial = partial.replace("./", "")
+    if partial.startswith(".\\"):
+        partial = partial.replace(".\\", "")
+    if partial.find("\\") >= 0:
+        partial = re.sub("\\\\", "/", partial)
+
+    return f"https://raw.githubusercontent.com/{owner}/{repo_name}/{repo_ref}/{urllib.parse.quote(partial)}"
+
+
+def convert_to_raw_user_content_gitlab(partial, owner, repo_name, repo_ref):
+    if partial.startswith("./"):
+        partial = partial.replace("./", "")
+    if partial.startswith(".\\"):
+        partial = partial.replace(".\\", "")
+    return f"https://gitlab.com/{owner}/{repo_name}/-/blob/{repo_ref}/{urllib.parse.quote(partial)}"
+
+
+def process_repository_files(repo_dir, filtered_resp, repo_type, owner="", repo_name="", repo_ref=""):
+    """
+    Method that given a folder, it recognizes whether there are notebooks, dockerfiles, docs, script files or
+    ontologies.
+    Parameters
+    ----------
+    repo_dir: path to the dir to analyze
+    filtered_resp: JSON object to be completed by this method
+    repo_type: GITHUB, GITLAB or LOCAL
+    owner: owner of the repo (only for github/gitlab repos)
+    repo_name: repository name (only for github/gitlab repos)
+    repo_ref: branch (only for github/gitlab repos)
+
+    Returns
+    -------
+    A JSON object (filtered_resp) with the findings and the text of the readme
+    """
+    notebooks = []
+    dockerfiles = []
+    docs = []
+    script_files = []
+    ontologies = []
+    text = ""
+    for dir_path, dir_names, filenames in os.walk(repo_dir):
+        repo_relative_path = os.path.relpath(dir_path, repo_dir)
+        for filename in filenames:
+            file_path = os.path.join(repo_relative_path, filename)
+            if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
+                if repo_type == constants.RepositoryType.GITHUB:
+                    dockerfiles.append(convert_to_raw_user_content_github(file_path, owner, repo_name, repo_ref))
+                elif repo_type == constants.RepositoryType.GITLAB:
+                    dockerfiles.append(convert_to_raw_user_content_gitlab(file_path, owner, repo_name, repo_ref))
+                else:
+                    dockerfiles.append(os.path.join(repo_dir, file_path))
+
+            if filename.lower().endswith(".ipynb"):
+                if repo_type == constants.RepositoryType.GITHUB:
+                    notebooks.append(convert_to_raw_user_content_github(file_path, owner, repo_name, repo_ref))
+                elif repo_type == constants.RepositoryType.GITLAB:
+                    notebooks.append(convert_to_raw_user_content_gitlab(file_path, owner, repo_name, repo_ref))
+                else:
+                    notebooks.append(os.path.join(repo_dir, file_path))
+
+            if "README" == filename.upper() or "README.MD" == filename.upper():
+                if repo_relative_path == ".":
+                    try:
+                        with open(os.path.join(dir_path, filename), "rb") as data_file:
+                            data_file_text = data_file.read()
+                            text = data_file_text.decode("utf-8")
+                            if repo_type == constants.RepositoryType.GITHUB:
+                                filtered_resp['readmeUrl'] = convert_to_raw_user_content_github(filename, owner,
+                                                                                                repo_name,
+                                                                                                repo_ref)
+                    except:
+                        print("README Error: error while reading file content")
+            if "LICENSE" == filename.upper() or "LICENSE.MD" == filename.upper():
+                try:
+                    with open(os.path.join(dir_path, filename), "rb") as data_file:
+                        file_text = data_file.read()
+                        filtered_resp["licenseText"] = markdown_utils.unmark(file_text)
+                except:
+                    # TO DO: try different encodings
+                    if repo_type == constants.RepositoryType.GITHUB:
+                        filtered_resp["licenseFile"] = convert_to_raw_user_content_github(filename, owner, repo_name,
+                                                                                          repo_ref)
+                    elif repo_type == constants.RepositoryType.GITLAB:
+                        filtered_resp["licenseFile"] = convert_to_raw_user_content_gitlab(filename, owner, repo_name,
+                                                                                          repo_ref)
+                    else:
+                        filtered_resp["licenseFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+
+            if "CODE_OF_CONDUCT" == filename.upper() or "CODE_OF_CONDUCT.MD" == filename.upper():
+                if repo_type == constants.RepositoryType.GITHUB:
+                    filtered_resp["codeOfConduct"] = convert_to_raw_user_content_github(filename, owner, repo_name,
+                                                                                        repo_ref)
+                elif repo_type == constants.RepositoryType.GITLAB:
+                    filtered_resp["codeOfConduct"] = convert_to_raw_user_content_gitlab(filename, owner, repo_name,
+                                                                                        repo_ref)
+                else:
+                    filtered_resp["codeOfConduct"] = os.path.join(repo_dir, repo_relative_path, filename)
+
+            if "CONTRIBUTING" == filename.upper() or "CONTRIBUTING.MD" == filename.upper():
+                try:
+                    with open(os.path.join(dir_path, filename), "r") as data_file:
+                        file_text = data_file.read()
+                        filtered_resp["contributingGuidelines"] = markdown_utils.unmark(file_text)
+                except:
+                    if repo_type == constants.RepositoryType.GITHUB:
+                        filtered_resp["contributingGuidelinesFile"] = convert_to_raw_user_content_github(filename,
+                                                                                                         owner,
+                                                                                                         repo_name,
+                                                                                                         repo_ref)
+                    elif repo_type == constants.RepositoryType.GITLAB:
+                        filtered_resp["contributingGuidelinesFile"] = convert_to_raw_user_content_gitlab(filename,
+                                                                                                         owner,
+                                                                                                         repo_name,
+                                                                                                         repo_ref)
+                    else:
+                        filtered_resp["contributingGuidelinesFile"] = os.path.join(repo_dir, repo_relative_path,
+                                                                                   filename)
+            if "ACKNOWLEDGMENT" in filename.upper():
+                try:
+                    with open(os.path.join(dir_path, filename), "r") as data_file:
+                        file_text = data_file.read()
+                        filtered_resp["acknowledgments"] = markdown_utils.unmark(file_text)
+                except:
+                    if repo_type == constants.RepositoryType.GITHUB:
+                        filtered_resp["acknowledgmentsFile"] = convert_to_raw_user_content_github(filename, owner,
+                                                                                                  repo_name,
+                                                                                                  repo_ref)
+                    elif repo_type == constants.RepositoryType.GITLAB:
+                        filtered_resp["acknowledgmentsFile"] = convert_to_raw_user_content_gitlab(filename, owner,
+                                                                                                  repo_name,
+                                                                                                  repo_ref)
+                    else:
+                        filtered_resp["acknowledgmentsFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+            if "CONTRIBUTORS" == filename.upper() or "CONTRIBUTORS.MD" == filename.upper():
+                try:
+                    with open(os.path.join(dir_path, filename), "r") as data_file:
+                        file_text = data_file.read()
+                        filtered_resp["contributors"] = markdown_utils.unmark(file_text)
+                except:
+                    if repo_type == constants.RepositoryType.GITHUB:
+                        filtered_resp["contributorsFile"] = convert_to_raw_user_content_github(filename, owner,
+                                                                                               repo_name,
+                                                                                               repo_ref)
+                    elif repo_type == constants.RepositoryType.GITLAB:
+                        filtered_resp["contributorsFile"] = convert_to_raw_user_content_gitlab(filename, owner,
+                                                                                               repo_name,
+                                                                                               repo_ref)
+                    else:
+                        filtered_resp["contributorsFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+            if "CITATION" == filename.upper() or "CITATION.CFF" == filename.upper():
+                try:
+                    with open(os.path.join(dir_path, filename), "r") as data_file:
+                        file_text = data_file.read()
+                        filtered_resp["citation"] = markdown_utils.unmark(file_text)
+                except:
+                    if repo_type == constants.RepositoryType.GITHUB:
+                        filtered_resp["citationFile"] = convert_to_raw_user_content_github(filename, owner,
+                                                                                           repo_name,
+                                                                                           repo_ref)
+                    elif repo_type == constants.RepositoryType.GITLAB:
+                        filtered_resp["citationFile"] = convert_to_raw_user_content_gitlab(filename, owner,
+                                                                                           repo_name,
+                                                                                           repo_ref)
+                    else:
+                        filtered_resp["citationFile"] = os.path.join(repo_dir, repo_relative_path, filename)
+
+            if filename.endswith(".sh"):
+                if repo_type == constants.RepositoryType.GITHUB:
+                    script_files.append(convert_to_raw_user_content_github(file_path, owner, repo_name, repo_ref))
+                elif repo_type == constants.RepositoryType.GITLAB:
+                    script_files.append(convert_to_raw_user_content_gitlab(file_path, owner, repo_name, repo_ref))
+                else:
+                    script_files.append(os.path.join(repo_dir, file_path))
+
+            if filename.endswith(".ttl") or filename.endswith(".owl") or filename.endswith(".nt") or filename. \
+                    endswith(".xml"):
+                uri = extract_ontologies.is_file_ontology(os.path.join(repo_dir, file_path))
+                if uri is not None:
+                    # and not any(o['uri'] == uri for o in ontologies): This checks if the onto is not already
+                    # there, but we return all ontologies we find right now. Filtering is up to users
+                    file_url = ""
+                    if repo_type == constants.RepositoryType.GITHUB:
+                        file_url = convert_to_raw_user_content_github(file_path, owner, repo_name, repo_ref)
+                    elif repo_type == constants.RepositoryType.GITLAB:
+                        file_url = convert_to_raw_user_content_gitlab(file_path, owner, repo_name, repo_ref)
+                    else:
+                        file_url = os.path.join(repo_dir, file_path)
+                    onto = {
+                        "uri": uri,
+                        "file_url": file_url
+                    }
+                    ontologies.append(onto)
+
+        for dir_name in dir_names:
+            if dir_name.lower() == "docs":
+                if repo_relative_path == ".":
+                    docs_path = dir_name
+                else:
+                    if repo_relative_path.find("\\") >= 0:
+                        new_repo_relative_path = repo_relative_path.replace("\\", "/")
+                        docs_path = os.path.join(new_repo_relative_path, dir_name)
+                    else:
+                        docs_path = os.path.join(repo_relative_path, dir_name)
+
+                names = os.listdir(os.path.join(repo_dir, docs_path))
+                for name in names:
+                    if name.lower().endswith(".pdf") or name.lower().endswith(".md") or name.lower().endswith(
+                            ".html") or name.lower().endswith(".htm"):
+                        if repo_type == constants.RepositoryType.GITHUB:
+                            docs.append(
+                                f"https://github.com/{owner}/{repo_name}/tree/{urllib.parse.quote(repo_ref)}/{docs_path}")
+                        elif repo_type == constants.RepositoryType.GITLAB:
+                            docs.append(
+                                f"https://gitlab.com/{owner}/{repo_name}/-/tree/{urllib.parse.quote(repo_ref)}/{docs_path}")
+                        else:
+                            docs.append(os.path.join(repo_dir, docs_path))
+                        break
+
+    if len(notebooks) > 0:
+        filtered_resp["hasExecutableNotebook"] = notebooks
+    if len(dockerfiles) > 0:
+        filtered_resp["hasBuildFile"] = dockerfiles
+    if len(docs) > 0:
+        filtered_resp["hasDocumentation"] = docs
+    if len(script_files) > 0:
+        filtered_resp["hasScriptFile"] = script_files
+    if len(ontologies) > 0:
+        filtered_resp["ontologies"] = ontologies
+    return text, filtered_resp
diff --git a/src/somef/regular_expressions.py b/src/somef/regular_expressions.py
new file mode 100644
index 00000000..40f73537
--- /dev/null
+++ b/src/somef/regular_expressions.py
@@ -0,0 +1,476 @@
+import re
+import markdown
+import requests
+import validators
+from urllib.parse import urlparse
+
+
+def extract_title(unfiltered_text):
+    html_text = markdown.markdown(unfiltered_text)
+    splitted = html_text.split("\n")
+    index = 0
+    limit = len(splitted)
+    output = ""
+    regex = r'<[^<>]+>'
+    while index < limit:
+        line = splitted[index]
+        if line.startswith("<h"):
+            if line.startswith("<h1>"):
+                output = re.sub(regex, '', line)
+            break
+        index += 1
+    return output
+
+
+def extract_title_old(unfiltered_text):
+    """
+    Function to extract a title based on the first header in the readme file
+    Parameters
+    ----------
+    unfiltered_text unfiltered text of the title
+
+    Returns
+    -------
+    Full title of the repo (if found)
+    """
+    underline_header = re.findall('.+[\n]={3,}[\n]', unfiltered_text)
+    # header declared with ====
+    title = ""
+    if len(underline_header) != 0:
+        title = re.split('.+[=]+[\n]+', unfiltered_text)[0].strip()
+    else:
+        # The first occurrence is assumed to be the title.
+        title = re.findall(r'# .+', unfiltered_text)[0]
+        # Remove initial #
+        if title is not None and len(title) > 0:
+            title = title[1:].strip()
+    # Remove other markup (links, etc.)
+    if "[!" in title:
+        title = re.split('\[\!', title)[0].strip()
+    # title = re.sub(r'/\[\!([^\[\]]*)\]\((.*?)\)', '',title)
+    return title
+
+
+def extract_readthedocs(readme_text) -> object:
+    """
+    Function to extract readthedocs links from text
+    Parameters
+    ----------
+    unfiltered_text text readme
+
+    Returns
+    -------
+    Links to the readthedocs documentation
+    """
+    # regex = r'http[s]?://[\w]+.readthedocs.io/'
+    regex = r'http[s]?://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]+.readthedocs.io/'
+    readthedocs_links = re.findall(regex, readme_text)
+    print("Extraction of readthedocs links from readme completed.\n")
+    # remove duplicates (links like [readthedocs](readthedocs) are found twice
+    return list(dict.fromkeys(readthedocs_links))
+
+
+def extract_support_channels(readme_text):
+    """
+    Function to extract Gitter Chat, Reddit and Discord links from text
+    Parameters
+    ----------
+    readme_text text readme
+
+    Returns
+    -------
+    Link to the Gitter Chat
+    """
+    results = []
+
+    index_gitter_chat = readme_text.find("[![Gitter chat]")
+    if index_gitter_chat > 0:
+        init = readme_text.find(")](", index_gitter_chat)
+        end = readme_text.find(")", init + 3)
+        gitter_chat = readme_text[init + 3:end]
+        results.append(gitter_chat)
+
+    init = readme_text.find("(https://www.reddit.com/r/")
+    if init > 0:
+        end = readme_text.find(")", init)
+        repo_status = readme_text[init + 1:end]
+        results.append(repo_status)
+
+    init = readme_text.find("(https://discord.com/invite/")
+    if init > 0:
+        end = readme_text.find(")", init)
+        repo_status = readme_text[init + 1:end]
+        results.append(repo_status)
+
+    return results
+
+
+def extract_repo_status(unfiltered_text):
+    repo_status = ""
+    init = unfiltered_text.find("[![Project Status:")
+    if init > 0:
+        end = unfiltered_text.find("](", init)
+        repo_status = unfiltered_text[init + 3:end]
+        repo_status = repo_status.replace("Project Status: ", "")
+    return repo_status
+
+
+def extract_arxiv_links(unfiltered_text):
+    result_links = [m.start() for m in re.finditer('https://arxiv.org/', unfiltered_text)]
+    result_refs = [m.start() for m in re.finditer('arXiv:', unfiltered_text)]
+    results = []
+    for position in result_links:
+        end = unfiltered_text.find(')', position)
+        link = unfiltered_text[position:end]
+        results.append(link)
+    for position in result_refs:
+        end = unfiltered_text.find('}', position)
+        link = unfiltered_text[position:end]
+        results.append(link.replace('arXiv:', 'https://arxiv.org/abs/'))
+
+    return results
+
+
+def extract_wiki_links(unfiltered_text, repo_url):
+    links = re.findall(r"\[[^\]]*\]\((.*?)?\)", unfiltered_text)
+    output = []
+    ends = 0
+    for link in links:
+        if validators.url(link):
+            if link.endswith("/wiki"):
+                output.append(link)
+            else:
+                ends = unfiltered_text.find("(" + link + ")") - 1
+                if unfiltered_text[:ends].find("[") != -1:
+                    position = unfiltered_text[:ends].rindex("[") + 1
+                    if unfiltered_text[position:ends].find("wiki") >= 0:
+                        output.append(link)
+            unfiltered_text = unfiltered_text[ends + len(link):]
+
+    # to check if a wiki url is available in the repository
+    if repo_url != "" and repo_url is not None and validators.url(repo_url):
+        if repo_url.endswith("/"):
+            repo_url = repo_url + "wiki"
+        else:
+            repo_url = repo_url + "/wiki"
+        wiki = requests.get(repo_url, allow_redirects=False)
+        if wiki.status_code == 200:
+            output.append(repo_url)
+
+    return list(dict.fromkeys(output))
+
+
+# TO DO: join with image detection in a single method
+def extract_logo(unfiltered_text, repo_url):
+    logo = ""
+    index_logo = unfiltered_text.lower().find("![logo]")
+    if index_logo >= 0:
+        init = unfiltered_text.find("(", index_logo)
+        end = unfiltered_text.find(")", init)
+        logo = unfiltered_text[init + 1:end]
+    else:
+        # This is problematic if alt label is used
+        # TO DO
+        result = [_.start() for _ in re.finditer("<img src=", unfiltered_text)]
+        if len(result) > 0:
+            for index_img in result:
+                init = unfiltered_text.find("\"", index_img)
+                end = unfiltered_text.find("\"", init + 1)
+                img = unfiltered_text[init + 1:end]
+                if img.find("logo") > 0:
+                    logo = img
+
+    if logo != "" and not logo.startswith("http") and repo_url is not None:
+        if repo_url.find("github.com") > 0:
+            if repo_url.find("/tree/") > 0:
+                repo_url = repo_url.replace("/tree/", "/")
+            else:
+                repo_url = repo_url + "/master/"
+            repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
+            if not repo_url.endswith("/"):
+                repo_url = repo_url + "/"
+            logo = repo_url + logo
+        else:
+            if not repo_url.endswith("/"):
+                repo_url = repo_url + "/"
+            logo = repo_url + logo
+    print(logo)
+    return logo
+
+
+def extract_images(unfiltered_text, repo_url):
+    logo = ""
+    has_logo = False
+    images = []
+    repo = False
+    repo_name = ""
+    if repo_url != None and repo_url != "":
+        url = urlparse(repo_url)
+        path_components = url.path.split('/')
+        repo_name = path_components[2]
+        repo = True
+
+    html_text = markdown.markdown(unfiltered_text)
+    img_md = re.findall(r"!\[[^\]]*\]\((.*?)?\)", html_text)
+    result = [_.start() for _ in re.finditer("<img ", html_text)]
+    for img in img_md:
+        if not has_logo and repo:
+            start = img.rindex("/")
+            if img.find(repo_name, start) > 0:
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            elif get_alt_text_md(html_text, img) == repo_name or get_alt_text_md(html_text, img).upper() == "LOGO":
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            else:
+                start = img.rindex("/")
+                if img.upper().find("LOGO", start) > 0:
+                    logo = rename_github_image(img, repo_url)
+                    has_logo = True
+                else:
+                    images.append(rename_github_image(img, repo_url))
+        else:
+            images.append(rename_github_image(img, repo_url))
+    for index_img in result:
+        init = html_text.find("src=\"", index_img)
+        end = html_text.find("\"", init + 5)
+        img = html_text[init + 5:end]
+        if not has_logo and repo:
+            start = img.rindex("/")
+            image_name = img[start:]
+            if image_name.find(repo_name) > 0 or image_name.upper().find("LOGO") > 0:
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            elif get_alt_text_img(html_text, index_img) == repo_name or get_alt_text_img(html_text,
+                                                                                         index_img).upper() == "LOGO":
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            else:
+                images.append(rename_github_image(img, repo_url))
+        else:
+            start = img.rindex("/")
+            if img.upper().find("LOGO", start) > 0:
+                logo = rename_github_image(img, repo_url)
+                has_logo = True
+            else:
+                images.append(rename_github_image(img, repo_url))
+
+    return logo, images
+
+
+# TO DO: join with logo detection
+def extract_images_old(unfiltered_text, repo_url):
+    logo = ""
+    images = []
+    html_text = markdown.markdown(unfiltered_text)
+    # print(html_text)
+    result = [_.start() for _ in re.finditer("<img ", html_text)]
+    if len(result) > 0:
+        for index_img in result:
+            init = html_text.find("src=\"", index_img)
+            end = html_text.find("\"", init + 5)
+            img = html_text[init + 5:end]
+            if img.find("logo") < 0:
+                if not img.startswith("http") and repo_url is not None:
+                    if repo_url.find("/tree/") > 0:
+                        repo_url = repo_url.replace("/tree/", "/")
+                    else:
+                        repo_url = repo_url + "/master/"
+                    repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
+                    if not repo_url.endswith("/"):
+                        repo_url = repo_url + "/"
+                    img = repo_url + img
+                images.append(img)
+            else:
+                if not img.startswith("http") and repo_url is not None:
+                    if repo_url.find("/tree/") > 0:
+                        repo_url = repo_url.replace("/tree/", "/")
+                    else:
+                        repo_url = repo_url + "/master/"
+                    repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
+                    if not repo_url.endswith("/"):
+                        repo_url = repo_url + "/"
+                    img = repo_url + img
+                logo = img
+
+    if logo == "" and repo_url != "" and repo_url != None:
+        print(repo_url)
+        url = urlparse(repo_url)
+        path_components = url.path.split('/')
+        repo_name = path_components[2]
+
+        for image in images:
+            start = image.rindex("/")
+            if image.find(repo_name, start) > 0:
+                logo = image
+                images.remove(image)
+                break
+
+    return logo, images
+
+
+def extract_support(unfiltered_text):
+    results = []
+    init = unfiltered_text.find("(https://www.reddit.com/r/")
+    if init > 0:
+        end = unfiltered_text.find(")", init)
+        repo_status = unfiltered_text[init + 1:end]
+        results.append(repo_status)
+
+    init = unfiltered_text.find("(https://discord.com/invite/")
+    if init > 0:
+        end = unfiltered_text.find(")", init)
+        repo_status = unfiltered_text[init + 1:end]
+        results.append(repo_status)
+
+    return results
+
+
+def extract_package_distributions(unfiltered_text):
+    output = ""
+    index_package_distribution = unfiltered_text.find("[![PyPI]")
+    if index_package_distribution > 0:
+        init = unfiltered_text.find(")](", index_package_distribution)
+        end = unfiltered_text.find(")", init + 3)
+        package_distribution = unfiltered_text[init + 3:end]
+        output = requests.get(package_distribution).url
+
+    return output
+
+
+def extract_colab_links(text):
+    """
+    Method designed to find colab notebooks in readmes
+    Parameters
+    ----------
+    text: markdown text where the links to colab will be searched.
+
+    Returns
+    -------
+    A list of colab links found in the text passed as a parameter.
+    """
+    output = []
+    links = re.findall(r"\[(.*?)?\]\(([^)]+)\)", text)
+    for link in links:
+        link_url = link[1]
+        if link_url.startswith("https://colab.research.google.com/drive"):
+            output.append(link_url)
+    return output
+
+
+def remove_links_images(text):
+    # process images
+    images = re.findall(r"!\[(.*?)?\]\((.*?)?\)", text)
+    for image in images:
+        link_text = image[1]
+        pos = text.find(link_text)
+        if pos != -1:
+            init = text[:pos].rindex("![")
+            end = text[pos:].index(")")
+            image_text = text[init:pos + end + 1]
+            text = text.replace(image_text, "")
+
+    # process links
+    links = re.findall(r"\[(.*?)?\]\(([^)]+)\)", text)
+    for link in links:
+        link_text = link[1]
+        pos = text.find(link_text)
+        if pos != -1:
+            init = text[:pos].rindex("[")
+            end = text[pos:].index(")")
+            link_text = text[init:pos + end + 1]
+            text = text.replace(link_text, "")
+
+    # remove blank spaces and \n
+    return text.strip()
+
+
+def extract_bibtex(readme_text) -> object:
+    """
+    Function takes readme text as input (cleaned from markdown notation) and runs a regex expression on top of it.
+    Returns list of bibtex citations
+    """
+    regex = r'\@[a-zA-Z]+\{[.\n\S\s]+?[author|title][.\n\S\s]+?[author|title][.\n\S\s]+?\n\}'
+    citations = re.findall(regex, readme_text)
+    return citations
+
+
+def extract_dois(readme_text) -> object:
+    """
+    Function that takes the text of a readme file and searches if there are any DOIs badges.
+    Parameters
+    ----------
+    readme_text Text of the readme
+
+    Returns
+    -------
+    DOIs/identifiers associated with this software component
+    """
+    # regex = r'\[\!\[DOI\]([^\]]+)\]\(([^)]+)\)'
+    # regex = r'\[\!\[DOI\]\(.+\)\]\(([^)]+)\)'
+    regex = r'\[\!\[DOI\]([^\]]+)\]\(([^)]+)\)'
+    dois = re.findall(regex, readme_text)
+    print("Extraction of DOIS from readme completed.\n")
+    return dois
+
+
+def extract_binder_links(readme_text) -> object:
+    """
+    Function that does a regex to extract binder links used as reference in the readme.
+    There could be multiple binder links for one reprository
+    Parameters
+    ----------
+    readme_text
+
+    Returns
+    -------
+    Links with binder notebooks/scripts that are ready to be executed.
+    """
+    regex = r'\[\!\[Binder\]([^\]]+)\]\(([^)]+)\)'
+    binder_links = re.findall(regex, readme_text)
+    print("Extraction of Binder links from readme completed.\n")
+    # remove duplicates
+    collabs = extract_colab_links(readme_text)
+    binder_links += collabs
+    return list(dict.fromkeys(binder_links))
+
+
+
+def rename_github_image(img, repo_url):
+    if not img.startswith("http") and repo_url is not None and repo_url != "":
+        if repo_url.find("/tree/") > 0:
+            repo_url = repo_url.replace("/tree/", "/")
+        else:
+            repo_url = repo_url + "/master/"
+        repo_url = repo_url.replace("github.com", "raw.githubusercontent.com")
+        if not repo_url.endswith("/"):
+            repo_url = repo_url + "/"
+        img = repo_url + img
+    return img
+
+
+def get_alt_text_md(text, image):
+    stop = text.find(image) - 2
+    start = text[:stop].rindex("![") + 2
+    return text[start:stop]
+
+
+def get_alt_text_img(html_text, index):
+    end = html_text.find(">", index)
+    output = ""
+    if html_text.find("alt=", index, end) > 0:
+        texto = html_text[index:end]
+        init = texto.find("alt=\"") + 5
+        end = texto.index("\"", init)
+        output = texto[init:end]
+    return output
+
+
+def get_alt_text_html(text, image):
+    stop = text.find(image)
+    start = text[:stop].rindex("<img")
+    if text[start:stop].find("alt=") != -1:
+        start = text.find("alt=", start) + 4
+        stop = text.find("\"", start) - 1
+        return text[start:stop]
+    return ""


Test patch:
diff --git a/src/somef/test/test_cli.py b/src/somef/test/test_cli.py
index 8e29444c..ca77e30e 100644
--- a/src/somef/test/test_cli.py
+++ b/src/somef/test/test_cli.py
@@ -1,7 +1,10 @@
+import json
+import os
 import unittest
 
-import os
-from somef.cli import *
+import validators
+
+from somef import cli
 
 test_data_path = "test_data/"
 test_data_repositories = "test_data/repositories/"
@@ -9,117 +12,6 @@
 
 class TestCli(unittest.TestCase):
 
-    def test_extract_bibtex(self):
-        with open(test_data_path + "test_extract_bibtex.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_bibtex(test_text)
-            assert "@inproceedings" in c[0]
-
-    def test_extract_dois(self):
-        with open(test_data_path + "test_extract_dois.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_dois(test_text)
-            assert len(c) == 2
-
-    def test_extract_binder_links(self):
-        with open(test_data_path + "test_extract_binder_links.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_binder_links(test_text)
-            assert len(c) == 2
-
-    def test_extract_title_underline(self):
-        with open(test_data_path + "test_extract_title_underline.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_title(test_text)
-            assert "Taguette" == c
-
-    def test_extract_title_hash(self):
-        with open(test_data_path + "test_extract_title_hash.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_title(test_text)
-            assert "T2WML: A Cell-Based Language To Map Tables Into Wikidata Records" == c
-
-    def test_extract_title_with_md(self):
-        with open(test_data_path + "test_extract_title_with_md.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_title(test_text)
-            # print(c)
-            assert "SOMEF" == c
-
-    def test_extract_readthedocs_1(self):
-        with open(test_data_path + "test_extract_readthedocs_1.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_readthedocs(test_text)
-            assert ["https://oba.readthedocs.io/"] == c
-
-    def test_extract_readthedocs_2(self):
-        with open(test_data_path + "test_extract_readthedocs_2.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_readthedocs(test_text)
-            # print(c)
-            assert ["https://kgtk.readthedocs.io/"] == c
-
-    def test_extract_readthedocs_3(self):
-        test_text = """
-        See full documentation at [https://somef.readthedocs.io/en/latest/](https://somef.readthedocs.io/en/latest/)
-        """
-        c = extract_readthedocs(test_text)
-        print(c)
-        assert ["https://somef.readthedocs.io/"] == c
-
-    def test_extract_readthedocs_issue_407(self):
-        with open(test_data_path + "test_extract_readthedocs_3.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_readthedocs(test_text)
-            print(c)
-            assert ["https://owl-to-oas.readthedocs.io/"] == c
-
-    def test_extract_gitter_chat(self):
-        with open(test_data_path + "test_extract_gitter_chat.txt", "r") as data_file:
-            test_text = data_file.read()
-            c = extract_support_channels(test_text)
-            # print(c)
-            assert "https://gitter.im/OpenGeoscience/geonotebook" in c
-
-    # This repo does no longer have a file with ACK.
-    # def test_issue_166(self):
-    #     header = {}
-    #     header['accept'] = 'application/vnd.github.v3+json'
-    #     text, github_data = load_repository_metadata("https://github.com/tensorflow/tensorflow/tree/v2.6.0", header)
-    #     assert len(github_data['acknowledgments']) > 0
-
-    def test_repo_status(self):
-        with open(test_data_path + "test_repo_status.txt", "r") as data_file:
-            test_text = data_file.read()
-            repo_status = extract_repo_status(test_text)
-            assert len(repo_status) > 0
-
-    # This issue should be changed to making a snapshot of the zip file
-    def test_issue_171(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "rdflib-6.0.2")
-        assert len(github_data['contributors']) > 0
-
-    # This issue should be changed to making a snapshot of the zip file
-    def test_issue_209(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "rdflib-6.0.2")
-        assert len(github_data['hasScriptFile']) > 0
-
-    def test_issue_181(self):
-        with open(test_data_path + "test_issue_181.txt", "r") as data_file:
-            test_text = data_file.read()
-            arxiv_links = extract_arxiv_links(test_text)
-            assert len(arxiv_links) > 0
-
-    # This issue should be changed to making a snapshot of the zip file
-    def test_issue_211(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "probot-12.1.1")
-        assert len(github_data['contributingGuidelines']) > 0 and ('license' in github_data.keys() or 'licenseText' in github_data.keys())
-
-    # This issue should be changed to making a snapshot of the zip file
-    def test_issue_218(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "captum")
-        assert len(github_data['citation']) > 0
-
     # Commenting this test because it downloads a big repository, and just checks that a ttl has a field.
     # The original issue was about having some fields missing in ttl. This does not seem to do that.
     # TO REVIEW.
@@ -152,231 +44,93 @@ def test_issue_280(self):
             repo_set.remove(remove_url)
         assert len(repo_set) > 0
 
-    # This issue should be changed to making a snapshot of the zip file
-    def test_issue_268(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "probot-12.1.1")
-        assert len(github_data['licenseText']) > 0
-
-    # Commenting this issue: this repo does no longer have an ACK file
-    # def test_issue_210(self):
-    #     from somef import cli
-    #     cli.run_cli(threshold=0.8,
-    #                 ignore_classifiers=False,
-    #                 repo_url="https://github.com/tensorflow/tensorflow/tree/v2.6.0",
-    #                 doc_src=None,
-    #                 in_file=None,
-    #                 output=None,
-    #                 graph_out=None,
-    #                 graph_format="turtle",
-    #                 codemeta_out="test-tensorflow-2.6.0.json-ld",
-    #                 pretty=True,
-    #                 missing=False)
-    #     text_file = open("test-tensorflow-2.6.0.json-ld", "r")
-    #     data = text_file.read()
-    #     text_file.close()
-    #     assert data.find("\"acknowledgments\":") >= 0
-
-    def test_issue_286(self):
-        credentials_file = Path(
-            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
-        ).expanduser()
-        if credentials_file.exists():
-            with credentials_file.open("r") as fh:
-                file_paths = json.load(fh)
-        else:
-            sys.exit("Error: Please provide a config.json file.")
-        header = {}
-        if 'Authorization' in file_paths.keys():
-            header['Authorization'] = file_paths['Authorization']
-        header['accept'] = 'application/vnd.github.v3+json'
-        text, github_data = load_repository_metadata("https://gitlab.com/gitlab-org/ci-sample-projects/platform-team",
-                                                     header)
-        assert len(github_data['downloadUrl']) > 0
-
-    def test_issue_291(self):
-        repo_url = "https://github.com/dgarijo/Widoco"
-        with open(test_data_path + "README-widoco.md", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, repo_url)
-            assert (not logo == "")
-
-    def test_issue_291_2(self):
-        repo_url = "https://github.com/usc-isi-i2/kgtk/"
-        with open(test_data_path + "test_logo_uscisii2.txt", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, repo_url)
-            assert (not logo == "")
-
-    def test_issue_291_3(self):
-        repo_url = "https://github.com/tensorflow/tensorflow/"
-        with open(test_data_path + "test_logo_tensorflow.txt", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, repo_url)
-            assert (not logo == "")
-
-    def test_issue_images(self):
-        repo_url = "https://github.com/usc-isi-i2/kgtk/"
-        with open(test_data_path + "test_issue_images.txt", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, repo_url)
-            assert len(images) > 0
-
-    def test_issue_285(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "RDFChess")
-        assert ('license' not in github_data) == True
-
-    def test_issue_no_readme(self):
-        credentials_file = Path(
-            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
-        ).expanduser()
-        if credentials_file.exists():
-            with credentials_file.open("r") as fh:
-                file_paths = json.load(fh)
-        else:
-            sys.exit("Error: Please provide a config.json file.")
-        header = {}
-        if 'Authorization' in file_paths.keys():
-            header['Authorization'] = file_paths['Authorization']
-        header['accept'] = 'application/vnd.github.v3+json'
-        text, github_data = load_repository_metadata("https://github.com/oeg-upm/OpenRefineExtension_Transformation",                                                     header)
-        assert ('codeRepository' in github_data) == True
-
-    def test_issue_270(self):
-        with open(test_data_path + "test_issue_270.txt", "r") as data_file:
-            test_text = data_file.read()
-            support_channels = extract_support_channels(test_text)
-            assert len(support_channels) == 2
-
     def test_issue_311(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "repostatus-README.md",
-                in_file=None,
-                output=None,
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=test_data_path + "test-repostatus-311.json-ld",
-                pretty=True,
-                missing=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "repostatus-README.md",
+                    in_file=None,
+                    output=None,
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=test_data_path + "test-repostatus-311.json-ld",
+                    pretty=True,
+                    missing=False)
         text_file = open(test_data_path + "test-repostatus-311.json-ld", "r")
         data = text_file.read()
         text_file.close()
         assert data.find("\"repoStatus\":") < 0
         os.remove(test_data_path + "test-repostatus-311.json-ld")
 
-    def test_issue_284_issue_272(self):
-        credentials_file = Path(
-            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
-        ).expanduser()
-        if credentials_file.exists():
-            with credentials_file.open("r") as fh:
-                file_paths = json.load(fh)
-        else:
-            sys.exit("Error: Please provide a config.json file.")
-        header = {}
-        if 'Authorization' in file_paths.keys():
-            header['Authorization'] = file_paths['Authorization']
-        header['accept'] = 'application/vnd.github.v3+json'
-        text, github_data = load_repository_metadata("https://github.com/3b1b/manim", header)
-        assert (('stargazersCount' in github_data) == True) and (('longTitle' in github_data) == False)
-
-    # Merging with previous issue to avoid multiple requests.
-    # def test_issue_272(self):
-    #     header = {'accept': 'application/vnd.github.v3+json'}
-    #     text, github_data = load_repository_metadata("https://github.com/3b1b/manim", header)
-    #     assert ('longTitle' in github_data) == False
-
     def test_issue_281(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "repostatus-README.md",
-                in_file=None,
-                output=test_data_path + "test-281.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "repostatus-README.md",
+                    in_file=None,
+                    output=test_data_path + "test-281.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-281.json", "r")
         data = text_file.read()
         text_file.close()
         assert data.find("missingCategories") > 0
         os.remove(test_data_path + "test-281.json")
 
-    def test_logo(self):
-        with open(test_data_path + "test_logo.txt", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, "https://github.com/oeg-upm/Chowlk")
-            assert (not logo == "")
-
-    def test_logo2(self):
-        with open(test_data_path + "test_logo2.txt", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, "https://github.com/pytorch/pytorch")
-            assert (not logo == "")
-
-    def test_images(self):
-        with open(test_data_path + "test_images.txt", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, "https://github.com/pytorch/pytorch")
-            assert (len(images) > 0 and not logo == "")
-
     def test_issue_200(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-widoco.md",
-                in_file=None,
-                output=test_data_path + "test-200.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-widoco.md",
+                    in_file=None,
+                    output=test_data_path + "test-200.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-200.json", "r")
         data = text_file.read()
         text_file.close()
         assert data.find("parentHeader") > 0
         os.remove(test_data_path + "test-200.json")
 
-
     def test_issue_343(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-ya2o.md",
-                in_file=None,
-                output=test_data_path + "test-343.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-ya2o.md",
+                    in_file=None,
+                    output=test_data_path + "test-343.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-343.json", "r")
         data = text_file.read()
         text_file.close()
         assert data.find("parentHeader") > 0
         os.remove(test_data_path + "test-343.json")
 
-    def test_issue_285(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "repos-oeg/cogito-kgg-module")
-        assert ('license' not in github_data) == True
+
 
     def test_issue_355(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=None,
-                local_repo=test_data_repositories + "bimerr-epw",
-                in_file=None,
-                output=test_data_path + "repositories/repos_oeg/test-355.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=None,
+                    local_repo=test_data_repositories + "bimerr-epw",
+                    in_file=None,
+                    output=test_data_path + "repositories/repos_oeg/test-355.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=False)
         text_file = open(test_data_path + "repositories/repos_oeg/test-355.json", "r")
         data = text_file.read()
         print(data)
@@ -384,35 +138,19 @@ def test_issue_355(self):
         assert data.find("longTitle") > 0
         os.remove(test_data_path + "repositories/repos_oeg/test-355.json")
 
-    def test_no_repository_metadata(self):
-        credentials_file = Path(
-            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
-        ).expanduser()
-        if credentials_file.exists():
-            with credentials_file.open("r") as fh:
-                file_paths = json.load(fh)
-        else:
-            sys.exit("Error: Please provide a config.json file.")
-        header = {}
-        if 'Authorization' in file_paths.keys():
-            header['Authorization'] = file_paths['Authorization']
-        header['accept'] = 'application/vnd.github.v3+json'
-        text, github_data = load_repository_metadata("https://github.com/RDFLib/rdflib/tree/6.0.2", header, True)
-        assert ('releases' not in github_data) == True
-
     def test_issue_150(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-mapshaper.md",
-                local_repo=None,
-                in_file=None,
-                output=None,
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=test_data_path + "test-150.json-ld",
-                pretty=True,
-                missing=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-mapshaper.md",
+                    local_repo=None,
+                    in_file=None,
+                    output=None,
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=test_data_path + "test-150.json-ld",
+                    pretty=True,
+                    missing=False)
         text_file = open(test_data_path + "test-150.json-ld", "r")
         data = text_file.read()
         text_file.close()
@@ -420,17 +158,17 @@ def test_issue_150(self):
         os.remove(test_data_path + "test-150.json-ld")
 
     def test_issue_346(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-wothive.md",
-                in_file=None,
-                output=test_data_path + "test-346.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-wothive.md",
+                    in_file=None,
+                    output=test_data_path + "test-346.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-346.json", "r")
         data = text_file.read()
         text_file.close()
@@ -438,45 +176,35 @@ def test_issue_346(self):
         os.remove(test_data_path + "test-346.json")
 
     def test_issue_241(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-tensorflow-2.6.0.md",
-                in_file=None,
-                output=test_data_path + "test-241.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-tensorflow-2.6.0.md",
+                    in_file=None,
+                    output=test_data_path + "test-241.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-241.json", "r")
         data = text_file.read()
         text_file.close()
         assert data.find("packageDistribution") > 0
         os.remove(test_data_path + "test-241.json")
 
-    def test_issue_320(self):
-        with open(test_data_path + "README-urllib3.md", "r") as data_file:
-            test_text = data_file.read()
-            logo, images = extract_images(test_text, "https://github.com/urllib3/urllib3")
-            assert (not logo == "")
-
-    def test_issue_361(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "rdflib-6.0.2")
-        assert len(text) > 0
-
     def test_issue_380(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-raidologist.md",
-                in_file=None,
-                output=test_data_path + "test-380.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-raidologist.md",
+                    in_file=None,
+                    output=test_data_path + "test-380.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-380.json", "r")
         data = text_file.read()
         text_file.close()
@@ -484,17 +212,17 @@ def test_issue_380(self):
         os.remove(test_data_path + "test-380.json")
 
     def test_issue_379(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-raidologist.md",
-                in_file=None,
-                output=test_data_path + "test-379.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-raidologist.md",
+                    in_file=None,
+                    output=test_data_path + "test-379.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-379.json", "r")
         data = text_file.read()
         text_file.close()
@@ -504,17 +232,17 @@ def test_issue_379(self):
         os.remove(test_data_path + "test-379.json")
 
     def test_issue_383(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-mapeathor.md",
-                in_file=None,
-                output=test_data_path + "test-383.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-mapeathor.md",
+                    in_file=None,
+                    output=test_data_path + "test-383.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-383.json", "r")
         data = text_file.read()
         text_file.close()
@@ -522,17 +250,17 @@ def test_issue_383(self):
         os.remove(test_data_path + "test-383.json")
 
     def test_issue_384(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-fair-ontologies.md",
-                in_file=None,
-                output=test_data_path + "test-384.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-fair-ontologies.md",
+                    in_file=None,
+                    output=test_data_path + "test-384.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-384.json", "r")
         data = text_file.read()
         text_file.close()
@@ -540,17 +268,17 @@ def test_issue_384(self):
         os.remove(test_data_path + "test-384.json")
 
     def test_issue_378(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-easytv-annotator.md",
-                in_file=None,
-                output=test_data_path + "test-378.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-easytv-annotator.md",
+                    in_file=None,
+                    output=test_data_path + "test-378.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-378.json", "r")
         data = text_file.read()
         text_file.close()
@@ -565,17 +293,17 @@ def test_issue_378(self):
         os.remove(test_data_path + "test-378.json")
 
     def test_issue_378_2(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-easytv-annotator.md",
-                in_file=None,
-                output=test_data_path + "test-378-2.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-easytv-annotator.md",
+                    in_file=None,
+                    output=test_data_path + "test-378-2.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-378-2.json", "r")
         data = text_file.read()
         text_file.close()
@@ -584,80 +312,94 @@ def test_issue_378_2(self):
         assert_true = False
         for install in installation:
             if 'parentHeader' in install.keys():
-                if install['originalHeader'] == "Easy way" and "Installation" in install['parentHeader']\
+                if install['originalHeader'] == "Easy way" and "Installation" in install['parentHeader'] \
                         and "easytv-annotator" in install["parentHeader"]:
                     assert_true = True
         assert assert_true
         os.remove(test_data_path + "test-378-2.json")
 
-    def test_issue_389(self):
-        text, github_data = load_local_repository_metadata(test_data_repositories + "wot-hive")
-        print(github_data)
-        assert len(github_data['hasBuildFile']) > 0
-
     def test_issue_260(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-kgtk-notebooks.md",
-                in_file=None,
-                output=test_data_path + "test-260.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-kgtk-notebooks.md",
+                    in_file=None,
+                    output=test_data_path + "test-260.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-260.json", "r")
         data = text_file.read()
         text_file.close()
         assert data.find("executableExample") > 0
         os.remove(test_data_path + "test-260.json")
 
-    def test_issue_337(self):
-        text = """## Interactive web interface
-
-Visit the public website at [www.mapshaper.org](http://www.mapshaper.org) or use the web UI locally via the `mapshaper-gui` script. 
-
-All processing is done in the browser, so your data stays private, even when using the public website.
-
-The web UI works in recent desktop versions of Chrome, Firefox, Safari and Internet Explorer. Safari before v10.1 and IE before v10 are not supported.
-
+    def test_issue_319_1(self):
         """
-        text = remove_links_images(text)
-        assert text.find("[www.mapshaper.org](http://www.mapshaper.org)") == -1
+        Check if citation file format is recognized and returned
+        Returns
+        -------
 
-    def test_issue_319_1(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=None,
-                local_repo=test_data_repositories + "Widoco",
-                in_file=None,
-                output=test_data_path + "test-319-1.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        """
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=None,
+                    local_repo=test_data_repositories + "Widoco",
+                    in_file=None,
+                    output=test_data_path + "test-319-1.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-319-1.json", "r")
         data = text_file.read()
         text_file.close()
         assert data.find("citation file format") > 0
         os.remove(test_data_path + "test-319-1.json")
 
+    def test_issue_388(self):
+        """
+        Test checks that ontologies are recognized appropriately
+        Returns
+        -------
+
+        """
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=None,
+                    local_repo=test_data_repositories + "Widoco",
+                    in_file=None,
+                    output=test_data_path + "test-388.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=False)
+        text_file = open(test_data_path + "test-388.json", "r")
+        data = text_file.read()
+        json_content = json.loads(data)
+        text_file.close()
+        assert len(json_content["ontologies"]["excerpt"]) == 2
+        os.remove(test_data_path + "test-388.json")
+
     def test_issue_319_2(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=None,
-                local_repo=test_data_repositories + "kgtk-notebooks",
-                in_file=None,
-                output=test_data_path + "test-319-2.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=None,
+                    local_repo=test_data_repositories + "kgtk-notebooks",
+                    in_file=None,
+                    output=test_data_path + "test-319-2.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-319-2.json", "r")
         data = text_file.read()
         text_file.close()
@@ -665,17 +407,17 @@ def test_issue_319_2(self):
         os.remove(test_data_path + "test-319-2.json")
 
     def test_issue_385(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                doc_src=test_data_path + "README-tada-gam.md",
-                in_file=None,
-                output=test_data_path + "test-385.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    doc_src=test_data_path + "README-tada-gam.md",
+                    in_file=None,
+                    output=test_data_path + "test-385.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-385.json", "r")
         data = text_file.read()
         text_file.close()
@@ -685,143 +427,143 @@ def test_issue_385(self):
         os.remove(test_data_path + "test-385.json")
 
     def test_issue_398(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/dgarijo/Widoco",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-398.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/dgarijo/Widoco",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-398.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-398.json", "r")
         data = text_file.read()
         text_file.close()
         json_content = json.loads(data)
-        repo_Status = json_content['repoStatus']
-        confidende = repo_Status['confidence']
-        assert len(confidende) == 1
+        repo_status = json_content['repoStatus']
+        confidence = repo_status['confidence']
+        assert len(confidence) == 1
         os.remove(test_data_path + "test-398.json")
 
     def test_issue_393(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/oeg-upm/wot-hive/tree/main/docker/auroral-hive",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-393.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/oeg-upm/wot-hive/tree/main/docker/auroral-hive",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-393.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True)
         text_file = open(test_data_path + "test-393.json", "r")
         data = text_file.read()
         text_file.close()
         json_content = json.loads(data)
         acknowledgement = json_content['acknowledgement']
-        assert acknowledgement != None
+        assert acknowledgement is not None
         os.remove(test_data_path + "test-393.json")
 
     def test_issue_314(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/oeg-upm/wot-hive/tree/main/docker/auroral-hive",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-314.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/oeg-upm/wot-hive/tree/main/docker/auroral-hive",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-314.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=True)
         text_file = open(test_data_path + "test-314.json", "r")
         data = text_file.read()
         text_file.close()
         json_content = json.loads(data)
         acknowledgement = json_content['acknowledgement']
-        assert acknowledgement != None
+        assert acknowledgement is not None
         os.remove(test_data_path + "test-314.json")
 
     def test_issue_314_1(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/tensorflow/tensorflow/tree/v2.6.0",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-314-1.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/tensorflow/tensorflow/tree/v2.6.0",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-314-1.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=True)
         text_file = open(test_data_path + "test-314-1.json", "r")
         data = text_file.read()
         text_file.close()
         json_content = json.loads(data)
         image = json_content['image']
-        assert image != None
+        assert image is not None
         os.remove(test_data_path + "test-314-1.json")
 
     def test_issue_314_2(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://gitlab.com/jleblay/tokei",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-314-2.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://gitlab.com/jleblay/tokei",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-314-2.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=False,
+                    readme_only=True)
         text_file = open(test_data_path + "test-314-2.json", "r")
         data = text_file.read()
         text_file.close()
         json_content = json.loads(data)
         description = json_content['description']
-        assert description != None
+        assert description is not None
         os.remove(test_data_path + "test-314-2.json")
 
     def test_issue_314_3(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://gitlab.com/unboundedsystems/adapt/-/tree/release-0.1",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-314-3.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=True)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://gitlab.com/unboundedsystems/adapt/-/tree/release-0.1",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-314-3.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=True)
         text_file = open(test_data_path + "test-314-3.json", "r")
         data = text_file.read()
         text_file.close()
         json_content = json.loads(data)
         description = json_content['description']
-        assert description != None
+        assert description is not None
         os.remove(test_data_path + "test-314-3.json")
 
     def test_issue_403(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/oeg-upm/wot-hive",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-403.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/oeg-upm/wot-hive",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-403.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-403.json", "r")
         data = text_file.read()
         text_file.close()
@@ -832,59 +574,61 @@ def test_issue_403(self):
         os.remove(test_data_path + "test-403.json")
 
     def test_issue_408(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                local_repo=test_data_repositories + "bimerr-epw",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-408.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=test_data_repositories + "bimerr-epw",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-408.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-408.json", "r")
         data = text_file.read()
         text_file.close()
-        assert data.find("https://github.com/oeg-upm/bimerr-epw/tree/master/Code/TDATA2RDFANDV/static/rest_framework/docs") == -1
+        assert data.find(
+            "https://github.com/oeg-upm/bimerr-epw/tree/master/Code/TDATA2RDFANDV/static/rest_framework/docs") == -1
         os.remove(test_data_path + "test-408.json")
 
     def test_issue_408_1(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/oeg-upm/bimerr-epw/",
-                local_repo=None,
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-408-1.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/oeg-upm/bimerr-epw/",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-408-1.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-408-1.json", "r")
         data = text_file.read()
         text_file.close()
-        assert data.find("https://github.com/oeg-upm/bimerr-epw/tree/master/Code/TDATA2RDFANDV/static/rest_framework/docs") == -1
+        assert data.find(
+            "https://github.com/oeg-upm/bimerr-epw/tree/master/Code/TDATA2RDFANDV/static/rest_framework/docs") == -1
         os.remove(test_data_path + "test-408-1.json")
 
     def test_issue_225_406(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                local_repo=None,
-                doc_src=test_data_path+"README-mapshaper.md",
-                in_file=None,
-                output=test_data_path + "test-225.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=None,
+                    doc_src=test_data_path + "README-mapshaper.md",
+                    in_file=None,
+                    output=test_data_path + "test-225.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-225.json", "r")
         data = text_file.read()
         text_file.close()
@@ -892,19 +636,19 @@ def test_issue_225_406(self):
         os.remove(test_data_path + "test-225.json")
 
     def test_issue_406(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                local_repo=None,
-                doc_src=test_data_path+"README-OWL-To-OAS.md",
-                in_file=None,
-                output=test_data_path + "test-406.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=None,
+                    doc_src=test_data_path + "README-OWL-To-OAS.md",
+                    in_file=None,
+                    output=test_data_path + "test-406.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-406.json", "r")
         data = text_file.read()
         text_file.close()
@@ -912,19 +656,19 @@ def test_issue_406(self):
         os.remove(test_data_path + "test-406.json")
 
     def test_issue_255(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/mbloch/mapshaper/",
-                local_repo=None,
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-255.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/mbloch/mapshaper/",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-255.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-255.json", "r")
         data = text_file.read()
         text_file.close()
@@ -932,19 +676,19 @@ def test_issue_255(self):
         os.remove(test_data_path + "test-255.json")
 
     def test_issue_255_1(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/PyLops/pylops/",
-                local_repo=None,
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-255-1.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/PyLops/pylops/",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-255-1.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-255-1.json", "r")
         data = text_file.read()
         text_file.close()
@@ -952,19 +696,19 @@ def test_issue_255_1(self):
         os.remove(test_data_path + "test-255-1.json")
 
     def test_issue_375(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                local_repo=None,
-                doc_src=test_data_path+"README-pylops.md",
-                in_file=None,
-                output=test_data_path + "test-375.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=None,
+                    doc_src=test_data_path + "README-pylops.md",
+                    in_file=None,
+                    output=test_data_path + "test-375.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-375.json", "r")
         data = text_file.read()
         text_file.close()
@@ -972,19 +716,19 @@ def test_issue_375(self):
         os.remove(test_data_path + "test-375.json")
 
     def test_issue_136(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                local_repo=None,
-                doc_src=test_data_path+"README-widoco.md",
-                in_file=None,
-                output=test_data_path + "test-136.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=None,
+                    doc_src=test_data_path + "README-widoco.md",
+                    in_file=None,
+                    output=test_data_path + "test-136.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-136.json", "r")
         data = text_file.read()
         text_file.close()
@@ -992,19 +736,19 @@ def test_issue_136(self):
         os.remove(test_data_path + "test-136.json")
 
     def test_issue_136_1(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                local_repo=None,
-                doc_src=test_data_path+"README-tokei.md",
-                in_file=None,
-                output=test_data_path + "test-136-1.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=None,
+                    doc_src=test_data_path + "README-tokei.md",
+                    in_file=None,
+                    output=test_data_path + "test-136-1.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-136-1.json", "r")
         data = text_file.read()
         text_file.close()
@@ -1012,19 +756,19 @@ def test_issue_136_1(self):
         os.remove(test_data_path + "test-136-1.json")
 
     def test_issue_353(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/proycon/analiticcl",
-                local_repo=None,
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-353.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/proycon/analiticcl",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-353.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-353.json", "r")
         data = text_file.read()
         text_file.close()
@@ -1032,19 +776,19 @@ def test_issue_353(self):
         os.remove(test_data_path + "test-353.json")
 
     def test_issue_366(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url=None,
-                local_repo=test_data_repositories + "wot-hive",
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-366.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=test_data_repositories + "wot-hive",
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-366.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-366.json", "r")
         data = text_file.read()
         text_file.close()
@@ -1052,19 +796,19 @@ def test_issue_366(self):
         os.remove(test_data_path + "test-366.json")
 
     def test_issue_417(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/dgarijo/Widoco",
-                local_repo=None,
-                doc_src=None,
-                in_file=None,
-                output=None,
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=test_data_path + "test-417.json-ld",
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/dgarijo/Widoco",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=None,
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=test_data_path + "test-417.json-ld",
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-417.json-ld", "r")
         data = text_file.read()
         text_file.close()
@@ -1074,23 +818,23 @@ def test_issue_417(self):
         os.remove(test_data_path + "test-417.json-ld")
 
     def test_issue_382(self):
-        run_cli(threshold=0.8,
-                ignore_classifiers=False,
-                repo_url="https://github.com/oeg-upm/mapeathor",
-                local_repo=None,
-                doc_src=None,
-                in_file=None,
-                output=test_data_path + "test-382.json",
-                graph_out=None,
-                graph_format="turtle",
-                codemeta_out=None,
-                pretty=True,
-                missing=True,
-                readme_only=False)
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/oeg-upm/mapeathor",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-382.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
         text_file = open(test_data_path + "test-382.json", "r")
         data = text_file.read()
         text_file.close()
         json_content = json.loads(data)
         contact = json_content['contact']
-        assert contact != None
-        os.remove(test_data_path + "test-382.json")
\ No newline at end of file
+        assert contact is not None
+        os.remove(test_data_path + "test-382.json")
diff --git a/src/somef/test/test_data/README-tensorflow-2.6.0.md b/src/somef/test/test_data/README-tensorflow-2.6.0.md
new file mode 100644
index 00000000..f5c7590b
--- /dev/null
+++ b/src/somef/test/test_data/README-tensorflow-2.6.0.md
@@ -0,0 +1,156 @@
+<div align="center">
+  <img src="https://www.tensorflow.org/images/tf_logo_social.png">
+</div>
+
+[![Python](https://img.shields.io/pypi/pyversions/tensorflow.svg?style=plastic)](https://badge.fury.io/py/tensorflow)
+[![PyPI](https://badge.fury.io/py/tensorflow.svg)](https://badge.fury.io/py/tensorflow)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4724125.svg)](https://doi.org/10.5281/zenodo.4724125)
+
+**`Documentation`** |
+------------------- |
+[![Documentation](https://img.shields.io/badge/api-reference-blue.svg)](https://www.tensorflow.org/api_docs/) |
+
+[TensorFlow](https://www.tensorflow.org/) is an end-to-end open source platform
+for machine learning. It has a comprehensive, flexible ecosystem of
+[tools](https://www.tensorflow.org/resources/tools),
+[libraries](https://www.tensorflow.org/resources/libraries-extensions), and
+[community](https://www.tensorflow.org/community) resources that lets
+researchers push the state-of-the-art in ML and developers easily build and
+deploy ML-powered applications.
+
+TensorFlow was originally developed by researchers and engineers working on the
+Google Brain team within Google's Machine Intelligence Research organization to
+conduct machine learning and deep neural networks research. The system is
+general enough to be applicable in a wide variety of other domains, as well.
+
+TensorFlow provides stable [Python](https://www.tensorflow.org/api_docs/python)
+and [C++](https://www.tensorflow.org/api_docs/cc) APIs, as well as
+non-guaranteed backward compatible API for
+[other languages](https://www.tensorflow.org/api_docs).
+
+Keep up-to-date with release announcements and security updates by subscribing
+to
+[announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
+See all the [mailing lists](https://www.tensorflow.org/community/forums).
+
+## Install
+
+See the [TensorFlow install guide](https://www.tensorflow.org/install) for the
+[pip package](https://www.tensorflow.org/install/pip), to
+[enable GPU support](https://www.tensorflow.org/install/gpu), use a
+[Docker container](https://www.tensorflow.org/install/docker), and
+[build from source](https://www.tensorflow.org/install/source).
+
+To install the current release, which includes support for
+[CUDA-enabled GPU cards](https://www.tensorflow.org/install/gpu) *(Ubuntu and
+Windows)*:
+
+```
+$ pip install tensorflow
+```
+
+A smaller CPU-only package is also available:
+
+```
+$ pip install tensorflow-cpu
+```
+
+To update TensorFlow to the latest version, add `--upgrade` flag to the above
+commands.
+
+*Nightly binaries are available for testing using the
+[tf-nightly](https://pypi.python.org/pypi/tf-nightly) and
+[tf-nightly-cpu](https://pypi.python.org/pypi/tf-nightly-cpu) packages on PyPi.*
+
+#### *Try your first TensorFlow program*
+
+```shell
+$ python
+```
+
+```python
+>>> import tensorflow as tf
+>>> tf.add(1, 2).numpy()
+3
+>>> hello = tf.constant('Hello, TensorFlow!')
+>>> hello.numpy()
+b'Hello, TensorFlow!'
+```
+
+For more examples, see the
+[TensorFlow tutorials](https://www.tensorflow.org/tutorials/).
+
+## Contribution guidelines
+
+**If you want to contribute to TensorFlow, be sure to review the
+[contribution guidelines](CONTRIBUTING.md). This project adheres to TensorFlow's
+[code of conduct](CODE_OF_CONDUCT.md). By participating, you are expected to
+uphold this code.**
+
+**We use [GitHub issues](https://github.com/tensorflow/tensorflow/issues) for
+tracking requests and bugs, please see
+[TensorFlow Discuss](https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss)
+for general questions and discussion, and please direct specific questions to
+[Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow).**
+
+The TensorFlow project strives to abide by generally accepted best practices in
+open-source software development:
+
+[![Fuzzing Status](https://oss-fuzz-build-logs.storage.googleapis.com/badges/tensorflow.svg)](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:tensorflow)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/1486/badge)](https://bestpractices.coreinfrastructure.org/projects/1486)
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v1.4%20adopted-ff69b4.svg)](CODE_OF_CONDUCT.md)
+
+## Continuous build status
+
+You can find more community-supported platforms and configurations in the
+[TensorFlow SIG Build community builds table](https://github.com/tensorflow/build#community-supported-tensorflow-builds).
+
+### Official Builds
+
+Build Type                    | Status                                                                                                                                                                                             | Artifacts
+----------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------
+**Linux CPU**                 | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-cc.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-cc.html)                             | [PyPI](https://pypi.org/project/tf-nightly/)
+**Linux GPU**                 | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-gpu-py3.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-gpu-py3.html)                   | [PyPI](https://pypi.org/project/tf-nightly-gpu/)
+**Linux XLA**                 | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-xla.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/ubuntu-xla.html)                           | TBA
+**macOS**                     | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/macos-py2-cc.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/macos-py2-cc.html)                       | [PyPI](https://pypi.org/project/tf-nightly/)
+**Windows CPU**               | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/windows-cpu.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/windows-cpu.html)                         | [PyPI](https://pypi.org/project/tf-nightly/)
+**Windows GPU**               | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/windows-gpu.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/windows-gpu.html)                         | [PyPI](https://pypi.org/project/tf-nightly-gpu/)
+**Android**                   | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/android.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/android.html)                                 | [![Download](https://api.bintray.com/packages/google/tensorflow/tensorflow/images/download.svg)](https://bintray.com/google/tensorflow/tensorflow/_latestVersion)
+**Raspberry Pi 0 and 1**      | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/rpi01-py3.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/rpi01-py3.html)                             | [Py3](https://storage.googleapis.com/tensorflow-nightly/tensorflow-1.10.0-cp34-none-linux_armv6l.whl)
+**Raspberry Pi 2 and 3**      | [![Status](https://storage.googleapis.com/tensorflow-kokoro-build-badges/rpi23-py3.svg)](https://storage.googleapis.com/tensorflow-kokoro-build-badges/rpi23-py3.html)                             | [Py3](https://storage.googleapis.com/tensorflow-nightly/tensorflow-1.10.0-cp34-none-linux_armv7l.whl)
+**Libtensorflow MacOS CPU**   | Status Temporarily Unavailable | [Nightly Binary](https://storage.googleapis.com/libtensorflow-nightly/prod/tensorflow/release/macos/latest/macos_cpu_libtensorflow_binaries.tar.gz) [Official GCS](https://storage.googleapis.com/tensorflow/)
+**Libtensorflow Linux CPU**   | Status Temporarily Unavailable | [Nightly Binary](https://storage.googleapis.com/libtensorflow-nightly/prod/tensorflow/release/ubuntu_16/latest/cpu/ubuntu_cpu_libtensorflow_binaries.tar.gz) [Official GCS](https://storage.googleapis.com/tensorflow/)
+**Libtensorflow Linux GPU**   | Status Temporarily Unavailable | [Nightly Binary](https://storage.googleapis.com/libtensorflow-nightly/prod/tensorflow/release/ubuntu_16/latest/gpu/ubuntu_gpu_libtensorflow_binaries.tar.gz) [Official GCS](https://storage.googleapis.com/tensorflow/)
+**Libtensorflow Windows CPU** | Status Temporarily Unavailable | [Nightly Binary](https://storage.googleapis.com/libtensorflow-nightly/prod/tensorflow/release/windows/latest/cpu/windows_cpu_libtensorflow_binaries.tar.gz) [Official GCS](https://storage.googleapis.com/tensorflow/)
+**Libtensorflow Windows GPU** | Status Temporarily Unavailable | [Nightly Binary](https://storage.googleapis.com/libtensorflow-nightly/prod/tensorflow/release/windows/latest/gpu/windows_gpu_libtensorflow_binaries.tar.gz) [Official GCS](https://storage.googleapis.com/tensorflow/)
+
+## Resources
+
+*   [TensorFlow.org](https://www.tensorflow.org)
+*   [TensorFlow Tutorials](https://www.tensorflow.org/tutorials/)
+*   [TensorFlow Official Models](https://github.com/tensorflow/models/tree/master/official)
+*   [TensorFlow Examples](https://github.com/tensorflow/examples)
+*   [DeepLearning.AI TensorFlow Developer Professional Certificate](https://www.coursera.org/specializations/tensorflow-in-practice)
+*   [TensorFlow: Data and Deployment from Coursera](https://www.coursera.org/specializations/tensorflow-data-and-deployment)
+*   [Getting Started with TensorFlow 2 from Coursera](https://www.coursera.org/learn/getting-started-with-tensor-flow2)
+*   [TensorFlow: Advanced Techniques from Coursera](https://www.coursera.org/specializations/tensorflow-advanced-techniques)
+*   [Intro to TensorFlow for A.I, M.L, and D.L from Coursera](https://www.coursera.org/learn/introduction-tensorflow)
+*   [Intro to TensorFlow for Deep Learning from Udacity](https://www.udacity.com/course/intro-to-tensorflow-for-deep-learning--ud187)
+*   [Introduction to TensorFlow Lite from Udacity](https://www.udacity.com/course/intro-to-tensorflow-lite--ud190)
+*   [Machine Learning with TensorFlow on GCP](https://www.coursera.org/specializations/machine-learning-tensorflow-gcp)
+*   [TensorFlow Codelabs](https://codelabs.developers.google.com/?cat=TensorFlow)
+*   [TensorFlow Blog](https://blog.tensorflow.org)
+*   [Learn ML with TensorFlow](https://www.tensorflow.org/resources/learn-ml)
+*   [TensorFlow Twitter](https://twitter.com/tensorflow)
+*   [TensorFlow YouTube](https://www.youtube.com/channel/UC0rqucBdTuFTjJiefW5t-IQ)
+*   [TensorFlow model optimization roadmap](https://www.tensorflow.org/model_optimization/guide/roadmap)
+*   [TensorFlow White Papers](https://www.tensorflow.org/about/bib)
+*   [TensorBoard Visualization Toolkit](https://github.com/tensorflow/tensorboard)
+
+Learn more about the
+[TensorFlow community](https://www.tensorflow.org/community) and how to
+[contribute](https://www.tensorflow.org/community/contribute).
+
+## License
+
+[Apache License 2.0](LICENSE)
diff --git a/src/somef/test/test_data/README-wothive.md b/src/somef/test/test_data/README-wothive.md
new file mode 100644
index 00000000..182dd2b3
--- /dev/null
+++ b/src/somef/test/test_data/README-wothive.md
@@ -0,0 +1,111 @@
+
+![logo](https://github.com/oeg-upm/wot-hive/blob/AndreaCimminoArriaga-wothive-logo/logo.png)
+
+[![Version](https://img.shields.io/badge/Version-0.2.2-orange)](https://github.com/oeg-upm/wot-hive/releases/tag/v0.2.2) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
+
+WoT Hive is an implementation of a [W3C Web of Things directory ](https://www.w3.org/TR/wot-discovery/). This implementation is compliant with the standard specification but aims at providing enriched features thanks to the usage of other W3C standards related to Semantic Web technologies.
+
+**Checkout our [wiki for more documentation](https://github.com/oeg-upm/wot-hive/wiki)**
+
+**Temporally the functionality that introduces the registration information in the Things has been disable (version 0.2.0 and above). Also, at the light of a new update in the [Thing Descriptions the semantic validation lacks of a proper shacl shape file](https://github.com/w3c/wot-thing-description/issues/1345) and therefore it is strongly recomended to disable this kind of validation.**
+
+##  Docker quick start
+Copy this receipt in a *docker-compose.yml* file
+
+```yaml
+version: '2'
+services:
+  triplestore:
+    image: acimmino/auroral-fuseky:latest
+    environment:
+     ADMIN_PASSWORD: pw123
+    volumes:
+    - triplestore:/fuseki
+    ports:
+      - '3030:3030'
+  wothive:
+    image: acimmino/wot-hive:auroral-dev
+    # volumes:
+    # - ./configuration.json:/usr/src/wothive/configuration.json
+    ports:
+      - '9000:9000'
+volumes:
+  triplestore:
+```
+
+Run the docker command
+
+```bash
+docker-compose up
+```
+
+
+[OPTIONAL] Uncomment wothive volume to bind your own configuration file for the wothive. Default file is configured to connect the fuseki service running in the docker-compose network.
+
+[OPTIONAL 2] If you want to change the fuseki service location (Move outside docker for instance), you can change the configuration of the wothive also via API. A `POST` request must be sent to `/configuration/triplestore` containing the in the body the following JSON. This 
+
+```json
+{
+    "updateEnpoint": "http://triplestore:3030/sparql",
+    "queryEnpoint": "http://triplestore:3030/sparql",
+    "queryUsingGET": true
+}
+```
+
+##  Jar quick start  
+##### `Requires a triple store service publishing a SPARQL endpoint to store the Thing Descriptions`
+#### 1. Download the WoT Hive service
+Download the latest release of WoT Hive into a folder. Notice that the releases have several files that must be downloaded and placed in the same folder:
+* **log4j.properties** allows to customise the logs of the service
+* **schema.json** allows to perform JSON schema validation over the Thing Descriptions
+* **shape.ttl** allows to perform SHACL shapes validation over the Thing Descriptions
+* **wothive.jar** is the jar of the service
+
+Once downloaded all the resources in the same folder the service can be ran using the command
+
+```bash
+java -jar wothive.jar
+```
+
+When the service is up and running a file called *configuration.json* will be created in the directory of the jar.  The service will run by default in port 9000. 
+#### 2. Set up the triple store
+In order to connect the WoT Hive to a remote triple store a `POST` request must be sent to `/configuration/triplestore` containing the in the body the following JSON
+
+```json
+{
+    "updateEnpoint": "http://localhost:3030/sparql",
+    "queryEnpoint": "http://localhost:3030/sparql",
+    "queryUsingGET": true
+}
+```
+
+Notice that `"queryEndpoint"`and `"updateEndpoint"` must have as value the correct endpoints of the triple store for either querying or inserting data. Finally, if the triple store implements the SPARQL protocol through `GET` requests then leave `"queryUsingGET": true`, otherwise, for using `POST`set it to false `"queryUsingGET": false`.
+
+## WoT Hive API
+
+| Endpoint 	| Method 	| Headers 	| Reference 	| Description 	|
+|---	|---	|---	|---	|---	|
+| `/.well-known/wot-thing-description` 	| `GET` 	| `N/A` 	| [Introduction Mechanim](https://w3c.github.io/wot-discovery/#introduction-well-known) 	| Provides the Thing Description of the WoT Hive directory 	|
+| `/configuration` 	| `GET` 	| `N/A` 	| [Management](https://w3c.github.io/wot-discovery/#exploration-directory-api-management) 	| Provides a JSON with the all the configurations of the WoT Hive 	|
+| `/configuration` 	| `POST` 	| `N/A` 	| [Management](https://w3c.github.io/wot-discovery/#exploration-directory-api-management) 	| The body of the request must contain a JSON with all the configurations of the WoT Hive. 	|
+| `/api/things{?offset,limit,sort_by,sort_order}` 	| `GET` 	| `Accept`: `application/td+json`or `text/turtle`  	| [Listing](https://w3c.github.io/wot-discovery/#exploration-directory-api-registration-listing) 	| Provides a listing of the stored Thing Descriptions in JSON-LD framed or Turtle 	|
+| `/api/things` 	| `POST` 	| `Content-Type`: `application/td+json` 	| [Creation (Anonymous)](https://w3c.github.io/wot-discovery/#exploration-directory-api-registration-creation) 	| Creates an [anonymous Thing Description](https://w3c.github.io/wot-discovery/#dfn-wot-anonymous-thing-description), provided in the body as JSON-LD framed. The generated `:id` is output in the response headers  	|
+| `/api/things/{:id}` 	| `GET` 	| `Accept`: `application/td+json`or `text/turtle` 	| [Retrieval](https://w3c.github.io/wot-discovery/#exploration-directory-api-registration-retrieval) 	| Retrieves the Thing Description with the provided id, in either JSON-LD framed or turtle 	|
+| `/api/things/{:id}` 	| `PUT` 	| `Content-Type`: `application/td+json`or `text/turtle` 	| [Creation](https://w3c.github.io/wot-discovery/#exploration-directory-api-registration-creation) or [Update](https://w3c.github.io/wot-discovery/#exploration-directory-api-registration-update) 	| Creates an Thing Description, provided in the body as JSON-LD framed or turtle 	|
+| `/api/things/{:id}` 	| `PATCH` 	| `Content-Type`: `application/merge-patch+json` 	| [Partial Update](https://w3c.github.io/wot-discovery/#exploration-directory-api-registration-update) 	| Partially updates an existing Thing Description, the updates must be provided in JSON-LD framed 	|
+| `/api/things/{:id}` 	| `DELETE` 	| `N/A` 	| [Deletion](https://w3c.github.io/wot-discovery/#exploration-directory-api-registration-deletion) 	| Partially updates an existing Thing Description, the updates must be provided in JSON-LD framed 	|
+| `api/search/jsonpath{?query}` 	| `GET` 	| `N/A` 	| [JSON path search](https://w3c.github.io/wot-discovery/#jsonpath-semantic) 	| Filters existing Thing Descriptions based on the provided JSON path, the output will be always in JSON-LD framed 	|
+| `api/search/sparql{?query}` 	| `GET` 	| `Accept` : `application/sparql-results+json`, `application/sparql-results+xml`, `text/csv`, or `text/tab-separated-values` 	| [SPARQL search](https://w3c.github.io/wot-discovery/#search-semantic) 	| Solves a SPARQL query following the [standard](https://www.w3.org/TR/sparql11-protocol/<br>), results format are in JSON by default if no header is specified. Otherwise available formats are JSON(application/sparql-results+json), XML (application/sparql-results+xml), CSV (text/csv), or TSV (text/tab-separated-values)  	|
+| `api/events{?diff}` 	| `GET` 	| `N/A` 	| [Notifications](https://w3c.github.io/wot-discovery/#exploration-directory-api-notification) 	| Subscribe to all the events of the service (`create`, `update`, and `delete`) using the Server-Sends-Events (SSE) protocol 	|
+| `api/events/create{?diff}` 	| `GET` 	| `N/A` 	| [Notifications](https://w3c.github.io/wot-discovery/#exploration-directory-api-notification) 	| Subscribe to all the `create` events of the service using the Server-Sends-Events (SSE) protocol 	|
+| `api/events/update{?diff}` 	| `GET` 	| `N/A` 	| [Notifications](https://w3c.github.io/wot-discovery/#exploration-directory-api-notification) 	| Subscribe to all the `update` events of the service using the Server-Sends-Events (SSE) protocol 	|
+| `api/events/delete{?diff}` 	| `GET` 	| `N/A` 	| [Notifications](https://w3c.github.io/wot-discovery/#exploration-directory-api-notification) 	| Subscribe to all the `delete` events of the service using the Server-Sends-Events (SSE) protocol 	|
+
+[Validation](https://w3c.github.io/wot-discovery/#validation) can  be configured to ran using the JSON schema of the Thing Descriptions and/or their SHACL shapes.
+
+---
+### Acknowledgements
+This project has been partially funded by the European project AURORAL from the European Union's Horizont 2020 research and innovation programme under grant agreement Nº101016854.
+
+<img src="https://user-images.githubusercontent.com/4105186/141472288-1b15e0ba-8ae1-414a-a849-222b6bc27754.png" height="75" /> <img src="https://www.auroral.eu/img/logos/bandeira.png"  height="80" />
+
diff --git a/src/somef/test/test_data/repositories/Widoco/example_onto/ontology.nt b/src/somef/test/test_data/repositories/Widoco/example_onto/ontology.nt
new file mode 100644
index 00000000..4770584c
--- /dev/null
+++ b/src/somef/test/test_data/repositories/Widoco/example_onto/ontology.nt
@@ -0,0 +1,153 @@
+<https://w3id.org/example> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Ontology> .
+<https://w3id.org/example> <http://www.w3.org/2002/07/owl#versionIRI> <https://w3id.org/example/1.0.1> .
+<https://w3id.org/example> <http://purl.org/dc/terms/abstract> "An example vocabulary designed to illustrate how to publish vocabularies on the Web following the FAIR principles"@en .
+<https://w3id.org/example> <http://purl.org/dc/elements/1.1/description> "This is an example ontology to illustrate some of the annotations that should be included"@en .
+<https://w3id.org/example> <http://purl.org/dc/elements/1.1/title> "The example ontology"@en .
+<https://w3id.org/example> <http://purl.org/dc/terms/created> "February 5th, 2020"@en .
+<https://w3id.org/example> <http://purl.org/dc/terms/creator> "Daniel Garijo"@en .
+<https://w3id.org/example> <http://purl.org/dc/terms/creator> "Maria Poveda-Villalon"@en .
+<https://w3id.org/example> <http://purl.org/dc/terms/license> <http://creativecommons.org/licenses/by/2.0/> .
+<https://w3id.org/example> <http://purl.org/vocab/vann/preferredNamespacePrefix> "exo"@en .
+<https://w3id.org/example> <http://purl.org/vocab/vann/preferredNamespaceUri> "https://w3id.org/example" .
+<https://w3id.org/example> <http://schema.org/citation> "Cite this vocabulary as: Garijo, D. and Poveda-Villalon, M. The example ontology 1.0.1."@en .
+<https://w3id.org/example> <http://www.w3.org/2000/01/rdf-schema#comment> "An example vocabulary designed to illustrate how to publish vocabularies on the Web following the FAIR principles. This vocabulary describes three simple classes with 3 properties and a data property."@en .
+<https://w3id.org/example> <http://www.w3.org/2002/07/owl#backwardCompatibleWith> <https://w3id.org/example/1.0.0> .
+<https://w3id.org/example> <http://www.w3.org/2002/07/owl#priorVersion> <https://w3id.org/example/1.0.0> .
+<https://w3id.org/example> <http://www.w3.org/2002/07/owl#versionInfo> "1.0.1"@en .
+# 
+# 
+# #################################################################
+# #
+# #    Annotation properties
+# #
+# #################################################################
+# 
+# 
+# http://purl.org/dc/elements/1.1/abstract
+<http://purl.org/dc/elements/1.1/abstract> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/dc/elements/1.1/created
+<http://purl.org/dc/elements/1.1/created> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/dc/elements/1.1/description
+<http://purl.org/dc/elements/1.1/description> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/dc/elements/1.1/publisher
+<http://purl.org/dc/elements/1.1/publisher> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/dc/elements/1.1/title
+<http://purl.org/dc/elements/1.1/title> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/dc/terms/created
+<http://purl.org/dc/terms/created> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/dc/terms/creator
+<http://purl.org/dc/terms/creator> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/dc/terms/license
+<http://purl.org/dc/terms/license> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/vocab/vann/example
+<http://purl.org/vocab/vann/example> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/vocab/vann/preferredNamespacePrefix
+<http://purl.org/vocab/vann/preferredNamespacePrefix> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://purl.org/vocab/vann/preferredNamespaceUri
+<http://purl.org/vocab/vann/preferredNamespaceUri> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://schema.org/author
+<http://schema.org/author> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://schema.org/citation
+<http://schema.org/citation> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# http://www.linkedmodel.org/schema/vaem#rationale
+<http://www.linkedmodel.org/schema/vaem#rationale> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#AnnotationProperty> .
+# 
+# 
+# 
+# #################################################################
+# #
+# #    Datatypes
+# #
+# #################################################################
+# 
+# 
+# http://www.w3.org/2001/XMLSchema#date
+<http://www.w3.org/2001/XMLSchema#date> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2000/01/rdf-schema#Datatype> .
+# 
+# 
+# 
+# #################################################################
+# #
+# #    Object Properties
+# #
+# #################################################################
+# 
+# 
+# https://w3id.org/example#hasMember
+<https://w3id.org/example#hasMember> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#ObjectProperty> .
+<https://w3id.org/example#hasMember> <http://www.w3.org/2000/01/rdf-schema#domain> <https://w3id.org/example#Organization> .
+<https://w3id.org/example#hasMember> <http://www.w3.org/2000/01/rdf-schema#range> <https://w3id.org/example#Researcher> .
+<https://w3id.org/example#hasMember> <http://www.w3.org/2000/01/rdf-schema#comment> "This example property indicates that an Organization has a Researcher as member"@en .
+<https://w3id.org/example#hasMember> <http://www.w3.org/2000/01/rdf-schema#label> "has member"@en .
+# 
+# https://w3id.org/example#hasMentor
+<https://w3id.org/example#hasMentor> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#ObjectProperty> .
+<https://w3id.org/example#hasMentor> <http://www.w3.org/2000/01/rdf-schema#domain> <https://w3id.org/example#Student> .
+<https://w3id.org/example#hasMentor> <http://www.w3.org/2000/01/rdf-schema#range> <https://w3id.org/example#Researcher> .
+<https://w3id.org/example#hasMentor> <http://www.w3.org/2000/01/rdf-schema#comment> "This property links the students to the researcher who mentored them"@en .
+<https://w3id.org/example#hasMentor> <http://www.w3.org/2000/01/rdf-schema#label> "has mentor"@en .
+# 
+# https://w3id.org/example#partOf
+<https://w3id.org/example#partOf> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#ObjectProperty> .
+<https://w3id.org/example#partOf> <http://www.w3.org/2000/01/rdf-schema#domain> <https://w3id.org/example#Researcher> .
+<https://w3id.org/example#partOf> <http://www.w3.org/2000/01/rdf-schema#range> <https://w3id.org/example#Organization> .
+<https://w3id.org/example#partOf> <http://www.w3.org/2000/01/rdf-schema#comment> "This property links instances from Researcher to Organization"@en .
+<https://w3id.org/example#partOf> <http://www.w3.org/2000/01/rdf-schema#label> "part of"@en .
+# 
+# 
+# 
+# #################################################################
+# #
+# #    Data properties
+# #
+# #################################################################
+# 
+# 
+# https://w3id.org/example#foundedIn
+<https://w3id.org/example#foundedIn> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#DatatypeProperty> .
+<https://w3id.org/example#foundedIn> <http://www.w3.org/2000/01/rdf-schema#domain> <https://w3id.org/example#Organization> .
+<https://w3id.org/example#foundedIn> <http://www.w3.org/2000/01/rdf-schema#range> <http://www.w3.org/2001/XMLSchema#date> .
+<https://w3id.org/example#foundedIn> <http://www.w3.org/2000/01/rdf-schema#comment> "Date when an organization was founded"@en .
+<https://w3id.org/example#foundedIn> <http://www.w3.org/2000/01/rdf-schema#label> "founded in"@en .
+# 
+# 
+# 
+# #################################################################
+# #
+# #    Classes
+# #
+# #################################################################
+# 
+# 
+# https://w3id.org/example#Organization
+<https://w3id.org/example#Organization> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class> .
+<https://w3id.org/example#Organization> <http://purl.org/vocab/vann/example> "University of Southern California"@en .
+<https://w3id.org/example#Organization> <http://www.w3.org/2000/01/rdf-schema#comment> "An organized body of people with a particular purpose, especially a business, society, association, etc."@en .
+<https://w3id.org/example#Organization> <http://www.w3.org/2000/01/rdf-schema#label> "Organization"@en .
+# 
+# https://w3id.org/example#Researcher
+<https://w3id.org/example#Researcher> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class> .
+<https://w3id.org/example#Researcher> <http://purl.org/vocab/vann/example> "An example using the Researcher class, e.g., Bob is a professor at the University of Southern California."@en .
+<https://w3id.org/example#Researcher> <http://www.linkedmodel.org/schema/vaem#rationale> "The reason why this concept was added to the ontology. This could reflect some agreement or use cases that may need to be reflected here. For example: The concept Researcher was added to the ontology to represent those authors of scientific publications that belong to a public institution."@en .
+<https://w3id.org/example#Researcher> <http://www.w3.org/2000/01/rdf-schema#comment> "A definition of your class. For example, A researcher is a person who publishes scientific papers, writes research proposals and mentors students"@en .
+<https://w3id.org/example#Researcher> <http://www.w3.org/2000/01/rdf-schema#label> "Researcher"@en .
+# 
+# https://w3id.org/example#Student
+<https://w3id.org/example#Student> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2002/07/owl#Class> .
+<https://w3id.org/example#Student> <http://www.w3.org/2000/01/rdf-schema#comment> "Student is a class introduced in the version 1.0.1 of the ontology. "@en .
+<https://w3id.org/example#Student> <http://www.w3.org/2000/01/rdf-schema#label> "Student"@en .
+# 
+# Generated by the OWL API (version 5.1.14) https://github.com/owlcs/owlapi/
diff --git a/src/somef/test/test_data_to_graph.py b/src/somef/test/test_data_to_graph.py
index 64e16d9d..ab283912 100644
--- a/src/somef/test/test_data_to_graph.py
+++ b/src/somef/test/test_data_to_graph.py
@@ -2,6 +2,7 @@
 
 from somef.data_to_graph import DataGraph
 
+
 class FlattenDict(unittest.TestCase):
     def test_combine_dict(self):
 
diff --git a/src/somef/test/test_extract_ontologies.py b/src/somef/test/test_extract_ontologies.py
new file mode 100644
index 00000000..7191e8be
--- /dev/null
+++ b/src/somef/test/test_extract_ontologies.py
@@ -0,0 +1,23 @@
+import unittest
+
+import os
+
+from somef import extract_ontologies
+
+
+class TestOntologies(unittest.TestCase):
+    def test_is_ontology(self):
+        """
+        This test checks if a file containing an ontology is really there.
+        -------
+        """
+        onto = extract_ontologies.is_file_ontology("test_data/repositories/Widoco/example_onto/ontology.ttl")
+        assert("https://w3id.org/example" in onto)
+
+    def test_is_ontology_fake(self):
+        """
+        This test checks that a RDF file with no ontology is not detected, as it should not.
+        -------
+        """
+        onto = extract_ontologies.is_file_ontology("test_data/repositories/Widoco/example_onto/test.ttl")
+        assert(onto is None)
\ No newline at end of file
diff --git a/src/somef/test/test_header_analysis.py b/src/somef/test/test_header_analysis.py
index ff3c6172..037dc981 100644
--- a/src/somef/test/test_header_analysis.py
+++ b/src/somef/test/test_header_analysis.py
@@ -5,6 +5,7 @@
 # Test data for tests
 test_data_path = "test_data/"
 
+
 class TestHeaderAnalysis(unittest.TestCase):
 
     def test_extract_header_content_hash(self):
@@ -58,7 +59,7 @@ def test_issue_237(self):
         with open(test_data_path + "README-manim.md", "r") as data_file:
             file_text = data_file.read()
             json, results = extract_categories_using_headers(file_text)
-            element = results[2]
+            element = results[0]
             split = element.split("\n")
             assert len(split) > 1
             print(split)
diff --git a/src/somef/test/test_parser_somef.py b/src/somef/test/test_parser_somef.py
index e04cd657..b5c97030 100644
--- a/src/somef/test/test_parser_somef.py
+++ b/src/somef/test/test_parser_somef.py
@@ -1,11 +1,12 @@
 import unittest
 
-from somef.parser_somef import extract_headers, extract_headers_with_tags, extract_content_per_header, extract_bash, \
-    extract_blocks_excerpts, extract_text_excerpts_header, extract_headers_parents
+from somef.parser_somef import extract_headers, extract_headers_with_tags, extract_content_per_header, \
+    extract_bash, extract_blocks_excerpts, extract_text_excerpts_header, extract_headers_parents
 
 # Test data for tests
 test_data_path = "test_data/"
 
+
 class TestParserSomef(unittest.TestCase):
 
     def test_extract_header_content(self):
@@ -41,16 +42,14 @@ def test_extract_blocks_excerpts(self):
             excerpts = extract_blocks_excerpts(content)
             assert len(excerpts) == 48
 
-
     def test_extract_text_excerpts_header(self):
         with open(test_data_path + "README-widoco.md", "r") as data_file:
             text = data_file.read()
             excerpts = extract_text_excerpts_header(text)
             assert len(excerpts.index) == 48
 
-
     def test_extract_headers_parents(self):
         with open(test_data_path + "README-widoco.md", "r") as data_file:
             text = data_file.read()
             parents = extract_headers_parents(text)
-            assert len(parents) == 15
\ No newline at end of file
+            assert len(parents) == 15
diff --git a/src/somef/test/test_process_repository.py b/src/somef/test/test_process_repository.py
new file mode 100644
index 00000000..d4939ca3
--- /dev/null
+++ b/src/somef/test/test_process_repository.py
@@ -0,0 +1,142 @@
+import unittest
+
+import os
+import sys
+import json
+
+from somef import process_repository
+from pathlib import Path
+
+test_data_path = "test_data/"
+test_data_repositories = "test_data/repositories/"
+
+
+class TestProcessRepository(unittest.TestCase):
+
+    def test_issue_171(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "rdflib-6.0.2")
+        assert len(github_data['contributors']) > 0
+
+    def test_issue_209(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "rdflib-6.0.2")
+        assert len(github_data['hasScriptFile']) > 0
+
+    def test_issue_211(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "probot-12.1.1")
+        assert len(github_data['contributingGuidelines']) > 0 and (
+                'license' in github_data.keys() or 'licenseText' in github_data.keys())
+
+    def test_issue_218(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "captum")
+        assert len(github_data['citation']) > 0
+
+    def test_issue_285(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories +
+                                                                              "repos-oeg/cogito-kgg-module")
+        assert 'license' not in github_data
+
+    def test_issue_361(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "rdflib-6.0.2")
+        assert len(text) > 0
+
+    def test_issue_389(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "wot-hive")
+        print(github_data)
+        assert len(github_data['hasBuildFile']) > 0
+
+    def test_issue_no_readme(self):
+        credentials_file = Path(
+            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
+        ).expanduser()
+        if credentials_file.exists():
+            with credentials_file.open("r") as fh:
+                file_paths = json.load(fh)
+        else:
+            sys.exit("Error: Please provide a config.json file.")
+        header = {}
+        if 'Authorization' in file_paths.keys():
+            header['Authorization'] = file_paths['Authorization']
+        header['accept'] = 'application/vnd.github.v3+json'
+        text, github_data = process_repository.load_github_repository_metadata(
+            "https://github.com/oeg-upm/OpenRefineExtension_Transformation", header)
+        assert 'codeRepository' in github_data
+
+    # This issue should be changed to making a snapshot of the zip file
+    def test_issue_268(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "probot-12.1.1")
+        assert len(github_data['licenseText']) > 0
+
+    # Commenting this issue: this repo does no longer have an ACK file
+    # def test_issue_210(self):
+    #     from somef import cli
+    #     cli.run_cli(threshold=0.8,
+    #                 ignore_classifiers=False,
+    #                 repo_url="https://github.com/tensorflow/tensorflow/tree/v2.6.0",
+    #                 doc_src=None,
+    #                 in_file=None,
+    #                 output=None,
+    #                 graph_out=None,
+    #                 graph_format="turtle",
+    #                 codemeta_out="test-tensorflow-2.6.0.json-ld",
+    #                 pretty=True,
+    #                 missing=False)
+    #     text_file = open("test-tensorflow-2.6.0.json-ld", "r")
+    #     data = text_file.read()
+    #     text_file.close()
+    #     assert data.find("\"acknowledgments\":") >= 0
+
+    def test_issue_286(self):
+        credentials_file = Path(
+            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
+        ).expanduser()
+        if credentials_file.exists():
+            with credentials_file.open("r") as fh:
+                file_paths = json.load(fh)
+        else:
+            sys.exit("Error: Please provide a config.json file.")
+        header = {}
+        if 'Authorization' in file_paths.keys():
+            header['Authorization'] = file_paths['Authorization']
+        header['accept'] = 'application/vnd.github.v3+json'
+        text, gitlab_data = process_repository.load_gitlab_repository_metadata(
+            "https://gitlab.com/gitlab-org/ci-sample-projects/platform-team",
+            header)
+        assert len(gitlab_data['downloadUrl']) > 0
+
+    def test_issue_285(self):
+        text, github_data = process_repository.load_local_repository_metadata(test_data_repositories + "RDFChess")
+        assert 'license' not in github_data
+
+    def test_no_repository_metadata(self):
+        credentials_file = Path(
+            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
+        ).expanduser()
+        if credentials_file.exists():
+            with credentials_file.open("r") as fh:
+                file_paths = json.load(fh)
+        else:
+            sys.exit("Error: Please provide a config.json file.")
+        header = {}
+        if 'Authorization' in file_paths.keys():
+            header['Authorization'] = file_paths['Authorization']
+        header['accept'] = 'application/vnd.github.v3+json'
+        text, github_data = process_repository.load_github_repository_metadata(
+            "https://github.com/RDFLib/rdflib/tree/6.0.2",
+            header, True)
+        assert 'releases' not in github_data
+
+    def test_issue_284_issue_272(self):
+        credentials_file = Path(
+            os.getenv("SOMEF_CONFIGURATION_FILE", '~/.somef/config.json')
+        ).expanduser()
+        if credentials_file.exists():
+            with credentials_file.open("r") as fh:
+                file_paths = json.load(fh)
+        else:
+            sys.exit("Error: Please provide a config.json file.")
+        header = {}
+        if 'Authorization' in file_paths.keys():
+            header['Authorization'] = file_paths['Authorization']
+        header['accept'] = 'application/vnd.github.v3+json'
+        text, github_data = process_repository.load_github_repository_metadata("https://github.com/3b1b/manim", header)
+        assert (('stargazersCount' in github_data) and ('longTitle' not in github_data))
diff --git a/src/somef/test/test_regular_expressions.py b/src/somef/test/test_regular_expressions.py
new file mode 100644
index 00000000..1be4b56f
--- /dev/null
+++ b/src/somef/test/test_regular_expressions.py
@@ -0,0 +1,164 @@
+import unittest
+
+from somef import regular_expressions
+
+test_data_path = "test_data/"
+test_data_repositories = "test_data/repositories/"
+
+
+class TestCli(unittest.TestCase):
+
+    def test_extract_bibtex(self):
+        with open(test_data_path + "test_extract_bibtex.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_bibtex(test_text)
+            assert "@inproceedings" in c[0]
+
+    def test_extract_dois(self):
+        with open(test_data_path + "test_extract_dois.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_dois(test_text)
+            assert len(c) == 2
+
+    def test_extract_binder_links(self):
+        with open(test_data_path + "test_extract_binder_links.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_binder_links(test_text)
+            assert len(c) == 2
+
+    def test_extract_title_underline(self):
+        with open(test_data_path + "test_extract_title_underline.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_title(test_text)
+            assert "Taguette" == c
+
+    def test_extract_title_hash(self):
+        with open(test_data_path + "test_extract_title_hash.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_title(test_text)
+            assert "T2WML: A Cell-Based Language To Map Tables Into Wikidata Records" == c
+
+    def test_extract_title_with_md(self):
+        with open(test_data_path + "test_extract_title_with_md.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_title(test_text)
+            # print(c)
+            assert "SOMEF" == c
+
+    def test_extract_readthedocs_1(self):
+        with open(test_data_path + "test_extract_readthedocs_1.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_readthedocs(test_text)
+            assert ["https://oba.readthedocs.io/"] == c
+
+    def test_extract_readthedocs_2(self):
+        with open(test_data_path + "test_extract_readthedocs_2.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_readthedocs(test_text)
+            # print(c)
+            assert ["https://kgtk.readthedocs.io/"] == c
+
+    def test_extract_readthedocs_3(self):
+        test_text = """
+        See full documentation at [https://somef.readthedocs.io/en/latest/](https://somef.readthedocs.io/en/latest/)
+        """
+        c = regular_expressions.extract_readthedocs(test_text)
+        print(c)
+        assert ["https://somef.readthedocs.io/"] == c
+
+    def test_extract_readthedocs_issue_407(self):
+        with open(test_data_path + "test_extract_readthedocs_3.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_readthedocs(test_text)
+            print(c)
+            assert ["https://owl-to-oas.readthedocs.io/"] == c
+
+    def test_extract_gitter_chat(self):
+        with open(test_data_path + "test_extract_gitter_chat.txt", "r") as data_file:
+            test_text = data_file.read()
+            c = regular_expressions.extract_support_channels(test_text)
+            # print(c)
+            assert "https://gitter.im/OpenGeoscience/geonotebook" in c
+
+    def test_repo_status(self):
+        with open(test_data_path + "test_repo_status.txt", "r") as data_file:
+            test_text = data_file.read()
+            repo_status = regular_expressions.extract_repo_status(test_text)
+            assert len(repo_status) > 0
+
+    def test_issue_291(self):
+        repo_url = "https://github.com/dgarijo/Widoco"
+        with open(test_data_path + "README-widoco.md", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, repo_url)
+            assert (not logo == "")
+
+    def test_issue_291_2(self):
+        repo_url = "https://github.com/usc-isi-i2/kgtk/"
+        with open(test_data_path + "test_logo_uscisii2.txt", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, repo_url)
+            assert (not logo == "")
+
+    def test_issue_291_3(self):
+        repo_url = "https://github.com/tensorflow/tensorflow/"
+        with open(test_data_path + "test_logo_tensorflow.txt", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, repo_url)
+            assert (not logo == "")
+
+    def test_issue_images(self):
+        repo_url = "https://github.com/usc-isi-i2/kgtk/"
+        with open(test_data_path + "test_issue_images.txt", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, repo_url)
+            assert len(images) > 0
+
+    def test_issue_181(self):
+        with open(test_data_path + "test_issue_181.txt", "r") as data_file:
+            test_text = data_file.read()
+            arxiv_links = regular_expressions.extract_arxiv_links(test_text)
+            assert len(arxiv_links) > 0
+
+    def test_issue_270(self):
+        with open(test_data_path + "test_issue_270.txt", "r") as data_file:
+            test_text = data_file.read()
+            support_channels = regular_expressions.extract_support_channels(test_text)
+            assert len(support_channels) == 2
+
+    def test_logo(self):
+        with open(test_data_path + "test_logo.txt", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, "https://github.com/oeg-upm/Chowlk")
+            assert (not logo == "")
+
+    def test_logo2(self):
+        with open(test_data_path + "test_logo2.txt", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, "https://github.com/pytorch/pytorch")
+            assert (not logo == "")
+
+    def test_images(self):
+        with open(test_data_path + "test_images.txt", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, "https://github.com/pytorch/pytorch")
+            assert (len(images) > 0 and not logo == "")
+
+    def test_issue_320(self):
+        with open(test_data_path + "README-urllib3.md", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, "https://github.com/urllib3/urllib3")
+            assert (not logo == "")
+
+    def test_issue_337(self):
+        text = """## Interactive web interface
+
+Visit the public website at [www.mapshaper.org](http://www.mapshaper.org) or use the web UI locally via the `mapshaper-gui` script. 
+
+All processing is done in the browser, so your data stays private, even when using the public website.
+
+The web UI works in recent desktop versions of Chrome, Firefox, Safari and Internet Explorer. Safari before v10.1 and IE before v10 are not supported.
+
+        """
+        text = regular_expressions.remove_links_images(text)
+        assert text.find("[www.mapshaper.org](http://www.mapshaper.org)") == -1