Repository: KnowledgeCaptureAndDiscovery/somef (language: python)

Issue:
Test export schemas
Some of the exports to graph need to be better tested. I have noticed that some fields in the json, such as acknowledgements, don't seem to be included in the turtle equivalent
Non-valid URL makes SOMEF-CLI fail.
Example:

```
somef describe  -r ithub.com/dgarijo/Widoco -o out.json -p
SOftware Metadata Extraction Framework (SOMEF) Command Line Interface
Loading Repository ithub.com/dgarijo/Widoco Information....
Error: repository must come from github
Extracting information using headers
Extracting headers and content.
Labeling headers.
File to analyze has no headers
Information extracted. 

Splitting text into valid excerpts for classification
Extraction of bibtex citation from readme completed. 

Text Successfully split. 

Checking Thresholds for Classified Excerpts.
All Excerpts below the given Threshold Removed. 

Extraction of bibtex citation from readme completed. 

Extraction of DOIS from readme completed.

Extraction of Binder links from readme completed.

Traceback (most recent call last):
  File "/home/dgarijo/Documents/GitHub/somef/env_3.9/bin/somef", line 11, in <module>
    load_entry_point('somef', 'console_scripts', 'somef')()
  File "/home/dgarijo/Documents/GitHub/somef/env_3.9/lib/python3.9/site-packages/click/core.py", line 1137, in __call__
    return self.main(*args, **kwargs)
  File "/home/dgarijo/Documents/GitHub/somef/env_3.9/lib/python3.9/site-packages/click/core.py", line 1062, in main
    rv = self.invoke(ctx)
  File "/home/dgarijo/Documents/GitHub/somef/env_3.9/lib/python3.9/site-packages/click/core.py", line 1668, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/home/dgarijo/Documents/GitHub/somef/env_3.9/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/home/dgarijo/Documents/GitHub/somef/env_3.9/lib/python3.9/site-packages/click/core.py", line 763, in invoke
    return __callback(*args, **kwargs)
  File "/home/dgarijo/Documents/GitHub/somef/src/somef/__main__.py", line 128, in describe
    cli.run_cli(**kwargs)
  File "/home/dgarijo/Documents/GitHub/somef/src/somef/cli.py", line 1027, in run_cli
    repo_data = cli_get_data(threshold, ignore_classifiers, repo_url=repo_url)
  File "/home/dgarijo/Documents/GitHub/somef/src/somef/cli.py", line 986, in cli_get_data
    title = extract_title(unfiltered_text)
  File "/home/dgarijo/Documents/GitHub/somef/src/somef/cli.py", line 625, in extract_title
    title = re.findall(r'#.+', unfiltered_text)[0]
IndexError: list index out of range
```
Change licenseFile -> license
Now we point to the contents of the file, so it should be the license.
Solve some issues
Modified code to solve issues #224 and #280


Discussion / hints:
Should do RML mappings for this, as it may seem like a more effective way of handling the transformations

Instead of license, it should be licenseText
This is fixed
@ma-garcia, please add both license File and license text in the JSON until we have a reconciliation service. I think it's a better idea. 


Fix patch:
diff --git a/setup.py b/setup.py
index f6c64e22..68b9d0cf 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,8 @@
     "textblob",
     "rdflib",
     "rdflib-jsonld",
-    "xgboost"
+    "xgboost",
+    "validators"
 ]
 
 
diff --git a/src/somef/cli.py b/src/somef/cli.py
index 3d856cfe..7c609d1c 100644
--- a/src/somef/cli.py
+++ b/src/somef/cli.py
@@ -13,6 +13,7 @@
 import tempfile
 import time
 import urllib
+import validators
 import zipfile
 from io import StringIO
 from os import path
@@ -778,7 +779,7 @@ def format_output(git_data, repo_data):
     """
     print("formatting output")
     file_exploration = ['hasExecutableNotebook', 'hasBuildFile', 'hasDocumentation', 'codeOfConduct',
-                        'contributingGuidelines', 'licenseFile', 'acknowledgments', 'contributors', 'hasScriptFile']
+                        'contributingGuidelines', 'licenseFile', 'licenseText', 'acknowledgments', 'contributors', 'hasScriptFile']
     for i in git_data.keys():
         # print(i)
         # print(git_data[i])
@@ -1011,6 +1012,11 @@ def run_cli(*,
             pretty=False,
             missing=False
             ):
+    # check if it is a valid url
+    if repo_url:
+        if not validators.url(repo_url):
+            print("Not a valid repository url. Please check the url provided")
+            return None
     multiple_repos = in_file is not None
     if multiple_repos:
         with open(in_file, "r") as in_handle:
@@ -1019,8 +1025,20 @@ def run_cli(*,
 
         # convert to a set to ensure uniqueness (we don't want to get the same data multiple times)
         repo_set = set(repo_list)
-
-        repo_data = [cli_get_data(threshold, ignore_classifiers, repo_url=repo_url) for repo_url in repo_set]
+        # check if the urls in repo_set if are valids
+        remove_urls = []
+        for repo_elem in repo_set:
+            if not validators.url(repo_elem):
+                print("Not a valid repository url. Please check the url provided: " + repo_elem)
+                #repo_set.remove(repo_url)
+                remove_urls.append(repo_elem)
+        # remove non valid urls in repo_set
+        for remove_url in remove_urls:
+            repo_set.remove(remove_url)
+        if len(repo_set) > 0:
+            repo_data = [cli_get_data(threshold, ignore_classifiers, repo_url=repo_url) for repo_url in repo_set]
+        else:
+            return None
 
     else:
         if repo_url:
@@ -1042,7 +1060,7 @@ def run_cli(*,
 
         print("Saving Knowledge Graph ttl data to", graph_out)
         with open(graph_out, "wb") as out_file:
-            out_file.write(data_graph.g.serialize(format=graph_format))
+            out_file.write(data_graph.g.serialize(format=graph_format, encoding="UTF-8"))
 
     if codemeta_out is not None:
         save_codemeta_output(repo_data, codemeta_out, pretty=pretty)
diff --git a/src/somef/data_to_graph.py b/src/somef/data_to_graph.py
index ae3b5e76..ab0b0bd2 100644
--- a/src/somef/data_to_graph.py
+++ b/src/somef/data_to_graph.py
@@ -17,6 +17,10 @@ def bind_prefixes(self, prefixes):
     def add_somef_data(self, somef_data):
         # process the somef output into data
         data = DataGraph.process_somef(somef_data)
+        if 'name' not in data.keys():
+            data['name'] = 'name'
+        if 'fullName' not in data.keys():
+            data['fullName'] = 'fullName'
         # add the prefixes that we use in the software_schema
         self.prefixes.update(software_prefixes)
         self.bind_prefixes(software_prefixes)
diff --git a/src/somef/schema/software_schema.py b/src/somef/schema/software_schema.py
index ba4f6d3a..4406fabd 100644
--- a/src/somef/schema/software_schema.py
+++ b/src/somef/schema/software_schema.py
@@ -29,7 +29,11 @@
         "@path": "citation",
         "@type": "xsd:string"
     },
-    "sd:hasInstallInstructions": {
+    "sd:hasAcknowledgments": {
+        "@path": "acknowledgments",
+        "@type": "xsd:string"
+    },
+    "sd:hasInstallationInstructions": {
         "@path": "installation",
         "@type": "xsd:string"
     },
@@ -47,7 +51,7 @@
         "@path": "usage",
         "@type": "xsd:string"
     },
-    "sd:downloadUrl": [
+    "sd:hasDownloadUrl": [
         {
             "@path": "downloadUrl",
             "@type": "xsd:anyURI"
@@ -79,9 +83,13 @@
         "@type": "xsd:string"
     },
     "sd:license": {
-        "@path": ["license", "url"],
+        "@path": ["license", "licenseFile", "url"],
         "@type": "xsd:anyURI"
     },
+    "sd:licenseText": {
+        "@path": ["licenseText"],
+        "@type": "xsd:string"
+    },
     "sd:keywords": {
         "@path": "topics",
         "@type": "xsd:string"
@@ -121,9 +129,17 @@
         "@type": "xsd:anyURI"
     },
     "sd:hasCodeOfConduct": {
-        "@path": ["code_of_conduct", "excerpt"],
+        "@path": "codeOfConduct",
         "@type": "xsd:anyURI"
     },
+    "sd:dateCreated": {
+        "@path": "dateCreated",
+        "@type": "xsd:dateTime"
+    },
+    "sd:dateModified": {
+        "@path": "dateModified",
+        "@type": "xsd:dateTime"
+    },
     "sd:hasAcknowledgement": {
         "@path": ["acknowledgement", "excerpt"],
         "@type": "xsd:string"
@@ -173,12 +189,63 @@
         "@path": "hasDocumentation",
         "@type": "xsd:anyURI"
     },
+    "sd:referencePublication": {
+        "@path": "arxivLinks",
+        "@type": "xsd:string"
+    },
     "sd:hasBuildFile": {
         "@path": "hasBuildFile",
         "@type": "xsd:string"
     },
-    "sd:hasExecutableNotebook": {
-        "@path": "hasExecutableNotebook",
+    "sd:identifier": {
+        "@path": "identifier",
+        "@type": "xsd:string"
+    },
+    "sd:issueTracker": {
+        "@path": "issueTracker",
+        "@type": "xsd:anyURI"
+    },
+    "sd:hasLongName": {
+        "@path": "long_title",
+        "@type": "xsd:string"
+    },
+    "sd:readme": {
+        "@path": "readme_url",
+        "@type": "xsd:anyURI"
+    },
+    "sd:contributor": {
+        "@path": "contributors",
+        "@type": "xsd:string"
+    },
+    "sd:contributingGuidelines": {
+        "@path": "contributingGuidelines",
         "@type": "xsd:string"
+    },
+    "sd:datePublished": {
+        "@path": "datePublished",
+        "@type": "xsd:dateTime"
+    },
+    "sd:hasExample": {
+        "@path": "executable_example",
+        "@type": "xsd:string"
+    },
+    "sd:hasSupportScriptLocation": {
+        "@path": "hasScriptFile",
+        "@type": "xsd:anyURI"
+    },
+    "sd:hasExecutableInstructions": [
+        {
+            "@path": "invocation",
+            "@type": "xsd:string"
+        },
+        {
+            "@path": "run",
+            "@type": "xsd:string"
+        }
+    ],
+    "sd:supportDetails": {
+        "@path": "support_channel",
+        "@type": "xsd:anyURI"
     }
-}
\ No newline at end of file
+
+}


Test patch:
diff --git a/src/somef/test/input-test.txt b/src/somef/test/input-test.txt
new file mode 100644
index 00000000..1529ca41
--- /dev/null
+++ b/src/somef/test/input-test.txt
@@ -0,0 +1,2 @@
+https://github.com/dgarijo/Widoco/
+htps:/github.com/dgarijo/Widoco/
\ No newline at end of file
diff --git a/src/somef/test/test_cli.py b/src/somef/test/test_cli.py
index 0d6a2458..5948328e 100644
--- a/src/somef/test/test_cli.py
+++ b/src/somef/test/test_cli.py
@@ -1,4 +1,5 @@
 import unittest
+import io
 
 from somef.cli import *
 
@@ -196,3 +197,38 @@ def test_issue_218(self):
         header['accept'] = 'application/vnd.github.v3+json'
         text, github_data = load_repository_metadata("https://github.com/pytorch/captum/tree/v0.4.0", header)
         assert len(github_data['citation']) > 0
+
+    def test_issue_224(self):
+        repo_data = cli_get_data(0.8, False, repo_url="https://github.com/tensorflow/tensorflow/tree/v2.6.0")
+        data_graph = DataGraph()
+        data_graph.add_somef_data(repo_data)
+        output = io.StringIO()
+        print(data_graph, file=output)
+        content = output.getvalue()
+        output.close()
+        assert content.find("sd:dateCreated") >= 0
+
+    def test_issue_280(self):
+        with open("input-test.txt", "r") as in_handle:
+            # get the line (with the final newline omitted) if the line is not empty
+            repo_list = [line[:-1] for line in in_handle if len(line) > 1]
+
+        # convert to a set to ensure uniqueness (we don't want to get the same data multiple times)
+        repo_set = set(repo_list)
+        # check if the urls in repo_set if are valids
+        remove_urls = []
+        for repo_elem in repo_set:
+            if not validators.url(repo_elem):
+                print("Not a valid repository url. Please check the url provided: " + repo_elem)
+                # repo_set.remove(repo_url)
+                remove_urls.append(repo_elem)
+        # remove non valid urls in repo_set
+        for remove_url in remove_urls:
+            repo_set.remove(remove_url)
+        assert len(repo_set) > 0
+
+    def test_issue_268(self):
+        header = {}
+        header['accept'] = 'application/vnd.github.v3+json'
+        text, github_data = load_repository_metadata("https://github.com/probot/probot/tree/v12.1.1", header)
+        assert len(github_data['licenseText']) > 0
\ No newline at end of file