Repository: KnowledgeCaptureAndDiscovery/somef (language: python)

Issue:
Error in logo extraction
In https://raw.githubusercontent.com/oeg-upm/licensius/master/README.md
It detects the jitpack URL as the logo because it's called `licensius.svg`.

We could remove URLs from jitpack (and other repos when we find them)
Fixed issue #443



Discussion / hints:

Fix patch:
diff --git a/src/somef/parser_somef.py b/src/somef/parser_somef.py
index 635cfbfb..5b3600c0 100644
--- a/src/somef/parser_somef.py
+++ b/src/somef/parser_somef.py
@@ -407,11 +407,13 @@ def remove_tags_new(header_parents):
     output = {}
     regex = r'<[^<>]+>'
     for key in header_parents.keys():
-        new_key = re.sub(regex, '', key)
+        #new_key = re.sub(regex, '', key)
+        new_key = key[4:len(key)-5]
         new_list = []
         for value in header_parents[key]:
             if key != value:
-                new_value = re.sub(regex, '', value)
+                #new_value = re.sub(regex, '', value)
+                new_value = value[4:len(value)-5]
                 new_list.append(new_value)
         output[new_key] = new_list
 
@@ -451,7 +453,7 @@ def is_header(header):
 
 def get_tag_content(header):
     init = header.index(">")
-    end = header.index("</h")
+    end = header.rindex("</h")
     return replace_html_tags(header[init+1:end])
 
 
diff --git a/src/somef/regular_expressions.py b/src/somef/regular_expressions.py
index a22efe09..3466b7ea 100644
--- a/src/somef/regular_expressions.py
+++ b/src/somef/regular_expressions.py
@@ -221,7 +221,11 @@ def extract_images(unfiltered_text, repo_url, local_repo):
     img_md = re.findall(r"!\[[^\]]*\]\((.*?)?\)", html_text)
     result = [_.start() for _ in re.finditer("<img ", html_text)]
     for img in img_md:
-        if not has_logo and repo:
+        # if the image contains jitpack.io, the element is not processed
+        if img.find("jitpack.io") > 0 or img.find("/badge") >= 0 or img.find("/travis-ci.org") >= 0 \
+                or img.find("img.shields.io") >= 0:
+            None
+        elif not has_logo and repo:
             start = img.rindex("/")
             if img.find(repo_name, start) > 0:
                 logo = rename_github_image(img, repo_url, local_repo)
@@ -242,8 +246,14 @@ def extract_images(unfiltered_text, repo_url, local_repo):
         init = html_text.find("src=\"", index_img)
         end = html_text.find("\"", init + 5)
         img = html_text[init + 5:end]
-        if not has_logo and repo:
-            start = img.rindex("/")
+        # if the image contains jitpack.io, the element is not processed
+        if img.find("jitpack.io") > 0 or img.find("/badge") >= 0 or img.find("/travis-ci.org") >= 0 \
+                or img.find("img.shields.io") >= 0:
+            None
+        elif not has_logo and repo:
+            start = 0
+            if img.find("/") > 0:
+                start = img.rindex("/")
             image_name = img[start:]
             if image_name.find(repo_name) > 0 or image_name.upper().find("LOGO") > 0:
                 logo = rename_github_image(img, repo_url, local_repo)


Test patch:
diff --git a/src/somef/test/test_cli.py b/src/somef/test/test_cli.py
index 2b387743..6b8a04e9 100644
--- a/src/somef/test/test_cli.py
+++ b/src/somef/test/test_cli.py
@@ -884,4 +884,70 @@ def test_issue_428(self):
         data = text_file.read()
         text_file.close()
         assert data.find("Manim is an engine for precise programmatic animations, designed for creating explanatory math videos.") > 0
-        os.remove(test_data_path + "test-428.json")
\ No newline at end of file
+        os.remove(test_data_path + "test-428.json")
+
+    def test_issue_443(self):
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url=None,
+                    local_repo=None,
+                    doc_src=test_data_path + "README-epw2rdf-contents.md",
+                    in_file=None,
+                    output=test_data_path + "test-443.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
+        text_file = open(test_data_path + "test-443.json", "r")
+        data = text_file.read()
+        text_file.close()
+        json_content = json.loads(data)
+        installation = json_content['installation']
+        assert installation is not None
+        os.remove(test_data_path + "test-443.json")
+
+    def test_issue_443_2(self):
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/oeg-upm/awesome-semantic-web",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-443.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
+        text_file = open(test_data_path + "test-443.json", "r")
+        data = text_file.read()
+        text_file.close()
+        json_content = json.loads(data)
+        installation = json_content['installation']
+        assert installation is not None
+        os.remove(test_data_path + "test-443.json")
+
+    def test_issue_443_3(self):
+        cli.run_cli(threshold=0.8,
+                    ignore_classifiers=False,
+                    repo_url="https://github.com/oeg-upm/pcake",
+                    local_repo=None,
+                    doc_src=None,
+                    in_file=None,
+                    output=test_data_path + "test-443.json",
+                    graph_out=None,
+                    graph_format="turtle",
+                    codemeta_out=None,
+                    pretty=True,
+                    missing=True,
+                    readme_only=False)
+        text_file = open(test_data_path + "test-443.json", "r")
+        data = text_file.read()
+        text_file.close()
+        json_content = json.loads(data)
+        description = json_content['description']
+        assert description is not None
+        os.remove(test_data_path + "test-443.json")
\ No newline at end of file
diff --git a/src/somef/test/test_data/README-PPool.md b/src/somef/test/test_data/README-PPool.md
new file mode 100644
index 00000000..cdce2d1f
--- /dev/null
+++ b/src/somef/test/test_data/README-PPool.md
@@ -0,0 +1,26 @@
+# PPool
+PPool is a Pool for Processes in Python with locks. The main advantage is the ability to use `Lock`s from  `multiprocessing`
+
+[![PyPI version](https://badge.fury.io/py/PPool.svg)](https://badge.fury.io/py/PPool)
+
+# Install
+```
+pip install PPool
+```
+
+# pypi
+`https://pypi.org/project/PPool/`
+
+
+# Example
+```
+from PPool.Pool import Pool
+params = [('A', 2),('B', 3),('C', 4), ('D', 5)]
+
+def foo(name, num):
+    print(name+str(num))
+
+pool = Pool(max_num_of_processes=3, func=foo, params_list=params)
+
+pool.run()
+```
\ No newline at end of file
diff --git a/src/somef/test/test_data/README-epw2rdf-contents.md b/src/somef/test/test_data/README-epw2rdf-contents.md
new file mode 100644
index 00000000..e1194594
--- /dev/null
+++ b/src/somef/test/test_data/README-epw2rdf-contents.md
@@ -0,0 +1,13 @@
+
+# **Note  epw2rdf contents** <h0>
+
+## **Install libraries for Python3** <h2>
+
+The `requirements.txt` file should list all Python libraries that your notebooks
+depend on, and they will be installed using:
+
+```
+pip install -r requirements.txt
+```
+
+Should also install Java
\ No newline at end of file
diff --git a/src/somef/test/test_data/README-epw2rdf.md b/src/somef/test/test_data/README-epw2rdf.md
new file mode 100644
index 00000000..bd8c2310
--- /dev/null
+++ b/src/somef/test/test_data/README-epw2rdf.md
@@ -0,0 +1,10 @@
+# **Note** <h0>
+
+## **Install libraries for Python3** <h2>
+
+The `requirements.txt` file should list all Python libraries that your notebooks
+depend on, and they will be installed using:
+
+```
+pip install -r requirements.txt
+```
\ No newline at end of file
diff --git a/src/somef/test/test_data/README-licensius.md b/src/somef/test/test_data/README-licensius.md
new file mode 100644
index 00000000..0abf9cd4
--- /dev/null
+++ b/src/somef/test/test_data/README-licensius.md
@@ -0,0 +1,59 @@
+# licensius
+
+[![](https://jitpack.io/v/oeg-upm/licensius.svg)](https://jitpack.io/#oeg-upm/licensius)
+
+Collection of RDF licensing projects. See: 
+- https://github.com/oeg-upm/licensius/tree/master/odrlapi Evaluator for the newest ODRL spec
+- https://github.com/oeg-upm/licensius/tree/master/rdflicense Collection of common licenses as RDF
+- https://github.com/oeg-upm/licensius/tree/master/jodrlapi Java library to manipulate ODRL 2.1 expressions
+- https://github.com/oeg-upm/licensius/tree/master/licensius-core Java library to find common licenses in HTML web pages
+- https://github.com/oeg-upm/licensius/tree/master/licensius-ws Java HTTP Rest API to find common licenses in HTML web pages
+
+See also: http://licensius.com
+
+
+## odrlapi Evaluator for the newest ODRL spec
+
+This software project is a an evalutor of ODRL expressions according to the W3C ODRL Recommendations:
+
+- ODRL Information Model https://www.w3.org/TR/odrl-model/
+- ODRL Vocabulary and Expression 2.2 https://www.w3.org/TR/odrl-vocab/
+
+You can see the evaluator here: http://odrlapi.appspot.com/
+The code is here: https://github.com/oeg-upm/licensius/tree/master/odrlapi 
+
+## rdflicense Collection of common licenses as RDF
+
+This is a collection of common licenses as RDF. 
+http://rdflicense.linkeddata.es/
+
+Source doe here: https://github.com/oeg-upm/licensius/tree/master/rdflicense
+
+See also 
+A Dataset of RDF Licenses, V. Rodríguez-Doncel, S. Villata, A. Gómez-Pérez, in Proc. of the 27th Int. Conf. on Legal Knowledge and Information System (JURIX), R. Hoekstra (Ed.), ISBN 978-1-61499-467-1, pp. 187-189, IOS Press, (2014) [https://delicias.dia.fi.upm.es/members/vrodriguez/pdf/2014.11.jurix.dataset.pdf]
+
+
+## jodrlapi Java library to manipulate ODRL 2.1 expressions
+
+- https://github.com/oeg-upm/licensius/tree/master/jodrlapi Java library to manipulate ODRL 2.1 expressions
+
+Java API to manipulate ODRL2 expressions.
+This API permits creating ODRL expressions in its RDF serialization by using simple Java methods.
+For more info, see http://oeg-upm.github.io/odrlapi/
+
+## licensius-core Java library to find common licenses in HTML web pages
+
+https://github.com/oeg-upm/licensius/tree/master/licensius-core
+
+## licensius-web Java HTTP Rest API to find common licenses in HTML web pages
+
+See the API here: http://www.licensius.com/apidoc/index.html
+https://github.com/oeg-upm/licensius/tree/master/licensius-ws 
+
+## other projects pending to be migrated here:
+Linked Data Rights - http://purl.oclc.org/NET/ldr/ns#
+Rights Information for Language Resources - http://purl.org/NET/ms-rights#
+
+
+
+Víctor Rodríguez Doncel 
diff --git a/src/somef/test/test_data/README-loupe-api.md b/src/somef/test/test_data/README-loupe-api.md
new file mode 100644
index 00000000..27c2b19c
--- /dev/null
+++ b/src/somef/test/test_data/README-loupe-api.md
@@ -0,0 +1,13 @@
+loupe-api
+=========
+[![Build Status](https://travis-ci.org/ldp4j/ldp4j.svg?branch=master)](https://travis-ci.org/ldp4j/ldp4j)
+[![GitHub license](https://img.shields.io/badge/license-Apache%202-blue.svg)](https://raw.githubusercontent.com/nandana/ld-sniffer/master/LICENSE)
+[![Java version](https://img.shields.io/badge/java-1.8-green.svg)](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
+[![GitHub issues](https://img.shields.io/github/issues/nandana/loupe-api.svg)](https://github.com/nandana/ld-sniffer/issues)
+[![Maintenance](https://img.shields.io/maintenance/yes/2017.svg?maxAge=2592000?style=plastic)](https://twitter.com/nandanamihindu)
+
+The Loupe API is a web service for proifiling Linked Data and currently three operations are available as illustrated in following figure.
+
+![Swagger Docs](https://raw.githubusercontent.com/nandana/loupe-api/master/src/main/resources/img/api-docs.png)
+
+Loupe API is available online at [http://api.loupe.linkeddata.es/](http://api.loupe.linkeddata.es/)
diff --git a/src/somef/test/test_regular_expressions.py b/src/somef/test/test_regular_expressions.py
index 029757e8..19939b7c 100644
--- a/src/somef/test/test_regular_expressions.py
+++ b/src/somef/test/test_regular_expressions.py
@@ -194,3 +194,23 @@ def test_issue_427(self):
             logo, images = regular_expressions.extract_images(test_text, None, test_data_repositories + "Widoco")
             assert (logo.find('test_data') > 0)
 
+    def test_issue_446(self):
+        """Test that jitpack URLs are not returned"""
+        with open(test_data_path + "README-licensius.md", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, "https://github.com/oeg-upm/licensius/", None)
+            assert (logo == "")
+
+    def test_issue_446_1(self):
+        """Test that jitpack URLs are not returned"""
+        with open(test_data_path + "README-PPool.md", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, "https://github.com/oeg-upm/PPool/", None)
+            assert (logo == "" and len(images) == 0)
+
+    def test_issue_446_2(self):
+        """Test that jitpack URLs are not returned"""
+        with open(test_data_path + "README-loupe-api.md", "r") as data_file:
+            test_text = data_file.read()
+            logo, images = regular_expressions.extract_images(test_text, "https://github.com/oeg-upm/PPool/", None)
+            assert (logo == "" and len(images) == 1)
\ No newline at end of file