Repository: KnowledgeCaptureAndDiscovery/somef (language: python)

Issue:
Detect Docker compose files
These are useful for concatenating services, and we should extract them too. These files are usually docker-compose.yml files.
Codemeta export errors
I have noticed the following:
- Issues has an incorrect URL
- Code repo uses the git+ format instead of the repo URL
- Download URL should not be there if there are no releases to download.
- Logos do not appear. These could be added as "image" which is a schema.org property


Discussion / hints:

Fix patch:
diff --git a/src/somef/cli.py b/src/somef/cli.py
index 76e0f4ed..95a6a191 100644
--- a/src/somef/cli.py
+++ b/src/somef/cli.py
@@ -250,6 +250,10 @@ def get_path(obj, path):
     filtered_resp = {}
     if not ignore_github_metadata:
         filtered_resp = do_crosswalk(general_resp, github_crosswalk_table)
+        if "issueTracker" in filtered_resp:
+            issueTracker = filtered_resp["issueTracker"]
+            issueTracker = issueTracker.replace("{/number}","")
+            filtered_resp["issueTracker"] = issueTracker
 
     # add download URL
     filtered_resp["downloadUrl"] = f"https://github.com/{owner}/{repo_name}/releases"
@@ -374,7 +378,7 @@ def get_path(obj, path):
         for dirpath, dirnames, filenames in os.walk(repo_dir):
             repo_relative_path = os.path.relpath(dirpath, repo_dir)
             for filename in filenames:
-                if filename == "Dockerfile":
+                if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
                     dockerfiles.append(os.path.join(repo_relative_path, filename))
                     #dockerfiles.append(repo_relative_path + "/" + filename)
                 if filename.lower().endswith(".ipynb"):
@@ -734,7 +738,7 @@ def get_path(obj, path):
         for dirpath, dirnames, filenames in os.walk(repo_dir):
             repo_relative_path = os.path.relpath(dirpath, repo_dir)
             for filename in filenames:
-                if filename == "Dockerfile":
+                if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
                     # dockerfiles.append(os.path.join(repo_relative_path, filename))
                     dockerfiles.append(repo_relative_path + "/" + filename)
                 if filename.lower().endswith(".ipynb"):
@@ -871,7 +875,7 @@ def load_local_repository_metadata(local_repo):
     for dirpath, dirnames, filenames in os.walk(repo_dir):
         repo_relative_path = os.path.relpath(dirpath, repo_dir)
         for filename in filenames:
-            if filename == "Dockerfile":
+            if filename == "Dockerfile" or filename.lower() == "docker-compose.yml":
                 dockerfiles.append(os.path.join(repo_dir, repo_relative_path, filename))
             if filename.lower().endswith(".ipynb"):
                 notebooks.append(os.path.join(repo_dir, repo_relative_path, filename))
@@ -1779,6 +1783,22 @@ def format_output(git_data, repo_data, gitlab_url=False):
                 if i == 'hasExecutableNotebook':
                     repo_data[i] = {'excerpt': git_data[i], 'confidence': [1.0], 'technique': 'File Exploration',
                                     'format': 'jupyter notebook'}
+                elif i == 'hasBuildFile':
+                    docker_files = []
+                    docker_compose = []
+                    for data in git_data[i]:
+                        if data.lower().endswith('docker-compose.yml'):
+                            docker_compose.append(data)
+                        else:
+                            docker_files.append(data)
+                    repo_data[i] = []
+                    if len(docker_files) > 0:
+                        repo_data[i].insert(0, {'excerpt': docker_files, 'confidence': [1.0], 'technique': 'File Exploration',
+                                    'format': 'Docker file'})
+                    if len(docker_compose) > 0:
+                        repo_data[i].insert(0, {'excerpt': docker_compose, 'confidence': [1.0], 'technique': 'File Exploration',
+                                    'format': 'Docker compose file'})
+                    #TODO docker
                 else:
                     repo_data[i] = {'excerpt': git_data[i], 'confidence': [1.0], 'technique': 'File Exploration'}
             elif git_data[i] != "" and git_data[i] != []:
@@ -1886,7 +1906,7 @@ def average_confidence(x):
     if "license" in repo_data:
         codemeta_output["license"] = data_path(["license", "excerpt"])
     if code_repository is not None:
-        codemeta_output["codeRepository"] = "git+" + code_repository + ".git"
+        codemeta_output["codeRepository"] = code_repository
         codemeta_output["issueTracker"] = code_repository + "/issues"
     if "dateCreated" in repo_data:
         codemeta_output["dateCreated"] = format_date(data_path(["dateCreated", "excerpt"]))
@@ -1896,6 +1916,8 @@ def average_confidence(x):
         codemeta_output["downloadUrl"] = data_path(["downloadUrl", "excerpt"])
     if "name" in repo_data:
         codemeta_output["name"] = data_path(["name", "excerpt"])
+    if "logo" in repo_data:
+        codemeta_output["logo"] = data_path(["logo", "excerpt"])
     if "releases" in repo_data:
         codemeta_output["releaseNotes"] = release_path(["body"])
         codemeta_output["version"] = release_path(["tag_name"])


Test patch:
diff --git a/src/somef/test/test_cli.py b/src/somef/test/test_cli.py
index b7e80399..8e29444c 100644
--- a/src/somef/test/test_cli.py
+++ b/src/somef/test/test_cli.py
@@ -592,6 +592,7 @@ def test_issue_378_2(self):
 
     def test_issue_389(self):
         text, github_data = load_local_repository_metadata(test_data_repositories + "wot-hive")
+        print(github_data)
         assert len(github_data['hasBuildFile']) > 0
 
     def test_issue_260(self):
@@ -1008,4 +1009,88 @@ def test_issue_136_1(self):
         data = text_file.read()
         text_file.close()
         assert data.find("doi") >= 0
-        os.remove(test_data_path + "test-136-1.json")
\ No newline at end of file
+        os.remove(test_data_path + "test-136-1.json")
+
+    def test_issue_353(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url="https://github.com/proycon/analiticcl",
+                local_repo=None,
+                doc_src=None,
+                in_file=None,
+                output=test_data_path + "test-353.json",
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=None,
+                pretty=True,
+                missing=True,
+                readme_only=False)
+        text_file = open(test_data_path + "test-353.json", "r")
+        data = text_file.read()
+        text_file.close()
+        assert data.find("description") >= 0
+        os.remove(test_data_path + "test-353.json")
+
+    def test_issue_366(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url=None,
+                local_repo=test_data_repositories + "wot-hive",
+                doc_src=None,
+                in_file=None,
+                output=test_data_path + "test-366.json",
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=None,
+                pretty=True,
+                missing=True,
+                readme_only=False)
+        text_file = open(test_data_path + "test-366.json", "r")
+        data = text_file.read()
+        text_file.close()
+        assert data.find("Docker file") >= 0
+        os.remove(test_data_path + "test-366.json")
+
+    def test_issue_417(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url="https://github.com/dgarijo/Widoco",
+                local_repo=None,
+                doc_src=None,
+                in_file=None,
+                output=None,
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=test_data_path + "test-417.json-ld",
+                pretty=True,
+                missing=True,
+                readme_only=False)
+        text_file = open(test_data_path + "test-417.json-ld", "r")
+        data = text_file.read()
+        text_file.close()
+        json_content = json.loads(data)
+        issue_tracker = json_content['issueTracker']
+        assert issue_tracker == 'https://api.github.com/repos/dgarijo/Widoco/issues'
+        os.remove(test_data_path + "test-417.json-ld")
+
+    def test_issue_382(self):
+        run_cli(threshold=0.8,
+                ignore_classifiers=False,
+                repo_url="https://github.com/oeg-upm/mapeathor",
+                local_repo=None,
+                doc_src=None,
+                in_file=None,
+                output=test_data_path + "test-382.json",
+                graph_out=None,
+                graph_format="turtle",
+                codemeta_out=None,
+                pretty=True,
+                missing=True,
+                readme_only=False)
+        text_file = open(test_data_path + "test-382.json", "r")
+        data = text_file.read()
+        text_file.close()
+        json_content = json.loads(data)
+        contact = json_content['contact']
+        assert contact != None
+        os.remove(test_data_path + "test-382.json")
\ No newline at end of file
diff --git a/src/somef/test/test_data/repositories/wot-hive/docker/docker-compose.yml b/src/somef/test/test_data/repositories/wot-hive/docker/docker-compose.yml
new file mode 100644
index 00000000..fb17a7f9
--- /dev/null
+++ b/src/somef/test/test_data/repositories/wot-hive/docker/docker-compose.yml
@@ -0,0 +1,26 @@
+version: "3.3"
+
+services:
+
+  traefik:
+    image: "traefik:v2.6"
+    container_name: "traefik"
+    command:
+      #- "--log.level=DEBUG"
+      - "--api.insecure=true"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.web.address=:80"
+    ports:
+      - "80:80"
+      - "8080:8080"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+
+  whoami:
+    image: "traefik/whoami"
+    container_name: "simple-service"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.whoami.rule=Host(`whoami.localhost`)"
+      - "traefik.http.routers.whoami.entrypoints=web"
\ No newline at end of file