Repository: ARMmbed/mbed-tools

Issue:
Fix `mbed-os.lib` for the following examples
### Description

<!--
A detailed description of what is being reported. Please include steps to reproduce the problem.

Things to consider sharing:
- What version of the package is being used (pip show mbed-tools)?
- What is the host platform and version (e.g. macOS 10.15.2, Windows 10, Ubuntu 18.04 LTS)?
-->

```
$mbed-tools -vvv clone mbed-os-example-ble
Cloning Mbed program 'mbed-os-example-ble'
Resolving program library dependencies.
DEBUG: Searching for mbed-os.lib file at path C:\Work\Workspace\trial\mbed-os-example-ble
DEBUG: Searching for mbed-os.lib file at path C:\Work\Workspace\trial
DEBUG: Searching for mbed-os.lib file at path C:\Work\Workspace
DEBUG: Searching for mbed-os.lib file at path C:\Work
DEBUG: No mbed-os.lib file found.
INFO: Cloning Mbed program from URL 'https://github.com/armmbed/mbed-os-example-ble'.
DEBUG: Failed checking if running in CYGWIN due to: FileNotFoundError(2, 'The system cannot find the file specified', None, 2, None)
DEBUG: Popen(['git', 'version'], cwd=c:\Work\Workspace\trial, universal_newlines=False, shell=None, istream=None)
DEBUG: Popen(['git', 'clone', '--progress', '-v', 'https://github.com/armmbed/mbed-os-example-ble', 'mbed-os-example-ble'], cwd=c:\Work\Workspace\trial, universal_newlines=True, shell=None, istream=None)
INFO: This program does not contain an mbed_app.json config file.
WARNING: No CMakeLists.txt found in the program root. Creating it...
ERROR: This repository does not contain a valid Mbed program at the top level. Cloned programs must contain an mbed-os.lib file containing the URL to the Mbed OS repository. It is possible you have cloned a repository containing multiple mbed-programs. If this is the case, you should cd to a directory containing a program before performing any other operations.
```

Following examples are effected
```
mbed-tools -vvv clone mbed-os-example-mbed-crypto
mbed-tools -vvv clone mbed-os-example-ble
mbed-tools -vvv clone mbed-os-example-nfc
```
mbed-os-example-ble example is the BLE_button one.

On

```
$mbed-tools --version 
3.2.2
```
### Issue request type

<!--
Please add only one `x` to one of the following types. Do not fill multiple types (split the issue otherwise).


For questions please use https://forums.mbed.com/
-->

- [ ] Enhancement
- [x ] Bug


Discussion:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 761e422..2e1058c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -99,7 +99,7 @@ matrix:
 
     - <<: *mbed-tools-test
       name: "Test ble example (NRF52840_DK)"
-      env: EXAMPLE_NAME=mbed-os-example-ble TARGET_NAME=NRF52840_DK SUBEXAMPLE_NAME=BLE_Button CACHE_NAME=ble-ble-button-NRF52840_DK
+      env: EXAMPLE_NAME=mbed-os-example-ble TARGET_NAME=NRF52840_DK SUBEXAMPLE_NAME=BLE_LED CACHE_NAME=ble-ble-led-NRF52840_DK
 
     - <<: *mbed-tools-test
       name: "Test cellular example (WIO_3G)"
diff --git a/docs/howto/mbed-tools-howto.md b/docs/howto/mbed-tools-howto.md
index ca880bc..d1ea748 100644
--- a/docs/howto/mbed-tools-howto.md
+++ b/docs/howto/mbed-tools-howto.md
@@ -207,7 +207,7 @@ Use CMake to build your application:
 
 We can use a single command which will configure (set up your target and toolchain) and build the project.
 
-    1. To build and configure:
+1. To build and configure:
 
     ```
     mbed-tools build -m <target> -t <toolchain>
@@ -221,7 +221,7 @@ We can use a single command which will configure (set up your target and toolcha
     mbed-tools build -m K64F -t GCC_ARM
     ```
 
-    1. To perform an iterative build on previously configured target:
+1. To perform an iterative build on previously configured target:
 
     ```
     mbed-tools build
diff --git a/news/202010261000.doc b/news/202010261000.doc
new file mode 100644
index 0000000..367a400
--- /dev/null
+++ b/news/202010261000.doc
@@ -0,0 +1,1 @@
+Fix formatting issue in docs
diff --git a/news/202010261100.misc b/news/202010261100.misc
new file mode 100644
index 0000000..cc6bfdb
--- /dev/null
+++ b/news/202010261100.misc
@@ -0,0 +1,1 @@
+Update CI to use BLE LED example
diff --git a/news/62.bugfix b/news/62.bugfix
new file mode 100644
index 0000000..56b12d0
--- /dev/null
+++ b/news/62.bugfix
@@ -0,0 +1,1 @@
+Do not check for mbed-os.lib in MbedProgram::from_url as it was making clone fail on examples containing multiple programs.
diff --git a/news/66.bugfix b/news/66.bugfix
new file mode 100644
index 0000000..f920f1e
--- /dev/null
+++ b/news/66.bugfix
@@ -0,0 +1,1 @@
+Fix https to ssh redirection
diff --git a/src/mbed_tools/project/_internal/libraries.py b/src/mbed_tools/project/_internal/libraries.py
index cc417bf..4e95694 100644
--- a/src/mbed_tools/project/_internal/libraries.py
+++ b/src/mbed_tools/project/_internal/libraries.py
@@ -42,6 +42,10 @@ class MbedLibReference:
         """
         raw_ref = self.reference_file.read_text().strip()
         url, sep, ref = raw_ref.partition("#")
+
+        if url.endswith("/"):
+            url = url[:-1]
+
         return git_utils.GitReference(repo_url=url, ref=ref)
 
 
@@ -50,7 +54,7 @@ class LibraryReferences:
     """Manages library references in an MbedProgram."""
 
     root: Path
-    ignore_paths: List[Path]
+    ignore_paths: List[str]
 
     def resolve(self) -> None:
         """Recursively clone all dependencies defined in .lib files."""
@@ -106,4 +110,4 @@ class LibraryReferences:
 
     def _in_ignore_path(self, lib_reference_path: Path) -> bool:
         """Check if a library reference is in a path we want to ignore."""
-        return any(p in lib_reference_path.parents for p in self.ignore_paths)
+        return any(p in lib_reference_path.parts for p in self.ignore_paths)
diff --git a/src/mbed_tools/project/_internal/project_data.py b/src/mbed_tools/project/_internal/project_data.py
index 2ea7022..b956e8b 100644
--- a/src/mbed_tools/project/_internal/project_data.py
+++ b/src/mbed_tools/project/_internal/project_data.py
@@ -113,8 +113,7 @@ class MbedProgramFiles:
 
         cmakelists_file = root_path / CMAKELISTS_FILE_NAME
         if not cmakelists_file.exists():
-            logger.warning("No CMakeLists.txt found in the program root. Creating it...")
-            render_cmakelists_template(cmakelists_file, root_path.stem)
+            logger.warning("No CMakeLists.txt found in the program root.")
 
         return cls(
             app_config_file=app_config,
diff --git a/src/mbed_tools/project/mbed_program.py b/src/mbed_tools/project/mbed_program.py
index 4c7fda7..65e94fb 100644
--- a/src/mbed_tools/project/mbed_program.py
+++ b/src/mbed_tools/project/mbed_program.py
@@ -26,7 +26,6 @@ class MbedProgram:
     """Represents an Mbed program.
 
     An `MbedProgram` consists of:
-        * A git repository
         * A copy of, or reference to, `MbedOS`
         * A set of `MbedProgramFiles`
         * A collection of references to external libraries, defined in .lib files located in the program source tree
@@ -42,38 +41,21 @@ class MbedProgram:
         self.files = program_files
         self.root = self.files.mbed_os_ref.parent
         self.mbed_os = mbed_os
-        self.lib_references = LibraryReferences(root=self.root, ignore_paths=[self.mbed_os.root])
+        self.lib_references = LibraryReferences(root=self.root, ignore_paths=[self.mbed_os.root.name])
 
     @classmethod
-    def from_url(cls, url: str, dst_path: Path, check_mbed_os: bool = True) -> "MbedProgram":
+    def from_url(cls, url: str, dst_path: Path) -> "MbedProgram":
         """Fetch an Mbed program from a remote URL.
 
         Args:
             url: URL of the remote program repository.
             dst_path: Destination path for the cloned program.
-
-        Raises:
-            ExistingProgram: `dst_path` already contains an Mbed program.
         """
-        if _tree_contains_program(dst_path):
-            raise ExistingProgram(
-                f"The destination path '{dst_path}' already contains an Mbed program. Please set the destination path "
-                "to an empty directory."
-            )
         logger.info(f"Cloning Mbed program from URL '{url}'.")
         git_utils.clone(url, dst_path)
-
         program_files = MbedProgramFiles.from_existing(dst_path)
-        if not program_files.mbed_os_ref.exists():
-            raise ProgramNotFound(
-                "This repository does not contain a valid Mbed program at the top level. "
-                "Cloned programs must contain an mbed-os.lib file containing the URL to the Mbed OS repository. It is "
-                "possible you have cloned a repository containing multiple mbed-programs. If this is the case, you "
-                "should cd to a directory containing a program before performing any other operations."
-            )
-
         try:
-            mbed_os = MbedOS.from_existing(dst_path / MBED_OS_DIR_NAME, check_mbed_os)
+            mbed_os = MbedOS.from_existing(dst_path / MBED_OS_DIR_NAME, False)
         except ValueError as mbed_err:
             raise MbedOSNotFound(f"{mbed_err}")
 
diff --git a/src/mbed_tools/project/project.py b/src/mbed_tools/project/project.py
index f8301cc..26e69b5 100644
--- a/src/mbed_tools/project/project.py
+++ b/src/mbed_tools/project/project.py
@@ -26,7 +26,7 @@ def clone_project(url: str, dst_path: Any = None, recursive: bool = False) -> No
     if not dst_path:
         dst_path = pathlib.Path(git_data["dst_path"])
 
-    program = MbedProgram.from_url(url, dst_path, check_mbed_os=False)
+    program = MbedProgram.from_url(url, dst_path)
     if recursive:
         program.resolve_libraries()
 


Test patch:
diff --git a/tests/project/_internal/test_libraries.py b/tests/project/_internal/test_libraries.py
index 9bed897..76ee133 100644
--- a/tests/project/_internal/test_libraries.py
+++ b/tests/project/_internal/test_libraries.py
@@ -18,7 +18,7 @@ class TestLibraryReferences(TestCase):
         lib = make_mbed_lib_reference(fs_root, ref_url="https://git")
         mock_clone.side_effect = lambda url, dst_dir: dst_dir.mkdir()
 
-        lib_refs = LibraryReferences(fs_root, ignore_paths=[fs_root / "mbed-os"])
+        lib_refs = LibraryReferences(fs_root, ignore_paths=["mbed-os"])
         lib_refs.resolve()
 
         mock_clone.assert_called_once_with(lib.get_git_reference().repo_url, lib.source_code_path)
@@ -40,7 +40,7 @@ class TestLibraryReferences(TestCase):
             lib2.source_code_path.mkdir(),
         )
 
-        lib_refs = LibraryReferences(fs_root, ignore_paths=[fs_root / "mbed-os"])
+        lib_refs = LibraryReferences(fs_root, ignore_paths=["mbed-os"])
         lib_refs.resolve()
 
         self.assertTrue(lib.is_resolved())
@@ -53,7 +53,7 @@ class TestLibraryReferences(TestCase):
         fs_root = pathlib.Path(fs, "foo")
         make_mbed_lib_reference(fs_root, ref_url="https://git", resolved=True)
 
-        lib_refs = LibraryReferences(fs_root, ignore_paths=[fs_root / "mbed-os"])
+        lib_refs = LibraryReferences(fs_root, ignore_paths=["mbed-os"])
         lib_refs.checkout(force=False)
 
         mock_checkout.assert_not_called()
@@ -65,7 +65,7 @@ class TestLibraryReferences(TestCase):
         fs_root = pathlib.Path(fs, "foo")
         lib = make_mbed_lib_reference(fs_root, ref_url="https://git#lajdhalk234", resolved=True)
 
-        lib_refs = LibraryReferences(fs_root, ignore_paths=[fs_root / "mbed-os"])
+        lib_refs = LibraryReferences(fs_root, ignore_paths=["mbed-os"])
         lib_refs.checkout(force=False)
 
         mock_checkout.assert_called_once_with(mock_init.return_value, lib.get_git_reference().ref, force=False)
@@ -78,7 +78,7 @@ class TestLibraryReferences(TestCase):
         make_mbed_lib_reference(fs_root, ref_url="https://git")
         mock_clone.side_effect = lambda url, dst_dir: dst_dir.mkdir()
 
-        lib_refs = LibraryReferences(fs_root, ignore_paths=[fs_root / "mbed-os"])
+        lib_refs = LibraryReferences(fs_root, ignore_paths=["mbed-os"])
         lib_refs.resolve()
 
         mock_checkout.assert_not_called()
@@ -91,7 +91,19 @@ class TestLibraryReferences(TestCase):
         lib = make_mbed_lib_reference(fs_root, ref_url="https://git#lajdhalk234")
         mock_clone.side_effect = lambda url, dst_dir: dst_dir.mkdir()
 
-        lib_refs = LibraryReferences(fs_root, ignore_paths=[fs_root / "mbed-os"])
+        lib_refs = LibraryReferences(fs_root, ignore_paths=["mbed-os"])
         lib_refs.resolve()
 
         mock_checkout.assert_called_once_with(None, lib.get_git_reference().ref)
+
+    @patchfs
+    @mock.patch("mbed_tools.project._internal.git_utils.checkout", autospec=True)
+    @mock.patch("mbed_tools.project._internal.git_utils.init", autospec=True)
+    def test_does_not_resolve_references_in_ignore_paths(self, mock_init, mock_checkout, mock_clone, fs):
+        fs_root = pathlib.Path(fs, "mbed-os")
+        make_mbed_lib_reference(fs_root, ref_url="https://git#lajdhalk234")
+
+        lib_refs = LibraryReferences(fs_root, ignore_paths=["mbed-os"])
+        lib_refs.resolve()
+
+        mock_clone.assert_not_called()
diff --git a/tests/project/_internal/test_mbed_program.py b/tests/project/_internal/test_mbed_program.py
index fee3f18..39f8edc 100644
--- a/tests/project/_internal/test_mbed_program.py
+++ b/tests/project/_internal/test_mbed_program.py
@@ -61,44 +61,25 @@ class TestInitialiseProgram(TestCase):
         self.assertEqual(program.files, MbedProgramFiles.from_existing(program_root))
 
     @patchfs
-    def test_from_url_raises_if_dest_dir_contains_program(self, fs):
+    @mock.patch("mbed_tools.project.mbed_program.git_utils.clone", autospec=True)
+    def test_from_url_returns_valid_program(self, mock_clone, fs):
         fs_root = pathlib.Path(fs, "foo")
-        make_mbed_program_files(fs_root)
         url = "https://valid"
-
-        with self.assertRaises(ExistingProgram):
-            MbedProgram.from_url(url, fs_root)
-
-    @patchfs
-    @mock.patch("mbed_tools.project._internal.git_utils.clone", autospec=True)
-    def test_from_url_raises_if_cloned_repo_is_not_program(self, mock_repo, fs):
-        fs_root = pathlib.Path(fs, "foo")
-        fs_root.mkdir()
-        url = "https://validrepo.com"
-        mock_repo.side_effect = lambda url, dst_dir: dst_dir.mkdir()
-
-        with self.assertRaises(ProgramNotFound):
-            MbedProgram.from_url(url, fs_root / "corrupt-prog")
-
-    @patchfs
-    @mock.patch("mbed_tools.project._internal.git_utils.clone", autospec=True)
-    def test_from_url_raises_if_check_mbed_os_is_true_and_mbed_os_dir_nonexistent(self, mock_clone, fs):
-        fs_root = pathlib.Path(fs, "foo")
-        url = "https://validrepo.com"
         mock_clone.side_effect = lambda *args: make_mbed_program_files(fs_root)
+        program = MbedProgram.from_url(url, fs_root)
 
-        with self.assertRaises(MbedOSNotFound):
-            MbedProgram.from_url(url, fs_root, check_mbed_os=True)
+        self.assertEqual(program.files, MbedProgramFiles.from_existing(fs_root))
+        mock_clone.assert_called_once_with(url, fs_root)
 
     @patchfs
     @mock.patch("mbed_tools.project.mbed_program.git_utils.clone", autospec=True)
-    def test_from_url_returns_valid_program(self, mock_clone, fs):
+    def test_from_url_does_not_raise_with_odd_program_layout(self, mock_clone, fs):
         fs_root = pathlib.Path(fs, "foo")
-        url = "https://valid"
-        mock_clone.side_effect = lambda *args: make_mbed_program_files(fs_root)
-        program = MbedProgram.from_url(url, fs_root, False)
+        fs_root.mkdir()
+        url = "https://nested"
+        mock_clone.side_effect = lambda *args: make_mbed_program_files(fs_root / "nested")
+        MbedProgram.from_url(url, fs_root)
 
-        self.assertEqual(program.files, MbedProgramFiles.from_existing(fs_root))
         mock_clone.assert_called_once_with(url, fs_root)
 
     @patchfs
diff --git a/tests/project/_internal/test_project_data.py b/tests/project/_internal/test_project_data.py
index 51c0fad..aaa59d1 100644
--- a/tests/project/_internal/test_project_data.py
+++ b/tests/project/_internal/test_project_data.py
@@ -10,7 +10,6 @@ from unittest import TestCase, mock
 from mbed_tools.project._internal.project_data import (
     MbedProgramFiles,
     MbedOS,
-    CMAKELISTS_FILE_NAME,
     MAIN_CPP_FILE_NAME,
 )
 from tests.project.factories import make_mbed_lib_reference, make_mbed_program_files, make_mbed_os_files, patchfs
@@ -65,28 +64,6 @@ class TestMbedProgramFiles(TestCase):
         self.assertTrue(program.mbed_os_ref.exists())
         self.assertTrue(program.cmakelists_file.exists())
 
-    @patchfs
-    def test_from_existing_calls_render_cmakelists_template_with_program_name_if_file_nonexistent(self, fs):
-        with mock.patch("mbed_tools.project._internal.project_data.render_cmakelists_template") as render_cmake:
-            root = pathlib.Path(fs, "foo")
-            make_mbed_program_files(root)
-            render_cmake.return_value = "foo"
-
-            program_files = MbedProgramFiles.from_existing(root)
-
-            render_cmake.assert_called_with(program_files.cmakelists_file, root.stem)
-
-    @patchfs
-    def test_from_existing_skips_rendering_cmake_template_if_file_exists(self, fs):
-        with mock.patch("mbed_tools.project._internal.project_data.render_cmakelists_template") as render_cmake:
-            root = pathlib.Path(fs, "foo")
-            make_mbed_program_files(root)
-            (root / CMAKELISTS_FILE_NAME).touch()
-
-            MbedProgramFiles.from_existing(root)
-
-            render_cmake.assert_not_called()
-
 
 class TestMbedLibReference(TestCase):
     @patchfs
@@ -108,13 +85,15 @@ class TestMbedLibReference(TestCase):
         root = pathlib.Path(fs, "foo")
         url = "https://github.com/mylibrepo"
         ref = "latest"
-        full_ref = f"{url}#{ref}"
-        lib = make_mbed_lib_reference(root, ref_url=full_ref)
+        references = [f"{url}#{ref}", f"{url}/#{ref}"]
+
+        for full_ref in references:
+            lib = make_mbed_lib_reference(root, ref_url=full_ref)
 
-        reference = lib.get_git_reference()
+            reference = lib.get_git_reference()
 
-        self.assertEqual(reference.repo_url, url)
-        self.assertEqual(reference.ref, ref)
+            self.assertEqual(reference.repo_url, url)
+            self.assertEqual(reference.ref, ref)
 
 
 class TestMbedOS(TestCase):
diff --git a/tests/project/factories.py b/tests/project/factories.py
index 49084bb..f1040d7 100644
--- a/tests/project/factories.py
+++ b/tests/project/factories.py
@@ -5,6 +5,11 @@
 from functools import wraps
 from tempfile import TemporaryDirectory
 from mbed_tools.project._internal.libraries import MbedLibReference
+from mbed_tools.project._internal.project_data import (
+    CMAKELISTS_FILE_NAME,
+    APP_CONFIG_FILE_NAME,
+    MBED_OS_REFERENCE_FILE_NAME,
+)
 
 
 def patchfs(func):
@@ -16,12 +21,13 @@ def patchfs(func):
     return wrapper
 
 
-def make_mbed_program_files(root, config_file_name="mbed_app.json"):
+def make_mbed_program_files(root, config_file_name=APP_CONFIG_FILE_NAME):
     if not root.exists():
         root.mkdir()
 
-    (root / "mbed-os.lib").touch()
+    (root / MBED_OS_REFERENCE_FILE_NAME).touch()
     (root / config_file_name).touch()
+    (root / CMAKELISTS_FILE_NAME).touch()
 
 
 def make_mbed_lib_reference(root, name="mylib.lib", resolved=False, ref_url=None):
diff --git a/tests/project/test_mbed_project.py b/tests/project/test_mbed_project.py
index 24efd3a..12f8247 100644
--- a/tests/project/test_mbed_project.py
+++ b/tests/project/test_mbed_project.py
@@ -32,13 +32,13 @@ class TestCloneProject(TestCase):
         url = "https://git.com/gitorg/repo"
         clone_project(url, recursive=False)
 
-        mock_program.from_url.assert_called_once_with(url, pathlib.Path(url.rsplit("/", maxsplit=1)[-1]), False)
+        mock_program.from_url.assert_called_once_with(url, pathlib.Path(url.rsplit("/", maxsplit=1)[-1]))
 
     def test_resolves_libs_when_recursive_is_true(self, mock_program):
         url = "https://git.com/gitorg/repo"
         clone_project(url, recursive=True)
 
-        mock_program.from_url.assert_called_once_with(url, pathlib.Path(url.rsplit("/", maxsplit=1)[-1]), False)
+        mock_program.from_url.assert_called_once_with(url, pathlib.Path(url.rsplit("/", maxsplit=1)[-1]))
         mock_program.from_url.return_value.resolve_libraries.assert_called_once()