Repository: ros-infrastructure/rosdistro

Issue:
KeyError with SourceDependencyWalker
Seeing these in our builds since updating to the current source master:

```
  File "env/local/lib/python2.7/site-packages/rosinstall_generator/distro.py", line 87, in get_recursive_dependencies
    dependencies |= walker.get_recursive_depends(pkg_name, ['buildtool', 'build', 'run', 'test'], ros_packages_only=True, ignore_pkgs=dependencies | excludes, limit_depth=limit_depth)
  File "env/local/lib/python2.7/site-packages/rosdistro/dependency_walker.py", line 78, in get_recursive_depends
    deps = self.get_depends(next_pkg_to_check, depend_type, ros_packages_only=ros_packages_only)
  File "env/local/lib/python2.7/site-packages/rosdistro/dependency_walker.py", line 61, in get_depends
    deps = self._get_dependencies(pkg_name, depend_type)
  File "env/local/lib/python2.7/site-packages/rosdistro/dependency_walker.py", line 120, in _get_dependencies
    pkg = self._get_package(pkg_name)
  File "env/local/lib/python2.7/site-packages/rosdistro/dependency_walker.py", line 133, in _get_package
    repo = self._distribution_instance.repositories[self._distribution_instance.source_packages[pkg_name].repository_name].source_repository
KeyError: 'package_which_definitely_does_exist'
```

@dirk-thomas I should have a patch for you shortly; just going through the delta between https://github.com/mikepurvis/rosdistro-1/releases/tag/0.4.7%2Bmpu1 and the current master to figure out where things went wrong.

(To clarify, there's no regression from current functionality, since `SourceDependencyWalker` is only used with a rosdistro which has a source cache...)

Comments:

Patch:
diff --git a/src/rosdistro/manifest_provider/git.py b/src/rosdistro/manifest_provider/git.py
index 8951983..3eca46f 100644
--- a/src/rosdistro/manifest_provider/git.py
+++ b/src/rosdistro/manifest_provider/git.py
@@ -65,14 +65,16 @@ def git_source_manifest_provider(repo):
             cache = { '_ref': result['output'] }
 
             # Find package.xml files inside the repo.
-            for package_path in catkin_pkg.packages.find_package_paths(git_repo_path):
+            for package_path in find_package_paths(git_repo_path):
+                if package_path == '.':
+                    package_path = ''
                 with open(os.path.join(git_repo_path, package_path, 'package.xml'), 'r') as f:
                     package_xml = f.read()
                 try:
                     name = parse_package_string(package_xml).name
                 except InvalidPackage:
                     raise RuntimeError('Unable to parse package.xml file found in %s' % repo.url)
-                cache[name] = [ path[len(git_repo_path)+1:], package_xml ]
+                cache[name] = [ package_path, package_xml ]
 
     except Exception as e:
         raise RuntimeError('Unable to fetch source package.xml files: %s' % e)
diff --git a/src/rosdistro/manifest_provider/github.py b/src/rosdistro/manifest_provider/github.py
index 7960c3d..6305d47 100644
--- a/src/rosdistro/manifest_provider/github.py
+++ b/src/rosdistro/manifest_provider/github.py
@@ -83,7 +83,7 @@ def github_source_manifest_provider(repo):
         authheader = 'Basic %s' % base64.b64encode('%s:%s' % (GITHUB_USER, GITHUB_PASSWORD))
         req.add_header('Authorization', authheader)
     try:
-        tree_json = json.load(urlopen(req))
+        tree_json = json.loads(urlopen(req).read().decode('utf-8'))
         logger.debug('- load repo tree from %s' % tree_url)
     except URLError as e:
         raise RuntimeError('Unable to fetch JSON tree from %s: %s' % (tree_url, e))
@@ -107,14 +107,14 @@ def github_source_manifest_provider(repo):
                 return False
             if parent == '':
                 return True
-    package_xml_paths = filter(package_xml_in_parent, package_xml_paths)
+    package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths))
 
     cache = { '_ref': tree_json['sha'] }
     for package_xml_path in package_xml_paths:
         url = 'https://raw.githubusercontent.com/%s/%s/%s' % \
             (path, cache['_ref'], package_xml_path + '/package.xml' if package_xml_path else 'package.xml')
         logger.debug('- load package.xml from %s' % url)
-        package_xml = urlopen(url).read()
+        package_xml = urlopen(url).read().decode('utf-8')
         name = parse_package_string(package_xml).name
         cache[name] = [ package_xml_path, package_xml ]
 


Test patch:
diff --git a/test/test_manifest_providers.py b/test/test_manifest_providers.py
index 73c3336..3bce35e 100644
--- a/test/test_manifest_providers.py
+++ b/test/test_manifest_providers.py
@@ -4,15 +4,16 @@ import os
 
 from rosdistro.manifest_provider.bitbucket import bitbucket_manifest_provider
 from rosdistro.manifest_provider.cache import CachedManifestProvider, sanitize_xml
-from rosdistro.manifest_provider.git import git_manifest_provider
-from rosdistro.manifest_provider.github import github_manifest_provider
+from rosdistro.manifest_provider.git import git_manifest_provider, git_source_manifest_provider
+from rosdistro.manifest_provider.github import github_manifest_provider, github_source_manifest_provider
 from rosdistro.release_repository_specification import ReleaseRepositorySpecification
+from rosdistro.source_repository_specification import SourceRepositorySpecification
 
 import rosdistro.vcs
 
 
 def test_bitbucket():
-    assert '</package>' in bitbucket_manifest_provider('indigo', _rospeex_repo(), 'rospeex_msgs')
+    assert '</package>' in bitbucket_manifest_provider('indigo', _rospeex_release_repo(), 'rospeex_msgs')
 
 
 def test_cached():
@@ -21,21 +22,57 @@ def test_cached():
             self.release_package_xmls = {}
     dc = FakeDistributionCache()
     cache = CachedManifestProvider(dc, [github_manifest_provider])
-    assert '</package>' in cache('kinetic', _genmsg_repo(), 'genmsg')
+    assert '</package>' in cache('kinetic', _genmsg_release_repo(), 'genmsg')
 
 
 def test_git():
-    assert '</package>' in git_manifest_provider('kinetic', _genmsg_repo(), 'genmsg')
+    assert '</package>' in git_manifest_provider('kinetic', _genmsg_release_repo(), 'genmsg')
 
 
 def test_git_legacy():
     rosdistro.vcs.Git._client_version = '1.7.0'
-    assert '</package>' in git_manifest_provider('kinetic', _genmsg_repo(), 'genmsg')
+    assert '</package>' in git_manifest_provider('kinetic', _genmsg_release_repo(), 'genmsg')
     rosdistro.vcs.Git._client_version = None
 
 
 def test_github():
-    assert '</package>' in github_manifest_provider('kinetic', _genmsg_repo(), 'genmsg')
+    assert '</package>' in github_manifest_provider('kinetic', _genmsg_release_repo(), 'genmsg')
+
+
+def test_git_source():
+    repo_cache = git_source_manifest_provider(_genmsg_source_repo())
+
+    # This hash corresponds to the 0.5.7 tag.
+    assert repo_cache['_ref'] == '81b66fe5eb00043c43894ddeee07e738d9b9712f'
+
+    package_path, package_xml = repo_cache['genmsg']
+    assert '' == package_path
+    assert '<version>0.5.7</version>' in package_xml
+
+
+def test_github_source():
+    repo_cache = github_source_manifest_provider(_genmsg_source_repo())
+
+    # This hash corresponds to the 0.5.7 tag.
+    assert repo_cache['_ref'] == '81b66fe5eb00043c43894ddeee07e738d9b9712f'
+
+    package_path, package_xml = repo_cache['genmsg']
+    assert '' == package_path
+    assert '<version>0.5.7</version>' in package_xml
+
+
+def test_git_source_multi():
+    repo_cache = git_source_manifest_provider(_ros_source_repo())
+    assert repo_cache['_ref']
+    package_path, package_xml = repo_cache['roslib']
+    assert package_path == 'core/roslib'
+
+
+def test_github_source_multi():
+    repo_cache = github_source_manifest_provider(_ros_source_repo())
+    assert repo_cache['_ref']
+    package_path, package_xml = repo_cache['roslib']
+    assert package_path == 'core/roslib'
 
 
 def test_sanitize():
@@ -46,15 +83,26 @@ def test_sanitize():
     assert '<a>français</a>' in sanitize_xml('<a> français  </a>')
 
 
-def _genmsg_repo():
+def _genmsg_release_repo():
     return ReleaseRepositorySpecification('genmsg', {
         'url': 'https://github.com/ros-gbp/genmsg-release.git',
         'tags': {'release': 'release/kinetic/{package}/{version}'},
         'version': '0.5.7-1'
     })
 
+def _genmsg_source_repo():
+    return SourceRepositorySpecification('genmsg', {
+        'url': 'https://github.com/ros/genmsg.git',
+        'version': '0.5.7'
+    })
+
+def _ros_source_repo():
+    return SourceRepositorySpecification('ros', {
+        'url': 'https://github.com/ros/ros.git',
+        'version': 'kinetic-devel'
+    })
 
-def _rospeex_repo():
+def _rospeex_release_repo():
     return ReleaseRepositorySpecification('rospeex', {
         'packages': ['rospeex', 'rospeex_msgs'],
         'tags': {'release': 'release/indigo/{package}/{version}'},