Repository: osbuild/osbuild-composer
Pull request #103 (2019-11-14 10:30:39)

Fetch packages from user sources
When users add their own sources they are now able to fetch packages from these sources. The package list is generated from the base repo and the user-defined repos. This allows users to add packages not found in the base repo to their blueprints.


Patch:
diff --git a/cmd/osbuild-composer/main.go b/cmd/osbuild-composer/main.go
index 0088a8cf7..3322c497e 100644
--- a/cmd/osbuild-composer/main.go
+++ b/cmd/osbuild-composer/main.go
@@ -40,11 +40,6 @@ func main() {
 
 	rpm := rpmmd.NewRPMMD()
 
-	packages, err := rpm.FetchPackageList([]rpmmd.RepoConfig{repo})
-	if err != nil {
-		panic(err)
-	}
-
 	var logger *log.Logger
 	if verbose {
 		logger = log.New(os.Stdout, "", 0)
@@ -58,7 +53,7 @@ func main() {
 	store := store.New(&stateFile)
 
 	jobAPI := jobqueue.New(logger, store)
-	weldrAPI := weldr.New(rpm, repo, packages, logger, store)
+	weldrAPI := weldr.New(rpm, repo, logger, store)
 
 	go jobAPI.Serve(jobListener)
 	weldrAPI.Serve(weldrListener)
diff --git a/internal/rpmmd/repository.go b/internal/rpmmd/repository.go
index 291924c91..86fca6ea6 100644
--- a/internal/rpmmd/repository.go
+++ b/internal/rpmmd/repository.go
@@ -8,6 +8,8 @@ import (
 	"os/exec"
 	"sort"
 	"time"
+
+	"github.com/osbuild/osbuild-composer/internal/store"
 )
 
 type RepoConfig struct {
@@ -159,3 +161,20 @@ func (packages PackageList) Search(name string) (int, int) {
 
 	return first, last - first
 }
+
+func SourceToRepo(source store.SourceConfig) RepoConfig {
+	var repo RepoConfig
+
+	repo.Name = source.Name
+	repo.Id = source.Name
+
+	if source.Type == "yum-baseurl" {
+		repo.BaseURL = source.URL
+	} else if source.Type == "yum-metalink" {
+		repo.Metalink = source.URL
+	} else if source.Type == "yum-mirrorlist" {
+		repo.MirrorList = source.URL
+	}
+
+	return repo
+}
diff --git a/internal/weldr/api.go b/internal/weldr/api.go
index 82551a99f..df97766f6 100644
--- a/internal/weldr/api.go
+++ b/internal/weldr/api.go
@@ -32,15 +32,14 @@ type API struct {
 	router *httprouter.Router
 }
 
-func New(rpmmd rpmmd.RPMMD, repo rpmmd.RepoConfig, packages rpmmd.PackageList, logger *log.Logger, store *store.Store) *API {
+func New(rpmmd rpmmd.RPMMD, repo rpmmd.RepoConfig, logger *log.Logger, store *store.Store) *API {
 	// This needs to be shared with the worker API so that they can communicate with each other
 	// builds := make(chan queue.Build, 200)
 	api := &API{
-		store:    store,
-		rpmmd:    rpmmd,
-		repo:     repo,
-		packages: packages,
-		logger:   logger,
+		store:  store,
+		rpmmd:  rpmmd,
+		repo:   repo,
+		logger: logger,
 	}
 
 	api.router = httprouter.New()
@@ -312,13 +311,29 @@ func (api *API) modulesListAllHandler(writer http.ResponseWriter, request *http.
 		return
 	}
 
-	total := uint(len(api.packages))
+	var repos []rpmmd.RepoConfig
+	repos = append(repos, api.repo)
+	for _, source := range api.store.GetAllSources() {
+		repos = append(repos, rpmmd.SourceToRepo(source))
+	}
+
+	packages, err := api.rpmmd.FetchPackageList(repos)
+	if err != nil {
+		errors := responseError{
+			ID:  "ModulesError",
+			Msg: fmt.Sprintf("msg: %s", err.Error()),
+		}
+		statusResponseError(writer, http.StatusBadRequest, errors)
+		return
+	}
+
+	total := uint(len(packages))
 	start := min(offset, total)
 	n := min(limit, total-start)
 
 	modules := make([]modulesListModule, n)
 	for i := uint(0); i < n; i++ {
-		modules[i] = modulesListModule{api.packages[start+i].Name, "rpm"}
+		modules[i] = modulesListModule{packages[start+i].Name, "rpm"}
 	}
 
 	json.NewEncoder(writer).Encode(modulesListReply{
@@ -347,6 +362,22 @@ func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Req
 		return
 	}
 
+	var repos []rpmmd.RepoConfig
+	repos = append(repos, api.repo)
+	for _, source := range api.store.GetAllSources() {
+		repos = append(repos, rpmmd.SourceToRepo(source))
+	}
+
+	packages, err := api.rpmmd.FetchPackageList(repos)
+	if err != nil {
+		errors := responseError{
+			ID:  "ModulesError",
+			Msg: fmt.Sprintf("msg: %s", err.Error()),
+		}
+		statusResponseError(writer, http.StatusBadRequest, errors)
+		return
+	}
+
 	// we don't support glob-matching, but cockpit-composer surrounds some
 	// queries with asterisks; this is crude, but solves that case
 	for i := range names {
@@ -357,7 +388,7 @@ func (api *API) modulesListHandler(writer http.ResponseWriter, request *http.Req
 	total := uint(0)
 	end := offset + limit
 	for i, name := range names {
-		for _, pkg := range api.packages {
+		for _, pkg := range packages {
 			if strings.Contains(pkg.Name, name) {
 				total++
 				if total > offset && total < end {
@@ -432,6 +463,22 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
 		return
 	}
 
+	var repos []rpmmd.RepoConfig
+	repos = append(repos, api.repo)
+	for _, source := range api.store.GetAllSources() {
+		repos = append(repos, rpmmd.SourceToRepo(source))
+	}
+
+	packages, err := api.rpmmd.FetchPackageList(repos)
+	if err != nil {
+		errors := responseError{
+			ID:  "ModulesError",
+			Msg: fmt.Sprintf("msg: %s", err.Error()),
+		}
+		statusResponseError(writer, http.StatusBadRequest, errors)
+		return
+	}
+
 	projects := make([]project, 0)
 	for i, name := range names {
 		// remove leading / from first name
@@ -439,7 +486,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
 			name = name[1:]
 		}
 
-		first, n := api.packages.Search(name)
+		first, n := packages.Search(name)
 		if n == 0 {
 			errors := responseError{
 				ID:  "UnknownModule",
@@ -451,7 +498,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
 
 		// get basic info from the first package, but collect build
 		// information from all that have the same name
-		pkg := api.packages[first]
+		pkg := packages[first]
 		project := project{
 			Name:        pkg.Name,
 			Summary:     pkg.Summary,
@@ -460,7 +507,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
 		}
 
 		project.Builds = make([]build, n)
-		for i, pkg := range api.packages[first : first+n] {
+		for i, pkg := range packages[first : first+n] {
 			project.Builds[i] = build{
 				Arch:      pkg.Arch,
 				BuildTime: pkg.BuildTime,
@@ -471,7 +518,7 @@ func (api *API) modulesInfoHandler(writer http.ResponseWriter, request *http.Req
 		}
 
 		if modulesRequested {
-			project.Dependencies, _ = api.rpmmd.Depsolve([]string{pkg.Name}, []rpmmd.RepoConfig{api.repo})
+			project.Dependencies, _ = api.rpmmd.Depsolve([]string{pkg.Name}, repos)
 		}
 
 		projects = append(projects, project)
@@ -637,7 +684,14 @@ func (api *API) blueprintsDepsolveHandler(writer http.ResponseWriter, request *h
 				specs[i] += "-*-*.*"
 			}
 		}
-		dependencies, _ := api.rpmmd.Depsolve(specs, []rpmmd.RepoConfig{api.repo})
+
+		var repos []rpmmd.RepoConfig
+		repos = append(repos, api.repo)
+		for _, source := range api.store.GetAllSources() {
+			repos = append(repos, rpmmd.SourceToRepo(source))
+		}
+
+		dependencies, _ := api.rpmmd.Depsolve(specs, repos)
 
 		blueprints = append(blueprints, entry{blueprint, dependencies})
 	}


Test patch:
diff --git a/internal/weldr/api_test.go b/internal/weldr/api_test.go
index e2b7d3ba2..b9c903f39 100644
--- a/internal/weldr/api_test.go
+++ b/internal/weldr/api_test.go
@@ -16,7 +16,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
+	rpmmd_mock "github.com/osbuild/osbuild-composer/internal/mocks/rpmmd"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 	"github.com/osbuild/osbuild-composer/internal/store"
 	"github.com/osbuild/osbuild-composer/internal/weldr"
@@ -159,13 +159,8 @@ func testRoute(t *testing.T, api *weldr.API, external bool, method, path, body s
 func createWeldrAPI(fixture rpmmd_mock.Fixture) (*weldr.API, *store.Store) {
 	s := store.New(nil)
 	rpm := rpmmd_mock.NewRPMMDMock(fixture)
-	packageList, err := rpm.FetchPackageList([]rpmmd.RepoConfig{repo})
 
-	if err != nil {
-		panic(err)
-	}
-
-	return weldr.New(rpm, repo, packageList, nil, s), s
+	return weldr.New(rpm, repo, nil, s), s
 }
 
 func TestBasic(t *testing.T) {