Repository: kumahq/kuma
Pull request #1196 (2020-11-23 11:51:33)

chore(kuma-cp) add customizable API (bp #1174)
This is an automated backport of pull request #1174 done by Mergify
---


<details>
<summary>Mergify commands and options</summary>

<br />

More conditions and actions can be found in the [documentation](https://docs.mergify.io/).

You can also trigger Mergify actions by commenting on this pull request:

- `@Mergifyio refresh` will re-evaluate the rules
- `@Mergifyio rebase` will rebase this PR on its base branch
- `@Mergifyio update` will merge the base branch into this PR
- `@Mergifyio backport <destination>` will backport this PR on `<destination>` branch

Additionally, on Mergify [dashboard](https://dashboard.mergify.io/) you can:

- look at your merge queues
- generate the Mergify configuration with the config editor.

Finally, you can contact us on https://mergify.io/
</details>



Patch:
diff --git a/pkg/api-server/customization/api_manager.go b/pkg/api-server/customization/api_manager.go
new file mode 100644
index 000000000..43a64e075
--- /dev/null
+++ b/pkg/api-server/customization/api_manager.go
@@ -0,0 +1,34 @@
+package customization
+
+import (
+	"github.com/emicklei/go-restful"
+)
+
+type APIInstaller interface {
+	Install(container *restful.Container)
+}
+
+type APIManager interface {
+	APIInstaller
+	Add(ws *restful.WebService)
+}
+
+type APIList struct {
+	list []*restful.WebService
+}
+
+func NewAPIList() *APIList {
+	return &APIList{
+		list: []*restful.WebService{},
+	}
+}
+
+func (c *APIList) Add(ws *restful.WebService) {
+	c.list = append(c.list, ws)
+}
+
+func (c *APIList) Install(container *restful.Container) {
+	for _, ws := range c.list {
+		container.Add(ws)
+	}
+}
diff --git a/pkg/api-server/server.go b/pkg/api-server/server.go
index 4b6aff70b..a0d91ff83 100644
--- a/pkg/api-server/server.go
+++ b/pkg/api-server/server.go
@@ -13,6 +13,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/kumahq/kuma/pkg/api-server/customization"
+
 	"github.com/emicklei/go-restful"
 	http_prometheus "github.com/slok/go-http-metrics/metrics/prometheus"
 	"github.com/slok/go-http-metrics/middleware"
@@ -71,7 +73,7 @@ func init() {
 	}
 }
 
-func NewApiServer(resManager manager.ResourceManager, defs []definitions.ResourceWsDefinition, cfg *kuma_cp.Config, enableGUI bool, metrics metrics.Metrics) (*ApiServer, error) {
+func NewApiServer(resManager manager.ResourceManager, wsManager customization.APIInstaller, defs []definitions.ResourceWsDefinition, cfg *kuma_cp.Config, enableGUI bool, metrics metrics.Metrics) (*ApiServer, error) {
 	serverConfig := cfg.ApiServer
 	container := restful.NewContainer()
 
@@ -135,6 +137,9 @@ func NewApiServer(resManager manager.ResourceManager, defs []definitions.Resourc
 	} else {
 		container.ServeMux.HandleFunc("/gui/", newApiServer.notAvailableHandler)
 	}
+
+	wsManager.Install(container)
+
 	return newApiServer, nil
 }
 
@@ -327,7 +332,7 @@ func SetupServer(rt runtime.Runtime) error {
 			}
 		}
 	}
-	apiServer, err := NewApiServer(rt.ResourceManager(), definitions.All, &cfg, enableGUI, rt.Metrics())
+	apiServer, err := NewApiServer(rt.ResourceManager(), rt.APIInstaller(), definitions.All, &cfg, enableGUI, rt.Metrics())
 	if err != nil {
 		return err
 	}
diff --git a/pkg/core/bootstrap/bootstrap.go b/pkg/core/bootstrap/bootstrap.go
index 5546a0043..a2b262e23 100644
--- a/pkg/core/bootstrap/bootstrap.go
+++ b/pkg/core/bootstrap/bootstrap.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"net"
 
+	"github.com/kumahq/kuma/pkg/api-server/customization"
+
 	"github.com/pkg/errors"
 
 	"github.com/kumahq/kuma/pkg/dns/resolver"
@@ -46,7 +48,7 @@ func buildRuntime(cfg kuma_cp.Config) (core_runtime.Runtime, error) {
 	if err := initializeMetrics(builder); err != nil {
 		return nil, err
 	}
-	if err := initializeBootstrap(cfg, builder); err != nil {
+	if err := initializeBeforeBootstrap(cfg, builder); err != nil {
 		return nil, err
 	}
 	if err := initializeResourceStore(cfg, builder); err != nil {
@@ -86,6 +88,11 @@ func buildRuntime(cfg kuma_cp.Config) (core_runtime.Runtime, error) {
 	builder.WithLeaderInfo(leaderInfoComponent)
 
 	builder.WithLookupIP(lookup.CachedLookupIP(net.LookupIP, cfg.General.DNSCacheTTL))
+	builder.WithAPIManager(customization.NewAPIList())
+
+	if err := initializeAfterBootstrap(cfg, builder); err != nil {
+		return nil, err
+	}
 
 	rt, err := builder.Build()
 	if err != nil {
@@ -142,13 +149,26 @@ func startReporter(runtime core_runtime.Runtime) error {
 	}))
 }
 
-func initializeBootstrap(cfg kuma_cp.Config, builder *core_runtime.Builder) error {
+func initializeBeforeBootstrap(cfg kuma_cp.Config, builder *core_runtime.Builder) error {
+	for name, plugin := range core_plugins.Plugins().BootstrapPlugins() {
+		if (cfg.Environment == config_core.KubernetesEnvironment && name == core_plugins.Universal) ||
+			(cfg.Environment == config_core.UniversalEnvironment && name == core_plugins.Kubernetes) {
+			continue
+		}
+		if err := plugin.BeforeBootstrap(builder, nil); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func initializeAfterBootstrap(cfg kuma_cp.Config, builder *core_runtime.Builder) error {
 	for name, plugin := range core_plugins.Plugins().BootstrapPlugins() {
 		if (cfg.Environment == config_core.KubernetesEnvironment && name == core_plugins.Universal) ||
 			(cfg.Environment == config_core.UniversalEnvironment && name == core_plugins.Kubernetes) {
 			continue
 		}
-		if err := plugin.Bootstrap(builder, nil); err != nil {
+		if err := plugin.AfterBootstrap(builder, nil); err != nil {
 			return err
 		}
 	}
diff --git a/pkg/core/plugins/interfaces.go b/pkg/core/plugins/interfaces.go
index e92e501b3..00acaec85 100644
--- a/pkg/core/plugins/interfaces.go
+++ b/pkg/core/plugins/interfaces.go
@@ -24,7 +24,8 @@ type MutablePluginContext = core_runtime.Builder
 // Unlike other plugins, can mutate plugin context directly.
 type BootstrapPlugin interface {
 	Plugin
-	Bootstrap(*MutablePluginContext, PluginConfig) error
+	BeforeBootstrap(*MutablePluginContext, PluginConfig) error
+	AfterBootstrap(*MutablePluginContext, PluginConfig) error
 }
 
 // ResourceStorePlugin is responsible for instantiating a particular ResourceStore.
diff --git a/pkg/core/runtime/builder.go b/pkg/core/runtime/builder.go
index f54950344..ca27acd99 100644
--- a/pkg/core/runtime/builder.go
+++ b/pkg/core/runtime/builder.go
@@ -3,6 +3,8 @@ package runtime
 import (
 	"context"
 
+	api_server "github.com/kumahq/kuma/pkg/api-server/customization"
+
 	"github.com/pkg/errors"
 
 	"github.com/kumahq/kuma/pkg/events"
@@ -39,6 +41,7 @@ type BuilderContext interface {
 	LeaderInfo() component.LeaderInfo
 	Metrics() metrics.Metrics
 	EventReaderFactory() events.ListenerFactory
+	APIManager() api_server.APIManager
 }
 
 var _ BuilderContext = &Builder{}
@@ -62,6 +65,7 @@ type Builder struct {
 	lif      lookup.LookupIPFunc
 	metrics  metrics.Metrics
 	erf      events.ListenerFactory
+	apim     api_server.APIManager
 	*runtimeInfo
 }
 
@@ -166,6 +170,11 @@ func (b *Builder) WithEventReaderFactory(erf events.ListenerFactory) *Builder {
 	return b
 }
 
+func (b *Builder) WithAPIManager(apim api_server.APIManager) *Builder {
+	b.apim = apim
+	return b
+}
+
 func (b *Builder) Build() (Runtime, error) {
 	if b.cm == nil {
 		return nil, errors.Errorf("ComponentManager has not been configured")
@@ -203,6 +212,9 @@ func (b *Builder) Build() (Runtime, error) {
 	if b.erf == nil {
 		return nil, errors.Errorf("EventReaderFactory has not been configured")
 	}
+	if b.apim == nil {
+		return nil, errors.Errorf("APIManager has not been configured")
+	}
 	return &runtime{
 		RuntimeInfo: b.runtimeInfo,
 		RuntimeContext: &runtimeContext{
@@ -221,6 +233,7 @@ func (b *Builder) Build() (Runtime, error) {
 			lif:      b.lif,
 			metrics:  b.metrics,
 			erf:      b.erf,
+			apim:     b.apim,
 		},
 		Manager: b.cm,
 	}, nil
@@ -277,3 +290,7 @@ func (b *Builder) Metrics() metrics.Metrics {
 func (b *Builder) EventReaderFactory() events.ListenerFactory {
 	return b.erf
 }
+
+func (b *Builder) APIManager() api_server.APIManager {
+	return b.apim
+}
diff --git a/pkg/core/runtime/runtime.go b/pkg/core/runtime/runtime.go
index 4acf598b8..727399584 100644
--- a/pkg/core/runtime/runtime.go
+++ b/pkg/core/runtime/runtime.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"sync"
 
+	api_server "github.com/kumahq/kuma/pkg/api-server/customization"
+
 	"github.com/kumahq/kuma/pkg/core/datasource"
 	"github.com/kumahq/kuma/pkg/dns/resolver"
 
@@ -53,6 +55,7 @@ type RuntimeContext interface {
 	LookupIP() lookup.LookupIPFunc
 	Metrics() metrics.Metrics
 	EventReaderFactory() events.ListenerFactory
+	APIInstaller() api_server.APIInstaller
 }
 
 var _ Runtime = &runtime{}
@@ -107,6 +110,7 @@ type runtimeContext struct {
 	lif      lookup.LookupIPFunc
 	metrics  metrics.Metrics
 	erf      events.ListenerFactory
+	apim     api_server.APIInstaller
 }
 
 func (rc *runtimeContext) Metrics() metrics.Metrics {
@@ -172,3 +176,6 @@ func (rc *runtimeContext) LeaderInfo() component.LeaderInfo {
 func (rc *runtimeContext) LookupIP() lookup.LookupIPFunc {
 	return rc.lif
 }
+func (rc *runtimeContext) APIInstaller() api_server.APIInstaller {
+	return rc.apim
+}
diff --git a/pkg/plugins/bootstrap/k8s/plugin.go b/pkg/plugins/bootstrap/k8s/plugin.go
index d0c7779bb..3cd5bb75c 100644
--- a/pkg/plugins/bootstrap/k8s/plugin.go
+++ b/pkg/plugins/bootstrap/k8s/plugin.go
@@ -23,7 +23,7 @@ func init() {
 	core_plugins.Register(core_plugins.Kubernetes, &plugin{})
 }
 
-func (p *plugin) Bootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig) error {
+func (p *plugin) BeforeBootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig) error {
 	scheme := kube_runtime.NewScheme()
 	mgr, err := kube_ctrl.NewManager(
 		kube_ctrl.GetConfigOrDie(),
@@ -52,6 +52,10 @@ func (p *plugin) Bootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig)
 	return nil
 }
 
+func (p *plugin) AfterBootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig) error {
+	return nil
+}
+
 type kubeComponentManager struct {
 	kube_ctrl.Manager
 }
diff --git a/pkg/plugins/bootstrap/universal/plugin.go b/pkg/plugins/bootstrap/universal/plugin.go
index dda74987d..f416e9c8b 100644
--- a/pkg/plugins/bootstrap/universal/plugin.go
+++ b/pkg/plugins/bootstrap/universal/plugin.go
@@ -15,7 +15,7 @@ func init() {
 	core_plugins.Register(core_plugins.Universal, &plugin{})
 }
 
-func (p *plugin) Bootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig) error {
+func (p *plugin) BeforeBootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig) error {
 	leaderElector, err := plugin_leader.NewLeaderElector(b)
 	if err != nil {
 		return err
@@ -23,3 +23,7 @@ func (p *plugin) Bootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig)
 	b.WithComponentManager(component.NewManager(leaderElector))
 	return nil
 }
+
+func (p *plugin) AfterBootstrap(b *core_runtime.Builder, _ core_plugins.PluginConfig) error {
+	return nil
+}


Test patch:
diff --git a/pkg/api-server/customization/api_manager_test.go b/pkg/api-server/customization/api_manager_test.go
new file mode 100644
index 000000000..e5bcb3e5c
--- /dev/null
+++ b/pkg/api-server/customization/api_manager_test.go
@@ -0,0 +1,67 @@
+package customization_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/emicklei/go-restful"
+
+	"github.com/kumahq/kuma/pkg/api-server/customization"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	api_server_config "github.com/kumahq/kuma/pkg/config/api-server"
+	"github.com/kumahq/kuma/pkg/metrics"
+	"github.com/kumahq/kuma/pkg/plugins/resources/memory"
+)
+
+var _ = Describe("API Manager", func() {
+
+	It("should return the config", func() {
+		// given
+		cfg := api_server_config.DefaultApiServerConfig()
+
+		// setup
+		resourceStore := memory.NewStore()
+		metrics, err := metrics.NewMetrics("Standalone")
+		Expect(err).ToNot(HaveOccurred())
+		wsManager := customization.NewAPIList()
+
+		ws := new(restful.WebService).Path("foo")
+		ws.Route(ws.GET("baz").To(func(request *restful.Request, response *restful.Response) {
+			_ = response.WriteAsJson("bar")
+		}))
+		wsManager.Add(ws)
+
+		apiServer := createTestApiServer(resourceStore, cfg, true, metrics, wsManager)
+
+		stop := make(chan struct{})
+		go func() {
+			defer GinkgoRecover()
+			err := apiServer.Start(stop)
+			Expect(err).ToNot(HaveOccurred())
+		}()
+		port := strings.Split(apiServer.Address(), ":")[1]
+
+		// wait for the server
+		Eventually(func() error {
+			_, err := http.Get(fmt.Sprintf("http://localhost:%s/foo", port))
+			return err
+		}, "3s").ShouldNot(HaveOccurred())
+
+		// when
+		resp, err := http.Get(fmt.Sprintf("http://localhost:%s/foo/baz", port))
+		Expect(err).ToNot(HaveOccurred())
+
+		// then
+		body, err := ioutil.ReadAll(resp.Body)
+		Expect(err).ToNot(HaveOccurred())
+
+		// when
+		Expect(string(body)).To(Equal("\"bar\"\n"))
+		close(stop)
+	})
+})
diff --git a/pkg/api-server/customization/customization_suite_test.go b/pkg/api-server/customization/customization_suite_test.go
new file mode 100644
index 000000000..f9251d842
--- /dev/null
+++ b/pkg/api-server/customization/customization_suite_test.go
@@ -0,0 +1,53 @@
+package customization_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	api_server "github.com/kumahq/kuma/pkg/api-server"
+	"github.com/kumahq/kuma/pkg/api-server/customization"
+	"github.com/kumahq/kuma/pkg/api-server/definitions"
+	config_api_server "github.com/kumahq/kuma/pkg/config/api-server"
+	kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
+	"github.com/kumahq/kuma/pkg/core/resources/manager"
+	"github.com/kumahq/kuma/pkg/core/resources/store"
+	core_metrics "github.com/kumahq/kuma/pkg/metrics"
+	"github.com/kumahq/kuma/pkg/test"
+)
+
+func TestWs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "API Server Customization")
+}
+
+func createTestApiServer(store store.ResourceStore, config *config_api_server.ApiServerConfig, enableGUI bool, metrics core_metrics.Metrics, wsManager customization.APIManager) *api_server.ApiServer {
+	// we have to manually search for port and put it into config. There is no way to retrieve port of running
+	// http.Server and we need it later for the client
+	port, err := test.GetFreePort()
+	Expect(err).NotTo(HaveOccurred())
+	config.HTTP.Port = uint32(port)
+
+	port, err = test.GetFreePort()
+	Expect(err).NotTo(HaveOccurred())
+	config.HTTPS.Port = uint32(port)
+	if config.HTTPS.TlsKeyFile == "" {
+		config.HTTPS.TlsKeyFile = filepath.Join("..", "..", "..", "test", "certs", "server-key.pem")
+		config.HTTPS.TlsCertFile = filepath.Join("..", "..", "..", "test", "certs", "server-cert.pem")
+		config.Auth.ClientCertsDir = filepath.Join("..", "..", "..", "test", "certs", "client")
+	}
+
+	defs := definitions.All
+	resources := manager.NewResourceManager(store)
+
+	if wsManager == nil {
+		wsManager = customization.NewAPIList()
+	}
+	cfg := kuma_cp.DefaultConfig()
+	cfg.ApiServer = config
+	apiServer, err := api_server.NewApiServer(resources, wsManager, defs, &cfg, enableGUI, metrics)
+	Expect(err).ToNot(HaveOccurred())
+	return apiServer
+}
diff --git a/pkg/api-server/resource_api_client_test.go b/pkg/api-server/resource_api_client_test.go
index 37eb191b8..53d58a014 100644
--- a/pkg/api-server/resource_api_client_test.go
+++ b/pkg/api-server/resource_api_client_test.go
@@ -6,6 +6,8 @@ import (
 	"net/http"
 	"path/filepath"
 
+	"github.com/kumahq/kuma/pkg/api-server/customization"
+
 	api_server "github.com/kumahq/kuma/pkg/api-server"
 	"github.com/kumahq/kuma/pkg/api-server/definitions"
 	config_api_server "github.com/kumahq/kuma/pkg/config/api-server"
@@ -117,9 +119,10 @@ func createTestApiServer(store store.ResourceStore, config *config_api_server.Ap
 
 	defs := append(definitions.All, SampleTrafficRouteWsDefinition)
 	resources := manager.NewResourceManager(store)
+	wsManager := customization.NewAPIList()
 	cfg := kuma_cp.DefaultConfig()
 	cfg.ApiServer = config
-	apiServer, err := api_server.NewApiServer(resources, defs, &cfg, enableGUI, metrics)
+	apiServer, err := api_server.NewApiServer(resources, wsManager, defs, &cfg, enableGUI, metrics)
 	Expect(err).ToNot(HaveOccurred())
 	return apiServer
 }
diff --git a/pkg/test/runtime/runtime.go b/pkg/test/runtime/runtime.go
index 95784847e..e4bcf7edf 100644
--- a/pkg/test/runtime/runtime.go
+++ b/pkg/test/runtime/runtime.go
@@ -3,6 +3,8 @@ package runtime
 import (
 	"net"
 
+	"github.com/kumahq/kuma/pkg/api-server/customization"
+
 	kuma_cp "github.com/kumahq/kuma/pkg/config/app/kuma-cp"
 	config_manager "github.com/kumahq/kuma/pkg/core/config/manager"
 	"github.com/kumahq/kuma/pkg/core/datasource"
@@ -65,6 +67,7 @@ func BuilderFor(cfg kuma_cp.Config) *core_runtime.Builder {
 	builder.WithLeaderInfo(&component.LeaderInfoComponent{})
 	builder.WithLookupIP(net.LookupIP)
 	builder.WithEventReaderFactory(events.NewEventBus())
+	builder.WithAPIManager(customization.NewAPIList())
 
 	_ = initializeConfigManager(cfg, builder)
 	_ = initializeDNSResolver(cfg, builder)