Repository: kubeless/kubeless
Pull request #1074 (2019-08-15 14:57:29)

consider resources for initContainers if provided in function deploym…
**Issue Ref**: #960
 
**Description**: 

This PR allows to provide default resources for initContainers if they need to be created. 

abinet: I think there is a general problem with CI build right now.
`go tools vet` is deprecated and should be replaced by `go vet`. But `go vet` does not work properly with individual files (see https://circleci.com/gh/kubeless/kubeless/2633?utm_campaign=vcs-integration-link&utm_medium=referral&utm_source=github-build-link)

From other POV vetting is already included into `go test` by default:

```
As part of building a test binary, go test runs go vet on the package
and its test source files to identify significant problems. If go vet
finds any problems, go test reports those and does not run the test
binary. Only a high-confidence subset of the default go vet checks are
used. That subset is: 'atomic', 'bool', 'buildtags', 'nilfunc', and
'printf'. You can see the documentation for these and other vet tests
via "go doc cmd/vet". To disable the running of go vet, use the
-vet=off flag.
```

So that's why I'm going to remove ./script/validate-vet from the validation step.

Patch:
diff --git a/Makefile b/Makefile
index ca778566..ab61c612 100644
--- a/Makefile
+++ b/Makefile
@@ -66,7 +66,6 @@ test:
 	$(GO) test $(GO_FLAGS) $(GO_PACKAGES)
 
 validation:
-	./script/validate-vet
 	./script/validate-lint
 	./script/validate-gofmt
 	./script/validate-git-marks
diff --git a/docs/advanced-function-deployment.md b/docs/advanced-function-deployment.md
index 30edf969..106a38b1 100644
--- a/docs/advanced-function-deployment.md
+++ b/docs/advanced-function-deployment.md
@@ -61,6 +61,14 @@ spec:
     spec:
       template:
         spec:
+          initContainers:
+          - resources:
+              limits:
+                cpu: 200m
+                memory: 200Mi
+              requests:
+                cpu: 200m
+                memory: 200Mi
           containers:
           - env:
             - name: FOO
@@ -83,6 +91,8 @@ spec:
 ```
 
 Would create a function with the environment variable `FOO`, using CPU and memory limits and mounting the secret `my-secret` as a volume. Note that you can also specify a default template for a Deployment spec in the [controller configuration](/docs/function-controller-configuration).
+The resource configuration in `initContainers` will be applied to all of the initial containers in the target deployment (like `provision`, `compile` etc.)   
+
 
 ## Custom Service
 
diff --git a/docs/runtimes.md b/docs/runtimes.md
index bf53d4aa..c098656e 100644
--- a/docs/runtimes.md
+++ b/docs/runtimes.md
@@ -436,6 +436,14 @@ You can deploy them using the command:
 kubeless function deploy fibonacci --from-file fibonacci.cs --handler module.handler --dependencies fibonacci.csproj --runtime dotnetcore2.0
 ```
 
+##### `nuget.config`
+
+If you happen to be using custom nuget repositories through a `nuget.config` file, you'll need to include the file along with the code inside a `.zip` file and then you can deploy the function with the `nuget.config` using the command:
+
+```bash
+kubeless function deploy custom-deps --from-file custom-deps.zip --handler module.handler --dependencies custom-deps.csproj --runtime dotnetcore2.0
+```
+
 ### Ballerina
 
 #### Example
diff --git a/pkg/langruntime/langruntime.go b/pkg/langruntime/langruntime.go
index a39c6758..0057a9d5 100644
--- a/pkg/langruntime/langruntime.go
+++ b/pkg/langruntime/langruntime.go
@@ -247,7 +247,7 @@ func parseEnv(env map[string]string) []v1.EnvVar {
 }
 
 // GetBuildContainer returns a Container definition based on a runtime
-func (l *Langruntimes) GetBuildContainer(runtime, depsChecksum string, env []v1.EnvVar, installVolume v1.VolumeMount) (v1.Container, error) {
+func (l *Langruntimes) GetBuildContainer(runtime, depsChecksum string, env []v1.EnvVar, installVolume v1.VolumeMount, resources v1.ResourceRequirements) (v1.Container, error) {
 	runtimeInf, err := l.GetRuntimeInfo(runtime)
 	if err != nil {
 		return v1.Container{}, err
@@ -289,6 +289,7 @@ func (l *Langruntimes) GetBuildContainer(runtime, depsChecksum string, env []v1.
 		ImagePullPolicy: v1.PullIfNotPresent,
 		WorkingDir:      installVolume.MountPath,
 		Env:             env,
+		Resources:       resources,
 	}, nil
 }
 
@@ -316,7 +317,7 @@ func (l *Langruntimes) UpdateDeployment(dpm *v1beta1.Deployment, volPath, runtim
 }
 
 // GetCompilationContainer returns a Container definition based on a runtime
-func (l *Langruntimes) GetCompilationContainer(runtime, funcName string, env []v1.EnvVar, installVolume v1.VolumeMount) (*v1.Container, error) {
+func (l *Langruntimes) GetCompilationContainer(runtime, funcName string, env []v1.EnvVar, installVolume v1.VolumeMount, resources v1.ResourceRequirements) (*v1.Container, error) {
 	versionInf, err := l.findRuntimeVersion(runtime)
 	if err != nil {
 		return nil, err
@@ -343,5 +344,6 @@ func (l *Langruntimes) GetCompilationContainer(runtime, funcName string, env []v
 		VolumeMounts:    []v1.VolumeMount{installVolume},
 		ImagePullPolicy: v1.PullIfNotPresent,
 		WorkingDir:      installVolume.MountPath,
+		Resources:       resources,
 	}, nil
 }
diff --git a/pkg/utils/kubelessutil.go b/pkg/utils/kubelessutil.go
index 14e35dea..1e5598de 100644
--- a/pkg/utils/kubelessutil.go
+++ b/pkg/utils/kubelessutil.go
@@ -67,7 +67,7 @@ func appendToCommand(orig string, command ...string) string {
 	return strings.Join(command, " && ")
 }
 
-func getProvisionContainer(function, checksum, fileName, handler, contentType, runtime, prepareImage string, runtimeVolume, depsVolume v1.VolumeMount, lr *langruntime.Langruntimes) (v1.Container, error) {
+func getProvisionContainer(function, checksum, fileName, handler, contentType, runtime, prepareImage string, runtimeVolume, depsVolume v1.VolumeMount, resources v1.ResourceRequirements, lr *langruntime.Langruntimes) (v1.Container, error) {
 	prepareCommand := ""
 	originFile := path.Join(depsVolume.MountPath, fileName)
 
@@ -139,6 +139,7 @@ func getProvisionContainer(function, checksum, fileName, handler, contentType, r
 		Args:            []string{prepareCommand},
 		VolumeMounts:    []v1.VolumeMount{runtimeVolume, depsVolume},
 		ImagePullPolicy: v1.PullIfNotPresent,
+		Resources:       resources,
 	}, nil
 }
 
@@ -338,6 +339,12 @@ func populatePodSpec(funcObj *kubelessApi.Function, lr *langruntime.Langruntimes
 		},
 	)
 	// prepare init-containers if some function is specified
+
+	resources := v1.ResourceRequirements{}
+	if len(funcObj.Spec.Deployment.Spec.Template.Spec.InitContainers) > 0 {
+		resources = funcObj.Spec.Deployment.Spec.Template.Spec.InitContainers[0].Resources
+	}
+
 	if funcObj.Spec.Function != "" {
 		fileName, err := getFileName(funcObj.Spec.Handler, funcObj.Spec.FunctionContentType, funcObj.Spec.Runtime, lr)
 		if err != nil {
@@ -360,6 +367,7 @@ func populatePodSpec(funcObj *kubelessApi.Function, lr *langruntime.Langruntimes
 			provisionImage,
 			runtimeVolumeMount,
 			srcVolumeMount,
+			resources,
 			lr,
 		)
 		if err != nil {
@@ -390,7 +398,7 @@ func populatePodSpec(funcObj *kubelessApi.Function, lr *langruntime.Langruntimes
 		if err != nil {
 			return fmt.Errorf("Unable to obtain dependencies checksum: %v", err)
 		}
-		depsInstallContainer, err := lr.GetBuildContainer(funcObj.Spec.Runtime, depsChecksum, envVars, runtimeVolumeMount)
+		depsInstallContainer, err := lr.GetBuildContainer(funcObj.Spec.Runtime, depsChecksum, envVars, runtimeVolumeMount, resources)
 		if err != nil {
 			return err
 		}
@@ -404,7 +412,7 @@ func populatePodSpec(funcObj *kubelessApi.Function, lr *langruntime.Langruntimes
 
 	// add compilation init container if needed
 	_, funcName, _ := splitHandler(funcObj.Spec.Handler)
-	compContainer, err := lr.GetCompilationContainer(funcObj.Spec.Runtime, funcName, envVars, runtimeVolumeMount)
+	compContainer, err := lr.GetCompilationContainer(funcObj.Spec.Runtime, funcName, envVars, runtimeVolumeMount, resources)
 	if err != nil {
 		return err
 	}
diff --git a/script/validate-vet b/script/validate-vet
index 2e8e018b..146e47f8 100755
--- a/script/validate-vet
+++ b/script/validate-vet
@@ -23,7 +23,7 @@ unset IFS
 failed=0
 for f in "${files[@]}"; do
 	# we use "git show" here to validate that what's committed passes go tool vet
-	if ! go tool vet "$f"; then
+	if ! go vet "$f"; then
 		failed=1
 	fi
 done


Test patch:
diff --git a/pkg/langruntime/langruntime_test.go b/pkg/langruntime/langruntime_test.go
index 37b0c397..eab306e1 100644
--- a/pkg/langruntime/langruntime_test.go
+++ b/pkg/langruntime/langruntime_test.go
@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/client-go/kubernetes/fake"
 )
 
@@ -105,14 +106,15 @@ func TestGetBuildContainer(t *testing.T) {
 	lr.ReadConfigMap()
 
 	// It should throw an error if there is not an image available
-	_, err := lr.GetBuildContainer("notExists", "", []v1.EnvVar{}, v1.VolumeMount{})
+	_, err := lr.GetBuildContainer("notExists", "", []v1.EnvVar{}, v1.VolumeMount{}, v1.ResourceRequirements{})
 	if err == nil {
 		t.Error("Expected to throw an error")
 	}
 
 	// It should return the proper build image for python
 	vol1 := v1.VolumeMount{Name: "v1", MountPath: "/v1"}
-	c, err := lr.GetBuildContainer("python2.7", "abc123", []v1.EnvVar{}, vol1)
+	resources := v1.ResourceRequirements{Limits: v1.ResourceList{v1.ResourceLimitsCPU: resource.MustParse("100m")}}
+	c, err := lr.GetBuildContainer("python2.7", "abc123", []v1.EnvVar{}, vol1, resources)
 	if err != nil {
 		t.Errorf("Unexpected error: %s", err)
 	}
@@ -128,6 +130,7 @@ func TestGetBuildContainer(t *testing.T) {
 			{Name: "KUBELESS_INSTALL_VOLUME", Value: "/v1"},
 			{Name: "KUBELESS_DEPS_FILE", Value: "/v1/requirements.txt"},
 		},
+		Resources: v1.ResourceRequirements{Limits: v1.ResourceList{v1.ResourceLimitsCPU: resource.MustParse("100m")}},
 	}
 	if !reflect.DeepEqual(expectedContainer, c) {
 		t.Errorf("Unexpected result. Expecting:\n %+v\nReceived:\n %+v", expectedContainer, c)
diff --git a/pkg/utils/kubelessutil_test.go b/pkg/utils/kubelessutil_test.go
index 9cdccc16..a33d0d06 100644
--- a/pkg/utils/kubelessutil_test.go
+++ b/pkg/utils/kubelessutil_test.go
@@ -11,6 +11,7 @@ import (
 
 	"k8s.io/api/core/v1"
 	"k8s.io/api/extensions/v1beta1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes/fake"
@@ -518,6 +519,17 @@ func TestEnsureDeployment(t *testing.T) {
 		"bar": "foo",
 	}
 	f1 := getDefaultFunc(f1Name, ns)
+
+	f1.Spec.Deployment.Spec.Template.Spec.InitContainers = []v1.Container{
+		{
+			Resources: v1.ResourceRequirements{
+				Limits: v1.ResourceList{
+					v1.ResourceLimitsCPU: resource.MustParse("100m"),
+				},
+			},
+		},
+	}
+
 	f1Port := f1.Spec.ServiceSpec.Ports[0].Port
 	f1.ObjectMeta.Labels = funcLabels
 	f1.Spec.Deployment.ObjectMeta = metav1.ObjectMeta{
@@ -640,6 +652,10 @@ func TestEnsureDeployment(t *testing.T) {
 	if dpm.Spec.Template.Spec.InitContainers[0].Image != "unzip" {
 		t.Errorf("Unexpected init image %s", dpm.Spec.Template.Spec.InitContainers[0].Image)
 	}
+	if dpm.Spec.Template.Spec.InitContainers[0].Resources.Limits == nil {
+		t.Errorf("Resources must be set for init container")
+	}
+
 }
 
 func TestEnsureDeploymentWithoutFuncNorHandler(t *testing.T) {
@@ -848,7 +864,9 @@ func TestGetProvisionContainer(t *testing.T) {
 
 	rvol := v1.VolumeMount{Name: "runtime", MountPath: "/runtime"}
 	dvol := v1.VolumeMount{Name: "deps", MountPath: "/deps"}
-	c, err := getProvisionContainer("test", "sha256:abc1234", "test.func", "test.foo", "text", "python2.7", "unzip", rvol, dvol, lr)
+	resources := v1.ResourceRequirements{Limits: v1.ResourceList{v1.ResourceLimitsCPU: resource.MustParse("100m")}}
+
+	c, err := getProvisionContainer("test", "sha256:abc1234", "test.func", "test.foo", "text", "python2.7", "unzip", rvol, dvol, resources, lr)
 	if err != nil {
 		t.Errorf("Unexpected error: %s", err)
 	}
@@ -859,13 +877,14 @@ func TestGetProvisionContainer(t *testing.T) {
 		Args:            []string{"echo 'abc1234  /deps/test.func' > /tmp/func.sha256 && sha256sum -c /tmp/func.sha256 && cp /deps/test.func /runtime/test.py && cp /deps/requirements.txt /runtime"},
 		VolumeMounts:    []v1.VolumeMount{rvol, dvol},
 		ImagePullPolicy: v1.PullIfNotPresent,
+		Resources:       v1.ResourceRequirements{Limits: v1.ResourceList{v1.ResourceLimitsCPU: resource.MustParse("100m")}},
 	}
 	if !reflect.DeepEqual(expectedContainer, c) {
 		t.Errorf("Unexpected result:\n %+v", c)
 	}
 
 	// If the content type is encoded it should decode it
-	c, err = getProvisionContainer("Zm9vYmFyCg==", "sha256:abc1234", "test.func", "test.foo", "base64", "python2.7", "unzip", rvol, dvol, lr)
+	c, err = getProvisionContainer("Zm9vYmFyCg==", "sha256:abc1234", "test.func", "test.foo", "base64", "python2.7", "unzip", rvol, dvol, resources, lr)
 	if err != nil {
 		t.Errorf("Unexpected error: %s", err)
 	}
@@ -883,7 +902,7 @@ func TestGetProvisionContainer(t *testing.T) {
 	}
 
 	// It should skip the dependencies installation if the runtime is not supported
-	c, err = getProvisionContainer("function", "sha256:abc1234", "test.func", "test.foo", "text", "cobol", "unzip", rvol, dvol, lr)
+	c, err = getProvisionContainer("function", "sha256:abc1234", "test.func", "test.foo", "text", "cobol", "unzip", rvol, dvol, resources, lr)
 	if err != nil {
 		t.Errorf("Unexpected error: %s", err)
 	}
@@ -892,13 +911,13 @@ func TestGetProvisionContainer(t *testing.T) {
 	}
 
 	// It should extract the file in case it is a Zip
-	c, err = getProvisionContainer("Zm9vYmFyCg==", "sha256:abc1234", "test.zip", "test.foo", "base64+zip", "python2.7", "unzip", rvol, dvol, lr)
+	c, err = getProvisionContainer("Zm9vYmFyCg==", "sha256:abc1234", "test.zip", "test.foo", "base64+zip", "python2.7", "unzip", rvol, dvol, resources, lr)
 	if !strings.Contains(c.Args[0], "unzip -o /tmp/func.decoded -d /runtime") {
 		t.Errorf("Unexpected command: %s", c.Args[0])
 	}
 
 	// If the content type is url it should use curl
-	c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url", "python2.7", "unzip", rvol, dvol, lr)
+	c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url", "python2.7", "unzip", rvol, dvol, resources, lr)
 	if err != nil {
 		t.Errorf("Unexpected error: %s", err)
 	}
@@ -907,7 +926,7 @@ func TestGetProvisionContainer(t *testing.T) {
 	}
 
 	// If the content type is url it should use curl
-	c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url+zip", "python2.7", "unzip", rvol, dvol, lr)
+	c, err = getProvisionContainer("https://raw.githubusercontent.com/test/test/test/test.py", "sha256:abc1234", "", "test.foo", "url+zip", "python2.7", "unzip", rvol, dvol, resources, lr)
 	if err != nil {
 		t.Errorf("Unexpected error: %s", err)
 	}