Repository: flant/shell-operator
Pull request #78 (2019-11-26 11:43:39)

test: add unit and integration tests as Github Actions


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 47c9093..c659ccc 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -12,9 +12,9 @@ jobs:
     working_directory: /tmp/flant/shell-operator
     steps:
       - checkout
-      - run: go mod download
-      - run: go build -v ./cmd/shell-operator
-      - run: go test -race -v ./pkg/...
+      #- run: go mod download
+      #- run: go build -v ./cmd/shell-operator
+      #- run: go test -race -v ./...
       # TODO add vet, lint, speller
 
   build_image_branch:
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
new file mode 100644
index 0000000..80409c4
--- /dev/null
+++ b/.github/workflows/main.yaml
@@ -0,0 +1,156 @@
+name: Main
+on: [push]
+jobs:
+  download_go_modules:
+    name: Download modules
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go 1.12
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.12
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - name: Download modules
+        run: go mod download
+        shell: bash
+
+      - name: Pack go modules
+        run: tar -czvf go_modules.tar.gz -C $HOME/go/pkg/mod .
+        shell: bash
+
+
+
+      # FIXME: https://github.community/t5/GitHub-Actions/Caching-files-between-GitHub-Action-executions/m-p/30974#M630
+      - name: Upload go modules artifact
+        uses: actions/upload-artifact@master
+        with:
+          name: go_modules
+          path: go_modules.tar.gz
+
+      - name: Build
+        run: go build -v ./cmd/shell-operator
+        shell: bash
+
+#
+#      - name: Upload binary artifact
+#        uses: actions/upload-artifact@master
+#        with:
+#          name: shell-operator
+#          path: shell-operator.tar.gz
+#
+#      - name: Upload unit test binary artifact
+#        uses: actions/upload-artifact@master
+#        with:
+#          name: unit_test
+#          path: unit_test.tar.gz
+
+
+  build:
+    name: Build and test
+    needs: download_go_modules
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Set up Go 1.12
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.12
+        id: go
+
+      - name: Download go modules artifact
+        uses: actions/download-artifact@master
+        with:
+          name: go_modules
+          path: .
+
+      - name: Unpack go modules
+        run: mkdir -p $HOME/go/pkg/mod && tar -xzvf go_modules.tar.gz -C $HOME/go/pkg/mod
+        shell: bash
+
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v1
+
+      - name: Build
+        run: go build -v ./cmd/shell-operator
+        shell: bash
+
+  unit_tests:
+    name: Unit tests
+    needs: download_go_modules
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Set up Go 1.12
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.12
+        id: go
+
+      - name: Download go modules artifact
+        uses: actions/download-artifact@master
+        with:
+          name: go_modules
+          path: .
+
+      - name: Unpack go modules
+        run: mkdir -p $HOME/go/pkg/mod && tar -xzvf go_modules.tar.gz -C $HOME/go/pkg/mod
+        shell: bash
+
+      - name: Prepare environment
+        run: |
+          export COVERAGE_DIR=$GITHUB_WORKSPACE/tests_coverage/unit_tests/${{ matrix.os }}
+          mkdir -p $COVERAGE_DIR
+          echo ::set-env name=COVERAGE_DIR::$COVERAGE_DIR
+        shell: bash
+
+      - name: Checkout code
+        uses: actions/checkout@v1
+
+      - name: Test shell-operator packages
+        run: go test -tags test -v ./cmd/... ./pkg/... ./test/utils
+
+  integration_tests:
+    name: Integration tests
+    needs: download_go_modules
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-latest]
+        k8s_version: [1.13, 1.16]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Set up Go 1.12
+        uses: actions/setup-go@v1
+        with:
+          go-version: 1.12
+        id: go
+
+      - name: Download go modules artifact
+        uses: actions/download-artifact@master
+        with:
+          name: go_modules
+          path: .
+
+      - name: Unpack go modules
+        run: mkdir -p $HOME/go/pkg/mod && tar -xzvf go_modules.tar.gz -C $HOME/go/pkg/mod
+        shell: bash
+
+      - name: Checkout code
+        uses: actions/checkout@v1
+
+      - name: Prepare environment
+        run: |
+          go build github.com/onsi/ginkgo/ginkgo
+          echo ::set-env name=KIND_CLUSTER_VERSION::${{ matrix.k8s_version }}
+# race flag ?
+      - name: Run integration tests
+        run: |
+          ./ginkgo --tags 'integration test' -p -r test/integration
diff --git a/cmd/shell-operator/main.go b/cmd/shell-operator/main.go
index 3dc9848..ecb7a18 100644
--- a/cmd/shell-operator/main.go
+++ b/cmd/shell-operator/main.go
@@ -45,7 +45,9 @@ func main() {
 			operator.Run()
 
 			// Block action by waiting signals from OS.
-			utils_signal.WaitForProcessInterruption()
+			utils_signal.WaitForProcessInterruption(func() {
+				operator.Stop()
+			})
 
 			return nil
 		})
diff --git a/go.mod b/go.mod
index f6715d5..5cd16b2 100644
--- a/go.mod
+++ b/go.mod
@@ -11,18 +11,23 @@ require (
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/kennygrant/sanitize v1.2.4
+	github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega v1.5.0
 	github.com/prometheus/client_golang v1.0.0
 	github.com/sirupsen/logrus v1.4.2
 	github.com/stretchr/testify v1.4.0
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	golang.org/x/tools v0.0.0-20190617190820-da514acc4774
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5
 	gopkg.in/satori/go.uuid.v1 v1.2.0
-	gopkg.in/yaml.v2 v2.2.2
+	gopkg.in/yaml.v2 v2.2.4
+	k8s.io/api v0.0.0-20190409092523-d687e77c8ae9
 	k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b
 	k8s.io/client-go v0.0.0-20190411052641-7a6b4715b709
 	k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 // indirect
+	sigs.k8s.io/kind v0.0.0-20191024075107-85a46de11816
 )
 
 // replace github.com/go-openapi/validate => ../go-openapi-validate
diff --git a/go.sum b/go.sum
index eb8f2b8..4d8da54 100644
--- a/go.sum
+++ b/go.sum
@@ -2,21 +2,27 @@ cloud.google.com/go v0.0.0-20160913182117-3b1ae45394a2/go.mod h1:aQUYkXzVsufM+Dw
 cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/Azure/go-autorest v11.1.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053 h1:H/GMMKYPkEIC3DF/JWQz8Pdd+Feifov2EIgGfNpeogI=
+github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053/go.mod h1:xW8sBma2LE3QxFSzCnH9qe6gAE2yO9GvQaWwX89HxbE=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -24,17 +30,23 @@ github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550 h1:mV9jbLoSW/8m4VK16ZkHTozJa8sesK5u5kTMFysTYac=
 github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
+github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f h1:HTLgtnIbx2CVK74aTk1D4XbOh6tOHsPShw7UPOJTQzM=
 github.com/flant/go-openapi-validate v0.19.4-0.20190926112101-38fbca4ac77f/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is=
 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
 github.com/go-openapi/analysis v0.19.2 h1:ophLETFestFZHk3ji7niPEL4d466QjW+0Tdg5VyDq7E=
 github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
@@ -43,11 +55,13 @@ github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9sn
 github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
 github.com/go-openapi/errors v0.19.2 h1:a2kIyV3w+OS3S97zxUndRVD46+FhGOUBDFY7nmu4CsY=
 github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
 github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
 github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
 github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
 github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
 github.com/go-openapi/jsonreference v0.19.2 h1:o20suLFB4Ri0tuzpWtyHlh7E7HnkqTNLq6aR6WVNS1w=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
@@ -56,6 +70,7 @@ github.com/go-openapi/loads v0.19.2 h1:rf5ArTHmIJxyV5Oiks+Su0mUens1+AjpkPoWr5xFR
 github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
 github.com/go-openapi/runtime v0.19.4 h1:csnOgcgAiuGoM/Po7PEpKDoNulCcF3FGbSnbHfxgjMI=
 github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
 github.com/go-openapi/spec v0.19.2 h1:SStNd1jRcYtfKCN7R0laGNs80WYYvn5CbBjM2sOmCrE=
 github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
@@ -66,6 +81,7 @@ github.com/go-openapi/strfmt v0.19.0 h1:0Dn9qy1G9+UJfRU7TR8bmdGxb4uifB7HNrJjOnV0
 github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
 github.com/go-openapi/strfmt v0.19.2 h1:clPGfBnJohokno0e+d7hs6Yocrzjlgz6EsQSDncCRnE=
 github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
 github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
@@ -75,21 +91,27 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903 h1:LbsanbbD6LieFkXbj9YNNBupiGHJgFeLpO0j0Fza1h8=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d h1:7XGaL1e6bYS1yIonGp9761ExpPPV1ui0SAC59Yube9k=
 github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.3.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
 github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
 github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@@ -103,12 +125,16 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
 github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -118,9 +144,11 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190620125010-da37f6c1e481/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
@@ -129,17 +157,28 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3 h1:EooPXg51Tn+xmWPXJUGCnJhJSpeuMlBmfJVcqIRmmv8=
 github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
@@ -154,13 +193,21 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -180,6 +227,7 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8 h1:1wopBVtVdWnn03fZelqdXT
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k=
 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -191,6 +239,7 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
 golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20170412232759-a6bd8cefa181/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -200,6 +249,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -210,6 +260,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f h1:25KHgbfyiSm6vwQLbM3zZIe1v9p/3ea4Rz+nnM5K/i4=
 golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190621203818-d432491b9138/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -218,9 +270,12 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
 golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190617190820-da514acc4774 h1:CQVOmarCBFzTx0kbOU0ru54Cvot8SdSrNYjZPhQl+gk=
 golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -236,6 +291,7 @@ gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o=
 gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY=
 gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE=
 gopkg.in/satori/go.uuid.v1 v1.2.0 h1:AH9uksa7bGe9rluapecRKBCpZvxaBEyu0RepitcD0Hw=
@@ -245,18 +301,38 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652 h1:VKvJ/mQ4BgCjZUDggYFxTe0qv9jPMHsZPD4Xt91Y5H4=
+gopkg.in/yaml.v3 v3.0.0-20191010095647-fc94e3f71652/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+k8s.io/api v0.0.0-20190313235455-40a48860b5ab/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
+k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
 k8s.io/api v0.0.0-20190409092523-d687e77c8ae9 h1:c9UEl5z8gk1DGh/g3snETZ+a52YeR9VdbX/3BQ4PHas=
 k8s.io/api v0.0.0-20190409092523-d687e77c8ae9/go.mod h1:FQEUn50aaytlU65qqBn/w+5ugllHwrBzKm7DzbnXdzE=
+k8s.io/apimachinery v0.0.0-20190313205120-d7deff9243b1/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
+k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
 k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b h1:fVkKJL9FIpA8LSJyHVM00MP45q1WJ7+af77vcxmQP4g=
 k8s.io/apimachinery v0.0.0-20190409092423-760d1845f48b/go.mod h1:FW86P8YXVLsbuplGMZeb20J3jYHscrDqw4jELaFJvRU=
 k8s.io/client-go v0.0.0-20190411052641-7a6b4715b709 h1:5ROFGMvCLRe8mkMzReTZI/LD8A/V6h6QgNKzHW7Bbu0=
 k8s.io/client-go v0.0.0-20190411052641-7a6b4715b709/go.mod h1:4IOfimLkjvlSoc9wyI1VEwkNUG20XFNp7qO6XkH2gdI=
+k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
+k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
 k8s.io/klog v0.0.0-20190306015804-8e90cee79f82 h1:SHucoAy7lRb+w5oC/hbXyZg+zX+Wftn6hD4tGzHCVqA=
 k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30 h1:TRb4wNWoBVrH9plmkp2q86FIDppkbrEXdXlxU3a3BMI=
 k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
+k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4=
 k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
 k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7 h1:8r+l4bNWjRlsFYlQJnKJ2p7s1YQPj4XyXiJVqDHRx7c=
 k8s.io/utils v0.0.0-20190308190857-21c4ce38f2a7/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
+sigs.k8s.io/kind v0.0.0-20191024075107-85a46de11816 h1:A/Z5WQbGDmbOj5umRxpstEBdmGAO01pp5qpupNbKfjs=
+sigs.k8s.io/kind v0.0.0-20191024075107-85a46de11816/go.mod h1:OLjBz2mDaA0leJCal/rumRCymPWFD4VWXaw+i8XZSa4=
+sigs.k8s.io/kind v0.5.1/go.mod h1:L+Kcoo83/D1+ryU5P2VFbvYm0oqbkJn9zTZq0KNxW68=
+sigs.k8s.io/kustomize/v3 v3.1.1-0.20190821175718-4b67a6de1296/go.mod h1:ztX4zYc/QIww3gSripwF7TBOarBTm5BvyAMem0kCzOE=
+sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
diff --git a/pkg/shell-operator/operator.go b/pkg/shell-operator/operator.go
index c6452f7..132c5ca 100644
--- a/pkg/shell-operator/operator.go
+++ b/pkg/shell-operator/operator.go
@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -58,14 +59,10 @@ var (
 func Init() (err error) {
 	log.Debug("MAIN Init")
 
-	if app.WorkingDir != "" {
-		WorkingDir = app.WorkingDir
-	} else {
-		WorkingDir, err = os.Getwd()
-		if err != nil {
-			log.Errorf("MAIN Fatal: Cannot determine a working dir: %s", err)
-			return err
-		}
+	WorkingDir, err := filepath.Abs(app.WorkingDir)
+	if err != nil {
+		log.Errorf("MAIN Fatal: Cannot determine a current dir: %s", err)
+		return err
 	}
 	if exists, _ := utils_file.DirExists(WorkingDir); !exists {
 		log.Errorf("MAIN Fatal: working dir '%s' is not exists", WorkingDir)
@@ -163,6 +160,14 @@ func Run() {
 	go TasksRunner()
 }
 
+// Stop closes event handle loop and pushes Stop task to wait until current task is done.
+func Stop() {
+	StopHandleEventsFromManagersCh <- struct{}{}
+	// FIXME: Double push to prevent Pop in task handlers.
+	TasksQueue.Push(task.NewTask(task.Stop, "Stop"))
+	TasksQueue.Push(task.NewTask(task.Stop, "Stop"))
+}
+
 func HandleEventsFromManagers() {
 	for {
 		select {
diff --git a/pkg/utils/signal/signal.go b/pkg/utils/signal/signal.go
index 877101b..3b4fc72 100644
--- a/pkg/utils/signal/signal.go
+++ b/pkg/utils/signal/signal.go
@@ -1,20 +1,49 @@
 package utils
 
 import (
-	log "github.com/sirupsen/logrus"
 	"os"
 	"os/signal"
 	"syscall"
+
+	log "github.com/sirupsen/logrus"
 )
 
-func WaitForProcessInterruption() {
+// WaitForProcessInterruption wait for SIGINT or SIGTERM and run a callback function.
+//
+// First signal start a callback function, which should call os.Exit(0).
+// Next signal will call os.Exit(128 + signal-value).
+// If no cb is given,
+func WaitForProcessInterruption(cb ...func()) {
+	allowedCount := 1
 	interruptCh := make(chan os.Signal, 1)
+
+	forcedExit := func(s os.Signal) {
+		log.Infof("Forced shutdown by '%s' signal", s.String())
+
+		signum := 0
+		switch v := s.(type) {
+		case syscall.Signal:
+			signum = int(v)
+		}
+		os.Exit(128 + signum)
+	}
+
 	signal.Notify(interruptCh, syscall.SIGINT, syscall.SIGTERM)
 	for {
 		select {
 		case sig := <-interruptCh:
-			log.Infof("Grace shutdown with %s signal", sig.String())
-			return
+			allowedCount--
+			switch allowedCount {
+			case 0:
+				if len(cb) > 0 {
+					log.Infof("Grace shutdown by '%s' signal", sig.String())
+					cb[0]()
+				} else {
+					forcedExit(sig)
+				}
+			case -1:
+				forcedExit(sig)
+			}
 		}
 	}
 }


Test patch:
diff --git a/pkg/hook/hook_binding_context_test.go b/pkg/hook/hook_binding_context_test.go
index 16bb3f1..336efc8 100644
--- a/pkg/hook/hook_binding_context_test.go
+++ b/pkg/hook/hook_binding_context_test.go
@@ -47,7 +47,7 @@ func Test_BindingContext_Convert_V1(t *testing.T) {
 	assert.Equal(t, "", bc1_0.FilterResult)
 
 	data, err := json.Marshal(bc1_0)
-	if assert.NoError(t, err) {
+	if !assert.NoError(t, err) {
 		t.Logf("bindingContext: %v", string(data))
 	}
 }
@@ -79,7 +79,7 @@ func Test_BindingContext_Convert_V0(t *testing.T) {
 	assert.Equal(t, "add", bc0_0.ResourceEvent)
 
 	data, err := json.Marshal(bc0_0)
-	if assert.NoError(t, err) {
+	if !assert.NoError(t, err) {
 		t.Logf("bindingContext: %v", string(data))
 	}
 }
diff --git a/pkg/hook/kube_event/hooks_controller_test.go b/pkg/hook/kube_event/hooks_controller_test.go
index b95cf64..d70f4df 100644
--- a/pkg/hook/kube_event/hooks_controller_test.go
+++ b/pkg/hook/kube_event/hooks_controller_test.go
@@ -32,14 +32,14 @@ func (m *MockKubeEventsManager) Start() {
 	return
 }
 
-func (m *MockKubeEventsManager) StopMonitor(configId string) error {
-	return nil
-}
-
 func (m *MockKubeEventsManager) StartMonitor(configId string) {
 	return
 }
 
+func (m *MockKubeEventsManager) StopMonitor(configId string) error {
+	return nil
+}
+
 func (m *MockKubeEventsManager) Ch() chan kube_events_manager.KubeEvent {
 	return nil
 }
diff --git a/pkg/kube/client_test.go b/pkg/kube/client_test.go
index 5cd28b9..3828311 100644
--- a/pkg/kube/client_test.go
+++ b/pkg/kube/client_test.go
@@ -1,54 +1,36 @@
 package kube
 
 import (
-	"fmt"
 	"testing"
 
-	"k8s.io/apimachinery/pkg/runtime/schema"
+	"github.com/stretchr/testify/assert"
 )
 
-func Test_Discover(t *testing.T) {
-	// Test works only with real cluster.
-	t.SkipNow()
-
-	err := Init(InitOptions{})
-	if err != nil {
-		t.Error(err)
-	}
-
-	lists, err := Kubernetes.Discovery().ServerPreferredResources()
-	if err != nil {
-		t.Error(err)
+func Test_EqualOneOf(t *testing.T) {
+	tests := []struct {
+		name    string
+		term    string
+		choices []string
+		result  bool
+	}{
+		{
+			"1",
+			"Pod",
+			[]string{"pod", "deployment"},
+			true,
+		},
+		{
+			"2",
+			"Pod",
+			[]string{"pood", "deployment"},
+			false,
+		},
 	}
 
-	fmt.Printf("Got %d lists\n", len(lists))
-
-	for _, list := range lists {
-		fmt.Printf("\n\n")
-		fmt.Printf("%s has %d resources\n", list.GroupVersion, len(list.APIResources))
-
-		if len(list.APIResources) == 0 {
-			continue
-		}
-
-		gv, err := schema.ParseGroupVersion(list.GroupVersion)
-		if err != nil {
-			t.Error(err)
-			continue
-		}
-
-		for _, resource := range list.APIResources {
-			gvr := schema.GroupVersionResource{
-				Resource: resource.Name,
-				Group:    gv.Group,
-				Version:  gv.Version,
-			}
-
-			fmt.Printf("%30s %30s %30s %d:%+v\n",
-				gvr.GroupVersion().String(),
-				resource.Kind,
-				fmt.Sprintf("%+v", append([]string{resource.Name}, resource.ShortNames...)),
-				len(resource.Verbs), resource.Verbs)
-		}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			res := equalToOneOf(tt.term, tt.choices...)
+			assert.Equal(t, tt.result, res)
+		})
 	}
 }
diff --git a/pkg/kube_events_manager/kube_events_manager_test.go b/pkg/kube_events_manager/kube_events_manager_test.go
index 18f87a4..9abceda 100644
--- a/pkg/kube_events_manager/kube_events_manager_test.go
+++ b/pkg/kube_events_manager/kube_events_manager_test.go
@@ -85,6 +85,8 @@ func Test_MainKubeEventsManager_Run(t *testing.T) {
 
 // FIXME: sometimes fails, skip for now.
 //
+// TODO: fake client is limited, move it to integration tests
+//
 // Test_MainKubeEventsManager_HandleEvents
 // Scenario:
 // - create new KubeEventManager, start informers
@@ -250,3 +252,194 @@ func Test_MainKubeEventsManager_HandleEvents(t *testing.T) {
 	}
 
 }
+
+// Test_FakeClient_CatchUpdates try to catch updates from fake client with additional WatchReactor
+func Test_FakeClient_CatchUpdates(t *testing.T) {
+	t.SkipNow()
+	timeout := time.Duration(30 * time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	// Add GVR
+	kube.Kubernetes = fake.NewSimpleClientset()
+	fakeDiscovery, ok := kube.Kubernetes.Discovery().(*fakediscovery.FakeDiscovery)
+	if !ok {
+		t.Fatalf("couldn't convert Discovery() to *FakeDiscovery")
+	}
+
+	fakeDiscovery.FakedServerVersion = &version.Info{
+		GitCommit: "v1.0.0",
+	}
+
+	podGvr := schema.GroupVersionResource{
+		Group:    "",
+		Version:  "v1",
+		Resource: "pods",
+	}
+
+	fakeDiscovery.Resources = []*metav1.APIResourceList{
+		&metav1.APIResourceList{
+			GroupVersion: "v1",
+			APIResources: []metav1.APIResource{
+				{
+					Kind:    "Pod",
+					Name:    "pods",
+					Verbs:   metav1.Verbs{"get", "list", "watch", "create", "patch", "update"},
+					Group:   "",
+					Version: "v1",
+				},
+			},
+		},
+	}
+
+	// Configure dynamic client
+	scheme := runtime.NewScheme()
+	objs := []runtime.Object{}
+
+	dc := fakedynamic.NewSimpleDynamicClient(scheme, objs...)
+	/*
+		dc.PrependWatchReactor("*", func(action testing2.Action) (handled bool, ret watch.Interface, err error) {
+			switch v := action.(type) {
+			case testing2.CreateAction:
+				gvr := action.GetResource()
+				obj := v.GetObject()
+				fmt.Printf("create object: %s %+v\n", gvr.String(), obj)
+			case testing2.UpdateAction:
+				gvr := action.GetResource()
+				obj := v.GetObject()
+
+				name, _, _ := metaFromEventObject(obj)
+
+				oldObj, _ := dc.Resource(gvr).Namespace(action.GetNamespace()).Get(name, metav1.GetOptions{})
+
+				fmt.Printf("updated object: %s\n>>> old: %+v\n>>> new: %+v\n", gvr.String(), oldObj.Object, obj)
+
+			default:
+				return false, nil, nil
+			}
+			return false, nil, nil
+		})
+	*/
+	kube.DynamicClient = dc
+
+	obj := &unstructured.Unstructured{
+		Object: map[string]interface{}{
+			"apiVersion": "v1",
+			"kind":       "Pod",
+			"metadata": map[string]interface{}{
+				"namespace": "default",
+				"name":      "pod-0",
+			},
+			"spec": "pod-0",
+		},
+	}
+	_, _ = kube.DynamicClient.Resource(podGvr).Namespace("default").Create(obj, metav1.CreateOptions{}, []string{}...)
+
+	//// Init() replacement
+	mgr := NewKubeEventsManager()
+	mgr.WithContext(ctx)
+	KubeEventCh = make(chan KubeEvent, 10)
+
+	// monitor with 3 namespaces and 4 object names and all event types
+	monitor := &MonitorConfig{
+		ApiVersion: "v1",
+		Kind:       "Pod",
+		EventTypes: []WatchEventType{WatchEventAdded, WatchEventModified, WatchEventDeleted},
+		NamespaceSelector: &NamespaceSelector{
+			NameSelector: &NameSelector{
+				MatchNames: []string{"default"},
+			},
+		},
+	}
+	monitor.Metadata.ConfigId = "ConfigId"
+
+	_, err := mgr.AddMonitor("test", monitor, log.WithField("test", "yes"))
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	mgr.Start()
+
+	fmt.Printf("mgr Started\n")
+
+	// First event — Synchronization, second and third are Event
+	eventCounter := 0
+	done := false
+	state := struct {
+		podsCreated bool
+		gotPod1     bool
+		gotPod2     bool
+	}{}
+	for {
+		fmt.Printf("Start select\n")
+		select {
+		case ev := <-KubeEventCh:
+			eventCounter = eventCounter + 1
+			fmt.Printf("Got event: %d %#v\n", eventCounter, ev)
+			//t.Logf("Got event: %d %#v\n", eventCounter, ev)
+
+			if !state.podsCreated {
+				assert.Equal(t, "Synchronization", ev.Type)
+				assert.Equal(t, "ConfigId", ev.ConfigId)
+				assert.Len(t, ev.Objects, 1)
+
+				obj.Object["spec"] = "pod-0-new-spec"
+
+				_, _ = kube.DynamicClient.Resource(podGvr).Namespace("default").Update(obj, metav1.UpdateOptions{}, []string{}...)
+
+				// Inject an event into the fake client.
+				obj1 := &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "Pod",
+						"metadata": map[string]interface{}{
+							"namespace": "default",
+							"name":      "pod-1",
+						},
+						"spec": "pod-1",
+					},
+				}
+				_, _ = kube.DynamicClient.Resource(podGvr).Namespace("default").Create(obj1, metav1.CreateOptions{}, []string{}...)
+				// Inject second event into the fake client.
+				obj2 := &unstructured.Unstructured{
+					Object: map[string]interface{}{
+						"apiVersion": "v1",
+						"kind":       "Pod",
+						"metadata": map[string]interface{}{
+							"namespace": "default",
+							"name":      "pod-2",
+						},
+						"spec": "pod-2",
+					},
+				}
+				_, _ = kube.DynamicClient.Resource(podGvr).Namespace("default").Create(obj2, metav1.CreateOptions{}, []string{}...)
+				t.Logf("DynamicClient Created pod\n")
+				state.podsCreated = true
+				break
+			}
+
+			assert.Equal(t, "Event", ev.Type)
+			assert.Equal(t, "ConfigId", ev.ConfigId)
+			assert.Equal(t, WatchEventAdded, ev.WatchEvents[0])
+			metadata := ev.Object["metadata"].(map[string]interface{})
+			assert.Contains(t, metadata, "name")
+			if metadata["name"] == "pod-1" {
+				state.gotPod1 = true
+			}
+			if metadata["name"] == "pod-2" {
+				state.gotPod2 = true
+			}
+			if state.gotPod1 && state.gotPod2 {
+				done = true
+			}
+		case <-ctx.Done():
+			t.Error("Kube events manager did not get the added pod")
+			done = true
+		}
+		if done {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+}
diff --git a/test/integration/kubeclient/kube_client_test.go b/test/integration/kubeclient/kube_client_test.go
new file mode 100644
index 0000000..3aa00c7
--- /dev/null
+++ b/test/integration/kubeclient/kube_client_test.go
@@ -0,0 +1,31 @@
+// +build integration
+
+package kubeclient_test
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/flant/shell-operator/pkg/kube"
+)
+
+var _ = Describe("Kubernetes API client package", func() {
+	When("client connect outside of the cluster", func() {
+
+		It("should list deployments", func() {
+			list, err := kube.Kubernetes.AppsV1().Deployments("").List(metav1.ListOptions{})
+			Ω(err).Should(Succeed())
+			Ω(list.Items).Should(Not(HaveLen(0)))
+		})
+
+		It("should find GroupVersionResource for Pod by kind", func() {
+			gvr, err := kube.GroupVersionResourceByKind("Pod")
+			Ω(err).Should(Succeed())
+			Ω(gvr.Resource).Should(Equal("pods"))
+			Ω(gvr.Group).Should(Equal(""))
+			Ω(gvr.Version).Should(Equal("v1"))
+		})
+	})
+})
diff --git a/test/integration/kubeclient/suite_test.go b/test/integration/kubeclient/suite_test.go
new file mode 100644
index 0000000..005fc73
--- /dev/null
+++ b/test/integration/kubeclient/suite_test.go
@@ -0,0 +1,41 @@
+// +build integration
+
+package kubeclient_test
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/flant/shell-operator/pkg/kube"
+	. "github.com/flant/shell-operator/test/utils"
+)
+
+var ClusterName = "kube-client-test"
+var ConfigPath string
+
+func TestSuite(t *testing.T) {
+	clusterVer := KindClusterVersion()
+	if clusterVer != "" {
+		ClusterName = fmt.Sprintf("%s-%s", ClusterName, clusterVer)
+	}
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "kube client suite")
+}
+
+var _ = SynchronizedBeforeSuite(func() []byte {
+	Ω(KindCreateCluster(ClusterName)).Should(Succeed())
+	fmt.Printf("Use kind flavour of k8s cluster v%s with node image %s\n", KindClusterVersion(), KindNodeImage())
+	return []byte{}
+}, func([]byte) {
+	// Initialize kube client out-of-cluster
+	ConfigPath = KindGetKubeconfigPath(ClusterName)
+	Ω(kube.Init(kube.InitOptions{KubeContext: "", KubeConfig: ConfigPath})).Should(Succeed())
+})
+
+var _ = SynchronizedAfterSuite(func() {}, func() {
+	Ω(KindDeleteCluster(ClusterName)).Should(Succeed())
+})
diff --git a/test/utils/assert.go b/test/utils/assert.go
new file mode 100644
index 0000000..8077bb0
--- /dev/null
+++ b/test/utils/assert.go
@@ -0,0 +1,25 @@
+// +build test
+
+package utils
+
+import "strings"
+
+func HasField(m map[string]string, key string) bool {
+	_, ok := m[key]
+	return ok
+}
+
+func FieldContains(m map[string]string, key string, s string) bool {
+	v, ok := m[key]
+	return ok && strings.Contains(v, s)
+}
+
+func FieldHasPrefix(m map[string]string, key string, s string) bool {
+	v, ok := m[key]
+	return ok && strings.HasPrefix(v, s)
+}
+
+func FieldEquals(m map[string]string, key string, s string) bool {
+	v, ok := m[key]
+	return ok && v == s
+}
diff --git a/test/utils/assert_test.go b/test/utils/assert_test.go
new file mode 100644
index 0000000..28a4e02
--- /dev/null
+++ b/test/utils/assert_test.go
@@ -0,0 +1,29 @@
+// +build test
+
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_MapAsserts(t *testing.T) {
+	m := map[string]string{
+		"foo": "bar",
+		"baz": "quux",
+	}
+
+	assert.True(t, HasField(m, "foo"))
+	assert.True(t, HasField(m, "baz"))
+	assert.False(t, HasField(m, "bar"))
+
+	assert.True(t, FieldEquals(m, "foo", "bar"))
+	assert.False(t, FieldEquals(m, "foo", "quux"))
+
+	assert.True(t, FieldContains(m, "baz", "uu"))
+	assert.False(t, FieldContains(m, "baz", "ZZ"))
+
+	assert.True(t, FieldHasPrefix(m, "baz", "qu"))
+	assert.False(t, FieldHasPrefix(m, "baz", "uux"))
+}
diff --git a/test/utils/directory.go b/test/utils/directory.go
new file mode 100644
index 0000000..553f0e6
--- /dev/null
+++ b/test/utils/directory.go
@@ -0,0 +1,59 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	utils_file "github.com/flant/shell-operator/pkg/utils/file"
+)
+
+// ChooseExistedDirectoryPath returns first non-empty existed directory path from arguments.
+//
+// Each argument is prefixed with current directory if is not started with /
+//
+// Example:
+// dir, err := ExtractExistedDirectoryPath(os.Getenv("DIR"), defaultDir)
+
+func ChooseExistedDirectoryPath(paths ...string) (path string, err error) {
+	for _, p := range paths {
+		if p != "" {
+			path = p
+			break
+		}
+	}
+
+	if path == "" {
+		return "", nil
+	}
+
+	path, err = ToAbsolutePath(path)
+	if err != nil {
+		return
+	}
+
+	if exists, _ := utils_file.DirExists(path); !exists {
+		return "", fmt.Errorf("no working dir")
+	}
+
+	return "", nil
+}
+
+func ToAbsolutePath(path string) (string, error) {
+	if filepath.IsAbs(path) {
+		return path, nil
+	}
+
+	if strings.HasPrefix(path, "./") {
+		cwd, _ := os.Getwd()
+		return strings.Replace(path, ".", cwd, 1), nil
+	}
+
+	p, err := filepath.Abs(path)
+	if err != nil {
+		return "", err
+	}
+
+	return p, nil
+}
diff --git a/test/utils/jsonloganalyzer.go b/test/utils/jsonloganalyzer.go
new file mode 100644
index 0000000..54ed770
--- /dev/null
+++ b/test/utils/jsonloganalyzer.go
@@ -0,0 +1,281 @@
+// +build test
+
+package utils
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/onsi/gomega/types"
+)
+
+type JsonLogAnalyzer struct {
+	Matchers      [][]*JsonLogMatcher
+	OnStopFn      func()
+	Index         int
+	finished      bool
+	groupFinished bool // flag to reset group of matchers after previous group finish
+	err           error
+}
+
+func NewJsonLogAnalyzer() *JsonLogAnalyzer {
+	return &JsonLogAnalyzer{
+		Matchers:      make([][]*JsonLogMatcher, 0),
+		Index:         0,
+		finished:      false,
+		groupFinished: true,
+	}
+}
+
+func (r *JsonLogAnalyzer) AddGroup(analyzers ...*JsonLogMatcher) {
+	r.Matchers = append(r.Matchers, analyzers)
+}
+
+func (r *JsonLogAnalyzer) OnStop(fn func()) {
+	r.OnStopFn = fn
+}
+
+func (a *JsonLogAnalyzer) Finished() bool {
+	return a.finished
+}
+
+func (a *JsonLogAnalyzer) Error() error {
+	return a.err
+}
+
+func (a *JsonLogAnalyzer) Reset() {
+	for _, matchers := range a.Matchers {
+		for _, matcher := range matchers {
+			matcher.Reset()
+		}
+	}
+	a.Index = 0
+	a.groupFinished = true
+	a.finished = false
+	a.err = nil
+}
+
+func (a *JsonLogAnalyzer) HandleLine(line string) {
+	//fmt.Printf("Got line: %s\n", line)
+	//defer func() {
+	//	fmt.Printf("analyzer.HandleLine done: ind=%d fin=%v err=%v\n", a.Index, a.finished, a.err)
+	//}()
+	if a.finished {
+		return
+	}
+
+	matchers := a.Matchers[a.Index]
+	if a.groupFinished {
+		for _, matcher := range matchers {
+			matcher.Reset()
+		}
+		a.groupFinished = false
+	}
+
+	groupFinished := true
+	for _, matcher := range matchers {
+		matcher.HandleLine(line)
+		if matcher.Error() != nil {
+			a.err = matcher.Error()
+			a.finished = true
+			break
+		}
+		if !matcher.Finished() {
+			groupFinished = false
+		}
+	}
+
+	if !a.finished {
+		if groupFinished {
+			a.Index++
+			a.groupFinished = groupFinished
+		}
+		if a.Index == len(a.Matchers) {
+			a.finished = true
+		}
+	}
+
+	if a.finished && a.OnStopFn != nil {
+		//fmt.Printf("analyzer.HandleLine OnStopFn()\n")
+		a.OnStopFn()
+	}
+}
+
+type MatcherStep struct {
+	Matcher    func(JsonLogRecord) bool
+	OnMatch    func(JsonLogRecord) error
+	NotMatcher func(JsonLogRecord) bool
+	OnNotMatch func(JsonLogRecord) error
+}
+
+func LineMatcher(matchFn func(JsonLogRecord) bool, onMatch ...func(JsonLogRecord) error) *MatcherStep {
+	res := &MatcherStep{
+		Matcher: matchFn,
+	}
+	if len(onMatch) > 0 {
+		res.OnMatch = onMatch[0]
+	}
+	return res
+}
+
+func AwaitMatch(matchFn func(JsonLogRecord) bool, onMatch ...func(JsonLogRecord) error) *MatcherStep {
+	return LineMatcher(matchFn, onMatch...)
+}
+
+func LineNotMatcher(matchFn func(JsonLogRecord) bool, onMatch ...func(JsonLogRecord) error) *MatcherStep {
+	res := &MatcherStep{
+		NotMatcher: matchFn,
+	}
+	if len(onMatch) > 0 {
+		res.OnNotMatch = onMatch[0]
+	} else {
+		res.OnNotMatch = func(r JsonLogRecord) error {
+			return fmt.Errorf("log line matches negative matcher: %+v", r)
+		}
+	}
+	return res
+}
+
+func AwaitNotMatch(notMatchFn func(JsonLogRecord) bool, onMatch ...func(JsonLogRecord) error) *MatcherStep {
+	return LineNotMatcher(notMatchFn, onMatch...)
+}
+
+type JsonLogMatcher struct {
+	Steps    []*MatcherStep
+	Index    int
+	finished bool
+	err      error
+}
+
+func NewJsonLogMatcher(steps ...*MatcherStep) *JsonLogMatcher {
+	return &JsonLogMatcher{
+		Steps:    steps,
+		Index:    0,
+		finished: false,
+	}
+}
+
+func (m *JsonLogMatcher) HandleRecord(r JsonLogRecord) {
+
+}
+
+func (m *JsonLogMatcher) HandleLine(line string) {
+	//fmt.Printf("Matcher Got line: %s\n", line)
+	//defer func() {
+	//	fmt.Printf("matcher.HandleLine done: ind=%d fin=%v err=%v\n", m.Index, m.finished, m.err)
+	//}()
+	defer func() {
+		e := recover()
+		if e != nil {
+			m.finished = true
+		}
+	}()
+
+	step := m.Steps[m.Index]
+
+	// NotMatcher step should always handle line. Matcher step should not handle line after finish.
+	if m.finished && step.Matcher != nil {
+		return
+	}
+	//Ω(line).Should(HavePrefix("{"))
+
+	logRecord, err := NewJsonLogRecord().FromString(line)
+	//Ω(err).ShouldNot(HaveOccurred())
+
+	if step.Matcher != nil {
+		res := step.Matcher(logRecord)
+		if res {
+			if step.OnMatch != nil {
+				//fmt.Printf("matcher.HandleLine OnMatch()\n")
+				err = step.OnMatch(logRecord)
+				//fmt.Printf("matcher.HandleLine OnMatchDone()\n")
+				if err != nil {
+					m.err = err
+					m.finished = true
+					return
+				}
+			}
+			if m.Index == len(m.Steps)-1 {
+				m.finished = true
+			} else {
+				m.Index++
+			}
+		}
+	}
+	if step.NotMatcher != nil {
+		// NotMatcher is a non-blocking step, it should be always considered as finished
+		m.finished = true
+		res := step.NotMatcher(logRecord)
+		if res {
+			if step.OnNotMatch != nil {
+				err := step.OnNotMatch(logRecord)
+				if err != nil {
+					m.err = err
+					return
+				}
+			}
+			if m.Index == len(m.Steps)-1 {
+				m.finished = true
+			} else {
+				m.Index++
+			}
+		}
+	}
+}
+
+func (m *JsonLogMatcher) Finished() bool {
+	return m.finished
+}
+
+func (m *JsonLogMatcher) Error() error {
+	return m.err
+}
+
+func (m *JsonLogMatcher) Reset() {
+	m.Index = 0
+	m.finished = false
+	m.err = nil
+}
+
+func FinishAllMatchersSuccessfully() types.GomegaMatcher {
+	return &FinishAllMatchersSuccessfullyMatcher{}
+}
+
+type FinishAllMatchersSuccessfullyMatcher struct {
+	finished bool
+	err      error
+}
+
+func (matcher *FinishAllMatchersSuccessfullyMatcher) Match(actual interface{}) (success bool, err error) {
+	analyzer, ok := actual.(*JsonLogAnalyzer)
+	if !ok {
+		return false, fmt.Errorf("FinishAllMatchersSuccessfully must be passed a JsonLogAnalyzer. Got %T\n", actual)
+	}
+
+	matcher.err = analyzer.Error()
+	matcher.finished = analyzer.Finished()
+
+	return matcher.finished && matcher.err == nil, nil
+}
+
+func (matcher *FinishAllMatchersSuccessfullyMatcher) FailureMessage(actual interface{}) (message string) {
+	msgs := []string{}
+	if !matcher.finished {
+		msgs = append(msgs, fmt.Sprintf("is finished"))
+	}
+	if matcher.err != nil {
+		msgs = append(msgs, fmt.Sprintf("has no error. Got %v", matcher.err))
+	}
+	return fmt.Sprintf("Expected JsonLogAnalyzer %s", strings.Join(msgs, " and "))
+}
+
+func (matcher *FinishAllMatchersSuccessfullyMatcher) NegatedFailureMessage(actual interface{}) (message string) {
+	msgs := []string{}
+	if matcher.finished {
+		msgs = append(msgs, "is not finished")
+	}
+	if matcher.err == nil {
+		msgs = append(msgs, "has error")
+	}
+	return fmt.Sprintf("Expected JsonLogAnalyzer %s", strings.Join(msgs, " or "))
+}
diff --git a/test/utils/jsonlogrecord.go b/test/utils/jsonlogrecord.go
new file mode 100644
index 0000000..68be624
--- /dev/null
+++ b/test/utils/jsonlogrecord.go
@@ -0,0 +1,45 @@
+// +build test
+
+package utils
+
+import (
+	"encoding/json"
+)
+
+type JsonLogRecord map[string]string
+
+func NewJsonLogRecord() JsonLogRecord {
+	return make(JsonLogRecord, 0)
+}
+
+func (j JsonLogRecord) FromString(s string) (JsonLogRecord, error) {
+	return j.FromBytes([]byte(s))
+}
+
+func (j JsonLogRecord) FromBytes(arr []byte) (JsonLogRecord, error) {
+	err := json.Unmarshal(arr, &j)
+	if err != nil {
+		return j, err
+	}
+	return j, nil
+}
+
+func (j JsonLogRecord) Map() map[string]string {
+	return j
+}
+
+func (j JsonLogRecord) HasField(key string) bool {
+	return HasField(j, key)
+}
+
+func (j JsonLogRecord) FieldContains(key string, s string) bool {
+	return FieldContains(j, key, s)
+}
+
+func (j JsonLogRecord) FieldHasPrefix(key string, s string) bool {
+	return FieldHasPrefix(j, key, s)
+}
+
+func (j JsonLogRecord) FieldEquals(key string, s string) bool {
+	return FieldEquals(j, key, s)
+}
diff --git a/test/utils/jsonlogrecord_test.go b/test/utils/jsonlogrecord_test.go
new file mode 100644
index 0000000..32f6106
--- /dev/null
+++ b/test/utils/jsonlogrecord_test.go
@@ -0,0 +1,29 @@
+// +build test
+
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_JsonLogRecord(t *testing.T) {
+	line := "{\"foo\":\"bar\", \"baz\":\"qrux\" }"
+
+	logRecord, err := NewJsonLogRecord().FromString(line)
+	assert.NoError(t, err)
+
+	assert.True(t, logRecord.HasField("foo"))
+	assert.True(t, logRecord.FieldEquals("foo", "bar"))
+	assert.False(t, logRecord.FieldEquals("foo", "Bar"))
+	assert.True(t, logRecord.HasField("baz"))
+	assert.True(t, logRecord.FieldContains("baz", "ux"))
+	assert.False(t, logRecord.FieldContains("baz", "zz"))
+
+	m := map[string]string{
+		"foo": "bar",
+		"baz": "qrux",
+	}
+	assert.Equal(t, m, logRecord.Map())
+}
diff --git a/test/utils/kind-k8s-cluster.go b/test/utils/kind-k8s-cluster.go
new file mode 100644
index 0000000..31ef629
--- /dev/null
+++ b/test/utils/kind-k8s-cluster.go
@@ -0,0 +1,84 @@
+// +build test
+
+package utils
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"sigs.k8s.io/kind/pkg/cluster"
+	"sigs.k8s.io/kind/pkg/cluster/create"
+	"sigs.k8s.io/kind/pkg/errors"
+	"sigs.k8s.io/kind/pkg/globals"
+)
+
+func KindCreateCluster(clusterName string) error {
+	// Check if the cluster name already exists
+	known, err := cluster.IsKnown(clusterName)
+	if err != nil {
+		return err
+	}
+	if known {
+		return fmt.Errorf("a cluster with the name %q already exists", clusterName)
+	}
+
+	// create a cluster context and create the cluster
+	ctx := cluster.NewContext(clusterName)
+	fmt.Printf("Creating cluster %q ...\n", clusterName)
+	if err = ctx.Create(
+		create.WithConfigFile(""),
+		create.WithNodeImage(KindNodeImage()),
+		create.WaitForReady(time.Second*300),
+	); err != nil {
+		if errs := errors.Errors(err); errs != nil {
+			for _, problem := range errs {
+				globals.GetLogger().Errorf("%v", problem)
+			}
+			return errors.New("aborting due to invalid configuration")
+		}
+		return errors.Wrap(err, "failed to create cluster")
+	}
+
+	return nil
+
+}
+
+func KindDeleteCluster(clusterName string) error {
+	// Delete the cluster
+	fmt.Printf("Deleting cluster %q ...\n", clusterName)
+	ctx := cluster.NewContext(clusterName)
+	if err := ctx.Delete(); err != nil {
+		return errors.Wrap(err, "failed to delete cluster")
+	}
+	return nil
+}
+
+func KindGetKubeconfigPath(clusterName string) string {
+	return cluster.NewContext(clusterName).KubeConfigPath()
+}
+
+func KindClusterVersion() string {
+	k8sVer := os.Getenv("KIND_CLUSTER_VERSION")
+	if k8sVer == "" {
+		k8sVer = "1.13"
+	}
+
+	return k8sVer
+}
+
+// KindNodeImage maps cluster Major.Minor version to image name
+//
+// See: https://hub.docker.com/r/kindest/node/tags
+func KindNodeImage() string {
+	images := map[string]string{
+		"1.11": "kindest/node:v1.11.10",
+		"1.12": "kindest/node:v1.12.10",
+		"1.13": "kindest/node:v1.13.10",
+		"1.14": "kindest/node:v1.14.6",
+		"1.15": "kindest/node:v1.15.3",
+		"1.16": "kindest/node:v1.16.2",
+	}
+
+	return images[KindClusterVersion()]
+}
diff --git a/test/utils/kubectl.go b/test/utils/kubectl.go
new file mode 100644
index 0000000..ab45716
--- /dev/null
+++ b/test/utils/kubectl.go
@@ -0,0 +1,71 @@
+// +build test
+
+package utils
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gexec"
+)
+
+type KubectlCmd struct {
+	ConfigPath string
+}
+
+func Kubectl(config string) *KubectlCmd {
+	return &KubectlCmd{
+		ConfigPath: config,
+	}
+}
+
+func (k *KubectlCmd) Apply(ns string, fileName string) {
+	args := []string{"apply"}
+	if ns != "" {
+		args = append(args, "-n")
+		args = append(args, ns)
+	}
+	args = append(args, "-f")
+	args = append(args, fileName)
+
+	cmd := exec.Command(GetKubectlPath(), args...)
+
+	k.Succeed(cmd)
+}
+
+func (k *KubectlCmd) ReplaceForce(ns string, fileName string) {
+	args := []string{"replace"}
+	if ns != "" {
+		args = append(args, "-n")
+		args = append(args, ns)
+	}
+	args = append(args, "--force")
+	args = append(args, "-f")
+	args = append(args, fileName)
+
+	cmd := exec.Command(GetKubectlPath(), args...)
+
+	k.Succeed(cmd)
+}
+
+func (k *KubectlCmd) Succeed(cmd *exec.Cmd) {
+	if k.ConfigPath != "" {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("KUBECONFIG=%s", k.ConfigPath))
+	}
+	session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
+	Ω(err).ShouldNot(HaveOccurred())
+	<-session.Exited
+	Ω(session).Should(gexec.Exit())
+	Ω(session.ExitCode()).Should(Equal(0))
+}
+
+func GetKubectlPath() string {
+	path := os.Getenv("KUBECTL_BINARY_PATH")
+	if path == "" {
+		return "kubectl"
+	}
+	return path
+}
diff --git a/test/utils/prometheus.go b/test/utils/prometheus.go
new file mode 100644
index 0000000..898c1f2
--- /dev/null
+++ b/test/utils/prometheus.go
@@ -0,0 +1,213 @@
+// +build test
+
+package utils
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/onsi/gomega/types"
+)
+
+type PromMetricValue struct {
+	Name   string
+	Labels map[string]string
+	Value  float64
+}
+
+func (pmv PromMetricValue) String() string {
+	labels := []string{}
+	for k, v := range pmv.Labels {
+		labels = append(labels, fmt.Sprintf("%s=%s", k, v))
+	}
+	l := ""
+	if len(labels) > 0 {
+		l = fmt.Sprintf("{%s}", strings.Join(labels, ","))
+	}
+
+	return fmt.Sprintf("%s%s %v", pmv.Name, l, pmv.Value)
+}
+
+func (pmv PromMetricValue) IsEmpty() bool {
+	return pmv.Name == "" && len(pmv.Labels) == 0 && pmv.Value == 0.0
+}
+
+var EmptyPromMetricsValue = PromMetricValue{}
+
+var metricMatchRe = regexp.MustCompile(`^([a-zA-Z_-]+)({(.*)})?\ ([0-9.eE+-])$`)
+
+func ParsePromMetricValue(line string) PromMetricValue {
+	parts := metricMatchRe.FindStringSubmatch(line)
+	if parts == nil {
+		return EmptyPromMetricsValue
+	}
+
+	// Last part is a float value
+	value, err := strconv.ParseFloat(parts[len(parts)-1], 64)
+	if err != nil {
+		return EmptyPromMetricsValue
+	}
+
+	labels := map[string]string{}
+
+	// labels should be in parts[3]
+	if len(parts) > 3 {
+		labels = ParsePromLabels(parts[3])
+	}
+
+	return PromMetricValue{
+		Name:   parts[1],
+		Labels: labels,
+		Value:  value,
+	}
+}
+
+func ParsePromLabels(labels string) map[string]string {
+	res := map[string]string{}
+
+	kvParts := strings.Split(labels, ",")
+
+	for _, kvPart := range kvParts {
+		kv := strings.SplitN(kvPart, "=", 2)
+		if len(kv) == 2 {
+			res[kv[0]] = kv[1]
+		}
+	}
+
+	return res
+}
+
+type PromScraper struct {
+	Address string
+	Metrics []PromMetricValue
+}
+
+func NewPromScraper(address string) *PromScraper {
+	return &PromScraper{
+		Address: address,
+	}
+}
+
+func (p *PromScraper) WithAddress(address string) {
+	p.Address = address
+}
+
+func (p *PromScraper) Scrape() error {
+	r, err := http.Get(p.Address)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		r.Body.Close()
+	}()
+
+	metrics := make([]PromMetricValue, 0)
+
+	scanner := bufio.NewScanner(r.Body)
+	for scanner.Scan() {
+		pmv := ParsePromMetricValue(scanner.Text())
+		if !pmv.IsEmpty() {
+			metrics = append(metrics, pmv)
+		}
+	}
+
+	p.Metrics = metrics
+
+	fmt.Printf("Got metrics: \n")
+	for _, pmv := range p.Metrics {
+		fmt.Printf("%s\n", pmv)
+	}
+
+	return nil
+}
+
+func (p *PromScraper) HasMetric(name string) {
+
+}
+
+func (p *PromScraper) MetricEquals(name string) {
+
+}
+
+// FindExact searches metric by name and labels
+func (p *PromScraper) FindExact(pmv PromMetricValue) PromMetricValue {
+	for _, m := range p.Metrics {
+		if m.Name == pmv.Name {
+			allLabelsEqual := true
+			for k, v := range m.Labels {
+				pmvl, ok := pmv.Labels[k]
+				if !ok || pmvl != v {
+					allLabelsEqual = false
+				}
+			}
+
+			if allLabelsEqual {
+				return m
+			}
+		}
+	}
+
+	return PromMetricValue{}
+}
+
+func PromMetric(name string, labels ...string) PromMetricValue {
+	labelsMap := map[string]string{}
+
+	for i := 0; i < len(labels); i += 2 {
+		k := labels[i]
+		v := ""
+		if i+1 < len(labels) {
+			v = labels[i+1]
+		}
+		labelsMap[k] = v
+	}
+	return PromMetricM(name, labelsMap)
+}
+
+func PromMetricM(name string, labels map[string]string) PromMetricValue {
+	return PromMetricValue{
+		Name:   name,
+		Labels: labels,
+	}
+}
+
+// gomega matchers
+
+func HaveMetric(pmv PromMetricValue) types.GomegaMatcher {
+	return &HaveMetricMatcher{
+		pmv: pmv,
+	}
+}
+
+type HaveMetricMatcher struct {
+	pmv         PromMetricValue
+	promScraper *PromScraper
+}
+
+func (matcher *HaveMetricMatcher) Match(actual interface{}) (success bool, err error) {
+	promScraper, ok := actual.(*PromScraper)
+	if !ok {
+		return false, fmt.Errorf("HaveMetric must be passed a *PromScraper. Got %T\n", actual)
+	}
+
+	matcher.promScraper = promScraper
+
+	return !promScraper.FindExact(matcher.pmv).IsEmpty(), nil
+}
+
+func (matcher *HaveMetricMatcher) FailureMessage(actual interface{}) (message string) {
+	return fmt.Sprintf("Expected ShellOperator has metric")
+}
+
+func (matcher *HaveMetricMatcher) NegatedFailureMessage(actual interface{}) (message string) {
+	return fmt.Sprintf("Expected ShellOperator has no metric")
+}
+
+func HaveMetricValue() types.GomegaMatcher {
+	return nil
+}
diff --git a/test/utils/shell-operator.go b/test/utils/shell-operator.go
new file mode 100644
index 0000000..ade4175
--- /dev/null
+++ b/test/utils/shell-operator.go
@@ -0,0 +1,115 @@
+// +build test
+
+package utils
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/onsi/gomega/types"
+)
+
+type ShellOperatorOptions struct {
+	CurrentDir string
+	BinPath    string
+	Command    string
+	Args       []string
+	LogType    string
+	WorkingDir string
+	KubeConfig string
+}
+
+func DefaultShellOperatorOptions(currentDir string, configPath string) ShellOperatorOptions {
+	return ShellOperatorOptions{
+		CurrentDir: currentDir,
+		BinPath:    GetShellOperatorPath(),
+		Args:       []string{"start"},
+		KubeConfig: configPath,
+		LogType:    "json",
+	}
+}
+
+func (o ShellOperatorOptions) Merge(opts ShellOperatorOptions) ShellOperatorOptions {
+	res := o
+	if opts.CurrentDir != "" {
+		o.CurrentDir = opts.CurrentDir
+	}
+	if opts.BinPath != "" {
+		o.BinPath = opts.BinPath
+	}
+	if opts.Command != "" {
+		o.Command = opts.Command
+	}
+	if opts.Args != nil {
+		o.Args = opts.Args
+	}
+	if opts.LogType != "" {
+		o.LogType = opts.LogType
+	}
+	if opts.WorkingDir != "" {
+		o.WorkingDir = opts.WorkingDir
+	}
+	if opts.KubeConfig != "" {
+		o.KubeConfig = opts.KubeConfig
+	}
+	return res
+}
+
+func ExecShellOperator(operatorOpts ShellOperatorOptions, opts CommandOptions) error {
+	if operatorOpts.BinPath == "" {
+		operatorOpts.BinPath = GetShellOperatorPath()
+	}
+	cmd := exec.Command(operatorOpts.BinPath, operatorOpts.Args...)
+	cmd.Env = os.Environ()
+	if operatorOpts.CurrentDir != "" {
+		cmd.Dir = operatorOpts.CurrentDir
+	}
+	// Environment variables
+	if operatorOpts.LogType != "" {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("LOG_TYPE=%s", operatorOpts.LogType))
+	}
+	if operatorOpts.KubeConfig != "" {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("SHELL_OPERATOR_KUBE_CONFIG=%s", operatorOpts.KubeConfig))
+	}
+	if operatorOpts.WorkingDir != "" {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("SHELL_OPERATOR_WORKING_DIR=%s", operatorOpts.WorkingDir))
+	}
+	return StreamedExecCommand(cmd, opts)
+}
+
+func GetShellOperatorPath() string {
+	path := os.Getenv("SHELL_OPERATOR_BINARY_PATH")
+	if path == "" {
+		return "shell-operator"
+	}
+	return path
+}
+
+func BeShellOperatorStopped() types.GomegaMatcher {
+	return &BeShellOperatorStoppedMatcher{}
+}
+
+type BeShellOperatorStoppedMatcher struct {
+	err error
+}
+
+func (matcher *BeShellOperatorStoppedMatcher) Match(actual interface{}) (success bool, err error) {
+	err, ok := actual.(error)
+	if !ok {
+		return false, fmt.Errorf("BeShellOperatorStopped must be passed an error. Got %T\n", actual)
+	}
+
+	matcher.err = err
+
+	return matcher.err != nil && strings.Contains(matcher.err.Error(), "command is stopped"), nil
+}
+
+func (matcher *BeShellOperatorStoppedMatcher) FailureMessage(actual interface{}) (message string) {
+	return fmt.Sprintf("Expected ShellOperator stopped. Got error: %v", matcher.err)
+}
+
+func (matcher *BeShellOperatorStoppedMatcher) NegatedFailureMessage(actual interface{}) (message string) {
+	return fmt.Sprintf("Expected ShellOperator stopped with error")
+}
diff --git a/test/utils/streamedexec.go b/test/utils/streamedexec.go
new file mode 100644
index 0000000..7e5efa2
--- /dev/null
+++ b/test/utils/streamedexec.go
@@ -0,0 +1,114 @@
+// +build test
+
+package utils
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"sync"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega/gexec"
+)
+
+type CommandOptions struct {
+	OutputLineHandler func(string)
+	StopCh            chan struct{}
+}
+
+type CommandError struct {
+	CommandError error
+	ExitCode     int
+}
+
+func (c *CommandError) Error() string {
+	return c.CommandError.Error()
+}
+
+func StreamedExecCommand(cmd *exec.Cmd, opts CommandOptions) error {
+	stdoutReadPipe, stdoutWritePipe, err := os.Pipe()
+	if err != nil {
+		return fmt.Errorf("unable to create os pipe for stdout: %s", err)
+	}
+
+	stderrReadPipe, stderrWritePipe, err := os.Pipe()
+	if err != nil {
+		return fmt.Errorf("unable to create os pipe for stderr: %s", err)
+	}
+
+	//outputReader := io.MultiReader(stdoutReadPipe, stderrReadPipe)
+
+	fmt.Printf("gexec.Start\n")
+	session, err := gexec.Start(cmd, stdoutWritePipe, stderrWritePipe)
+	if err != nil {
+		return fmt.Errorf("error starting command: %s", err)
+	}
+
+	var stopMsg string
+	var wg sync.WaitGroup
+	wg.Add(3)
+
+	go func() {
+		defer wg.Done()
+		if opts.StopCh != nil {
+			select {
+			case <-opts.StopCh:
+				stopMsg = "command is stopped"
+				session.Interrupt().Wait(time.Second * 30)
+			case <-session.Exited:
+			}
+		} else {
+			<-session.Exited
+		}
+
+		// Initiate EOF to stop consumeLines
+		stdoutWritePipe.Close()
+		stderrWritePipe.Close()
+	}()
+
+	go func() {
+		defer wg.Done()
+		defer gexecRecover(session)
+		defer ginkgo.GinkgoRecover()
+		consumeLines(stdoutReadPipe, session, opts)
+	}()
+
+	go func() {
+		defer wg.Done()
+		defer gexecRecover(session)
+		defer ginkgo.GinkgoRecover()
+		consumeLines(stderrReadPipe, session, opts)
+	}()
+
+	wg.Wait()
+
+	if exitCode := session.ExitCode(); exitCode != 0 {
+		cmdErr := fmt.Errorf("command failed, exit code %d", exitCode)
+		if stopMsg != "" {
+			cmdErr = fmt.Errorf(stopMsg)
+		}
+		return &CommandError{
+			CommandError: cmdErr,
+			ExitCode:     exitCode,
+		}
+	}
+	return nil
+}
+
+func consumeLines(r io.Reader, session *gexec.Session, opts CommandOptions) {
+	scanner := bufio.NewScanner(r)
+	for scanner.Scan() {
+		opts.OutputLineHandler(scanner.Text())
+	}
+}
+
+func gexecRecover(session *gexec.Session) {
+	e := recover()
+	if e != nil && session != nil {
+		session.Kill()
+	}
+}