Repository: elastic/synthetics (ts)

Issue:
Ndjson parsing is broken if data is bit large 
It will throw an error if data is returned like this since it's not able to safely parse the json

`Error SyntaxError: Unexpected end of JSON input could not parse data '{"createdMonitors":[],"updatedMonitors":["production-logging-ap-northeast-1"],"staleMonitors":["production-metrics-gcp-asia-east1","production-metrics-eu-west-1","production-metrics-azure-westus2","production-metrics-azure-westeurope","production-metrics-azure-uksouth","production-metrics-azure-southeastasia","production-metrics-azure-southcentralus","production-metrics-azure-southafricanorth","production-metrics-azure-northeurope","production-metrics-azure-japaneast","production-metrics-azure-francecentral","production-metrics-azure-eastus2","production-metrics-azure-eastus","production-me'`


issue can be reproduced on cloud against 8.5.1 or 8.5.1 and on the latest version of the agent !!

Pull request:
fix: parse buffered NDJSON response from kibana correctly
+ fix #660 
+ When Kibana NDJSON responses are buffered, this would result in parsing error on the chunking code as buffer length can be different when working on local vs on cloud (proxies).
+ To account for these varying buffer lengths, We consume all the stream body from Kibana as buffers and then parse the NDJSON response to avoid intereleaved parsing errors. 

Patch:
diff --git a/.gitignore b/.gitignore
index 8399ae0..d83fc53 100644
--- a/.gitignore
+++ b/.gitignore
@@ -119,3 +119,5 @@ __tests__/e2e/junit.xml
 
 .idea/
 seccomp/build
+
+test-projects/
diff --git a/src/generator/utils.ts b/src/generator/utils.ts
index a37bfc0..0231543 100644
--- a/src/generator/utils.ts
+++ b/src/generator/utils.ts
@@ -47,6 +47,7 @@ export function replaceTemplates(input: string, values: Record<string, any>) {
       if (Array.isArray(finalValue)) {
         return String(finalValue)
           .split(',')
+          .filter(Boolean)
           .map(f => `'${f}'`);
       }
       if (typeof finalValue == 'number') {
diff --git a/src/helpers.ts b/src/helpers.ts
index 46c023e..4d3f33d 100644
--- a/src/helpers.ts
+++ b/src/helpers.ts
@@ -322,10 +322,10 @@ export function wrapFnWithLocation<A extends unknown[], R>(
 }
 
 // Safely parse ND JSON (Newline delimitted JSON) chunks
-export function safeNDJSONParse(chunks: string[]) {
-  // chunks may not be at proper newline boundaries, so we make sure everything is split
+export function safeNDJSONParse(data: string | string[]) {
+  // data may not be at proper newline boundaries, so we make sure everything is split
   // on proper newlines
-  chunks = Array.isArray(chunks) ? chunks : [chunks];
+  const chunks = Array.isArray(data) ? data : [data];
   const lines = chunks.join('\n').split(/\r?\n/);
   return lines
     .filter(l => l.match(/\S/)) // remove blank lines
diff --git a/src/push/index.ts b/src/push/index.ts
index 7f55e88..18831d6 100644
--- a/src/push/index.ts
+++ b/src/push/index.ts
@@ -110,26 +110,27 @@ export async function pushMonitors({
     const { error, message } = await body.json();
     throw formatAPIError(statusCode, error, message);
   }
-  body.setEncoding('utf-8');
+  const allchunks = [];
   for await (const data of body) {
-    // Its kind of hacky for now where Kibana streams the response by
-    // writing the data as NDJSON events (data can be interleaved), we
-    // distinguish the final data by checking if the event was a progress vs complete event
-    const chunks = safeNDJSONParse(data);
-    for (const chunk of chunks) {
-      if (typeof chunk === 'string') {
-        // TODO: add progress back for all states once we get the fix
-        // on kibana side
-        keepStale && apiProgress(chunk);
-        continue;
-      }
-      const { failedMonitors, failedStaleMonitors } = chunk;
-      if (failedMonitors && failedMonitors.length > 0) {
-        throw formatFailedMonitors(failedMonitors);
-      }
-      if (failedStaleMonitors.length > 0) {
-        throw formatStaleMonitors(failedStaleMonitors);
-      }
+    allchunks.push(Buffer.from(data));
+  }
+  const chunks = safeNDJSONParse(Buffer.concat(allchunks).toString('utf-8'));
+  // Its kind of hacky for now where Kibana streams the response by
+  // writing the data as NDJSON events (data can be interleaved), we
+  // distinguish the final data by checking if the event was a progress vs complete event
+  for (const chunk of chunks) {
+    if (typeof chunk === 'string') {
+      // TODO: add progress back for all states once we get the fix
+      // on kibana side
+      keepStale && apiProgress(chunk);
+      continue;
+    }
+    const { failedMonitors, failedStaleMonitors } = chunk;
+    if (failedMonitors && failedMonitors.length > 0) {
+      throw formatFailedMonitors(failedMonitors);
+    }
+    if (failedStaleMonitors.length > 0) {
+      throw formatStaleMonitors(failedStaleMonitors);
     }
   }
 }
diff --git a/templates/journeys/lightweight-example.yaml b/templates/journeys/lightweight-example.yaml
new file mode 100644
index 0000000..33381fc
--- /dev/null
+++ b/templates/journeys/lightweight-example.yaml
@@ -0,0 +1,5 @@
+- type: http
+  name: Todos Lightweight
+  id: todos-lightweight
+  urls: "https://elastic.github.io/synthetics-demo/"
+  schedule: '@every 1m'
\ No newline at end of file


Test patch:
diff --git a/__tests__/generator/index.test.ts b/__tests__/generator/index.test.ts
index 698f53f..97dbfac 100644
--- a/__tests__/generator/index.test.ts
+++ b/__tests__/generator/index.test.ts
@@ -44,7 +44,7 @@ describe('Generator', () => {
   });
   it('generate synthetics project - NPM', async () => {
     // In a few development environments, this test may take
-    // a few millisseconds more than the default 15000ms timeout
+    // a few milliseconds more than the default 15000ms timeout
     jest.setTimeout(30000);
     const cli = new CLIMock().args(['init', scaffoldDir]).run({
       env: {
diff --git a/__tests__/generator/utils.test.ts b/__tests__/generator/utils.test.ts
new file mode 100644
index 0000000..56e2f52
--- /dev/null
+++ b/__tests__/generator/utils.test.ts
@@ -0,0 +1,51 @@
+/**
+ * MIT License
+ *
+ * Copyright (c) 2020-present, Elastic NV
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+import { readFileSync } from 'fs';
+import { join } from 'path';
+import { replaceTemplates } from '../../src/generator/utils';
+
+describe('Generator utils', () => {
+  it('does not add empty values in array', async () => {
+    const template = readFileSync(
+      join(__dirname, '..', '..', 'templates/synthetics.config.ts'),
+      'utf-8'
+    );
+    const values = {
+      locations: ['us_east'],
+      privateLocations: [''],
+      schedule: 30,
+      id: 'test',
+      space: 'kbn',
+      url: 'foo:bar',
+    };
+    const finalValue = replaceTemplates(template, values);
+
+    expect(finalValue).toContain("locations: ['us_east'],");
+    expect(finalValue).toContain('privateLocations: [],');
+
+    expect(finalValue).not.toContain("['']");
+  });
+});
diff --git a/__tests__/push/index.test.ts b/__tests__/push/index.test.ts
index 9fa8fe6..36218ce 100644
--- a/__tests__/push/index.test.ts
+++ b/__tests__/push/index.test.ts
@@ -263,6 +263,12 @@ heartbeat.monitors:
           req.on('data', chunks => {
             const schema = JSON.parse(chunks.toString()) as APISchema;
             res.write(JSON.stringify(schema.monitors[0].name) + '\n');
+            if (!schema.keep_stale) {
+              // write more than the stream buffer to check the broken NDJSON data
+              res.write(
+                JSON.stringify(Buffer.from('a'.repeat(70000)).toString()) + '\n'
+              );
+            }
           });
           req.on('end', () => {
             res.end(JSON.stringify(apiRes));
diff --git a/__tests__/utils/test-config.ts b/__tests__/utils/test-config.ts
index 99f3b81..a2848dd 100644
--- a/__tests__/utils/test-config.ts
+++ b/__tests__/utils/test-config.ts
@@ -80,7 +80,11 @@ export class CLIMock {
   run(spawnOverrides?: { cwd?: string; env?: NodeJS.ProcessEnv }): CLIMock {
     this.process = spawn(
       'node',
-      [join(__dirname, '..', '..', 'dist', 'cli.js'), ...this.cliArgs],
+      [
+        '--no-warnings',
+        join(__dirname, '..', '..', 'dist', 'cli.js'),
+        ...this.cliArgs,
+      ],
       {
         env: process.env,
         stdio: 'pipe',