Repository: fluent/fluent-bit (C)

Issue:
TCP input plugin dropping records
## Bug Report

**Describe the bug**
The 1.9.x releases contain a regression that makes TCP input plugin dropping records. The actual breaking commit is [f3ac138b0](https://github.com/fluent/fluent-bit/commit/f3ac138b0)

**To Reproduce**
1. Create a test data set having 64k records:
```
for i in $(seq 1 65535); do echo '{"foo":"bar"}'; done > tcp.log
```

2. Create `fluent-bit.conf`:
```
[SERVICE]
  Flush         10
  Log_Level     debug
  Daemon        off
[INPUT]
  Name          tcp
  Listen        0.0.0.0
  Port          5170
  Chunk_Size    32
  Buffer_Size   64
  Format        json
  Mem_Buf_Limit 30MB
[OUTPUT]
  Name stdout
  Match *
```

3. Start **working** 1.8.15 daemon:
```
docker run -it --rm -p 5170:5170 -v /path/to/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf fluent/fluent-bit:1.8.15
```

4. Send test data:
```
cat tcp.log | nc localhost 5170
```
```
Fluent Bit v1.8.15
* Copyright (C) 2015-2021 The Fluent Bit Authors
...
[2022/04/21 15:42:58] [debug] [input chunk] update output instances with new chunk size diff=46800
[2022/04/21 15:42:58] [debug] [input chunk] update output instances with new chunk size diff=14920
[2022/04/21 15:43:05] [debug] [task] created task=0x7fea70637280 id=0 OK
[2022/04/21 15:43:05] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] tcp.0: [1650555777.674115534, {"foo"=>"bar"}]
[1] tcp.0: [1650555777.674137953, {"foo"=>"bar"}]
...
[65533] tcp.0: [1650555778.342504882, {"foo"=>"bar"}]
[65534] tcp.0: [1650555778.342512844, {"foo"=>"bar"}]
[2022/04/21 15:43:06] [debug] [out coro] cb_destroy coro_id=0
```
> All records reach the output!

5. Start **non-working** 1.9.2 daemon:
```
docker run -it --rm -p 5170:5170 -v /path/to/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf fluent/fluent-bit:1.9.2
```

6. Send test data:
```
cat tcp.log | nc localhost 5170
```
```
Fluent Bit v1.9.2
* Copyright (C) 2015-2022 The Fluent Bit Authors
...
[2022/04/21 15:46:36] [debug] [tcp:tcp.0] created event channels: read=21 write=22
[2022/04/21 15:46:36] [ info] [input:tcp:tcp.0] listening on 0.0.0.0:5170
[2022/04/21 15:46:36] [debug] [stdout:stdout.0] created event channels: read=24 write=25
[2022/04/21 15:46:36] [debug] [router] match rule tcp.0:stdout.0
[2022/04/21 15:46:36] [ info] [sp] stream processor started
[2022/04/21 15:46:36] [ info] [output:stdout:stdout.0] worker #0 started
[2022/04/21 15:47:11] [debug] [input:tcp:tcp.0] JSON incomplete, waiting for more data...
[2022/04/21 15:47:11] [debug] [input:tcp:tcp.0] JSON incomplete, waiting for more data...
[2022/04/21 15:47:11] [debug] [input:tcp:tcp.0] JSON incomplete, waiting for more data...
[2022/04/21 15:47:11] [debug] [input:tcp:tcp.0] JSON incomplete, waiting for more data...
```
> I've seen processed record count varying between 0, 512, 1024, but never the amount of input.

7.  Revert the mentioned commit and rebuild image
```
git checkout -b v1.9.2-patched v1.9.2
git revert f3ac138b0
docker buildx build --no-cache --load --platform "linux/amd64" --target=production -f dockerfiles/Dockerfile -t fluent/fluent-bit:1.9.2-patched .
```

8. Start **patched** 1.9.2 daemon:
```
docker run -it --rm -p 5170:5170 -v /path/to/fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf fluent/fluent-bit:1.9.2-patched
``` 

9. Send test data:
```
cat tcp.log | nc localhost 5170
```
```
Fluent Bit v1.9.2
* Copyright (C) 2015-2022 The Fluent Bit Authors
...
[2022/04/21 15:56:29] [debug] [tcp:tcp.0] created event channels: read=21 write=22
[2022/04/21 15:56:29] [ info] [input:tcp:tcp.0] listening on 0.0.0.0:5170
[2022/04/21 15:56:29] [debug] [stdout:stdout.0] created event channels: read=24 write=25
[2022/04/21 15:56:29] [debug] [router] match rule tcp.0:stdout.0
[2022/04/21 15:56:29] [ info] [sp] stream processor started
[2022/04/21 15:56:29] [ info] [output:stdout:stdout.0] worker #0 started
[2022/04/21 15:56:34] [debug] [input chunk] update output instances with new chunk size diff=21940
[2022/04/21 15:56:34] [debug] [input chunk] update output instances with new chunk size diff=46800
...
[2022/04/21 15:56:34] [debug] [input chunk] update output instances with new chunk size diff=46800
[2022/04/21 15:56:34] [debug] [input chunk] update output instances with new chunk size diff=46800
[2022/04/21 15:56:34] [debug] [input chunk] update output instances with new chunk size diff=25160
[2022/04/21 15:56:38] [debug] [task] created task=0x7f7a05e423f0 id=0 OK
[2022/04/21 15:56:38] [debug] [output:stdout:stdout.0] task_id=0 assigned to thread #0
[0] tcp.0: [1650556594.057870047, {"foo"=>"bar"}]
[1] tcp.0: [1650556594.057911742, {"foo"=>"bar"}]
...
[65533] tcp.0: [1650556594.703312414, {"foo"=>"bar"}]
[65534] tcp.0: [1650556594.703320516, {"foo"=>"bar"}]
[2022/04/21 15:56:39] [debug] [out flush] cb_destroy coro_id=0
[2022/04/21 15:56:39] [debug] [task] destroy task=0x7f7a05e423f0 (task_id=0)
```
> Works again just like in 1.8 releases!

**Expected behavior**

1.9 releases should behave in a similar way as the earlier 1.8 releases. 

**Additional context**

Especially systems using TCP keepalive sessions with TCP input plugin are badly misbehaving at the moment.

The following patch is enough to fix the issue (but it might have some other issues):
```
diff --git a/src/flb_pack.c b/src/flb_pack.c
index 40e1bde5d..e7f33a1bd 100644
--- a/src/flb_pack.c
+++ b/src/flb_pack.c
@@ -364,9 +364,13 @@ int flb_pack_json_state(const char *js, size_t len,
         int i;
         int found = 0;

-        for (i = 1; i < state->tokens_count; i++) {
+        for (i = 1; i < state->tokens_size; i++) {
             t = &state->tokens[i];

+            if (t->start < (state->tokens[i - 1]).start) {
+                break;
+            }
+
             if (t->parent == -1 && (t->end != 0)) {
                 found++;
                 delim = i;
```



Discussion:
@edsiper any comments?
@edsiper  I think below line causes this issue.
https://github.com/fluent/fluent-bit/commit/f3ac138b0#diff-7228b3553f609319f8501a9d8815f08fbb6c451a69dbccb9faec6bdaff57cbdeR368

If jsmn API returns `JSMN_ERROR_PART`, `state->tokens_count` doesn't increase. 
https://github.com/fluent/fluent-bit/blob/v1.9.3/src/flb_pack.c#L77

Then fluent-bit tries to iterate below loop.
https://github.com/fluent/fluent-bit/blob/v1.9.3/src/flb_pack.c#L367

However it doesn't iterate since `state->tokens_count` is 0.
Then `found` never increase and the function returns `FLB_ERR_JSON_PART`.
https://github.com/fluent/fluent-bit/blob/v1.9.3/src/flb_pack.c#L380

`in_tcp` reports `JSON incomplete` since the API(`flb_pack_json_state`) returns `FLB_ERR_JSON_PART`.





Patch:
diff --git a/src/flb_pack.c b/src/flb_pack.c
index 1adea691ede..249b075ee94 100644
--- a/src/flb_pack.c
+++ b/src/flb_pack.c
@@ -361,24 +361,33 @@ int flb_pack_json_state(const char *js, size_t len,
          * are OK in the array of tokens, if any, process them and adjust
          * the JSMN context/buffers.
          */
+
+        /*
+         * jsmn_parse updates jsmn_parser members. (state->parser)
+         * A member 'toknext' points next incomplete object token.
+         * We use toknext - 1 as an index of last member of complete JSON.
+         */
         int i;
         int found = 0;
 
-        for (i = 1; i < state->tokens_count; i++) {
+        if (state->parser.toknext == 0) {
+            return ret;
+        }
+
+        for (i = (int)state->parser.toknext - 1; i >= 1; i--) {
             t = &state->tokens[i];
 
             if (t->parent == -1 && (t->end != 0)) {
                 found++;
                 delim = i;
+                break;
             }
         }
 
-        if (found > 0) {
-            state->tokens_count += delim;
-        }
-        else {
-            return ret;
+        if (found == 0) {
+            return ret; /* FLB_ERR_JSON_PART */
         }
+        state->tokens_count += delim;
     }
     else if (ret != 0) {
         return ret;


Test patch:
diff --git a/tests/internal/pack.c b/tests/internal/pack.c
index 4b635394c00..4b1ec0d6dc5 100644
--- a/tests/internal/pack.c
+++ b/tests/internal/pack.c
@@ -634,6 +634,154 @@ void test_json_pack_bug1278()
     }
 }
 
+static int check_msgpack_val(msgpack_object obj, int expected_type, char *expected_val)
+{
+    int len;
+
+    if (!TEST_CHECK(obj.type == expected_type)) {
+        TEST_MSG("type mismatch\nexpected=%d got=%d", expected_type, obj.type);
+        return -1;
+    }
+    switch(obj.type) {
+    case MSGPACK_OBJECT_MAP:
+        if(!TEST_CHECK(obj.via.map.size == atoi(expected_val))) {
+            TEST_MSG("map size mismatch\nexpected=%s got=%d", expected_val, obj.via.map.size);
+            return -1;
+        }
+        break;
+
+    case MSGPACK_OBJECT_ARRAY:
+        if(!TEST_CHECK(obj.via.array.size == atoi(expected_val))) {
+            TEST_MSG("array size mismatch\nexpected=%s got=%d", expected_val, obj.via.array.size);
+            return -1;
+        }
+        break;
+
+    case MSGPACK_OBJECT_STR:
+        len = strlen(expected_val);
+        if (!TEST_CHECK(obj.via.str.size == strlen(expected_val))) {
+            TEST_MSG("str size mismatch\nexpected=%d got=%d", len, obj.via.str.size);
+            return -1;
+        }
+        else if(!TEST_CHECK(strncmp(expected_val, obj.via.str.ptr ,len) == 0)) {
+            TEST_MSG("str mismatch\nexpected=%.*s got=%.*s", len, expected_val, len, obj.via.str.ptr);
+            return -1;
+        }
+        break;
+
+    case MSGPACK_OBJECT_POSITIVE_INTEGER:
+        if(!TEST_CHECK(obj.via.u64 == (uint64_t)atoi(expected_val))) {
+            TEST_MSG("int mismatch\nexpected=%s got=%"PRIu64, expected_val, obj.via.u64);
+            return -1;
+        }
+        break;
+
+    case MSGPACK_OBJECT_BOOLEAN:
+        if (obj.via.boolean) {
+            if(!TEST_CHECK(strncasecmp(expected_val, "true",4) == 0)) {
+                TEST_MSG("bool mismatch\nexpected=%s got=true", expected_val);
+                return -1;
+            }
+        }
+        else {
+            if(!TEST_CHECK(strncasecmp(expected_val, "false",5) == 0)) {
+                TEST_MSG("bool mismatch\nexpected=%s got=false", expected_val);
+                return -1;
+            }
+        }
+        break;
+
+    default:
+        TEST_MSG("unknown type %d", obj.type);
+        return -1;
+    }
+
+    return 0;
+}
+
+/*
+ * https://github.com/fluent/fluent-bit/issues/5336
+ * Pack "valid JSON + partial JSON"
+ */
+#define JSON_BUG5336 "{\"int\":10, \"string\":\"hello\", \"bool\":true, \"array\":[0,1,2]}"
+void test_json_pack_bug5336()
+{
+    int ret;
+    char *json_valid = JSON_BUG5336;
+    size_t len = strlen(json_valid);
+
+    char *json_incomplete = JSON_BUG5336 JSON_BUG5336;
+    char *out = NULL;
+    int out_size;
+    struct flb_pack_state state;
+    int i;
+
+    msgpack_unpacked result;
+    msgpack_object obj;
+    size_t off = 0;
+
+    int loop_cnt = 0;
+
+    for (i=len; i<len*2; i++) {
+        loop_cnt++;
+
+        flb_pack_state_init(&state);
+
+        /* Pass small string size to create incomplete JSON */
+        ret = flb_pack_json_state(json_incomplete, i, &out, &out_size, &state);
+        if (!TEST_CHECK(ret != FLB_ERR_JSON_INVAL)) {
+            TEST_MSG("%ld: FLB_ERR_JSON_INVAL\njson=%.*s", i-len, i, json_incomplete);
+            exit(EXIT_FAILURE);
+        }
+        else if(!TEST_CHECK(ret != FLB_ERR_JSON_PART)) {
+            TEST_MSG("%ld: FLB_ERR_JSON_PART\njson=%.*s", i-len, i, json_incomplete);
+            exit(EXIT_FAILURE);
+        }
+
+        /* unpack parsed data */
+        msgpack_unpacked_init(&result);
+        off = 0;
+        TEST_CHECK(msgpack_unpack_next(&result, out, out_size, &off) == MSGPACK_UNPACK_SUCCESS);
+
+        TEST_CHECK(check_msgpack_val(result.data, MSGPACK_OBJECT_MAP, "4" /*map size*/) == 0);
+
+        /* "int":10 */
+        obj = result.data.via.map.ptr[0].key;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_STR, "int") == 0);
+        obj = result.data.via.map.ptr[0].val;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_POSITIVE_INTEGER, "10") == 0);
+
+        /* "string":"hello"*/
+        obj = result.data.via.map.ptr[1].key;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_STR, "string") == 0);
+        obj = result.data.via.map.ptr[1].val;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_STR, "hello") == 0);
+
+        /* "bool":true */
+        obj = result.data.via.map.ptr[2].key;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_STR, "bool") == 0);
+        obj = result.data.via.map.ptr[2].val;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_BOOLEAN, "true") == 0);
+
+        /* "array":[0,1,2] */
+        obj = result.data.via.map.ptr[3].key;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_STR, "array") == 0);
+        obj = result.data.via.map.ptr[3].val;
+        TEST_CHECK(check_msgpack_val(obj, MSGPACK_OBJECT_ARRAY, "3" /*array size*/) == 0);
+        TEST_CHECK(check_msgpack_val(obj.via.array.ptr[0], MSGPACK_OBJECT_POSITIVE_INTEGER, "0") == 0);
+        TEST_CHECK(check_msgpack_val(obj.via.array.ptr[1], MSGPACK_OBJECT_POSITIVE_INTEGER, "1") == 0);
+        TEST_CHECK(check_msgpack_val(obj.via.array.ptr[2], MSGPACK_OBJECT_POSITIVE_INTEGER, "2") == 0);
+
+        msgpack_unpacked_destroy(&result);
+        flb_free(out);
+        flb_pack_state_reset(&state);
+    }
+
+    if(!TEST_CHECK(loop_cnt == len)) {
+        TEST_MSG("loop_cnt expect=%ld got=%d", len, loop_cnt);
+    }
+}
+
 TEST_LIST = {
     /* JSON maps iteration */
     { "json_pack"          , test_json_pack },
@@ -644,6 +792,7 @@ TEST_LIST = {
     { "json_dup_keys"      , test_json_dup_keys},
     { "json_pack_bug342"   , test_json_pack_bug342},
     { "json_pack_bug1278"  , test_json_pack_bug1278},
+    { "json_pack_bug5336"  , test_json_pack_bug5336},
 
     /* Mixed bytes, check JSON encoding */
     { "utf8_to_json", test_utf8_to_json},
diff --git a/tests/runtime/in_tcp.c b/tests/runtime/in_tcp.c
index 36aa07cc98d..75151c86a30 100644
--- a/tests/runtime/in_tcp.c
+++ b/tests/runtime/in_tcp.c
@@ -60,6 +60,28 @@ static void clear_output_num()
     set_output_num(0);
 }
 
+static int cb_count_msgpack(void *record, size_t size, void *data)
+{
+    msgpack_unpacked result;
+    size_t off = 0;
+
+    if (!TEST_CHECK(data != NULL)) {
+        flb_error("data is NULL");
+    }
+
+    /* Iterate each item array and apply rules */
+    msgpack_unpacked_init(&result);
+    while (msgpack_unpack_next(&result, record, size, &off) == MSGPACK_UNPACK_SUCCESS) {
+        pthread_mutex_lock(&result_mutex);
+        num_output++;
+        pthread_mutex_unlock(&result_mutex);
+    }
+    msgpack_unpacked_destroy(&result);
+
+    flb_free(record);
+    return 0;
+}
+
 /* Callback to check expected results */
 static int cb_check_result_json(void *record, size_t size, void *data)
 {
@@ -354,10 +376,77 @@ void flb_test_format_none_separator()
     test_ctx_destroy(ctx);
 }
 
+/*
+ * Ingest 64k records.
+ * https://github.com/fluent/fluent-bit/issues/5336
+ */
+void flb_test_issue_5336()
+{
+    struct flb_lib_out_cb cb_data;
+    struct test_ctx *ctx;
+    flb_sockfd_t fd;
+    int ret;
+    int num;
+    ssize_t w_size;
+    int not_used;
+    int i;
+    int count = 65535;
+
+    char *buf = "{\"test\":\"msg\"}";
+    size_t size = strlen(buf);
+
+    clear_output_num();
+
+    cb_data.cb = cb_count_msgpack;
+    cb_data.data = &not_used;
+
+    ctx = test_ctx_create(&cb_data);
+    if (!TEST_CHECK(ctx != NULL)) {
+        TEST_MSG("test_ctx_create failed");
+        exit(EXIT_FAILURE);
+    }
+
+    ret = flb_output_set(ctx->flb, ctx->o_ffd,
+                         "match", "*",
+                         NULL);
+    TEST_CHECK(ret == 0);
+
+    /* Start the engine */
+    ret = flb_start(ctx->flb);
+    TEST_CHECK(ret == 0);
+
+    /* use default host/port */
+    fd = connect_tcp(NULL, -1);
+    if (!TEST_CHECK(fd >= 0)) {
+        exit(EXIT_FAILURE);
+    }
+
+    for (i=0; i<count; i++) {
+        w_size = send(fd, buf, size, 0);
+        if (!TEST_CHECK(w_size == size)) {
+            TEST_MSG("failed to send, count=%d errno=%d",i, errno);
+            flb_socket_close(fd);
+            exit(EXIT_FAILURE);
+        }
+    }
+
+    /* waiting to flush */
+    flb_time_msleep(2500);
+
+    num = get_output_num();
+    if (!TEST_CHECK(num == count))  {
+        TEST_MSG("got %d, expected: %d", num, count);
+    }
+
+    flb_socket_close(fd);
+    test_ctx_destroy(ctx);
+}
+
 TEST_LIST = {
     {"tcp", flb_test_tcp},
     {"format_none", flb_test_format_none},
     {"format_none_separator", flb_test_format_none_separator},
+    {"65535_records_issue_5336", flb_test_issue_5336},
     {NULL, NULL}
 };