Repository: fluent/fluent-bit (C)

Issue:
filter_lua changes the JSON schema
## Bug Report

I have JSON logs in my project, and I've tested the **Fluent Bit 2.1.7** version on **Debian**. The Linux version is `5.10.0-19-cloud-amd64 (debian-kernel@lists.debian.org) (gcc-10 (Debian 10.2.1-6) 10.2.1 20210110, GNU ld (GNU Binutils for Debian) 2.35.2) #1 SMP Debian 5.10.149-2 (2022-10-21)`.

I realized that Fluent Bit is changing the JSON schema. Here's an example:

Input:` {"aaa": null, "bbb": "cccc"}`
Stdout:` {"date": 1689498354.316911, "token": "11111", "bbb": "cccc", "extra": {}, "hostname": "ip-xxx-xx-xx-xx"}`

The `aaa` field is missing. Here's another example:

Input: `{"foo": [], "bar": {}}`
Stdout: `{"date": 1689498914.5041, "token": "11111", "extra": {}, "foo": {}, "bar": {}, "hostname": "ip-xxx-xx-xx-xx"}`

The `foo` has changed to an object, but it should be a list

<!--- A clear and concise description of what the bug is. -->

**To Reproduce**

The parser.conf
```
[PARSER]
    Name        json-test
    Format      json
    Time_Key    time
    Time_Format %s.%L
    Time_Keep   On
```

The fluent.conf
```
[SERVICE]
    Daemon                          off
    Log_Level                       debug
    Parsers_file                    /tmp/parser.conf

[INPUT]
    Name                tail
    Tag                 json-test
    Path                /tmp/log.log
    Parser              json-test
    Read_from_Head      false
    Buffer_Chunk_Size   1m
    Buffer_Max_Size     1m

[FILTER]
    Name    record_modifier
    Match   *
    Record  hostname ${HOSTNAME}
    Record  token 11111

[FILTER]
    Name    Lua
    Match   *
    call    append_tag
    code    function append_tag(tag, timestamp, record) new_record = record new_record["extra"] = {} return 1, timestamp, new_record end


[OUTPUT]
    Name            stdout
    Match           *
    format          json_lines

```

Append a log `echo '{"foo": [], "bar": {}}' >> /tmp/log.log`
Append a log `echo '{"aaa": null, "bbb": "cccc"}' >> /tmp/log.log`

**Expected behaviors**
`{"date":1689499573.713323,"token":"11111","extra":{},"foo":[],"bar":{},"hostname":"ip-xxx-xx-xx-xx"}`
`{"date":1689499619.142707,"token":"11111","aaa":null,"bbb":"cccc","extra":{},"hostname":"ip-xxx-xx-xx-xx"}`


**Environment**
<!--- Include as many relevant details about the environment you experienced the bug in -->
* Version used: Fluent Bit v2.1.7
* Server Info: `Linux version 5.10.0-19-cloud-amd64 (debian-kernel@lists.debian.org) (gcc-10 (Debian 10.2.1-6) 10.2.1 20210110, GNU ld (GNU Binutils for Debian) 2.35.2) #1 SMP Debian 5.10.149-2 (2022-10-21)`


Discussion: > The foo has changed to an object, but it should be a list

For this case, could you try `type_array_key` property for filter_lua plugin ?
https://docs.fluentbit.io/manual/pipeline/filters/lua#config

filter_lua converted empty array -> empty map.
The property is to prevent it.

```
[INPUT]
    Name dummy
    Dummy {"foo":[], "bar":{}}

[FILTER]
    Name    Lua
    Match   *
    call    append_tag
    code    function append_tag(tag, timestamp, record) new_record = record new_record["extra"] = {} return 1, timestamp, new_record end
    Type_Array_Key foo

[OUTPUT]
    Name            stdout
    Match           *
    format          json_lines
```
> The aaa field is missing.

This is Lua spec.
https://www.lua.org/pil/2.5.html
```
Also like global variables, you can assign nil to a table field to delete it.
```

We convert `json` -> `msgpack` -> `lua assosiative array`.
nil is a special value for lua associative array. It is the value to delete key/value pair.
> > The foo has changed to an object, but it should be a list
> 
> For this case, could you try `type_array_key` property for filter_lua plugin ? https://docs.fluentbit.io/manual/pipeline/filters/lua#config
> 
> filter_lua converted empty array -> empty map. The property is to prevent it.
> 

Thank you for your reply. But Do you think it's the right way? Because my log lines could contain many empty lists. And I could not predict it. Also, There could be nested empty lists. Is there any way to prevent it?

For example a logline:
`{"foo": [], "bar": {"cc": [], "dd": {"cc": []}}}`

I agree with @enesgur , Considering when we are reading Kubernetes container logs. We can not predict which key would have empty lists.  



Fix (patch):
diff --git a/include/fluent-bit/flb_lua.h b/include/fluent-bit/flb_lua.h
index b2787430284..fb4ad99b49d 100644
--- a/include/fluent-bit/flb_lua.h
+++ b/include/fluent-bit/flb_lua.h
@@ -33,7 +33,8 @@
 
 enum flb_lua_l2c_type_enum {
     FLB_LUA_L2C_TYPE_INT,
-    FLB_LUA_L2C_TYPE_ARRAY
+    FLB_LUA_L2C_TYPE_ARRAY,
+    FLB_LUA_L2C_TYPE_MAP
 };
 
 struct flb_lua_l2c_type {
@@ -47,6 +48,27 @@ struct flb_lua_l2c_config {
     struct mk_list l2c_types;  /* data types (lua -> C) */
 };
 
+
+/*
+ * Metatable for Lua table.
+ * https://www.lua.org/manual/5.1/manual.html#2.8
+ */
+struct flb_lua_metadata {
+    int initialized;
+    int data_type; /* Map or Array */
+};
+
+static inline int flb_lua_metadata_init(struct flb_lua_metadata *meta)
+{
+    if (meta == NULL) {
+        return -1;
+    }
+    meta->initialized = FLB_TRUE;
+    meta->data_type = -1;
+
+    return 0;
+}
+
 /* convert from negative index to positive index */
 static inline int flb_lua_absindex(lua_State *l , int index)
 {
diff --git a/src/flb_lua.c b/src/flb_lua.c
index a374752326c..f872b512aa4 100644
--- a/src/flb_lua.c
+++ b/src/flb_lua.c
@@ -53,12 +53,35 @@ int flb_lua_is_valid_func(lua_State *lua, flb_sds_t func)
     return ret;
 }
 
+static int flb_lua_setmetatable(lua_State *l, struct flb_lua_metadata *meta, int index)
+{
+    int abs_index;
+
+    if (meta->initialized != FLB_TRUE) {
+        return -1;
+    }
+    abs_index = flb_lua_absindex(l, index);
+
+    lua_createtable(l, 0, 1);
+
+    /* data type */
+    lua_pushlstring(l, "type", 4);
+    lua_pushinteger(l, meta->data_type);
+    lua_settable(l, -3); /* point created table */
+
+    lua_setmetatable(l, abs_index);
+
+    return 0;
+}
+
 int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader)
 {
     int ret = 0;
     mpack_tag_t tag;
     uint32_t length;
     uint32_t i;
+    int index;
+    struct flb_lua_metadata meta;
 
     tag = mpack_read_tag(reader);
     switch (mpack_tag_type(&tag)) {
@@ -88,8 +111,12 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader)
             reader->data += length;
             break;
         case mpack_type_array:
+            flb_lua_metadata_init(&meta);
+            meta.data_type = FLB_LUA_L2C_TYPE_ARRAY;
+
             length = mpack_tag_array_count(&tag);
             lua_createtable(l, length, 0);
+            index = lua_gettop(l); /* save index of created table */
             for (i = 0; i < length; i++) {
                 ret = flb_lua_pushmpack(l, reader);
                 if (ret) {
@@ -97,10 +124,16 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader)
                 }
                 lua_rawseti(l, -2, i+1);
             }
+            flb_lua_setmetatable(l, &meta, index);
+
             break;
         case mpack_type_map:
+            flb_lua_metadata_init(&meta);
+            meta.data_type = FLB_LUA_L2C_TYPE_MAP;
+
             length = mpack_tag_map_count(&tag);
             lua_createtable(l, length, 0);
+            index = lua_gettop(l); /* save index of created table */
             for (i = 0; i < length; i++) {
                 ret = flb_lua_pushmpack(l, reader);
                 if (ret) {
@@ -112,6 +145,8 @@ int flb_lua_pushmpack(lua_State *l, mpack_reader_t *reader)
                 }
                 lua_settable(l, -3);
             }
+            flb_lua_setmetatable(l, &meta, index);
+
             break;
         default:
             return -1;
@@ -123,6 +158,8 @@ void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o)
 {
     int i;
     int size;
+    int index;
+    struct flb_lua_metadata meta;
 
     lua_checkstack(l, 3);
 
@@ -161,28 +198,38 @@ void flb_lua_pushmsgpack(lua_State *l, msgpack_object *o)
             break;
 
         case MSGPACK_OBJECT_ARRAY:
+            flb_lua_metadata_init(&meta);
+            meta.data_type = FLB_LUA_L2C_TYPE_ARRAY;
+
             size = o->via.array.size;
             lua_createtable(l, size, 0);
+            index = lua_gettop(l); /* save index of created table */
             if (size != 0) {
                 msgpack_object *p = o->via.array.ptr;
                 for (i = 0; i < size; i++) {
                     flb_lua_pushmsgpack(l, p+i);
-                    lua_rawseti (l, -2, i+1);
+                    lua_rawseti (l, index, i+1);
                 }
             }
+            flb_lua_setmetatable(l, &meta, index);
             break;
 
         case MSGPACK_OBJECT_MAP:
+            flb_lua_metadata_init(&meta);
+            meta.data_type = FLB_LUA_L2C_TYPE_MAP;
+
             size = o->via.map.size;
             lua_createtable(l, 0, size);
+            index = lua_gettop(l); /* save index of created table */
             if (size != 0) {
                 msgpack_object_kv *p = o->via.map.ptr;
                 for (i = 0; i < size; i++) {
                     flb_lua_pushmsgpack(l, &(p+i)->key);
                     flb_lua_pushmsgpack(l, &(p+i)->val);
-                    lua_settable(l, -3);
+                    lua_settable(l, index);
                 }
             }
+            flb_lua_setmetatable(l, &meta, index);
             break;
     }
 }
@@ -277,10 +324,10 @@ int flb_lua_arraylength(lua_State *l, int index)
     return max;
 }
 
-static void lua_toarray(lua_State *l,
-                        msgpack_packer *pck,
-                        int index,
-                        struct flb_lua_l2c_config *l2cc)
+static void lua_toarray_msgpack(lua_State *l,
+                                msgpack_packer *pck,
+                                int index,
+                                struct flb_lua_l2c_config *l2cc)
 {
     int len;
     int i;
@@ -319,7 +366,6 @@ static void lua_toarray_mpack(lua_State *l,
 
 static void try_to_convert_data_type(lua_State *l,
                                      msgpack_packer *pck,
-                                     int index,
                                      struct flb_lua_l2c_config *l2cc)
 {
     size_t   len;
@@ -351,7 +397,7 @@ static void try_to_convert_data_type(lua_State *l,
             l2c = mk_list_entry(head, struct flb_lua_l2c_type, _head);
             if (!strncmp(l2c->key, tmp, len) && l2c->type == FLB_LUA_L2C_TYPE_ARRAY) {
                 flb_lua_tomsgpack(l, pck, -1, l2cc);
-                lua_toarray(l, pck, 0, l2cc);
+                lua_toarray_msgpack(l, pck, 0, l2cc);
                 return;
             }
         }
@@ -364,7 +410,6 @@ static void try_to_convert_data_type(lua_State *l,
 
 static void try_to_convert_data_type_mpack(lua_State *l,
                                            mpack_writer_t *writer,
-                                           int index,
                                            struct flb_lua_l2c_config *l2cc)
 {
     size_t   len;
@@ -407,6 +452,89 @@ static void try_to_convert_data_type_mpack(lua_State *l,
     flb_lua_tompack(l, writer, 0, l2cc);
 }
 
+static int flb_lua_getmetatable(lua_State *l, int index, struct flb_lua_metadata *meta)
+{
+    int lua_ret;
+    int abs_index;
+    const char *str;
+    size_t len;
+
+    if (meta->initialized != FLB_TRUE) {
+        return -1;
+    }
+
+    lua_ret = lua_getmetatable(l, index);
+    if (lua_ret == 0) {
+        /* no metadata */
+        return -1;
+    }
+    else if (lua_type(l, -1) != LUA_TTABLE) {
+        /* invalid metatable? */
+        lua_pop(l, 1);
+        return -1;
+    }
+
+    lua_pushnil(l);
+    abs_index = flb_lua_absindex(l, -2);
+    while (lua_next(l, abs_index) != 0) {
+        if (lua_type(l, -2) != LUA_TSTRING) {
+            /* key is not a string */
+            flb_debug("key is not a string");
+            lua_pop(l, 1);
+            continue;
+        }
+
+        str = lua_tolstring(l, -2, &len); /* key */
+
+        if (len == 4 && strncmp(str, "type", 4) == 0) {
+            /* data_type */
+            if (lua_type(l, -1) != LUA_TNUMBER) {
+                /* value is not data type */
+                flb_debug("type is not num. type=%s", lua_typename(l, lua_type(l, -1)));
+                lua_pop(l, 1);
+                continue;
+            }
+            meta->data_type = (int)lua_tointeger(l, -1);
+        }
+        lua_pop(l, 1);
+    }
+    lua_pop(l, 1); /* pop metatable */
+
+    return 0;
+}
+
+static void lua_tomap_mpack(lua_State *l,
+                            mpack_writer_t *writer,
+                            int index,
+                            struct flb_lua_l2c_config *l2cc)
+{
+    int len;
+
+    len = 0;
+    lua_pushnil(l);
+    while (lua_next(l, -2) != 0) {
+        lua_pop(l, 1);
+        len++;
+    }
+    mpack_write_tag(writer, mpack_tag_map(len));
+
+    lua_pushnil(l);
+
+    if (l2cc->l2c_types_num > 0) {
+        /* type conversion */
+        while (lua_next(l, -2) != 0) {
+            try_to_convert_data_type_mpack(l, writer, l2cc);
+            lua_pop(l, 1);
+        }
+    } else {
+        while (lua_next(l, -2) != 0) {
+            flb_lua_tompack(l, writer, -1, l2cc);
+            flb_lua_tompack(l, writer, 0, l2cc);
+            lua_pop(l, 1);
+        }
+    }
+}
+
 void flb_lua_tompack(lua_State *l,
                      mpack_writer_t *writer,
                      int index,
@@ -414,6 +542,8 @@ void flb_lua_tompack(lua_State *l,
 {
     int len;
     int i;
+    int use_metatable = FLB_FALSE;
+    struct flb_lua_metadata meta;
 
     switch (lua_type(l, -1 + index)) {
         case LUA_TSTRING:
@@ -445,6 +575,23 @@ void flb_lua_tompack(lua_State *l,
                 mpack_write_false(writer);
             break;
         case LUA_TTABLE:
+            flb_lua_metadata_init(&meta);
+            if (flb_lua_getmetatable(l, -1 + index, &meta) == 0 &&
+                meta.data_type >= 0) {
+                use_metatable = FLB_TRUE;
+            }
+            if (use_metatable) {
+                if (meta.data_type == FLB_LUA_L2C_TYPE_ARRAY) {
+                    /* array */
+                    lua_toarray_mpack(l, writer, 0, l2cc);
+                }
+                else {
+                    /* map */
+                    lua_tomap_mpack(l, writer, -1 + index, l2cc);
+                }
+                break;
+            }
+
             len = flb_lua_arraylength(l, -1 + index);
             if (len > 0) {
                 mpack_write_tag(writer, mpack_tag_array(len));
@@ -453,31 +600,9 @@ void flb_lua_tompack(lua_State *l,
                     flb_lua_tompack(l, writer, 0, l2cc);
                     lua_pop(l, 1);
                 }
-            } else
-            {
-                len = 0;
-                lua_pushnil(l);
-                while (lua_next(l, -2) != 0) {
-                    lua_pop(l, 1);
-                    len++;
-                }
-                mpack_write_tag(writer, mpack_tag_map(len));
-
-                lua_pushnil(l);
-
-                if (l2cc->l2c_types_num > 0) {
-                    /* type conversion */
-                    while (lua_next(l, -2) != 0) {
-                        try_to_convert_data_type_mpack(l, writer, index, l2cc);
-                        lua_pop(l, 1);
-                    }
-                } else {
-                    while (lua_next(l, -2) != 0) {
-                        flb_lua_tompack(l, writer, -1, l2cc);
-                        flb_lua_tompack(l, writer, 0, l2cc);
-                        lua_pop(l, 1);
-                    }
-                }
+            }
+            else {
+                lua_tomap_mpack(l, writer, -1 + index, l2cc);
             }
             break;
         case LUA_TNIL:
@@ -497,6 +622,41 @@ void flb_lua_tompack(lua_State *l,
     }
 }
 
+static inline void lua_tomap_msgpack(lua_State *l,
+                                     msgpack_packer *pck,
+                                     int index,
+                                     struct flb_lua_l2c_config *l2cc)
+{
+    int len;
+    int abs_index;
+
+    abs_index = flb_lua_absindex(l, index);
+
+    len = 0;
+    lua_pushnil(l);
+    while (lua_next(l, abs_index) != 0) {
+        lua_pop(l, 1);
+        len++;
+    }
+    msgpack_pack_map(pck, len);
+
+    lua_pushnil(l);
+
+    if (l2cc->l2c_types_num > 0) {
+        /* type conversion */
+        while (lua_next(l, abs_index) != 0) {
+            try_to_convert_data_type(l, pck, l2cc);
+            lua_pop(l, 1);
+        }
+    } else {
+        while (lua_next(l, abs_index) != 0) {
+            flb_lua_tomsgpack(l, pck, -1, l2cc);
+            flb_lua_tomsgpack(l, pck, 0, l2cc);
+            lua_pop(l, 1);
+        }
+    }
+}
+
 void flb_lua_tomsgpack(lua_State *l,
                        msgpack_packer *pck,
                        int index,
@@ -504,6 +664,8 @@ void flb_lua_tomsgpack(lua_State *l,
 {
     int len;
     int i;
+    int use_metatable = FLB_FALSE;
+    struct flb_lua_metadata meta;
 
     switch (lua_type(l, -1 + index)) {
         case LUA_TSTRING:
@@ -536,6 +698,23 @@ void flb_lua_tomsgpack(lua_State *l,
                 msgpack_pack_false(pck);
             break;
         case LUA_TTABLE:
+            flb_lua_metadata_init(&meta);
+            if (flb_lua_getmetatable(l, -1 + index, &meta) == 0 &&
+                meta.data_type >= 0) {
+                use_metatable = FLB_TRUE;
+            }
+            if (use_metatable) {
+                if (meta.data_type == FLB_LUA_L2C_TYPE_ARRAY) {
+                    /* array */
+                    lua_toarray_msgpack(l, pck, 0, l2cc);
+                }
+                else {
+                    /* map */
+                    lua_tomap_msgpack(l, pck, -1 + index, l2cc);
+                }
+                break;
+            }
+
             len = flb_lua_arraylength(l, -1 + index);
             if (len > 0) {
                 msgpack_pack_array(pck, len);
@@ -544,31 +723,9 @@ void flb_lua_tomsgpack(lua_State *l,
                     flb_lua_tomsgpack(l, pck, 0, l2cc);
                     lua_pop(l, 1);
                 }
-            } else
-            {
-                len = 0;
-                lua_pushnil(l);
-                while (lua_next(l, -2) != 0) {
-                    lua_pop(l, 1);
-                    len++;
-                }
-                msgpack_pack_map(pck, len);
-
-                lua_pushnil(l);
-
-                if (l2cc->l2c_types_num > 0) {
-                    /* type conversion */
-                    while (lua_next(l, -2) != 0) {
-                        try_to_convert_data_type(l, pck, index, l2cc);
-                        lua_pop(l, 1);
-                    }
-                } else {
-                    while (lua_next(l, -2) != 0) {
-                        flb_lua_tomsgpack(l, pck, -1, l2cc);
-                        flb_lua_tomsgpack(l, pck, 0, l2cc);
-                        lua_pop(l, 1);
-                    }
-                }
+            }
+            else {
+                lua_tomap_msgpack(l, pck, -1 + index, l2cc);
             }
             break;
         case LUA_TNIL: