Repository: lf-edge/ekuiper
Pull request #3186 (2024-09-10 09:30:28)

docs(conn): connection guide 


Patch:
diff --git a/docs/directory.json b/docs/directory.json
index d836b8db..468a8bce 100644
--- a/docs/directory.json
+++ b/docs/directory.json
@@ -348,6 +348,10 @@
               ]
             }
           ]
+        },
+        {
+          "title": "连接管理",
+          "path": "guide/connections/overview"
         }
       ]
     },
@@ -410,6 +414,10 @@
         {
           "title": "错误码",
           "path": "operation/usage/error_code"
+        },
+        {
+          "title": "数据链路追踪",
+          "path": "operation/usage/trace_data"
         }
       ]
     },
@@ -542,6 +550,10 @@
         {
           "title": "规则试运行",
           "path": "api/restapi/ruletest"
+        },
+        {
+          "title": "数据链路追踪",
+          "path": "api/restapi/trace"
         }
       ]
     },
@@ -1112,6 +1124,10 @@
               ]
             }
           ]
+        },
+        {
+          "title": "Connection Management",
+          "path": "guide/connections/overview"
         }
       ]
     },
@@ -1228,6 +1244,10 @@
               "path": "operation/manager-ui/plugins_in_manager"
             }
           ]
+        },
+        {
+          "title": "Tracing Data",
+          "path": "operation/usage/trace_data"
         }
       ]
     },
@@ -1299,6 +1319,10 @@
         {
           "title": "Rule Test",
           "path": "api/restapi/ruletest"
+        },
+        {
+          "title": "Trace Data",
+          "path": "api/restapi/trace"
         }
       ]
     },
diff --git a/docs/en_US/guide/connections/overview.md b/docs/en_US/guide/connections/overview.md
new file mode 100644
index 00000000..fe22a29b
--- /dev/null
+++ b/docs/en_US/guide/connections/overview.md
@@ -0,0 +1,134 @@
+# Connection Management
+
+Sources and Sinks are used for interacting with external systems, which involve connecting to external resources. This
+chapter primarily explains how eKuiper manages connections.
+
+## Connection Types
+
+Different connection types vary in complexity. For example, MQTT long connections require attention to connection
+status, and there may be disconnections during rule execution that necessitate automatic reconnection. On the other
+hand, HTTP connections are stateless by default, making their state management simpler. To unify the management of
+complex connection resources, including creation, reuse, automatic reconnection, and obtaining connection status,
+eKuiper v2 introduced an internal connection pool component and adapted a series of connection types:
+
+- MQTT Connection
+- Neuron Connection
+- EdgeX Connection
+- SQL Connection
+- HTTP Connection (including REST sink, HTTP Pull source, and HTTP push source connections)
+- WebSocket Connection
+
+Other connection types may be gradually integrated in subsequent versions. Connection types integrated into the
+connection pool can be independently created via API and accessed.
+
+In eKuiper, the lifecycle management of various connections is divided into three types:
+
+1. **Connection Attached to Rule**: By default, connections are managed by the Source/Sink implementation itself, with
+   their lifecycle controlled by the rule in use. The connection resources will only start connecting when the rule is
+   started; when the rule ends, the connection will be closed. In the example below, we create a data stream `memStream`
+   of type `memory`. Since this type is not integrated into the connection pool, the connection will only be established
+   when a rule using this stream is started.
+
+    ```sql
+    create stream memStream () WITH (TYPE="mqtt", DATASOURCE="demo")
+    ```
+
+2. **Anonymous Connection Resource Managed by Connection Pool**: Some connection types are adapted to the connection
+   pool management interface, with their lifecycle managed by the connection pool. When a rule containing these types of
+   connections is started, the rule will obtain an anonymous (actual resource id generated by the rule and not shared)
+   resource from the connection pool. In the example below, we create a data stream `mqttStream` of type `mqtt`. The
+   connection is anonymous, and since this type is adapted to the connection pool, we can retrieve this connection via
+   the connection API. When the rule is deleted, the corresponding connection will also be deleted.
+
+    ```sql
+    create stream mqttStream () WITH (TYPE="mqtt", DATASOURCE="demo")
+    ```
+
+3. **User-Created Connection Resource**: Users can add, delete, modify, and query resources via
+   the [Connection Management API](../../api/restapi/connection.md). Resources created via the API must specify a unique
+   id, which can be referenced in rules. **Note**: Only connection resources adapted to the connection pool can be
+   managed via the API. Connections created in this way are independent physical connections that run immediately after
+   creation and do not depend on rules. They can be shared by multiple rules or multiple sources/sinks.
+
+**Note**: User-created connections are physical connections that will automatically reconnect until the connection is
+successful.
+
+### Connection Reuse
+
+User-created connection resources can run independently, and multiple rules can reference this named resource.
+Connection reuse is configured through the `connectionSelector` configuration item. Users only need to create the
+connection resource once for reuse, improving connection management efficiency and simplifying the configuration
+process.
+
+1. **Create Resource**: As shown in the example below, create a connection with id `mqttcon1` via the API. The
+   connection parameters required can be configured in `props`. After the connection is successfully created, `mqttcon1`
+   can be found in the connection list API.
+
+    ```shell
+    POST http://localhost:9081/connections
+    {
+      "id": "mqttcon1"
+      "typ":"mqtt",
+      "props": {
+        server: "tcp://127.0.0.1:1883"
+      }
+    }
+    ```
+
+2. **Use in Data Source**: When configuring the MQTT source (`$ekuiper/etc/mqtt_source.yaml`), you can reference the
+   above connection configuration via `connectionSelector`, for example, both `demo_conf` and `demo2_conf` will
+   reference the connection configuration of `mqttcon1`.
+
+```yaml
+#Override the global configurations
+demo_conf: #Conf_key
+  qos: 0
+  connectionSelector: mqttcon1
+  servers: [ tcp://10.211.55.6:1883, tcp://127.0.0.1 ]
+
+#Override the global configurations
+demo2_conf: #Conf_key
+  qos: 0
+  connentionSelector: mqttcon1
+  servers: [ tcp://10.211.55.6:1883, tcp://127.0.0.1 ]
+```
+
+Based on `demo_conf` and `demo2_conf`, create two data streams `demo` and `demo2`:
+
+```text
+demo (
+    ...
+  ) WITH (DATASOURCE="test/", FORMAT="JSON", CONF_KEY="demo_conf");
+
+demo2 (
+    ...
+  ) WITH (DATASOURCE="test2/", FORMAT="JSON", CONF_KEY="demo2_conf");
+
+```
+
+When corresponding rules reference the above data streams, the source parts of the rules will share the connection.
+Here, `DATASOURCE` corresponds to the MQTT subscription topic, and the `qos` in the configuration item will be used as
+the `Qos` for subscription. In the example configuration above, `demo` subscribes to topic `test/` with Qos 0,
+and `demo2` subscribes to topic `test2/` with Qos 0.
+
+::: tip
+
+For MQTT sources, if two streams have the same `DATASOURCE` but different `qos` values, only the rule that starts first
+will trigger the subscription.
+
+:::
+
+You can also reuse the defined connection resource in the rule's action via `connentionSelector`.
+
+## Connection Status
+
+Connection status is divided into three types:
+
+1. **Connected**, represented by 1 in metrics.
+2. **Connecting**, represented by 0 in metrics.
+3. **Disconnected**, represented by 1 in metrics.
+
+Users can retrieve the connection status via the connection API. Additionally, users can view the connection status in
+the rule's source/sink metrics, for example, the `source_demo_0_connection_status` metric indicates the connection
+status of the `demo` stream. For a complete list of supported connection metrics, please refer to
+the [Metrics List](../../operation/usage/monitor_with_prometheus.md#metric-types).
diff --git a/docs/en_US/guide/connector.md b/docs/en_US/guide/connector.md
index af102e25..6642d823 100644
--- a/docs/en_US/guide/connector.md
+++ b/docs/en_US/guide/connector.md
@@ -73,79 +73,6 @@ For specialized data dispatch requirements or integrations with particular platf
 
 [Data templates](./sinks/data_template.md) in eKuiper allow for "secondary processing" of analysis results to cater to the diverse formatting requirements of different sink systems. Utilizing the Golang template system, eKuiper provides mechanisms for dynamic data transformation, conditional outputs, and iterative processing. This ensures compatibility and precise formatting for various sinks.
 
-## Connection Selector
-
-The connector selector is a powerful feature in eKuiper that allows users to define a connection once and reuse it across multiple configurations. It ensures efficient connection management and reduces redundancy.
-
-To define a global connection configuration, use the `connectionSelector` key to name your connection, e.g., `mqtt.localConnection`. Override global configurations with custom configurations but reference the same `connectionSelector`.
-
-For example, consider the configurations `demo_conf` and `demo2_conf`:
-
-```yaml
-#Override the global configurations
-demo_conf: #Conf_key
-  qos: 0
-  connectionSelector: mqtt.localConnection
-  servers: [tcp://10.211.55.6:1883, tcp://127.0.0.1]
-
-#Override the global configurations
-demo2_conf: #Conf_key
-  qos: 0
-  connentionSelector: mqtt.localConnection
-  servers: [tcp://10.211.55.6:1883, tcp://127.0.0.1]
-```
-
-Both configurations reference the same `connectionSelector`, indicating that they utilize the same MQTT connection. When streams `demo` and `demo2` are defined based on these configurations:
-
-```text
-demo (
-    ...
-  ) WITH (DATASOURCE="test/", FORMAT="JSON", CONF_KEY="demo_conf");
-
-demo2 (
-    ...
-  ) WITH (DATASOURCE="test2/", FORMAT="JSON", CONF_KEY="demo2_conf");
-
-```
-
-They inherently share the MQTT connection. Specifically:
-
-- The stream `demo` subscribes to the MQTT topic `test/` with a QoS of 0.
-- The stream `demo2` subscribes to `test2/`, also with a QoS of 0.
-
-::: tip
-
-For MQTT sources, if two streams have the same `DATASOURCE` but differing `qos` values, only the rule started first will trigger a subscription.
-
-:::
-
-**Configuration**
-
-The actual connection profiles, like `mqtt.localConnection`, are usually defined in a separate file, such as `connections/connection.yaml`.
-
-Example
-
-```yaml
-mqtt:
-  localConnection: #connection key
-    server: "tcp://127.0.0.1:1883"
-    username: ekuiper
-    password: password
-    #certificationPath: /var/kuiper/xyz-certificate.pem
-    #privateKeyPath: /var/kuiper/xyz-private.pem.ke
-    #insecureSkipVerify: false
-    #protocolVersion: 3
-    clientid: ekuiper
-  cloudConnection: #connection key
-    server: "tcp://broker.emqx.io:1883"
-    username: user1
-    password: password
-    #certificationPath: /var/kuiper/xyz-certificate.pem
-    #privateKeyPath: /var/kuiper/xyz-private.pem.ke
-    #insecureSkipVerify: false
-    #protocolVersion: 3
-```
-
 ## Batch Configuration
 
 For advanced data stream processing, eKuiper offers an array of connectors like Memory, File, MQTT, and more. To streamline the integration, eKuiper’s REST API introduces the capability for batch configuration, allowing users to simultaneously import or export multiple configurations.
diff --git a/docs/en_US/guide/rules/overview.md b/docs/en_US/guide/rules/overview.md
index 27e67422..38e9f984 100644
--- a/docs/en_US/guide/rules/overview.md
+++ b/docs/en_US/guide/rules/overview.md
@@ -301,3 +301,6 @@ When we try to send a record to the stream, the status of the rule is obtained a
 ```
 
 It can be seen that `records_in_total` and `records_out_total` of each operator have changed from 0 to 1, which means that the operator has received a record and passed a record to the next operator, and finally sent to the `sink` and the `sink` wrote 1 record.
+
+If Prometheus configuration is enabled, these metrics will also be collected by Prometheus. For a complete list of
+operational metrics, please refer to the [Metrics List](../../operation/usage/monitor_with_prometheus.md#metric-types).
diff --git a/docs/en_US/operation/usage/monitor_with_prometheus.md b/docs/en_US/operation/usage/monitor_with_prometheus.md
index df2bba2a..ee220d48 100644
--- a/docs/en_US/operation/usage/monitor_with_prometheus.md
+++ b/docs/en_US/operation/usage/monitor_with_prometheus.md
@@ -70,6 +70,14 @@ After version 1.6.1, we added two more exception-related metrics to facilitate t
 - last_exception: the error message of the last exception.
 - last_exception_time: the time of the last exception.
 
+After version 2.0.0, we added connection-related metrics for source/sink.
+
+- connection_status: Connection status. 1 for connected, 0 for connecting, -1 for disconnected.
+- connection_last_connected_time: The last successful connection time.
+- connection_last_disconnected_time: The last disconnection time.
+- connection_last_disconnected_message: The message of the last disconnection exception.
+- connection_last_try_time: The last reconnection attempt time.
+
 The numeric types of these metrics can all be monitored using Prometheus. In the next section we will describe how to configure the Prometheus service in eKuiper.
 
 ## Configuring the Prometheus Service in eKuiper
diff --git a/docs/en_US/operation/usage/trace_data.md b/docs/en_US/operation/usage/trace_data.md
index cbfdb8f5..ddaf9e84 100644
--- a/docs/en_US/operation/usage/trace_data.md
+++ b/docs/en_US/operation/usage/trace_data.md
@@ -8,6 +8,7 @@ You can expose Open Telemetry data to the remote Open Telemetry Collector throug
 
 ```yaml
 openTelemetry:
+  serviceName: kuiperd-service
   enableRemoteCollector: false
   remoteEndpoint: localhost:4318
   localTraceCapacity: 2048
diff --git a/docs/zh_CN/guide/connections/overview.md b/docs/zh_CN/guide/connections/overview.md
new file mode 100644
index 00000000..90794987
--- /dev/null
+++ b/docs/zh_CN/guide/connections/overview.md
@@ -0,0 +1,114 @@
+# 连接管理
+
+Source 和 Sink 用于与外部系统的交互，其中都会包含连接外部资源的动作。本章主要讲解 eKuiper 中对连接的管理。
+
+## 连接类型
+
+不同的连接类型复杂度各不相同，例如 MQTT 长连接需要关注连接的状态，规则运行中可能出现连接的断连，需要自动重连等复杂的管理；而
+HTTP 连接默认为无状态的连接，状态管理较为简单。为了统一管理复杂的连接资源创建，复用，自动重连以及获取连接状态等功能，eKuiper
+v2 增加了内部的连接池组件，并适配了一系列连接类型：
+
+- MQTT 连接
+- Neuron 连接
+- EdgeX 连接
+- SQL 连接
+- HTTP 连接 （包括 REST sink，HTTP Pull source，HTTP push source 使用的连接）
+- WebSocket 连接
+
+其余连接类型可能会在后续版本中陆续接入。接入连接池的连接类型可通过 API 进行资源的独立创建，并获取 API。
+
+eKuiper 中对各种连接的生命周期的管理分为 3 种：
+
+1. 连接附属于规则：默认情况下，连接由 Source/Sink 的实现自行管理，其生命周期由使用的规则进行控制。规则启动时，使用到的连接资源才会开始连接；规则结束时，连接将被关闭。在下例中，我们创建了
+   memory 类型的数据流 memStream。由于该类型未接入连接池，只有当使用该流的规则启动时，才会进行连接。
+
+    ```sql
+    create stream memStream () WITH (TYPE="mqtt", DATASOURCE="demo")
+    ```
+
+2. 连接池管理的匿名连接资源：部分连接类型适配了连接池管理接口，其生命周期由连接池管理。当启动包含这些类型连接的规则时，规则会从连接池获取匿名（实际资源
+   id 由规则生成，且不会被共享）资源。在下例中，我们创建了 mqtt 类型的数据流 mqttStream。连接为匿名连接，由于该类型适配了连接池，我们可以在连接
+   API 中获取到该连接。规则删除时，对应连接也会删除。
+
+    ```sql
+    create stream mqttStream () WITH (TYPE="mqtt", DATASOURCE="demo")
+    ```
+
+3. 用户创建的连接资源：用户可通过[连接管理API](../../api/restapi/connection.md) 进行资源的增删改查。通过 API 创建的资源必须指定唯一的
+   id，规则中可引用此处创建的规则资源。**请注意**：只有适配连接池的连接资源可通过 API
+   进行管理。这种连接类型创建的连接为独立的物理连接，创建完后会立即运行，无需依附于规则。它可以被多个规则，或者多个
+   source/sink 共用。
+
+**请注意**：用户创建的连接为实体连接，会自动重连直到连接成功为止。
+
+### 连接重用
+
+用户创建的连接资源可以独立运行，多个规则可以引用该命名资源。连接重用是通过 `connectionSelector`
+配置项进行配置。用户只需创建一次连接资源即可复用，提升连接管理效率，简化配置流程。
+
+1. 创建资源，如下例通过 API 创建 id 为 mqttcon1 的连接。可在 props 中配置连接需要的参数。连接创建成功后，mqttcon1 可在连接列表
+   API 中找到。
+
+    ```shell
+    POST http://localhost:9081/connections
+    {
+      "id": "mqttcon1"
+      "typ":"mqtt",
+      "props": {
+        server: "tcp://127.0.0.1:1883"
+      }
+    }
+    ```
+
+2. 在数据源中使用。在配置 MQTT 源（`$ekuiper/etc/mqtt_source.yaml`）时，可通过 `connectionSelector`
+   引用以上连接配置，例如`demo_conf` 和 `demo2_conf` 都将引用 `mqttcon1` 的连接配置。
+
+```yaml
+#Override the global configurations
+demo_conf: #Conf_key
+   qos: 0
+   connectionSelector: mqttcon1
+   servers: [ tcp://10.211.55.6:1883, tcp://127.0.0.1 ]
+
+#Override the global configurations
+demo2_conf: #Conf_key
+   qos: 0
+   connentionSelector: mqttcon1
+   servers: [ tcp://10.211.55.6:1883, tcp://127.0.0.1 ]
+```
+
+基于 `demo_conf` 和 `demo2_conf` 分别创建两个数据流 `demo` 和 `demo2`：
+
+```text
+demo (
+    ...
+  ) WITH (DATASOURCE="test/", FORMAT="JSON", CONF_KEY="demo_conf");
+
+demo2 (
+    ...
+  ) WITH (DATASOURCE="test2/", FORMAT="JSON", CONF_KEY="demo2_conf");
+
+```
+
+当相应的规则分别引用以上数据流时，规则之间的源部分将共享连接。在这里 `DATASOURCE` 对应 mqtt 订阅的 topic，配置项中的 `qos`
+将用作订阅时的 `Qos`。在以上示例配置中，`demo` 以 Qos 0 订阅 topic `test/`，`demo2` 以 Qos 0 订阅 topic `test2/` 。
+
+::: tip
+
+对于MQTT源，如果两个流具有相同的 `DATASOURCE` 但 `qos` 值不同，则只有先启动的规则才会触发订阅。
+
+:::
+
+也可以在规则的 action 中，通过 connentionSelector 重用定义的连接资源。
+
+## 连接状态
+
+连接状态分成 3 种：
+
+1. 已连接，指标中用 1 表示。
+2. 连接中，指标中用 0 表示。
+3. 未连接，指标中用 1 表示。
+
+用户可通过连接 API 获取连接的状态。同时，用户也可通过规则的指标查看规则 source/sink
+中连接的状态，例如 `source_demo_0_connection_status` 指标表示 demo
+流的连接状态。所有支持的连接指标请查看[指标列表](../../operation/usage/monitor_with_prometheus.md#运行指标)。
diff --git a/docs/zh_CN/guide/connector.md b/docs/zh_CN/guide/connector.md
index 20eec63d..9e055a39 100644
--- a/docs/zh_CN/guide/connector.md
+++ b/docs/zh_CN/guide/connector.md
@@ -73,72 +73,6 @@ eKuiper Sink 连接器负责将 eKuiper 处理后的数据发送到各种目标
 
 eKuiper [数据模板](./sinks/data_template.md) 支持用户对分析结果进行"二次处理"，以满足不同接收系统的多样化格式要求。利用 Golang 模板系统，eKuiper 提供了动态数据转换、条件输出和迭代处理的机制，确保了与各种接收器的兼容性和精确格式化。
 
-## 连接器的重用
-
-eKuiper 支持通过 `connectionSelector` 配置项对连接器进行重用，用户只需一次定义即可在多个配置中重用，提升连接管理效率，简化配置流程。
-
-**配置**
-
-以 MQTT 数据源为例，您可首先在连接配置文件 `connections/connection.yaml` 中定义 MQTT 全局连接信息，例如  `mqtt.localConnection` 和 `mqtt.cloudConnection`。
-
-```yaml
-mqtt:
-  localConnection: #connection key
-    server: "tcp://127.0.0.1:1883"
-    username: ekuiper
-    password: password
-    #certificationPath: /var/kuiper/xyz-certificate.pem
-    #privateKeyPath: /var/kuiper/xyz-private.pem.ke
-    #insecureSkipVerify: false
-    #protocolVersion: 3
-    clientid: ekuiper
-  cloudConnection: #connection key
-    server: "tcp://broker.emqx.io:1883"
-    username: user1
-    password: password
-    #certificationPath: /var/kuiper/xyz-certificate.pem
-    #privateKeyPath: /var/kuiper/xyz-private.pem.ke
-    #insecureSkipVerify: false
-    #protocolVersion: 3
-```
-
-在配置 MQTT 源（`$ekuiper/etc/mqtt_source.yaml`）时，可通过 `connectionSelector` 引用以上连接配置，例如`demo_conf` 和 `demo2_conf` 都将引用 `mqtt.localConnection` 的连接配置。
-
-```yaml
-#Override the global configurations
-demo_conf: #Conf_key
-  qos: 0
-  connectionSelector: mqtt.localConnection
-  servers: [tcp://10.211.55.6:1883, tcp://127.0.0.1]
-
-#Override the global configurations
-demo2_conf: #Conf_key
-  qos: 0
-  connentionSelector: mqtt.localConnection
-  servers: [tcp://10.211.55.6:1883, tcp://127.0.0.1]
-```
-
-基于 `demo_conf` 和 `demo2_conf` 分别创建两个数据流 `demo` 和 `demo2`：
-
-```text
-demo (
-    ...
-  ) WITH (DATASOURCE="test/", FORMAT="JSON", CONF_KEY="demo_conf");
-
-demo2 (
-    ...
-  ) WITH (DATASOURCE="test2/", FORMAT="JSON", CONF_KEY="demo2_conf");
-
-```
-
-当相应的规则分别引用以上数据流时，规则之间的源部分将共享连接。在这里 `DATASOURCE` 对应 mqtt 订阅的 topic，配置项中的 `qos` 将用作订阅时的 `Qos`。在以上示例配置中，`demo` 以 Qos 0 订阅 topic `test/`，`demo2` 以 Qos 0 订阅 topic `test2/` 。
-
-::: tip
-
-对于MQTT源，如果两个流具有相同的 `DATASOURCE` 但 `qos` 值不同，则只有先启动的规则才会触发订阅。
-
-:::
-
 ## 批量配置
 
 eKuiper 提供了 Memory、File、MQTT 等多种数据连接器。为进一步简化用户的配置流程，eKuiper 通过 REST API 引入了批量配置功能，支持用户同时导入或导出多个配置。
diff --git a/docs/zh_CN/guide/rules/overview.md b/docs/zh_CN/guide/rules/overview.md
index c04be75c..69c466bc 100644
--- a/docs/zh_CN/guide/rules/overview.md
+++ b/docs/zh_CN/guide/rules/overview.md
@@ -283,3 +283,6 @@ cronDatetimeRange 支持结构体数组，你可以声明一组时间段来表
 ```
 
 可以看到每个算子的 `records_in_total` 与 `records_out_total` 都由 0 变为了 1，代表了该算子接收到了一条记录，并向下一个算子传递了一条记录，最终在 `sink` 端向 sink 写入了 1 条记录。
+
+若开启 Prometheus 配置，这些指标也会收集到 Prometheus
+中。全部的运行指标列表请查看[指标列表](../../operation/usage/monitor_with_prometheus.md#运行指标)。
diff --git a/docs/zh_CN/operation/usage/monitor_with_prometheus.md b/docs/zh_CN/operation/usage/monitor_with_prometheus.md
index 4a625637..4d52cbf7 100644
--- a/docs/zh_CN/operation/usage/monitor_with_prometheus.md
+++ b/docs/zh_CN/operation/usage/monitor_with_prometheus.md
@@ -70,6 +70,14 @@ kuiper_rule_count: eKuiper 中有多少条规则运行，多少条规则暂停
 - last_exception：最近一次的异常的错误信息。
 - last_exception_time：最近一次异常的发生时间。
 
+在 2.0.0 版本之后，我们为 source/sink 添加了连接相关指标。
+
+- connection_status：连接状态。1 为已连接，0 为连接中，-1 为未连接。
+- connection_last_connected_time：最近一次连接成功时间
+- connection_last_disconnected_time：最近一次断连时间
+- connection_last_disconnected_message：最近一次断连异常的消息
+- connection_last_try_time：最近一次重连时间
+
 这些运行指标中的数值类型指标均可使用 Prometheus 进行监控。下一节我们将描述如何配置 eKuiper 中的 Prometheus 服务。
 
 ## 配置 eKuiper 的 Prometheus 服务
diff --git a/docs/zh_CN/operation/usage/trace_data.md b/docs/zh_CN/operation/usage/trace_data.md
index b035d3a5..8c29ea11 100644
--- a/docs/zh_CN/operation/usage/trace_data.md
+++ b/docs/zh_CN/operation/usage/trace_data.md
@@ -8,6 +8,7 @@ eKuiper 的规则是一个持续运行的流式计算任务。规则用于处理
 
 ```yaml
 openTelemetry:
+  serviceName: kuiperd-service
   enableRemoteCollector: false
   remoteEndpoint: localhost:4318
   localTraceCapacity: 2048
diff --git a/etc/kuiper.yaml b/etc/kuiper.yaml
index 36581793..15749431 100644
--- a/etc/kuiper.yaml
+++ b/etc/kuiper.yaml
@@ -142,6 +142,7 @@ portable:
   initTimeout: 5s
 
 openTelemetry:
+  serviceName: kuiperd-service
   enableRemoteCollector: false
   remoteEndpoint: localhost:4318
   localTraceCapacity: 2048
diff --git a/internal/io/neuron/sink.go b/internal/io/neuron/sink.go
index a1b64e99..8b24efc3 100644
--- a/internal/io/neuron/sink.go
+++ b/internal/io/neuron/sink.go
@@ -22,10 +22,12 @@ import (
 	"github.com/lf-edge/ekuiper/contract/v2/api"
 
 	"github.com/lf-edge/ekuiper/v2/internal/conf"
+	"github.com/lf-edge/ekuiper/v2/internal/xsql"
 	"github.com/lf-edge/ekuiper/v2/pkg/cast"
 	"github.com/lf-edge/ekuiper/v2/pkg/connection"
 	"github.com/lf-edge/ekuiper/v2/pkg/errorx"
 	"github.com/lf-edge/ekuiper/v2/pkg/nng"
+	"github.com/lf-edge/ekuiper/v2/pkg/tracer"
 )
 
 type c struct {
@@ -100,6 +102,7 @@ func (s *sink) Collect(ctx api.StreamContext, data api.MessageTuple) error {
 		if err != nil {
 			return err
 		}
+		r = extractSpanContextIntoData(ctx, data, r)
 		return s.cli.Send(ctx, r)
 	} else {
 		return s.SendMapToNeuron(ctx, data)
@@ -160,7 +163,7 @@ func (s *sink) SendMapToNeuron(ctx api.StreamContext, tuple api.MessageTuple) er
 			for _, k := range keys {
 				t.TagName = k
 				t.Value = el[k]
-				err := doPublish(ctx, s.cli, t)
+				err := doPublish(ctx, s.cli, tuple, t)
 				if err != nil {
 					return err
 				}
@@ -169,7 +172,7 @@ func (s *sink) SendMapToNeuron(ctx api.StreamContext, tuple api.MessageTuple) er
 			for k, v := range el {
 				t.TagName = k
 				t.Value = v
-				err := doPublish(ctx, s.cli, t)
+				err := doPublish(ctx, s.cli, tuple, t)
 				if err != nil {
 					return err
 				}
@@ -188,7 +191,7 @@ func (s *sink) SendMapToNeuron(ctx api.StreamContext, tuple api.MessageTuple) er
 				ctx.GetLogger().Errorf("Error get the value of tag %s: %v", t.TagName, err)
 				continue
 			}
-			err := doPublish(ctx, s.cli, t)
+			err := doPublish(ctx, s.cli, tuple, t)
 			if err != nil {
 				return err
 			}
@@ -197,11 +200,12 @@ func (s *sink) SendMapToNeuron(ctx api.StreamContext, tuple api.MessageTuple) er
 	return nil
 }
 
-func doPublish(ctx api.StreamContext, cli *nng.Sock, t *neuronTemplate) error {
+func doPublish(ctx api.StreamContext, cli *nng.Sock, tuple api.MessageTuple, t *neuronTemplate) error {
 	r, err := json.Marshal(t)
 	if err != nil {
 		return fmt.Errorf("Error marshall the tag payload %v: %v", t, err)
 	}
+	r = extractSpanContextIntoData(ctx, tuple, r)
 	err = cli.Send(ctx, r)
 	if err != nil {
 		return errorx.NewIOErr(fmt.Sprintf(`Error publish the tag payload %s: %v`, t.TagName, err))
@@ -210,6 +214,21 @@ func doPublish(ctx api.StreamContext, cli *nng.Sock, t *neuronTemplate) error {
 	return nil
 }
 
+func extractSpanContextIntoData(ctx api.StreamContext, data interface{}, sendBytes []byte) []byte {
+	if tracerCtx, ok := data.(xsql.HasTracerCtx); ok && ctx.IsTraceEnabled() {
+		_, span := tracer.GetTracer().Start(tracerCtx.GetTracerCtx(), ctx.GetOpId())
+		traceID := span.SpanContext().TraceID()
+		spanID := span.SpanContext().SpanID()
+		defer span.End()
+		r := NeuronTraceHeader
+		r = append(r, traceID[:]...)
+		r = append(r, spanID[:]...)
+		r = append(r, sendBytes...)
+		return r
+	}
+	return sendBytes
+}
+
 func GetSink() api.Sink {
 	return &sink{}
 }
diff --git a/internal/io/neuron/source.go b/internal/io/neuron/source.go
index 5013e6d0..69dbafcc 100644
--- a/internal/io/neuron/source.go
+++ b/internal/io/neuron/source.go
@@ -15,13 +15,16 @@
 package neuron
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"time"
 
 	"github.com/lf-edge/ekuiper/contract/v2/api"
 	"go.nanomsg.org/mangos/v3"
+	"go.opentelemetry.io/otel/trace"
 
+	"github.com/lf-edge/ekuiper/v2/internal/topo/node/tracenode"
 	"github.com/lf-edge/ekuiper/v2/pkg/infra"
 	"github.com/lf-edge/ekuiper/v2/pkg/nng"
 	"github.com/lf-edge/ekuiper/v2/pkg/timex"
@@ -32,6 +35,15 @@ const (
 	PROTOCOL         = "pair"
 )
 
+var (
+	NeuronTraceHeader           = []byte{0x0A, 0xCE}
+	NeuronTraceIDStartIndex     = len(NeuronTraceHeader)
+	NeuronTraceIDEndIndex       = NeuronTraceIDStartIndex + 16
+	NeuronTraceSpanIDStartIndex = NeuronTraceIDEndIndex
+	NeuronTraceSpanIDEndIndex   = NeuronTraceSpanIDStartIndex + 8
+	NeuronTraceHeaderLen        = 2 + 16 + 8
+)
+
 type source struct {
 	c     *nng.SockConf
 	cli   *nng.Sock
@@ -84,7 +96,8 @@ func (s *source) Subscribe(ctx api.StreamContext, ingest api.BytesIngest, ingest
 				// no receiving deadline, will wait until the socket closed
 				if msg, err := s.cli.Recv(); err == nil {
 					ctx.GetLogger().Debugf("nng received message %s", string(msg))
-					ingest(ctx, msg, nil, timex.GetNow())
+					rawData, meta := extractTraceMeta(ctx, msg)
+					ingest(ctx, rawData, meta, timex.GetNow())
 				} else if err == mangos.ErrClosed {
 					ctx.GetLogger().Infof("neuron connection closed, retry after 1 second")
 					ingestErr(ctx, errors.New("neuron connection closed"))
@@ -112,3 +125,31 @@ func (s *source) Close(ctx api.StreamContext) error {
 func GetSource() api.Source {
 	return &source{}
 }
+
+func extractTraceMeta(ctx api.StreamContext, data []byte) ([]byte, map[string]interface{}) {
+	rawData := data
+	// extract rawData
+	if len(data) > NeuronTraceHeaderLen && bytes.Equal(data[:2], NeuronTraceHeader) {
+		rawData = data[NeuronTraceHeaderLen:]
+	}
+	if !ctx.IsTraceEnabled() {
+		return rawData, nil
+	}
+	meta := make(map[string]interface{})
+	var traced bool
+	var tracerCtx api.StreamContext
+	var span trace.Span
+	if len(data) > NeuronTraceHeaderLen && bytes.Equal(data[:2], NeuronTraceHeader) {
+		traceID := data[NeuronTraceIDStartIndex:NeuronTraceIDEndIndex]
+		spanID := data[NeuronTraceSpanIDStartIndex:NeuronTraceSpanIDEndIndex]
+		traced, tracerCtx, span = tracenode.StartTraceByID(ctx, [16]byte(traceID), [8]byte(spanID))
+	} else {
+		traced, tracerCtx, span = tracenode.StartTrace(ctx, ctx.GetOpId())
+	}
+	if traced {
+		meta["traceId"] = span.SpanContext().TraceID().String()
+		meta["traceCtx"] = tracerCtx
+		defer span.End()
+	}
+	return rawData, meta
+}
diff --git a/internal/topo/node/tracenode/trace_node.go b/internal/topo/node/tracenode/trace_node.go
index 8f5e2aaf..cf1f7580 100644
--- a/internal/topo/node/tracenode/trace_node.go
+++ b/internal/topo/node/tracenode/trace_node.go
@@ -82,6 +82,20 @@ func StartTrace(ctx api.StreamContext, opName string) (bool, api.StreamContext,
 	return true, ingestCtx, span
 }
 
+func StartTraceByID(ctx api.StreamContext, traceID [16]byte, spanID [8]byte) (bool, api.StreamContext, trace.Span) {
+	if !ctx.IsTraceEnabled() {
+		return false, nil, nil
+	}
+	spanCtx, span := tracer.GetTracer().Start(trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
+		TraceID: traceID,
+		SpanID:  spanID,
+	})), ctx.GetOpId())
+	span.SetAttributes(attribute.String(RuleKey, ctx.GetRuleId()))
+	ingestCtx := topoContext.WithContext(spanCtx)
+	ingestCtx.IsTraceEnabled()
+	return true, ingestCtx, span
+}
+
 func ToStringRow(r xsql.Row) string {
 	d := r.Clone().ToMap()
 	b, _ := json.Marshal(d)
diff --git a/pkg/tracer/tracer.go b/pkg/tracer/tracer.go
index fa95fc10..6fc2d7a9 100644
--- a/pkg/tracer/tracer.go
+++ b/pkg/tracer/tracer.go
@@ -77,6 +77,7 @@ func (g *GlobalTracerManager) SetTracer(enableRemote bool, serviceName, endpoint
 	if err != nil {
 		return err
 	}
+	g.SpanExporter = exporter
 	opts = append(opts, sdktrace.WithBatcher(exporter))
 	tp := sdktrace.NewTracerProvider(opts...)
 	otel.SetTracerProvider(tp)


Test patch:
diff --git a/internal/io/neuron/sink_test.go b/internal/io/neuron/sink_test.go
index 4a6533e2..9cc39285 100644
--- a/internal/io/neuron/sink_test.go
+++ b/internal/io/neuron/sink_test.go
@@ -20,7 +20,11 @@ import (
 
 	"github.com/lf-edge/ekuiper/contract/v2/api"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/trace"
 
+	"github.com/lf-edge/ekuiper/v2/internal/topo/context"
+	topoContext "github.com/lf-edge/ekuiper/v2/internal/topo/context"
 	"github.com/lf-edge/ekuiper/v2/internal/xsql"
 	"github.com/lf-edge/ekuiper/v2/pkg/mock"
 	mockContext "github.com/lf-edge/ekuiper/v2/pkg/mock/context"
@@ -249,3 +253,40 @@ func TestSinkProvision(t *testing.T) {
 	err = s.Close(ctx)
 	assert.NoError(t, err)
 }
+
+func TestExtractSinkTraceData(t *testing.T) {
+	data := &mockData{
+		traceID: "1234567890abcdef",
+		spanID:  "12345678",
+	}
+	rawData := []byte(`{"a":1}`)
+	ctx := mockContext.NewMockContext("1", "2")
+	ctx.EnableTracer(true)
+	got := extractSpanContextIntoData(ctx, data, rawData)
+	require.Equal(t, NeuronTraceHeader, got[:len(NeuronTraceHeader)])
+	require.Equal(t, []byte(data.traceID), got[NeuronTraceIDStartIndex:NeuronTraceIDEndIndex])
+	require.Equal(t, rawData, got[NeuronTraceHeaderLen:])
+}
+
+type mockData struct {
+	traceID string
+	spanID  string
+}
+
+func (m *mockData) GetTracerCtx() api.StreamContext {
+	var traceID [16]byte
+	for i := 0; i < 16; i++ {
+		traceID[i] = m.traceID[i]
+	}
+	var spanID [8]byte
+	for i := 0; i < 8; i++ {
+		spanID[i] = m.spanID[i]
+	}
+	ctx := trace.ContextWithSpanContext(context.Background(), trace.NewSpanContext(trace.SpanContextConfig{
+		TraceID: traceID,
+		SpanID:  spanID,
+	}))
+	return topoContext.WithContext(ctx)
+}
+
+func (m *mockData) SetTracerCtx(ctx api.StreamContext) {}
diff --git a/internal/io/neuron/source_test.go b/internal/io/neuron/source_test.go
index f6049511..1727eb79 100644
--- a/internal/io/neuron/source_test.go
+++ b/internal/io/neuron/source_test.go
@@ -15,10 +15,12 @@
 package neuron
 
 import (
+	"encoding/hex"
 	"testing"
 
 	"github.com/lf-edge/ekuiper/contract/v2/api"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	_ "go.nanomsg.org/mangos/v3/transport/ipc"
 
 	"github.com/lf-edge/ekuiper/v2/pkg/mock"
@@ -82,3 +84,40 @@ func TestProvision(t *testing.T) {
 	err = s.Close(ctx)
 	assert.NoError(t, err)
 }
+
+func TestExtractSourceTraceData(t *testing.T) {
+	rawData := `{"a":1}`
+	traceID := "0123456789abcdef"
+	spanID := "12345678"
+	traceData := []byte{}
+	traceData = append(traceData, NeuronTraceHeader...)
+	traceData = append(traceData, []byte(traceID)...)
+	traceData = append(traceData, []byte(spanID)...)
+	traceData = append(traceData, []byte(rawData)...)
+	tests := []struct {
+		data    []byte
+		traceID string
+		spanID  string
+	}{
+		{
+			data:    traceData,
+			traceID: traceID,
+			spanID:  spanID,
+		},
+		{
+			data: []byte(rawData),
+		},
+	}
+	ctx := mockContext.NewMockContext("1", "2")
+	ctx.EnableTracer(true)
+	for _, tc := range tests {
+		gotData, meta := extractTraceMeta(ctx, tc.data)
+		require.Equal(t, []byte(rawData), gotData)
+		if len(tc.traceID) > 0 {
+			require.Equal(t, meta["traceId"], hex.EncodeToString([]byte(tc.traceID)))
+		} else {
+			_, ok := meta["traceId"]
+			require.True(t, ok)
+		}
+	}
+}
diff --git a/pkg/tracer/tracer_test.go b/pkg/tracer/tracer_test.go
index 488322d0..43f88a4e 100644
--- a/pkg/tracer/tracer_test.go
+++ b/pkg/tracer/tracer_test.go
@@ -34,3 +34,11 @@ func TestSaveLoadTracerConfig(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, newConfig, traceConfig)
 }
+
+func TestSetTracer(t *testing.T) {
+	conf.IsTesting = true
+	conf.InitConf()
+	err := SetTracer(&TracerConfig{})
+	require.NoError(t, err)
+	require.NotNil(t, globalTracerManager.SpanExporter)
+}