Repository: encapsulate-xyz/avail-ansible
Pull request #9: EHT-1072 Substrate Authority Monitoring
Author: login: kowshikvajrala
Created: 2025-03-27T09:27:23Z | Merged: 2025-03-28T07:17:54Z



Diff:
diff --git a/group_vars/all.yml b/group_vars/all.yml
index a694c89..824a26c 100644
--- a/group_vars/all.yml
+++ b/group_vars/all.yml
@@ -5,6 +5,7 @@ org: encapsulate
 domain_regex: ^([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$
 monitor_server_dns: monitor.encapsulate.xyz
 ansible_port: 8192
+pushgateway_port: 9091
 
 architecture_map:
   x86_64: amd64
@@ -16,6 +17,9 @@ default_node_types:
 
 tmp_dir: /tmp
 
+nvm:
+  path: /opt/nvm
+
 avail:
   validator:
     source_url: https://github.com/availproject/avail/releases/download
diff --git a/group_vars/default.yml b/group_vars/default.yml
new file mode 100644
index 0000000..71af733
--- /dev/null
+++ b/group_vars/default.yml
@@ -0,0 +1,3 @@
+---
+nvm:
+  version: 0.40.1
diff --git a/group_vars/mainnet.yml b/group_vars/mainnet.yml
index e376ef9..3a79dec 100644
--- a/group_vars/mainnet.yml
+++ b/group_vars/mainnet.yml
@@ -3,5 +3,8 @@
 avail:
   validator:
     version: v2.2.5.0
+    grandpa_authority_id: "0x9454703654730ee4b61c9c546810bc786c915227b6bdec3c4de132bf76e1e728"
+    node:
+      version: v18.20.5
   fullnode:
     version: v2.2.5.0
diff --git a/group_vars/testnet.yml b/group_vars/testnet.yml
index bd2c4e4..7e7ad8e 100644
--- a/group_vars/testnet.yml
+++ b/group_vars/testnet.yml
@@ -3,5 +3,8 @@
 avail:
   validator:
     version: v2.2.5.1
+    grandpa_authority_id: "0x641eedeb227a27cb7649d88aa2095afb653787aa55ce2b7ede1f3e216d60fff7"
+    node:
+      version: v18.20.5
   fullnode:
     version: v2.2.5.1
diff --git a/roles/common/handlers/main.yml b/roles/common/handlers/main.yml
index 134ca45..44987e7 100644
--- a/roles/common/handlers/main.yml
+++ b/roles/common/handlers/main.yml
@@ -13,8 +13,15 @@
   ansible.builtin.systemd:
     name: "{{ service_identifier }}"
     state: restarted
+    enabled: true
 
 - name: Stop service
   ansible.builtin.systemd:
     name: "{{ service_identifier }}"
     state: stopped
+
+- name: Restart timer
+  ansible.builtin.systemd:
+    name: "{{ service_identifier }}.timer"
+    state: restarted
+    enabled: true
diff --git a/roles/metrics/defaults/main.yml b/roles/metrics/defaults/main.yml
new file mode 100644
index 0000000..c32cdbf
--- /dev/null
+++ b/roles/metrics/defaults/main.yml
@@ -0,0 +1,19 @@
+---
+# maintains directories and paths
+metrics_home_dir: /opt/{{ service_identifier }}
+
+metrics_bin_dir: "{{ metrics_home_dir }}/bin"
+metrics_env_dir: "{{ metrics_home_dir }}/environment"
+metrics_log_dir: "{{ metrics_home_dir }}/log"
+
+metrics_directories:
+  - "{{ metrics_home_dir }}"
+  - "{{ metrics_bin_dir }}"
+  - "{{ metrics_env_dir }}"
+  - "{{ metrics_log_dir }}"
+
+metrics_log_file_path: "{{ metrics_log_dir }}/{{ service_identifier }}.log"
+
+metrics_script_file_path: "{{ metrics_bin_dir }}/{{ metrics_script_file_name }}"
+
+metrics_environment_general_file_path: "{{ metrics_env_dir }}/{{ metrics_environment_general_file_name }}"
diff --git a/roles/metrics/meta/main.yml b/roles/metrics/meta/main.yml
new file mode 100644
index 0000000..497f59b
--- /dev/null
+++ b/roles/metrics/meta/main.yml
@@ -0,0 +1,10 @@
+---
+dependencies:
+  - role: morgangraphics.ansible-role-nvm
+    nvm_version: "{{ nvm.version }}"
+    nodejs_version: "{{ avail.validator.node.version }}"
+    nvm_dir: "{{ nvm.path }}"
+    nvm_commands:
+      - npm install -g @polkadot/api 
+      - npm install -g prom-client 
+      - npm install -g axios
diff --git a/roles/metrics/tasks/main.yml b/roles/metrics/tasks/main.yml
new file mode 100644
index 0000000..b131d4c
--- /dev/null
+++ b/roles/metrics/tasks/main.yml
@@ -0,0 +1,44 @@
+---
+- name: Include user creation task
+  ansible.builtin.include_tasks: tasks/create_user_and_directories.yml
+  vars:
+    home: "{{ metrics_home_dir }}"
+    directories: "{{ metrics_directories }}"
+    log_file_path: "{{ metrics_log_file_path }}"
+
+- name: Copy metrics script
+  ansible.builtin.template:
+    src: "authority_metrics.js.j2"
+    dest: "{{ metrics_script_file_path }}"
+    owner: "{{ service_identifier }}"
+    group: "{{ service_identifier }}"
+    mode: "0644"
+
+- name: Copy general environment file
+  ansible.builtin.template:
+    src: general.env.j2
+    dest: "{{ metrics_environment_general_file_path }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Copy avail metrics systemd service file
+  ansible.builtin.template:
+    src: authority_metrics.service.j2
+    dest: "/etc/systemd/system/{{ service_identifier }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload service
+
+- name: Copy avail metrics timer service file
+  ansible.builtin.template:
+    src: authority_metrics.timer.j2
+    dest: "/etc/systemd/system/{{ service_identifier }}.timer"
+    owner: root
+    group: root
+    mode: "0644"
+  notify:
+    - Reload service
+    - Restart timer
diff --git a/roles/metrics/templates/authority_metrics.js.j2 b/roles/metrics/templates/authority_metrics.js.j2
new file mode 100644
index 0000000..e5bcf6c
--- /dev/null
+++ b/roles/metrics/templates/authority_metrics.js.j2
@@ -0,0 +1,96 @@
+const { ApiPromise, WsProvider } = require('@polkadot/api');
+const client = require('prom-client');
+const axios = require('axios');
+
+const TARGET_AUTHORITY = '{{ avail.validator.grandpa_authority_id }}';
+const PUSHGATEWAY_URL = 'http://{{ monitor_server_dns }}:{{ pushgateway_port }}';
+const JOB_NAME = 'chain_exporter';
+const RPC_TIMEOUT_MS = 30000;
+const PUSH_TIMEOUT_MS = 30000;
+
+function log(msg) {
+  const timestamp = new Date().toISOString();
+  console.log(`[${timestamp}] ${msg}`);
+}
+
+async function connectToApiWithTimeout(endpoint, timeoutMs) {
+  const provider = new WsProvider(endpoint);
+
+  return new Promise((resolve, reject) => {
+    const timeout = setTimeout(() => {
+      reject(new Error(`Connection to RPC endpoint timed out after ${timeoutMs / 1000} seconds.`));
+    }, timeoutMs);
+
+    provider.on('connected', async () => {
+      try {
+        const api = await ApiPromise.create({ provider });
+        clearTimeout(timeout);
+        resolve(api);
+      } catch (err) {
+        clearTimeout(timeout);
+        reject(new Error('Failed to create API instance after connecting.'));
+      }
+    });
+
+    provider.on('error', (err) => {
+      clearTimeout(timeout);
+      reject(new Error(`WebSocket connection error: ${err.message}`));
+    });
+  });
+}
+
+async function main() {
+  try {
+    log('Attempting to connect to Substrate RPC...');
+    const api = await connectToApiWithTimeout('ws://127.0.0.1:{{ avail.validator.ports.rpc_port }}', RPC_TIMEOUT_MS);
+    log('Connected to Substrate RPC.');
+
+    const grandpa = await api.query.grandpa.authorities();
+
+    let isTargetFound = false;
+
+    if (grandpa && grandpa.length > 0) {
+      grandpa.forEach(([id]) => {
+        const idHex = id.toString();
+        if (idHex.toLowerCase() === TARGET_AUTHORITY.toLowerCase()) {
+          isTargetFound = true;
+        }
+      });
+    }
+
+    const result = isTargetFound ? 1 : 0;
+
+    const registry = new client.Registry();
+    const matchMetric = new client.Gauge({
+      name: 'substrate_node_authority_role',
+      help: 'Whether the target GRANDPA authority was found (1 = yes, 0 = no)',
+      labelNames: ['authority', 'env', 'project', 'type'],
+    });
+
+    const labels = {
+      authority: TARGET_AUTHORITY,
+      type: 'validator',
+    };
+
+    matchMetric.set(labels, result);
+    registry.registerMetric(matchMetric);
+
+    log('Pushing metrics to Pushgateway...');
+    await axios.post(
+      `${PUSHGATEWAY_URL}/metrics/job/${JOB_NAME}/project/{{ project }}/env/{{ env }}`,
+      await registry.metrics(),
+      {
+        headers: { 'Content-Type': 'text/plain' },
+        timeout: PUSH_TIMEOUT_MS,
+      }
+    );
+
+    log('Metrics successfully pushed to Pushgateway.');
+    await api.disconnect();
+  } catch (err) {
+    log(`Error: ${err.message || err}`);
+    process.exit(1);
+  }
+}
+
+main();
diff --git a/roles/metrics/templates/authority_metrics.service.j2 b/roles/metrics/templates/authority_metrics.service.j2
new file mode 100644
index 0000000..9fca203
--- /dev/null
+++ b/roles/metrics/templates/authority_metrics.service.j2
@@ -0,0 +1,17 @@
+[Unit]
+Description=Check Avail Authority Status
+
+[Service]
+Type=oneshot
+User={{ service_identifier }}
+Group={{ service_identifier }}
+WorkingDirectory={{ metrics_home_dir }}
+EnvironmentFile={{ metrics_environment_general_file_path }}
+ExecStart=/bin/bash -c 'source {{ nvm.path }}/nvm.sh && node {{ metrics_script_file_path }} 2>/dev/null'
+LimitNOFILE=65536
+MemoryMax=64G
+StandardOutput=append:{{ metrics_log_file_path }}
+StandardError=inherit
+
+[Install]
+WantedBy=multi-user.target
diff --git a/roles/metrics/templates/authority_metrics.timer.j2 b/roles/metrics/templates/authority_metrics.timer.j2
new file mode 100644
index 0000000..8d2adf7
--- /dev/null
+++ b/roles/metrics/templates/authority_metrics.timer.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=Run Check Avail Authority Status every 30 minutes
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=30min
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/roles/metrics/templates/general.env.j2 b/roles/metrics/templates/general.env.j2
new file mode 100644
index 0000000..fd85822
--- /dev/null
+++ b/roles/metrics/templates/general.env.j2
@@ -0,0 +1 @@
+NODE_PATH={{ nvm.path }}/versions/node/{{ avail.validator.node.version }}/lib/node_modules
diff --git a/roles/metrics/vars/main.yml b/roles/metrics/vars/main.yml
new file mode 100644
index 0000000..c6ed9eb
--- /dev/null
+++ b/roles/metrics/vars/main.yml
@@ -0,0 +1,7 @@
+---
+# maintains file name, binary names, and anything which is not a directory or path
+
+metrics_script_file_name: authority-metrics.js
+
+# Environment file names
+metrics_environment_general_file_name: "{{ service_identifier }}.env"
diff --git a/roles/node/templates/mainnet/systemd.service.j2 b/roles/node/templates/mainnet/systemd.service.j2
index 5b59580..3e37e54 100644
--- a/roles/node/templates/mainnet/systemd.service.j2
+++ b/roles/node/templates/mainnet/systemd.service.j2
@@ -18,9 +18,9 @@ ExecStart={{ node_bin_dir }}/{{ service_identifier }} --chain mainnet \
 Restart=always
 RestartSec=4
 LimitNOFILE=65536
-MemoryLimit=64G
+MemoryMax=64G
 StandardOutput=append:{{ node_log_file_path }}
 StandardError=inherit
 
 [Install]
-WantedBy=multi-user.target
\ No newline at end of file
+WantedBy=multi-user.target
diff --git a/roles/node/templates/testnet/systemd.service.j2 b/roles/node/templates/testnet/systemd.service.j2
index aea0792..58ca7bf 100644
--- a/roles/node/templates/testnet/systemd.service.j2
+++ b/roles/node/templates/testnet/systemd.service.j2
@@ -18,9 +18,9 @@ ExecStart={{ node_bin_dir }}/{{ service_identifier }} --chain turing \
 Restart=always
 RestartSec=4
 LimitNOFILE=65536
-MemoryLimit=64G
+MemoryMax=64G
 StandardOutput=append:{{ node_log_file_path }}
 StandardError=inherit
 
 [Install]
-WantedBy=multi-user.target
\ No newline at end of file
+WantedBy=multi-user.target
diff --git a/roles/requirements.yml b/roles/requirements.yml
new file mode 100644
index 0000000..01399bc
--- /dev/null
+++ b/roles/requirements.yml
@@ -0,0 +1,5 @@
+---
+roles:
+  - name: morgangraphics.ansible-role-nvm
+    src: morgangraphics.ansible-role-nvm
+    version: v2.0.0
diff --git a/setup_metrics.yml b/setup_metrics.yml
new file mode 100644
index 0000000..56c6216
--- /dev/null
+++ b/setup_metrics.yml
@@ -0,0 +1,26 @@
+---
+- name: Set up Metrics
+  hosts: all
+  become: true
+  gather_facts: true
+  vars:
+    project: "{{ group_names[0] }}"
+    node_service_identifier: "{{ group_names[0] }}{{ '' if type in default_node_types else '-' + type }}"
+    system_architecture: "{{ architecture_map[ansible_architecture] | default(ansible_architecture) }}"
+  vars_files:
+    - group_vars/all.yml
+    - group_vars/default.yml
+    - group_vars/{{ env }}.yml
+
+  tasks:
+    - name: Set service_identifier
+      ansible.builtin.set_fact:
+        service_identifier: "{{ node_service_identifier }}-metrics"
+    
+    - name: Include common handlers
+      ansible.builtin.include_role:
+        name: common
+
+    - name: Include metrics role
+      ansible.builtin.include_role:
+        name: metrics


Reviews:
user: login: aditya-manit
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-27T12:24:38Z
user: login: kowshikvajrala
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T05:20:49Z
user: login: aditya-manit
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T05:27:41Z
user: login: aditya-manit
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T06:19:33Z
user: login: aditya-manit
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T06:34:27Z
user: login: aditya-manit
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T06:38:55Z
user: login: aditya-manit
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-28T07:17:49Z

Review comments:
diff_hunk: @@ -0,0 +1,50 @@
+---
path: roles/node/tasks/authority_metrics.yml
user: login: aditya-manit
body: please move it to another role called metrics, follow the approach similar to supra telegraph and promtail
created_at: 2025-03-27T12:20:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,62 @@
+const { ApiPromise, WsProvider } = require('@polkadot/api');
+const client = require('prom-client');
+const axios = require('axios');
+
+const TARGET_AUTHORITY = '{{ avail.validator.grandpa_authority_id }}';
+const PUSHGATEWAY_URL = 'http://{{ monitor_server_dns }}:9091';
path: roles/node/templates/authority_metrics.js.j2
user: login: aditya-manit
body: add the push gateway port in group vars please
created_at: 2025-03-27T12:21:15Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,62 @@
+const { ApiPromise, WsProvider } = require('@polkadot/api');
+const client = require('prom-client');
+const axios = require('axios');
+
+const TARGET_AUTHORITY = '{{ avail.validator.grandpa_authority_id }}';
+const PUSHGATEWAY_URL = 'http://{{ monitor_server_dns }}:9091';
+const JOB_NAME = 'chain_exporter';
+
+async function main() {
+  try {
+    const provider = new WsProvider('ws://localhost:{{ avail.validator.ports.rpc_port }}');
+    const api = await ApiPromise.create({ provider });
+
+    const grandpa = await api.query.grandpa.authorities();
+
+    let isTargetFound = false;
+
+    if (grandpa && grandpa.length > 0) {
+      grandpa.forEach(([id]) => {
+        const idHex = id.toString();
+        if (idHex.toLowerCase() === TARGET_AUTHORITY.toLowerCase()) {
+          isTargetFound = true;
+        }
+      });
+    }
+
+    const result = isTargetFound ? 1 : 0;
+
+    // Setup Prometheus Gauge metric
+    const registry = new client.Registry();
+    const matchMetric = new client.Gauge({
+      name: 'substrate_node_authority_role',
+      help: 'Whether the target GRANDPA authority was found (1 = yes, 0 = no)',
+      labelNames: ['authority', 'env', 'project'],
+    });
+
+    // Example label values
+    const labels = {
+      authority: TARGET_AUTHORITY,
+      env: '{{ env }}',
+      project: '{{ project }}',
path: roles/node/templates/authority_metrics.js.j2
user: login: aditya-manit
body: lets also add type
created_at: 2025-03-27T12:22:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,17 @@
+[Unit]
+Description=Check Avail Authority Status
+
+[Service]
+Type=oneshot
+User={{ service_identifier }}
+Group={{ service_identifier }}
+WorkingDirectory={{ node_home_dir }}
+EnvironmentFile={{ node_environment_general_file_path }}
+ExecStart=/bin/bash -c 'source {{ nvm.path }}/nvm.sh && node {{ node_metrics_script_file_path }}'
path: roles/node/templates/authority_metrics.service.j2
user: login: aditya-manit
body: There is a pre exec start, see if you can use that to source {{ nvm.path }}/nvm.sh
created_at: 2025-03-27T12:23:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,17 @@
+[Unit]
+Description=Check Avail Authority Status
+
+[Service]
+Type=oneshot
+User={{ service_identifier }}
+Group={{ service_identifier }}
+WorkingDirectory={{ node_home_dir }}
+EnvironmentFile={{ node_environment_general_file_path }}
+ExecStart=/bin/bash -c 'source {{ nvm.path }}/nvm.sh && node {{ node_metrics_script_file_path }}'
path: roles/node/templates/authority_metrics.service.j2
user: login: kowshikvajrala
body: Environment loaded from execprestart will not persist in exec start
created_at: 2025-03-28T05:20:49Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,17 @@
+[Unit]
+Description=Check Avail Authority Status
+
+[Service]
+Type=oneshot
+User={{ service_identifier }}
+Group={{ service_identifier }}
+WorkingDirectory={{ node_home_dir }}
+EnvironmentFile={{ node_environment_general_file_path }}
+ExecStart=/bin/bash -c 'source {{ nvm.path }}/nvm.sh && node {{ node_metrics_script_file_path }}'
path: roles/node/templates/authority_metrics.service.j2
user: login: aditya-manit
body: ah okay, yeah its usually used for checks
created_at: 2025-03-28T05:27:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,49 @@
+---
+- name: Include user creation task
+  ansible.builtin.include_tasks: tasks/create_user_and_directories.yml
+  vars:
+    home: "{{ metrics_home_dir }}"
+    directories: "{{ metrics_directories }}"
+    log_file_path: "{{ metrics_log_file_path }}"
+
+- name: Copy metrics script
+  ansible.builtin.template:
+    src: "authority_metrics.js.j2"
+    dest: "{{ metrics_script_file_path }}"
+    owner: "{{ service_identifier }}"
+    group: "{{ service_identifier }}"
+    mode: "0644"
+
+- name: Copy general environment file
+  ansible.builtin.template:
+    src: general.env.j2
+    dest: "{{ metrics_environment_general_file_path }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Copy avail metrics systemd service file
+  ansible.builtin.template:
+    src: authority_metrics.service.j2
+    dest: "/etc/systemd/system/{{ service_identifier }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Copy avail metrics timer service file
+  ansible.builtin.template:
+    src: authority_metrics.timer.j2
+    dest: "/etc/systemd/system/{{ service_identifier }}.timer"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Reload service
+  ansible.builtin.systemd:
+    daemon_reload: true
path: roles/metrics/tasks/main.yml
user: login: aditya-manit
body: We already have a handler for this, lets use that
created_at: 2025-03-28T06:17:34Z
author_association: CONTRIBUTOR
diff_hunk: @@ -0,0 +1,49 @@
+---
+- name: Include user creation task
+  ansible.builtin.include_tasks: tasks/create_user_and_directories.yml
+  vars:
+    home: "{{ metrics_home_dir }}"
+    directories: "{{ metrics_directories }}"
+    log_file_path: "{{ metrics_log_file_path }}"
+
+- name: Copy metrics script
+  ansible.builtin.template:
+    src: "authority_metrics.js.j2"
+    dest: "{{ metrics_script_file_path }}"
+    owner: "{{ service_identifier }}"
+    group: "{{ service_identifier }}"
+    mode: "0644"
+
+- name: Copy general environment file
+  ansible.builtin.template:
+    src: general.env.j2
+    dest: "{{ metrics_environment_general_file_path }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Copy avail metrics systemd service file
+  ansible.builtin.template:
+    src: authority_metrics.service.j2
+    dest: "/etc/systemd/system/{{ service_identifier }}.service"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Copy avail metrics timer service file
+  ansible.builtin.template:
+    src: authority_metrics.timer.j2
+    dest: "/etc/systemd/system/{{ service_identifier }}.timer"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Reload service
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Enable and start timer service
+  ansible.builtin.systemd:
+    name: "{{ service_identifier }}.timer"
+    enabled: true
+    state: restarted
path: roles/metrics/tasks/main.yml
user: login: aditya-manit
body: Lets write a new handler and use that here

```
- name: Restart timer
  ansible.builtin.systemd:
    name: "{{ service_identifier }}.timer"
    state: restarted
    enabled: true
```
created_at: 2025-03-28T06:18:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -18,3 +18,9 @@
   ansible.builtin.systemd:
     name: "{{ service_identifier }}"
     state: stopped
+
+- name: Start timer
path: roles/common/handlers/main.yml
user: login: aditya-manit
body: use restarted, so that if there are any new changes, it will pick them up
created_at: 2025-03-28T06:38:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -18,3 +18,9 @@
   ansible.builtin.systemd:
     name: "{{ service_identifier }}"
     state: stopped
+
+- name: Start timer
path: roles/common/handlers/main.yml
user: login: aditya-manit
body: also please add enable true to Restart service too, looks like we didnt update in this repo
created_at: 2025-03-28T06:38:50Z
author_association: CONTRIBUTOR

PR comments:
user: login: notion-workspace[bot]
created_at: 2025-03-27T09:27:27Z
body: <a href="https://www.notion.so/Support-services-monitoring-1b514f565a5080feab9bda752d39d243?pvs=8&n=github_linkback">Support services monitoring</a>
author_association: NONE
user: login: notion-workspace[bot]
created_at: 2025-03-27T10:51:58Z
body: <a href="https://www.notion.so/Substrate-authority-monitoring-1b514f565a5080018101d79b56ee7aaa?pvs=8&n=github_linkback">Substrate authority monitoring</a>
author_association: NONE