Repository: transitiverobotics/transitive
Pull request #82: device config refreshed on file change
Author: login: jlurgo
Created: 2025-05-07T18:07:19Z | Merged: 2025-05-15T16:37:40Z

- first approach
- changes to the config.json file gets reflected on global.config object
- global object is proxied so we can refresh device data on mqttsync reacting to config changes
- I have some limited testing abilities, I can use some help on that from
  - But as far as I could test:
    - Frontend device details reflects live changes in global.rosReleases, updating the dropboxes selections, but it needs a page reload
    - I'm failing to start capabilities, unshare.sh fails, but adding a log line there I can see that the TRCONFIG env var gets updated when I hit restart for a capability on the frontend

addresses https://github.com/transitiverobotics/issues/issues/592



Diff:
diff --git a/robot-agent/commands.js b/robot-agent/commands.js
index 37fa19a2..bc3469f7 100644
--- a/robot-agent/commands.js
+++ b/robot-agent/commands.js
@@ -58,17 +58,14 @@ const commands = {
   updateConfig: ({modifier}) => {
     log.debug('updateConfig', modifier);
     // now set it in `global.config` and write it back to disk
-    _.forEach(modifier, (value, path) => _.set(global.config, path, value));
-    log.debug('backing up old config and writing new', global.config);
+    const newConfig = clone(global.config);
+    _.forEach(modifier, (value, path) => _.set(newConfig, path, value));
+    log.debug('backing up old config and writing new', newConfig);
     try {
       fs.copyFileSync('./config.json', './config.json.bak');
     } catch (e) {}
-    fs.writeFileSync('./config.json', JSON.stringify(global.config, true, 2),
+    fs.writeFileSync('./config.json', JSON.stringify(newConfig, true, 2),
       {encoding: 'utf8'});
-
-    global.data.update(`${global.AGENT_PREFIX}/info/config`,
-      clone(global.config)
-    );
   },
 
   upgradeNodejs: () => {
diff --git a/robot-agent/config.js b/robot-agent/config.js
index 2093bf3f..888985e5 100644
--- a/robot-agent/config.js
+++ b/robot-agent/config.js
@@ -1,5 +1,7 @@
 const fs = require('fs');
 const dotenv = require('dotenv');
+const { clone } = require('@transitive-sdk/utils');
+const { getInstalledPackages, updatePackageConfigFile } = require('./utils');
 
 dotenv.config({path: './.env'});
 dotenv.config({path: './.env_user'});
@@ -7,12 +9,37 @@ dotenv.config({path: './.env_user'});
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfigFromFile = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+    const packages = getInstalledPackages();
+    // generate config.json for each package
+    packages.forEach(updatePackageConfigFile);
+    // update the config in the fleet data store
+    if (global.data) {
+      // update the config in the fleet data store
+      console.log(`Updating global config in ${global.AGENT_PREFIX}/info/config`);
+      global.data.update(`${global.AGENT_PREFIX}/info/config`, clone(global.config));
+    }
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfigFromFile();
+
+fs.watch('./', { persistence: false }, (eventType, filename) => {
+  if (filename !== 'config.json') {
+    return;
+  }
+  // check if the file exists, eventType == 'rename' triggers when the file is created or deleted
+  // but we only want to reload if the file is changed or created
+  if (eventType == 'rename' && !fs.existsSync('./config.json')) {
+    return;
+  }        
+  console.log('config.json changed, reloading...');
+  refreshGlobalConfigFromFile();
+});
 
 /** Set the `key` in the fleet config to `value` */
 const updateFleetConfig = (key, value) => fleetConfig[key] = value;
diff --git a/robot-agent/mqtt.js b/robot-agent/mqtt.js
index acd773f0..555a8d1b 100644
--- a/robot-agent/mqtt.js
+++ b/robot-agent/mqtt.js
@@ -122,7 +122,7 @@ mqttClient.on('connect', function(connackPacket) {
       global.data = data; // #hacky; need this in commands.js
       global.AGENT_PREFIX = AGENT_PREFIX;
 
-      mqttSync.publish(`${AGENT_PREFIX}/info`);
+      mqttSync.publish(`${AGENT_PREFIX}/info`, {atomic: true});
       mqttSync.publish(`${AGENT_PREFIX}/status`);
 
       // for ongoing ping checking (not to be confused with ping RPC command)
diff --git a/robot-agent/startPackage.sh b/robot-agent/startPackage.sh
index ac6c7caa..c2a76074 100755
--- a/robot-agent/startPackage.sh
+++ b/robot-agent/startPackage.sh
@@ -53,6 +53,7 @@ pid=
 
 BASE=$PWD
 STATUS_FILE="$BASE/status.json"
+CONFIG_FILE="$BASE/config.json"
 
 # trap SIGUSR1 and restart the node application
 # The `-` in front of the pid ensures that the entire process group gets killed,
@@ -104,6 +105,23 @@ do
     echo $CURRENT_MODULES_VERSION > .compiled_modules_version
   fi;
 
+  if [ -e $CONFIG_FILE ]; then
+    # Config file exists
+    if [ -s $CONFIG_FILE ]; then
+      # Config file is not empty
+      export TRCONFIG=$(cat $CONFIG_FILE)
+    else
+      # If the config file is empty
+      echo "Config file $CONFIG_FILE is empty"
+      export TRCONFIG="{}"
+    fi
+  else    
+    echo "Config file $CONFIG_FILE not found"
+     export TRCONFIG="{}"
+  fi
+  
+  echo "TRCONFIG: $TRCONFIG"
+
   cd "$BASE/node_modules/$1"
   export PASSWORD=$(cat ../../../password)
   npm start &
diff --git a/robot-agent/utils.js b/robot-agent/utils.js
index ce3b4ded..2371be71 100644
--- a/robot-agent/utils.js
+++ b/robot-agent/utils.js
@@ -47,6 +47,7 @@ const addPackage = (addedPkg) => {
   fs.copyFileSync(`${constants.TRANSITIVE_DIR}/.npmrc`, `${dir}/.npmrc`);
   fs.writeFileSync(`${dir}/package.json`,
     `{ "dependencies": {"${addedPkg}": "*"} }`);
+  updatePackageConfigFile(addedPkg);
   startPackage(addedPkg);
 };
 
@@ -66,6 +67,20 @@ const removePackage = (pkg) => {
   });
 };
 
+/** update the config.json file for the package with the current global config
+  and the package specific config.*/
+const updatePackageConfigFile = (packageName) => {
+  const config = {
+    global: global.config.global,      // global, shared config
+    package: global.config[packageName] || {} // pkg specific config
+  };
+  log.info(`Updating ${packageName} config.json file`, config);
+  // update the config.json file for this package
+  fs.writeFileSync(`${basePath}/${packageName}/config.json`,
+    JSON.stringify(config, null, 2));
+};
+
+
 /** restart the named package by sending a SIGUSR1 to its startPackage.sh process
   e.g., name = '@transitive-robotics/health-monitoring'
 */
@@ -116,11 +131,8 @@ const startPackage = (name) => {
       fs.mkdirSync(path.dirname(logFile), {recursive: true});
       const out = fs.openSync(logFile, 'a');
 
-      const config = {
-        global: global.config.global,      // global, shared config
-        package: global.config[name] || {} // pkg specific config
-      };
 
+      // package is started with passed config
       const subprocess = spawn(`${os.homedir()}/.transitive/unshare.sh`,
         [`/home/bin/startPackage.sh ${name}`],
         { stdio: ['ignore', out, out], // so it can continue without us
@@ -130,7 +142,6 @@ const startPackage = (name) => {
             process.env, // TODO: is this safe? we may *not* want capabilities to see this
             {
               TRPACKAGE: name,
-              TRCONFIG: JSON.stringify(config),
               TR_ROS_RELEASES: config?.global?.rosReleases?.join(' ')
             })
         });
@@ -337,5 +348,6 @@ module.exports = {
   rotateAllLogs,
   killAllPackages,
   ensureDesiredPackages,
-  upgradeNodejs
+  upgradeNodejs,
+  updatePackageConfigFile
 };


Reviews:
user: login: chfritz
body: Left some comments. The part that is definitely still missing is writing the capability specific config to the capability's package folder, and reloading that config in the while loop in startPackage.js. Otherwise a "normal" restart of the capability from the Portal will not pick up the changes.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-07T18:22:04Z
user: login: jlurgo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-07T19:15:27Z
user: login: jlurgo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-07T19:15:33Z
user: login: chfritz
body: This approach should work. But I just realized that this is very similar to what DataCache provides: essentially a variable with change listeners. I think that might be more concise to use here.

Let's make sure we get you a working dev setup where you can run capabilities locally tomorrow.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T20:43:22Z
user: login: chfritz
body: The `fs.watch` doesn't seem to work reliably. To test I edited the config.json file repeatedly using `vi`. It only triggers on the first edit, but not subsequent ones.

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-12T20:53:29Z
user: login: chfritz
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T00:57:52Z
user: login: jlurgo
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-14T16:34:04Z
user: login: chfritz
body: The new code looks good. But I found an issue: when you install a new capability, it doesn't receive any config until the user edits config.json the next time (which may be never).

state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-14T21:31:08Z
user: login: chfritz
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T16:32:10Z

Review comments:
diff_hunk: @@ -7,12 +7,22 @@ dotenv.config({path: './.env_user'});
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfig = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfig();
+
+fs.watchFile('./config.json', {interval: 1000}, (curr, prev) => {
path: robot-agent/config.js
user: login: chfritz
body: watchFile doesn't seem to use inotify (from the OS) but polling instead which is not very efficient.
> Using [fs.watch()](https://nodejs.org/docs/latest-v20.x/api/fs.html#fswatchfilename-options-listener) is more efficient than fs.watchFile and fs.unwatchFile. fs.watch should be used instead of fs.watchFile and fs.unwatchFile when possible.

Can we try `watch` instead?
created_at: 2025-05-07T18:18:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7,12 +7,22 @@ dotenv.config({path: './.env_user'});
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfig = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfig();
+
+fs.watchFile('./config.json', {interval: 1000}, (curr, prev) => {
+  if (curr.mtime !== prev.mtime) {
path: robot-agent/config.js
user: login: chfritz
body: This should not be necessary when using `watch`, as the eventType should already indicate whether a change occurred.
created_at: 2025-05-07T18:18:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -133,7 +133,19 @@ mqttClient.on('connect', function(connackPacket) {
           {ping, pong: Date.now()});
       });
 
+      // publish device info including config
       staticInfo();
+      const globalProxy = new Proxy(global, {
+        set(target, property, value) {
+          target[property] = value;
+          if (property == 'config') {
+            console.log('config changed, republishing staticInfo');
+            staticInfo(); // Republish staticInfo on config change
+          }
+          return true;
+        }
+      });
+      global = globalProxy;
path: robot-agent/mqtt.js
user: login: chfritz
body: Replacing global with a proxy seems a bit risky and hard to read. People usually assume what `global` is what it does and doesn't do. Can we instead just change the code where global.config is being set (in `refreshGlobalConfig`)?

created_at: 2025-05-07T18:20:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -133,7 +133,19 @@ mqttClient.on('connect', function(connackPacket) {
           {ping, pong: Date.now()});
       });
 
+      // publish device info including config
       staticInfo();
+      const globalProxy = new Proxy(global, {
+        set(target, property, value) {
+          target[property] = value;
+          if (property == 'config') {
+            console.log('config changed, republishing staticInfo');
+            staticInfo(); // Republish staticInfo on config change
+          }
+          return true;
+        }
+      });
+      global = globalProxy;
path: robot-agent/mqtt.js
user: login: jlurgo
body: addressed
created_at: 2025-05-07T19:15:27Z
author_association: COLLABORATOR
diff_hunk: @@ -7,12 +7,22 @@ dotenv.config({path: './.env_user'});
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfig = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfig();
+
+fs.watchFile('./config.json', {interval: 1000}, (curr, prev) => {
path: robot-agent/config.js
user: login: jlurgo
body: addressed

created_at: 2025-05-07T19:15:33Z
author_association: COLLABORATOR
diff_hunk: @@ -7,12 +7,24 @@ dotenv.config({path: './.env_user'});
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const configChangeHandlers = [];
+
+const refreshGlobalConfig = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+    // Notify all handlers of the config change
+    configChangeHandlers.forEach(handler => handler(global.config));
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfig();
+
+fs.watch('./config.json', {interval: 1000}, (curr, prev) => {
path: robot-agent/config.js
user: login: chfritz
body: The internal option is not used by `fs.watch`.
created_at: 2025-05-08T20:34:35Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,18 +1,32 @@
 const fs = require('fs');
 const dotenv = require('dotenv');
+const { clone } = require('@transitive-sdk/utils');
 
 dotenv.config({path: './.env'});
 dotenv.config({path: './.env_user'});
 
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfigFromFile = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+    if (global.data) {
+      // update the config in the fleet data store
+      console.log(`Updating global config in ${global.AGENT_PREFIX}/info/config`);
+      global.data.update(`${global.AGENT_PREFIX}/info/config`, clone(global.config));
+    }
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfigFromFile();
+
+fs.watch('./config.json', (curr, prev) => {
path: robot-agent/config.js
user: login: chfritz
body: `(curr, prev)` seems to still be left over from the previous use of watchFile, but it's very confusing, because that's not what the `watch` callback receives: https://nodejs.org/docs/latest-v20.x/api/fs.html#fswatchfilename-options-listener



created_at: 2025-05-12T20:26:39Z
author_association: CONTRIBUTOR
diff_hunk: @@ -104,6 +105,21 @@ do
     echo $CURRENT_MODULES_VERSION > .compiled_modules_version
   fi;
 
+  # Check if the config file exists, then update TRCONFIG with the contents of the file
+  if [ -e $CONFIG_FILE ]; then
+    # Check if the config file is empty
path: robot-agent/startPackage.sh
user: login: chfritz
body: comment doesn't match test, should say `Check if the config file is not empty` (or switch if and else case to avoid the negation).
created_at: 2025-05-12T20:28:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -7,6 +7,7 @@ const _ = require('lodash');
 
 const { toFlatObject, getLogger } = require('@transitive-sdk/utils');
 const constants = require('./constants');
+const { registerConfigChangeHandler } = require('./config');
path: robot-agent/utils.js
user: login: chfritz
body: No longer used, right?
created_at: 2025-05-12T20:32:54Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,18 +1,72 @@
 const fs = require('fs');
 const dotenv = require('dotenv');
+const { clone } = require('@transitive-sdk/utils');
+const { getInstalledPackages, updatePackageConfigFile } = require('./utils');
 
 dotenv.config({path: './.env'});
 dotenv.config({path: './.env_user'});
 
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfigFromFile = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+    const packages = getInstalledPackages();
+    // generate config.json for each package
+    packages.forEach(updatePackageConfigFile);
+    // update the config in the fleet data store
+    if (global.data) {
+      // update the config in the fleet data store
+      console.log(`Updating global config in ${global.AGENT_PREFIX}/info/config`);
+      global.data.update(`${global.AGENT_PREFIX}/info/config`, clone(global.config));
+    }
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfigFromFile();
+
+let watcher;
+
+// Wait for the file to exist before using it
+// This is useful for VI or other editors that creates a new file
+// on save, which can cause the watcher to trigger before the file is ready
+// to be read.
+const waitForFile = (filePath, callback) => {
path: robot-agent/config.js
user: login: chfritz
body: Good point about needing to wait for the file to exists. But we should poll the filesystem twice a second for potentially every if the user never creates the file. Let's just create it as an empty JSON object, `{}`, if it doesn't exist yet.
created_at: 2025-05-14T00:50:51Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,18 +1,72 @@
 const fs = require('fs');
 const dotenv = require('dotenv');
+const { clone } = require('@transitive-sdk/utils');
+const { getInstalledPackages, updatePackageConfigFile } = require('./utils');
 
 dotenv.config({path: './.env'});
 dotenv.config({path: './.env_user'});
 
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfigFromFile = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+    const packages = getInstalledPackages();
+    // generate config.json for each package
+    packages.forEach(updatePackageConfigFile);
+    // update the config in the fleet data store
+    if (global.data) {
+      // update the config in the fleet data store
+      console.log(`Updating global config in ${global.AGENT_PREFIX}/info/config`);
+      global.data.update(`${global.AGENT_PREFIX}/info/config`, clone(global.config));
+    }
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfigFromFile();
+
+let watcher;
+
+// Wait for the file to exist before using it
+// This is useful for VI or other editors that creates a new file
+// on save, which can cause the watcher to trigger before the file is ready
+// to be read.
+const waitForFile = (filePath, callback) => {
path: robot-agent/config.js
user: login: chfritz
body: Alternatively we could watch the directory itself.
created_at: 2025-05-14T00:53:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,18 +1,72 @@
 const fs = require('fs');
 const dotenv = require('dotenv');
+const { clone } = require('@transitive-sdk/utils');
+const { getInstalledPackages, updatePackageConfigFile } = require('./utils');
 
 dotenv.config({path: './.env'});
 dotenv.config({path: './.env_user'});
 
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfigFromFile = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+    const packages = getInstalledPackages();
+    // generate config.json for each package
+    packages.forEach(updatePackageConfigFile);
+    // update the config in the fleet data store
+    if (global.data) {
+      // update the config in the fleet data store
+      console.log(`Updating global config in ${global.AGENT_PREFIX}/info/config`);
+      global.data.update(`${global.AGENT_PREFIX}/info/config`, clone(global.config));
+    }
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfigFromFile();
+
+let watcher;
+
+// Wait for the file to exist before using it
+// This is useful for VI or other editors that creates a new file
+// on save, which can cause the watcher to trigger before the file is ready
+// to be read.
+const waitForFile = (filePath, callback) => {
+  const interval = setInterval(() => {
+    if (fs.existsSync(filePath)) {
+      clearInterval(interval);
+      callback();
+    }
+  }, 500);
+};
+
+const startWatchingConfig = () => {
+  if (watcher) {
+    watcher.close();
+    console.log('Previous watcher closed.');
+  }
+
+  waitForFile('./config.json', () => {
+    try {
+      watcher = fs.watch('./config.json', { persistence: false }, (eventType, filename) => {
+        console.log('config.json changed');
+        waitForFile('./config.json', () => {
+          refreshGlobalConfigFromFile();
+        });
+        startWatchingConfig(); // Restart the watcher
+      });
+
+      console.log('Started watching config.json');
+    } catch (err) {
+      console.error('Error setting up watcher:', err);
+    }
+  });
path: robot-agent/config.js
user: login: chfritz
body: I just tested the following:
```js
fs.watch('/tmp', console.log)
```
and then ran `touch /tmp/foo` and `vi /tmp/foo`. Each time I made such a change I got (among other things)
```
change foo
```
I think this gives us a much better way to watch the config.json for changes: just watch the folder. This will also solve the wait-for-file issue.

created_at: 2025-05-14T00:57:19Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,18 +1,72 @@
 const fs = require('fs');
 const dotenv = require('dotenv');
+const { clone } = require('@transitive-sdk/utils');
+const { getInstalledPackages, updatePackageConfigFile } = require('./utils');
 
 dotenv.config({path: './.env'});
 dotenv.config({path: './.env_user'});
 
 global.config = {};
 const fleetConfig = {};
 
-try {
-  global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
-  console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
-} catch (e) {
-  console.log('No config.json file found or not valid JSON, proceeding without.');
-}
+const refreshGlobalConfigFromFile = () => {
+  try {
+    global.config = JSON.parse(fs.readFileSync('./config.json', {encoding: 'utf8'}));
+    console.log(`Using config:\n${JSON.stringify(global.config, true, 2)}`);
+    const packages = getInstalledPackages();
+    // generate config.json for each package
+    packages.forEach(updatePackageConfigFile);
+    // update the config in the fleet data store
+    if (global.data) {
+      // update the config in the fleet data store
+      console.log(`Updating global config in ${global.AGENT_PREFIX}/info/config`);
+      global.data.update(`${global.AGENT_PREFIX}/info/config`, clone(global.config));
+    }
+  } catch (e) {
+    console.log('No config.json file found or not valid JSON, proceeding without.');
+  }
+};
+refreshGlobalConfigFromFile();
+
+let watcher;
+
+// Wait for the file to exist before using it
+// This is useful for VI or other editors that creates a new file
+// on save, which can cause the watcher to trigger before the file is ready
+// to be read.
+const waitForFile = (filePath, callback) => {
+  const interval = setInterval(() => {
+    if (fs.existsSync(filePath)) {
+      clearInterval(interval);
+      callback();
+    }
+  }, 500);
+};
+
+const startWatchingConfig = () => {
+  if (watcher) {
+    watcher.close();
+    console.log('Previous watcher closed.');
+  }
+
+  waitForFile('./config.json', () => {
+    try {
+      watcher = fs.watch('./config.json', { persistence: false }, (eventType, filename) => {
+        console.log('config.json changed');
+        waitForFile('./config.json', () => {
+          refreshGlobalConfigFromFile();
+        });
+        startWatchingConfig(); // Restart the watcher
+      });
+
+      console.log('Started watching config.json');
+    } catch (err) {
+      console.error('Error setting up watcher:', err);
+    }
+  });
path: robot-agent/config.js
user: login: jlurgo
body: Thanks Christian! That's clever, code looks much better now, also supports file creation.
Only issue I see it that when editing with vi the watch handler is triggered several times, I filtered a lot of the calls, but we still have 3 or 4 valid calls for every change, would that be acceptable?
I quickly tried to throttle or debounce the call but it didn't work, should I keep trying or is it ok like this?
created_at: 2025-05-14T16:34:04Z
author_association: COLLABORATOR

PR comments:
user: login: jlurgo
created_at: 2025-05-08T18:40:17Z
body: @chfritz I made some progress, only thing I couldn't test is if TRCONFIG env var is being correctly read and set from the per-package config.json on every restart
author_association: COLLABORATOR
user: login: jlurgo
created_at: 2025-05-13T16:12:53Z
body: > The `fs.watch` doesn't seem to work reliably. To test I edited the config.json file repeatedly using `vi`. It only triggers on the first edit, but not subsequent ones.

I was testing edits with VS code and it worked repeatedly, but now I tested with vim and it doesn't

ChatGPT says:

> The issue you're experiencing is likely due to how vim handles file writes. By default, vim creates a new file, writes the content to it, and then replaces the original file. This process can break `fs.watch` because the file descriptor being watched is no longer valid after the replacement.
> 
> To fix this, you can use the [fs.watchFile](vscode-file://vscode-app/usr/share/code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.html) method instead of [fs.watch](vscode-file://vscode-app/usr/share/code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.html). The [fs.watchFile](vscode-file://vscode-app/usr/share/code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.html) method uses polling to detect changes, which is more reliable in scenarios like this.

I switched to watchFile and works great
author_association: COLLABORATOR
user: login: jlurgo
created_at: 2025-05-13T18:10:20Z
body: I implemented rewatching the file after every change, which needed some extra work for VI
author_association: COLLABORATOR
user: login: jlurgo
created_at: 2025-05-13T18:42:05Z
body: And also to generate the per package config.json files immediately after any change on the device's config.json
author_association: COLLABORATOR
user: login: jlurgo
created_at: 2025-05-15T16:08:38Z
body: * config also gets generated on agent restart, that should happen more frequently than config.json changes

I just added a config.json refresh on package install, would that be enough?
author_association: COLLABORATOR
user: login: chfritz
created_at: 2025-05-15T16:30:57Z
body: Yes, that seems to work, just tested it. Thanks.
author_association: CONTRIBUTOR