Repository: LLNL/Caliper (language: python)

Issue:
Caliper-2.9.1/src/services/memusage/MemUsageService.cpp(25): warning #1478: function "mallinfo" (declared at line 114 of "/usr/include/malloc.h") was declared deprecated           struct mallinfo mi = mallinfo();



Discussion / hints:

Fix patch:
diff --git a/src/caliper/controllers/controllers.cpp b/src/caliper/controllers/controllers.cpp
index b9c7853ca..81b0950e6 100644
--- a/src/caliper/controllers/controllers.cpp
+++ b/src/caliper/controllers/controllers.cpp
@@ -768,6 +768,37 @@ const char* builtin_option_specs = R"json(
      "category"    : "metric",
      "config"      : { "CALI_UMPIRE_ALLOCATOR_FILTER": "{}" }
     },
+    {
+     "name"        : "mem.pages",
+     "description" : "Memory pages used via /proc/self/statm",
+     "type"        : "bool",
+     "category"    : "metric",
+     "services"    : [ "memstat" ],
+     "query"  :
+     [
+       { "level"   : "local",
+         "let"     :
+           [ "mem.vmsize = first(max#memstat.vmsize,memstat.vmsize)",
+             "mem.vmrss = first(max#memstat.vmrss,memstat.vmrss)",
+             "mem.data = first(max#memstat.data,memstat.data)"
+           ],
+         "select"  :
+           [
+            { "expr": "max(mem.vmsize)", "as": "VmSize", "unit": "pages" },
+            { "expr": "max(mem.vmrss)", "as": "VmRSS", "unit": "pages" },
+            { "expr": "max(mem.data)", "as": "Data", "unit": "pages" }
+           ]
+       },
+       { "level"   : "cross",
+         "select"  :
+           [
+            { "expr": "max(max#mem.vmsize)", "as": "VmSize (max)", "unit": "pages" },
+            { "expr": "max(max#mem.vmrss)", "as": "VmRSS (max)", "unit": "pages" },
+            { "expr": "max(max#mem.data)", "as": "Data (max)", "unit": "pages" }
+           ]
+       }
+     ]
+    },
     {
      "name"        : "mem.highwatermark",
      "description" : "Report memory high-water mark",
diff --git a/src/services/memusage/CMakeLists.txt b/src/services/memusage/CMakeLists.txt
index 6cd178cf8..d7051bb91 100644
--- a/src/services/memusage/CMakeLists.txt
+++ b/src/services/memusage/CMakeLists.txt
@@ -1,5 +1,5 @@
 set(CALIPER_MEMUSAGE_SOURCES
-  MemUsageService.cpp)
+  MemStatService.cpp)
 
 add_service_sources(${CALIPER_MEMUSAGE_SOURCES})
-add_caliper_service("memusage")
+add_caliper_service("memstat")
diff --git a/src/services/memusage/MemStatService.cpp b/src/services/memusage/MemStatService.cpp
new file mode 100644
index 000000000..024788306
--- /dev/null
+++ b/src/services/memusage/MemStatService.cpp
@@ -0,0 +1,132 @@
+// Copyright (c) 2015-2022, Lawrence Livermore National Security, LLC.
+// See top-level LICENSE file for details.
+
+#include "caliper/CaliperService.h"
+
+#include "caliper/Caliper.h"
+#include "caliper/SnapshotRecord.h"
+
+#include "caliper/common/Attribute.h"
+#include "caliper/common/Log.h"
+
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+using namespace cali;
+
+namespace
+{
+
+inline std::array<uint64_t, 6> parse_statm(const char* buf, ssize_t max)
+{
+    std::array<uint64_t, 6> numbers = { 0, 0, 0, 0, 0, 0 };
+    int nn = 0;
+
+    for (ssize_t n = 0; n < max && nn < 6; ++n) {
+        char c = buf[n];
+        if (c >= '0' && c <= '9')
+            numbers[nn] = numbers[nn] * 10 + (static_cast<unsigned>(c) - static_cast<unsigned>('0'));
+        else if (c == ' ')
+            ++nn;
+    }
+
+    return numbers;
+}
+
+class MemstatService
+{
+    Attribute m_vmsize_attr;
+    Attribute m_vmrss_attr;
+    Attribute m_vmdata_attr;
+
+    int       m_fd;
+
+    unsigned  m_failed;
+
+    void snapshot_cb(Caliper*, SnapshotBuilder& rec) {
+        char buf[80];
+        ssize_t ret = pread(m_fd, buf, sizeof(buf), 0);
+
+        if (ret < 0) {
+            ++m_failed;
+            return;
+        }
+
+        auto val = parse_statm(buf, ret);
+
+        rec.append(m_vmsize_attr, cali_make_variant_from_uint(val[0]));
+        rec.append(m_vmrss_attr,  cali_make_variant_from_uint(val[1]));
+        rec.append(m_vmdata_attr, cali_make_variant_from_uint(val[5]));
+    }
+
+    void finish_cb(Caliper*, Channel* channel) {
+        if (m_failed > 0)
+            Log(0).stream() << channel->name()
+                            << ": memstat: failed to read /proc/self/statm "
+                            << m_failed << " times\n";
+    }
+
+    MemstatService(Caliper* c, int fd)
+        : m_fd     { fd },
+          m_failed { 0  }
+    {
+        m_vmsize_attr =
+            c->create_attribute("memstat.vmsize", CALI_TYPE_UINT,
+                CALI_ATTR_SCOPE_PROCESS |
+                CALI_ATTR_ASVALUE       |
+                CALI_ATTR_AGGREGATABLE);
+        m_vmrss_attr =
+            c->create_attribute("memstat.vmrss", CALI_TYPE_UINT,
+                CALI_ATTR_SCOPE_PROCESS |
+                CALI_ATTR_ASVALUE       |
+                CALI_ATTR_AGGREGATABLE);
+        m_vmdata_attr =
+            c->create_attribute("memstat.data", CALI_TYPE_UINT,
+                CALI_ATTR_SCOPE_PROCESS |
+                CALI_ATTR_ASVALUE       |
+                CALI_ATTR_AGGREGATABLE);
+    }
+
+public:
+
+    static void memstat_register(Caliper* c, Channel* channel) {
+        int fd = open("/proc/self/statm", O_RDONLY | O_NONBLOCK);
+
+        if (fd < 0) {
+            Log(0).perror(errno, "open(\"/proc/self/statm\")");
+            return;
+        }
+
+        MemstatService* instance = new MemstatService(c, fd);
+
+        channel->events().snapshot.connect(
+            [instance](Caliper* c, Channel*, SnapshotView, SnapshotBuilder& rec){
+                instance->snapshot_cb(c, rec);
+            });
+        channel->events().finish_evt.connect(
+            [instance](Caliper* c, Channel* channel){
+                instance->finish_cb(c, channel);
+                close(instance->m_fd);
+                delete instance;
+            });
+
+        Log(1).stream() << channel->name() << ": registered memstat service\n";
+    }
+};
+
+const char* memstat_spec = R"json(
+{
+    "name"        : "memstat",
+    "description" : "Record process memory info from /proc/self/statm"
+}
+)json";
+
+}
+
+namespace cali
+{
+
+CaliperService memstat_service { ::memstat_spec, ::MemstatService::memstat_register };
+
+}
\ No newline at end of file
diff --git a/src/services/memusage/MemUsageService.cpp b/src/services/memusage/MemUsageService.cpp
deleted file mode 100644
index 47ebb3845..000000000
--- a/src/services/memusage/MemUsageService.cpp
+++ /dev/null
@@ -1,67 +0,0 @@
-// Copyright (c) 2015-2022, Lawrence Livermore National Security, LLC.
-// See top-level LICENSE file for details.
-
-#include "caliper/CaliperService.h"
-
-#include "caliper/Caliper.h"
-#include "caliper/SnapshotRecord.h"
-
-#include "caliper/common/Attribute.h"
-#include "caliper/common/Log.h"
-
-#include <malloc.h>
-
-using namespace cali;
-
-namespace
-{
-
-Attribute malloc_total_bytes_attr;
-Attribute malloc_bytes_attr;
-
-void snapshot_cb(Caliper* c, Channel* chn, SnapshotView, SnapshotBuilder& rec)
-{
-    struct mallinfo mi = mallinfo();
-
-    int      total = mi.arena + mi.hblkhd;
-    Variant v_prev =
-        c->exchange(malloc_total_bytes_attr, Variant(total));
-
-    rec.append(malloc_bytes_attr, Variant(total - v_prev.to_int()));
-}
-
-void post_init_cb(Caliper* c, Channel* chn)
-{
-    // set the initial malloc value on the blackboard
-
-    struct mallinfo mi = mallinfo();
-    c->set(malloc_total_bytes_attr, Variant(mi.arena + mi.hblkhd));
-}
-
-void memusage_register(Caliper* c, Channel* chn)
-{
-    malloc_total_bytes_attr =
-        c->create_attribute("malloc.total.bytes", CALI_TYPE_UINT,
-                            CALI_ATTR_SCOPE_PROCESS |
-                            CALI_ATTR_ASVALUE       |
-                            CALI_ATTR_AGGREGATABLE);
-    malloc_bytes_attr =
-        c->create_attribute("malloc.bytes", CALI_TYPE_INT,
-                            CALI_ATTR_SCOPE_PROCESS |
-                            CALI_ATTR_ASVALUE       |
-                            CALI_ATTR_AGGREGATABLE);
-
-    chn->events().post_init_evt.connect(post_init_cb);
-    chn->events().snapshot.connect(snapshot_cb);
-
-    Log(1).stream() << chn->name() << ": Registered memusage service" << std::endl;
-}
-
-}
-
-namespace cali
-{
-
-CaliperService memusage_service = { "memusage", ::memusage_register };
-
-}
diff --git a/src/services/monitor/CMakeLists.txt b/src/services/monitor/CMakeLists.txt
index 260cdd266..e3ff42de2 100644
--- a/src/services/monitor/CMakeLists.txt
+++ b/src/services/monitor/CMakeLists.txt
@@ -1,9 +1,11 @@
 set(CALIPER_MONITOR_SOURCES
   LoopMonitor.cpp
-  RegionMonitor.cpp)
+  RegionMonitor.cpp
+  Timeseries.cpp)
 
 add_service_sources(${CALIPER_MONITOR_SOURCES})
 
 add_caliper_service("loop_monitor")
 add_caliper_service("region_monitor")
+add_caliper_service("timeseries")
 # add_caliper_service("thread_monitor")
diff --git a/src/services/monitor/Timeseries.cpp b/src/services/monitor/Timeseries.cpp
new file mode 100644
index 000000000..3915c7762
--- /dev/null
+++ b/src/services/monitor/Timeseries.cpp
@@ -0,0 +1,142 @@
+// Copyright (c) 2015-2023, Lawrence Livermore National Security, LLC.
+// See top-level LICENSE file for details.
+
+#include "caliper/CaliperService.h"
+
+#include "../Services.h"
+
+#include "caliper/Caliper.h"
+#include "caliper/ChannelController.h"
+#include "caliper/SnapshotRecord.h"
+
+#include "caliper/common/Log.h"
+#include "caliper/common/RuntimeConfig.h"
+
+#include <chrono>
+#include <vector>
+
+using namespace cali;
+
+namespace
+{
+
+inline double get_timestamp()
+{
+    return 1e-6 * std::chrono::duration_cast<std::chrono::microseconds>(
+        std::chrono::system_clock::now().time_since_epoch()).count();
+}
+
+class TimeseriesService
+{
+    Attribute m_timestamp_attr;
+    Attribute m_snapshot_attr;
+    Attribute m_duration_attr;
+
+    ChannelController m_timeprofile;
+
+    unsigned  m_snapshots;
+
+    void snapshot_cb(Caliper* c, Channel* channel, SnapshotView info, SnapshotBuilder& srec) {
+        double ts_now = get_timestamp();
+        Variant v_prev = c->exchange(m_timestamp_attr, Variant(ts_now));
+
+        std::array<Entry, 2> ts_entries = {
+            Entry(m_timestamp_attr, v_prev),
+            Entry(m_snapshot_attr, cali_make_variant_from_uint(m_snapshots))
+        };
+
+        Channel* prof_chn = m_timeprofile.channel();
+
+        c->flush(prof_chn, info, [c,channel,info,ts_entries](CaliperMetadataAccessInterface&, const std::vector<Entry>& frec){
+                std::vector<Entry> rec;
+                rec.reserve(frec.size() + info.size() + ts_entries.size());
+                rec.insert(rec.end(), frec.begin(), frec.end());
+                rec.insert(rec.end(), info.begin(), info.end());
+                rec.insert(rec.end(), ts_entries.begin(), ts_entries.end());
+
+                channel->events().process_snapshot(c, channel, SnapshotView(), SnapshotView(rec.size(), rec.data()));
+            });
+
+        c->clear(prof_chn);
+
+        srec.append(ts_entries.size(), ts_entries.data());
+        srec.append(m_duration_attr, Variant(ts_now - v_prev.to_double()));
+
+        ++m_snapshots;
+    }
+
+    void post_init_cb(Caliper* c, Channel*) {
+        c->set(m_timestamp_attr, Variant(get_timestamp()));
+        m_timeprofile.start();
+    }
+
+    void finish_cb(Caliper*, Channel* channel) {
+        Log(1).stream() << channel->name() << ": timeseries: Processed "
+            << m_snapshots << " snapshots\n";
+    }
+
+    TimeseriesService(Caliper* c, Channel* channel)
+        : m_timeprofile("timeseries.timeprofile", CALI_CHANNEL_LEAVE_INACTIVE,
+            {   { "CALI_CHANNEL_FLUSH_ON_EXIT",      "false" },
+                { "CALI_CHANNEL_CONFIG_CHECK",       "false" },
+                { "CALI_EVENT_ENABLE_SNAPSHOT_INFO", "false" },
+                { "CALI_SERVICES_ENABLE", "aggregate,event,timer" },
+                { "CALI_AGGREGATE_KEY",   "*,mpi.rank" }
+            }),
+            m_snapshots(0)
+    {
+        m_timestamp_attr =
+            c->create_attribute("timeseries.starttime", CALI_TYPE_DOUBLE,
+                                CALI_ATTR_ASVALUE |
+                                CALI_ATTR_SKIP_EVENTS);
+        m_snapshot_attr =
+            c->create_attribute("timeseries.snapshot", CALI_TYPE_UINT,
+                                CALI_ATTR_ASVALUE |
+                                CALI_ATTR_SKIP_EVENTS);
+        m_duration_attr =
+            c->create_attribute("timeseries.duration", CALI_TYPE_DOUBLE,
+                                CALI_ATTR_ASVALUE     |
+                                CALI_ATTR_SKIP_EVENTS |
+                                CALI_ATTR_AGGREGATABLE);
+    }
+
+public:
+
+    static const char* s_spec;
+
+    static void create(Caliper* c, Channel* channel) {
+        TimeseriesService* instance = new TimeseriesService(c, channel);
+
+        channel->events().snapshot.connect(
+            [instance](Caliper* c, Channel* channel, SnapshotView info, SnapshotBuilder& rec){
+                instance->snapshot_cb(c, channel, info, rec);
+            });
+        channel->events().post_init_evt.connect(
+            [instance](Caliper* c, Channel* channel){
+                instance->post_init_cb(c, channel);
+            });
+        channel->events().finish_evt.connect(
+            [instance](Caliper* c, Channel* channel){
+                instance->finish_cb(c, channel);
+                delete instance;
+            });
+
+        Log(1).stream() << channel->name() << ": Registered timeseries service\n";
+    }
+};
+
+const char* TimeseriesService::s_spec = R"json(
+{
+    "name"        : "timeseries",
+    "description" : "Run a sub-profile for time series profiling"
+}
+)json";
+
+} // namespace [anonymous]
+
+namespace cali
+{
+
+CaliperService timeseries_service { ::TimeseriesService::s_spec, ::TimeseriesService::create };
+
+}


Test patch:
diff --git a/test/ci_app_tests/test_memusageservice.py b/test/ci_app_tests/test_memusageservice.py
index 8eb75f0a2..73c7f2f5c 100644
--- a/test/ci_app_tests/test_memusageservice.py
+++ b/test/ci_app_tests/test_memusageservice.py
@@ -7,12 +7,12 @@
 class CaliperMemusageServiceTest(unittest.TestCase):
     """ Caliper MemUsage service test case """
 
-    def test_memusage(self):
+    def test_memstat(self):
         target_cmd = [ './ci_test_basic' ]
         query_cmd  = [ '../../src/tools/cali-query/cali-query', '-e' ]
 
         caliper_config = {
-            'CALI_SERVICES_ENABLE'   : 'event,memusage,trace,recorder',
+            'CALI_SERVICES_ENABLE'   : 'event,memstat,trace,recorder',
             'CALI_RECORDER_FILENAME' : 'stdout',
             'CALI_LOG_VERBOSITY'     : '0'
         }
@@ -23,29 +23,10 @@ def test_memusage(self):
         self.assertTrue(len(snapshots) > 1)
 
         self.assertTrue(calitest.has_snapshot_with_keys(
-            snapshots, { 'malloc.total.bytes',
+            snapshots, { 'memstat.vmsize',
+                         'memstat.data',
                          'phase',
                          'iteration' }))
 
-    def test_memusage_agg(self):
-        target_cmd = [ './ci_test_basic' ]
-        query_cmd  = [ '../../src/tools/cali-query/cali-query', '-e' ]
-
-        caliper_config = {
-            'CALI_SERVICES_ENABLE'   : 'event,memusage,aggregate,recorder',
-            'CALI_RECORDER_FILENAME' : 'stdout',
-            'CALI_LOG_VERBOSITY'     : '0'
-        }
-
-        query_output = calitest.run_test_with_query(target_cmd, query_cmd, caliper_config)
-        snapshots = calitest.get_snapshots_from_text(query_output)
-
-        self.assertTrue(len(snapshots) > 1)
-
-        self.assertTrue(calitest.has_snapshot_with_keys(
-            snapshots, { 'max#malloc.total.bytes',
-                         'min#malloc.total.bytes',
-                         'phase' }))
-
 if __name__ == "__main__":
     unittest.main()