Repository: mars-project/mars

Issue:
[BUG] Error when executing a task with prometheus metric
<!--
Thank you for your contribution!

Please review https://github.com/mars-project/mars/blob/master/CONTRIBUTING.rst before opening an issue.
-->

**Describe the bug**
There is an error when executing a task with prometheus metric as follow:

```python
In [1]: import numpy as np
   ...:
   ...: import mars
   ...: import mars.dataframe as md
   ...:
   ...: session = mars.new_session(n_worker=1, n_cpu=2, web=True, config={"metrics.backend": "prometheus"})
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-1-5090e9d921e8> in <module>
      4 import mars.dataframe as md
      5
----> 6 session = mars.new_session(n_worker=1, n_cpu=2, web=True, config={"metrics.backend": "prometheus"})

~/Projects/git/git-mars/mars/deploy/oscar/session.py in new_session(address, session_id, backend, default, new, **kwargs)
   2038
   2039     session = SyncSession.init(
-> 2040         address, session_id=session_id, backend=backend, new=new, **kwargs
   2041     )
   2042     if default:

~/Projects/git/git-mars/mars/deploy/oscar/session.py in init(cls, address, session_id, backend, new, **kwargs)
   1634         coro = _IsolatedSession.init(address, session_id, backend, new=new, **kwargs)
   1635         fut = asyncio.run_coroutine_threadsafe(coro, isolation.loop)
-> 1636         isolated_session = fut.result()
   1637         return SyncSession(address, session_id, isolated_session, isolation)
   1638

~/.pyenv/versions/3.7.9/lib/python3.7/concurrent/futures/_base.py in result(self, timeout)
    433                 raise CancelledError()
    434             elif self._state == FINISHED:
--> 435                 return self.__get_result()
    436             else:
    437                 raise TimeoutError()

~/.pyenv/versions/3.7.9/lib/python3.7/concurrent/futures/_base.py in __get_result(self)
    382     def __get_result(self):
    383         if self._exception:
--> 384             raise self._exception
    385         else:
    386             return self._result

~/Projects/git/git-mars/mars/deploy/oscar/session.py in init(cls, address, session_id, backend, new, timeout, **kwargs)
    846             return (
    847                 await new_cluster_in_isolation(
--> 848                     address, timeout=timeout, backend=backend, **kwargs
    849                 )
    850             ).session

~/Projects/git/git-mars/mars/deploy/oscar/local.py in new_cluster_in_isolation(address, n_worker, n_cpu, mem_bytes, cuda_devices, subprocess_start_method, backend, config, web,
 timeout, n_supervisor_process)
     89         n_supervisor_process,
     90     )
---> 91     await cluster.start()
     92     return await LocalClient.create(cluster, timeout)
     93

~/Projects/git/git-mars/mars/deploy/oscar/local.py in start(self)
    217         await self._start_worker_pools()
    218         # start service
--> 219         await self._start_service()
    220
    221         if self._web:

~/Projects/git/git-mars/mars/deploy/oscar/local.py in _start_service(self)
    272                 self.supervisor_address,
    273                 band_to_resource,
--> 274                 config=self._config,
    275             )
    276

~/Projects/git/git-mars/mars/deploy/oscar/service.py in start_worker(address, lookup_address, band_to_resource, modules, config, mark_ready)
     82         config["modules"] = modules
     83     await start_services(
---> 84         NodeRole.WORKER, config, address=address, mark_ready=mark_ready
     85     )
     86     logger.debug("Mars worker started at %s", address)

~/Projects/git/git-mars/mars/services/core.py in start_services(node_role, config, address, mark_ready)
    172     for entries in svc_entries_list:
    173         instances = [svc_entry.get_instance(address, config) for svc_entry in entries]
--> 174         await asyncio.gather(*[inst.start() for inst in instances])
    175
    176     if mark_ready and "cluster" in service_names:

~/Projects/git/git-mars/mars/services/scheduling/worker/service.py in start(self)
     78             data_prepare_timeout=data_prepare_timeout,
     79             uid=SubtaskExecutionActor.default_uid(),
---> 80             address=address,
     81         )
     82

~/Projects/git/git-mars/mars/oscar/api.py in create_actor(actor_cls, uid, address, *args, **kwargs)
     25 async def create_actor(actor_cls, *args, uid=None, address=None, **kwargs) -> ActorRef:
     26     ctx = get_context()
---> 27     return await ctx.create_actor(actor_cls, *args, uid=uid, address=address, **kwargs)
     28
     29

~/Projects/git/git-mars/mars/oscar/backends/context.py in create_actor(self, actor_cls, uid, address, *args, **kwargs)
    110         future = await self._call(address, create_actor_message, wait=False)
    111         result = await self._wait(future, address, create_actor_message)
--> 112         return self._process_result_message(result)
    113
    114     async def has_actor(self, actor_ref: ActorRef) -> bool:

~/Projects/git/git-mars/mars/oscar/backends/context.py in _process_result_message(message)
     74             return message.result
     75         else:
---> 76             raise message.as_instanceof_cause()
     77
     78     async def _wait(self, future: asyncio.Future, address: str, message: _MessageBase):

~/Projects/git/git-mars/mars/oscar/backends/pool.py in create_actor(self, message)
    519                 )
    520
--> 521             actor = message.actor_cls(*message.args, **message.kwargs)
    522             actor.uid = actor_id
    523             actor.address = address = self.external_address

~/Projects/git/git-mars/mars/services/scheduling/worker/execution.py in __init__(self, subtask_max_retries, enable_kill_slot, data_prepare_timeout)
    158             "mars.band.submitted_subtask_count",
    159             "The count of submitted subtasks to the current band.",
--> 160             ("band",),
    161         )
    162         self._finished_subtask_count = Metrics.counter(

~/Projects/git/git-mars/mars/metrics/api.py in counter(name, description, tag_keys)
     82     @staticmethod
     83     def counter(name, description: str = "", tag_keys: Optional[Tuple[str]] = None):
---> 84         return _backends_cls[_metric_backend].Counter(name, description, tag_keys)
     85
     86     @staticmethod

~/Projects/git/git-mars/mars/metrics/backends/metric.py in __init__(self, name, description, tag_keys)
     82         self, name: str, description: str = "", tag_keys: Optional[Tuple[str]] = None
     83     ):
---> 84         super().__init__(name, description, tag_keys)
     85         self._count = 0
     86

~/Projects/git/git-mars/mars/metrics/backends/metric.py in __init__(self, name, description, tag_keys)
     39         self._description = description
     40         self._tag_keys = tag_keys or tuple()
---> 41         self._init()
     42
     43     @property

~/Projects/git/git-mars/mars/metrics/backends/prometheus/prometheus_metric.py in _init(self)
     30     def _init(self):
     31         self._metric = (
---> 32             pc.Gauge(self._name, self._description, self._tag_keys) if pc else None
     33         )
     34

~/.pyenv/versions/3.7.9/envs/gmars379/lib/python3.7/site-packages/prometheus_client/metrics.py in __init__(self, name, documentation, labelnames, namespace, subsystem, unit, re
gistry, _labelvalues, multiprocess_mode)
    342             unit=unit,
    343             registry=registry,
--> 344             _labelvalues=_labelvalues,
    345         )
    346         self._kwargs['multiprocess_mode'] = self._multiprocess_mode

~/.pyenv/versions/3.7.9/envs/gmars379/lib/python3.7/site-packages/prometheus_client/metrics.py in __init__(self, name, documentation, labelnames, namespace, subsystem, unit, re
gistry, _labelvalues)
    106
    107         if not METRIC_NAME_RE.match(self._name):
--> 108             raise ValueError('Invalid metric name: ' + self._name)
    109
    110         if self._is_parent():

ValueError: [address=127.0.0.1:50365, pid=8489] Invalid metric name: mars.band.submitted_subtask_count
```

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version: 3.7.9
2. The version of Mars you use: master
3. prometheus-client version: 0.11.0





Discussion:

Resolving patch:
diff --git a/mars/metrics/__init__.py b/mars/metrics/__init__.py
--- a/mars/metrics/__init__.py
+++ b/mars/metrics/__init__.py
@@ -13,5 +13,5 @@
 # limitations under the License.
 
 from .api import Metrics
-from .api import init_metrics
+from .api import init_metrics, shutdown_metrics
 from .api import record_time_cost_percentile, Percentile
diff --git a/mars/metrics/api.py b/mars/metrics/api.py
--- a/mars/metrics/api.py
+++ b/mars/metrics/api.py
@@ -26,6 +26,7 @@
 
 logger = logging.getLogger(__name__)
 
+_init = False
 _metric_backend = "console"
 _backends_cls = {
     "console": console_metric,
@@ -35,6 +36,10 @@
 
 
 def init_metrics(backend="console", config: Dict[str, Any] = None):
+    global _init
+    if _init is True:
+        return
+
     backend = backend or "console"
     if backend not in _backends_cls:
         raise NotImplementedError(f"Do not support metric backend {backend}")
@@ -43,17 +48,29 @@ def init_metrics(backend="console", config: Dict[str, Any] = None):
     if _metric_backend == "prometheus":
         try:
             from prometheus_client import start_http_server
+            from ..utils import get_next_port
 
             port = config.get("port", 0) if config else 0
+            port = port or get_next_port()
             start_http_server(port)
-            logger.info("Finished startup prometheus http server and port is %d", port)
+            logger.warning(
+                "Finished startup prometheus http server and port is %d", port
+            )
         except ImportError:
             logger.warning(
                 "Failed to start prometheus http server because there is no prometheus_client"
             )
+    _init = True
     logger.info("Finished initialize the metrics with backend %s", _metric_backend)
 
 
+def shutdown_metrics():
+    global _metric_backend
+    _metric_backend = "console"
+    global _init
+    _init = False
+
+
 class Metrics:
     """
     A factory to generate different types of metrics.
diff --git a/mars/metrics/backends/prometheus/prometheus_metric.py b/mars/metrics/backends/prometheus/prometheus_metric.py
--- a/mars/metrics/backends/prometheus/prometheus_metric.py
+++ b/mars/metrics/backends/prometheus/prometheus_metric.py
@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
+import socket
+
 from typing import Optional, Dict
 
 from ....utils import lazy_import
@@ -28,16 +31,28 @@
 
 class PrometheusMetricMixin(AbstractMetric):
     def _init(self):
-        self._metric = (
-            pc.Gauge(self._name, self._description, self._tag_keys) if pc else None
+        # Prometheus metric name must match the regex `[a-zA-Z_:][a-zA-Z0-9_:]*`
+        # `.` is a common character in metrics, so here replace it with `:`
+        self._name = self._name.replace(".", ":")
+        self._tag_keys = self._tag_keys + (
+            "host",
+            "pid",
         )
+        self._tags = {"host": socket.gethostname(), "pid": os.getpid()}
+        try:
+            self._metric = (
+                pc.Gauge(self._name, self._description, self._tag_keys) if pc else None
+            )
+        except ValueError:  # pragma: no cover
+            self._metric = None
 
     def _record(self, value=1, tags: Optional[Dict[str, str]] = None):
         if self._metric:
-            if tags:
-                self._metric.labels(**tags).set(value)
+            if tags is not None:
+                tags.update(self._tags)
             else:
-                self._metric.set(value)
+                tags = self._tags
+            self._metric.labels(**tags).set(value)
 
 
 class Counter(PrometheusMetricMixin, AbstractCounter):


Test patch:
diff --git a/mars/metrics/backends/prometheus/tests/test_prometheus_metric.py b/mars/metrics/backends/prometheus/tests/test_prometheus_metric.py
--- a/mars/metrics/backends/prometheus/tests/test_prometheus_metric.py
+++ b/mars/metrics/backends/prometheus/tests/test_prometheus_metric.py
@@ -54,7 +54,8 @@ def test_counter(start_prometheus_http_server):
     c = Counter("test_counter", "A test counter", ("service", "tenant"))
     assert c.name == "test_counter"
     assert c.description == "A test counter"
-    assert c.tag_keys == ("service", "tenant")
+    assert set(["host", "pid"]).issubset(set(c.tag_keys))
+    assert set(["service", "tenant"]).issubset(set(c.tag_keys))
     assert c.type == "counter"
     c.record(1, {"service": "mars", "tenant": "test"})
     verify_metric("test_counter", 1.0)
@@ -66,7 +67,7 @@ def test_gauge(start_prometheus_http_server):
     g = Gauge("test_gauge", "A test gauge")
     assert g.name == "test_gauge"
     assert g.description == "A test gauge"
-    assert g.tag_keys == ()
+    assert set(["host", "pid"]).issubset(set(g.tag_keys))
     assert g.type == "gauge"
     g.record(0.1)
     verify_metric("test_gauge", 0.1)
@@ -78,7 +79,7 @@ def test_meter(start_prometheus_http_server):
     m = Meter("test_meter")
     assert m.name == "test_meter"
     assert m.description == ""
-    assert m.tag_keys == ()
+    assert set(["host", "pid"]).issubset(set(m.tag_keys))
     assert m.type == "meter"
     num = 3
     while num > 0:
@@ -92,7 +93,7 @@ def test_histogram(start_prometheus_http_server):
     h = Histogram("test_histogram")
     assert h.name == "test_histogram"
     assert h.description == ""
-    assert h.tag_keys == ()
+    assert set(["host", "pid"]).issubset(set(h.tag_keys))
     assert h.type == "histogram"
     num = 3
     while num > 0:
diff --git a/mars/metrics/tests/test_metric_api.py b/mars/metrics/tests/test_metric_api.py
--- a/mars/metrics/tests/test_metric_api.py
+++ b/mars/metrics/tests/test_metric_api.py
@@ -22,6 +22,7 @@
     _percentile_builder,
     Percentile,
     record_time_cost_percentile,
+    shutdown_metrics,
 )
 
 
@@ -33,16 +34,22 @@ def init():
 def test_init_metrics():
     init_metrics()
     assert api._metric_backend == "console"
+    shutdown_metrics()
     init_metrics("console")
     assert api._metric_backend == "console"
+    shutdown_metrics()
     init_metrics(backend="console")
     assert api._metric_backend == "console"
+    shutdown_metrics()
     init_metrics("prometheus")
     assert api._metric_backend == "prometheus"
+    shutdown_metrics()
     init_metrics(backend="prometheus", config={"port": 0})
     assert api._metric_backend == "prometheus"
+    shutdown_metrics()
     init_metrics("ray")
     assert api._metric_backend == "ray"
+    shutdown_metrics()
     with pytest.raises(NotImplementedError):
         init_metrics("not_exist")