Repository: zalando/patroni

Issue:
Set different directory for configuration files
Hello,

On debian, the configuration files (`postgresql.conf`, `pg_hba.conf`, etc) are not stored in the data directory. It would be great to be able to configure the location of this separate directory. Patroni could override existing configuration files where they are used to be.

It's quite confusing to see configuration files created in the data directory as a debian sysadmin. We are tempted to ignore them but they are effectively used by Patroni. On the other hand, Patroni ignores the configuration files in `/etc`. Using only one location should avoid confusion.

I will understand if you are not agree with this. I was a CentOS administrator few years ago, configuration files where stored in the data directory and everything was ok. Now that I have to manage plenty of debian systems with PostgreSQL running on it, things have changed. This pull request is reflecting this feeling.

The default is to store configuration files in the data directory. This setting is targeting custom installations like debian and any others moving configuration files out of the data directory.


Discussion:

Resolving patch:
diff --git a/patroni/postgresql.py b/patroni/postgresql.py
--- a/patroni/postgresql.py
+++ b/patroni/postgresql.py
@@ -99,6 +99,7 @@ def __init__(self, config):
         self._bin_dir = config.get('bin_dir') or ''
         self._database = config.get('database', 'postgres')
         self._data_dir = config['data_dir']
+        self._config_dir = config.get('config_dir') or self._data_dir
         self._pending_restart = False
         self.__thread_ident = current_thread().ident
 
@@ -120,9 +121,9 @@ def __init__(self, config):
         self.__cb_called = False
         self.__cb_pending = None
         config_base_name = config.get('config_base_name', 'postgresql')
-        self._postgresql_conf = os.path.join(self._data_dir, config_base_name + '.conf')
+        self._postgresql_conf = os.path.join(self._config_dir, config_base_name + '.conf')
         self._postgresql_base_conf_name = config_base_name + '.base.conf'
-        self._postgresql_base_conf = os.path.join(self._data_dir, self._postgresql_base_conf_name)
+        self._postgresql_base_conf = os.path.join(self._config_dir, self._postgresql_base_conf_name)
         self._recovery_conf = os.path.join(self._data_dir, 'recovery.conf')
         self._postmaster_pid = os.path.join(self._data_dir, 'postmaster.pid')
         self._trigger_file = config.get('recovery_conf', {}).get('trigger_file') or 'promote'
@@ -162,11 +163,11 @@ def __init__(self, config):
 
     @property
     def _configuration_to_save(self):
-        configuration = [self._postgresql_conf]
+        configuration = [os.path.basename(self._postgresql_conf)]
         if 'custom_conf' not in self.config:
-            configuration.append(self._postgresql_base_conf)
+            configuration.append(os.path.basename(self._postgresql_base_conf))
         if not self.config['parameters'].get('hba_file'):
-            configuration.append(os.path.join(self._data_dir, 'pg_hba.conf'))
+            configuration.append('pg_hba.conf')
         return configuration
 
     @property
@@ -826,8 +827,9 @@ def start(self, timeout=None, block_callbacks=False, task=None):
                 return False
 
             start_initiated = time.time()
-            proc = call_self(['pg_ctl_start', self._pgcommand('postgres'), '-D', self._data_dir] + options,
-                             close_fds=True, preexec_fn=os.setsid, stdout=subprocess.PIPE,
+            proc = call_self(['pg_ctl_start', self._pgcommand('postgres'), '-D', self._data_dir,
+                             '--config-file={}'.format(self._postgresql_conf)] + options, close_fds=True,
+                             preexec_fn=os.setsid, stdout=subprocess.PIPE,
                              env={p: os.environ[p] for p in ('PATH', 'LC_ALL', 'LANG') if p in os.environ})
             pid = int(proc.stdout.readline().strip())
             proc.wait()
@@ -1060,6 +1062,9 @@ def _write_postgresql_conf(self):
         with open(self._postgresql_conf, 'w') as f:
             f.write('# Do not edit this file manually!\n# It will be overwritten by Patroni!\n')
             f.write("include '{0}'\n\n".format(self.config.get('custom_conf') or self._postgresql_base_conf_name))
+            f.write("data_directory = '{}'\n".format(self._data_dir))
+            f.write("hba_file = '{}'\n".format(os.path.join(self._config_dir, 'pg_hba.conf')))
+            f.write("ident_file = '{}'\n".format(os.path.join(self._config_dir, 'pg_ident.conf')))
             for name, value in sorted(self._server_parameters.items()):
                 f.write("{0} = '{1}'\n".format(name, value))
 
@@ -1070,7 +1075,7 @@ def is_healthy(self):
         return True
 
     def write_pg_hba(self, config):
-        with open(os.path.join(self._data_dir, 'pg_hba.conf'), 'a') as f:
+        with open(os.path.join(self._config_dir, 'pg_hba.conf'), 'a') as f:
             f.write('\n{}\n'.format('\n'.join(config)))
 
     def _replace_pg_hba(self):
@@ -1081,7 +1086,7 @@ def _replace_pg_hba(self):
         :returns: True if pg_hba.conf was rewritten.
         """
         if not self.config['parameters'].get('hba_file') and self.config.get('pg_hba'):
-            with open(os.path.join(self._data_dir, 'pg_hba.conf'), 'w') as f:
+            with open(os.path.join(self._config_dir, 'pg_hba.conf'), 'w') as f:
                 f.write('# Do not edit this file manually!\n# It will be overwritten by Patroni!\n')
                 for line in self.config['pg_hba']:
                     f.write('{0}\n'.format(line))
@@ -1366,8 +1371,10 @@ def save_configuration_files(self):
         """
         try:
             for f in self._configuration_to_save:
-                if os.path.isfile(f):
-                    shutil.copy(f, f + '.backup')
+                config_file = os.path.join(self._config_dir, f)
+                backup_file = os.path.join(self._data_dir, f + '.backup')
+                if os.path.isfile(config_file):
+                    shutil.copy(config_file, backup_file)
         except IOError:
             logger.exception('unable to create backup copies of configuration files')
 
@@ -1375,8 +1382,10 @@ def restore_configuration_files(self):
         """ restore a previously saved postgresql.conf """
         try:
             for f in self._configuration_to_save:
-                if not os.path.isfile(f) and os.path.isfile(f + '.backup'):
-                    shutil.copy(f + '.backup', f)
+                config_file = os.path.join(self._config_dir, f)
+                backup_file = os.path.join(self._data_dir, f + '.backup')
+                if not os.path.isfile(config_file) and os.path.isfile(backup_file):
+                    shutil.copy(backup_file, config_file)
         except IOError:
             logger.exception('unable to restore configuration files from backup')
 


Test patch:
diff --git a/tests/test_config.py b/tests/test_config.py
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -39,6 +39,7 @@ def test_reload_local_configuration(self):
             'PATRONI_POSTGRESQL_LISTEN': '0.0.0.0:5432',
             'PATRONI_POSTGRESQL_CONNECT_ADDRESS': '127.0.0.1:5432',
             'PATRONI_POSTGRESQL_DATA_DIR': 'data/postgres0',
+            'PATRONI_POSTGRESQL_CONFIG_DIR': 'data/postgres0',
             'PATRONI_POSTGRESQL_PGPASS': '/tmp/pgpass0',
             'PATRONI_ETCD_HOST': '127.0.0.1:2379',
             'PATRONI_ETCD_URL': 'https://127.0.0.1:2379',
diff --git a/tests/test_postgresql.py b/tests/test_postgresql.py
--- a/tests/test_postgresql.py
+++ b/tests/test_postgresql.py
@@ -168,9 +168,11 @@ class TestPostgresql(unittest.TestCase):
     @patch.object(Postgresql, 'is_running', Mock(return_value=True))
     def setUp(self):
         self.data_dir = 'data/test0'
+        self.config_dir = self.data_dir
         if not os.path.exists(self.data_dir):
             os.makedirs(self.data_dir)
-        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir, 'retry_timeout': 10,
+        self.p = Postgresql({'name': 'test0', 'scope': 'batman', 'data_dir': self.data_dir,
+                             'config_dir': self.config_dir, 'retry_timeout': 10,
                              'listen': '127.0.0.2, 127.0.0.3:5432', 'connect_address': '127.0.0.2:5432',
                              'authentication': {'superuser': {'username': 'test', 'password': 'test'},
                                                 'replication': {'username': 'replicator', 'password': 'rep-pass'}},
@@ -502,7 +504,7 @@ def test_bootstrap(self):
         config = {'users': {'replicator': {'password': 'rep-pass', 'options': ['replication']}}}
 
         self.p.bootstrap(config)
-        with open(os.path.join(self.data_dir, 'pg_hba.conf')) as f:
+        with open(os.path.join(self.config_dir, 'pg_hba.conf')) as f:
             lines = f.readlines()
             self.assertTrue('host all all 0.0.0.0/0 md5\n' in lines)