Repository: opendatacube/datacube-core

Issue:
Running datacube system init on an existing database fails when adding new columns
### Expected behaviour
Updating the database structure should not cause the init process to fail

### Actual behaviour

An active database may have locking happening which causes the `datacube system init` command to fail. Logs:

> datacube -v system init
> 2020-07-24 01:28:43,559 362 datacube INFO Running datacube command: /env/bin/datacube -v system init
> Initialising database...
> 2020-07-24 01:28:43,808 362 datacube.drivers.postgres._core INFO Ensuring user roles.
> 2020-07-24 01:28:43,818 362 datacube.drivers.postgres._core INFO Adding role grants.
> 2020-07-24 01:28:43,839 362 datacube.drivers.postgres._core INFO Adding Update Triggers
> Traceback (most recent call last):
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
>     cursor, statement, parameters, context
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
>     cursor.execute(statement, parameters)
> psycopg2.errors.DeadlockDetected: deadlock detected
> DETAIL:  Process 16680 waits for AccessExclusiveLock on relation 16492 of database 16592; blocked by process 18866.
> Process 18866 waits for AccessShareLock on relation 16431 of database 16592; blocked by process 16680.
> HINT:  See server log for query details.
> 
> 
> The above exception was the direct cause of the following exception:
> 
> Traceback (most recent call last):
>   File "/env/bin/datacube", line 8, in <module>
>     sys.exit(cli())
>   File "/env/lib/python3.6/site-packages/click/core.py", line 829, in __call__
>     return self.main(*args, **kwargs)
>   File "/env/lib/python3.6/site-packages/click/core.py", line 782, in main
>     rv = self.invoke(ctx)
>   File "/env/lib/python3.6/site-packages/click/core.py", line 1259, in invoke
>     return _process_result(sub_ctx.command.invoke(sub_ctx))
>   File "/env/lib/python3.6/site-packages/click/core.py", line 1259, in invoke
>     return _process_result(sub_ctx.command.invoke(sub_ctx))
>   File "/env/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
>     return ctx.invoke(self.callback, **ctx.params)
>   File "/env/lib/python3.6/site-packages/click/core.py", line 610, in invoke
>     return callback(*args, **kwargs)
>   File "/env/lib/python3.6/site-packages/datacube/ui/click.py", line 196, in new_func
>     return f(parsed_config, *args, **kwargs)
>   File "/env/lib/python3.6/site-packages/datacube/ui/click.py", line 228, in with_index
>     return f(index, *args, **kwargs)
>   File "/env/lib/python3.6/site-packages/datacube/scripts/system.py", line 48, in database_init
>     with_permissions=init_users)
>   File "/env/lib/python3.6/site-packages/datacube/index/index.py", line 60, in init_db
>     is_new = self._db.init(with_permissions=with_permissions)
>   File "/env/lib/python3.6/site-packages/datacube/drivers/postgres/_connections.py", line 182, in init
>     _core.update_schema(self._engine)
>   File "/env/lib/python3.6/site-packages/datacube/drivers/postgres/_core.py", line 166, in update_schema
>     install_timestamp_trigger(c)
>   File "/env/lib/python3.6/site-packages/datacube/drivers/postgres/_triggers.py", line 50, in install_timestamp_trigger
>     conn.execute(UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name))
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1006, in execute
>     return self._execute_text(object_, multiparams, params)
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1181, in _execute_text
>     parameters,
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1318, in _execute_context
>     e, statement, parameters, cursor, context
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1512, in _handle_dbapi_exception
>     sqlalchemy_exception, with_traceback=exc_info[2], from_=e
>   File "/env/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
>     raise exception
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
>     cursor, statement, parameters, context
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
>     cursor.execute(statement, parameters)
> sqlalchemy.exc.OperationalError: (psycopg2.errors.DeadlockDetected) deadlock detected
> DETAIL:  Process 16680 waits for AccessExclusiveLock on relation 16492 of database 16592; blocked by process 18866.
> Process 18866 waits for AccessShareLock on relation 16431 of database 16592; blocked by process 16680.
> HINT:  See server log for query details.
> 
> [SQL: 
> alter table agdc.dataset_source add column if not exists updated
> timestamptz not null default now();
> ]
> (Background on this error at: http://sqlalche.me/e/13/e3q8)
> jovyan@jupyter-alexgleith:~$ datacube -v system init
> 2020-07-24 01:45:37,650 386 datacube INFO Running datacube command: /env/bin/datacube -v system init
> Initialising database...
> 2020-07-24 01:45:37,901 386 datacube.drivers.postgres._core INFO Ensuring user roles.
> 2020-07-24 01:45:37,911 386 datacube.drivers.postgres._core INFO Adding role grants.
> 2020-07-24 01:45:37,918 386 datacube.drivers.postgres._core INFO Adding Update Triggers
> Traceback (most recent call last):
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
>     cursor, statement, parameters, context
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
>     cursor.execute(statement, parameters)
> psycopg2.errors.DeadlockDetected: deadlock detected
> DETAIL:  Process 16605 waits for AccessExclusiveLock on relation 16492 of database 16592; blocked by process 18557.
> Process 18557 waits for AccessShareLock on relation 16431 of database 16592; blocked by process 16605.
> HINT:  See server log for query details.
> 
> 
> The above exception was the direct cause of the following exception:
> 
> Traceback (most recent call last):
>   File "/env/bin/datacube", line 8, in <module>
>     sys.exit(cli())
>   File "/env/lib/python3.6/site-packages/click/core.py", line 829, in __call__
>     return self.main(*args, **kwargs)
>   File "/env/lib/python3.6/site-packages/click/core.py", line 782, in main
>     rv = self.invoke(ctx)
>   File "/env/lib/python3.6/site-packages/click/core.py", line 1259, in invoke
>     return _process_result(sub_ctx.command.invoke(sub_ctx))
>   File "/env/lib/python3.6/site-packages/click/core.py", line 1259, in invoke
>     return _process_result(sub_ctx.command.invoke(sub_ctx))
>   File "/env/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
>     return ctx.invoke(self.callback, **ctx.params)
>   File "/env/lib/python3.6/site-packages/click/core.py", line 610, in invoke
>     return callback(*args, **kwargs)
>   File "/env/lib/python3.6/site-packages/datacube/ui/click.py", line 196, in new_func
>     return f(parsed_config, *args, **kwargs)
>   File "/env/lib/python3.6/site-packages/datacube/ui/click.py", line 228, in with_index
>     return f(index, *args, **kwargs)
>   File "/env/lib/python3.6/site-packages/datacube/scripts/system.py", line 48, in database_init
>     with_permissions=init_users)
>   File "/env/lib/python3.6/site-packages/datacube/index/index.py", line 60, in init_db
>     is_new = self._db.init(with_permissions=with_permissions)
>   File "/env/lib/python3.6/site-packages/datacube/drivers/postgres/_connections.py", line 182, in init
>     _core.update_schema(self._engine)
>   File "/env/lib/python3.6/site-packages/datacube/drivers/postgres/_core.py", line 166, in update_schema
>     install_timestamp_trigger(c)
>   File "/env/lib/python3.6/site-packages/datacube/drivers/postgres/_triggers.py", line 50, in install_timestamp_trigger
>     conn.execute(UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name))
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1006, in execute
>     return self._execute_text(object_, multiparams, params)
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1181, in _execute_text
>     parameters,
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1318, in _execute_context
>     e, statement, parameters, cursor, context
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1512, in _handle_dbapi_exception
>     sqlalchemy_exception, with_traceback=exc_info[2], from_=e
>   File "/env/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
>     raise exception
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
>     cursor, statement, parameters, context
>   File "/env/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
>     cursor.execute(statement, parameters)
> sqlalchemy.exc.OperationalError: (psycopg2.errors.DeadlockDetected) deadlock detected
> DETAIL:  Process 16605 waits for AccessExclusiveLock on relation 16492 of database 16592; blocked by process 18557.
> Process 18557 waits for AccessShareLock on relation 16431 of database 16592; blocked by process 16605.
> HINT:  See server log for query details.
> 
> [SQL: 
> alter table agdc.dataset_source add column if not exists updated
> timestamptz not null default now();
> ]
> (Background on this error at: http://sqlalche.me/e/13/e3q8)

### Steps to reproduce the behaviour

Run `datacube system init` on an existing database with lots of records and some activity.

### Environment information

datacube 1.8.2.dev...



Discussion:
This did not go well running on NCI either.

> datacube -v system init
> 2020-07-29 11:12:14,393 3122514 datacube INFO Running datacube command: /g/data/v10/public/modules/dea/unstable/bin/datacube -v system init
> Initialising database...
> 2020-07-29 11:12:15,029 3122514 datacube.drivers.postgres._core INFO Ensuring user roles.
> 2020-07-29 11:12:15,035 3122514 datacube.drivers.postgres._core INFO Adding role grants.
> 2020-07-29 11:12:15,043 3122514 datacube.drivers.postgres._core INFO Adding 'updated' fields and triggers to schema.
> Traceback (most recent call last):
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
>     cursor, statement, parameters, context
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
>     cursor.execute(statement, parameters)
> psycopg2.OperationalError: server closed the connection unexpectedly
> 	This probably means the server terminated abnormally
> 	before or while processing the request.
> 
> 
> The above exception was the direct cause of the following exception:
> 
> Traceback (most recent call last):
>   File "/g/data/v10/public/modules/dea/unstable/bin/datacube", line 8, in <module>
>     sys.exit(cli())
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/click/core.py", line 829, in __call__
>     return self.main(*args, **kwargs)
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/click/core.py", line 782, in main
>     rv = self.invoke(ctx)
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/click/core.py", line 1259, in invoke
>     return _process_result(sub_ctx.command.invoke(sub_ctx))
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/click/core.py", line 1259, in invoke
>     return _process_result(sub_ctx.command.invoke(sub_ctx))
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
>     return ctx.invoke(self.callback, **ctx.params)
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/click/core.py", line 610, in invoke
>     return callback(*args, **kwargs)
>   File "/g/data/v10/public/modules/dea/unstable/lib/python3.6/site-packages/datacube/ui/click.py", line 196, in new_func
>     return f(parsed_config, *args, **kwargs)
>   File "/g/data/v10/public/modules/dea/unstable/lib/python3.6/site-packages/datacube/ui/click.py", line 228, in with_index
>     return f(index, *args, **kwargs)
>   File "/g/data/v10/public/modules/dea/unstable/lib/python3.6/site-packages/datacube/scripts/system.py", line 48, in database_init
>     with_permissions=init_users)
>   File "/g/data/v10/public/modules/dea/unstable/lib/python3.6/site-packages/datacube/index/index.py", line 60, in init_db
>     is_new = self._db.init(with_permissions=with_permissions)
>   File "/g/data/v10/public/modules/dea/unstable/lib/python3.6/site-packages/datacube/drivers/postgres/_connections.py", line 182, in init
>     _core.update_schema(self._engine)
>   File "/g/data/v10/public/modules/dea/unstable/lib/python3.6/site-packages/datacube/drivers/postgres/_core.py", line 191, in update_schema
>     install_timestamp_trigger(c)
>   File "/g/data/v10/public/modules/dea/unstable/lib/python3.6/site-packages/datacube/drivers/postgres/_core.py", line 52, in install_timestamp_trigger
>     connection.execute(UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name))
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1006, in execute
>     return self._execute_text(object_, multiparams, params)
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1181, in _execute_text
>     parameters,
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1318, in _execute_context
>     e, statement, parameters, cursor, context
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1512, in _handle_dbapi_exception
>     sqlalchemy_exception, with_traceback=exc_info[2], from_=e
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 178, in raise_
>     raise exception
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1278, in _execute_context
>     cursor, statement, parameters, context
>   File "/g/data/v10/public/modules/dea-env/20200713/lib/python3.6/site-packages/sqlalchemy/engine/default.py", line 593, in do_execute
>     cursor.execute(statement, parameters)
> sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) server closed the connection unexpectedly
> 	This probably means the server terminated abnormally
> 	before or while processing the request.
> 
> [SQL: 
> alter table agdc.dataset add column if not exists updated
> timestamptz not null default now();
> ]
> (Background on this error at: http://sqlalche.me/e/13/e3q8)
The above error on NCI isn't actually of any use. The update process took an hour and still didn't complete, so I killed it.
huge databases are like that though, it will take forever to update all the indexes. 
Recommended new approach : 

- Leave dataset_location as it is
- Add updated field to: dataset, metadata_type and dataset_type tables
- Add triggers on above tables to update updated field
- Default to null
- No not null constraint
- Add added field to: dataset_source
- Populate with now()
- Default to now()
- Only add this field if it doesn’t exist, like how the ^ is implemented currently
- Do NOT make changes in the schema, but do comment.

It is possible some databases in use have been initialized with the current `datacube system init`, these will need to be manually rolled back. Overall datacube-core needs more managed approach to schema updats and rollbacks.

Resolving patch:
diff --git a/datacube/drivers/postgres/_core.py b/datacube/drivers/postgres/_core.py
--- a/datacube/drivers/postgres/_core.py
+++ b/datacube/drivers/postgres/_core.py
@@ -8,6 +8,7 @@
 from datacube.drivers.postgres.sql import (INSTALL_TRIGGER_SQL_TEMPLATE,
                                            SCHEMA_NAME, TYPES_INIT_SQL,
                                            UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE,
+                                           ADDED_COLUMN_MIGRATE_SQL_TEMPLATE,
                                            UPDATE_TIMESTAMP_SQL,
                                            escape_pg_identifier,
                                            pg_column_exists, pg_exists)
@@ -39,19 +40,21 @@ def install_timestamp_trigger(connection):
     TABLE_NAMES = [
         _schema.METADATA_TYPE.name,
         _schema.PRODUCT.name,
-        _schema.DATASET_SOURCE.name,
         _schema.DATASET.name,
-        _schema.DATASET_LOCATION.name
     ]
     # Create trigger capture function
     connection.execute(UPDATE_TIMESTAMP_SQL)
 
     for name in TABLE_NAMES:
-        # Add update_at columns
-        # HACK: Make this more SQLAlchemy with add_column on Table objects
+        # Add update columns
         connection.execute(UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name))
         connection.execute(INSTALL_TRIGGER_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=name))
 
+def install_added_column(connection):
+    from . import _schema
+    TABLE_NAME = _schema.DATASET_LOCATION.name
+    connection.execute(ADDED_COLUMN_MIGRATE_SQL_TEMPLATE.format(schema=SCHEMA_NAME, table=TABLE_NAME))
+
 
 def schema_qualified(name):
     """
@@ -104,6 +107,8 @@ def ensure_db(engine, with_permissions=True):
             METADATA.create_all(c)
             _LOG.info("Creating triggers.")
             install_timestamp_trigger(c)
+            _LOG.info("Creating added column.")
+            install_added_column(c)
             c.execute('commit')
         except:
             c.execute('rollback')
@@ -183,12 +188,13 @@ def update_schema(engine: Engine):
     # -> If you need to write one, look at the Git history of this
     #    function for some examples.
     
-    # Post 1.8 DB Federation triggers
+    # Post 1.8 DB Incremental Sync triggers
     if not pg_column_exists(engine, schema_qualified('dataset'), 'updated'):
-        _LOG.info("Adding 'updated' fields and triggers to schema.")
+        _LOG.info("Adding 'updated'/'added' fields and triggers to schema.")
         c = engine.connect()
         c.execute('begin')
         install_timestamp_trigger(c)
+        install_added_column(c)
         c.execute('commit')
         c.close()
     else:
diff --git a/datacube/drivers/postgres/_schema.py b/datacube/drivers/postgres/_schema.py
--- a/datacube/drivers/postgres/_schema.py
+++ b/datacube/drivers/postgres/_schema.py
@@ -125,4 +125,7 @@
 
     # Note that the `updated` column is not included here to maintain backwards-compatibility
     # with pre-1.8.3 datacubes (and it is not used by any internal ODC functionality yet anyway)
+
+    # This table is immutable and uses a migrations based `added` column to keep track of new
+    # dataset locations being added. The added column defaults to `now()`
 )
diff --git a/datacube/drivers/postgres/sql.py b/datacube/drivers/postgres/sql.py
--- a/datacube/drivers/postgres/sql.py
+++ b/datacube/drivers/postgres/sql.py
@@ -39,7 +39,12 @@ def visit_create_view(element, compiler, **kw):
 
 UPDATE_COLUMN_MIGRATE_SQL_TEMPLATE = """
 alter table {schema}.{table} add column if not exists updated
-timestamptz not null default now();
+timestamptz default null;
+"""
+
+ADDED_COLUMN_MIGRATE_SQL_TEMPLATE = """
+alter table {schema}.{table} add column if not exists added
+timestamptz default now();
 """
 
 INSTALL_TRIGGER_SQL_TEMPLATE = """


Test patch:
diff --git a/integration_tests/index/test_update_columns.py b/integration_tests/index/test_update_columns.py
new file mode 100644
--- /dev/null
+++ b/integration_tests/index/test_update_columns.py
@@ -0,0 +1,94 @@
+"""
+Test creation of added/updated columns during
+`datacube system init`
+"""
+from datacube.drivers.postgres.sql import SCHEMA_NAME
+from datacube.drivers.postgres import _schema
+
+
+COLUMN_PRESENCE = """
+select exists (select 1 from information_schema.columns
+where table_schema='{schema}' and table_name='{table}' and column_name='{column}');
+"""
+
+DROP_COLUMN = """
+alter table {schema}.{table} drop column {column}
+"""
+
+TRIGGER_PRESENCE = """
+select tgname
+from pg_trigger
+where not tgisinternal
+and tgrelid = '{schema}.{table}'::regclass;
+"""
+
+
+def check_column(conn, table_name: str, column_name: str) -> bool:
+    column_result = conn.execute(
+        COLUMN_PRESENCE.format(
+            schema=SCHEMA_NAME, table=table_name, column=column_name
+        )
+    ).fetchone()
+    return column_result == (True,)
+
+
+def check_trigger(conn, table_name: str) -> bool:
+    trigger_result = conn.execute(
+        TRIGGER_PRESENCE.format(schema=SCHEMA_NAME, table=table_name)
+    ).fetchone()
+    if trigger_result is None:
+        return False
+    return 'row_update_time' in trigger_result[0]
+
+
+def drop_column(conn, table: str, column: str):
+    conn.execute(DROP_COLUMN.format(
+        schema=SCHEMA_NAME, table=table, column=column))
+
+
+def test_added_column(clirunner, uninitialised_postgres_db):
+    # Run on an empty database.
+    result = clirunner(["system", "init"])
+    assert "Created." in result.output
+
+    with uninitialised_postgres_db.connect() as connection:
+        assert check_column(connection, _schema.METADATA_TYPE.name, "updated")
+        assert not check_column(connection, _schema.METADATA_TYPE.name, "fake_column")
+        assert check_column(connection, _schema.PRODUCT.name, "updated")
+        assert not check_column(connection, _schema.PRODUCT.name, "fake_column")
+        assert check_column(connection, _schema.DATASET.name, "updated")
+        assert not check_column(connection, _schema.DATASET.name, "fake_column")
+        assert check_column(connection, _schema.DATASET_LOCATION.name, "added")
+        assert not check_column(connection, _schema.DATASET_LOCATION.name, "fake_column")
+
+        # Check for triggers
+        assert check_trigger(connection, _schema.METADATA_TYPE.name)
+        assert check_trigger(connection, _schema.PRODUCT.name)
+        assert check_trigger(connection, _schema.DATASET.name)
+        assert not check_trigger(connection, _schema.DATASET_LOCATION.name)
+
+
+def test_readd_column(clirunner, uninitialised_postgres_db):
+    # Run on an empty database. drop columns and readd
+    result = clirunner(["system", "init"])
+    assert "Created." in result.output
+
+    with uninitialised_postgres_db.connect() as connection:
+        # Drop all the columns for an init rerun
+        drop_column(connection, _schema.METADATA_TYPE.name, "updated")
+        drop_column(connection, _schema.PRODUCT.name, "updated")
+        drop_column(connection, _schema.DATASET.name, "updated")
+        drop_column(connection, _schema.DATASET_LOCATION.name, "added")
+
+        assert not check_column(connection, _schema.METADATA_TYPE.name, "updated")
+        assert not check_column(connection, _schema.PRODUCT.name, "updated")
+        assert not check_column(connection, _schema.DATASET.name, "updated")
+        assert not check_column(connection, _schema.DATASET_LOCATION.name, "added")
+
+    result = clirunner(["system", "init"])
+
+    with uninitialised_postgres_db.connect() as connection:
+        assert check_column(connection, _schema.METADATA_TYPE.name, "updated")
+        assert check_column(connection, _schema.PRODUCT.name, "updated")
+        assert check_column(connection, _schema.DATASET.name, "updated")
+        assert check_column(connection, _schema.DATASET_LOCATION.name, "added")