Repository: input-output-hk/mithril
Pull request #570 (2022-10-25 15:24:09)

Configurable Aggregator snapshot bucket
## Content
This PR includes a configurable `Snapshot Bucket Name` for the Aggregator that is used by the `GCPFileUploader`.
This will be used when deploying to the `testing`, `pre-release` and `release` environments.

## Pre-submit checklist

- Branch
  - [x] Commit sequence broadly makes sense
  - [x] Key commits have useful messages
- PR
  - [x] No clippy warnings in the CI
  - [x] Self-reviewed the diff
  - [x] Useful pull request description
  - [x] Reviewer requested
- Documentation
  - [x] Update documentation website

## Issue(s)
Relates to #542 



Patch:
diff --git a/docs/root/manual/developer-docs/nodes/mithril-aggregator.md b/docs/root/manual/developer-docs/nodes/mithril-aggregator.md
index 416518317..eaa178cef 100644
--- a/docs/root/manual/developer-docs/nodes/mithril-aggregator.md
+++ b/docs/root/manual/developer-docs/nodes/mithril-aggregator.md
@@ -299,6 +299,7 @@ General parameters:
 | `snapshot_directory` | `--snapshot-directory` | - | `SNAPSHOT_DIRECTORY` | Directory to store local snapshots of the **Cardano Node** | `.` | - | :heavy_check_mark: |
 | `snapshot_store_type` | - | - | `SNAPSHOT_STORE_TYPE` | Type of snapshot store to use | - | `gcp` or `local` | :heavy_check_mark: |
 | `snapshot_uploader_type` | - | - | `SNAPSHOT_UPLOADER_TYPE` | Type of snapshot uploader to use | - | `gcp` or `local` | :heavy_check_mark: |
+| `snapshot_bucket_name` | - | - | `SNAPSHOT_BUCKET_NAME` | Name of the bucket where the snapshots are stored  | - | `snapshot-bucket` | :heavy_check_mark: | Required if `snapshot_uploader_type` is `gcp`
 | `run_interval` | - | - | `RUN_INTERVAL` | Interval between two runtime cycles in ms | - | `60000` | :heavy_check_mark: |
 | `url_snapshot_manifest` | - | - | `URL_SNAPSHOT_MANIFEST` | Snapshots manifest location | - | Only if `snapshot_store_type` is `gcp`, else it should be `` | :heavy_check_mark: |
 
diff --git a/mithril-aggregator/config/preview.json b/mithril-aggregator/config/preview.json
index 48401a803..d3d125085 100644
--- a/mithril-aggregator/config/preview.json
+++ b/mithril-aggregator/config/preview.json
@@ -11,7 +11,8 @@
     "url_snapshot_manifest": "https://storage.googleapis.com/cardano-preview/snapshots.json",
     "snapshot_store_type": "local",
     "snapshot_uploader_type": "gcp",
+    "snapshot_bucket_name": "cardano-testnet",
     "stores_directory": "./mithril-aggregator/stores",
     "genesis_verification_key": "5b3132372c37332c3132342c3136312c362c3133372c3133312c3231332c3230372c3131372c3139382c38352c3137362c3139392c3136322c3234312c36382c3132332c3131392c3134352c31332c3233322c3234332c34392c3232392c322c3234392c3230352c3230352c33392c3233352c34345d",
     "limit_keys_in_stores": 5
-}
+}
\ No newline at end of file
diff --git a/mithril-aggregator/src/command_args.rs b/mithril-aggregator/src/command_args.rs
index 09f716db2..3700aa10d 100644
--- a/mithril-aggregator/src/command_args.rs
+++ b/mithril-aggregator/src/command_args.rs
@@ -297,7 +297,7 @@ impl ServeCommand {
         debug!("SERVE command"; "config" => format!("{:?}", config));
         // Init dependencies
         let snapshot_store = config.build_snapshot_store()?;
-        let snapshot_uploader = config.build_snapshot_uploader();
+        let snapshot_uploader = config.build_snapshot_uploader()?;
 
         let sqlite_db_path = Some(config.get_sqlite_file());
 
diff --git a/mithril-aggregator/src/configuration.rs b/mithril-aggregator/src/configuration.rs
index 4bb3354c4..784b88193 100644
--- a/mithril-aggregator/src/configuration.rs
+++ b/mithril-aggregator/src/configuration.rs
@@ -49,6 +49,9 @@ pub struct Configuration {
     /// Type of snapshot uploader to use
     pub snapshot_uploader_type: SnapshotUploaderType,
 
+    /// Bucket name where the snapshots are stored if snapshot_uploader_type is Gcp
+    pub snapshot_bucket_name: Option<String>,
+
     /// Server listening IP
     pub server_ip: String,
 
@@ -109,7 +112,11 @@ impl Configuration {
     pub fn build_snapshot_store(&self) -> Result<Arc<dyn SnapshotStore>, Box<dyn Error>> {
         match self.snapshot_store_type {
             SnapshotStoreType::Gcp => Ok(Arc::new(RemoteSnapshotStore::new(
-                Box::new(GcpFileUploader::default()),
+                Box::new(GcpFileUploader::new(
+                    self.snapshot_bucket_name.to_owned().ok_or_else(|| {
+                        ConfigError::Message("missing snapshot bucket name".to_string())
+                    })?,
+                )),
                 self.url_snapshot_manifest.clone(),
             ))),
             SnapshotStoreType::Local => Ok(Arc::new(LocalSnapshotStore::new(
@@ -123,15 +130,17 @@ impl Configuration {
     }
 
     /// Create a snapshot uploader from configuration settings.
-    pub fn build_snapshot_uploader(&self) -> Arc<dyn SnapshotUploader> {
+    pub fn build_snapshot_uploader(&self) -> Result<Arc<dyn SnapshotUploader>, Box<dyn Error>> {
         match self.snapshot_uploader_type {
-            SnapshotUploaderType::Gcp => Arc::new(RemoteSnapshotUploader::new(Box::new(
-                GcpFileUploader::default(),
-            ))),
-            SnapshotUploaderType::Local => Arc::new(LocalSnapshotUploader::new(
+            SnapshotUploaderType::Gcp => Ok(Arc::new(RemoteSnapshotUploader::new(Box::new(
+                GcpFileUploader::new(self.snapshot_bucket_name.to_owned().ok_or_else(|| {
+                    ConfigError::Message("missing snapshot bucket name".to_string())
+                })?),
+            )))),
+            SnapshotUploaderType::Local => Ok(Arc::new(LocalSnapshotUploader::new(
                 self.get_server_url(),
                 &self.snapshot_directory,
-            )),
+            ))),
         }
     }
 
diff --git a/mithril-aggregator/src/dependency.rs b/mithril-aggregator/src/dependency.rs
index 1db15e545..d7c08184c 100644
--- a/mithril-aggregator/src/dependency.rs
+++ b/mithril-aggregator/src/dependency.rs
@@ -193,6 +193,7 @@ pub mod tests {
                 .to_string(),
             snapshot_store_type: SnapshotStoreType::Local,
             snapshot_uploader_type: SnapshotUploaderType::Local,
+            snapshot_bucket_name: None,
             server_ip: "0.0.0.0".to_string(),
             server_port: 8000,
             run_interval: 5000,
diff --git a/mithril-aggregator/src/tools/remote_file_uploader.rs b/mithril-aggregator/src/tools/remote_file_uploader.rs
index 2b507451f..ba5d2b927 100644
--- a/mithril-aggregator/src/tools/remote_file_uploader.rs
+++ b/mithril-aggregator/src/tools/remote_file_uploader.rs
@@ -25,11 +25,10 @@ pub struct GcpFileUploader {
     bucket: String,
 }
 
-impl Default for GcpFileUploader {
-    fn default() -> Self {
-        Self {
-            bucket: "cardano-testnet".to_string(),
-        }
+impl GcpFileUploader {
+    /// GcpFileUploader factory
+    pub fn new(bucket: String) -> Self {
+        Self { bucket }
     }
 }
 
diff --git a/mithril-infra/docker-compose.yaml b/mithril-infra/docker-compose.yaml
index 324c98bb3..7b9b5cecb 100644
--- a/mithril-infra/docker-compose.yaml
+++ b/mithril-infra/docker-compose.yaml
@@ -65,6 +65,7 @@ services:
       - URL_SNAPSHOT_MANIFEST=https://storage.googleapis.com/cardano-${NETWORK}/snapshots.json
       - SNAPSHOT_STORE_TYPE=local
       - SNAPSHOT_UPLOADER_TYPE=gcp
+      - SNAPSHOT_BUCKET_NAME=cardano-testnet
       - DATA_STORES_DIRECTORY=/mithril-aggregator
       - STORE_RETENTION_LIMIT=5
       - CARDANO_NODE_SOCKET_PATH=/ipc/node.socket


Test patch:
diff --git a/mithril-aggregator/config/testnet.json b/mithril-aggregator/config/testnet.json
deleted file mode 100644
index 974e6037e..000000000
--- a/mithril-aggregator/config/testnet.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-    "cardano_cli_path": "cardano-cli",
-    "cardano_node_socket_path": "/tmp/cardano.sock",
-    "network": "testnet",
-    "run_interval": 60000,
-    "protocol_parameters": {
-        "k": 5,
-        "m": 100,
-        "phi_f": 0.65
-    },
-    "url_snapshot_manifest": "https://storage.googleapis.com/cardano-testnet/snapshots.json",
-    "snapshot_store_type": "local",
-    "snapshot_uploader_type": "gcp",
-    "stores_directory": "./mithril-aggregator/stores"
-}
diff --git a/mithril-aggregator/tests/test_extensions/dependency.rs b/mithril-aggregator/tests/test_extensions/dependency.rs
index 7e6bbd9ba..c7a026d64 100644
--- a/mithril-aggregator/tests/test_extensions/dependency.rs
+++ b/mithril-aggregator/tests/test_extensions/dependency.rs
@@ -37,6 +37,7 @@ pub async fn initialize_dependencies(
             .to_string(),
         snapshot_store_type: SnapshotStoreType::Local,
         snapshot_uploader_type: SnapshotUploaderType::Local,
+        snapshot_bucket_name: None,
         server_ip: "0.0.0.0".to_string(),
         server_port: 8000,
         run_interval: 5000,
diff --git a/mithril-client/config/testnet.json b/mithril-client/config/testnet.json
deleted file mode 100644
index 6c4cc6fc4..000000000
--- a/mithril-client/config/testnet.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "network": "testnet",
-    "aggregator_endpoint": "https://aggregator.api.mithril.network/aggregator"
-}
\ No newline at end of file
diff --git a/mithril-signer/config/testnet.json b/mithril-signer/config/testnet.json
deleted file mode 100644
index bc8a11b06..000000000
--- a/mithril-signer/config/testnet.json
+++ /dev/null
@@ -1,8 +0,0 @@
-{
-    "network": "testnet",
-    "aggregator_endpoint": "https://aggregator.api.mithril.network/aggregator",
-    "party_id": 0,
-    "run_interval": 20000,
-    "db_directory": "/db",
-    "data_stores_directory": "./mithril-signer/stores"
-}
\ No newline at end of file