Repository: golemfactory/yagna (language: python)

Issue:
Provider can set "Outbound for all" Rule on or off
**As a** Provider
**I want** to be able to turn Outbound for all Rule on or off
**so that** either only trusted Outbound connections are made or more tasks will be received by me.


Discussion / hints:

Fix patch:
diff --git a/Cargo.lock b/Cargo.lock
index 8b86652d5e..328c1a4b22 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1668,6 +1668,12 @@ dependencies = [
  "migrations_macros",
 ]
 
+[[package]]
+name = "diff"
+version = "0.1.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
+
 [[package]]
 name = "difflib"
 version = "0.4.0"
@@ -4191,6 +4197,15 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "output_vt100"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66"
+dependencies = [
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "pairing_ce"
 version = "0.24.2"
@@ -4519,6 +4534,18 @@ dependencies = [
  "termtree",
 ]
 
+[[package]]
+name = "pretty_assertions"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755"
+dependencies = [
+ "ctor",
+ "diff",
+ "output_vt100",
+ "yansi",
+]
+
 [[package]]
 name = "prettytable-rs"
 version = "0.8.0"
@@ -8135,6 +8162,7 @@ dependencies = [
  "num_cpus",
  "path-clean",
  "predicates",
+ "pretty_assertions",
  "regex",
  "semver 0.11.0",
  "serde",
@@ -8147,6 +8175,7 @@ dependencies = [
  "strum 0.24.1",
  "strum_macros 0.24.3",
  "sys-info",
+ "tempdir",
  "test-case",
  "thiserror",
  "tokio 1.22.0",
diff --git a/agent/provider/Cargo.toml b/agent/provider/Cargo.toml
index 9ff2cae3ca..c54ef55364 100644
--- a/agent/provider/Cargo.toml
+++ b/agent/provider/Cargo.toml
@@ -57,12 +57,12 @@ serde_json = "1.0"
 shared_child = "0.3.4"
 signal-hook = "0.3"
 structopt = "0.3.20"
-strum ="0.24"
+strum = "0.24"
 strum_macros = "0.24"
 sys-info = "0.8.0"
 thiserror = "1.0.14"
 tokio = { version = "1", features = ["process", "signal"] }
-tokio-stream = {version = "0.1.6", features = ["sync"] }
+tokio-stream = { version = "0.1.6", features = ["sync"] }
 url = "2.1.1"
 walkdir = "2.3.1"
 yansi = "0.5.0"
@@ -81,5 +81,7 @@ test-case = "2.1"
 predicates = "2.1"
 serial_test = "0.9"
 shlex = "1.1"
+tempdir = "0.3"
+pretty_assertions = "1.3"
 
 ya-manifest-test-utils = "0.1"
diff --git a/agent/provider/src/cli.rs b/agent/provider/src/cli.rs
index bdb28cf95d..a4f720e863 100644
--- a/agent/provider/src/cli.rs
+++ b/agent/provider/src/cli.rs
@@ -5,6 +5,7 @@ pub mod exe_unit;
 pub mod keystore;
 pub mod preset;
 pub mod profile;
+pub mod rule;
 pub mod whitelist;
 
 use crate::startup_config::ProviderConfig;
diff --git a/agent/provider/src/cli/rule.rs b/agent/provider/src/cli/rule.rs
new file mode 100644
index 0000000000..659dcdb2f1
--- /dev/null
+++ b/agent/provider/src/cli/rule.rs
@@ -0,0 +1,60 @@
+use anyhow::Result;
+use structopt::StructOpt;
+use ya_manifest_utils::rules::{Mode, RuleStore};
+
+use crate::startup_config::ProviderConfig;
+
+#[derive(StructOpt, Clone, Debug)]
+pub enum RuleCommand {
+    /// Set Modes for specific Rules
+    Set(SetRule),
+    /// List active Rules and their information
+    List,
+}
+
+#[derive(StructOpt, Clone, Debug)]
+pub enum SetRule {
+    Everyone {
+        #[structopt(subcommand)]
+        mode: Mode,
+    },
+    AuditedPayload {
+        #[structopt(long)]
+        certificate: Option<String>,
+        #[structopt(subcommand)]
+        mode: Mode,
+    },
+}
+
+impl RuleCommand {
+    pub fn run(self, config: ProviderConfig) -> Result<()> {
+        match self {
+            RuleCommand::Set(set_rule) => set(set_rule, config),
+            RuleCommand::List => list(config),
+        }
+    }
+}
+
+fn set(set_rule: SetRule, config: ProviderConfig) -> Result<()> {
+    let rules = RuleStore::load_or_create(&config.rules_file)?;
+
+    match set_rule {
+        SetRule::Everyone { mode } => rules.set_everyone_mode(mode),
+        SetRule::AuditedPayload { certificate, mode } => match certificate {
+            Some(_) => todo!("Setting rule for specific certificate isn't implemented yet"),
+            None => rules.set_default_audited_payload_mode(mode),
+        },
+    }
+}
+
+fn list(config: ProviderConfig) -> Result<()> {
+    let rules = RuleStore::load_or_create(&config.rules_file)?;
+
+    if config.json {
+        rules.print()?;
+    } else {
+        todo!("Printing pretty table isn't implemented yet")
+    }
+
+    Ok(())
+}
diff --git a/agent/provider/src/main.rs b/agent/provider/src/main.rs
index 1e61e35a60..e84b182bec 100644
--- a/agent/provider/src/main.rs
+++ b/agent/provider/src/main.rs
@@ -26,6 +26,7 @@ async fn main() -> anyhow::Result<()> {
     config.globals_file = data_dir.join(config.globals_file);
     config.presets_file = data_dir.join(config.presets_file);
     config.hardware_file = data_dir.join(config.hardware_file);
+    config.rules_file = data_dir.join(config.rules_file);
 
     match cli_args.commands {
         Commands::Run(args) => {
@@ -46,5 +47,6 @@ async fn main() -> anyhow::Result<()> {
         Commands::Keystore(keystore_cmd) => keystore_cmd.run(config),
         Commands::Whitelist(whitelist_cmd) => whitelist_cmd.run(config),
         Commands::Clean(clean_cmd) => clean_cmd.run(config),
+        Commands::Rule(outbound_cmd) => outbound_cmd.run(config),
     }
 }
diff --git a/agent/provider/src/market/negotiator/builtin/manifest.rs b/agent/provider/src/market/negotiator/builtin/manifest.rs
index fac65d0404..80fca14800 100644
--- a/agent/provider/src/market/negotiator/builtin/manifest.rs
+++ b/agent/provider/src/market/negotiator/builtin/manifest.rs
@@ -6,6 +6,7 @@ use ya_agreement_utils::{Error, OfferDefinition};
 use ya_manifest_utils::matching::domain::SharedDomainMatchers;
 use ya_manifest_utils::matching::Matcher;
 use ya_manifest_utils::policy::{CertPermissions, Keystore, Match, Policy, PolicyConfig};
+use ya_manifest_utils::rules::RuleStore;
 use ya_manifest_utils::{
     decode_manifest, AppManifest, Feature, CAPABILITIES_PROPERTY,
     DEMAND_MANIFEST_CERT_PERMISSIONS_PROPERTY, DEMAND_MANIFEST_CERT_PROPERTY,
@@ -14,10 +15,10 @@ use ya_manifest_utils::{
 
 use crate::market::negotiator::*;
 
-#[derive(Default)]
 pub struct ManifestSignature {
     enabled: bool,
     keystore: Keystore,
+    rulestore: RuleStore,
     whitelist_matcher: SharedDomainMatchers,
 }
 
@@ -27,7 +28,7 @@ impl NegotiatorComponent for ManifestSignature {
         demand: &ProposalView,
         offer: ProposalView,
     ) -> anyhow::Result<NegotiationResult> {
-        if self.enabled.not() {
+        if self.enabled.not() || self.rulestore.always_accept_outbound() {
             log::trace!("Manifest signature verification disabled.");
             return acceptance(offer);
         }
@@ -100,10 +101,13 @@ impl From<PolicyConfig> for ManifestSignature {
         };
 
         let whitelist_matcher = config.domain_patterns.matchers.clone();
+        //TODO Nones should be errors or config should not wrap stores inside Option
         let keystore = config.trusted_keys.unwrap_or_default();
+        let rulestore = config.rules_config.unwrap_or_default();
         ManifestSignature {
             enabled,
             keystore,
+            rulestore,
             whitelist_matcher,
         }
     }
diff --git a/agent/provider/src/provider_agent.rs b/agent/provider/src/provider_agent.rs
index 9b029a57f8..2a05c17e08 100644
--- a/agent/provider/src/provider_agent.rs
+++ b/agent/provider/src/provider_agent.rs
@@ -3,6 +3,7 @@ use anyhow::{anyhow, Error};
 use futures::{FutureExt, StreamExt, TryFutureExt};
 use ya_client::net::NetApi;
 use ya_manifest_utils::matching::domain::{DomainPatterns, DomainWhitelistState, DomainsMatcher};
+use ya_manifest_utils::rules::RuleStore;
 
 use std::convert::TryFrom;
 use std::path::{Path, PathBuf};
@@ -129,6 +130,7 @@ pub struct ProviderAgent {
     log_handler: LoggerHandle,
     networks: Vec<NetworkName>,
     keystore_monitor: FileMonitor,
+    rulestore_monitor: FileMonitor,
     net_api: NetApi,
     domain_whitelist: WhitelistManager,
 }
@@ -185,11 +187,14 @@ impl ProviderAgent {
         let cert_dir = config.cert_dir_path()?;
         let keystore = load_keystore(&cert_dir)?;
 
+        let rulestore = RuleStore::load_or_create(&config.rules_file)?;
+
         args.market.session_id = format!("{}-{}", name, std::process::id());
         args.runner.session_id = args.market.session_id.clone();
         args.payment.session_id = args.market.session_id.clone();
         let policy_config = &mut args.market.negotiator_config.composite_config.policy_config;
         policy_config.trusted_keys = Some(keystore.clone());
+        policy_config.rules_config = Some(rulestore.clone());
 
         let networks = args.node.account.networks.clone();
         for n in networks.iter() {
@@ -211,6 +216,7 @@ impl ProviderAgent {
         let mut hardware = hardware::Manager::try_new(&config)?;
         hardware.spawn_monitor(&config.hardware_file)?;
         let keystore_monitor = spawn_keystore_monitor(cert_dir, keystore)?;
+        let rulestore_monitor = spawn_rulestore_monitor(rulestore)?;
         let mut domain_whitelist = WhitelistManager::try_new(&config.domain_whitelist_file)?;
         domain_whitelist.spawn_monitor(&config.domain_whitelist_file)?;
         policy_config.domain_patterns = domain_whitelist.get_state();
@@ -233,6 +239,7 @@ impl ProviderAgent {
             log_handler,
             networks,
             keystore_monitor,
+            rulestore_monitor,
             net_api,
             domain_whitelist,
         })
@@ -498,6 +505,22 @@ fn spawn_keystore_monitor<P: AsRef<Path>>(
     Ok(monitor)
 }
 
+fn spawn_rulestore_monitor(rulestore: RuleStore) -> Result<FileMonitor, Error> {
+    let path = rulestore.path.clone();
+    let handler = move |p: PathBuf| match rulestore.reload() {
+        Ok(()) => {
+            log::info!("rulestore updated from {}", p.display());
+        }
+        Err(e) => log::warn!("Error updating rulestore from {}: {e}", p.display()),
+    };
+    let monitor = FileMonitor::spawn_with(
+        path,
+        FileMonitor::on_modified(handler),
+        FileMonitorConfig::silent(),
+    )?;
+    Ok(monitor)
+}
+
 impl Actor for ProviderAgent {
     type Context = Context<Self>;
 
@@ -591,6 +614,7 @@ impl Handler<Shutdown> for ProviderAgent {
         let runner = self.runner.clone();
         let log_handler = self.log_handler.clone();
         self.keystore_monitor.stop();
+        self.rulestore_monitor.stop();
         self.domain_whitelist.stop();
 
         async move {
diff --git a/agent/provider/src/startup_config.rs b/agent/provider/src/startup_config.rs
index 2e4abf7d6d..ed8d88f40a 100644
--- a/agent/provider/src/startup_config.rs
+++ b/agent/provider/src/startup_config.rs
@@ -21,6 +21,7 @@ use crate::cli::exe_unit::ExeUnitsConfig;
 use crate::cli::keystore::KeystoreConfig;
 pub use crate::cli::preset::PresetsConfig;
 use crate::cli::profile::ProfileConfig;
+use crate::cli::rule::RuleCommand;
 use crate::cli::whitelist::WhitelistConfig;
 pub(crate) use crate::config::globals::GLOBALS_JSON;
 use crate::execution::{ExeUnitsRegistry, TaskRunnerConfig};
@@ -33,6 +34,7 @@ lazy_static::lazy_static! {
     static ref DEFAULT_PLUGINS_DIR : PathBuf = default_plugins();
 }
 pub(crate) const DOMAIN_WHITELIST_JSON: &str = "domain_whitelist.json";
+pub(crate) const RULES_JSON: &str = "rules.json";
 pub(crate) const PRESETS_JSON: &str = "presets.json";
 pub(crate) const HARDWARE_JSON: &str = "hardware.json";
 pub(crate) const CERT_DIR: &str = "cert-dir";
@@ -82,6 +84,8 @@ pub struct ProviderConfig {
     pub presets_file: PathBuf,
     #[structopt(skip = HARDWARE_JSON)]
     pub hardware_file: PathBuf,
+    #[structopt(skip = RULES_JSON)]
+    pub rules_file: PathBuf,
     /// Max number of available CPU cores
     #[structopt(
         long,
@@ -238,6 +242,8 @@ pub enum Commands {
     Whitelist(WhitelistConfig),
     /// Clean up disk space
     Clean(CleanConfig),
+    /// Manage Rule config
+    Rule(RuleCommand),
 }
 
 #[derive(Debug)]
diff --git a/utils/manifest-utils/src/lib.rs b/utils/manifest-utils/src/lib.rs
index 6c9a31558d..0d5c6c7cf5 100644
--- a/utils/manifest-utils/src/lib.rs
+++ b/utils/manifest-utils/src/lib.rs
@@ -1,6 +1,7 @@
 pub mod manifest;
 pub mod matching;
 pub mod policy;
+pub mod rules;
 pub mod util;
 
 pub use manifest::*;
diff --git a/utils/manifest-utils/src/policy.rs b/utils/manifest-utils/src/policy.rs
index 0a9fe3de68..21947fb0e3 100644
--- a/utils/manifest-utils/src/policy.rs
+++ b/utils/manifest-utils/src/policy.rs
@@ -18,6 +18,7 @@ use structopt::StructOpt;
 use strum::{Display, EnumIter, EnumString, EnumVariantNames, IntoEnumIterator, VariantNames};
 
 use crate::matching::domain::DomainWhitelistState;
+use crate::rules::RuleStore;
 use crate::util::{cert_to_id, format_permissions, CertBasicDataVisitor, X509Visitor};
 
 pub(crate) const PERMISSIONS_FILE: &str = "cert-permissions.json";
@@ -48,6 +49,8 @@ pub struct PolicyConfig {
     pub trusted_keys: Option<Keystore>,
     #[structopt(skip)]
     pub domain_patterns: DomainWhitelistState,
+    #[structopt(skip)]
+    pub rules_config: Option<RuleStore>,
 }
 
 impl PolicyConfig {
diff --git a/utils/manifest-utils/src/rules.rs b/utils/manifest-utils/src/rules.rs
new file mode 100644
index 0000000000..e4ff498f83
--- /dev/null
+++ b/utils/manifest-utils/src/rules.rs
@@ -0,0 +1,147 @@
+use std::{
+    fs::OpenOptions,
+    io::BufReader,
+    path::{Path, PathBuf},
+    sync::{Arc, RwLock},
+};
+
+use anyhow::Result;
+use serde::{Deserialize, Serialize};
+use structopt::StructOpt;
+use strum::Display;
+
+#[derive(Clone, Debug, Default)]
+pub struct RuleStore {
+    pub path: PathBuf,
+    config: Arc<RwLock<RulesConfig>>,
+}
+
+impl RuleStore {
+    pub fn load_or_create(rules_file: &Path) -> Result<Self> {
+        if rules_file.exists() {
+            log::debug!("Loading rule from: {}", rules_file.display());
+            let file = OpenOptions::new().read(true).open(rules_file)?;
+
+            Ok(Self {
+                config: Arc::new(serde_json::from_reader(BufReader::new(file))?),
+                path: rules_file.to_path_buf(),
+            })
+        } else {
+            log::debug!("Creating default Rules configuration");
+            let config = Default::default();
+
+            let store = Self {
+                config: Arc::new(RwLock::new(config)),
+                path: rules_file.to_path_buf(),
+            };
+            store.save()?;
+
+            Ok(store)
+        }
+    }
+
+    fn save(&self) -> Result<()> {
+        log::debug!("Saving RuleStore to: {}", self.path.display());
+        Ok(std::fs::write(
+            &self.path,
+            serde_json::to_string_pretty(&*self.config.read().unwrap())?,
+        )?)
+    }
+
+    pub fn reload(&self) -> Result<()> {
+        log::debug!("Reloading RuleStore from: {}", self.path.display());
+        let new_rule_store = Self::load_or_create(&self.path)?;
+
+        self.replace(new_rule_store);
+
+        Ok(())
+    }
+
+    fn replace(&self, other: Self) {
+        let store = std::mem::take(&mut (*other.config.write().unwrap()));
+
+        *self.config.write().unwrap() = store;
+    }
+
+    pub fn set_everyone_mode(&self, mode: Mode) -> Result<()> {
+        log::debug!("Setting outbound everyone mode: {mode}");
+        self.config.write().unwrap().outbound.everyone = mode;
+
+        self.save()
+    }
+
+    pub fn set_default_audited_payload_mode(&self, mode: Mode) -> Result<()> {
+        log::debug!("Setting outbound audited_payload default mode: {mode}");
+        self.config
+            .write()
+            .unwrap()
+            .outbound
+            .audited_payload
+            .default
+            .mode = mode;
+
+        self.save()
+    }
+
+    pub fn always_accept_outbound(&self) -> bool {
+        self.config.read().unwrap().outbound.everyone == Mode::All
+    }
+
+    pub fn print(&self) -> Result<()> {
+        println!(
+            "{}",
+            serde_json::to_string_pretty(&*self.config.read().unwrap())?
+        );
+
+        Ok(())
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct RulesConfig {
+    outbound: OutboundConfig,
+}
+
+impl Default for RulesConfig {
+    fn default() -> Self {
+        Self {
+            outbound: OutboundConfig {
+                enabled: true,
+                everyone: Mode::None,
+                audited_payload: CertRules {
+                    default: CertRule {
+                        mode: Mode::All,
+                        description: "Default setting".into(),
+                    },
+                },
+            },
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct OutboundConfig {
+    enabled: bool,
+    everyone: Mode,
+    audited_payload: CertRules,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct CertRules {
+    default: CertRule,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+pub struct CertRule {
+    mode: Mode,
+    description: String,
+}
+
+#[derive(StructOpt, Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Display)]
+#[serde(rename_all = "kebab-case")]
+pub enum Mode {
+    All,
+    None,
+    Whitelist,
+}


Test patch:
diff --git a/agent/provider/tests/rule_cli.rs b/agent/provider/tests/rule_cli.rs
new file mode 100644
index 0000000000..82d00fa7ed
--- /dev/null
+++ b/agent/provider/tests/rule_cli.rs
@@ -0,0 +1,103 @@
+use std::path::Path;
+
+use assert_cmd::Command;
+use pretty_assertions::assert_eq;
+use serde_json::json;
+use tempdir::TempDir;
+use test_case::test_case;
+
+#[test]
+fn rule_list_cmd_should_print_default_rules() {
+    let data_dir = prepare_test_dir();
+
+    let output = Command::cargo_bin("ya-provider")
+        .unwrap()
+        .env("DATA_DIR", data_dir.path().to_str().unwrap())
+        .arg("rule")
+        .arg("list")
+        .arg("--json")
+        .output()
+        .unwrap();
+
+    let result: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap();
+
+    assert_eq!(
+        result,
+        json!({
+          "outbound": {
+            "enabled": true,
+            "everyone": "none",
+            "audited-payload": {
+              "default": {
+                "mode": "all",
+                "description": "Default setting"
+              }
+            }
+          }
+        })
+    );
+}
+
+#[test_case("all")]
+#[test_case("none")]
+#[test_case("whitelist")]
+fn rule_set_should_edit_everyone_mode(mode: &str) {
+    let rule = "everyone";
+    let data_dir = prepare_test_dir();
+
+    Command::cargo_bin("ya-provider")
+        .unwrap()
+        .env("DATA_DIR", data_dir.path().to_str().unwrap())
+        .arg("rule")
+        .arg("set")
+        .arg(rule)
+        .arg(mode)
+        .assert()
+        .success();
+
+    let result = list_rules_command(data_dir.path());
+
+    assert_eq!(&result["outbound"][rule], mode);
+}
+
+#[test_case("audited-payload", "all")]
+#[test_case("audited-payload", "none")]
+#[test_case("audited-payload", "whitelist")]
+fn rule_set_should_edit_default_modes_for_certificate_rules(rule: &str, mode: &str) {
+    let data_dir = prepare_test_dir();
+
+    Command::cargo_bin("ya-provider")
+        .unwrap()
+        .env("DATA_DIR", data_dir.path().to_str().unwrap())
+        .arg("rule")
+        .arg("set")
+        .arg(rule)
+        .arg(mode)
+        .assert()
+        .success();
+
+    let result = list_rules_command(data_dir.path());
+
+    assert_eq!(&result["outbound"][rule]["default"]["mode"], mode);
+    assert_eq!(
+        &result["outbound"][rule]["default"]["description"],
+        "Default setting"
+    );
+}
+
+fn list_rules_command(data_dir: &Path) -> serde_json::Value {
+    let output = Command::cargo_bin("ya-provider")
+        .unwrap()
+        .env("DATA_DIR", data_dir.to_str().unwrap())
+        .arg("rule")
+        .arg("list")
+        .arg("--json")
+        .output()
+        .unwrap();
+
+    serde_json::from_slice(&output.stdout).unwrap()
+}
+
+fn prepare_test_dir() -> TempDir {
+    TempDir::new("rule-cli-test-data-dir").unwrap()
+}