Repository: pantsbuild/pants

Issue:
Allow a strong-weak cycle to be entered from either end
For example, for the following cyclic graph:
```
digraph {
  A -> B [label=strong];
  B -> A [label=weak, style=dashed];
}
```
... there are two potential entrypoints. The current behavior is that:
*  If a caller requests `A` first:
    1. `A` will start and request `B` with a strong edge, which will succeed.
    2. `B` will start and request `A` with a weak edge, which will detect a cycle and
       return None.
    3. `B` will complete as Dirty.
    4. `A` will complete as Clean.
* If a caller requests `B` first:
    1. `B` will run and request `A` with a weak edge, which will succeed.
    2. `A` will run and request `B` with a strong edge, which will cycle detect and see a cycle
       of running nodes  which would cause it to deadlock if it requested `B`. It will fail with a cycle
       error, because it is not able to "cancel" or restart the computation of `B` (which would allow
      `B` to instead observe `None` rather than continuing to wait on `A`).

----

It should be possible to make this case consistent when `B` is requested first by allowing `B` to run multiple times (ie, until it reaches a consumer-specific fixpoint).

Remove weak Gets
This ended up not being as robust as we wanted. Now it's leftover code.


Discussion:

Resolving patch:
diff --git a/src/python/pants/engine/internals/native.py b/src/python/pants/engine/internals/native.py
--- a/src/python/pants/engine/internals/native.py
+++ b/src/python/pants/engine/internals/native.py
@@ -73,7 +73,6 @@ def generator_send(
                     res.output_type,
                     res.input_type,
                     res.input,
-                    res.weak,
                 )
             elif type(res) in (tuple, list):
                 # GetMulti.
@@ -83,7 +82,6 @@ def generator_send(
                             get.output_type,
                             get.input_type,
                             get.input,
-                            get.weak,
                         )
                         for get in res
                     )
diff --git a/src/python/pants/engine/internals/selectors.py b/src/python/pants/engine/internals/selectors.py
--- a/src/python/pants/engine/internals/selectors.py
+++ b/src/python/pants/engine/internals/selectors.py
@@ -122,10 +122,6 @@ class Get(GetConstraints, Generic[_Output, _Input]):
     infer from the input variable [1]. Likewise, the short form must use inline construction of the
     input in order to convey the input type to the engine.
 
-    The `weak` parameter is an experimental extension: a "weak" Get will return None rather than the
-    requested value iff the dependency caused by the Get would create a cycle in the dependency
-    graph.
-
     [1] The engine needs to determine all rule and Get input and output types statically before
     executing any rules. Since Gets are declared inside function bodies, the only way to extract this
     information is through a parse of the rule function. The parse analysis is rudimentary and cannot
@@ -135,9 +131,7 @@ class Get(GetConstraints, Generic[_Output, _Input]):
     """
 
     @overload
-    def __init__(
-        self, output_type: Type[_Output], input_arg0: _Input, *, weak: bool = False
-    ) -> None:
+    def __init__(self, output_type: Type[_Output], input_arg0: _Input) -> None:
         ...
 
     @overload
@@ -146,8 +140,6 @@ def __init__(
         output_type: Type[_Output],
         input_arg0: Type[_Input],
         input_arg1: _Input,
-        *,
-        weak: bool = False,
     ) -> None:
         ...
 
@@ -156,15 +148,12 @@ def __init__(
         output_type: Type[_Output],
         input_arg0: Union[Type[_Input], _Input],
         input_arg1: Optional[_Input] = None,
-        *,
-        weak: bool = False,
     ) -> None:
         self.output_type = output_type
         self.input_type = self._validate_input_type(
             input_arg0 if input_arg1 is not None else type(input_arg0)
         )
         self.input = self._validate_input(input_arg1 if input_arg1 is not None else input_arg0)
-        self.weak = weak
 
         self._validate_output_type()
 


Test patch:
diff --git a/src/python/pants/engine/internals/scheduler_test.py b/src/python/pants/engine/internals/scheduler_test.py
--- a/src/python/pants/engine/internals/scheduler_test.py
+++ b/src/python/pants/engine/internals/scheduler_test.py
@@ -5,7 +5,7 @@
 from contextlib import contextmanager
 from dataclasses import dataclass
 from textwrap import dedent
-from typing import Any, FrozenSet
+from typing import Any
 
 from pants.engine.internals.engine_testutil import (
     assert_equal_with_printing,
@@ -121,20 +121,6 @@ async def error_msg_test_rule(union_wrapper: UnionWrapper) -> UnionX:
     raise AssertionError("The statement above this one should have failed!")
 
 
-class BooleanDeps(FrozenSet[bool]):
-    pass
-
-
-@rule
-async def boolean_cycle(key: bool) -> BooleanDeps:
-    """A rule with exactly two instances (bool == two keys), which depend on one another weakly."""
-    deps = {key}
-    dep = await Get(BooleanDeps, bool, not key, weak=True)
-    if dep is not None:
-        deps.update(dep)
-    return BooleanDeps(deps)
-
-
 class TypeCheckFailWrapper:
     """This object wraps another object which will be used to demonstrate a type check failure when
     the engine processes an `await Get(...)` statement."""
@@ -198,8 +184,6 @@ def rules(cls):
             QueryRule(A, (UnionWrapper,)),
             error_msg_test_rule,
             QueryRule(UnionX, (UnionWrapper,)),
-            boolean_cycle,
-            QueryRule(BooleanDeps, (bool,)),
             boolean_and_int,
             QueryRule(A, (int, bool)),
         )
@@ -245,10 +229,6 @@ def test_strict_equals(self):
         # type of a value in equality.
         assert A() == self.request(A, [1, True])
 
-    def test_weak_gets(self):
-        assert {True, False} == set(self.request(BooleanDeps, [True]))
-        assert {True, False} == set(self.request(BooleanDeps, [False]))
-
     @contextmanager
     def _assert_execution_error(self, expected_msg):
         with assert_execution_error(self, expected_msg):
diff --git a/src/rust/engine/graph/src/tests.rs b/src/rust/engine/graph/src/tests.rs
--- a/src/rust/engine/graph/src/tests.rs
+++ b/src/rust/engine/graph/src/tests.rs
@@ -11,7 +11,7 @@ use parking_lot::Mutex;
 use rand::{self, Rng};
 use tokio::time::{delay_for, timeout, Elapsed};
 
-use crate::{EdgeType, EntryId, Graph, InvalidationResult, Node, NodeContext, NodeError, RunToken};
+use crate::{EntryId, Graph, InvalidationResult, Node, NodeContext, NodeError};
 
 #[tokio::test]
 async fn create() {
@@ -84,7 +84,7 @@ async fn invalidate_and_rerun() {
 
   // Request with a different salt, which will cause both the middle and upper nodes to rerun since
   // their input values have changed.
-  let context = context.new_session(1).with_salt(1);
+  let context = context.new_run(1).with_salt(1);
   assert_eq!(
     graph.create(TNode::new(2), &context).await,
     Ok(vec![T(0, 0), T(1, 1), T(2, 1)])
@@ -136,8 +136,8 @@ async fn invalidate_randomly() {
   let graph = Arc::new(Graph::new());
 
   let invalidations = 10;
-  let sleep_per_invalidation = Duration::from_millis(500);
-  let range = 1000;
+  let sleep_per_invalidation = Duration::from_millis(100);
+  let range = 100;
 
   // Spawn a background thread to randomly invalidate in the relevant range. Hold its handle so
   // it doesn't detach.
@@ -236,6 +236,7 @@ async fn poll_cacheable() {
 
 #[tokio::test]
 async fn poll_uncacheable() {
+  let _logger = env_logger::try_init();
   let graph = Arc::new(Graph::new());
   // Create a context where the middle node is uncacheable.
   let context = {
@@ -289,7 +290,7 @@ async fn dirty_dependents_of_uncacheable_node() {
   );
 
   // Re-request the root in a new session and confirm that only the bottom node re-runs.
-  let context = context.new_session(1);
+  let context = context.new_run(1);
   assert_eq!(
     graph.create(TNode::new(2), &context).await,
     Ok(vec![T(0, 0), T(1, 0), T(2, 0)])
@@ -298,7 +299,7 @@ async fn dirty_dependents_of_uncacheable_node() {
 
   // Re-request with a new session and different salt, and confirm that everything re-runs bottom
   // up (the order of node cleaning).
-  let context = context.new_session(2).with_salt(1);
+  let context = context.new_run(2).with_salt(1);
   assert_eq!(
     graph.create(TNode::new(2), &context).await,
     Ok(vec![T(0, 1), T(1, 1), T(2, 1)])
@@ -338,13 +339,17 @@ async fn uncachable_node_only_runs_once() {
     graph.create(TNode::new(2), &context).await,
     Ok(vec![T(0, 0), T(1, 0), T(2, 0)])
   );
+  // TNode(0) and TNode(2) are cleared and dirtied (respectively) before completing, and
+  // so run twice each. But the uncacheable node runs once.
   assert_eq!(
-    context
-      .runs()
-      .into_iter()
-      .filter(|n| *n == TNode::new(1))
-      .count(),
-    1,
+    context.runs(),
+    vec![
+      TNode::new(2),
+      TNode::new(1),
+      TNode::new(0),
+      TNode::new(2),
+      TNode::new(0),
+    ]
   );
 }
 
@@ -457,6 +462,22 @@ async fn canceled_immediately() {
   );
 }
 
+#[tokio::test]
+async fn cyclic_failure() {
+  // Confirms that an attempt to create a cycle fails.
+  let graph = Arc::new(Graph::new());
+  let top = TNode::new(2);
+  let context = TContext::new(graph.clone()).with_dependencies(
+    // Request creation of a cycle by sending the bottom most node to the top.
+    vec![(TNode::new(0), Some(top))].into_iter().collect(),
+  );
+
+  assert_eq!(
+    graph.create(TNode::new(2), &context).await,
+    Err(TError::Cyclic)
+  );
+}
+
 #[tokio::test]
 async fn cyclic_dirtying() {
   // Confirms that a dirtied path between two nodes is able to reverse direction while being
@@ -490,82 +511,6 @@ async fn cyclic_dirtying() {
   assert_eq!(res, Ok(vec![T(1, 1), T(2, 1)]));
 }
 
-#[tokio::test]
-async fn cyclic_strong_strong() {
-  // A cycle between two nodes with strong edges.
-  let (graph, context) = cyclic_references(vec![]);
-  assert_eq!(
-    graph.create(TNode::new(1), &context).await,
-    Err(TError::Cyclic)
-  );
-  assert_eq!(
-    graph.create(TNode::new(0), &context).await,
-    Err(TError::Cyclic)
-  );
-}
-
-#[tokio::test]
-async fn cyclic_strong_weak_with_strong_first() {
-  // A cycle between two nodes with a strong dep from top to bottom and a weak dep from bottom
-  // to top, where we enter from the top first.
-  let _logger = env_logger::try_init();
-
-  let (graph, context) = cyclic_references(vec![TNode::new(1)]);
-  assert_eq!(
-    graph.create(TNode::new(1), &context).await,
-    Ok(vec![T(0, 0), T(1, 0)])
-  );
-  assert_eq!(
-    graph.create(TNode::new(0), &context).await,
-    Ok(vec![T(0, 0), T(1, 0), T(0, 0)])
-  );
-}
-
-///
-/// TODO: Ignored due to https://github.com/pantsbuild/pants/issues/10229.
-///
-#[tokio::test]
-#[ignore]
-async fn cyclic_strong_weak_with_weak_first() {
-  // A cycle between two nodes with a strong dep from top to bottom and a weak dep from bottom
-  // to top, where we enter from the bottom first.
-  let (graph, context) = cyclic_references(vec![TNode::new(1)]);
-  assert_eq!(
-    graph.create(TNode::new(0), &context).await,
-    Ok(vec![T(0, 0), T(1, 0), T(0, 0)])
-  );
-  assert_eq!(
-    graph.create(TNode::new(1), &context).await,
-    Ok(vec![T(0, 0), T(1, 0)])
-  );
-}
-
-#[tokio::test]
-async fn cyclic_weak_weak() {
-  // A cycle between two nodes, both with weak edges.
-  let (graph, context) = cyclic_references(vec![TNode::new(0), TNode::new(1)]);
-  assert_eq!(
-    graph.create(TNode::new(1), &context).await,
-    Ok(vec![T(0, 0), T(1, 0)])
-  );
-  assert_eq!(
-    graph.create(TNode::new(0), &context).await,
-    Ok(vec![T(1, 0), T(0, 0)])
-  );
-}
-
-fn cyclic_references(weak: Vec<TNode>) -> (Arc<Graph<TNode>>, TContext) {
-  let graph = Arc::new(Graph::new());
-  let top = TNode::new(1);
-  let context = TContext::new(graph.clone())
-    .with_dependencies(
-      // Request creation of a cycle by sending the bottom most node to the top.
-      vec![(TNode::new(0), Some(top))].into_iter().collect(),
-    )
-    .with_weak(weak.into_iter().collect());
-  (graph, context)
-}
-
 #[tokio::test]
 async fn critical_path() {
   use super::entry::Entry;
@@ -622,9 +567,7 @@ async fn critical_path() {
     for (src, dst) in &deps {
       let src = inner.nodes[&node_key(src)];
       let dst = inner.nodes[&node_key(dst)];
-      inner
-        .pg
-        .add_edge(src, dst, (EdgeType::Strong, RunToken::initial()));
+      inner.pg.add_edge(src, dst, 1.0);
     }
   }
 
@@ -781,13 +724,11 @@ impl TNode {
 ///
 #[derive(Clone)]
 struct TContext {
-  session_id: usize,
+  run_id: usize,
   // A value that is included in every value computed by this context. Stands in for "the state of the
   // outside world". A test that wants to "change the outside world" and observe its effect on the
   // graph should change the salt to do so.
   salt: usize,
-  // When dependencies on these nodes are requested, those dependencies will be weak.
-  weak: Arc<HashSet<TNode>>,
   // A mapping from source to optional destination that drives what values each TNode depends on.
   // If there is no entry in this map for a node, then TNode::run will default to requesting
   // the next smallest node. Finally, if a None entry is present, a node will have no
@@ -798,39 +739,34 @@ struct TContext {
   graph: Arc<Graph<TNode>>,
   aborts: Arc<Mutex<Vec<TNode>>>,
   runs: Arc<Mutex<Vec<TNode>>>,
-  entry_id_and_run_token: Option<(EntryId, RunToken)>,
+  entry_id: Option<EntryId>,
 }
 impl NodeContext for TContext {
   type Node = TNode;
-  type SessionId = usize;
+  type RunId = usize;
 
-  fn clone_for(&self, entry_id: EntryId, run_token: RunToken) -> TContext {
+  fn clone_for(&self, entry_id: EntryId) -> TContext {
     TContext {
-      session_id: self.session_id,
+      run_id: self.run_id,
       salt: self.salt,
-      weak: self.weak.clone(),
       edges: self.edges.clone(),
       delays: self.delays.clone(),
       uncacheable: self.uncacheable.clone(),
       graph: self.graph.clone(),
       aborts: self.aborts.clone(),
       runs: self.runs.clone(),
-      entry_id_and_run_token: Some((entry_id, run_token)),
+      entry_id: Some(entry_id),
     }
   }
 
-  fn session_id(&self) -> &usize {
-    &self.session_id
+  fn run_id(&self) -> &usize {
+    &self.run_id
   }
 
   fn graph(&self) -> &Graph<TNode> {
     &self.graph
   }
 
-  fn entry_id_and_run_token(&self) -> Option<(EntryId, RunToken)> {
-    self.entry_id_and_run_token
-  }
-
   fn spawn<F>(&self, future: F)
   where
     F: Future<Output = ()> + Send + 'static,
@@ -843,24 +779,18 @@ impl NodeContext for TContext {
 impl TContext {
   fn new(graph: Arc<Graph<TNode>>) -> TContext {
     TContext {
-      session_id: 0,
+      run_id: 0,
       salt: 0,
-      weak: Arc::default(),
       edges: Arc::default(),
       delays: Arc::default(),
       uncacheable: Arc::default(),
       graph,
-      aborts: Arc::default(),
-      runs: Arc::default(),
-      entry_id_and_run_token: None,
+      aborts: Arc::new(Mutex::new(Vec::new())),
+      runs: Arc::new(Mutex::new(Vec::new())),
+      entry_id: None,
     }
   }
 
-  fn with_weak(mut self, weak: HashSet<TNode>) -> TContext {
-    self.weak = Arc::new(weak);
-    self
-  }
-
   fn with_dependencies(mut self, edges: HashMap<TNode, Option<TNode>>) -> TContext {
     self.edges = Arc::new(edges);
     self
@@ -881,8 +811,8 @@ impl TContext {
     self
   }
 
-  fn new_session(mut self, new_session_id: usize) -> TContext {
-    self.session_id = new_session_id;
+  fn new_run(mut self, new_run_id: usize) -> TContext {
+    self.run_id = new_run_id;
     {
       let mut runs = self.runs.lock();
       runs.clear();
@@ -895,17 +825,7 @@ impl TContext {
   }
 
   async fn get(&self, dst: TNode) -> Result<Vec<T>, TError> {
-    if self.weak.contains(&dst) {
-      Ok(
-        self
-          .graph
-          .get_weak(self, dst)
-          .await?
-          .unwrap_or_else(Vec::new),
-      )
-    } else {
-      self.graph.get(self, dst).await
-    }
+    self.graph.get(self.entry_id, self, dst).await
   }
 
   fn abort_guard(&self, node: TNode) -> AbortGuard {