Repository: googleapis/google-cloud-ruby (Ruby)

Issue:
[Pubsub] Introduce a minimum lease for ModifyAckDeadline
To ensure that the client libraries do not send very tight ack_deadlines leading to ack expiration, we will introduce a minimum ack_deadline for client library driven ModifyAckDeadline RPCs. 
This is similar to the maximum_duration_per_lease_extenstion or client library driven ModifyAckDeadline RPCs, which our client libraries already support. The default minimum ack_deadline will be 1 minute, however, the users will be able to update it to a desired value.

Introduce a new variable min_duration_per_lease_extension in [inventory](https://github.com/googleapis/google-cloud-ruby/blob/main/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/inventory.rb), make it readable from subscriber (add a method like max_duration, update stream_inventory and coerce_inventory)  and accept user set value when creating subscription with inventory. Default value of the field should be 0 and if exactly once delivery is enabled it should be updated to 60 if the user has not defined a value. Also update the value to 60 seconds whenever the exactly_once setting for a stream is enabled and back to 0 if disabled.
Use the new variable to determine the delay for [renewing the lease in inventory](https://github.com/googleapis/google-cloud-ruby/blob/dce568ad26944f59d7d8a130c210a83abf410b53/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/inventory.rb#L164).


Discussion:

Patch:
diff --git a/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber.rb b/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber.rb
index 401edd3a19d9..40f6ca15e7a5 100644
--- a/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber.rb
+++ b/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber.rb
@@ -331,6 +331,16 @@ def max_duration_per_lease_extension
           @inventory[:max_duration_per_lease_extension]
         end
 
+        ##
+        # The minimum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
+        # redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
+        #
+        # @return [Integer] The minimum number of seconds.
+        #
+        def min_duration_per_lease_extension
+          @inventory[:min_duration_per_lease_extension]
+        end
+
         ##
         # @private
         def stream_inventory
@@ -339,6 +349,7 @@ def stream_inventory
             bytesize:                         @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
             extension:                        @inventory[:max_total_lease_duration],
             max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
+            min_duration_per_lease_extension: @inventory[:min_duration_per_lease_extension],
             use_legacy_flow_control:          @inventory[:use_legacy_flow_control]
           }
         end
@@ -394,6 +405,7 @@ def coerce_inventory inventory
           @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
           @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
           @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
+          @inventory[:min_duration_per_lease_extension] = Integer(@inventory[:min_duration_per_lease_extension] || 0)
           @inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
         end
 
diff --git a/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/inventory.rb b/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/inventory.rb
index 5417455f1884..5ce9ff7e5d3c 100644
--- a/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/inventory.rb
+++ b/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/inventory.rb
@@ -35,16 +35,18 @@ def self.from rec_msg
           attr_reader :bytesize
           attr_reader :extension
           attr_reader :max_duration_per_lease_extension
+          attr_accessor :min_duration_per_lease_extension
           attr_reader :use_legacy_flow_control
 
           def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
-                         use_legacy_flow_control:
+                         min_duration_per_lease_extension:, use_legacy_flow_control:
             super()
             @stream = stream
             @limit = limit
             @bytesize = bytesize
             @extension = extension
             @max_duration_per_lease_extension = max_duration_per_lease_extension
+            @min_duration_per_lease_extension = min_duration_per_lease_extension
             @use_legacy_flow_control = use_legacy_flow_control
             @inventory = {}
             @wait_cond = new_cond
@@ -162,6 +164,8 @@ def calc_target
           def calc_delay
             delay = (stream.subscriber.deadline - 3) * rand(0.8..0.9)
             delay = [delay, max_duration_per_lease_extension].min if max_duration_per_lease_extension.positive?
+            delay = [delay, min_duration_per_lease_extension].max if min_duration_per_lease_extension.positive? &&
+                                                                     stream.exactly_once_delivery_enabled
             delay
           end
         end
diff --git a/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/stream.rb b/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/stream.rb
index 945c32482945..ba7c00120b9d 100644
--- a/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/stream.rb
+++ b/google-cloud-pubsub/lib/google/cloud/pubsub/subscriber/stream.rb
@@ -47,7 +47,7 @@ class Stream
           attr_reader :sequencer
 
           ##
-          # @private Sequencer.
+          # @private exactly_once_delivery_enabled.
           attr_reader :exactly_once_delivery_enabled
 
           ##
@@ -249,8 +249,9 @@ def background_run
                 response = enum.next
                 new_exactly_once_delivery_enabled = response&.subscription_properties&.exactly_once_delivery_enabled
 
-                # Use synchronize so both changes happen atomically
+                # Use synchronize so changes happen atomically
                 synchronize do
+                  update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
                   @exactly_once_delivery_enabled = new_exactly_once_delivery_enabled unless new_exactly_once_delivery_enabled.nil? 
 
                   # Create receipt of received messages reception
@@ -293,6 +294,14 @@ def background_run
 
           # rubocop:enable all
 
+          # Updates min_duration_per_lease_extension to 60 when exactly_once_delivery_enabled
+          # and reverts back to default 0 when disabled.
+          # Skips if exactly_once_enabled is not modified.
+          def update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
+            return if new_exactly_once_delivery_enabled == @exactly_once_delivery_enabled
+            @inventory.min_duration_per_lease_extension = new_exactly_once_delivery_enabled ? 60 : 0
+          end
+
           def register_callback rec_msg
             if @sequencer
               # Add the message to the sequencer to invoke the callback.


Test patch:
diff --git a/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/inventory_test.rb b/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/inventory_test.rb
index 5ef8015b73b0..e82a2487094b 100644
--- a/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/inventory_test.rb
+++ b/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/inventory_test.rb
@@ -13,6 +13,7 @@
 # limitations under the License.
 
 require "helper"
+require "ostruct"
 
 describe Google::Cloud::PubSub::Subscriber, :inventory, :mock_pubsub do
   let(:topic_name) { "topic-name-goes-here" }
@@ -165,9 +166,32 @@
     _(mod_ack_hash[60].sort).must_equal ["ack-id-1111", "ack-id-1112", "ack-id-1113"]
   end
 
+  it "calculates delay considering min_duration_per_lease_extension" do
+    subscriber_mock = Minitest::Mock.new
+    subscriber_mock.expect :subscriber, OpenStruct.new({:deadline => 60})
+    subscriber_mock.expect :exactly_once_delivery_enabled, true
+    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, 
+                                                                 limit: 2, 
+                                                                 bytesize: 100_000, 
+                                                                 extension: 3600, 
+                                                                 max_duration_per_lease_extension: 0, 
+                                                                 min_duration_per_lease_extension: 61,
+                                                                 use_legacy_flow_control: false
+
+    inventory.add rec_msg1_grpc
+    delay = inventory.send :calc_delay
+    assert_equal 61, delay
+  end
+
   it "knows its count limit" do
     subscriber_mock = Minitest::Mock.new
-    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, limit: 2, bytesize: 100_000, extension: 3600, max_duration_per_lease_extension: 0, use_legacy_flow_control: false
+    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, 
+                                                                 limit: 2, 
+                                                                 bytesize: 100_000, 
+                                                                 extension: 3600, 
+                                                                 max_duration_per_lease_extension: 0, 
+                                                                 min_duration_per_lease_extension: 0,
+                                                                 use_legacy_flow_control: false
 
     inventory.add rec_msg1_grpc
     _(inventory).wont_be :full?
@@ -179,7 +203,13 @@
 
   it "knows its bytesize limit" do
     subscriber_mock = Minitest::Mock.new
-    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, limit: 1000, bytesize: 100, extension: 3600, max_duration_per_lease_extension: 0, use_legacy_flow_control: false
+    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, 
+                                                                 limit: 1000, 
+                                                                 bytesize: 100, 
+                                                                 extension: 3600, 
+                                                                 max_duration_per_lease_extension: 0, 
+                                                                 min_duration_per_lease_extension: 0,
+                                                                 use_legacy_flow_control: false
 
     inventory.add rec_msg1_grpc
     _(inventory).wont_be :full?
@@ -191,7 +221,13 @@
 
   it "removes expired items" do
     subscriber_mock = Minitest::Mock.new
-    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, limit: 1000, bytesize: 100_000, extension: 3600, max_duration_per_lease_extension: 0, use_legacy_flow_control: false
+    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, 
+                                                                 limit: 1000, 
+                                                                 bytesize: 100_000, 
+                                                                 extension: 3600, 
+                                                                 max_duration_per_lease_extension: 0,
+                                                                 min_duration_per_lease_extension: 0,
+                                                                 use_legacy_flow_control: false
 
     expired_time = Time.now - 7200
 
@@ -209,8 +245,27 @@
 
   it "knows its max_duration_per_lease_extension limit" do
     subscriber_mock = Minitest::Mock.new
-    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, limit: 1000, bytesize: 100, extension: 3600, max_duration_per_lease_extension: 10, use_legacy_flow_control: false
+    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, 
+                                                                 limit: 1000,
+                                                                 bytesize: 100, 
+                                                                 extension: 3600, 
+                                                                 max_duration_per_lease_extension: 10, 
+                                                                 min_duration_per_lease_extension: 0,
+                                                                 use_legacy_flow_control: false
 
     _(inventory.max_duration_per_lease_extension).must_equal 10
   end
+
+  it "knows its min_duration_per_lease_extension limit" do
+    subscriber_mock = Minitest::Mock.new
+    inventory = Google::Cloud::PubSub::Subscriber::Inventory.new subscriber_mock, 
+                                                                 limit: 1000, 
+                                                                 bytesize: 100, 
+                                                                 extension: 3600, 
+                                                                 max_duration_per_lease_extension: 0, 
+                                                                 min_duration_per_lease_extension: 10, 
+                                                                 use_legacy_flow_control: false
+
+    _(inventory.min_duration_per_lease_extension).must_equal 10
+  end
 end
diff --git a/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/stream_test.rb b/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/stream_test.rb
index 2c15d5ea844d..f1825b835be5 100644
--- a/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/stream_test.rb
+++ b/google-cloud-pubsub/test/google/cloud/pubsub/subscriber/stream_test.rb
@@ -54,4 +54,68 @@
     subscriber.stop
     subscriber.wait!
   end
+
+  it "should update min_duration_per_lease_extension when exactly_once_delivery is modified" do
+    pull_res1 = Google::Cloud::PubSub::V1::StreamingPullResponse.new received_messages: [rec_msg1_grpc],
+                                                                     subscription_properties: {
+                                                                         exactly_once_delivery_enabled: true
+                                                                     }   
+    response_groups = [[pull_res1]]
+
+    stub = StreamingPullStub.new response_groups
+    called = false
+
+    subscription.service.mocked_subscriber = stub
+    subscriber = subscription.listen streams: 1 do |msg|
+      assert msg.subscription.exactly_once_delivery_enabled
+      called = true
+    end
+
+    subscriber.on_error do |error|
+      # raise error
+      p error
+    end
+
+    subscriber.start
+
+    subscriber_retries = 0
+    until called
+      fail "total number of calls were never made" if subscriber_retries > 100
+      subscriber_retries += 1
+      sleep 0.01
+    end
+
+    assert_equal subscriber.stream_pool.first.inventory.min_duration_per_lease_extension, 60 
+    subscriber.stop
+    subscriber.wait!
+  end
+
+  it "should not update min_duration_per_lease_extension when exactly_once_delivery is not modified" do
+    pull_res1 = Google::Cloud::PubSub::V1::StreamingPullResponse.new received_messages: [rec_msg1_grpc],
+                                                                     subscription_properties: {
+                                                                         exactly_once_delivery_enabled: false
+                                                                     }   
+    response_groups = [[pull_res1]]
+
+    stub = StreamingPullStub.new response_groups
+    called = false
+
+    subscription.service.mocked_subscriber = stub
+    subscriber = subscription.listen streams: 1 do |msg|
+      called = true
+    end
+
+    subscriber.start
+
+    subscriber_retries = 0
+    until called
+      fail "total number of calls were never made" if subscriber_retries > 100
+      subscriber_retries += 1
+      sleep 0.01
+    end
+
+    assert_equal subscriber.stream_pool.first.inventory.min_duration_per_lease_extension, 0 
+    subscriber.stop
+    subscriber.wait!
+  end
 end
diff --git a/google-cloud-pubsub/test/google/cloud/pubsub/subscriber_test.rb b/google-cloud-pubsub/test/google/cloud/pubsub/subscriber_test.rb
index 8200e7bf114a..9a1ef5258872 100644
--- a/google-cloud-pubsub/test/google/cloud/pubsub/subscriber_test.rb
+++ b/google-cloud-pubsub/test/google/cloud/pubsub/subscriber_test.rb
@@ -54,7 +54,8 @@
     _(subscriber.inventory_extension).must_equal 3600 # deprecated Use #max_total_lease_duration.
     _(subscriber.max_total_lease_duration).must_equal 3600
     _(subscriber.max_duration_per_lease_extension).must_equal 0
-    _(subscriber.stream_inventory).must_equal({limit: 500, bytesize: 50000000, max_duration_per_lease_extension: 0, extension: 3600, use_legacy_flow_control: false})
+    _(subscriber.min_duration_per_lease_extension).must_equal 0
+    _(subscriber.stream_inventory).must_equal({limit: 500, bytesize: 50000000, max_duration_per_lease_extension: 0, min_duration_per_lease_extension: 0, extension: 3600, use_legacy_flow_control: false})
     _(subscriber.callback_threads).must_equal 8
     _(subscriber.push_threads).must_equal 4
   end
@@ -73,7 +74,8 @@
     _(subscriber.inventory_extension).must_equal 3600 # deprecated Use #max_total_lease_duration.
     _(subscriber.max_total_lease_duration).must_equal 3600
     _(subscriber.max_duration_per_lease_extension).must_equal 0
-    _(subscriber.stream_inventory).must_equal({limit: 250, bytesize: 12500000, max_duration_per_lease_extension: 0, extension: 3600, use_legacy_flow_control: false})
+    _(subscriber.min_duration_per_lease_extension).must_equal 0
+    _(subscriber.stream_inventory).must_equal({limit: 250, bytesize: 12500000, max_duration_per_lease_extension: 0, min_duration_per_lease_extension: 0, extension: 3600, use_legacy_flow_control: false})
     _(subscriber.callback_threads).must_equal callback_threads
     _(subscriber.push_threads).must_equal push_threads
 
@@ -92,6 +94,6 @@
     )
     _(subscriber.max_outstanding_messages).must_equal 999
     _(subscriber.use_legacy_flow_control?).must_equal true
-    _(subscriber.stream_inventory).must_equal({limit: 500, bytesize: 50000000, max_duration_per_lease_extension: 0, extension: 3600, use_legacy_flow_control: true})
+    _(subscriber.stream_inventory).must_equal({limit: 500, bytesize: 50000000, max_duration_per_lease_extension: 0, min_duration_per_lease_extension: 0, extension: 3600, use_legacy_flow_control: true})
   end
 end
diff --git a/google-cloud-pubsub/test/google/cloud/pubsub/subscription/listen_test.rb b/google-cloud-pubsub/test/google/cloud/pubsub/subscription/listen_test.rb
index fae7c1e6e327..374b8dc52ec3 100644
--- a/google-cloud-pubsub/test/google/cloud/pubsub/subscription/listen_test.rb
+++ b/google-cloud-pubsub/test/google/cloud/pubsub/subscription/listen_test.rb
@@ -230,6 +230,26 @@
     _(subscriber.use_legacy_flow_control?).must_equal false
   end
 
+  it "will set inventory min_duration_per_lease_extension while creating a Subscriber" do
+    subscriber = subscription.listen inventory: { min_duration_per_lease_extension: 10 } do |msg|
+      puts msg.msg_id
+    end
+    _(subscriber).must_be_kind_of Google::Cloud::PubSub::Subscriber
+    _(subscriber.subscription_name).must_equal subscription.name
+    _(subscriber.deadline).must_equal 60
+    _(subscriber.streams).must_equal 2
+    _(subscriber.inventory).must_equal 1000
+    _(subscriber.inventory_limit).must_equal 1000
+    _(subscriber.max_outstanding_messages).must_equal 1000
+    _(subscriber.inventory_bytesize).must_equal 100000000
+    _(subscriber.max_outstanding_bytes).must_equal 100000000
+    _(subscriber.inventory_extension).must_equal 3600
+    _(subscriber.max_total_lease_duration).must_equal 3600
+    _(subscriber.max_duration_per_lease_extension).must_equal 0
+    _(subscriber.min_duration_per_lease_extension).must_equal 10
+    _(subscriber.use_legacy_flow_control?).must_equal false
+  end
+
   it "will use inventory use_legacy_flow_control while creating a Subscriber" do
     subscriber = subscription.listen inventory: { use_legacy_flow_control: true } do |msg|
       puts msg.msg_id