Repository: SpeciesFileGroup/taxonworks
Pull request #4326: Slow cached map item translates
Author: login: kleintom
Created: 2025-05-05T14:10:22Z | Merged: 2025-05-15T21:29:38Z

This fixes the gazetteer branch being 3x slower than old development on recreating the cached map index. On OSF project locally it's now:
gaz branch: 1h2m
old dev branch: 1h20m

Side note: I noticed that most of the time creating translations for ADs for OSF (at least the version of OSF I have) is spent on just _one_ geographic_item: North America (61718). Checking for intersection with 3,669 ne_states GAs takes a _long_ time in that case. No fix for that yet.


Diff:
diff --git a/app/controllers/tasks/geographic_areas/usage_controller.rb b/app/controllers/tasks/geographic_areas/usage_controller.rb
index f2c8f0cee6..67f9e99e16 100644
--- a/app/controllers/tasks/geographic_areas/usage_controller.rb
+++ b/app/controllers/tasks/geographic_areas/usage_controller.rb
@@ -4,7 +4,7 @@ class Tasks::GeographicAreas::UsageController < ApplicationController
   # GET
   def index
     if params[:geographic_area_id].present?
-      @geogrpahic_areas = GeographicArea.where(id: params[:geographic_area_id])
+      @geographic_areas = GeographicArea.where(id: params[:geographic_area_id])
     else
       @geographic_areas = GeographicArea.all
     end
diff --git a/app/models/cached_map_item.rb b/app/models/cached_map_item.rb
index a8708461f7..a0491a1993 100644
--- a/app/models/cached_map_item.rb
+++ b/app/models/cached_map_item.rb
@@ -169,9 +169,16 @@ def self.translate_by_spatial_overlap(geographic_item_id, data_origin, buffer)
 
     # This is a fast first pass, pure intersection
     a = GeographicItem
-      .joins(:geographic_areas_geographic_items)
-      .where(geographic_areas_geographic_items: { data_origin: })
-      .merge(GeographicItem.intersecting(:multi_polygon, geographic_item_id))
+      .with(b: GeographicItem
+        .joins(:geographic_areas_geographic_items)
+        .where(geographic_areas_geographic_items: { data_origin: })
+      )
+      .from('b')
+      .select('b.*')
+      .where(
+        'ST_Intersects(b.geography, (SELECT geography FROM geographic_items ' \
+        'WHERE geographic_items.id = ?))', geographic_item_id
+      )
       .pluck(:id)
 
     return a if buffer.nil?
@@ -191,6 +198,10 @@ def self.translate_by_spatial_overlap(geographic_item_id, data_origin, buffer)
   #   area (and so has a chance of already being associated with a cached map
   #   item/translation)
   #
+  # @param search_existing_translates [Boolean]
+  #   true if existing cached_map_item_translations should be checked for a
+  #   match.
+  #
   # @param data_origin Array, String
   #   like `ne_states` or ['ne_states, 'ne_countries']
   #
@@ -199,7 +210,8 @@ def self.translate_by_spatial_overlap(geographic_item_id, data_origin, buffer)
   #   Typical use, do not apply for Georeferences, apply -10km for AssertedDistributions
   #
   def self.translate_geographic_item_id(
-    geographic_item_id, geographic_area_based, data_origin = nil, buffer = nil
+    geographic_item_id, geographic_area_based,
+    search_existing_translates = true, data_origin = nil, buffer = nil
   )
     return nil if data_origin.blank?
 
@@ -209,6 +221,13 @@ def self.translate_geographic_item_id(
 
     b = buffer
 
+    if search_existing_translates
+      a = translate_by_geographic_item_translation(
+        geographic_item_id, cached_map_type
+      )
+      return a if a.present?
+    end
+
     # All these methods depend on "prior knowledge" (not spatial calculations)
     if geographic_area_based
       a = translate_by_data_origin(geographic_item_id, data_origin)
@@ -280,19 +299,14 @@ def self.stubs(source_object, cached_map_type)
     if geographic_item_id
       h[:origin_geographic_item_id] = geographic_item_id
 
-      # We assume georefs won't match on an existing translation
-      if base_class_name != 'Georeference' &&
-         (a = translate_by_geographic_item_translation(geographic_item_id, cached_map_type)).present?
-        h[:geographic_item_id] = a
-      else
-        geographic_area_based = base_class_name == 'AssertedDistribution' &&
-          o.asserted_distribution_shape_type == 'GeographicArea'
-        h[:geographic_item_id] = translate_geographic_item_id(
-          geographic_item_id,
-          geographic_area_based,
-          cached_map_type.safe_constantize::SOURCE_GAZETEERS
-        )
-      end
+      geographic_area_based = base_class_name == 'AssertedDistribution' &&
+        o.asserted_distribution_shape_type == 'GeographicArea'
+      search_existing_translates = base_class_name == 'AssertedDistribution'
+
+      h[:geographic_item_id] = translate_geographic_item_id(
+        geographic_item_id, geographic_area_based, search_existing_translates,
+        cached_map_type.safe_constantize::SOURCE_GAZETEERS
+      )
 
       if h[:geographic_item_id].blank?
         h[:geographic_item_id] = [geographic_item_id]
diff --git a/app/models/collecting_event.rb b/app/models/collecting_event.rb
index 3891cc18fd..b394111fa6 100644
--- a/app/models/collecting_event.rb
+++ b/app/models/collecting_event.rb
@@ -668,9 +668,11 @@ def collecting_events_within_radius_of(distance)
       .where(GeographicItem.within_radius_of_item_sql(geographic_item_id, distance))
   end
 
+  # DEPRECATED (unused)
   # @return [Scope]
   # Find all (other) CEs which have GIs or EGIs (through georeferences) which intersect self
   def collecting_events_intersecting_with
+    # TODO may need to optimize through .intersecting
     pieces = GeographicItem.with_collecting_event_through_georeferences.intersecting('any', self.geographic_items.first.id).distinct
     gr     = [] # all collecting events for a geographic_item
 
diff --git a/app/models/geographic_item.rb b/app/models/geographic_item.rb
index 5447eee4c7..4032f1fd14 100644
--- a/app/models/geographic_item.rb
+++ b/app/models/geographic_item.rb
@@ -673,6 +673,7 @@ def with_collecting_event_through_georeferences
                           ).distinct
     end
 
+    # DEPRECATED (unused)
     # This is a geographic intersect, not geometric
     def intersecting(shape, *geographic_item_ids)
       shape = shape.to_s.downcase
diff --git a/app/views/tasks/geographic_items/debug/index.html.erb b/app/views/tasks/geographic_items/debug/index.html.erb
index 106c322531..185c80f4d3 100644
--- a/app/views/tasks/geographic_items/debug/index.html.erb
+++ b/app/views/tasks/geographic_items/debug/index.html.erb
@@ -49,7 +49,7 @@
     <div class="panel content separate-left">
       <h3> WebLevel1 (full translate) </h3>
       <% a = CachedMapItem.translate_geographic_item_id( @geographic_item.id,
-        true, # A fib, but should give the correct result in all cases
+        true, true, # A fib, but should give the correct result in all cases
         ['ne_states'])
       %>
       <div>
diff --git a/lib/tasks/maintenance/cached/maps.rake b/lib/tasks/maintenance/cached/maps.rake
index 8252571b9d..41c43aea05 100644
--- a/lib/tasks/maintenance/cached/maps.rake
+++ b/lib/tasks/maintenance/cached/maps.rake
@@ -103,9 +103,9 @@ namespace :tw do
             h = CachedMapItem.cached_map_name_hierarchy(o.geographic_item_id)
 
             z = CachedMapItem.where(geographic_item_id: o.geographic_item_id)
-              .where.not(untranslated: true)
+              .where(untranslated: [nil, false])
 
-            puts 'Size: ' + z.size.to_s
+            #puts 'Size: ' + z.size.to_s
 
             z.update_all(
               level0_geographic_name: h[:country],
@@ -113,8 +113,8 @@ namespace :tw do
               level2_geographic_name: h[:county]
             )
 
-            puts o.geographic_item_id
-            puts h
+            #puts o.geographic_item_id
+            #puts h
           end
           puts 'Done labelling cached map items.'
         end
@@ -255,16 +255,20 @@ namespace :tw do
           ids_in__ga.sort!
           ids_in__gz.sort!
 
-          Parallel.each(ids_in__ga, progress: 'build_cached_map_item_translations', in_processes: cached_rebuild_processes ) do |id|
+          Parallel.each(ids_in__ga, progress: 'build_cached_map_item_translations GA', in_processes: cached_rebuild_processes ) do |id|
             reconnected ||= CachedMapItemTranslation.connection.reconnect! || true
             process_asserted_distribution_translation(id, true)
           end
 
-          Parallel.each(ids_in__gz, progress: 'build_cached_map_item_translations', in_processes: cached_rebuild_processes ) do |id|
+          puts 'Geographic Area-based Asserted Distributions done.'
+
+          Parallel.each(ids_in__gz, progress: 'build_cached_map_item_translations GZ', in_processes: cached_rebuild_processes ) do |id|
             reconnected ||= CachedMapItemTranslation.connection.reconnect! || true
             process_asserted_distribution_translation(id, false)
           end
 
+          puts 'Gazetteer-based Asserted Distributions done.'
+
           puts 'Done.'
         end
 
@@ -277,7 +281,7 @@ namespace :tw do
           begin
             #  print "#{id}: "
             t = CachedMapItem.translate_geographic_item_id(
-              geographic_item_id, geographic_area_based, ['ne_states']
+              geographic_item_id, geographic_area_based, false, ['ne_states']
             )
             # if t.present?
             #   print t.join(', ')
diff --git a/spec/models/cached_map_item_spec.rb b/spec/models/cached_map_item_spec.rb
index d646b97bf2..cb2ea0d541 100644
--- a/spec/models/cached_map_item_spec.rb
+++ b/spec/models/cached_map_item_spec.rb
@@ -4,12 +4,12 @@
 
   include_context 'cached map scenario'
 
-  specify '#translate_geographic_item_id' do
-    expect(CachedMapItem.translate_geographic_item_id(gi2.id, true,  ['ne_states'])).to contain_exactly(gi1.id)
+  specify '#translate_geographic_item_id 1' do
+    expect(CachedMapItem.translate_geographic_item_id(gi2.id, true, true, ['ne_states'])).to contain_exactly(gi1.id)
   end
 
-  specify '#translate_geographic_item_id' do
-    expect(CachedMapItem.translate_geographic_item_id(gi3.id, true, ['ne_states'])).to contain_exactly(gi1.id)
+  specify '#translate_geographic_item_id 2' do
+    expect(CachedMapItem.translate_geographic_item_id(gi3.id, true, true, ['ne_states'])).to contain_exactly(gi1.id)
   end
 
   context 'Gazetteer-backed asserted distributions' do
diff --git a/spec/models/geographic_item_spec.rb b/spec/models/geographic_item_spec.rb
index d665440629..69eaabaf43 100644
--- a/spec/models/geographic_item_spec.rb
+++ b/spec/models/geographic_item_spec.rb
@@ -643,7 +643,8 @@
         )
       end
 
-      specify 'does not work with arbitrary geometry collection' do
+      # This seems to work in recent pg?
+      xspecify 'does not work with arbitrary geometry collection' do
         pending 'ST_Covers fails when input GeometryCollection has a line intersecting a polygon\'s interior'
         # The same test as the previous only the geometry collection in the
         # first argument also contains a line intersecting the interior of


Reviews:
user: login: LocoDelAssembly
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T14:18:37Z
user: login: kleintom
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-05T14:23:39Z

Review comments:
diff_hunk: @@ -103,18 +103,18 @@ namespace :tw do
             h = CachedMapItem.cached_map_name_hierarchy(o.geographic_item_id)
 
             z = CachedMapItem.where(geographic_item_id: o.geographic_item_id)
-              .where.not(untranslated: true)
+              .where('untranslated IS NULL OR untranslated = FALSE')
path: lib/tasks/maintenance/cached/maps.rake
user: login: LocoDelAssembly
body: Per chance does `.where(untranslated: [nil, false])` provide similar behavior?

```
CachedMapItem.where(geographic_item_id: 3).where(untranslated: [nil, false]).to_sql
 => "SELECT \"cached_map_items\".* FROM \"cached_map_items\" WHERE \"cached_map_items\".\"geographic_item_id\" = 3 AND (\"cached_map_items\".\"untranslated\" = FALSE OR \"cached_map_items\".\"untranslated\" IS NULL)"
 ```
created_at: 2025-05-05T14:18:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -103,18 +103,18 @@ namespace :tw do
             h = CachedMapItem.cached_map_name_hierarchy(o.geographic_item_id)
 
             z = CachedMapItem.where(geographic_item_id: o.geographic_item_id)
-              .where.not(untranslated: true)
+              .where('untranslated IS NULL OR untranslated = FALSE')
path: lib/tasks/maintenance/cached/maps.rake
user: login: kleintom
body: Yah, that should be the same, thanks - I'll change.
created_at: 2025-05-05T14:23:38Z
author_association: CONTRIBUTOR

PR comments:
user: login: mjy
created_at: 2025-05-05T14:14:00Z
body: > North America (61718). Checking for intersection with 3,669 ne_states GAs takes a long time in that case. 

Maybe we should just hard code/exception that?  Though translations for non Gaz should be almost "fixed", and we're not that worried about time on those.


author_association: MEMBER
user: login: LocoDelAssembly
created_at: 2025-05-05T20:56:48Z
body: ```
3.3.7 :023 > GeographicItem
3.3.7 :024 >       .joins(:geographic_areas_geographic_items)
3.3.7 :025 >       .where(geographic_areas_geographic_items: { data_origin: })
3.3.7 :026 > .merge(GeographicItem.intersecting(:multi_polygon, geographic_item_id)).select(:id).explain
  GeographicItem Load (1648.3ms)  SELECT "geographic_items"."id" FROM "geographic_items" INNER JOIN "geographic_areas_geographic_items" ON "geographic_areas_geographic_items"."geographic_item_id" = "geographic_items"."id" WHERE "geographic_areas_geographic_items"."data_origin" = $1 AND ST_Intersects(CASE ST_GeometryType(("geographic_items"."geography")::geometry) WHEN 'ST_MultiPolygon' THEN "geographic_items"."geography" ELSE NULL END, (SELECT "geographic_items"."geography" FROM "geographic_items" WHERE "geographic_items"."id" = 1))  [["data_origin", "ne_countries"]]
 => 
EXPLAIN SELECT "geographic_items"."id" FROM "geographic_items" INNER JOIN "geographic_areas_geographic_items" ON "geographic_areas_geographic_items"."geographic_item_id" = "geographic_items"."id" WHERE "geographic_areas_geographic_items"."data_origin" = $1 AND ST_Intersects(CASE ST_GeometryType(("geographic_items"."geography")::geometry) WHEN 'ST_MultiPolygon' THEN "geographic_items"."geography" ELSE NULL END, (SELECT "geographic_items"."geography" FROM "geographic_items" WHERE "geographic_items"."id" = 1)) [["data_origin", "ne_countries"]]
                                                                        QUERY PLAN
----------------------------------------------------------------------------------------------------------------------------------------------------------
 Nested Loop  (cost=9.15..5304.08 rows=1 width=4)
   InitPlan 1 (returns $0)
     ->  Index Scan using geographic_items_pkey on geographic_items geographic_items_1  (cost=0.42..8.44 rows=1 width=7468)
           Index Cond: (id = 1)
   ->  Index Scan using index_geographic_areas_geographic_items_on_data_origin on geographic_areas_geographic_items  (cost=0.29..131.27 rows=252 width=4)
         Index Cond: ((data_origin)::text = 'ne_countries'::text)
   ->  Index Scan using geographic_items_pkey on geographic_items  (cost=0.42..20.48 rows=1 width=4)
         Index Cond: (id = geographic_areas_geographic_items.geographic_item_id)
         Filter: st_intersects(CASE st_geometrytype((geography)::geometry) WHEN 'ST_MultiPolygon'::text THEN geography ELSE NULL::geography END, $0)
```
_(replaced `pluck` with `select` so I can execute `explain`)_

Why is `CASE` needed when comparing interaction and why is limited to MultiPolygon only?

Full SQL code:
```sql
SELECT "geographic_items"."id"
FROM "geographic_items"
  INNER JOIN "geographic_areas_geographic_items" ON "geographic_areas_geographic_items"."geographic_item_id" = "geographic_items"."id"
WHERE "geographic_areas_geographic_items"."data_origin" = $1
  AND ST_Intersects(
    CASE
      ST_GeometryType(("geographic_items"."geography")::geometry)
      WHEN 'ST_MultiPolygon' THEN "geographic_items"."geography"
      ELSE NULL
    END,
    (
      SELECT "geographic_items"."geography"
      FROM "geographic_items"
      WHERE "geographic_items"."id" = 1
    )
  )
  ```

All above is equivalent to calling `CachedMapItem.translate_by_spatial_overlap(1, 'ne_countries', nil)` (or close).

Not totally sure, but it looks like it is not actually using spatial indexes, it is scanning over other indicies and still computing intersection one by one.
author_association: CONTRIBUTOR
user: login: kleintom
created_at: 2025-05-05T21:33:46Z
body: > Why is `CASE` needed when comparing interaction and why is limited to MultiPolygon only?

Here's pre-gaz code: https://github.com/SpeciesFileGroup/taxonworks/blob/c46bf0fd808170facdedabad1461b6f262f56c49/app/models/cached_map_item.rb#L172-L176

I think the idea was that GA GIs are always multi-polygons, and the query is "get all GA ne_state GIs that intersect geographic_item_id", so the CASE was restricting to multi-polygon GIs to reduce work (I guess) since it knew what we wanted had to be a multi-polygon.

The version you reference here - my first gaz version - was written when I was less comfortable with what I was doing and just aimed to replicate the previous code. Technically I did but for whatever reason the execution of the query changed and made it much slower - my latest version in this PR removes the multi_polygon CASE because it ensures that the intersection only ranges over the GA GIs of type ne_states, which are all automatically multi_polygon.
author_association: CONTRIBUTOR
user: login: LocoDelAssembly
created_at: 2025-05-06T00:00:12Z
body: Ah I see:
```ruby
taxon-works(prod)> GeographicItem.group(:type).count
=> {"GeographicItem::MultiPolygon"=>41462, "GeographicItem::Point"=>125651, "GeographicItem::Polygon"=>7559}
taxon-works(prod)> GeographicItem.joins(:geographic_areas_geographic_items).group(:type).count
=> {"GeographicItem::MultiPolygon"=>41462}
```
So indeed only MultiPolygon when tied to a geographic area.

Sorry, forgot to run experiments on this PR rather than `development` branch :facepalm: 
author_association: CONTRIBUTOR
user: login: LocoDelAssembly
created_at: 2025-05-06T00:47:47Z
body: With same example it runs a bit more than 10 times faster:

```
3.3.7 :018 > GeographicItem
3.3.7 :019 >       .with(b: GeographicItem
3.3.7 :020 >         .joins(:geographic_areas_geographic_items)
3.3.7 :021 >         .where(geographic_areas_geographic_items: { data_origin: })
3.3.7 :022 >       )
3.3.7 :023 >       .from('b')
3.3.7 :024 >       .select('b.*')
3.3.7 :025 >       .where(
3.3.7 :026 >         'ST_Intersects(b.geography, (SELECT geography FROM geographic_items ' \
3.3.7 :027 >         'WHERE geographic_items.id = ?))', geographic_item_id
3.3.7 :028 >       ).select(:id).explain
  GeographicItem Load (166.4ms)  WITH "b" AS (SELECT "geographic_items".* FROM "geographic_items" INNER JOIN "geographic_areas_geographic_items" ON "geographic_areas_geographic_items"."geographic_item_id" = "geographic_items"."id" WHERE "geographic_areas_geographic_items"."data_origin" = $1) SELECT b.*, "id" FROM b WHERE (ST_Intersects(b.geography, (SELECT geography FROM geographic_items WHERE geographic_items.id = $2)))  [["data_origin", "ne_countries"], [nil, 1]]
 => 
EXPLAIN WITH "b" AS (SELECT "geographic_items".* FROM "geographic_items" INNER JOIN "geographic_areas_geographic_items" ON "geographic_areas_geographic_items"."geographic_item_id" = "geographic_items"."id" WHERE "geographic_areas_geographic_items"."data_origin" = $1) SELECT b.*, "id" FROM b WHERE (ST_Intersects(b.geography, (SELECT geography FROM geographic_items WHERE geographic_items.id = $2))) [["data_origin", "ne_countries"], [nil, 1]]
                                                                        QUERY PLAN
----------------------------------------------------------------------------------------------------------------------------------------------------------
 Hash Join  (cost=294.02..425.67 rows=1 width=39258)
   Hash Cond: (geographic_areas_geographic_items.geographic_item_id = geographic_items.id)
   InitPlan 1 (returns $0)
     ->  Index Scan using geographic_items_pkey on geographic_items geographic_items_1  (cost=0.42..8.44 rows=1 width=7468)
           Index Cond: (id = 1)
   ->  Index Scan using index_geographic_areas_geographic_items_on_data_origin on geographic_areas_geographic_items  (cost=0.29..131.27 rows=252 width=4)
         Index Cond: ((data_origin)::text = 'ne_countries'::text)
   ->  Hash  (cost=285.08..285.08 rows=17 width=39254)
         ->  Index Scan using index_geographic_items_on_geography on geographic_items  (cost=0.28..285.08 rows=17 width=39254)
               Index Cond: (geography && $0)
               Filter: st_intersects(geography, $0)
(11 rows)
 
3.3.7 :029 > CachedMapItem.translate_by_spatial_overlap(1, 'ne_countries', nil)
  GeographicItem Pluck (165.2ms)  WITH "b" AS (SELECT "geographic_items".* FROM "geographic_items" INNER JOIN "geographic_areas_geographic_items" ON "geographic_areas_geographic_items"."geographic_item_id" = "geographic_items"."id" WHERE "geographic_areas_geographic_items"."data_origin" = $1) SELECT "id" FROM b WHERE (ST_Intersects(b.geography, (SELECT geography FROM geographic_items WHERE geographic_items.id = $2)))  [["data_origin", "ne_countries"], [nil, 1]]
 => [41, 144, 179, 186, 194, 1, 30432] 
```

An alternative but without any speed improvement:
```ruby
3.3.7 :042 > GeographicItem.
3.3.7 :043 >   joins(:geographic_areas_geographic_items).
3.3.7 :044 >   where(geographic_areas_geographic_items: { data_origin: }).
3.3.7 :045 > where('ST_Intersects(geography, (SELECT geography FROM geographic_items WHERE geographic_items.id = ?))', geographic_item_id).pluck(:id)
  GeographicItem Pluck (169.0ms)  SELECT "geographic_items"."id" FROM "geographic_items" INNER JOIN "geographic_areas_geographic_items" ON "geographic_areas_geographic_items"."geographic_item_id" = "geographic_items"."id" WHERE "geographic_areas_geographic_items"."data_origin" = $1 AND (ST_Intersects(geography, (SELECT geography FROM geographic_items WHERE geographic_items.id = $2)))  [["data_origin", "ne_countries"], [nil, 1]]
 => [41, 144, 179, 186, 194, 1, 30432] 
```
```ruby
    a = GeographicItem
        .joins(:geographic_areas_geographic_items)
        .where(geographic_areas_geographic_items: { data_origin: })
        .where('ST_Intersects(geography, (SELECT geography FROM geographic_items WHERE geographic_items.id = ?))', geographic_item_id)
```
I have also experimented with removing `pluck` and using `a.select(:id)` on second query when buffer is present. Took almost a minute just to compute `explain`, so two steps are really required.

Do you agree on merging @mjy?
author_association: CONTRIBUTOR