Repository: ruby/rubygems (Ruby)

Issue:
Edge case difference in behavior from the standard `Kernel.require`
There's a slight difference in behavior between our extended `require` and the standard `require` provided by ruby by default. Sometimes, `$LOADED_FEATURES` cache is not respected when `$LOAD_PATH` priority has changed.

The issue can be reproduced as follows:

```
$ ruby -v
ruby 2.7.0p0 (2019-12-25 revision 647ee6f091) [x86_64-linux]

$ gem list reline

*** LOCAL GEMS ***

reline (0.1.3, default: 0.1.2)

$ ruby --disable-gems -e 'require "reline"; puts Reline::VERSION; $LOAD_PATH.unshift "/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib"; puts require("reline")'
0.1.2
false

$ ruby -e 'require "reline"; puts Reline::VERSION; $LOAD_PATH.unshift "/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0"; puts require("reline")'
0.1.3
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/version.rb:2: warning: already initialized constant Reline::VERSION
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/version.rb:2: warning: previous definition of VERSION was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/config.rb:6: warning: already initialized constant Reline::Config::DEFAULT_PATH
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/config.rb:6: warning: previous definition of DEFAULT_PATH was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/config.rb:8: warning: already initialized constant Reline::Config::KEYSEQ_PATTERN
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/config.rb:8: warning: previous definition of KEYSEQ_PATTERN was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/config.rb:14: warning: already initialized constant Reline::Config::VARIABLE_NAMES
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/config.rb:14: warning: previous definition of VARIABLE_NAMES was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/config.rb:41: warning: already initialized constant Reline::Config::VARIABLE_NAME_SYMBOLS
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/config.rb:41: warning: previous definition of VARIABLE_NAME_SYMBOLS was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/key_actor/base.rb:2: warning: already initialized constant Reline::KeyActor::Base::MAPPING
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/key_actor/base.rb:2: warning: previous definition of MAPPING was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/key_actor/emacs.rb:2: warning: already initialized constant Reline::KeyActor::Emacs::MAPPING
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/key_actor/emacs.rb:2: warning: previous definition of MAPPING was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/key_actor/vi_command.rb:2: warning: already initialized constant Reline::KeyActor::ViCommand::MAPPING
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/key_actor/vi_command.rb:2: warning: previous definition of MAPPING was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/key_actor/vi_insert.rb:2: warning: already initialized constant Reline::KeyActor::ViInsert::MAPPING
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/key_actor/vi_insert.rb:2: warning: previous definition of MAPPING was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/kill_ring.rb:3: warning: already initialized constant Reline::KillRing::State::FRESH
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/kill_ring.rb:3: warning: previous definition of FRESH was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/kill_ring.rb:4: warning: already initialized constant Reline::KillRing::State::CONTINUED
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/kill_ring.rb:4: warning: previous definition of CONTINUED was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/kill_ring.rb:5: warning: already initialized constant Reline::KillRing::State::PROCESSED
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/kill_ring.rb:5: warning: previous definition of PROCESSED was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/kill_ring.rb:6: warning: already initialized constant Reline::KillRing::State::YANK
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/kill_ring.rb:6: warning: previous definition of YANK was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/kill_ring.rb:9: warning: already initialized constant Reline::KillRing::RingPoint
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/kill_ring.rb:9: warning: previous definition of RingPoint was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode.rb:2: warning: already initialized constant Reline::Unicode::EscapedPairs
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode.rb:2: warning: previous definition of EscapedPairs was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode.rb:36: warning: already initialized constant Reline::Unicode::EscapedChars
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode.rb:36: warning: previous definition of EscapedChars was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode/east_asian_width.rb:6: warning: already initialized constant Reline::Unicode::EastAsianWidth::TYPE_F
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode/east_asian_width.rb:6: warning: previous definition of TYPE_F was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode/east_asian_width.rb:13: warning: already initialized constant Reline::Unicode::EastAsianWidth::TYPE_H
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode/east_asian_width.rb:13: warning: previous definition of TYPE_H was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode/east_asian_width.rb:24: warning: already initialized constant Reline::Unicode::EastAsianWidth::TYPE_W
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode/east_asian_width.rb:24: warning: previous definition of TYPE_W was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode/east_asian_width.rb:139: warning: already initialized constant Reline::Unicode::EastAsianWidth::TYPE_NA
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode/east_asian_width.rb:139: warning: previous definition of TYPE_NA was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode/east_asian_width.rb:150: warning: already initialized constant Reline::Unicode::EastAsianWidth::TYPE_A
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode/east_asian_width.rb:150: warning: previous definition of TYPE_A was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/unicode/east_asian_width.rb:333: warning: already initialized constant Reline::Unicode::EastAsianWidth::TYPE_N
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/unicode/east_asian_width.rb:333: warning: previous definition of TYPE_N was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:21: warning: already initialized constant Reline::LineEditor::VI_MOTIONS
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:21: warning: previous definition of VI_MOTIONS was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:43: warning: already initialized constant Reline::LineEditor::CompletionState::NORMAL
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:43: warning: previous definition of NORMAL was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:44: warning: already initialized constant Reline::LineEditor::CompletionState::COMPLETION
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:44: warning: previous definition of COMPLETION was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:45: warning: already initialized constant Reline::LineEditor::CompletionState::MENU
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:45: warning: previous definition of MENU was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:46: warning: already initialized constant Reline::LineEditor::CompletionState::JOURNEY
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:46: warning: previous definition of JOURNEY was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:47: warning: already initialized constant Reline::LineEditor::CompletionState::MENU_WITH_PERFECT_MATCH
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:47: warning: previous definition of MENU_WITH_PERFECT_MATCH was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:48: warning: already initialized constant Reline::LineEditor::CompletionState::PERFECT_MATCH
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:48: warning: previous definition of PERFECT_MATCH was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:51: warning: redefining constant Struct::CompletionJourneyData
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:51: warning: already initialized constant Reline::LineEditor::CompletionJourneyData
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:51: warning: previous definition of CompletionJourneyData was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:52: warning: redefining constant Struct::MenuInfo
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:52: warning: already initialized constant Reline::LineEditor::MenuInfo
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:52: warning: previous definition of MenuInfo was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:54: warning: already initialized constant Reline::LineEditor::CSI_REGEXP
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:54: warning: previous definition of CSI_REGEXP was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:55: warning: already initialized constant Reline::LineEditor::OSC_REGEXP
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:55: warning: previous definition of OSC_REGEXP was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:56: warning: already initialized constant Reline::LineEditor::NON_PRINTING_START
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:56: warning: previous definition of NON_PRINTING_START was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:57: warning: already initialized constant Reline::LineEditor::NON_PRINTING_END
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:57: warning: previous definition of NON_PRINTING_END was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/line_editor.rb:58: warning: already initialized constant Reline::LineEditor::WIDTH_SCANNER
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/line_editor.rb:58: warning: previous definition of WIDTH_SCANNER was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:12: warning: already initialized constant Reline::FILENAME_COMPLETION_PROC
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline.rb:12: warning: previous definition of FILENAME_COMPLETION_PROC was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:13: warning: already initialized constant Reline::USERNAME_COMPLETION_PROC
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline.rb:13: warning: previous definition of USERNAME_COMPLETION_PROC was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:15: warning: redefining constant Struct::Key
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:15: warning: already initialized constant Reline::Key
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline.rb:15: warning: previous definition of Key was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:16: warning: already initialized constant Reline::CursorPos
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline.rb:16: warning: previous definition of CursorPos was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:19: warning: already initialized constant Reline::Core::ATTR_READER_NAMES
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline.rb:19: warning: previous definition of ATTR_READER_NAMES was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:409: warning: already initialized constant Reline::HISTORY
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline.rb:430: warning: previous definition of HISTORY was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/ansi.rb:4: warning: already initialized constant Reline::ANSI::RAW_KEYSTROKE_CONFIG
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/ansi.rb:12: warning: previous definition of RAW_KEYSTROKE_CONFIG was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline.rb:423: warning: already initialized constant Reline::IOGate
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline.rb:428: warning: previous definition of IOGate was here
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/2.7.0/reline/general_io.rb:4: warning: already initialized constant Reline::GeneralIO::RAW_KEYSTROKE_CONFIG
/home/deivid/.rbenv/versions/2.7.0/lib/ruby/gems/2.7.0/gems/reline-0.1.3/lib/reline/general_io.rb:12: warning: previous definition of RAW_KEYSTROKE_CONFIG was here
true
```

Ruby's require will respect a previously required file, not matter what changes are made to the `$LOAD_PATH`. Our monkeypatched require will not.

This issue was originally reported by @voxik at #3131 but since there are two separate bugs in there, I'm splitting the second one into its own ticket.

This issue is related to:

  - [ ] Network problems
  - [ ] Installing a library
  - [ ] Publishing a library
  - [ ] The command line `gem`
  - [x] Other

I will abide by the [code of conduct](https://github.com/rubygems/rubygems/blob/master/CODE_OF_CONDUCT.md).

Rubygems 3.1.x performance regression due to #1868
#1868 introduced code to scan through the $LOAD_PATH to test for the presence of files in the -I args of the ruby cli.

What I think was missed there is that the the $LOAD_PATH includes the lib directories of all of the activated gems:

e.g. from some debugging output of `puts $LOAD_PATH[0...load_path_insert_index]`:

```
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/test-kitchen-2.5.0/bin/../lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/concurrent-ruby-1.1.6/lib/concurrent-ruby
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/i18n-1.8.2/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/thread_safe-0.3.6/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/tzinfo-1.2.7/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/minitest-5.14.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/activesupport-5.2.4.2/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/public_suffix-4.0.5/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/addressable-2.7.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/mixlib-shellout-3.0.9/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/mixlib-cli-2.1.6/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/appbundler-0.13.2/lib
[...]
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/pry-byebug-3.9.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/slop-3.6.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/pry-remote-0.1.8/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/pry-stack_explorer-0.4.9.3/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/rexml-3.2.4/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/ruby-prof-1.3.2/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/extensions/x86_64-linux/2.7.0/ruby-prof-1.3.2
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/ruby-shadow-2.5.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/extensions/x86_64-linux/2.7.0/ruby-shadow-2.5.0
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/stove-7.1.6/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/yard-0.9.25/lib
```

It looks like what is happening is this is now an extra scan through every single activated gem, times the three suffixes which every require call has to go through in addition to all the processing that was there in rubygems 2.x

The difference is fairly substantial, this is rubygems 3.1.2 performance:

```
# time /opt/chef-workstation/bin/kitchen list
/opt/chef-workstation/bin/kitchen list  13.02s user 5.64s system 49% cpu 37.690 total
```

versus commenting out the code block from that PR (in its current form):

```
# time /opt/chef-workstation/bin/kitchen list
/opt/chef-workstation/bin/kitchen list  2.42s user 0.61s system 48% cpu 6.256 total
```

I think that if the file to be required is an internal gem that it is particularly badly performant because it has to fail this loop first going through all the gems, then it has to go through all the same logic as before until finally falling back to scanning through the internal ruby gems.

One solution might be to delete the now-duplicate second scan of activated gems, but it also seems like this loop is particularly slow for some reason (the 600% speedup after deleting it suggests something else is going on).

Another solution would be to tweak the loop so that it was only looping through the command line -I flags (as I think was intended, but I don't know enough about how rubygems handles the $LOAD_PATH).

Additionally, this is on ruby 2.7.1 so the UNTAINT call should be hitting the empty-proc side of that, so it shouldn't be untainting all the strings.
`require` changes behavior comparing to plain Ruby
~~~
$ for psych in psych{1..3}; do mkdir $psych; echo "puts '$psych'" > $psych/psych.rb; done

$ find .
.
./psych2
./psych2/psych.rb
./psych1
./psych1/psych.rb
./psych3
./psych3/psych.rb

$ ruby -e '(1..3).each {|psych| $:.unshift("psych#{psych}"); require "psych" }'
psych1
psych2
psych3

$ ruby --disable-gems -e '(1..3).each {|psych| $:.unshift("psych#{psych}"); require "psych" }'
psych1
~~~

I have mentioned this test first time in \[[1]\] and this might be very well duplicate or resolved by #3131 or #3491. I just think this is nice and clean example which would be worth of its test case, because RubyGems should not modify the behavior of original Ruby `require` method in this way.

[1]: https://bugs.ruby-lang.org/issues/16904#note-8


Discussion:


> Another solution would be to tweak the loop so that it was only looping through the command line -I flags (as I think was intended, but I don't know enough about how rubygems handles the $LOAD_PATH).

Yes, that's what was intended :grimacing:

Can you try this patch and see if it fixes the regression?

```diff
-        $LOAD_PATH[0...load_path_insert_index].each do |lp|
+        rubyopt_rubylib_load_path_index = load_path_insert_index - Gem.loaded_specs.values.count {|s| !s.default_gem? }
+
+        $LOAD_PATH[0...rubyopt_rubylib_load_path_index].each do |lp|

```
This is just a candidate that can be speed up further to not look into all loaded specs every time, but I think this initial version should already make things much faster for you?
I'm pretty sure I can make that fail because the user can twiddle manually with the $LOAD_PATH and even without that, in testing yesterday I managed to get a binstubs library path first in the $LOAD_PATH with the -I flags after it, and then all the rest of the gems coming later.
(think i may have more for you on Monday though because this is $DAYJOB on the weekend for me, and I need a solid break...)
> I'm pretty sure I can make that fail because the user can twiddle manually with the $LOAD_PATH and even without that, in testing yesterday I managed to get a binstubs library path first in the $LOAD_PATH with the -I flags after it, and then all the rest of the gems coming later.

I think it would work for both manually prepending (path overrides default gems) and appending (path does not override default gems) entries to the `$LOAD_PATH`. For other `$LOAD_PATH` manipulations the behaviour would be as undefined as it is right now.

But I might be missing something.
That code works and is much faster, but its still not quite accurate.

I threw some debugging in there and I'm still seeing the loop going through this bit of my $LOAD_PATH:

```
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/chef-bin-16.1.0/bin/../lib
/foo/bar
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/concurrent-ruby-1.1.6/lib/concurrent-ruby
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/i18n-1.8.2/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/thread_safe-0.3.6/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/tzinfo-1.2.7/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/activesupport-5.2.4.2/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/public_suffix-4.0.5/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/addressable-2.7.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/mixlib-shellout-3.0.9/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/mixlib-cli-2.1.6/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/appbundler-0.13.2/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/artifactory-3.0.13/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/ast-2.4.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/aws-eventstream-1.1.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/aws-partitions-1.316.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/jmespath-1.4.0/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/aws-sigv4-1.1.3/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/aws-sdk-core-3.96.0/lib
```

First line is the binstub itself, second like is from an `-I/foo/bar` I threw in the binstub, then there's a few very non-default gems that shouldn't be in that list.  That isn't truncated so there's only 17 extras coming from somewhere.

I tried just `Gem.loaded_specs.values.count` without removing the default gems and that ultimately counted more gems than in Gem.load_path_insert_index and ultimately produced `[0...-1]` ranges.

I suspect that you may be missing that stuff like native gems have at last two entries in the LOAD_PATH?

```
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/nokogiri-1.11.0.rc2/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/extensions/x86_64-linux/2.7.0/nokogiri-1.11.0.rc2
```
Yeah there's 15 `extensions` directories, plus this one:

```
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/google-api-client-0.34.1/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/google-api-client-0.34.1/generated
```
And must be one more that I'm missing.
yeah this one:

```
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/knife-cloud-3.0.1/lib
/opt/chef-workstation/embedded/lib/ruby/gems/2.7.0/gems/knife-cloud-3.0.1/spec
```

that's one of ours, i'm going to have to fix that one, that's terrible.
Wow, thanks for the investigation!

So I was indeed missing something: the fact that some gems add more than one path to the `$LOAD_PATH` when they get activated.

But it should be fixable, I'll give it a try tomorrow! :+1:
So yeah the only other thing is that binstub line that comes first, that's duplicated with the active gem later.  That seems correct to me though.  That is technically "manually" added to the LOAD_PATH by the binstub, and should be loaded at a higher priority than the `-I` flags, and higher priority than any rubygems or internal gems (i think).
I pushed an initial attempt here: https://github.com/rubygems/rubygems/compare/require_perf_regression, in case you want to give it a try. Will look into adding a test for it, but if you can double check the patch I appreciate it.
FInal PR is #3639!

Thanks! I confirm this is still an issue even with #3639 applied, so I'll have a look at it.



Patch:
diff --git a/lib/rubygems.rb b/lib/rubygems.rb
index 843cb49e4a32..ca80326459a9 100644
--- a/lib/rubygems.rb
+++ b/lib/rubygems.rb
@@ -590,22 +590,25 @@ def self.load_path_insert_index
 
     index = $LOAD_PATH.index RbConfig::CONFIG['sitelibdir']
 
-    index
+    index || 0
+  end
+
+  ##
+  # The number of paths in the `$LOAD_PATH` from activated gems. Used to
+  # prioritize `-I` and `ENV['RUBYLIB`]` entries during `require`.
+
+  def self.activated_gem_paths
+    @activated_gem_paths ||= 0
   end
 
   ##
   # Add a list of paths to the $LOAD_PATH at the proper place.
 
   def self.add_to_load_path(*paths)
-    insert_index = load_path_insert_index
+    @activated_gem_paths = activated_gem_paths + paths.size
 
-    if insert_index
-      # gem directories must come after -I and ENV['RUBYLIB']
-      $LOAD_PATH.insert(insert_index, *paths)
-    else
-      # we are probably testing in core, -I and RUBYLIB don't apply
-      $LOAD_PATH.unshift(*paths)
-    end
+    # gem directories must come after -I and ENV['RUBYLIB']
+    $LOAD_PATH.insert(Gem.load_path_insert_index, *paths)
   end
 
   @yaml_loaded = false
diff --git a/lib/rubygems/core_ext/kernel_require.rb b/lib/rubygems/core_ext/kernel_require.rb
index ed24111bd5d6..115ae0cb50fd 100644
--- a/lib/rubygems/core_ext/kernel_require.rb
+++ b/lib/rubygems/core_ext/kernel_require.rb
@@ -39,48 +39,40 @@ def require(path)
 
     path = path.to_path if path.respond_to? :to_path
 
-    # Ensure -I beats a default gem
-    # https://github.com/rubygems/rubygems/pull/1868
-    resolved_path = begin
-      rp = nil
-      Gem.suffixes.each do |s|
-        load_path_insert_index = Gem.load_path_insert_index
-        break unless load_path_insert_index
-
-        $LOAD_PATH[0...load_path_insert_index].each do |lp|
-          safe_lp = lp.dup.tap(&Gem::UNTAINT)
-          begin
-            if File.symlink? safe_lp # for backward compatibility
-              next
+    if spec = Gem.find_unresolved_default_spec(path)
+      # Ensure -I beats a default gem
+      resolved_path = begin
+        rp = nil
+        load_path_check_index = Gem.load_path_insert_index - Gem.activated_gem_paths
+        Gem.suffixes.each do |s|
+          $LOAD_PATH[0...load_path_check_index].each do |lp|
+            safe_lp = lp.dup.tap(&Gem::UNTAINT)
+            begin
+              if File.symlink? safe_lp # for backward compatibility
+                next
+              end
+            rescue SecurityError
+              RUBYGEMS_ACTIVATION_MONITOR.exit
+              raise
             end
-          rescue SecurityError
-            RUBYGEMS_ACTIVATION_MONITOR.exit
-            raise
-          end
 
-          full_path = File.expand_path(File.join(safe_lp, "#{path}#{s}"))
-          if File.file?(full_path)
-            rp = full_path
-            break
+            full_path = File.expand_path(File.join(safe_lp, "#{path}#{s}"))
+            if File.file?(full_path)
+              rp = full_path
+              break
+            end
           end
+          break if rp
         end
-        break if rp
+        rp
       end
-      rp
-    end
 
-    if resolved_path
-      RUBYGEMS_ACTIVATION_MONITOR.exit
-      return gem_original_require(resolved_path)
-    end
-
-    if spec = Gem.find_unresolved_default_spec(path)
       begin
         Kernel.send(:gem, spec.name, Gem::Requirement.default_prerelease)
       rescue Exception
         RUBYGEMS_ACTIVATION_MONITOR.exit
         raise
-      end
+      end unless resolved_path
     end
 
     # If there are no unresolved deps, then we can use just try


Test patch:
diff --git a/lib/rubygems/test_case.rb b/lib/rubygems/test_case.rb
index a05a2898d184..53dd495aeff0 100644
--- a/lib/rubygems/test_case.rb
+++ b/lib/rubygems/test_case.rb
@@ -384,6 +384,7 @@ def setup
     Gem::Security.reset
 
     Gem.loaded_specs.clear
+    Gem.instance_variable_set(:@activated_gem_paths, 0)
     Gem.clear_default_specs
     Bundler.reset!
 
diff --git a/test/rubygems/test_require.rb b/test/rubygems/test_require.rb
index f36892f8cc46..2b11e26dfeb5 100644
--- a/test/rubygems/test_require.rb
+++ b/test/rubygems/test_require.rb
@@ -45,6 +45,35 @@ def refute_require(path)
     refute require(path), "'#{path}' was not yet required"
   end
 
+  def test_respect_loaded_features_caching_like_standard_require
+    dir = Dir.mktmpdir("test_require", @tempdir)
+
+    lp1 = File.join dir, 'foo1'
+    foo1 = File.join lp1, 'foo.rb'
+
+    FileUtils.mkdir_p lp1
+    File.open(foo1, 'w') { |f| f.write "class Object; HELLO = 'foo1' end" }
+
+    lp = $LOAD_PATH.dup
+
+    $LOAD_PATH.unshift lp1
+    assert_require 'foo'
+    assert_equal "foo1", ::Object::HELLO
+
+    lp2 = File.join dir, 'foo2'
+    foo2 = File.join lp2, 'foo.rb'
+
+    FileUtils.mkdir_p lp2
+    File.open(foo2, 'w') { |f| f.write "class Object; HELLO = 'foo2' end" }
+
+    $LOAD_PATH.unshift lp2
+    refute_require 'foo'
+    assert_equal "foo1", ::Object::HELLO
+  ensure
+    $LOAD_PATH.replace lp
+    Object.send :remove_const, :HELLO if Object.const_defined? :HELLO
+  end
+
   # Providing -I on the commandline should always beat gems
   def test_dash_i_beats_gems
     a1 = util_spec "a", "1", {"b" => "= 1"}, "lib/test_gem_require_a.rb"
@@ -420,6 +449,17 @@ def test_default_gem_require_activates_just_once
     assert_equal 0, times_called
   end
 
+  def test_second_gem_require_does_not_resolve_path_manually_before_going_through_standard_require
+    a1 = util_spec "a", "1", nil, "lib/test_gem_require_a.rb"
+    install_gem a1
+
+    assert_require "test_gem_require_a"
+
+    stub(:gem_original_require, ->(path) { assert_equal "test_gem_require_a", path }) do
+      require "test_gem_require_a"
+    end
+  end
+
   def test_realworld_default_gem
     testing_ruby_repo = !ENV["GEM_COMMAND"].nil?
     skip "this test can't work under ruby-core setup" if testing_ruby_repo || java_platform?