Repository: spiral/framework (php)

Issue:
Slow collection of annotated routes
### Description

Every time app boots annotated routes (`@Route(...)`) are collected from php files (unless ROUTE_CACHE variable is set to true). Every php file is checked and every class method is parsed in this process, which is very slow for projects with 500+ php files. In my project with around 1000 files in it routes collection takes about 1.5-2 seconds and this is a lot of time, considering app reboots after every change in code ("reload" feature of RR). Collection process also slows CLI commands execution as app has to do it for every command, even not related to routes.

I'd suggest storing modification time for every class file in cache along with the associated routes.

### Additional Info

| Q                   | A
| ------------------- | ---
| Framework Version   | 2.8.3
| PHP version         | 8.0.6
| Operating system    | Linux


Pull request description:
Added ability to use scopes for indexing files with specific scopes
fixes #483

| Q             | A
| ------------- | ---
| Bugfix?       | ❌
| Breaks BC?    | ❌
| New feature?  | ✔️
| Issues        | #483


This PR adds the ability to configure specific scopes directories for tokenizer.


```php
final class RouteLocator
{
    public function __construct(private ScopedClassesInterface $locator)  {}

    public function findDeclarations(): array
    {
          foreach ($this->locator->getScopedClasses('routes') as $class) {
               // ...
          }
    }
}
```


```php
// tokenizer.php

return [
            'directories' => [
                 directory('app')
            ],
            'exclude' => [
                    directory('resources'),
                    directory('config'),
                    'tests',
                    'migrations',
            ],
            'scopes' => [
                'routes' => [
                    'directories' => [
                          directory('app/Controllers')
                    ],
                ],
               'consoleCommands' => [
                    'directories' => [
                          directory('app/Commands')
                    ],
                ],
            ]
];
```


Patch:
diff --git a/src/AnnotatedRoutes/src/RouteLocator.php b/src/AnnotatedRoutes/src/RouteLocator.php
index e33e6c13f..531c1d9cb 100644
--- a/src/AnnotatedRoutes/src/RouteLocator.php
+++ b/src/AnnotatedRoutes/src/RouteLocator.php
@@ -13,17 +13,16 @@ namespace Spiral\Router;
 
 use Spiral\Attributes\ReaderInterface;
 use Spiral\Router\Annotation\Route;
-use Spiral\Tokenizer\ClassLocator;
+use Spiral\Tokenizer\ScopedClassesInterface;
 
 final class RouteLocator
 {
-    /** @var ClassLocator */
-    private $locator;
+    private ScopedClassesInterface $locator;
 
     /** @var ReaderInterface */
     private $reader;
 
-    public function __construct(ClassLocator $locator, ReaderInterface $reader)
+    public function __construct(ScopedClassesInterface $locator, ReaderInterface $reader)
     {
         $this->locator = $locator;
         $this->reader = $reader;
@@ -47,7 +46,7 @@ final class RouteLocator
     public function findDeclarations(): array
     {
         $result = [];
-        foreach ($this->locator->getClasses() as $class) {
+        foreach ($this->locator->getScopedClasses('routes') as $class) {
             foreach ($class->getMethods() as $method) {
                 $route = $this->reader->firstFunctionMetadata($method, Route::class);
 
diff --git a/src/Framework/Command/Translator/IndexCommand.php b/src/Framework/Command/Translator/IndexCommand.php
index 0929b3780..2c0b2dc51 100644
--- a/src/Framework/Command/Translator/IndexCommand.php
+++ b/src/Framework/Command/Translator/IndexCommand.php
@@ -13,7 +13,7 @@ namespace Spiral\Command\Translator;
 
 use Spiral\Console\Command;
 use Spiral\Core\Container\SingletonInterface;
-use Spiral\Tokenizer\ClassesInterface;
+use Spiral\Tokenizer\ScopedClassesInterface;
 use Spiral\Tokenizer\InvocationsInterface;
 use Spiral\Translator\Catalogue\CatalogueManager;
 use Spiral\Translator\Config\TranslatorConfig;
@@ -32,13 +32,13 @@ final class IndexCommand extends Command implements SingletonInterface
      * @param TranslatorConfig     $config
      * @param CatalogueManager     $manager
      * @param InvocationsInterface $invocations
-     * @param ClassesInterface     $classes
+     * @param ScopedClassesInterface $classes
      */
     public function perform(
         TranslatorConfig $config,
         CatalogueManager $manager,
         InvocationsInterface $invocations,
-        ClassesInterface $classes
+        ScopedClassesInterface $classes
     ): void {
         $catalogue = $manager->load(
             $this->argument('locale') ?? $config->getDefaultLocale()
diff --git a/src/Framework/Console/CommandLocator.php b/src/Framework/Console/CommandLocator.php
index b70cfbb0c..a4911c7d6 100644
--- a/src/Framework/Console/CommandLocator.php
+++ b/src/Framework/Console/CommandLocator.php
@@ -13,24 +13,19 @@ namespace Spiral\Console;
 
 use Psr\Container\ContainerInterface;
 use Spiral\Console\Traits\LazyTrait;
-use Spiral\Tokenizer\ClassesInterface;
+use Spiral\Tokenizer\ScopedClassesInterface;
 use Symfony\Component\Console\Command\Command as SymfonyCommand;
 
 final class CommandLocator implements LocatorInterface
 {
     use LazyTrait;
 
-    /** @var ClassesInterface */
-    private $classes;
+    private ScopedClassesInterface $classes;
 
     /** @var ContainerInterface */
     private $container;
 
-    /**
-     * @param ClassesInterface   $classes
-     * @param ContainerInterface $container
-     */
-    public function __construct(ClassesInterface $classes, ContainerInterface $container)
+    public function __construct(ScopedClassesInterface $classes, ContainerInterface $container)
     {
         $this->classes = $classes;
         $this->container = $container;
@@ -42,7 +37,7 @@ final class CommandLocator implements LocatorInterface
     public function locateCommands(): array
     {
         $commands = [];
-        foreach ($this->classes->getClasses(SymfonyCommand::class) as $class) {
+        foreach ($this->classes->getScopedClasses('consoleCommands', SymfonyCommand::class) as $class) {
             if ($class->isAbstract()) {
                 continue;
             }
diff --git a/src/Prototype/src/PrototypeLocator.php b/src/Prototype/src/PrototypeLocator.php
index 1ed32230b..f5fe6f821 100644
--- a/src/Prototype/src/PrototypeLocator.php
+++ b/src/Prototype/src/PrototypeLocator.php
@@ -12,14 +12,13 @@ declare(strict_types=1);
 namespace Spiral\Prototype;
 
 use Spiral\Prototype\Traits\PrototypeTrait;
-use Spiral\Tokenizer\ClassesInterface;
+use Spiral\Tokenizer\ScopedClassesInterface;
 
 final class PrototypeLocator
 {
-    /** @var ClassesInterface */
-    private $classes;
+    private ScopedClassesInterface $classes;
 
-    public function __construct(ClassesInterface $classes)
+    public function __construct(ScopedClassesInterface $classes)
     {
         $this->classes = $classes;
     }
@@ -31,6 +30,6 @@ final class PrototypeLocator
      */
     public function getTargetClasses(): array
     {
-        return $this->classes->getClasses(PrototypeTrait::class);
+        return $this->classes->getScopedClasses('prototypes', PrototypeTrait::class);
     }
 }
diff --git a/src/Tokenizer/src/Bootloader/TokenizerBootloader.php b/src/Tokenizer/src/Bootloader/TokenizerBootloader.php
index 8f5f3ab5f..4aa106b2c 100644
--- a/src/Tokenizer/src/Bootloader/TokenizerBootloader.php
+++ b/src/Tokenizer/src/Bootloader/TokenizerBootloader.php
@@ -21,12 +21,15 @@ use Spiral\Tokenizer\ClassesInterface;
 use Spiral\Tokenizer\ClassLocator;
 use Spiral\Tokenizer\InvocationLocator;
 use Spiral\Tokenizer\InvocationsInterface;
+use Spiral\Tokenizer\ScopedClassesInterface;
+use Spiral\Tokenizer\ScopedClassLocator;
 use Spiral\Tokenizer\Tokenizer;
 
 final class TokenizerBootloader extends Bootloader implements SingletonInterface
 {
     protected const BINDINGS = [
-        ClassesInterface::class     => ClassLocator::class,
+        ScopedClassesInterface::class => ScopedClassLocator::class,
+        ClassesInterface::class => ClassLocator::class,
         InvocationsInterface::class => InvocationLocator::class,
     ];
 
diff --git a/src/Tokenizer/src/Config/TokenizerConfig.php b/src/Tokenizer/src/Config/TokenizerConfig.php
index 876606d67..45e2a1b37 100644
--- a/src/Tokenizer/src/Config/TokenizerConfig.php
+++ b/src/Tokenizer/src/Config/TokenizerConfig.php
@@ -20,12 +20,11 @@ final class TokenizerConfig extends InjectableConfig
 {
     public const CONFIG = 'tokenizer';
 
-    /**
-     * @var array
-     */
+    /** @var array<non-empty-string, array<int, non-empty-string>> */
     protected $config = [
         'directories' => [],
-        'exclude'     => [],
+        'exclude' => [],
+        'scopes' => [],
     ];
 
     public function getDirectories(): array
@@ -37,4 +36,18 @@ final class TokenizerConfig extends InjectableConfig
     {
         return $this->config['exclude'] ?? ['vendor', 'tests'];
     }
+
+    /**
+     * @return array{directories: array<string>, exclude: array<string>}
+     */
+    public function getScope(string $scope): array
+    {
+        $directories = $this->config['scopes'][$scope]['directories'] ?? $this->getDirectories();
+        $excludes = $this->config['scopes'][$scope]['exclude'] ?? $this->getExcludes();
+
+        return [
+            'directories' => $directories,
+            'exclude' => $excludes,
+        ];
+    }
 }
diff --git a/src/Tokenizer/src/ScopedClassLocator.php b/src/Tokenizer/src/ScopedClassLocator.php
new file mode 100644
index 000000000..d54be3579
--- /dev/null
+++ b/src/Tokenizer/src/ScopedClassLocator.php
@@ -0,0 +1,20 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Spiral\Tokenizer;
+
+final class ScopedClassLocator implements ScopedClassesInterface
+{
+    private Tokenizer $tokenizer;
+
+    public function __construct(Tokenizer $tokenizer)
+    {
+        $this->tokenizer = $tokenizer;
+    }
+
+    public function getScopedClasses(string $scope, $target = null): array
+    {
+        return $this->tokenizer->scopedClassLocator($scope)->getClasses($target);
+    }
+}
diff --git a/src/Tokenizer/src/ScopedClassesInterface.php b/src/Tokenizer/src/ScopedClassesInterface.php
new file mode 100644
index 000000000..58170aec6
--- /dev/null
+++ b/src/Tokenizer/src/ScopedClassesInterface.php
@@ -0,0 +1,21 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Spiral\Tokenizer;
+
+interface ScopedClassesInterface
+{
+    /**
+     * Index all available files and generate list of found classes for given scope with their names and filenames.
+     * Unreachable classes or files with conflicts must be skipped. This is SLOW method, should be
+     * used only for static analysis.
+     *
+     * @param string $scope  Scope name. If scope is not exist, global settings will be used.
+     * @param mixed $target  Class, interface or trait parent. By default - null (all classes).
+     *                       Parent (class) will also be included to classes list as one of
+     *                       results.
+     * @return \ReflectionClass[]
+     */
+    public function getScopedClasses(string $scope, $target = null): array;
+}
diff --git a/src/Tokenizer/src/Tokenizer.php b/src/Tokenizer/src/Tokenizer.php
index 7e4451afd..fb622a523 100644
--- a/src/Tokenizer/src/Tokenizer.php
+++ b/src/Tokenizer/src/Tokenizer.php
@@ -40,6 +40,16 @@ final class Tokenizer implements SingletonInterface, InjectorInterface
         $this->config = $config;
     }
 
+    /**
+     * Get pre-configured class locator for specific scope.
+     */
+    public function scopedClassLocator(string $scope): ClassesInterface
+    {
+        $dirs = $this->config->getScope($scope);
+
+        return $this->classLocator($dirs['directories'], $dirs['exclude']);
+    }
+
     /**
      * Get pre-configured class locator.
      */
diff --git a/src/Translator/src/Indexer.php b/src/Translator/src/Indexer.php
index 9f1e4af2d..6d718db02 100644
--- a/src/Translator/src/Indexer.php
+++ b/src/Translator/src/Indexer.php
@@ -12,7 +12,7 @@ declare(strict_types=1);
 namespace Spiral\Translator;
 
 use Spiral\Logger\Traits\LoggerTrait;
-use Spiral\Tokenizer\ClassesInterface;
+use Spiral\Tokenizer\ScopedClassesInterface;
 use Spiral\Tokenizer\InvocationsInterface;
 use Spiral\Tokenizer\Reflection\ReflectionArgument;
 use Spiral\Tokenizer\Reflection\ReflectionInvocation;
@@ -70,9 +70,9 @@ final class Indexer
      * Index and register i18n string located in default properties which belongs to TranslatorTrait
      * classes.
      */
-    public function indexClasses(ClassesInterface $locator): void
+    public function indexClasses(ScopedClassesInterface $locator): void
     {
-        foreach ($locator->getClasses(TranslatorTrait::class) as $class) {
+        foreach ($locator->getScopedClasses('translations', TranslatorTrait::class) as $class) {
             $strings = $this->fetchMessages($class, true);
             foreach ($strings as $string) {
                 $this->registerMessage($class->getName(), $string);


Test patch:
diff --git a/src/Console/tests/LazyTest.php b/src/Console/tests/LazyTest.php
index 9ec51af16..24ad880aa 100644
--- a/src/Console/tests/LazyTest.php
+++ b/src/Console/tests/LazyTest.php
@@ -14,6 +14,7 @@ use Spiral\Console\CommandLocator;
 use Spiral\Console\StaticLocator;
 use Spiral\Tests\Console\Fixtures\LazyLoadedCommand;
 use Spiral\Tokenizer\ClassesInterface;
+use Spiral\Tokenizer\ScopedClassesInterface;
 use Symfony\Component\Console\Command\LazyCommand;
 
 class LazyTest extends BaseTest
@@ -21,8 +22,8 @@ class LazyTest extends BaseTest
     public function testLazyCommandCreationInCommandLocator(): void
     {
         $locator = new CommandLocator(
-            new class() implements ClassesInterface {
-                public function getClasses($target = null): array
+            new class() implements ScopedClassesInterface {
+                public function getScopedClasses(string $scope, $target = null): array
                 {
                     return [
                         new \ReflectionClass(LazyLoadedCommand::class),
diff --git a/src/Prototype/tests/LocatorTest.php b/src/Prototype/tests/LocatorTest.php
index 2381ebc50..a4dc9bfe1 100644
--- a/src/Prototype/tests/LocatorTest.php
+++ b/src/Prototype/tests/LocatorTest.php
@@ -17,6 +17,10 @@ use Spiral\Tests\Prototype\Fixtures\HydratedClass;
 use Spiral\Tests\Prototype\Fixtures\TestClass;
 use Spiral\Tokenizer\ClassesInterface;
 use Spiral\Tokenizer\ClassLocator;
+use Spiral\Tokenizer\Config\TokenizerConfig;
+use Spiral\Tokenizer\ScopedClassesInterface;
+use Spiral\Tokenizer\ScopedClassLocator;
+use Spiral\Tokenizer\Tokenizer;
 use Symfony\Component\Finder\Finder;
 
 class LocatorTest extends TestCase
@@ -37,10 +41,15 @@ class LocatorTest extends TestCase
         $this->assertArrayNotHasKey(HydratedClass::class, $l->getTargetClasses());
     }
 
-    private function makeClasses(): ClassesInterface
+    private function makeClasses(): ScopedClassesInterface
     {
-        return new ClassLocator(
-            (new Finder())->in([__DIR__ . '/Fixtures'])->files()
-        );
+        return new ScopedClassLocator(new Tokenizer(new TokenizerConfig([
+            'directories' => [],
+            'scopes' => [
+                'prototypes' => [
+                    'directories' => [__DIR__ . '/Fixtures']
+                ]
+            ]
+        ])));
     }
 }
diff --git a/src/Tokenizer/tests/ClassLocatorTest.php b/src/Tokenizer/tests/ClassLocatorTest.php
index cf564e17a..f53d68191 100644
--- a/src/Tokenizer/tests/ClassLocatorTest.php
+++ b/src/Tokenizer/tests/ClassLocatorTest.php
@@ -139,7 +139,7 @@ class ClassLocatorTest extends TestCase
          */
         $locator->setLogger($logger);
 
-        $classes = $locator->getClasses(ClassB::class);
+        $locator->getClasses(ClassB::class);
 
         $this->assertStringContainsString(
             ' has includes and excluded from analysis',
diff --git a/src/Tokenizer/tests/ConfigTest.php b/src/Tokenizer/tests/ConfigTest.php
index 8b1ef99b7..0f1117f70 100644
--- a/src/Tokenizer/tests/ConfigTest.php
+++ b/src/Tokenizer/tests/ConfigTest.php
@@ -28,4 +28,58 @@ class ConfigTest extends TestCase
         ]);
         $this->assertSame(['a', 'b', 'c'], $config->getExcludes());
     }
+
+    public function testNonExistScopeShouldReturnDefaultDirectories()
+    {
+        $config = new TokenizerConfig([
+            'directories' => ['a'],
+            'exclude' => ['b'],
+            'scopes' => [
+                'foo' => [
+                    'directories' => ['c'],
+                    'exclude' => ['d'],
+                ]
+            ]
+        ]);
+
+        $this->assertSame([
+            'directories' => ['a'],
+            'exclude' => ['b'],
+        ], $config->getScope('bar'));
+    }
+
+    public function testExistsScopeShouldReturnDirectoriesFromIt()
+    {
+        $config = new TokenizerConfig([
+            'directories' => ['a'],
+            'exclude' => ['b'],
+            'scopes' => [
+                'foo' => [
+                    'directories' => ['c'],
+                    'exclude' => ['d'],
+                ],
+                'bar' => [
+                    'directories' => ['c'],
+                ],
+                'baz' => [
+                    'exclude' => ['d'],
+                ]
+            ]
+        ]);
+
+        $this->assertSame([
+            'directories' => ['c'],
+            'exclude' => ['d'],
+        ], $config->getScope('foo'));
+
+        $this->assertSame([
+            'directories' => ['c'],
+            'exclude' => ['b'],
+        ], $config->getScope('bar'));
+
+        $this->assertSame([
+            'directories' => ['a'],
+            'exclude' => ['d'],
+        ], $config->getScope('baz'));
+    }
 }
diff --git a/src/Tokenizer/tests/ScopedClassLocatorTest.php b/src/Tokenizer/tests/ScopedClassLocatorTest.php
new file mode 100644
index 000000000..86c669b2f
--- /dev/null
+++ b/src/Tokenizer/tests/ScopedClassLocatorTest.php
@@ -0,0 +1,76 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Spiral\Tests\Tokenizer;
+
+use PHPUnit\Framework\TestCase;
+use Spiral\Core\Container;
+use Spiral\Tests\Tokenizer\Classes\ClassA;
+use Spiral\Tests\Tokenizer\Classes\ClassB;
+use Spiral\Tests\Tokenizer\Classes\ClassC;
+use Spiral\Tests\Tokenizer\Classes\Inner\ClassD;
+use Spiral\Tokenizer\Config\TokenizerConfig;
+use Spiral\Tokenizer\ScopedClassesInterface;
+use Spiral\Tokenizer\ScopedClassLocator;
+use Spiral\Tokenizer\Tokenizer;
+
+final class ScopedClassLocatorTest extends TestCase
+{
+    private Container $container;
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+
+        $this->container = new Container();
+        $this->container->bind(Tokenizer::class, $this->getTokenizer());
+        $this->container->bindSingleton(ScopedClassesInterface::class, ScopedClassLocator::class);
+    }
+
+    public function testGetsClassesForExistsScope()
+    {
+        $classes = $this->container->get(ScopedClassesInterface::class)->getScopedClasses('foo');
+
+        $this->assertArrayHasKey(ClassD::class, $classes);
+
+        // Excluded
+        $this->assertArrayNotHasKey(self::class, $classes);
+        $this->assertArrayNotHasKey(ClassA::class, $classes);
+        $this->assertArrayNotHasKey(ClassB::class, $classes);
+        $this->assertArrayNotHasKey(ClassC::class, $classes);
+        $this->assertArrayNotHasKey('Spiral\Tests\Tokenizer\Classes\Excluded\ClassXX', $classes);
+        $this->assertArrayNotHasKey('Spiral\Tests\Tokenizer\Classes\Bad_Class', $classes);
+    }
+
+    public function testGetsClassesForNotExistScope()
+    {
+        $classes = $this->container->get(ScopedClassesInterface::class)->getScopedClasses('bar');
+
+        $this->assertArrayHasKey(self::class, $classes);
+        $this->assertArrayHasKey(ClassA::class, $classes);
+        $this->assertArrayHasKey(ClassB::class, $classes);
+        $this->assertArrayHasKey(ClassC::class, $classes);
+        $this->assertArrayHasKey(ClassD::class, $classes);
+
+        // Excluded
+        $this->assertArrayNotHasKey('Spiral\Tests\Tokenizer\Classes\Excluded\ClassXX', $classes);
+        $this->assertArrayNotHasKey('Spiral\Tests\Tokenizer\Classes\Bad_Class', $classes);
+    }
+
+    protected function getTokenizer(): Tokenizer
+    {
+        $config = new TokenizerConfig([
+            'directories' => [__DIR__],
+            'exclude' => ['Excluded'],
+            'scopes' => [
+                'foo' => [
+                    'directories' => [__DIR__.'/Classes/Inner'],
+                    'exclude' => [],
+                ],
+            ],
+        ]);
+
+        return new Tokenizer($config);
+    }
+}
diff --git a/src/Translator/tests/IndexerTest.php b/src/Translator/tests/IndexerTest.php
index c8f72041b..e8414d494 100644
--- a/src/Translator/tests/IndexerTest.php
+++ b/src/Translator/tests/IndexerTest.php
@@ -18,6 +18,8 @@ use Spiral\Tokenizer\ClassLocator;
 use Spiral\Tokenizer\Config\TokenizerConfig;
 use Spiral\Tokenizer\InvocationLocator;
 use Spiral\Tokenizer\InvocationsInterface;
+use Spiral\Tokenizer\ScopedClassesInterface;
+use Spiral\Tokenizer\ScopedClassLocator;
 use Spiral\Translator\Catalogue;
 use Spiral\Translator\Config\TranslatorConfig;
 use Spiral\Translator\Indexer;
@@ -65,7 +67,7 @@ class IndexerTest extends TestCase
             ]
         ]), $catalogue);
 
-        $indexer->indexClasses($this->tContainer()->get(ClassesInterface::class));
+        $indexer->indexClasses($this->tContainer()->get(ScopedClassesInterface::class));
 
         $this->assertTrue($catalogue->has('spiral', 'indexer-message'));
         $this->assertFalse($catalogue->has('spiral', 'not-message'));
@@ -80,12 +82,17 @@ class IndexerTest extends TestCase
     protected function tContainer(): Container
     {
         $container = new Container();
-        $container->bind(ClassesInterface::class, ClassLocator::class);
+        $container->bind(ScopedClassesInterface::class, ScopedClassLocator::class);
         $container->bind(InvocationsInterface::class, InvocationLocator::class);
 
         $container->bind(TokenizerConfig::class, new TokenizerConfig([
             'directories' => [__DIR__],
-            'exclude'     => []
+            'exclude'     => [],
+            'scopes' => [
+                'translations' => [
+                    'directories' => [__DIR__],
+                ]
+            ]
         ]));
 
         return $container;