Repository: apify/apify-cli (ts)

Issue:
Apify run should allow starting different npm scripts or files for testing
Right now, I have to change `npm start` script if I want to test with all the ENV injected.

We should allow either start from any file or with different npm scripts.

Pull request:
feat: support running other entrypoints with injected Apify environment variables
Closes #95 

Patch:
diff --git a/src/commands/run.ts b/src/commands/run.ts
index a8cedcf..523093c 100644
--- a/src/commands/run.ts
+++ b/src/commands/run.ts
@@ -1,4 +1,5 @@
 import { existsSync, renameSync } from 'node:fs';
+import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
 import process from 'node:process';
 
@@ -57,6 +58,17 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
             description: 'Deletes all records from the default key-value store in the local directory before the run starts, except for the "INPUT" key.',
             required: false,
         }),
+        entrypoint: Flags.string({
+            description: [
+                'Optional entrypoint for running with injected environment variables.',
+                '\n',
+                'For Python, it is the module name, or a path to a file.',
+                '\n',
+                'For node.js, it is the npm script name, or a path to a JS/MJS file.',
+                'You can also pass in a directory name, provided that directory contains an "index.js" file.',
+            ].join(' '),
+            required: false,
+        }),
     };
 
     async run() {
@@ -73,6 +85,7 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
         const packageJsonExists = existsSync(packageJsonPath);
         const mainPyExists = existsSync(mainPyPath);
         const isScrapyProject = projectType === PROJECT_TYPES.SCRAPY;
+        const { language, languageVersion } = detectLocalActorLanguage(cwd);
 
         if (!packageJsonExists && !mainPyExists && !isScrapyProject) {
             throw new Error(
@@ -81,6 +94,29 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
             );
         }
 
+        // Defaults (for node, the start script, for python, the src module)
+        let runType: RunType = language === LANGUAGE.NODEJS ? RunType.Script : RunType.Module;
+        let entrypoint: string = language === LANGUAGE.NODEJS ? 'start' : 'src';
+
+        if (this.flags.entrypoint) {
+            entrypoint = this.flags.entrypoint;
+
+            const entrypointPath = join(cwd, this.flags.entrypoint);
+
+            const entrypointStat = await stat(entrypointPath).catch(() => null);
+
+            // Directory -> We just try to run it as a module (in python, it needs to have a main.py file, in node.js, an index.(m)js file)
+            if (entrypointStat?.isDirectory()) {
+                runType = RunType.Module;
+            // entrypoint -> ./src/file for example (running custom scripts)
+            } else if (entrypointStat?.isFile()) {
+                runType = RunType.DirectFile;
+            // If it's not a file, or a directory, we just let it be a script
+            } else {
+                runType = RunType.Script;
+            }
+        }
+
         if (existsSync(LEGACY_LOCAL_STORAGE_DIR) && !existsSync(actualStoragePath)) {
             renameSync(LEGACY_LOCAL_STORAGE_DIR, actualStoragePath);
             warning(`The legacy 'apify_storage' directory was renamed to '${actualStoragePath}' to align it with Apify SDK v3.`
@@ -151,18 +187,10 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
             warning('You are not logged in with your Apify Account. Some features like Apify Proxy will not work. Call "apify login" to fix that.');
         }
 
-        const { language, languageVersion } = detectLocalActorLanguage(cwd);
         if (language === LANGUAGE.NODEJS) { // Actor is written in Node.js
             const currentNodeVersion = languageVersion;
             const minimumSupportedNodeVersion = minVersion(SUPPORTED_NODEJS_VERSION);
             if (currentNodeVersion) {
-                const serverJsFile = join(cwd, 'server.js');
-                const packageJson = await loadJsonFile<{ scripts: Record<string, string > }>(packageJsonPath);
-                if ((!packageJson.scripts || !packageJson.scripts.start) && !existsSync(serverJsFile)) {
-                    throw new Error('The "npm start" script was not found in package.json. Please set it up for your project. '
-                        + 'For more information about that call "apify help run".');
-                }
-
                 // --max-http-header-size=80000
                 // Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb.
                 // However they did not think about all the sites there with large headers,
@@ -173,9 +201,31 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
                     warning(`You are running Node.js version ${currentNodeVersion}, which is no longer supported. `
                         + `Please upgrade to Node.js version ${minimumSupportedNodeVersion} or later.`);
                 }
+
                 this.telemetryData.actorNodejsVersion = currentNodeVersion;
                 this.telemetryData.actorLanguage = LANGUAGE.NODEJS;
-                await execWithLog(getNpmCmd(), ['start'], { env, cwd });
+
+                // We allow "module" type directly in node too (it will work for a folder that has an `index.js` file)
+                if (runType === RunType.DirectFile || runType === RunType.Module) {
+                    await execWithLog('node', [entrypoint], { env, cwd });
+                } else {
+                    // TODO(vladfrangu): what is this for? Some old template maybe?
+                    // && !existsSync(serverJsFile)
+                    // const serverJsFile = join(cwd, 'server.js');
+                    const packageJson = await loadJsonFile<{ scripts: Record<string, string > }>(packageJsonPath);
+
+                    if (!packageJson.scripts) {
+                        throw new Error('No scripts were found in package.json. Please set it up for your project. '
+                            + 'For more information about that call "apify help run".');
+                    }
+
+                    if (!packageJson.scripts[entrypoint]) {
+                        throw new Error(`The script "${entrypoint}" was not found in package.json. Please set it up for your project. `
+                            + 'For more information about that call "apify help run".');
+                    }
+
+                    await execWithLog(getNpmCmd(), ['run', entrypoint], { env, cwd });
+                }
             } else {
                 error(`No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher to be able to run Node.js Actors locally.`);
             }
@@ -186,17 +236,20 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
             if (pythonVersion) {
                 if (isPythonVersionSupported(pythonVersion)) {
                     const pythonCommand = getPythonCommand(cwd);
-                    let executableLocation = 'src';
 
-                    if (isScrapyProject) {
+                    if (isScrapyProject && !this.flags.entrypoint) {
                         const project = new ScrapyProjectAnalyzer(cwd);
                         project.loadScrapyCfg();
                         if (project.configuration.hasKey('apify', 'mainpy_location')) {
-                            executableLocation = project.configuration.get('apify', 'mainpy_location')!;
+                            entrypoint = project.configuration.get('apify', 'mainpy_location')!;
                         }
                     }
 
-                    await execWithLog(pythonCommand, ['-m', executableLocation], { env, cwd });
+                    if (runType === RunType.Module) {
+                        await execWithLog(pythonCommand, ['-m', entrypoint], { env, cwd });
+                    } else {
+                        await execWithLog(pythonCommand, [entrypoint], { env, cwd });
+                    }
                 } else {
                     error(`Python Actors require Python 3.8 or higher, but you have Python ${pythonVersion}!`);
                     error('Please install Python 3.8 or higher to be able to run Python Actors locally.');
@@ -207,3 +260,9 @@ export class RunCommand extends ApifyCommand<typeof RunCommand> {
         }
     }
 }
+
+enum RunType {
+    DirectFile,
+    Module,
+    Script,
+}


Test patch:
diff --git a/test/commands/run.test.ts b/test/commands/run.test.ts
index 40c7ded..d053895 100644
--- a/test/commands/run.test.ts
+++ b/test/commands/run.test.ts
@@ -1,4 +1,4 @@
-import { existsSync, writeFileSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 
 import { APIFY_ENV_VARS } from '@apify/consts';
 import { loadJsonFileSync } from 'load-json-file';
@@ -97,6 +97,91 @@ describe('apify run', () => {
         expect(localEnvVars.TEST_LOCAL).toStrictEqual(testEnvVars.TEST_LOCAL);
     });
 
+    it(`run with env vars from "${LOCAL_CONFIG_PATH}" and custom script`, async () => {
+        const testEnvVars = {
+            TEST_LOCAL: 'testValue',
+        };
+
+        await LoginCommand.run(['--token', TEST_USER_TOKEN], import.meta.url);
+
+        const actCode = `
+        import { Actor } from 'apify';
+
+        Actor.main(async () => {
+            await Actor.setValue('OUTPUT', process.env);
+            await Actor.setValue('owo', 'uwu');
+            console.log('Done.');
+        });
+        `;
+
+        writeFileSync(joinPath('src/other.js'), actCode, { flag: 'w' });
+
+        const apifyJson = EMPTY_LOCAL_CONFIG;
+        apifyJson.environmentVariables = testEnvVars;
+        writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), apifyJson);
+
+        const pkgJson = readFileSync(joinPath('package.json'), 'utf8');
+        const parsedPkgJson = JSON.parse(pkgJson);
+        parsedPkgJson.scripts.other = 'node src/other.js';
+        writeFileSync(joinPath('package.json'), JSON.stringify(parsedPkgJson, null, 2), { flag: 'w' });
+
+        await RunCommand.run(['--entrypoint', 'other'], import.meta.url);
+
+        const actOutputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json');
+
+        const localEnvVars = loadJsonFileSync<Record<typeof APIFY_ENV_VARS[keyof typeof APIFY_ENV_VARS] | 'TEST_LOCAL', string>>(actOutputPath);
+        const auth = loadJsonFileSync<{ proxy: { password: string }; id: string; token: string }>(AUTH_FILE_PATH());
+
+        expect(localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD]).toStrictEqual(auth.proxy.password);
+        expect(localEnvVars[APIFY_ENV_VARS.USER_ID]).toStrictEqual(auth.id);
+        expect(localEnvVars[APIFY_ENV_VARS.TOKEN]).toStrictEqual(auth.token);
+        expect(localEnvVars.TEST_LOCAL).toStrictEqual(testEnvVars.TEST_LOCAL);
+
+        const actOutputPath2 = joinPath(getLocalKeyValueStorePath(), 'owo.json');
+        const actOutput2 = loadJsonFileSync(actOutputPath2);
+        expect(actOutput2).toStrictEqual('uwu');
+    });
+
+    it(`run with env vars from "${LOCAL_CONFIG_PATH}" and direct path to script`, async () => {
+        const testEnvVars = {
+            TEST_LOCAL: 'testValue',
+        };
+
+        await LoginCommand.run(['--token', TEST_USER_TOKEN], import.meta.url);
+
+        const actCode = `
+        import { Actor } from 'apify';
+
+        Actor.main(async () => {
+            await Actor.setValue('OUTPUT', process.env);
+            await Actor.setValue('two', 'can play');
+            console.log('Done.');
+        });
+        `;
+
+        writeFileSync(joinPath('src/other.js'), actCode, { flag: 'w' });
+
+        const apifyJson = EMPTY_LOCAL_CONFIG;
+        apifyJson.environmentVariables = testEnvVars;
+        writeJsonFileSync(joinPath(LOCAL_CONFIG_PATH), apifyJson);
+
+        await RunCommand.run(['--entrypoint', 'src/other.js'], import.meta.url);
+
+        const actOutputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json');
+
+        const localEnvVars = loadJsonFileSync<Record<typeof APIFY_ENV_VARS[keyof typeof APIFY_ENV_VARS] | 'TEST_LOCAL', string>>(actOutputPath);
+        const auth = loadJsonFileSync<{ proxy: { password: string }; id: string; token: string }>(AUTH_FILE_PATH());
+
+        expect(localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD]).toStrictEqual(auth.proxy.password);
+        expect(localEnvVars[APIFY_ENV_VARS.USER_ID]).toStrictEqual(auth.id);
+        expect(localEnvVars[APIFY_ENV_VARS.TOKEN]).toStrictEqual(auth.token);
+        expect(localEnvVars.TEST_LOCAL).toStrictEqual(testEnvVars.TEST_LOCAL);
+
+        const actOutputPath2 = joinPath(getLocalKeyValueStorePath(), 'two.json');
+        const actOutput2 = loadJsonFileSync(actOutputPath2);
+        expect(actOutput2).toStrictEqual('can play');
+    });
+
     it('run purge stores', async () => {
         const input = {
             myInput: 'value',