Repository: shazow/whatsabi (ts)

Issue:
loaders: Add code loading?
Right now we just load the metadata, but we're not too far from loading the code too.

I believe the Etherscan loader already returns the code in the response, we just need to plumb it through to the response.

The Sourcify loader would likely need another request to fetch the source, so might need to make it a flag?

Pull request description:
loaders: Add getSources() to ContractResult
- Added `getSources?: () => Promise<ContractSources>;` to `ContractResult`
- Currently only supported by the Etherscan loader
- `ContractSources` automatically deserializes the encoded source code structure. This will help once we add support for the Sourcify loader which uses a different format (and will require a separate request for each import).
- `ContractResult` is not yet accessible from `AutoloadResult`, so would need to use loaders directly to access it for now. (This will require a bit more plumbing, since autoload uses loadABI rather than loadContract, so we'll need yet another flag to include contract sources.)
- Fixes #111 

Usage:

```typescript
const result = await loader.getContract(address);
const sources = await result.getSources();

for (const s of sources) {
  console.log(s.path, " -> ", s.content.slice(25) + "...");
}
```

Patch:
diff --git a/Makefile b/Makefile
index 10eb9ad..e8efcac 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,9 @@ build:
 docs:
 	pnpm build:docs
 
+serve-docs:
+	python -m http.server -d ./docs
+
 watch:
 	tsc --project tsconfig.esm.json -w
 
diff --git a/README.md b/README.md
index 08dcf2a..949ba70 100644
--- a/README.md
+++ b/README.md
@@ -97,6 +97,8 @@ const loader = new whatsabi.loaders.MultiABILoader([
 const { abi, name, /* ... other metadata */ } = await loader.getContract(address));
 ```
 
+See [whatsabi.loaders](https://shazow.github.io/whatsabi/modules/whatsabi.loaders.html) for more examples of what our loaders can do, like loading verified contract source code and compiler settings.
+
 All together with our do-all-the-things helper:
 
 ```typescript
diff --git a/src/loaders.ts b/src/loaders.ts
index e8ef1e1..a503e6c 100644
--- a/src/loaders.ts
+++ b/src/loaders.ts
@@ -1,4 +1,30 @@
-import { addressWithChecksum, fetchJSON } from "./utils.js";
+/**
+ * @module loaders
+ * @example
+ * Verified contract source code:
+ * ```ts
+ * const loader = whatsabi.loaders.defaultsWithEnv(env);
+ * const result = await loader.getContract(address);
+ * const sources = await result.getSources();
+ *
+ * for (const s of sources) {
+ *   console.log(s.path, " -> ", s.content + "...");
+ * }
+ * ```
+ *
+ * @example
+ * Combine loaders with custom settings behind a single interface, or use {@link defaultsWithEnv} as a shortcut for this.
+ * ```ts
+ * const loader = new whatsabi.loaders.MultiABILoader([
+ *   new whatsabi.loaders.SourcifyABILoader({ chainId: 8453 }),
+ *   new whatsabi.loaders.EtherscanABILoader({
+ *     baseURL: "https://api.basescan.org/api",
+ *     apiKey: "...", // Replace the value with your API key
+ *   }),
+ * ]);
+ * ```
+ */
+import { fetchJSON } from "./utils.js";
 import * as errors from "./errors.js";
 
 export type ContractResult = {
@@ -9,8 +35,34 @@ export type ContractResult = {
     runs: number;
 
     ok: boolean; // False if no result is found
+
+    /**
+     * getSources returns the imports -> source code mapping for the contract, if available.
+     *
+     * Caveats:
+     * - Not all loaders support this, so the property could be undefined.
+     * - This call could trigger additional fetch requests, depending on the loader.
+     **/
+    getSources?: () => Promise<ContractSources>;
 }
 
+/**
+ * ContractSources is a list of source files.
+ * If the source was flattened, it will lack a path attribute.
+ *
+ * @example
+ * ```typescript
+ * [{"content": "pragma solidity =0.7.6;\n\nimport ..."}]
+ * ```
+ *
+ * @example
+ * ```typescript
+ * [{"path": "contracts/Foo.sol", "content:" "pragma solidity =0.7.6;\n\nimport ..."}]
+ * ```
+ **/
+export type ContractSources = Array<{ path?: string, content: string }>;
+
+
 const emptyContractResult: ContractResult = {
     ok: false,
 
@@ -38,7 +90,7 @@ export class MultiABILoader implements ABILoader {
         for (const loader of this.loaders) {
             try {
                 const r = await loader.getContract(address);
-                if (r && r.abi.length > 0) return Promise.resolve(r);
+                if (r && r.abi.length > 0) return r;
             } catch (err: any) {
                 if (err.status === 404) continue;
 
@@ -57,7 +109,7 @@ export class MultiABILoader implements ABILoader {
                 const r = await loader.loadABI(address);
 
                 // Return the first non-empty result
-                if (r.length > 0) return Promise.resolve(r);
+                if (r.length > 0) return r;
             } catch (err: any) {
                 throw new MultiABILoaderError("MultiABILoader loadABI error: " + err.message, {
                     context: { loader, address },
@@ -65,7 +117,7 @@ export class MultiABILoader implements ABILoader {
                 });
             }
         }
-        return Promise.resolve([]);
+        return [];
     }
 }
 
@@ -82,6 +134,26 @@ export class EtherscanABILoader implements ABILoader {
         this.baseURL = config.baseURL || "https://api.etherscan.io/api";
     }
 
+
+    /** Etherscan helper for converting the encoded SourceCode result arg to a decoded ContractSources. */
+    #toContractSources(result: { SourceCode: string }): ContractSources {
+        if (!result.SourceCode.startsWith("{{")) {
+            return [{ content: result.SourceCode }];
+        }
+
+        // Etherscan adds an extra {} to the encoded JSON
+        const s = JSON.parse(result.SourceCode.slice(1, result.SourceCode.length - 1));
+
+        // Flatten sources
+        // { "sources": {"foo.sol": {"content": "..."}}}
+        const sources = s.sources as Record<string, { content: string }>;
+        return Object.entries(sources).map(
+            ([path, source]) => {
+                return { path, content: source.content };
+            }
+        )
+    }
+
     async getContract(address: string): Promise<ContractResult> {
         let url = this.baseURL + '?module=contract&action=getsourcecode&address=' + address;
         if (this.apiKey) url += "&apikey=" + this.apiKey;
@@ -101,6 +173,17 @@ export class EtherscanABILoader implements ABILoader {
                 compilerVersion: result.CompilerVersion,
                 runs: result.Runs,
 
+                getSources: async () => {
+                    try {
+                        return this.#toContractSources(result);
+                    } catch (err: any) {
+                        throw new EtherscanABILoaderError("EtherscanABILoader getContract getSources error: " + err.message, {
+                            context: { url, address },
+                            cause: err,
+                        });
+                    }
+                },
+
                 ok: true,
             };
         } catch (err: any) {
@@ -153,68 +236,67 @@ export class SourcifyABILoader implements ABILoader {
         this.chainId = config?.chainId ?? 1;
     }
 
-    async getContract(address: string): Promise<ContractResult> {
-        // Sourcify doesn't like it when the address is not checksummed
-        address = addressWithChecksum(address);
+    static stripPathPrefix(path: string): string {
+        return path.replace(/^\/contracts\/(full|partial)_match\/\d*\/\w*\/(sources\/)?/, "");
+    }
+
+    async #loadContract(url: string): Promise<ContractResult> {
+        try {
+            const r = await fetchJSON(url);
+            const files: Array<{ name: string, path: string, content: string }> = r.files ?? r;
+
+            // Metadata is usually the first one
+            const metadata = files.find((f) => f.name === "metadata.json")
+            if (metadata === undefined) throw new SourcifyABILoaderError("metadata.json not found");
+
+            // Note: Sometimes metadata.json contains sources, but not always. So we can't rely on just the metadata.json
+            const m = JSON.parse(metadata.content);
 
+            return {
+                abi: m.output.abi,
+                name: m.output.devdoc?.title ?? null, // Sourcify includes a title from the Natspec comments
+                evmVersion: m.settings.evmVersion,
+                compilerVersion: m.compiler.version,
+                runs: m.settings.optimizer.runs,
+
+                // TODO: Paths will have a sourcify prefix, do we want to strip it to help normalize? It doesn't break anything keeping the prefix, so not sure.
+                // E.g. /contracts/full_match/1/0x1F98431c8aD98523631AE4a59f267346ea31F984/sources/contracts/interfaces/IERC20Minimal.sol
+                // Can use stripPathPrefix helper to do this, but maybe we want something like getSources({ normalize: true })?
+                getSources: async () => files.map(({ path, content }) => { return { path, content } }),
+
+                ok: true,
+            };
+        } catch (err: any) {
+            if (isSourcifyNotFound(err)) return emptyContractResult;
+            throw new SourcifyABILoaderError("SourcifyABILoader load contract error: " + err.message, {
+                context: { url },
+                cause: err,
+            });
+        }
+    }
+
+    async getContract(address: string): Promise<ContractResult> {
         {
             // Full match index includes verification settings that matches exactly
-            const url = "https://repo.sourcify.dev/contracts/full_match/" + this.chainId + "/" + address + "/metadata.json";
-            try {
-                const r = await fetchJSON(url);
-                return {
-                    abi: r.output.abi,
-                    name: r.output.devdoc?.title ?? null, // Sourcify includes a title from the Natspec comments
-                    evmVersion: r.settings.evmVersion,
-                    compilerVersion: r.compiler.version,
-                    runs: r.settings.optimizer.runs,
-
-                    ok: true,
-                };
-            } catch (err: any) {
-                if (!isSourcifyNotFound(err)) {
-                    throw new SourcifyABILoaderError("SourcifyABILoader getContract error: " + err.message, {
-                        context: { address, url },
-                        cause: err,
-                    });
-                }
-            }
+            const url = "https://sourcify.dev/server/files/" + this.chainId + "/" + address;
+            const r = await this.#loadContract(url);
+            if (r.ok) return r;
         }
 
         {
             // Partial match index is for verified contracts whose settings didn't match exactly
-            const url = "https://repo.sourcify.dev/contracts/partial_match/" + this.chainId + "/" + address + "/metadata.json"
-            try {
-                const r = await fetchJSON(url);
-                return {
-                    abi: r.output.abi,
-                    name: r.output.devdoc?.title ?? null, // Sourcify includes a title from the Natspec comments
-                    evmVersion: r.settings.evmVersion,
-                    compilerVersion: r.compiler.version,
-                    runs: r.settings.optimizer.runs,
-
-                    ok: true,
-                };
-            } catch (err: any) {
-                if (!isSourcifyNotFound(err)) {
-                    throw new SourcifyABILoaderError("SourcifyABILoader getContract error: " + err.message, {
-                        context: { address, url },
-                        cause: err,
-                    });
-                }
-            }
+            const url = "https://sourcify.dev/server/files/any/" + this.chainId + "/" + address;
+            const r = await this.#loadContract(url);
+            if (r.ok) return r;
         }
 
         return emptyContractResult;
     }
 
     async loadABI(address: string): Promise<any[]> {
-        // Sourcify doesn't like it when the address is not checksummed
-        address = addressWithChecksum(address);
-
         {
             // Full match index includes verification settings that matches exactly
-            const url = "https://repo.sourcify.dev/contracts/full_match/" + this.chainId + "/" + address + "/metadata.json";
+            const url = "https://sourcify.dev/server/repository/contracts/full_match/" + this.chainId + "/" + address + "/metadata.json";
             try {
                 return (await fetchJSON(url)).output.abi;
             } catch (err: any) {
@@ -229,7 +311,7 @@ export class SourcifyABILoader implements ABILoader {
 
         {
             // Partial match index is for verified contracts whose settings didn't match exactly
-            const url = "https://repo.sourcify.dev/contracts/partial_match/" + this.chainId + "/" + address + "/metadata.json";
+            const url = "https://sourcify.dev/server/repository/contracts/partial_match/" + this.chainId + "/" + address + "/metadata.json";
             try {
                 return (await fetchJSON(url)).output.abi;
             } catch (err: any) {
@@ -267,9 +349,9 @@ export class MultiSignatureLookup implements SignatureLookup {
             const r = await lookup.loadFunctions(selector);
 
             // Return the first non-empty result
-            if (r.length > 0) return Promise.resolve(r);
+            if (r.length > 0) return r;
         }
-        return Promise.resolve([]);
+        return [];
     }
 
     async loadEvents(hash: string): Promise<string[]> {
@@ -277,9 +359,9 @@ export class MultiSignatureLookup implements SignatureLookup {
             const r = await lookup.loadEvents(hash);
 
             // Return the first non-empty result
-            if (r.length > 0) return Promise.resolve(r);
+            if (r.length > 0) return r;
         }
-        return Promise.resolve([]);
+        return [];
     }
 }
 
@@ -356,20 +438,26 @@ type LoaderEnv = {
  *
  * @example
  * ```ts
- * whatsabi.autoload(address, {provider, ...defaultsWithEnv(process.env)})
+ * whatsabi.autoload(address, {provider, ...whatsabi.loaders.defaultsWithEnv(process.env)})
  * ```
  *
  * @example
  * ```ts
  * whatsabi.autoload(address, {
  *     provider,
- *     ...defaultsWithEnv({
+ *     ...whatsabi.loaders.defaultsWithEnv({
  *         SOURCIFY_CHAIN_ID: 42161,
  *         ETHERSCAN_BASE_URL: "https://api.arbiscan.io/api",
  *         ETHERSCAN_API_KEY: "MYSECRETAPIKEY",
  *     }),
  * })
  * ```
+ *
+ * @example
+ * Can be useful for stand-alone usage too!
+ * ```ts
+ * const { abiLoader, signatureLookup } = whatsabi.loaders.defaultsWithEnv(env);
+ * ```
  */
 export function defaultsWithEnv(env: LoaderEnv): Record<string, ABILoader | SignatureLookup> {
     return {


Test patch:
diff --git a/src/__tests__/examples.test.ts b/src/__tests__/examples.test.ts
index 1f40934..a2ac5e1 100644
--- a/src/__tests__/examples.test.ts
+++ b/src/__tests__/examples.test.ts
@@ -18,7 +18,7 @@ cached_test('README usage', async ({ provider, withCache }) => {
     )
 
     const selectors = whatsabi.selectorsFromBytecode(code); // Get the callable selectors
-    
+
     // console.log(selectors); // ["0x06fdde03", "0x46423aa7", "0x55944a42", ...]
     expect(selectors).toEqual(expect.arrayContaining(["0x06fdde03", "0x46423aa7", "0x55944a42"]));
 
@@ -32,9 +32,9 @@ cached_test('README usage', async ({ provider, withCache }) => {
         //        ...
         // ]
 
-        expect(abi).toContainEqual({"hash": "0x721c20121297512b72821b97f5326877ea8ecf4bb9948fea5bfcb6453074d37f", "type": "event"});
+        expect(abi).toContainEqual({ "hash": "0x721c20121297512b72821b97f5326877ea8ecf4bb9948fea5bfcb6453074d37f", "type": "event" });
         expect(abi).toContainEqual(
-            {"payable": true, "selector": "0xb3a34c4c", "type": "function", "stateMutability": "payable", "inputs": [{"type": "bytes", "name": ""}], "outputs": [{"type": "bytes", "name": ""}]},
+            { "payable": true, "selector": "0xb3a34c4c", "type": "function", "stateMutability": "payable", "inputs": [{ "type": "bytes", "name": "" }], "outputs": [{ "type": "bytes", "name": "" }] },
         );
     }
 
@@ -64,7 +64,7 @@ online_test('README autoload', async ({ provider }) => {
             // signatureLoader: whatsabi.loaders.defaultSignatureLookup,
 
             // There is a handy helper for adding the default loaders but with your own settings
-            ... whatsabi.loaders.defaultsWithEnv({
+            ...whatsabi.loaders.defaultsWithEnv({
                 SOURCIFY_CHAIN_ID: 42161,
                 ETHERSCAN_BASE_URL: "https://api.arbiscan.io/api",
                 //ETHERSCAN_API_KEY: "MYSECRETAPIKEY",
@@ -83,20 +83,28 @@ online_test('README autoload', async ({ provider }) => {
         });
         expect(result.abi).toContainEqual(
             // 'function name() pure returns (string contractName)'
-            {"inputs":[],"name":"name","outputs":[{"internalType":"string","name":"contractName","type":"string"}],"stateMutability":"pure","type":"function"}
+            { "inputs": [], "name": "name", "outputs": [{ "internalType": "string", "name": "contractName", "type": "string" }], "stateMutability": "pure", "type": "function" }
         );
 
         if (result.followProxies) {
-                result = await result.followProxies();
+            result = await result.followProxies();
         }
     }
 
     {
-            const { abi, address } = await whatsabi.autoload("0x4f8AD938eBA0CD19155a835f617317a6E788c868", {
-                    provider,
-                    followProxies: true,
-            });
-            expect(abi.length).toBeGreaterThan(0);
-            expect(address).toBe("0x964f84048f0d9bb24b82413413299c0a1d61ea9f");
+        const { abi, address } = await whatsabi.autoload("0x4f8AD938eBA0CD19155a835f617317a6E788c868", {
+            provider,
+            followProxies: true,
+        });
+        expect(abi.length).toBeGreaterThan(0);
+        expect(address).toBe("0x964f84048f0d9bb24b82413413299c0a1d61ea9f");
+    }
+
+    {
+        // Check defaultsWithEnv decomposition
+        const { abiLoader, signatureLookup } = whatsabi.loaders.defaultsWithEnv({});
+        expect(abiLoader).toBeDefined();
+        expect(signatureLookup).toBeDefined();
     }
+
 }, TIMEOUT);
diff --git a/src/__tests__/loaders.test.ts b/src/__tests__/loaders.test.ts
index 93c60aa..35c7115 100644
--- a/src/__tests__/loaders.test.ts
+++ b/src/__tests__/loaders.test.ts
@@ -12,7 +12,7 @@ import {
 } from "../loaders";
 import { selectorsFromABI } from "../index";
 
-import { online_test } from "./env";
+import { online_test, test } from "./env";
 
 const SLOW_ETHERSCAN_TIMEOUT = 30000;
 
@@ -40,7 +40,7 @@ describe('loaders module', () => {
 
   online_test('SourcifyABILoader', async () => {
     const loader = new SourcifyABILoader();
-    const abi = await loader.loadABI("0x7a250d5630b4cf539739df2c5dacb4c659f2488d");
+    const abi = await loader.loadABI("0x7a250d5630b4cf539739df2c5dacb4c659f2488d"); // Unchecksummed address
     const selectors = Object.values(selectorsFromABI(abi));
     const sig = "swapExactETHForTokens(uint256,address[],address,uint256)";
     expect(selectors).toContain(sig);
@@ -76,6 +76,12 @@ describe('loaders module', () => {
     expect(name).toBeFalsy()
   })
 
+  online_test('SourcifyABILoader_getContract_missing', async () => {
+    const loader = new SourcifyABILoader();
+    const r = await loader.getContract("0x0000000000000000000000000000000000000000");
+    expect(r.ok).toBeFalsy();
+  })
+
   online_test('SourcifyABILoader_getContract_UniswapV3Factory', async () => {
     const loader = new SourcifyABILoader();
     const {abi, name} = await loader.getContract("0x1F98431c8aD98523631AE4a59f267346ea31F984");
@@ -87,10 +93,14 @@ describe('loaders module', () => {
 
   online_test('EtherscanABILoader_getContract', async () => {
     const loader = new EtherscanABILoader({ apiKey: process.env["ETHERSCAN_API_KEY"] });
-    const {abi} = await loader.getContract("0x7a250d5630b4cf539739df2c5dacb4c659f2488d");
-    const selectors = Object.values(selectorsFromABI(abi));
+    const result = await loader.getContract("0x7a250d5630b4cf539739df2c5dacb4c659f2488d");
+    const selectors = Object.values(selectorsFromABI(result.abi));
     const sig = "swapExactETHForTokens(uint256,address[],address,uint256)";
     expect(selectors).toContain(sig);
+
+    const sources = result.getSources && await result.getSources();
+    expect(sources && sources[0].content).toContain("pragma solidity");
+
   }, SLOW_ETHERSCAN_TIMEOUT)
 
   online_test('EtherscanABILoader_getContract_UniswapV3Factory', async () => {
@@ -146,11 +156,16 @@ describe('loaders module', () => {
     const loader = new MultiABILoader([
       new EtherscanABILoader({ apiKey: process.env["ETHERSCAN_API_KEY"] }),
     ]);
-    const {abi, name} = await loader.getContract(address);
+    const res = await loader.getContract(address);
     const sig = "owner()";
-    const selectors = Object.values(selectorsFromABI(abi));
+    const selectors = Object.values(selectorsFromABI(res.abi));
     expect(selectors).toContain(sig);
-    expect(name).toEqual("UniswapV3Factory");
+    expect(res.name).toEqual("UniswapV3Factory");
+
+    const sources = res.getSources && await res.getSources();
+    expect(
+      sources?.find(s => s.path?.endsWith("contracts/libraries/UnsafeMath.sol"))?.content
+    ).contains("pragma solidity");
   }, SLOW_ETHERSCAN_TIMEOUT);
 
   online_test('SamczunSignatureLookup', async () => {
@@ -174,3 +189,15 @@ describe('loaders module', () => {
     expect(selectors).toContain(sig);
   })
 })
+
+describe('loaders helpers', () => {
+  test('SourcifyABILoader.stripPathPrefix', () => {
+    expect(
+      SourcifyABILoader.stripPathPrefix("/contracts/full_match/1/0x1F98431c8aD98523631AE4a59f267346ea31F984/sources/contracts/interfaces/IERC20Minimal.sol")
+    ).toEqual("contracts/interfaces/IERC20Minimal.sol");
+
+    expect(
+      SourcifyABILoader.stripPathPrefix("/contracts/full_match/1/0x1F98431c8aD98523631AE4a59f267346ea31F984/metadata.json")
+    ).toEqual("metadata.json");
+  });
+});