Repository: Thinkmill/keystatic
Pull request #217 (2023-04-04 05:06:17)

Add an all method to the collection reader API and `resolveLinkedFiles` option to the methods that read entries
This makes using the reader API easier for a lot of common uses. (e.g. `all` for getting slugs with titles for navigation and `resolveLinkedFiles` for when you're rendering a blog post and you know you'll definitely need everything in the entry)

changeset-bot[bot]: ###  🦋  Changeset detected

Latest commit: 0c330da2ce675cefecb7628c15361d7c8aefc55b

**The changes in this PR will be included in the next version bump.**

<details><summary>This PR includes changesets to release 6 packages</summary>

  | Name                           | Type  |
| ------------------------------ | ----- |
| @keystatic/core                | Patch |
| @keystatic/test-next-app       | Patch |
| @keystatic/test-astro          | Patch |
| localization                   | Patch |
| @keystatic/test-next-pages-dir | Patch |
| @keystatic/remix-test-app      | Patch |

</details>

Not sure what this means? [Click here  to learn what changesets are](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md).

[Click here if you're a maintainer who wants to add another changeset to this PR](https://github.com/Thinkmill/keystatic/new/reader-api?filename=.changeset/polite-boxes-glow.md&value=---%0A%22%40keystatic%2Ftest-next-app%22%3A%20patch%0A%22%40keystatic%2Fcore%22%3A%20patch%0A---%0A%0AAdd%20an%20all%20method%20to%20the%20collection%20reader%20API%20and%20%60resolveLinkedFiles%60%20option%20to%20the%20methods%20that%20read%20entries%0A)


vercel[bot]: [vc]: #OcKPCsWhc4mnq8ueDyT9yGUMVk1L+OqVq9YZBUitMWo=:eyJpc01vbm9yZXBvIjp0cnVlLCJ0eXBlIjoiZ2l0aHViIiwicHJvamVjdHMiOlt7Im5hbWUiOiJrZXlzdGF0aWMiLCJyb290RGlyZWN0b3J5Ijoia2V5c3RhdGljIiwiaW5zcGVjdG9yVXJsIjoiaHR0cHM6Ly92ZXJjZWwuY29tL3RoaW5rbWlsbC9rZXlzdGF0aWMvSENjZTFxNUYydVZFUHNRRFRHRUZYa1Vkc3VaRSIsIm5leHRDb21taXRTdGF0dXMiOiJQRU5ESU5HIn0seyJuYW1lIjoidm91c3NvaXIiLCJyb290RGlyZWN0b3J5IjoiZGVzaWduLXN5c3RlbS9kb2NzIiwiaW5zcGVjdG9yVXJsIjoiaHR0cHM6Ly92ZXJjZWwuY29tL3RoaW5rbWlsbC92b3Vzc29pci9HeGZxcTZoWExQaHRiTVc4WFZNNXlBM3JrYXUzIiwibmV4dENvbW1pdFN0YXR1cyI6IlBFTkRJTkcifV19
**The latest updates on your projects**. Learn more about [Vercel for Git ↗︎](https://vercel.link/github-learn-more)

| Name | Status | Preview | Updated (UTC) |
| :--- | :----- | :------ | :------ |
| **keystatic** | 🔄 Building ([Inspect](https://vercel.com/thinkmill/keystatic/HCce1q5F2uVEPsQDTGEFXkUdsuZE)) |  | Apr 4, 2023 5:06am |
| **voussoir** | 🔄 Building ([Inspect](https://vercel.com/thinkmill/voussoir/Gxfqq6hXLPhtbMW8XVM5yA3rkau3)) |  | Apr 4, 2023 5:06am |




Patch:
diff --git a/.changeset/slow-spoons-travel.md b/.changeset/slow-spoons-travel.md
new file mode 100644
index 00000000..b3cf0674
--- /dev/null
+++ b/.changeset/slow-spoons-travel.md
@@ -0,0 +1,5 @@
+---
+'@keystatic/core': patch
+---
+
+Add an `all` method to the collection reader API to load all entries in a collection and add `resolveLinkedFiles` option to the methods that read entries to eagerly load document fields (and potentially other kinds of fields in the future).
diff --git a/packages/keystatic/DocumentEditor/component-blocks/api.tsx b/packages/keystatic/DocumentEditor/component-blocks/api.tsx
index 778dcaf0..c218bd43 100644
--- a/packages/keystatic/DocumentEditor/component-blocks/api.tsx
+++ b/packages/keystatic/DocumentEditor/component-blocks/api.tsx
@@ -456,14 +456,10 @@ export type ValueForReading<Schema extends ComponentSchema> =
         infer Value
       >
     ? () => Promise<Value>
-    : Schema extends FormFieldWithFileNotRequiringContentsForReader<
-        any,
-        infer Value
-      >
-    ? Value
-    : Schema extends SlugFormField<infer Value, any>
-    ? Value
-    : Schema extends BasicFormField<infer Value>
+    : Schema extends
+        | FormFieldWithFileNotRequiringContentsForReader<any, infer Value>
+        | SlugFormField<infer Value, any>
+        | BasicFormField<infer Value>
     ? Value
     : Schema extends ObjectField<infer Value>
     ? {
@@ -483,6 +479,33 @@ export type ValueForReading<Schema extends ComponentSchema> =
     ? readonly ValueForReading<ElementField>[]
     : never;
 
+export type ValueForReadingDeep<Schema extends ComponentSchema> =
+  Schema extends ChildField
+    ? null
+    : Schema extends
+        | FormFieldWithFileRequiringContentsForReader<any, infer Value>
+        | FormFieldWithFileNotRequiringContentsForReader<any, infer Value>
+        | SlugFormField<infer Value, any>
+        | BasicFormField<infer Value>
+    ? Value
+    : Schema extends ObjectField<infer Value>
+    ? {
+        readonly [Key in keyof Value]: ValueForReadingDeep<Value[Key]>;
+      }
+    : Schema extends ConditionalField<infer DiscriminantField, infer Values>
+    ? {
+        readonly [Key in keyof Values]: {
+          readonly discriminant: DiscriminantStringToDiscriminantValue<
+            DiscriminantField,
+            Key
+          >;
+          readonly value: ValueForReadingDeep<Values[Key]>;
+        };
+      }[keyof Values]
+    : Schema extends ArrayField<infer ElementField>
+    ? readonly ValueForReadingDeep<ElementField>[]
+    : never;
+
 export type InferRenderersForComponentBlocks<
   ComponentBlocks extends Record<string, ComponentBlock<any>>
 > = {
diff --git a/packages/keystatic/src/reader.ts b/packages/keystatic/src/reader.ts
index c23187d7..f352028d 100644
--- a/packages/keystatic/src/reader.ts
+++ b/packages/keystatic/src/reader.ts
@@ -5,6 +5,7 @@ import {
   ObjectField,
   SlugFormField,
   ValueForReading,
+  ValueForReadingDeep,
 } from '../DocumentEditor/component-blocks/api';
 import fs from 'fs/promises';
 import path from 'path';
@@ -28,25 +29,71 @@ import {
 import { getValueAtPropPath } from '../DocumentEditor/component-blocks/utils';
 import { Dirent } from 'fs';
 
+type EntryReaderOpts = { resolveLinkedFiles?: boolean };
+
+type ValueForReadingWithMode<
+  Schema extends ComponentSchema,
+  Opts extends boolean | undefined
+> = Opts extends true ? ValueForReadingDeep<Schema> : ValueForReading<Schema>;
+
+type OptionalChain<
+  T extends {} | undefined,
+  Key extends keyof (T & {})
+> = T extends {} ? T[Key] : undefined;
+
 type CollectionReader<
   Schema extends Record<string, ComponentSchema>,
   SlugField extends string
 > = {
-  read: (id: string) => Promise<
+  read: <Opts extends [opts?: EntryReaderOpts]>(
+    slug: string,
+    ...opts: Opts & [opts?: EntryReaderOpts]
+  ) => Promise<
     | {
         [Key in keyof Schema]: SlugField extends Key
-          ? Schema[Key] extends SlugFormField<any, infer Value>
-            ? Value
-            : ValueForReading<Schema[Key]>
-          : ValueForReading<Schema[Key]>;
+          ? Schema[Key] extends SlugFormField<any, infer SlugSerializedValue>
+            ? SlugSerializedValue
+            : ValueForReadingWithMode<
+                Schema[Key],
+                OptionalChain<Opts[0], 'resolveLinkedFiles'>
+              >
+          : ValueForReadingWithMode<
+              Schema[Key],
+              OptionalChain<Opts[0], 'resolveLinkedFiles'>
+            >;
       }
     | null
   >;
+  all: <Opts extends [opts?: EntryReaderOpts]>(
+    ...opts: Opts & [opts?: EntryReaderOpts]
+  ) => Promise<
+    {
+      slug: string;
+      entry: {
+        [Key in keyof Schema]: SlugField extends Key
+          ? Schema[Key] extends SlugFormField<any, infer SlugSerializedValue>
+            ? SlugSerializedValue
+            : ValueForReadingWithMode<
+                Schema[Key],
+                OptionalChain<Opts[0], 'resolveLinkedFiles'>
+              >
+          : ValueForReadingWithMode<
+              Schema[Key],
+              OptionalChain<Opts[0], 'resolveLinkedFiles'>
+            >;
+      };
+    }[]
+  >;
   list: () => Promise<string[]>;
 };
 
 type SingletonReader<Schema extends Record<string, ComponentSchema>> = {
-  read: () => Promise<ValueForReading<ObjectField<Schema>> | null>;
+  read: <Opts extends [opts?: EntryReaderOpts]>(
+    ...opts: Opts & [opts?: EntryReaderOpts]
+  ) => Promise<ValueForReadingWithMode<
+    ObjectField<Schema>,
+    OptionalChain<Opts[0], 'resolveLinkedFiles'>
+  > | null>;
 };
 
 async function getAllEntries(
@@ -90,67 +137,84 @@ function collectionReader(
   const schema = fields.object(collectionConfig.schema);
   const glob = getSlugGlobForCollection(config, collection);
   const extension = getDataFileExtension(formatInfo);
-  return {
-    read: (slug: string) => {
-      return readItem(
-        schema,
-        formatInfo,
-        getCollectionItemPath(config, collection, slug),
-        {
-          field: collectionConfig.slugField,
-          slug,
-          glob,
-        },
-        repoPath
-      );
-    },
-    async list() {
-      const entries: { entry: Dirent; name: string }[] =
-        glob === '*'
-          ? (
-              await fs
-                .readdir(path.join(repoPath, collectionPath), {
-                  withFileTypes: true,
-                })
-                .catch(err => {
-                  if ((err as any).code === 'ENOENT') {
-                    return [];
-                  }
-                  throw err;
-                })
-            ).map(x => ({ entry: x, name: x.name }))
-          : await getAllEntries(path.join(repoPath, collectionPath), '');
-
-      return (
-        await Promise.all(
-          entries.map(async x => {
-            if (formatInfo.dataLocation === 'index') {
-              if (!x.entry.isDirectory()) return [];
-              try {
-                await fs.stat(
-                  path.join(
-                    repoPath,
-                    getEntryDataFilepath(
-                      `${collectionPath}/${x.name}`,
-                      formatInfo
-                    )
-                  )
-                );
-                return [x.name];
-              } catch (err) {
+  async function list() {
+    const entries: { entry: Dirent; name: string }[] =
+      glob === '*'
+        ? (
+            await fs
+              .readdir(path.join(repoPath, collectionPath), {
+                withFileTypes: true,
+              })
+              .catch(err => {
                 if ((err as any).code === 'ENOENT') {
                   return [];
                 }
                 throw err;
+              })
+          ).map(x => ({ entry: x, name: x.name }))
+        : await getAllEntries(path.join(repoPath, collectionPath), '');
+
+    return (
+      await Promise.all(
+        entries.map(async x => {
+          if (formatInfo.dataLocation === 'index') {
+            if (!x.entry.isDirectory()) return [];
+            try {
+              await fs.stat(
+                path.join(
+                  repoPath,
+                  getEntryDataFilepath(
+                    `${collectionPath}/${x.name}`,
+                    formatInfo
+                  )
+                )
+              );
+              return [x.name];
+            } catch (err) {
+              if ((err as any).code === 'ENOENT') {
+                return [];
               }
-            } else {
-              if (!x.entry.isFile() || !x.name.endsWith(extension)) return [];
-              return [x.name.slice(0, -extension.length)];
+              throw err;
             }
+          } else {
+            if (!x.entry.isFile() || !x.name.endsWith(extension)) return [];
+            return [x.name.slice(0, -extension.length)];
+          }
+        })
+      )
+    ).flat();
+  }
+  const read: CollectionReader<any, any>['read'] = (slug, ...args) =>
+    readItem(
+      schema,
+      formatInfo,
+      getCollectionItemPath(config, collection, slug),
+      {
+        field: collectionConfig.slugField,
+        slug,
+        glob,
+      },
+      repoPath,
+      args[0]
+    );
+
+  return {
+    read,
+    // TODO: this could drop the fs.stat call that list does for each item
+    // since we just immediately read it
+    all: async (...args) => {
+      const slugs = await list();
+      return (
+        await Promise.all(
+          slugs.map(async slug => {
+            const entry = await read(slug, args[0]);
+            if (entry === null) return [];
+            return [{ slug, entry }];
           })
         )
       ).flat();
     },
+    list,
   };
 }
 
@@ -165,7 +229,8 @@ async function readItem(
         glob: Glob;
       }
     | undefined,
-  repoPath: string
+  repoPath: string,
+  opts: EntryReaderOpts | undefined
 ) {
   let dataFile: Uint8Array;
   try {
@@ -194,60 +259,67 @@ async function readItem(
         }
   );
   const requiredFiles = getRequiredFiles(validated, schema, slugField?.slug);
-  for (const file of requiredFiles) {
-    const parentValue = getValueAtPropPath(
-      validated,
-      file.path.slice(0, -1)
-    ) as any;
-    const keyOnParent = file.path[file.path.length - 1];
-    const originalValue = parentValue[keyOnParent];
-    if (file.schema.serializeToFile.reader.requiresContentInReader) {
-      parentValue[keyOnParent] = async () => {
-        const loadedFiles = new Map<string, Uint8Array>();
-        if (file.file) {
-          const filepath = `${
-            file.file.parent
-              ? `${file.file.parent}${slugField ? slugField.slug : ''}`
-              : itemDir
-          }/${file.file.filename}`;
-          if (file.file.filename === extraFakeFile?.path) {
-            loadedFiles.set(filepath, extraFakeFile.contents);
-          } else {
-            const contents = await fs
-              .readFile(path.resolve(repoPath, filepath))
-              .catch(x => {
-                if ((x as any).code === 'ENOENT') return undefined;
-                throw x;
-              });
-            if (contents) {
-              loadedFiles.set(filepath, contents);
+  await Promise.all(
+    requiredFiles.map(async file => {
+      const parentValue = getValueAtPropPath(
+        validated,
+        file.path.slice(0, -1)
+      ) as any;
+      const keyOnParent = file.path[file.path.length - 1];
+      const originalValue = parentValue[keyOnParent];
+      if (file.schema.serializeToFile.reader.requiresContentInReader) {
+        const loadData = async () => {
+          const loadedFiles = new Map<string, Uint8Array>();
+          if (file.file) {
+            const filepath = `${
+              file.file.parent
+                ? `${file.file.parent}${slugField ? slugField.slug : ''}`
+                : itemDir
+            }/${file.file.filename}`;
+            if (file.file.filename === extraFakeFile?.path) {
+              loadedFiles.set(filepath, extraFakeFile.contents);
+            } else {
+              const contents = await fs
+                .readFile(path.resolve(repoPath, filepath))
+                .catch(x => {
+                  if ((x as any).code === 'ENOENT') return undefined;
+                  throw x;
+                });
+              if (contents) {
+                loadedFiles.set(filepath, contents);
+              }
             }
           }
+          return parseSerializedFormField(
+            originalValue,
+            file,
+            loadedFiles,
+            'read',
+            itemDir,
+            slugField?.slug,
+            validated,
+            schema
+          );
+        };
+        if (opts?.resolveLinkedFiles) {
+          parentValue[keyOnParent] = await loadData();
+        } else {
+          parentValue[keyOnParent] = loadData;
         }
-        return parseSerializedFormField(
+      } else {
+        parentValue[keyOnParent] = parseSerializedFormField(
           originalValue,
           file,
-          loadedFiles,
+          new Map(),
           'read',
           itemDir,
           slugField?.slug,
           validated,
           schema
         );
-      };
-    } else {
-      parentValue[keyOnParent] = parseSerializedFormField(
-        originalValue,
-        file,
-        new Map(),
-        'read',
-        itemDir,
-        slugField?.slug,
-        validated,
-        schema
-      );
-    }
-  }
+      }
+    })
+  );
 
   return validated;
 }
@@ -261,8 +333,8 @@ function singletonReader(
   const singletonPath = getSingletonPath(config, singleton);
   const schema = fields.object(config.singletons![singleton].schema);
   return {
-    read: () =>
-      readItem(schema, formatInfo, singletonPath, undefined, repoPath),
+    read: (...args) =>
+      readItem(schema, formatInfo, singletonPath, undefined, repoPath, args[0]),
   };
 }
 


Test patch:
diff --git a/packages/keystatic/reader.test.tsx b/packages/keystatic/reader.test.tsx
index 4a67482b..699d1572 100644
--- a/packages/keystatic/reader.test.tsx
+++ b/packages/keystatic/reader.test.tsx
@@ -114,3 +114,148 @@ test('read', async () => {
     ]
   `);
 });
+
+test('read deep', async () => {
+  const reader = createReader(path.join(__dirname, 'test-data'), localConfig);
+  const result = await reader.collections.posts.read('2023-is-finally-here', {
+    resolveLinkedFiles: true,
+  });
+  expect(result).toMatchInlineSnapshot(`
+    {
+      "authors": [
+        {
+          "bio": [
+            {
+              "children": [
+                {
+                  "text": "Dan Cousens",
+                },
+              ],
+              "type": "paragraph",
+            },
+          ],
+          "name": "Dan",
+        },
+      ],
+      "content": [
+        {
+          "children": [
+            {
+              "text": "Cool, and things are direct to GitHub?",
+            },
+          ],
+          "type": "paragraph",
+        },
+        {
+          "alt": "",
+          "children": [
+            {
+              "text": "",
+            },
+          ],
+          "src": "blank.png",
+          "title": undefined,
+          "type": "image",
+        },
+        {
+          "children": [
+            {
+              "text": "",
+            },
+          ],
+          "type": "paragraph",
+        },
+      ],
+      "heroImage": "heroImage.png",
+      "publishDate": null,
+      "title": "oh my there is dark mode, thank god Joss",
+    }
+  `);
+});
+
+test('read all shallow', async () => {
+  const reader = createReader(path.join(__dirname, 'test-data'), localConfig);
+  const result = await reader.collections.posts.all();
+  expect(result).toMatchInlineSnapshot(`
+    [
+      {
+        "entry": {
+          "authors": [
+            {
+              "bio": [Function],
+              "name": "Dan",
+            },
+          ],
+          "content": [Function],
+          "heroImage": "heroImage.png",
+          "publishDate": null,
+          "title": "oh my there is dark mode, thank god Joss",
+        },
+        "slug": "2023-is-finally-here",
+      },
+    ]
+  `);
+});
+
+test('read all deep', async () => {
+  const reader = createReader(path.join(__dirname, 'test-data'), localConfig);
+  const result = await reader.collections.posts.all({
+    resolveLinkedFiles: true,
+  });
+  expect(result).toMatchInlineSnapshot(`
+    [
+      {
+        "entry": {
+          "authors": [
+            {
+              "bio": [
+                {
+                  "children": [
+                    {
+                      "text": "Dan Cousens",
+                    },
+                  ],
+                  "type": "paragraph",
+                },
+              ],
+              "name": "Dan",
+            },
+          ],
+          "content": [
+            {
+              "children": [
+                {
+                  "text": "Cool, and things are direct to GitHub?",
+                },
+              ],
+              "type": "paragraph",
+            },
+            {
+              "alt": "",
+              "children": [
+                {
+                  "text": "",
+                },
+              ],
+              "src": "blank.png",
+              "title": undefined,
+              "type": "image",
+            },
+            {
+              "children": [
+                {
+                  "text": "",
+                },
+              ],
+              "type": "paragraph",
+            },
+          ],
+          "heroImage": "heroImage.png",
+          "publishDate": null,
+          "title": "oh my there is dark mode, thank god Joss",
+        },
+        "slug": "2023-is-finally-here",
+      },
+    ]
+  `);
+});