Repository: solid/community-server
Pull request #428 (2020-12-16 13:46:54)

feat: Support .meta files for pod provisioning
This opens up a lot more possibilities with the templates and will prevent random pod deletion after #426 .

This does encode what the metadata files should look like (content-type wise and extension) in the template generator, but generalizing that would cover most of the generator so probably would need a new class anyway if new template types should be supported.

RubenVerborgh: > This does encode what the metadata files should look like

Fine with me!

Patch:
diff --git a/src/pods/generate/TemplatedResourcesGenerator.ts b/src/pods/generate/TemplatedResourcesGenerator.ts
index ae16fd218..a35cfefc6 100644
--- a/src/pods/generate/TemplatedResourcesGenerator.ts
+++ b/src/pods/generate/TemplatedResourcesGenerator.ts
@@ -1,8 +1,14 @@
 import { promises as fsPromises } from 'fs';
 import { posix } from 'path';
+import { Parser } from 'n3';
 import { RepresentationMetadata } from '../../ldp/representation/RepresentationMetadata';
 import type { ResourceIdentifier } from '../../ldp/representation/ResourceIdentifier';
-import type { FileIdentifierMapper, FileIdentifierMapperFactory } from '../../storage/mapping/FileIdentifierMapper';
+import type {
+  FileIdentifierMapper,
+  FileIdentifierMapperFactory,
+  ResourceLink,
+} from '../../storage/mapping/FileIdentifierMapper';
+import { isContainerIdentifier } from '../../util/PathUtil';
 import { guardedStreamFrom } from '../../util/StreamUtil';
 import type { Resource, ResourcesGenerator } from './ResourcesGenerator';
 import type { TemplateEngine } from './TemplateEngine';
@@ -20,6 +26,7 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
   private readonly templateFolder: string;
   private readonly factory: FileIdentifierMapperFactory;
   private readonly engine: TemplateEngine;
+  private readonly metaExtension = '.meta';
 
   /**
    * A mapper is needed to convert the template file paths to identifiers relative to the given base identifier.
@@ -36,57 +43,132 @@ export class TemplatedResourcesGenerator implements ResourcesGenerator {
 
   public async* generate(location: ResourceIdentifier, options: Dict<string>): AsyncIterable<Resource> {
     const mapper = await this.factory.create(location.path, this.templateFolder);
-    yield* this.parseFolder(this.templateFolder, mapper, options);
+    const folderLink = await mapper.mapFilePathToUrl(this.templateFolder, true);
+    yield* this.parseFolder(folderLink, mapper, options);
   }
 
   /**
    * Generates results for all entries in the given folder, including the folder itself.
    */
-  private async* parseFolder(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
+  private async* parseFolder(folderLink: ResourceLink, mapper: FileIdentifierMapper, options: Dict<string>):
   AsyncIterable<Resource> {
-    // Generate representation for the container
-    const link = await mapper.mapFilePathToUrl(filePath, true);
-    yield {
-      identifier: link.identifier,
-      representation: {
-        binary: true,
-        data: guardedStreamFrom([]),
-        metadata: new RepresentationMetadata(link.identifier),
-      },
-    };
+    // Group resource links with their corresponding metadata links
+    const links = await this.groupLinks(this.generateLinks(folderLink.filePath, mapper));
+
+    // Remove root metadata if it exists
+    const metaLink = links[folderLink.identifier.path]?.meta;
+    // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+    delete links[folderLink.identifier.path];
 
-    // Generate representations for all resources in this container
-    const files = await fsPromises.readdir(filePath);
-    for (const childName of files) {
-      const childPath = joinPath(filePath, childName);
-      const childStats = await fsPromises.lstat(childPath);
-      if (childStats.isDirectory()) {
-        yield* this.parseFolder(childPath, mapper, options);
-      } else if (childStats.isFile()) {
-        yield this.generateDocument(childPath, mapper, options);
+    yield this.generateResource(folderLink, options, metaLink);
+
+    for (const { link, meta } of Object.values(links)) {
+      if (isContainerIdentifier(link.identifier)) {
+        yield* this.parseFolder(link, mapper, options);
+      } else {
+        yield this.generateResource(link, options, meta);
       }
     }
   }
 
   /**
-   * Generates a new Representation corresponding to the template file at the given location.
+   * Generates ResourceLinks for each entry in the given folder.
    */
-  private async generateDocument(filePath: string, mapper: FileIdentifierMapper, options: Dict<string>):
+  private async* generateLinks(folderPath: string, mapper: FileIdentifierMapper): AsyncIterable<ResourceLink> {
+    const files = await fsPromises.readdir(folderPath);
+    for (const name of files) {
+      const filePath = joinPath(folderPath, name);
+      const stats = await fsPromises.lstat(filePath);
+      yield mapper.mapFilePathToUrl(filePath, stats.isDirectory());
+    }
+  }
+
+  /**
+   * Parses a group of ResourceLinks so resources and their metadata are grouped together.
+   */
+  private async groupLinks(linkGen: AsyncIterable<ResourceLink>):
+  Promise<Record<string, { link: ResourceLink; meta?: ResourceLink }>> {
+    const links: Record<string, { link: ResourceLink; meta?: ResourceLink }> = { };
+    for await (const link of linkGen) {
+      const { path } = link.identifier;
+      if (this.isMeta(path)) {
+        const resourcePath = this.metaToResource(link.identifier).path;
+        links[resourcePath] = Object.assign(links[resourcePath] || {}, { meta: link });
+      } else {
+        links[path] = Object.assign(links[path] || {}, { link });
+      }
+    }
+    return links;
+  }
+
+  /**
+   * Generates a Resource object for the given ResourceLink.
+   * In the case of documents the corresponding template will be used.
+   * If a ResourceLink of metadata is provided the corresponding data will be added as metadata.
+   */
+  private async generateResource(link: ResourceLink, options: Dict<string>, metaLink?: ResourceLink):
   Promise<Resource> {
-    const link = await mapper.mapFilePathToUrl(filePath, false);
+    const data: string[] = [];
     const metadata = new RepresentationMetadata(link.identifier);
-    metadata.contentType = link.contentType;
 
-    const raw = await fsPromises.readFile(filePath, 'utf8');
-    const compiled = this.engine.apply(raw, options);
+    // Read file if it is not a container
+    if (!isContainerIdentifier(link.identifier)) {
+      const compiled = await this.parseTemplate(link.filePath, options);
+      data.push(compiled);
+      metadata.contentType = link.contentType;
+    }
+
+    // Add metadata from meta file if there is one
+    if (metaLink) {
+      const rawMetadata = await this.generateMetadata(metaLink, options);
+      metadata.addQuads(rawMetadata.quads());
+    }
 
     return {
       identifier: link.identifier,
       representation: {
         binary: true,
-        data: guardedStreamFrom([ compiled ]),
+        data: guardedStreamFrom(data),
         metadata,
       },
     };
   }
+
+  /**
+   * Generates a RepresentationMetadata using the given template.
+   */
+  private async generateMetadata(metaLink: ResourceLink, options: Dict<string>):
+  Promise<RepresentationMetadata> {
+    const identifier = this.metaToResource(metaLink.identifier);
+    const metadata = new RepresentationMetadata(identifier);
+
+    const data = await this.parseTemplate(metaLink.filePath, options);
+    const parser = new Parser({ format: metaLink.contentType, baseIRI: identifier.path });
+    const quads = parser.parse(data);
+    metadata.addQuads(quads);
+
+    return metadata;
+  }
+
+  /**
+   * Applies the given options to the template found at the given path.
+   */
+  private async parseTemplate(filePath: string, options: Dict<string>): Promise<string> {
+    const raw = await fsPromises.readFile(filePath, 'utf8');
+    return this.engine.apply(raw, options);
+  }
+
+  /**
+   * Verifies if the given path corresponds to a metadata file.
+   */
+  private isMeta(path: string): boolean {
+    return path.endsWith(this.metaExtension);
+  }
+
+  /**
+   * Converts a generated metadata identifier to the identifier of its corresponding resource.
+   */
+  private metaToResource(metaIdentifier: ResourceIdentifier): ResourceIdentifier {
+    return { path: metaIdentifier.path.slice(0, -this.metaExtension.length) };
+  }
 }
diff --git a/src/storage/DataAccessorBasedStore.ts b/src/storage/DataAccessorBasedStore.ts
index da97b8e34..cad36e803 100644
--- a/src/storage/DataAccessorBasedStore.ts
+++ b/src/storage/DataAccessorBasedStore.ts
@@ -1,6 +1,6 @@
 import arrayifyStream from 'arrayify-stream';
 import { DataFactory } from 'n3';
-import type { Quad } from 'rdf-js';
+import type { Quad, Term } from 'rdf-js';
 import { v4 as uuid } from 'uuid';
 import type { Representation } from '../ldp/representation/Representation';
 import { RepresentationMetadata } from '../ldp/representation/RepresentationMetadata';
@@ -8,6 +8,7 @@ import type { ResourceIdentifier } from '../ldp/representation/ResourceIdentifie
 import { INTERNAL_QUADS } from '../util/ContentTypes';
 import { BadRequestHttpError } from '../util/errors/BadRequestHttpError';
 import { ConflictHttpError } from '../util/errors/ConflictHttpError';
+import { InternalServerError } from '../util/errors/InternalServerError';
 import { MethodNotAllowedHttpError } from '../util/errors/MethodNotAllowedHttpError';
 import { NotFoundHttpError } from '../util/errors/NotFoundHttpError';
 import { NotImplementedHttpError } from '../util/errors/NotImplementedHttpError';
@@ -318,14 +319,13 @@ export class DataAccessorBasedStore implements ResourceStore {
    * @param suffix - Suffix of the URI. Can be the full URI, but only the last part is required.
    */
   protected isNewContainer(metadata: RepresentationMetadata, suffix?: string): boolean {
-    let isContainer: boolean;
-    try {
-      isContainer = this.isExistingContainer(metadata);
-    } catch {
-      const slug = suffix ?? metadata.get(HTTP.slug)?.value;
-      isContainer = Boolean(slug && isContainerPath(slug));
+    // Should not use `isExistingContainer` since the metadata might contain unrelated type triples
+    // It's not because there is no container type triple that the new resource is not a container
+    if (this.hasContainerType(metadata.getAll(RDF.type))) {
+      return true;
     }
-    return isContainer;
+    const slug = suffix ?? metadata.get(HTTP.slug)?.value;
+    return Boolean(slug && isContainerPath(slug));
   }
 
   /**
@@ -335,9 +335,16 @@ export class DataAccessorBasedStore implements ResourceStore {
    */
   protected isExistingContainer(metadata: RepresentationMetadata): boolean {
     const types = metadata.getAll(RDF.type);
-    if (types.length === 0) {
-      throw new Error('Unknown resource type.');
+    if (!types.some((type): boolean => type.value === LDP.Resource)) {
+      throw new InternalServerError('Unknown resource type.');
     }
+    return this.hasContainerType(types);
+  }
+
+  /**
+   * Checks in a list of types if any of them match a Container type.
+   */
+  protected hasContainerType(types: Term[]): boolean {
     return types.some((type): boolean => type.value === LDP.Container || type.value === LDP.BasicContainer);
   }
 
diff --git a/src/storage/accessors/FileDataAccessor.ts b/src/storage/accessors/FileDataAccessor.ts
index 744d67d70..6fef0fdb3 100644
--- a/src/storage/accessors/FileDataAccessor.ts
+++ b/src/storage/accessors/FileDataAccessor.ts
@@ -16,7 +16,7 @@ import type { Guarded } from '../../util/GuardedStream';
 import { isContainerIdentifier } from '../../util/PathUtil';
 import { parseQuads, pushQuad, serializeQuads } from '../../util/QuadUtil';
 import { generateContainmentQuads, generateResourceQuads } from '../../util/ResourceUtil';
-import { CONTENT_TYPE, DCTERMS, POSIX, RDF, XSD } from '../../util/UriConstants';
+import { CONTENT_TYPE, DCTERMS, LDP, POSIX, RDF, XSD } from '../../util/UriConstants';
 import { toNamedNode, toTypedLiteral } from '../../util/UriUtil';
 import type { FileIdentifierMapper, ResourceLink } from '../mapping/FileIdentifierMapper';
 import type { DataAccessor } from './DataAccessor';
@@ -210,7 +210,9 @@ export class FileDataAccessor implements DataAccessor {
    */
   private async writeMetadata(link: ResourceLink, metadata: RepresentationMetadata): Promise<boolean> {
     // These are stored by file system conventions
-    metadata.removeAll(RDF.type);
+    metadata.remove(RDF.type, toNamedNode(LDP.Resource));
+    metadata.remove(RDF.type, toNamedNode(LDP.Container));
+    metadata.remove(RDF.type, toNamedNode(LDP.BasicContainer));
     metadata.removeAll(CONTENT_TYPE);
     const quads = metadata.quads();
     const metadataLink = await this.getMetadataLink(link.identifier);
diff --git a/templates/.meta b/templates/.meta
new file mode 100644
index 000000000..5ecce5efa
--- /dev/null
+++ b/templates/.meta
@@ -0,0 +1,3 @@
+@prefix pim: <http://www.w3.org/ns/pim/space#>.
+
+<> a pim:Storage.


Test patch:
diff --git a/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts b/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts
index 3f48d1645..531bd5279 100644
--- a/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts
+++ b/test/unit/pods/generate/TemplatedResourcesGenerator.test.ts
@@ -1,6 +1,6 @@
 import type { ResourceIdentifier } from '../../../../src/ldp/representation/ResourceIdentifier';
+import { HandlebarsTemplateEngine } from '../../../../src/pods/generate/HandlebarsTemplateEngine';
 import { TemplatedResourcesGenerator } from '../../../../src/pods/generate/TemplatedResourcesGenerator';
-import type { TemplateEngine } from '../../../../src/pods/generate/TemplateEngine';
 import type {
   FileIdentifierMapper,
   FileIdentifierMapperFactory,
@@ -9,7 +9,6 @@ import type {
 import { ensureTrailingSlash, trimTrailingSlashes } from '../../../../src/util/PathUtil';
 import { readableToString } from '../../../../src/util/StreamUtil';
 import { mockFs } from '../../../util/Util';
-import Dict = NodeJS.Dict;
 
 jest.mock('fs');
 
@@ -30,13 +29,6 @@ class DummyFactory implements FileIdentifierMapperFactory {
   }
 }
 
-class DummyEngine implements TemplateEngine {
-  public apply(template: string, options: Dict<string>): string {
-    const keys = Object.keys(options);
-    return `${template}${keys.map((key): string => `{${key}:${options[key]}}`).join('')}`;
-  }
-}
-
 const genToArray = async<T>(iterable: AsyncIterable<T>): Promise<T[]> => {
   const arr: T[] = [];
   for await (const result of iterable) {
@@ -47,7 +39,8 @@ const genToArray = async<T>(iterable: AsyncIterable<T>): Promise<T[]> => {
 
 describe('A TemplatedResourcesGenerator', (): void => {
   const rootFilePath = 'templates';
-  const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new DummyEngine());
+  // Using handlebars engine since it's smaller than any possible dummy
+  const generator = new TemplatedResourcesGenerator(rootFilePath, new DummyFactory(), new HandlebarsTemplateEngine());
   let cache: { data: any };
   const template = '<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.';
   const location = { path: 'http://test.com/alice/' };
@@ -68,11 +61,11 @@ describe('A TemplatedResourcesGenerator', (): void => {
     expect(representation.binary).toBe(true);
     expect(representation.metadata.contentType).toBe('text/turtle');
     await expect(readableToString(representation.data)).resolves
-      .toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
+      .toEqual(`<${webId}> a <http://xmlns.com/foaf/0.1/Person>.`);
   });
 
-  it('creates the necessary containers and ignores non-files.', async(): Promise<void> => {
-    cache.data = { container: { container: { template }}, 2: 5 };
+  it('creates the necessary containers.', async(): Promise<void> => {
+    cache.data = { container: { container: { template }}};
     const result = await genToArray(generator.generate(location, { webId }));
     const identifiers = result.map((res): ResourceIdentifier => res.identifier);
     const id = { path: `${location.path}container/container/template` };
@@ -85,6 +78,37 @@ describe('A TemplatedResourcesGenerator', (): void => {
 
     const { representation } = result[3];
     await expect(readableToString(representation.data)).resolves
-      .toEqual(`<{{webId}}> a <http://xmlns.com/foaf/0.1/Person>.{webId:${webId}}`);
+      .toEqual(`<${webId}> a <http://xmlns.com/foaf/0.1/Person>.`);
+  });
+
+  it('adds metadata from .meta files.', async(): Promise<void> => {
+    const meta = '<> <pre:has> "metadata".';
+    cache.data = { '.meta': meta, container: { 'template.meta': meta, template }};
+
+    // Not using options since our dummy template generator generates invalid turtle
+    const result = await genToArray(generator.generate(location, { webId }));
+    const identifiers = result.map((res): ResourceIdentifier => res.identifier);
+    expect(identifiers).toEqual([
+      location,
+      { path: `${location.path}container/` },
+      { path: `${location.path}container/template` },
+    ]);
+    // Root has the 1 raw metadata triple (with <> changed to its identifier)
+    const rootMetadata = result[0].representation.metadata;
+    expect(rootMetadata.identifier.value).toBe(location.path);
+    expect(rootMetadata.quads()).toHaveLength(1);
+    expect(rootMetadata.get('pre:has')?.value).toBe('metadata');
+
+    // Container has no metadata triples
+    const contMetadata = result[1].representation.metadata;
+    expect(contMetadata.identifier.value).toBe(`${location.path}container/`);
+    expect(contMetadata.quads()).toHaveLength(0);
+
+    // Document has the 1 raw metadata triple (with <> changed to its identifier) and content-type
+    const docMetadata = result[2].representation.metadata;
+    expect(docMetadata.identifier.value).toBe(`${location.path}container/template`);
+    expect(docMetadata.quads()).toHaveLength(2);
+    expect(docMetadata.get('pre:has')?.value).toBe('metadata');
+    expect(docMetadata.contentType).toBe('text/turtle');
   });
 });
diff --git a/test/unit/storage/DataAccessorBasedStore.test.ts b/test/unit/storage/DataAccessorBasedStore.test.ts
index 3512dc413..560f092d8 100644
--- a/test/unit/storage/DataAccessorBasedStore.test.ts
+++ b/test/unit/storage/DataAccessorBasedStore.test.ts
@@ -10,6 +10,7 @@ import { DataAccessorBasedStore } from '../../../src/storage/DataAccessorBasedSt
 import { INTERNAL_QUADS } from '../../../src/util/ContentTypes';
 import { BadRequestHttpError } from '../../../src/util/errors/BadRequestHttpError';
 import { ConflictHttpError } from '../../../src/util/errors/ConflictHttpError';
+import { InternalServerError } from '../../../src/util/errors/InternalServerError';
 import { MethodNotAllowedHttpError } from '../../../src/util/errors/MethodNotAllowedHttpError';
 import { NotFoundHttpError } from '../../../src/util/errors/NotFoundHttpError';
 import { NotImplementedHttpError } from '../../../src/util/errors/NotImplementedHttpError';
@@ -83,7 +84,11 @@ describe('A DataAccessorBasedStore', (): void => {
     store = new DataAccessorBasedStore(accessor, identifierStrategy);
 
     containerMetadata = new RepresentationMetadata(
-      { [RDF.type]: [ DataFactory.namedNode(LDP.Container), DataFactory.namedNode(LDP.BasicContainer) ]},
+      { [RDF.type]: [
+        DataFactory.namedNode(LDP.Resource),
+        DataFactory.namedNode(LDP.Container),
+        DataFactory.namedNode(LDP.BasicContainer),
+      ]},
     );
     accessor.data[root] = { metadata: containerMetadata } as Representation;
 
@@ -270,12 +275,23 @@ describe('A DataAccessorBasedStore', (): void => {
     });
 
     it('will error if the ending slash does not match its resource type.', async(): Promise<void> => {
-      const resourceID = { path: `${root}resource/` };
+      const resourceID = { path: `${root}resource` };
+      representation.metadata.add(RDF.type, toNamedNode(LDP.Container));
       await expect(store.setRepresentation(resourceID, representation)).rejects.toThrow(
         new BadRequestHttpError('Containers should have a `/` at the end of their path, resources should not.'),
       );
     });
 
+    it('will error if the DataAccessor did not store the required type triples.', async(): Promise<void> => {
+      const resourceID = { path: `${root}resource` };
+      accessor.data[resourceID.path] = representation;
+      representation.metadata.identifier = namedNode(resourceID.path);
+      representation.metadata.removeAll(RDF.type);
+      await expect(store.setRepresentation(resourceID, representation)).rejects.toThrow(
+        new InternalServerError('Unknown resource type.'),
+      );
+    });
+
     it('errors when trying to create a container with non-RDF data.', async(): Promise<void> => {
       const resourceID = { path: `${root}container/` };
       representation.metadata.add(RDF.type, toNamedNode(LDP.Container));