Repository: ontodev/robot (java)

Issue:
Add a mirror command to cache remote ontology imports closure on local disk
Similar to https://github.com/owlcollab/owltools/wiki/Import-Chain-Mirroring

Pull request:
First pass at mirror operation and command
fixes #144

Adds a command `mirror`

TODO: add to examples/README

Patch:
diff --git a/examples/README.md b/examples/README.md
index 2961f6f7..18023205 100644
--- a/examples/README.md
+++ b/examples/README.md
@@ -147,6 +147,15 @@ Ontologies are shared in different formats. The default format used by ROBOT is
 
     robot convert --input annotated.owl --output results/annotated.obo
 
+## Mirroring
+
+Many ontologies make use of `owl:imports` to bring in other ontologies, or portions of other ontologies. Large import chains involving multiple large ontologies are more prone to run-time failure due to network errors or latency. It can therefore be beneficial to mirror or cache an external ontology's import chain locally. This can be though of as analogous to what happens with standard dependency management tools for software development.
+
+The following command will mirror a local 
+
+    robot mirror -i test.owl -d results/my-cache -o results/my-catalog.xml
+
+This will generate a directory `results/my-cache/purl.obolibrary.org/obo/ro/` with the imported ontologies as files. The file my-catalog.xml is a generated XML catalog mapping the source URIs to local files.
 
 ## Templating
 
diff --git a/examples/my-catalog.xml b/examples/my-catalog.xml
new file mode 100644
index 00000000..048fd03d
--- /dev/null
+++ b/examples/my-catalog.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<catalog prefer="public" xmlns="urn:oasis:names:tc:entity:xmlns:xml:catalog">
+
+  <!-- generated mapping -->
+  <uri name="http://purl.obolibrary.org/obo/ro/core.owl" uri="purl.obolibrary.org/obo/ro/core.owl"/>
+
+  <!-- generated mapping -->
+  <uri name="http://purl.obolibrary.org/obo/ro/annotations.owl" uri="purl.obolibrary.org/obo/ro/annotations.owl"/>
+
+  <!-- generated mapping -->
+  <uri name="http://purl.obolibrary.org/obo/ro/bfo-classes-minimal.owl" uri="purl.obolibrary.org/obo/ro/bfo-classes-minimal.owl"/>
+</catalog>
+
diff --git a/robot-command/src/main/java/org/obolibrary/robot/CommandLineInterface.java b/robot-command/src/main/java/org/obolibrary/robot/CommandLineInterface.java
index 3658ffd4..16eff4b6 100644
--- a/robot-command/src/main/java/org/obolibrary/robot/CommandLineInterface.java
+++ b/robot-command/src/main/java/org/obolibrary/robot/CommandLineInterface.java
@@ -29,6 +29,7 @@ public class CommandLineInterface {
         m.addCommand("filter",           new FilterCommand());
         m.addCommand("materialize",      new MaterializeCommand());
         m.addCommand("merge",            new MergeCommand());
+        m.addCommand("mirror",           new MirrorCommand());
         m.addCommand("query",            new QueryCommand());
         m.addCommand("reason",           new ReasonCommand());
         m.addCommand("reduce",           new ReduceCommand());
diff --git a/robot-command/src/main/java/org/obolibrary/robot/MirrorCommand.java b/robot-command/src/main/java/org/obolibrary/robot/MirrorCommand.java
new file mode 100644
index 00000000..68d3c2d3
--- /dev/null
+++ b/robot-command/src/main/java/org/obolibrary/robot/MirrorCommand.java
@@ -0,0 +1,126 @@
+package org.obolibrary.robot;
+
+import java.io.File;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
+import org.semanticweb.owlapi.model.OWLOntology;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Mirrors an ontology plus its imports closure locally
+ *
+ * @author <a href="mailto:cjmungall@lbl.gov">Chris Mungall</a>
+ */
+public class MirrorCommand implements Command {
+    /**
+     * Logger.
+     */
+    private static final Logger logger =
+        LoggerFactory.getLogger(MirrorCommand.class);
+
+    /**
+     * Store the command-line options for the command.
+     */
+    private Options options;
+
+    /**
+     * Initialize the command.
+     */
+    public MirrorCommand() {
+        Options o = CommandLineHelper.getCommonOptions();
+        o.addOption("i", "input",     true, "mirror ontology from a file");
+        o.addOption("I", "input-iri", true, "mirror ontology from an IRI");
+        o.addOption("d", "directory", true, "target directory");
+        o.addOption("o", "output",    true, "output file for catalog (default catalog-v001.xml)");
+        options = o;
+    }
+
+    /**
+     * Name of the command.
+     *
+     * @return name
+     */
+    public String getName() {
+        return "mirror";
+    }
+
+    /**
+     * Brief description of the command.
+     *
+     * @return description
+     */
+    public String getDescription() {
+        return "mirror ontology imports closure";
+    }
+
+    /**
+     * Command-line usage for the command.
+     *
+     * @return usage
+     */
+    public String getUsage() {
+        return "robot mirror --input <file> "
+             + "--directory <directory> "
+             + "--output <file>";
+    }
+
+    /**
+     * Command-line options for the command.
+     *
+     * @return options
+     */
+    public Options getOptions() {
+        return options;
+    }
+
+    /**
+     * Handle the command-line and file operations for the ReasonOperation.
+     *
+     * @param args strings to use as arguments
+     */
+    public void main(String[] args) {
+        try {
+            execute(null, args);
+        } catch (Exception e) {
+            CommandLineHelper.handleException(getUsage(), getOptions(), e);
+        }
+    }
+
+    /**
+     * Mirrors the input ontologgy and its imports closure
+     *
+     * @param state the state from the previous command, or null
+     * @param args the command-line arguments
+     * @return the input state unchanged, or a new state with the ontology
+     * @throws Exception on any problem
+     */
+    public CommandState execute(CommandState state, String[] args)
+            throws Exception {
+        CommandLine line = CommandLineHelper
+            .getCommandLine(getUsage(), getOptions(), args);
+        if (line == null) {
+            return null;
+        }
+
+        if (state == null) {
+            state = new CommandState();
+        }
+
+        IOHelper ioHelper = CommandLineHelper.getIOHelper(line);
+        state = CommandLineHelper.updateInputOntology(ioHelper, state, line);
+        OWLOntology ontology = state.getOntology();
+
+        String dir = CommandLineHelper.getDefaultValue(line, "directory", ".");
+        File catalogFile = CommandLineHelper.getOutputFile(line);
+        if (catalogFile == null) {
+            catalogFile = new File(dir + "/catalog-v001.xml");
+            logger.info("Using default catalog with specified directory:" + catalogFile);
+        }
+
+        MirrorOperation.mirror(ontology, new File(dir), catalogFile);
+        logger.info("Mirrored in " + catalogFile);
+        return state;
+    }
+}
diff --git a/robot-core/src/main/java/org/obolibrary/robot/MirrorOperation.java b/robot-core/src/main/java/org/obolibrary/robot/MirrorOperation.java
new file mode 100644
index 00000000..2f20ebe8
--- /dev/null
+++ b/robot-core/src/main/java/org/obolibrary/robot/MirrorOperation.java
@@ -0,0 +1,145 @@
+package org.obolibrary.robot;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.semanticweb.owlapi.model.IRI;
+import org.semanticweb.owlapi.model.OWLOntology;
+import org.semanticweb.owlapi.model.OWLOntologyID;
+import org.semanticweb.owlapi.model.OWLOntologyStorageException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Optional;
+
+/**
+ * Mirrors ontologies on filesystem
+ *
+ * @author cjm
+ *
+ */
+public class MirrorOperation {
+
+    /**
+     * Logger.
+     */
+    private static final Logger logger =
+            LoggerFactory.getLogger(MirrorOperation.class);
+
+
+    /**
+     * Mirrors ontologies on local filesystem
+     *
+     * @param ontologies
+     * @param baseFolder
+     * @param catalogFile
+     * @throws IOException
+     * @throws OWLOntologyStorageException
+     */
+    public static void mirror(OWLOntology rootOntology,
+            File baseFolder, File catalogFile) throws IOException, OWLOntologyStorageException {
+
+        logger.info("Mirroring ontologies: "+rootOntology);
+        Map<IRI, String> iriMap = new HashMap<>();
+        for (OWLOntology ont : rootOntology.getImportsClosure()) {
+            logger.info("Mirroring: "+ont);
+            validateImports(ont);
+
+            OWLOntologyID ontologyID = ont.getOntologyID();
+            Optional<IRI> ontologyIRI = ontologyID.getOntologyIRI();
+
+            // Not really sure why this is here, but apparently we can get
+            // an ontology without an IRI, in which case we'll generate one
+            // that is 'sort of' unique (only fails if two different machines run
+            // this tool at the exact same time).
+            //
+            if (!ontologyIRI.isPresent()) {
+                ontologyIRI = Optional.of(IRI.generateDocumentIRI());
+            }
+            // Always write the actualIRI
+            String localFilePath = getMirrorPathOfOntologyIRI(ontologyIRI.get());
+            IRI outputStream = IRI.create(new File(baseFolder, localFilePath));
+            ont.getOWLOntologyManager().saveOntology(ont, outputStream);
+            iriMap.put(ontologyIRI.get(), localFilePath);
+
+            //
+            // In case there is a difference between the source document IRI
+            // and the IRI of the resolved target (e.g., there is an HTTP
+            // redirect from a legacy IRI to a newer IRI), then write an entry
+            // in the catalog that points the legacy IRI to the newer, canonical one.
+            // Examples of this include:
+            //  http://purl.obolibrary.org/obo/so.owl
+            // which redirects to:
+            //  http://purl.obolibrary.org/obo/so-xp.obo.owl
+            //
+
+            IRI documentIRI = ont.getOWLOntologyManager().getOntologyDocumentIRI(ont);
+            if (!documentIRI.toString().startsWith("file:") &&
+                    documentIRI != ontologyIRI.get()) {
+                String sourceLocalFile = getMirrorPathOfOntologyIRI(ontologyIRI.get());
+                logger.info("Mirroring "+documentIRI+" in "+sourceLocalFile);
+                iriMap.put(documentIRI, sourceLocalFile);
+           }
+        }
+        writeCatalog(catalogFile, iriMap);
+
+    }
+
+    /**
+     * Writes a catalog-v001.xml file
+     *
+     * @param path
+     * @param iriMap
+     * @throws IOException
+     */
+    public static void writeCatalog(File catalogFile, Map<IRI,String> iriMap) throws IOException {
+        List<String> lines = new ArrayList<String>();
+        lines.add("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>");
+        lines.add("<catalog prefer=\"public\" xmlns=\"urn:oasis:names:tc:entity:xmlns:xml:catalog\">");
+        for (IRI iri : iriMap.keySet()) {
+            lines.add("");
+            lines.add("  <!-- generated mapping -->");
+            lines.add("  <uri name=\""+ iri +"\" uri=\""+ iriMap.get(iri) +"\"/>");
+        }
+        lines.add("</catalog>\n");
+        FileUtils.writeLines(catalogFile, lines);
+    }
+
+    /**
+     * Generates a local file path that should be in a 1:1 relationship with the ontology IRI
+     *
+     * The path can the point to a file that mirrors an individual ontology
+     *
+     * @param iri
+     * @return path
+     */
+    private static String getMirrorPathOfOntologyIRI(IRI iri) {
+        String iriString = iri.toString();
+        iriString = iriString.replaceFirst("http://", "").replaceFirst("https://", "");
+        iriString = iriString.replace(':', '_');
+        iriString = iriString.replace('\\', '_');
+        return iriString;
+    }
+
+
+    private static void validateImports(OWLOntology ontology) throws IOException {
+        Set<IRI> directImportDocuments = ontology.getDirectImportsDocuments();
+        Set<OWLOntology> directImports = ontology.getDirectImports();
+        if (directImports.size() < directImportDocuments.size()) {
+            // less imports than actually declared
+            // assume something went wrong, throw Exception
+            throw new IOException(
+                "The ontology has less actual imports then declared.\nActual: "
+                + directImports
+                + "\n Declared: "
+                + directImportDocuments
+            );
+        }
+    }
+}


Test patch:
diff --git a/robot-core/src/test/java/org/obolibrary/robot/CoreTest.java b/robot-core/src/test/java/org/obolibrary/robot/CoreTest.java
index 19823bcb..e93bc911 100644
--- a/robot-core/src/test/java/org/obolibrary/robot/CoreTest.java
+++ b/robot-core/src/test/java/org/obolibrary/robot/CoreTest.java
@@ -1,16 +1,22 @@
 package org.obolibrary.robot;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.StringWriter;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 
+import org.semanticweb.owlapi.apibinding.OWLManager;
 import org.semanticweb.owlapi.model.AxiomType;
 import org.semanticweb.owlapi.model.IRI;
 import org.semanticweb.owlapi.model.OWLDataFactory;
 import org.semanticweb.owlapi.model.OWLDeclarationAxiom;
 import org.semanticweb.owlapi.model.OWLOntology;
+import org.semanticweb.owlapi.model.OWLOntologyCreationException;
+import org.semanticweb.owlapi.model.OWLOntologyManager;
+
+import com.google.common.base.Optional;
 
 import uk.ac.manchester.cs.owl.owlapi.OWLDataFactoryImpl;
 
@@ -48,6 +54,37 @@ public class CoreTest {
         return ioh.loadOntology(this.getClass().getResourceAsStream(path));
     }
 
+    /**
+     * Load ontology using guessed catalog
+     * 
+     * @param path
+     * @return the loaded ontology
+     * @throws IOException
+     */
+    public OWLOntology loadOntologyWithCatalog(String path) throws IOException {
+        IOHelper ioh = new IOHelper();
+        String fullpath = this.getClass().getResource(path).getFile().toString();
+        return ioh.loadOntology(fullpath);
+    }
+    
+    /**
+     * Load ontology using explicit catalog
+     * 
+     * @param ontologyIRI
+     * @param catalogFile
+     * @return the loaded ontology
+     * @throws IOException
+     * @throws OWLOntologyCreationException 
+     */
+    public OWLOntology loadOntologyWithCatalog(IRI ontologyIRI, File catalogFile) throws IOException, OWLOntologyCreationException {
+        //TODO: move logic here to IOHelper
+        //IOHelper ioh = new IOHelper();
+        OWLOntologyManager manager =
+                OWLManager.createOWLOntologyManager();
+        manager.addIRIMapper(new CatalogXmlIRIMapper(catalogFile));
+        return manager.loadOntology(ontologyIRI);
+    }
+
     /**
      * Given an ontology path and an ontology, assert that they are the same,
      * and print a message to the test log.
diff --git a/robot-core/src/test/java/org/obolibrary/robot/MirrorOperationTest.java b/robot-core/src/test/java/org/obolibrary/robot/MirrorOperationTest.java
new file mode 100644
index 00000000..fdc7824d
--- /dev/null
+++ b/robot-core/src/test/java/org/obolibrary/robot/MirrorOperationTest.java
@@ -0,0 +1,76 @@
+package org.obolibrary.robot;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.Test;
+import org.semanticweb.owlapi.model.IRI;
+import org.semanticweb.owlapi.model.OWLOntology;
+import org.semanticweb.owlapi.model.OWLOntologyCreationException;
+import org.semanticweb.owlapi.model.OWLOntologyStorageException;
+
+/**
+ * Tests creating local cache 
+ * 
+ * @author cjm
+ *
+ */
+public class MirrorOperationTest extends CoreTest  {
+
+    /**
+     * Test MIREOT.
+     *
+     * @throws IOException on IO problems
+     * @throws OWLOntologyCreationException on ontology problems
+     * @throws OWLOntologyStorageException 
+     */
+    @Test
+    public void testMirror()
+            throws IOException, OWLOntologyCreationException, OWLOntologyStorageException {
+        testMirror("/import_test.owl");
+    }
+
+  
+    /**
+     * Mirrors a given ontology plus its import closure, and tests the results are
+     * identical when loaded
+     * 
+     * @param expectedPath
+     * @throws IOException
+     * @throws OWLOntologyCreationException
+     * @throws OWLOntologyStorageException
+     */
+    public void testMirror(String expectedPath)
+            throws IOException, OWLOntologyCreationException, OWLOntologyStorageException {
+        OWLOntology inputOntology = loadOntologyWithCatalog(expectedPath);
+        
+        File catalogFile = new File("target/mirror-catalog.xml");
+        MirrorOperation.mirror(inputOntology, new File("target"), catalogFile);
+
+        OWLOntology loadedOntology = loadOntologyWithCatalog(inputOntology.getOntologyID().getOntologyIRI().get(),
+                catalogFile);
+        assertIdentical(loadedOntology, inputOntology);
+        int n=0;
+        for (OWLOntology importedOntology : inputOntology.getImportsClosure()) {
+            n++;
+            boolean isMatched = false;
+            for (OWLOntology loadedImportedOntology : loadedOntology.getImportsClosure()) {
+                if (loadedImportedOntology.getOntologyID().equals(importedOntology.getOntologyID())) {
+                    assertIdentical(loadedImportedOntology, importedOntology);
+                    isMatched = true;
+                }
+            }
+            assertTrue(isMatched);         
+        }
+        assertEquals(2, n);
+        assertEquals(2, loadedOntology.getImportsClosure().size());
+               
+    }
+
+}
diff --git a/robot-core/src/test/resources/import_test.owl b/robot-core/src/test/resources/import_test.owl
new file mode 100644
index 00000000..666432f9
--- /dev/null
+++ b/robot-core/src/test/resources/import_test.owl
@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<rdf:RDF
+     xmlns:dc="http://purl.org/dc/elements/1.1/"
+     xmlns:foaf="http://xmlns.com/foaf/0.1/"
+     xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
+     xmlns:owl="http://www.w3.org/2002/07/owl#"
+     xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+  >
+  <owl:Ontology rdf:about="https://github.com/ontodev/robot/robot-core/src/test/resources/import_test.owl">
+    <owl:imports rdf:resource="https://github.com/ontodev/robot/robot-core/src/test/resources/simple.owl"/>
+  </owl:Ontology>
+</rdf:RDF>
+