Repository: common-workflow-language/cwltool

Issue:
--pack should preserve @namespace
---

## Expected Behavior
cwltool should resolve file formats specified as links (e.g., after `--pack`-ing a workflow)

## Steps to reproduce
```
cd /path/to/common-workflow-language/v1.0/v1.0
cwltool --pack formattest2.cwl > formattest2-packed.cwl
cwltool formattest2-packed.cwl formattest2-job.json
```

Expected output:
```
{
    "output": {
        "format": "http://edamontology.org/format_2330", 
        "checksum": "sha1$97fe1b50b4582cebc7d853796ebd62e3e163aa3f", 
        "basename": "output.txt", 
        "location": "file:///Users/anton/Programming/CWL/cwltool/cwltool/schemas/v1.0/output.txt", 
        "path": "/Users/anton/Programming/CWL/cwltool/cwltool/schemas/v1.0/output.txt", 
        "class": "File", 
        "size": 1111
    }
}
```

Produced output:
```
{
    "output": {
        "format": "edam:format_2330", 
        "checksum": "sha1$97fe1b50b4582cebc7d853796ebd62e3e163aa3f", 
        "basename": "output.txt", 
        "location": "file:///Users/anton/Programming/CWL/cwltool/cwltool/schemas/v1.0/output.txt", 
        "path": "/Users/anton/Programming/CWL/cwltool/cwltool/schemas/v1.0/output.txt", 
        "class": "File", 
        "size": 1111
    }
}
```

## Your Environment
* cwltool version: 
1.0.20171107133715



Discussion:
It should have lifted the $namespaces section, so the loader of the packed file is responsible for resolving it (by usual rules), is that missing?
Here's the packed version from cwltool 1.0.20171205210500
```
{
    "class": "CommandLineTool", 
    "doc": "Reverse each line using the `rev` command", 
    "hints": [
        {
            "dockerPull": "debian:stretch-slim", 
            "class": "DockerRequirement"
        }
    ], 
    "inputs": [
        {
            "type": "File", 
            "inputBinding": {}, 
            "format": "http://edamontology.org/format_2330", 
            "id": "#main/input"
        }
    ], 
    "outputs": [
        {
            "type": "File", 
            "outputBinding": {
                "glob": "output.txt"
            }, 
            "format": "$(inputs.input.format)", 
            "id": "#main/output"
        }
    ], 
    "baseCommand": "rev", 
    "stdout": "output.txt", 
    "id": "#main", 
    "cwlVersion": "v1.0"
}
```
So no, the namespace section isn't being copied over during `--pack`

FYI: the input object is using `edam` but it doesn't define it:
(formattest2-job.json)
```
{
    "input": {
        "class": "File",
        "location": "ref.fasta",
        "format": "edam:format_1929"
    }
}
```

Adding a `@namespaces` section to the input object doesn't change the final output when using the `--pack`'d version.

Reviewing the spec I'm not seeing a lot of guidance on how to process the input object with regard to namespaces.
So perhaps the real question is, do users need to duplicate `@namespaces` in input objects to make use of them?

If not, then `--pack` needs to preserve `@namespaces` and not collapse them.

Resolving patch:
diff --git a/cwltool/pack.py b/cwltool/pack.py
--- a/cwltool/pack.py
+++ b/cwltool/pack.py
@@ -149,6 +149,7 @@ def rewrite_id(r, mainuri):
 
     packed = {"$graph": [], "cwlVersion": metadata["cwlVersion"]
               }  # type: Dict[Text, Any]
+    namespaces = metadata.get('$namespaces', None)
 
     schemas = set()  # type: Set[Text]
     for r in sorted(runs):
@@ -185,5 +186,7 @@ def rewrite_id(r, mainuri):
         # duplicate 'cwlVersion' inside $graph when there is a single item
         # because we're printing contents inside '$graph' rather than whole dict
         packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"]
+    if namespaces:
+        packed["$graph"][0]["$namespaces"] = dict(cast(Dict, namespaces))
 
     return packed


Test patch:
diff --git a/tests/test_pack.py b/tests/test_pack.py
--- a/tests/test_pack.py
+++ b/tests/test_pack.py
@@ -5,6 +5,7 @@
 
 import os
 from functools import partial
+import tempfile
 
 import pytest
 from six import StringIO
@@ -47,7 +48,7 @@ def test_pack_missing_cwlVersion(self):
         # Testing single tool workflow
         document_loader, workflowobj, uri = fetch_document(
             get_data("tests/wf/hello_single_tool.cwl"))
-        document_loader, avsc_names, processobj, metadata, uri = validate_document(
+        document_loader, _, processobj, metadata, uri = validate_document(
             document_loader, workflowobj, uri)
         # generate pack output dict
         packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
@@ -57,7 +58,7 @@ def test_pack_missing_cwlVersion(self):
         # Testing single step workflow
         document_loader, workflowobj, uri = fetch_document(
             get_data("tests/wf/hello-workflow.cwl"))
-        document_loader, avsc_names, processobj, metadata, uri = validate_document(
+        document_loader, _, processobj, metadata, uri = validate_document(
             document_loader, workflowobj, uri)
         # generate pack output dict
         packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
@@ -103,7 +104,7 @@ def test_packed_workflow_execution(self):
         document_loader, avsc_names, processobj, metadata, uri = validate_document(
             document_loader, workflowobj, uri)
         packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
-        temp_packed_path = "/tmp/packedwf"
+        temp_packed_path = tempfile.mkstemp()[1]
         with open(temp_packed_path, 'w') as f:
             json.dump(packed, f)
         normal_output = StringIO()
@@ -115,4 +116,30 @@ def test_packed_workflow_execution(self):
                                 get_data(test_wf_job)],
                                stdout=normal_output), 0)
         self.assertEquals(json.loads(packed_output.getvalue()), json.loads(normal_output.getvalue()))
+        os.remove(temp_packed_path)
 
+    @pytest.mark.skipif(onWindows(),
+                        reason="Instance of cwltool is used, on Windows it invokes a default docker container"
+                               "which is not supported on AppVeyor")
+    def test_preserving_namespaces(self):
+        test_wf = "tests/wf/formattest.cwl"
+        test_wf_job = "tests/wf/formattest-job.json"
+        document_loader, workflowobj, uri = fetch_document(
+            get_data(test_wf))
+        document_loader, avsc_names, processobj, metadata, uri = validate_document(
+            document_loader, workflowobj, uri)
+        packed = json.loads(print_pack(document_loader, processobj, uri, metadata))
+        assert "$namespaces" in packed
+        temp_packed_path = tempfile.mkstemp()[1]
+        with open(temp_packed_path, 'w') as f:
+            json.dump(packed, f)
+        normal_output = StringIO()
+        packed_output = StringIO()
+        self.assertEquals(main(['--debug', get_data(temp_packed_path),
+                                get_data(test_wf_job)],
+                               stdout=packed_output), 0)
+        self.assertEquals(main([get_data(test_wf),
+                                get_data(test_wf_job)],
+                               stdout=normal_output), 0)
+        self.assertEquals(json.loads(packed_output.getvalue()), json.loads(normal_output.getvalue()))
+        os.remove(temp_packed_path)
diff --git a/tests/wf/formattest-job.json b/tests/wf/formattest-job.json
new file mode 100644
--- /dev/null
+++ b/tests/wf/formattest-job.json
@@ -0,0 +1,7 @@
+{
+    "input": {
+        "class": "File",
+        "location": "whale.txt",
+        "format": "edam:format_2330"
+    }
+}
diff --git a/tests/wf/formattest.cwl b/tests/wf/formattest.cwl
new file mode 100644
--- /dev/null
+++ b/tests/wf/formattest.cwl
@@ -0,0 +1,20 @@
+$namespaces:
+  edam: "http://edamontology.org/"
+cwlVersion: v1.0
+class: CommandLineTool
+doc: "Reverse each line using the `rev` command"
+inputs:
+  input:
+    type: File
+    inputBinding: {}
+    format: edam:format_2330
+
+outputs:
+  output:
+    type: File
+    outputBinding:
+      glob: output.txt
+    format: edam:format_2330
+
+baseCommand: rev
+stdout: output.txt
\ No newline at end of file