Repository: ogen-go/ogen
Pull request #657 (2022-11-03 18:23:16)

fix: use new `urlpath` package for conversion


Patch:
diff --git a/cmd/ogen/main.go b/cmd/ogen/main.go
index 2bfdbb1a..5d322f12 100644
--- a/cmd/ogen/main.go
+++ b/cmd/ogen/main.go
@@ -26,6 +26,7 @@ import (
 	"github.com/ogen-go/ogen/internal/location"
 	"github.com/ogen-go/ogen/internal/ogenversion"
 	"github.com/ogen-go/ogen/internal/ogenzap"
+	"github.com/ogen-go/ogen/internal/urlpath"
 )
 
 func cleanDir(targetDir string, files []os.DirEntry) (rerr error) {
@@ -153,7 +154,8 @@ func parseSpecPath(
 	logger *zap.Logger,
 ) (f file, opts gen.RemoteOptions, _ error) {
 	// FIXME(tdakkota): pass context.
-	if u, _ := url.Parse(p); u != nil {
+	containsDrive := runtime.GOOS == "windows" && filepath.VolumeName(p) != ""
+	if u, _ := url.Parse(p); u != nil && !containsDrive && u.Scheme != "" {
 		switch u.Scheme {
 		case "http", "https":
 			_, fileName := path.Split(u.Path)
@@ -185,15 +187,22 @@ func parseSpecPath(
 				Logger: logger,
 			}
 			return f, opts, nil
-		case "":
-		default:
-			if runtime.GOOS == "windows" && filepath.VolumeName(p) != "" {
-				break
+		case "file":
+			converted, err := urlpath.URLToFilePath(u)
+			if err != nil {
+				return f, opts, errors.Wrap(err, "convert url to file path")
 			}
+			p = converted
+		default:
 			return f, opts, errors.Errorf("unsupported scheme %q", u.Scheme)
 		}
 	}
 	p = filepath.Clean(p)
+
+	abs, err := filepath.Abs(p)
+	if err != nil {
+		return f, opts, err
+	}
 	_, fileName := filepath.Split(p)
 
 	data, err := readFile(p)
@@ -201,11 +210,16 @@ func parseSpecPath(
 		return f, opts, err
 	}
 
+	u, err := urlpath.URLFromFilePath(abs)
+	if err != nil {
+		return f, opts, errors.Wrap(err, "convert file path to url")
+	}
+
 	f = file{
 		data:     data,
 		fileName: fileName,
 		source:   p,
-		rootURL:  &url.URL{Path: filepath.ToSlash(p)},
+		rootURL:  u,
 	}
 	opts = gen.RemoteOptions{
 		HTTPClient: client,
diff --git a/internal/jsonpointer/resolve_ctx.go b/internal/jsonpointer/resolve_ctx.go
index 83fa30ab..b3c5ac2b 100644
--- a/internal/jsonpointer/resolve_ctx.go
+++ b/internal/jsonpointer/resolve_ctx.go
@@ -96,7 +96,6 @@ func (r *ResolveCtx) Key(ref string) (key RefKey, _ error) {
 	if err != nil {
 		return RefKey{}, err
 	}
-	u.Path = strings.TrimPrefix(u.Path, "/")
 	key.FromURL(u)
 	return key, nil
 }
diff --git a/internal/urlpath/urlpath.go b/internal/urlpath/urlpath.go
new file mode 100644
index 00000000..eef7233a
--- /dev/null
+++ b/internal/urlpath/urlpath.go
@@ -0,0 +1,144 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Code copied from
+// https://github.com/golang/go/blob/2ebe77a2fda1ee9ff6fd9a3e08933ad1ebaea039/src/cmd/go/internal/web/url.go
+// TODO(go.dev/issue/32456): if accepted, use the new API.
+
+// Package urlpath provides utilities for converting between file URLs and file paths.
+package urlpath
+
+import (
+	"errors"
+	"net/url"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+var errNotAbsolute = errors.New("path is not absolute")
+
+// URLToFilePath converts a file-scheme url to a file path.
+func URLToFilePath(u *url.URL) (string, error) {
+	if u.Scheme != "file" {
+		return "", errors.New("non-file URL")
+	}
+
+	checkAbs := func(path string) (string, error) {
+		if !filepath.IsAbs(path) {
+			return "", errNotAbsolute
+		}
+		return path, nil
+	}
+
+	if u.Path == "" {
+		if u.Host != "" || u.Opaque == "" {
+			return "", errors.New("file URL missing path")
+		}
+		return checkAbs(filepath.FromSlash(u.Opaque))
+	}
+
+	path, err := convertFileURLPath(u.Host, u.Path)
+	if err != nil {
+		return path, err
+	}
+	return checkAbs(path)
+}
+
+// URLFromFilePath converts the given absolute path to a URL.
+func URLFromFilePath(path string) (*url.URL, error) {
+	if !filepath.IsAbs(path) {
+		return nil, errNotAbsolute
+	}
+
+	// If path has a Windows volume name, convert the volume to a host and prefix
+	// per https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/.
+	if vol := filepath.VolumeName(path); vol != "" {
+		if strings.HasPrefix(vol, `\\`) {
+			path = filepath.ToSlash(path[2:])
+			i := strings.IndexByte(path, '/')
+
+			if i < 0 {
+				// A degenerate case.
+				// \\host.example.com (without a share name)
+				// becomes
+				// file://host.example.com/
+				return &url.URL{
+					Scheme: "file",
+					Host:   path,
+					Path:   "/",
+				}, nil
+			}
+
+			// \\host.example.com\Share\path\to\file
+			// becomes
+			// file://host.example.com/Share/path/to/file
+			return &url.URL{
+				Scheme: "file",
+				Host:   path[:i],
+				Path:   filepath.ToSlash(path[i:]),
+			}, nil
+		}
+
+		// C:\path\to\file
+		// becomes
+		// file:///C:/path/to/file
+		return &url.URL{
+			Scheme: "file",
+			Path:   "/" + filepath.ToSlash(path),
+		}, nil
+	}
+
+	// /path/to/file
+	// becomes
+	// file:///path/to/file
+	return &url.URL{
+		Scheme: "file",
+		Path:   filepath.ToSlash(path),
+	}, nil
+}
+
+func convertFileURLPath(host, path string) (string, error) {
+	if runtime.GOOS == "windows" {
+		return convertFileURLPathWindows(host, path)
+	}
+	switch host {
+	case "", "localhost":
+	default:
+		return "", errors.New("file URL specifies non-local host")
+	}
+	return filepath.FromSlash(path), nil
+}
+
+func convertFileURLPathWindows(host, path string) (string, error) {
+	if path == "" || path[0] != '/' {
+		return "", errNotAbsolute
+	}
+
+	path = filepath.FromSlash(path)
+
+	// We interpret Windows file URLs per the description in
+	// https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/.
+
+	// The host part of a file URL (if any) is the UNC volume name,
+	// but RFC 8089 reserves the authority "localhost" for the local machine.
+	if host != "" && host != "localhost" {
+		// A common "legacy" format omits the leading slash before a drive letter,
+		// encoding the drive letter as the host instead of part of the path.
+		// (See https://blogs.msdn.microsoft.com/freeassociations/2005/05/19/the-bizarre-and-unhappy-story-of-file-urls/.)
+		// We do not support that format, but we should at least emit a more
+		// helpful error message for it.
+		if filepath.VolumeName(host) != "" {
+			return "", errors.New("file URL encodes volume in host field: too few slashes?")
+		}
+		return `\\` + host + path, nil
+	}
+
+	// If host is empty, path must contain an initial slash followed by a
+	// drive letter and path. Remove the slash and verify that the path is valid.
+	if vol := filepath.VolumeName(path[1:]); vol == "" || strings.HasPrefix(vol, `\\`) {
+		return "", errors.New("file URL missing drive letter")
+	}
+	return path[1:], nil
+}
diff --git a/jsonschema/external.go b/jsonschema/external.go
index 566fd14e..41b13bb8 100644
--- a/jsonschema/external.go
+++ b/jsonschema/external.go
@@ -10,6 +10,8 @@ import (
 
 	"github.com/go-faster/errors"
 	"go.uber.org/zap"
+
+	"github.com/ogen-go/ogen/internal/urlpath"
 )
 
 // ExternalResolver resolves external links.
@@ -122,7 +124,13 @@ func (e externalResolver) Get(ctx context.Context, loc string) ([]byte, error) {
 	case "http", "https":
 		data, err = e.httpGet(ctx, u)
 	case "file", "":
-		data, err = e.readFile(u.Path)
+		var p string
+		p, err = urlpath.URLToFilePath(u)
+		if err != nil {
+			err = errors.Wrap(err, "convert url to file path")
+			break
+		}
+		data, err = e.readFile(p)
 	default:
 		return nil, errors.Errorf("unsupported scheme %q", scheme)
 	}


Test patch:
diff --git a/cmd/ogen/main_test.go b/cmd/ogen/main_test.go
deleted file mode 100644
index dc7a6064..00000000
--- a/cmd/ogen/main_test.go
+++ /dev/null
@@ -1,92 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"runtime"
-	"testing"
-
-	"github.com/go-faster/errors"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/zap/zaptest"
-)
-
-type roundTripFunc func(req *http.Request) (*http.Response, error)
-
-func (r roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
-	return r(req)
-}
-
-func Test_parseSpecPath(t *testing.T) {
-	testdata := []byte(`{}`)
-	urlPath := func(p string) *url.URL {
-		return &url.URL{Path: p}
-	}
-	urlParse := func(s string) *url.URL {
-		u, err := url.Parse(s)
-		require.NoError(t, err)
-		return u
-	}
-
-	type testCase struct {
-		input        string
-		httpData     []byte
-		fileData     []byte
-		wantFilename string
-		wantURL      *url.URL
-	}
-
-	tests := []testCase{
-		{"spec.json", nil, testdata, "spec.json", urlPath("spec.json")},
-		{"./spec.json", nil, testdata, "spec.json", urlPath("spec.json")},
-		{"_testdata/spec.json", nil, testdata, "spec.json", urlPath("_testdata/spec.json")},
-
-		{"http://example.com/spec.json", testdata, nil, "spec.json", urlParse("http://example.com/spec.json")},
-	}
-	if runtime.GOOS == "windows" {
-		tests = append(tests, []testCase{
-			{`_testdata\spec.json`, nil, testdata, "spec.json", urlPath("_testdata/spec.json")},
-			{`C:\_testdata\spec.json`, nil, testdata, "spec.json", urlPath("C:/_testdata/spec.json")},
-		}...)
-	}
-	for i, tt := range tests {
-		tt := tt
-		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
-			a := require.New(t)
-
-			var (
-				data     = tt.httpData
-				readFile = func(filename string) ([]byte, error) {
-					return nil, errors.Errorf("unexpected read file: %q", filename)
-				}
-				httpClient = &http.Client{
-					Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
-						return &http.Response{
-							StatusCode: http.StatusOK,
-							Body:       io.NopCloser(bytes.NewReader(data)),
-						}, nil
-					}),
-				}
-			)
-			if data == nil {
-				data = tt.fileData
-				readFile = func(filename string) ([]byte, error) {
-					return data, nil
-				}
-				httpClient = &http.Client{
-					Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
-						return nil, errors.Errorf("unexpected http request: %q", req.URL)
-					}),
-				}
-			}
-
-			f, _, err := parseSpecPath(tt.input, httpClient, readFile, zaptest.NewLogger(t))
-			a.NoError(err)
-			a.Equal(tt.wantFilename, f.fileName)
-			a.Equal(tt.wantURL, f.rootURL)
-		})
-	}
-}
diff --git a/internal/urlpath/urlpath_test.go b/internal/urlpath/urlpath_test.go
new file mode 100644
index 00000000..cb4c160a
--- /dev/null
+++ b/internal/urlpath/urlpath_test.go
@@ -0,0 +1,200 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package urlpath
+
+import (
+	"net/url"
+	"runtime"
+	"testing"
+)
+
+func TestURLToFilePath(t *testing.T) {
+	for _, tc := range urlTests() {
+		if tc.url == "" {
+			continue
+		}
+		tc := tc
+
+		t.Run(tc.url, func(t *testing.T) {
+			u, err := url.Parse(tc.url)
+			if err != nil {
+				t.Fatalf("url.Parse(%q): %v", tc.url, err)
+			}
+
+			path, err := URLToFilePath(u)
+			if err != nil {
+				if err.Error() == tc.wantErr {
+					return
+				}
+				if tc.wantErr == "" {
+					t.Fatalf("urlToFilePath(%v): %v; want <nil>", u, err)
+				} else {
+					t.Fatalf("urlToFilePath(%v): %v; want %s", u, err, tc.wantErr)
+				}
+			}
+
+			if path != tc.filePath || tc.wantErr != "" {
+				t.Fatalf("urlToFilePath(%v) = %q, <nil>; want %q, %s", u, path, tc.filePath, tc.wantErr)
+			}
+		})
+	}
+}
+
+func TestURLFromFilePath(t *testing.T) {
+	for _, tc := range urlTests() {
+		if tc.filePath == "" {
+			continue
+		}
+		tc := tc
+
+		t.Run(tc.filePath, func(t *testing.T) {
+			u, err := URLFromFilePath(tc.filePath)
+			if err != nil {
+				if err.Error() == tc.wantErr {
+					return
+				}
+				if tc.wantErr == "" {
+					t.Fatalf("urlFromFilePath(%v): %v; want <nil>", tc.filePath, err)
+				} else {
+					t.Fatalf("urlFromFilePath(%v): %v; want %s", tc.filePath, err, tc.wantErr)
+				}
+			}
+
+			if tc.wantErr != "" {
+				t.Fatalf("urlFromFilePath(%v) = <nil>; want error: %s", tc.filePath, tc.wantErr)
+			}
+
+			wantURL := tc.url
+			if tc.canonicalURL != "" {
+				wantURL = tc.canonicalURL
+			}
+			if u.String() != wantURL {
+				t.Errorf("urlFromFilePath(%v) = %v; want %s", tc.filePath, u, wantURL)
+			}
+		})
+	}
+}
+
+func urlTests() []urlTest {
+	if runtime.GOOS == "windows" {
+		return urlTestsWindows
+	}
+	return urlTestsOthers
+}
+
+type urlTest struct {
+	url          string
+	filePath     string
+	canonicalURL string // If empty, assume equal to url.
+	wantErr      string
+}
+
+var urlTestsOthers = []urlTest{
+	// Examples from RFC 8089:
+	{
+		url:      `file:///path/to/file`,
+		filePath: `/path/to/file`,
+	},
+	{
+		url:          `file:/path/to/file`,
+		filePath:     `/path/to/file`,
+		canonicalURL: `file:///path/to/file`,
+	},
+	{
+		url:          `file://localhost/path/to/file`,
+		filePath:     `/path/to/file`,
+		canonicalURL: `file:///path/to/file`,
+	},
+
+	// We reject non-local files.
+	{
+		url:     `file://host.example.com/path/to/file`,
+		wantErr: "file URL specifies non-local host",
+	},
+}
+
+var urlTestsWindows = []urlTest{
+	// Examples from https://blogs.msdn.microsoft.com/ie/2006/12/06/file-uris-in-windows/:
+
+	{
+		url:      `file://laptop/My%20Documents/FileSchemeURIs.doc`,
+		filePath: `\\laptop\My Documents\FileSchemeURIs.doc`,
+	},
+	{
+		url:      `file:///C:/Documents%20and%20Settings/davris/FileSchemeURIs.doc`,
+		filePath: `C:\Documents and Settings\davris\FileSchemeURIs.doc`,
+	},
+	{
+		url:      `file:///D:/Program%20Files/Viewer/startup.htm`,
+		filePath: `D:\Program Files\Viewer\startup.htm`,
+	},
+	{
+		url:          `file:///C:/Program%20Files/Music/Web%20Sys/main.html?REQUEST=RADIO`,
+		filePath:     `C:\Program Files\Music\Web Sys\main.html`,
+		canonicalURL: `file:///C:/Program%20Files/Music/Web%20Sys/main.html`,
+	},
+	{
+		url:      `file://applib/products/a-b/abc_9/4148.920a/media/start.swf`,
+		filePath: `\\applib\products\a-b\abc_9\4148.920a\media\start.swf`,
+	},
+	{
+		url:     `file:////applib/products/a%2Db/abc%5F9/4148.920a/media/start.swf`,
+		wantErr: "file URL missing drive letter",
+	},
+	{
+		url:     `C:\Program Files\Music\Web Sys\main.html?REQUEST=RADIO`,
+		wantErr: "non-file URL",
+	},
+
+	// The example "file://D:\Program Files\Viewer\startup.htm" errors out in
+	// url.Parse, so we substitute a slash-based path for testing instead.
+	{
+		url:     `file://D:/Program Files/Viewer/startup.htm`,
+		wantErr: "file URL encodes volume in host field: too few slashes?",
+	},
+
+	// The blog post discourages the use of non-ASCII characters because they
+	// depend on the user's current codepage. However, when we are working with Go
+	// strings we assume UTF-8 encoding, and our url package refuses to encode
+	// URLs to non-ASCII strings.
+	{
+		url:          `file:///C:/exampleㄓ.txt`,
+		filePath:     `C:\exampleㄓ.txt`,
+		canonicalURL: `file:///C:/example%E3%84%93.txt`,
+	},
+	{
+		url:      `file:///C:/example%E3%84%93.txt`,
+		filePath: `C:\exampleㄓ.txt`,
+	},
+
+	// Examples from RFC 8089:
+
+	// We allow the drive-letter variation from section E.2, because it is
+	// simpler to support than not to. However, we do not generate the shorter
+	// form in the reverse direction.
+	{
+		url:          `file:c:/path/to/file`,
+		filePath:     `c:\path\to\file`,
+		canonicalURL: `file:///c:/path/to/file`,
+	},
+
+	// We encode the UNC share name as the authority following section E.3.1,
+	// because that is what the Microsoft blog post explicitly recommends.
+	{
+		url:      `file://host.example.com/Share/path/to/file.txt`,
+		filePath: `\\host.example.com\Share\path\to\file.txt`,
+	},
+
+	// We decline the four- and five-slash variations from section E.3.2.
+	// The paths in these URLs would change meaning under path.Clean.
+	{
+		url:     `file:////host.example.com/path/to/file`,
+		wantErr: "file URL missing drive letter",
+	},
+	{
+		url:     `file://///host.example.com/path/to/file`,
+		wantErr: "file URL missing drive letter",
+	},
+}
diff --git a/jsonschema/resolve_test.go b/jsonschema/resolve_test.go
index f03942e1..7d48a477 100644
--- a/jsonschema/resolve_test.go
+++ b/jsonschema/resolve_test.go
@@ -3,6 +3,7 @@ package jsonschema
 import (
 	"context"
 	"encoding/json"
+	"strings"
 	"testing"
 
 	"github.com/go-faster/errors"
@@ -16,6 +17,7 @@ import (
 type external map[string]components
 
 func (e external) Get(_ context.Context, loc string) ([]byte, error) {
+	loc = strings.TrimPrefix(loc, "/")
 	r, ok := e[loc]
 	if !ok {
 		return nil, errors.Errorf("unexpected location %q", loc)
diff --git a/openapi/parser/resolve_external_test.go b/openapi/parser/resolve_external_test.go
index d6e12a71..2bd9ed33 100644
--- a/openapi/parser/resolve_external_test.go
+++ b/openapi/parser/resolve_external_test.go
@@ -10,6 +10,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"path"
+	"strings"
 	"testing"
 
 	"github.com/go-faster/errors"
@@ -25,6 +26,7 @@ import (
 type external map[string]any
 
 func (e external) Get(_ context.Context, loc string) ([]byte, error) {
+	loc = strings.TrimPrefix(loc, "/")
 	r, ok := e[loc]
 	if !ok {
 		return nil, errors.Errorf("unexpected location %q", loc)
@@ -214,7 +216,7 @@ func TestExternalReference(t *testing.T) {
 			Examples: []jsonschema.Example{exampleValue},
 		}
 		localExample = &openapi.Example{
-			Ref:   "#/components/examples/LocalExample",
+			Ref:   "foo.json#/components/examples/RemoteExample",
 			Value: exampleValue,
 		}
 		param = &openapi.Parameter{
@@ -233,7 +235,7 @@ func TestExternalReference(t *testing.T) {
 					Schema: schema,
 					Examples: map[string]*openapi.Example{
 						"ref": {
-							Ref:   "#/components/examples/LocalExample",
+							Ref:   "foo.json#/components/examples/RemoteExample",
 							Value: exampleValue,
 						},
 					},
@@ -310,7 +312,10 @@ func TestExternalReference(t *testing.T) {
 				"LocalParameter": param,
 			},
 			Examples: map[string]*openapi.Example{
-				"LocalExample": localExample,
+				"LocalExample": {
+					Ref:   "#/components/examples/LocalExample",
+					Value: exampleValue,
+				},
 			},
 			RequestBodies: map[string]*openapi.RequestBody{
 				"LocalRequestBody": requestBody,
@@ -464,30 +469,4 @@ func TestInitialLocation(t *testing.T) {
 		a.NoError(err)
 		check(a, parsed)
 	})
-	t.Run("File", func(t *testing.T) {
-		a := require.New(t)
-
-		rootURL := &url.URL{
-			Path: path.Join(rootPrefix, rootSpec),
-		}
-
-		var called []string
-		parsed, err := Parse(spec, Settings{
-			External: jsonschema.NewExternalResolver(jsonschema.ExternalOptions{
-				ReadFile: func(p string) ([]byte, error) {
-					called = append(called, p)
-					return fs.ReadFile(remotes, p)
-				},
-			}),
-			File:    location.NewFile(rootSpec, rootURL.String(), raw),
-			RootURL: rootURL,
-		})
-		a.NoError(err)
-		a.Equal([]string{
-			"_testdata/remotes/api/response.json",
-			"_testdata/remotes/schemas.json",
-		}, called)
-
-		check(a, parsed)
-	})
 }