Repository: bwplotka/mdox
Pull request #26 (2021-05-13 13:12:19)

Fix same link in different file error
Currently, if there are same links but in different files (come under same file glob passed to mdox) like [here](https://github.com/thanos-io/thanos/blame/main/docs/contributing/mentorship.md#L26) and [here](https://github.com/thanos-io/thanos/blame/main/docs/contributing/community.md#L36),  mdox throws the error below,
```
remote link https://slack.cncf.io/: URL already visited
```
Even though the link is valid.
This PR fixes this and adds a test case for the same. 

saswatamcode: Linter failure seems to be due to goreportcard.com being down.
bwplotka: You should be able to retry @saswatamcode since you are maintainer now!
bwplotka: Hm, do you know why this error was created in the first place? (: 
saswatamcode: I think this error was created to improve performance and avoid duplicate requests in colly. But here it seems to cause this issue...is there a better way to handle this other than using `AllowURLRevisit`? (:

Also, goreportcard seems to still be down so check would fail if I rerun...

Patch:
diff --git a/README.md b/README.md
index b06f25a..542029c 100644
--- a/README.md
+++ b/README.md
@@ -21,7 +21,7 @@ Just run `mdox fmt` and pass markdown files (or glob matching those).
 
 For example this README is formatted by the CI on every PR using [`mdox fmt -l *.md` command](https://github.com/bwplotka/mdox/blob/9e183714070f464b1ef089da3df8048aff1abeda/Makefile#L59).
 
-```bash mdox-gen-exec="mdox fmt --help"
+```bash mdox-exec="mdox fmt --help"
 usage: mdox fmt [<flags>] <files>...
 
 Formats in-place given markdown files uniformly following GFM (Github Flavored
@@ -39,7 +39,7 @@ Flags:
                                  directives prefixed with 'mdox-gen' to
                                  autogenerate code snippets. For example:
                                  
-                                   ```<lang> mdox-gen-exec="<executable + arguments>"
+                                   ```<lang> mdox-exec="<executable + arguments>"
                                  
                                  This directive runs executable with arguments
                                  and put its stderr and stdout output inside
@@ -64,19 +64,19 @@ Args:
 
 ### Code Generation
 
-It's not uncommon that documentation is explaining code or configuration snippets. One of the challenges of such documentation is keeping it up to date. This is where `mdox` code block directives comes handy! To ensure mdox will auto update code snippet add `mdox-gen-exec="<whatever command you want take output from>"` after language directive on code block.
+It's not uncommon that documentation is explaining code or configuration snippets. One of the challenges of such documentation is keeping it up to date. This is where `mdox` code block directives comes handy! To ensure mdox will auto update code snippet add `mdox-exec="<whatever command you want take output from>"` after language directive on code block.
 
 For example this Readme contains `mdox --help` which is has to be auto generated on every PR:
 
 ```markdown
-``` bash mdox-gen-exec="mdox fmt --help"
+``` bash mdox-exec="mdox fmt --help"
 ...
 ```
 
 This also enables auto updating snippets of code in code blocks using tools like `sed`. For example, below code block directive will auto update and insert lines 3 to 6 from main.go into code block.
 
 ```markdown
-```go mdox-gen-exec="go mdox-gen-exec="sed -n '3,6p' main.go"
+```go mdox-exec="sed -n '3,6p' main.go"
 ...
 ```
 
diff --git a/main.go b/main.go
index 6bf96d5..47b3fcf 100644
--- a/main.go
+++ b/main.go
@@ -118,7 +118,7 @@ func registerFmt(_ context.Context, app *extkingpin.App) {
 	checkOnly := cmd.Flag("check", "If true, fmt will not modify the given files, instead it will fail if files needs formatting").Bool()
 
 	disableGenCodeBlocksDirectives := cmd.Flag("code.disable-directives", `If false, fmt will parse custom fenced code directives prefixed with 'mdox-gen' to autogenerate code snippets. For example:
-	`+"```"+`<lang> mdox-gen-exec="<executable + arguments>"
+	`+"```"+`<lang> mdox-exec="<executable + arguments>"
 This directive runs executable with arguments and put its stderr and stdout output inside code block content, replacing existing one.`).Bool()
 	anchorDir := cmd.Flag("anchor-dir", "Anchor directory for all transformers. PWD is used if flag is not specified.").ExistingDir()
 	linksLocalizeForAddress := cmd.Flag("links.localize.address-regex", "If specified, all HTTP(s) links that target a domain and path matching given regexp will be transformed to relative to anchor dir path (if exists)."+
diff --git a/pkg/mdformatter/linktransformer/link.go b/pkg/mdformatter/linktransformer/link.go
index 9ffd16e..aef85dd 100644
--- a/pkg/mdformatter/linktransformer/link.go
+++ b/pkg/mdformatter/linktransformer/link.go
@@ -31,6 +31,10 @@ var (
 	IDNotFoundErr   = LookupError(errors.New("file exists, but does not have such id"))
 )
 
+const (
+	originalURLKey = "originalURLKey"
+)
+
 type chain struct {
 	chain []mdformatter.LinkTransformer
 }
@@ -150,15 +154,20 @@ func NewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) (m
 	}); err != nil {
 		return nil, err
 	}
+	v.c.OnRequest(func(request *colly.Request) {
+		v.rMu.Lock()
+		defer v.rMu.Unlock()
+		request.Ctx.Put(originalURLKey, request.URL.String())
+	})
 	v.c.OnScraped(func(response *colly.Response) {
 		v.rMu.Lock()
 		defer v.rMu.Unlock()
-		v.remoteLinks[response.Request.URL.String()] = nil
+		v.remoteLinks[response.Ctx.Get(originalURLKey)] = nil
 	})
 	v.c.OnError(func(response *colly.Response, err error) {
 		v.rMu.Lock()
 		defer v.rMu.Unlock()
-		v.remoteLinks[response.Request.URL.String()] = errors.Wrapf(err, "%q not accessible; status code %v", response.Request.URL.String(), response.StatusCode)
+		v.remoteLinks[response.Ctx.Get(originalURLKey)] = errors.Wrapf(err, "%q not accessible; status code %v", response.Request.URL.String(), response.StatusCode)
 	})
 	return v, nil
 }
diff --git a/pkg/mdformatter/mdgen/mdgen.go b/pkg/mdformatter/mdgen/mdgen.go
index 443b504..c8a3b62 100644
--- a/pkg/mdformatter/mdgen/mdgen.go
+++ b/pkg/mdformatter/mdgen/mdgen.go
@@ -19,7 +19,7 @@ import (
 const (
 	infoStringKeyLang     = "mdox-gen-lang"
 	infoStringKeyType     = "mdox-gen-type"
-	infoStringKeyExec     = "mdox-gen-exec"
+	infoStringKeyExec     = "mdox-exec"
 	infoStringKeyExitCode = "mdox-expect-exit-code"
 )
 


Test patch:
diff --git a/pkg/mdformatter/linktransformer/link_test.go b/pkg/mdformatter/linktransformer/link_test.go
index c4e6a6f..0a9395d 100644
--- a/pkg/mdformatter/linktransformer/link_test.go
+++ b/pkg/mdformatter/linktransformer/link_test.go
@@ -146,6 +146,23 @@ func TestValidator_TransformDestination(t *testing.T) {
 		testutil.Equals(t, 0, len(diff), diff.String())
 	})
 
+	t.Run("check valid but same link in diff files", func(t *testing.T) {
+		testFile := filepath.Join(tmpDir, "repo", "docs", "test", "valid-link.md")
+		testFile2 := filepath.Join(tmpDir, "repo", "docs", "test", "valid-link2.md")
+		testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/about\n"), os.ModePerm))
+		testutil.Ok(t, ioutil.WriteFile(testFile2, []byte("https://bwplotka.dev/about\n"), os.ModePerm))
+
+		diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2})
+		testutil.Ok(t, err)
+		testutil.Equals(t, 0, len(diff), diff.String())
+
+		diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, mdformatter.WithLinkTransformer(
+			MustNewValidator(logger, regexp.MustCompile(`^$`), anchorDir),
+		))
+		testutil.Ok(t, err)
+		testutil.Equals(t, 0, len(diff), diff.String())
+	})
+
 	t.Run("check valid local links", func(t *testing.T) {
 		testFile := filepath.Join(tmpDir, "repo", "docs", "test", "valid-local-links.md")
 		testutil.Ok(t, ioutil.WriteFile(testFile, []byte(`# yolo
diff --git a/pkg/mdformatter/mdgen/testdata/mdgen_formatted.md b/pkg/mdformatter/mdgen/testdata/mdgen_formatted.md
index 14737c1..a423640 100755
--- a/pkg/mdformatter/mdgen/testdata/mdgen_formatted.md
+++ b/pkg/mdformatter/mdgen/testdata/mdgen_formatted.md
@@ -1,6 +1,6 @@
 # Quick Tutorial
 
-```bash mdox-gen-exec="bash ./testdata/out.sh"
+```bash mdox-exec="bash ./testdata/out.sh"
 test output
 ```
 
@@ -26,16 +26,16 @@ The `--alertmanagers.config` and `--alertmanagers.config-file` flags allow speci
 
 The configuration format is the following:
 
-```yaml mdox-gen-exec="bash ./testdata/out2.sh"
+```yaml mdox-exec="bash ./testdata/out2.sh"
 test output2
 newline
 ```
 
-```bash mdox-expect-exit-code=2 mdox-gen-exec="bash ./testdata/out3.sh"
+```bash mdox-expect-exit-code=2 mdox-exec="bash ./testdata/out3.sh"
 test output3
 ```
 
-```bash mdox-gen-exec="sed -n '1,3p' ./testdata/out3.sh"
+```bash mdox-exec="sed -n '1,3p' ./testdata/out3.sh"
 #!/usr/bin/env bash
 
 echo "test output3"
diff --git a/pkg/mdformatter/mdgen/testdata/mdgen_not_formatted.md b/pkg/mdformatter/mdgen/testdata/mdgen_not_formatted.md
index e84b87c..3085b70 100644
--- a/pkg/mdformatter/mdgen/testdata/mdgen_not_formatted.md
+++ b/pkg/mdformatter/mdgen/testdata/mdgen_not_formatted.md
@@ -1,7 +1,7 @@
 Quick Tutorial
 ==============
 
-```bash mdox-gen-exec="bash ./testdata/out.sh"
+```bash mdox-exec="bash ./testdata/out.sh"
 a
 adf
 ```
@@ -29,15 +29,15 @@ The `--alertmanagers.config` and `--alertmanagers.config-file` flags allow speci
 
 The configuration format is the following:
 
-```yaml mdox-gen-exec="bash ./testdata/out2.sh"
+```yaml mdox-exec="bash ./testdata/out2.sh"
 alertmanagers:
 - http_config:
   api_version: v1
 ```
 
-```bash mdox-expect-exit-code=2 mdox-gen-exec="bash ./testdata/out3.sh"
+```bash mdox-expect-exit-code=2 mdox-exec="bash ./testdata/out3.sh"
 abc
 ```
 
-```bash mdox-gen-exec="sed -n '1,3p' ./testdata/out3.sh"
+```bash mdox-exec="sed -n '1,3p' ./testdata/out3.sh"
 ```