Repository: bwplotka/mdox
Pull request #33 (2021-05-20 11:44:27)

Add `links.validate.config` flag
This PR adds in the `--links.validate.config` flag which allows users to pass in path to YAML config with composable regex matchers. 

A sample yaml config would be,
```
validators:
  - regex: 'bwplotka\/mdox'
    token: '$(GITHUB_TOKEN)'
    type: "github"
  - regex: '(^http[s]?:\/\/)(www\.)?(fakelink[0-9]\.com\/)'
    type: "ignore"
  - regex: 'mycustomdomain'
    type: "roundtrip"
```

Here, all links are checked with default type `ignore`, i.e, links are not visited. If type is set to `roundtrip`, then links matching provided regex are visited by colly. For type `github` only repo name regex is taken, which then directs mdox to not visit any GitHub issue/PR link of that repo. User can also provide a token(env var substitution) to tackle GitHub API rate limiting.

Also, we don't check if a particular number is an issue or a PR since GitHub can automatically redirect such links to the correct page, for example the link`https://github.com/bwplotka/mdox/pull/30` is actually an issue and will be redirected. So such links can be safely skipped.


Patch:
diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml
index 915c522..7954572 100644
--- a/.github/workflows/go.yaml
+++ b/.github/workflows/go.yaml
@@ -55,4 +55,5 @@ jobs:
       - name: Run unit tests.
         env:
           GOBIN: /tmp/.bin
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: make test
\ No newline at end of file
diff --git a/README.md b/README.md
index 588a2af..f1520c9 100644
--- a/README.md
+++ b/README.md
@@ -60,9 +60,15 @@ Flags:
                                  exists).Absolute path links will be converted
                                  to relative links to anchor dir as well.
   -l, --links.validate           If true, all links will be validated
-      --links.validate.without-address-regex=^$  
-                                 If specified, all links will be validated,
-                                 except those matching the given target address.
+      --links.validate.config-file=<file-path>  
+                                 Path to YAML file for skipping link check, with
+                                 spec defined in
+                                 github.com/bwplotka/mdox/pkg/linktransformer.Config
+      --links.validate.config=<content>  
+                                 Alternative to 'links.validate.config-file'
+                                 flag (mutually exclusive). Content of YAML file
+                                 for skipping link check, with spec defined in
+                                 github.com/bwplotka/mdox/pkg/linktransformer.Config
 
 Args:
   <files>  Markdown file(s) to process.
diff --git a/main.go b/main.go
index 3766e97..c39eb4e 100644
--- a/main.go
+++ b/main.go
@@ -13,6 +13,7 @@ import (
 	"syscall"
 
 	"github.com/bwplotka/mdox/pkg/clilog"
+	"github.com/bwplotka/mdox/pkg/extflag"
 	"github.com/bwplotka/mdox/pkg/extkingpin"
 	"github.com/bwplotka/mdox/pkg/mdformatter"
 	"github.com/bwplotka/mdox/pkg/mdformatter/linktransformer"
@@ -126,7 +127,7 @@ This directive runs executable with arguments and put its stderr and stdout outp
 		"Absolute path links will be converted to relative links to anchor dir as well.").Regexp()
 	// TODO(bwplotka): Add cache in file?
 	linksValidateEnabled := cmd.Flag("links.validate", "If true, all links will be validated").Short('l').Bool()
-	linksValidateExceptDomains := cmd.Flag("links.validate.without-address-regex", "If specified, all links will be validated, except those matching the given target address.").Default(`^$`).Regexp()
+	linksValidateConfig := extflag.RegisterPathOrContent(cmd, "links.validate.config", "YAML file for skipping link check, with spec defined in github.com/bwplotka/mdox/pkg/linktransformer.Config", extflag.WithEnvSubstitution())
 
 	cmd.Run(func(ctx context.Context, logger log.Logger) (err error) {
 		var opts []mdformatter.Option
@@ -151,7 +152,11 @@ This directive runs executable with arguments and put its stderr and stdout outp
 
 		var linkTr []mdformatter.LinkTransformer
 		if *linksValidateEnabled {
-			v, err := linktransformer.NewValidator(logger, *linksValidateExceptDomains, anchorDir)
+			validateConfigContent, err := linksValidateConfig.Content()
+			if err != nil {
+				return err
+			}
+			v, err := linktransformer.NewValidator(logger, validateConfigContent, anchorDir)
 			if err != nil {
 				return err
 			}
diff --git a/pkg/extflag/pathorcontent.go b/pkg/extflag/pathorcontent.go
new file mode 100644
index 0000000..9a6c4ff
--- /dev/null
+++ b/pkg/extflag/pathorcontent.go
@@ -0,0 +1,130 @@
+// Copyright (c) Bartłomiej Płotka @bwplotka
+// Licensed under the Apache License 2.0.
+
+// Taken from Thanos project.
+//
+// Copyright (c) The Thanos Authors.
+// Licensed under the Apache License 2.0.
+package extflag
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"regexp"
+
+	"github.com/pkg/errors"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+// PathOrContent is a flag type that defines two flags to fetch bytes. Either from file (*-file flag) or content (* flag).
+type PathOrContent struct {
+	flagName string
+
+	envSubstitution bool
+	required        bool
+
+	path    *string
+	content *string
+}
+
+// Option is a functional option type for PathOrContent objects.
+type Option func(*PathOrContent)
+
+type FlagClause interface {
+	Flag(name, help string) *kingpin.FlagClause
+}
+
+// RegisterPathOrContent registers PathOrContent flag in kingpinCmdClause.
+func RegisterPathOrContent(cmd FlagClause, flagName string, help string, opts ...Option) *PathOrContent {
+	fileFlagName := fmt.Sprintf("%s-file", flagName)
+	contentFlagName := flagName
+
+	fileHelp := fmt.Sprintf("Path to %s", help)
+	fileFlag := cmd.Flag(fileFlagName, fileHelp).PlaceHolder("<file-path>").String()
+
+	contentHelp := fmt.Sprintf("Alternative to '%s' flag (mutually exclusive). Content of %s", fileFlagName, help)
+	contentFlag := cmd.Flag(contentFlagName, contentHelp).PlaceHolder("<content>").String()
+
+	p := &PathOrContent{
+		flagName:        flagName,
+		path:            fileFlag,
+		content:         contentFlag,
+		required:        false,
+		envSubstitution: false,
+	}
+	for _, opt := range opts {
+		opt(p)
+	}
+	return p
+}
+
+// Content returns the content of the file when given or directly the content that has been passed to the flag.
+// It returns an error when:
+// * The file and content flags are both not empty.
+// * The file flag is not empty but the file can't be read.
+// * The content is empty and the flag has been defined as required.
+func (p *PathOrContent) Content() ([]byte, error) {
+	fileFlagName := fmt.Sprintf("%s-file", p.flagName)
+
+	if len(*p.path) > 0 && len(*p.content) > 0 {
+		return nil, errors.Errorf("both %s and %s flags set.", fileFlagName, p.flagName)
+	}
+
+	var content []byte
+	if len(*p.path) > 0 {
+		c, err := ioutil.ReadFile(*p.path)
+		if err != nil {
+			return nil, errors.Wrapf(err, "loading file %s for %s", *p.path, fileFlagName)
+		}
+		content = c
+	} else {
+		content = []byte(*p.content)
+	}
+
+	if len(content) == 0 && p.required {
+		return nil, errors.Errorf("flag %s or %s is required for running this command and content cannot be empty.", fileFlagName, p.flagName)
+	}
+	if p.envSubstitution {
+		replace, err := expandEnv(content)
+		if err != nil {
+			return nil, err
+		}
+		content = replace
+	}
+	return content, nil
+}
+
+// WithRequired allows you to override default required option.
+func WithRequired() Option {
+	return func(p *PathOrContent) {
+		p.required = true
+	}
+}
+
+// WithRequired allows you to override default envSubstitution option.
+func WithEnvSubstitution() Option {
+	return func(p *PathOrContent) {
+		p.envSubstitution = true
+	}
+}
+
+// expandEnv returns content of YAML file with substituted environment variables.
+// Follows K8s convention, i.e $(...), as mentioned here https://kubernetes.io/docs/tasks/inject-data-application/define-interdependent-environment-variables/.
+func expandEnv(b []byte) (r []byte, err error) {
+	var envRe = regexp.MustCompile(`\$\(([a-zA-Z_0-9]+)\)`)
+	r = envRe.ReplaceAllFunc(b, func(n []byte) []byte {
+		if err != nil {
+			return nil
+		}
+		n = n[2 : len(n)-1]
+
+		v, ok := os.LookupEnv(string(n))
+		if !ok {
+			err = errors.Errorf("found reference to unset environment variable %q", n)
+			return nil
+		}
+		return []byte(v)
+	})
+	return r, err
+}
diff --git a/pkg/mdformatter/linktransformer/config.go b/pkg/mdformatter/linktransformer/config.go
new file mode 100644
index 0000000..78991a5
--- /dev/null
+++ b/pkg/mdformatter/linktransformer/config.go
@@ -0,0 +1,167 @@
+// Copyright (c) Bartłomiej Płotka @bwplotka
+// Licensed under the Apache License 2.0.
+
+package linktransformer
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math"
+	"net/http"
+	"regexp"
+	"strconv"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+type Config struct {
+	Version int
+
+	Validators []ValidatorConfig `yaml:"validators"`
+}
+
+type ValidatorConfig struct {
+	// Regex for type github is reponame matcher, like `bwplotka\/mdox`.
+	Regex string `yaml:"regex"`
+	// By default type is `ignore`. Could be `github` or `roundtrip`.
+	Type ValidatorType `yaml:"type"`
+	// GitHub repo token to avoid getting rate limited.
+	Token string `yaml:"token"`
+
+	ghValidator GitHubValidator
+	rtValidator RoundTripValidator
+	igValidator IgnoreValidator
+}
+
+type RoundTripValidator struct {
+	_regex *regexp.Regexp
+}
+
+type GitHubValidator struct {
+	_regex  *regexp.Regexp
+	_maxNum int
+}
+
+type IgnoreValidator struct {
+	_regex *regexp.Regexp
+}
+
+type ValidatorType string
+
+const (
+	roundtripValidator ValidatorType = "roundtrip"
+	githubValidator    ValidatorType = "github"
+	ignoreValidator    ValidatorType = "ignore"
+)
+
+const (
+	gitHubAPIURL = "https://api.github.com/repos/%v/%v?sort=created&direction=desc&per_page=1"
+)
+
+type GitHubResponse struct {
+	Number int `json:"number"`
+}
+
+func ParseConfig(c []byte) (Config, error) {
+	cfg := Config{}
+	dec := yaml.NewDecoder(bytes.NewReader(c))
+	dec.KnownFields(true)
+	if err := dec.Decode(&cfg); err != nil {
+		return Config{}, errors.Wrapf(err, "parsing YAML content %q", string(c))
+	}
+
+	if len(cfg.Validators) <= 0 {
+		return Config{}, errors.New("No validator provided")
+	}
+
+	// Evaluate regex for given validators.
+	for i := range cfg.Validators {
+		switch cfg.Validators[i].Type {
+		case roundtripValidator:
+			cfg.Validators[i].rtValidator._regex = regexp.MustCompile(cfg.Validators[i].Regex)
+		case githubValidator:
+			regex, maxNum, err := getGitHubRegex(cfg.Validators[i].Regex, cfg.Validators[i].Token)
+			if err != nil {
+				return Config{}, errors.Wrapf(err, "parsing GitHub Regex %v", err)
+			}
+			cfg.Validators[i].ghValidator._regex = regex
+			cfg.Validators[i].ghValidator._maxNum = maxNum
+		case ignoreValidator:
+			cfg.Validators[i].igValidator._regex = regexp.MustCompile(cfg.Validators[i].Regex)
+		default:
+			return Config{}, errors.New("Validator type not supported")
+		}
+	}
+	return cfg, nil
+}
+
+// getGitHubRegex returns GitHub pulls/issues regex from repo name.
+func getGitHubRegex(repoRe string, repoToken string) (*regexp.Regexp, int, error) {
+	// Get reponame from regex.
+	getRepo := regexp.MustCompile(`(?P<org>[A-Za-z0-9_.-]+)\\\/(?P<repo>[A-Za-z0-9_.-]+)`)
+	match := getRepo.FindStringSubmatch(repoRe)
+	if len(match) != 3 {
+		return nil, math.MaxInt64, errors.New("repo name regex not valid")
+	}
+	reponame := match[1] + "/" + match[2]
+
+	var pullNum []GitHubResponse
+	var issueNum []GitHubResponse
+	max := 0
+	// All GitHub API reqs need to have User-Agent: https://docs.github.com/en/rest/overview/resources-in-the-rest-api#user-agent-required.
+	client := &http.Client{}
+
+	// Check latest pull request number.
+	reqPull, err := http.NewRequest("GET", fmt.Sprintf(gitHubAPIURL, reponame, "pulls"), nil)
+	if err != nil {
+		return nil, math.MaxInt64, err
+	}
+	reqPull.Header.Set("User-Agent", "mdox")
+
+	// Check latest issue number and return whichever is greater.
+	reqIssue, err := http.NewRequest("GET", fmt.Sprintf(gitHubAPIURL, reponame, "issues"), nil)
+	if err != nil {
+		return nil, math.MaxInt64, err
+	}
+	reqIssue.Header.Set("User-Agent", "mdox")
+
+	if repoToken != "" {
+		reqPull.Header.Set("Authorization", "Bearer "+repoToken)
+		reqIssue.Header.Set("Authorization", "Bearer "+repoToken)
+	}
+
+	respPull, err := client.Do(reqPull)
+	if err != nil {
+		return nil, math.MaxInt64, err
+	}
+	if respPull.StatusCode != 200 {
+		return nil, math.MaxInt64, errors.New("pulls API request failed. status code: " + strconv.Itoa(respPull.StatusCode))
+	}
+	defer respPull.Body.Close()
+	if err := json.NewDecoder(respPull.Body).Decode(&pullNum); err != nil {
+		return nil, math.MaxInt64, err
+	}
+
+	respIssue, err := client.Do(reqIssue)
+	if err != nil {
+		return nil, math.MaxInt64, err
+	}
+	if respIssue.StatusCode != 200 {
+		return nil, math.MaxInt64, errors.New("issues API request failed. status code: " + strconv.Itoa(respIssue.StatusCode))
+	}
+	defer respIssue.Body.Close()
+	if err := json.NewDecoder(respIssue.Body).Decode(&issueNum); err != nil {
+		return nil, math.MaxInt64, err
+	}
+
+	if len(pullNum) > 0 {
+		max = pullNum[0].Number
+	}
+	if len(issueNum) > 0 && issueNum[0].Number > max {
+		max = issueNum[0].Number
+	}
+
+	return regexp.MustCompile(`(^http[s]?:\/\/)(www\.)?(github\.com\/)(` + repoRe + `)(\/pull\/|\/issues\/)`), max, nil
+}
diff --git a/pkg/mdformatter/linktransformer/link.go b/pkg/mdformatter/linktransformer/link.go
index 561fc34..acce8c7 100644
--- a/pkg/mdformatter/linktransformer/link.go
+++ b/pkg/mdformatter/linktransformer/link.go
@@ -110,9 +110,9 @@ func (l *localizer) TransformDestination(ctx mdformatter.SourceContext, destinat
 func (l *localizer) Close(mdformatter.SourceContext) error { return nil }
 
 type validator struct {
-	logger    log.Logger
-	anchorDir string
-	except    *regexp.Regexp
+	logger         log.Logger
+	anchorDir      string
+	validateConfig Config
 
 	localLinks  localLinksCache
 	rMu         sync.RWMutex
@@ -135,15 +135,23 @@ type futureResult struct {
 
 // NewValidator returns mdformatter.LinkTransformer that crawls all links.
 // TODO(bwplotka): Add optimization and debug modes - this is the main source of latency and pain.
-func NewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) (mdformatter.LinkTransformer, error) {
+func NewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string) (mdformatter.LinkTransformer, error) {
+	var err error
+	config := Config{}
+	if string(linksValidateConfig) != "" {
+		config, err = ParseConfig(linksValidateConfig)
+		if err != nil {
+			return nil, err
+		}
+	}
 	v := &validator{
-		logger:      logger,
-		anchorDir:   anchorDir,
-		except:      except,
-		localLinks:  map[string]*[]string{},
-		remoteLinks: map[string]error{},
-		c:           colly.NewCollector(colly.Async()),
-		destFutures: map[futureKey]*futureResult{},
+		logger:         logger,
+		anchorDir:      anchorDir,
+		validateConfig: config,
+		localLinks:     map[string]*[]string{},
+		remoteLinks:    map[string]error{},
+		c:              colly.NewCollector(colly.Async()),
+		destFutures:    map[futureKey]*futureResult{},
 	}
 	// Set very soft limits.
 	// E.g github has 50-5000 https://docs.github.com/en/free-pro-team@latest/rest/reference/rate-limit limit depending
@@ -173,8 +181,8 @@ func NewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) (m
 }
 
 // MustNewValidator returns mdformatter.LinkTransformer that crawls all links.
-func MustNewValidator(logger log.Logger, except *regexp.Regexp, anchorDir string) mdformatter.LinkTransformer {
-	v, err := NewValidator(logger, except, anchorDir)
+func MustNewValidator(logger log.Logger, linksValidateConfig []byte, anchorDir string) mdformatter.LinkTransformer {
+	v, err := NewValidator(logger, linksValidateConfig, anchorDir)
 	if err != nil {
 		panic(err)
 	}
@@ -232,10 +240,6 @@ func (v *validator) visit(filepath string, dest string) {
 		return
 	}
 	v.destFutures[k] = &futureResult{cases: 1, resultFn: func() error { return nil }}
-	if v.except.MatchString(dest) {
-		return
-	}
-
 	matches := remoteLinkPrefixRe.FindAllStringIndex(dest, 1)
 	if matches == nil {
 		// Relative or absolute path. Check if exists.
@@ -247,25 +251,12 @@ func (v *validator) visit(filepath string, dest string) {
 		}
 		return
 	}
-
-	// Result will be in future.
-	v.destFutures[k].resultFn = func() error { return v.remoteLinks[dest] }
-	v.rMu.RLock()
-	if _, ok := v.remoteLinks[dest]; ok {
-		v.rMu.RUnlock()
-		return
-	}
-	v.rMu.RUnlock()
-
-	v.rMu.Lock()
-	defer v.rMu.Unlock()
-	// We need to check again here to avoid race.
-	if _, ok := v.remoteLinks[dest]; ok {
-		return
-	}
-
-	if err := v.c.Visit(dest); err != nil {
-		v.remoteLinks[dest] = errors.Wrapf(err, "remote link %v", dest)
+	validator := v.validateConfig.GetValidatorForURL(dest)
+	if validator != nil {
+		matched, err := validator.IsValid(k, v)
+		if matched && err == nil {
+			return
+		}
 	}
 }
 
diff --git a/pkg/mdformatter/linktransformer/validator.go b/pkg/mdformatter/linktransformer/validator.go
new file mode 100644
index 0000000..1000813
--- /dev/null
+++ b/pkg/mdformatter/linktransformer/validator.go
@@ -0,0 +1,89 @@
+// Copyright (c) Bartłomiej Płotka @bwplotka
+// Licensed under the Apache License 2.0.
+
+package linktransformer
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type Validator interface {
+	IsValid(k futureKey, r *validator) (bool, error)
+}
+
+// GitHubValidator.IsValid skips visiting all github issue/PR links.
+func (v GitHubValidator) IsValid(k futureKey, r *validator) (bool, error) {
+	// Find rightmost index of match i.e, where regex match ends.
+	// This will be where issue/PR number starts. Split incase of section link and convert to int.
+	rightmostIndex := v._regex.FindStringIndex(k.dest)
+	stringNum := strings.Split(k.dest[rightmostIndex[1]:], "#")
+	num, err := strconv.Atoi(stringNum[0])
+	if err != nil {
+		return false, err
+	}
+	// If number in link does not exceed then link is valid.
+	if v._maxNum >= num {
+		return true, nil
+	}
+	return false, nil
+}
+
+// RoundTripValidator.IsValid returns true if url is checked by colly.
+func (v RoundTripValidator) IsValid(k futureKey, r *validator) (bool, error) {
+	// Result will be in future.
+	r.destFutures[k].resultFn = func() error { return r.remoteLinks[k.dest] }
+	r.rMu.RLock()
+	if _, ok := r.remoteLinks[k.dest]; ok {
+		r.rMu.RUnlock()
+		return true, nil
+	}
+	r.rMu.RUnlock()
+
+	r.rMu.Lock()
+	defer r.rMu.Unlock()
+	// We need to check again here to avoid race.
+	if _, ok := r.remoteLinks[k.dest]; ok {
+		return true, nil
+	}
+
+	if err := r.c.Visit(k.dest); err != nil {
+		r.remoteLinks[k.dest] = errors.Wrapf(err, "remote link %v", k.dest)
+		return false, nil
+	}
+	return true, nil
+}
+
+// IgnoreValidator.IsValid returns true if matched so that link in not checked.
+func (v IgnoreValidator) IsValid(k futureKey, r *validator) (bool, error) {
+	return true, nil
+}
+
+// GetValidatorForURL returns correct Validator by matching URL.
+func (v Config) GetValidatorForURL(URL string) Validator {
+	for _, val := range v.Validators {
+		switch val.Type {
+		case roundtripValidator:
+			if !val.rtValidator._regex.MatchString(URL) {
+				continue
+			}
+			return val.rtValidator
+		case githubValidator:
+			if !val.ghValidator._regex.MatchString(URL) {
+				continue
+			}
+			return val.ghValidator
+		case ignoreValidator:
+			if !val.igValidator._regex.MatchString(URL) {
+				continue
+			}
+			return val.igValidator
+		default:
+			panic("unexpected validator type")
+		}
+	}
+	// No config file passed, so all links must be checked.
+	return RoundTripValidator{}
+}


Test patch:
diff --git a/pkg/mdformatter/linktransformer/link_test.go b/pkg/mdformatter/linktransformer/link_test.go
index 4770634..880e969 100644
--- a/pkg/mdformatter/linktransformer/link_test.go
+++ b/pkg/mdformatter/linktransformer/link_test.go
@@ -140,7 +140,7 @@ func TestValidator_TransformDestination(t *testing.T) {
 		testutil.Equals(t, 0, len(diff), diff.String())
 
 		diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer(
-			MustNewValidator(logger, regexp.MustCompile(`^$`), anchorDir),
+			MustNewValidator(logger, []byte(""), anchorDir),
 		))
 		testutil.Ok(t, err)
 		testutil.Equals(t, 0, len(diff), diff.String())
@@ -157,7 +157,7 @@ func TestValidator_TransformDestination(t *testing.T) {
 		testutil.Equals(t, 0, len(diff), diff.String())
 
 		diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile, testFile2}, mdformatter.WithLinkTransformer(
-			MustNewValidator(logger, regexp.MustCompile(`^$`), anchorDir),
+			MustNewValidator(logger, []byte(""), anchorDir),
 		))
 		testutil.Ok(t, err)
 		testutil.Equals(t, 0, len(diff), diff.String())
@@ -175,7 +175,7 @@ func TestValidator_TransformDestination(t *testing.T) {
 		testutil.Equals(t, 0, len(diff), diff.String())
 
 		diff, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer(
-			MustNewValidator(logger, regexp.MustCompile(`^$`), anchorDir),
+			MustNewValidator(logger, []byte(""), anchorDir),
 		))
 		testutil.Ok(t, err)
 		testutil.Equals(t, 0, len(diff), diff.String())
@@ -198,7 +198,7 @@ func TestValidator_TransformDestination(t *testing.T) {
 		testutil.Equals(t, 0, len(diff), diff.String())
 
 		_, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer(
-			MustNewValidator(logger, regexp.MustCompile(`^$`), anchorDir),
+			MustNewValidator(logger, []byte(""), anchorDir),
 		))
 		testutil.NotOk(t, err)
 
@@ -224,22 +224,52 @@ func TestValidator_TransformDestination(t *testing.T) {
 		testutil.Equals(t, 0, len(diff), diff.String())
 
 		_, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer(
-			MustNewValidator(logger, regexp.MustCompile(`^$`), anchorDir),
+			MustNewValidator(logger, []byte(""), anchorDir),
 		))
 		testutil.NotOk(t, err)
 		testutil.Equals(t, fmt.Sprintf("%v%v: %v%v: \"https://bwplotka.dev/does-not-exists\" not accessible; status code 404: Not Found", tmpDir, filePath, relDirPath, filePath), err.Error())
 	})
 
-	t.Run("check 404 link, ignored", func(t *testing.T) {
+	t.Run("check valid & 404 link with validate config", func(t *testing.T) {
 		testFile := filepath.Join(tmpDir, "repo", "docs", "test", "invalid-link2.md")
-		testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://bwplotka.dev/does-not-exists\n"), os.ModePerm))
+		testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://www.github.com/ https://bwplotka.dev/does-not-exits\n"), os.ModePerm))
+
+		diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile})
+		testutil.Ok(t, err)
+		testutil.Equals(t, 0, len(diff), diff.String())
+
+		_, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer(
+			MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n  - regex: 'bwplotka'\n    type: 'ignore'\n"), anchorDir),
+		))
+		testutil.Ok(t, err)
+	})
+
+	t.Run("check 404 links with ignore validate config", func(t *testing.T) {
+		testFile := filepath.Join(tmpDir, "repo", "docs", "test", "links.md")
+		testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://fakelink1.com/ http://fakelink2.com/ https://www.fakelink3.com/\n"), os.ModePerm))
+
+		diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile})
+		testutil.Ok(t, err)
+		testutil.Equals(t, 0, len(diff), diff.String())
+
+		_, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer(
+			MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n  - regex: '(^http[s]?:\\/\\/)(www\\.)?(fakelink[0-9]\\.com\\/)'\n    type: 'ignore'\n"), anchorDir),
+		))
+		testutil.Ok(t, err)
+	})
+
+	t.Run("check github links with validate config", func(t *testing.T) {
+		testFile := filepath.Join(tmpDir, "repo", "docs", "test", "github-link.md")
+		testutil.Ok(t, ioutil.WriteFile(testFile, []byte("https://github.com/bwplotka/mdox/issues/23 https://github.com/bwplotka/mdox/pull/32 https://github.com/bwplotka/mdox/pull/27#pullrequestreview-659598194\n"), os.ModePerm))
+		// This is substituted in config using PathorContent flag. But need to pass it directly here.
+		repoToken := os.Getenv("GITHUB_TOKEN")
 
 		diff, err := mdformatter.IsFormatted(context.TODO(), logger, []string{testFile})
 		testutil.Ok(t, err)
 		testutil.Equals(t, 0, len(diff), diff.String())
 
 		_, err = mdformatter.IsFormatted(context.TODO(), logger, []string{testFile}, mdformatter.WithLinkTransformer(
-			MustNewValidator(logger, regexp.MustCompile(`://bwplotka.dev`), anchorDir),
+			MustNewValidator(logger, []byte("version: 1\n\nvalidators:\n  - regex: 'bwplotka\\/mdox'\n    token: '"+repoToken+"'\n    type: 'github'\n"), anchorDir),
 		))
 		testutil.Ok(t, err)
 	})