Repository: reviewdog/reviewdog
Pull request #1867 (2024-08-01 12:30:33)

Fix project config based run with github-[pr-]check and github-pr-review reporters
- Tool name was incorrect for github-[pr-]check reporters and check API does not work correctly for project config based run (ref: https://github.com/reviewdog/reviewdog/issues/1846)
- Deletion of outdated comment didn't work properly with github-pr-review reporter since tool name isn't correctly configured for project config based run

- [x] Updated Unreleased section in [CHANGELOG](https://github.com/reviewdog/reviewdog/blob/master/CHANGELOG.md) or it's not notable changes.




Patch:
diff --git a/.github/workflows/reviewdog.yml b/.github/workflows/reviewdog.yml
index f7a8f558..c2a07606 100644
--- a/.github/workflows/reviewdog.yml
+++ b/.github/workflows/reviewdog.yml
@@ -47,6 +47,12 @@ jobs:
           cat ./_testdata/custom_rdjson.json | \
             reviewdog -name="custom-rdjson" -f=rdjson -reporter=github-check -level=info
 
+      - name: reviewdog project run
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+            reviewdog -reporter=github-check
+
   reviewdog-pr:
     permissions:
       checks: write
@@ -160,6 +166,12 @@ jobs:
           gofmt -s -d . | \
             reviewdog -name="gofmt" -f=diff -f.diff.strip=0 -reporter=github-pr-review
 
+      - name: reviewdog project run
+        env:
+          REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+            reviewdog -reporter=github-pr-review
+
   golangci-lint:
     permissions:
       checks: write
diff --git a/.reviewdog.yml b/.reviewdog.yml
index a485a5de..383bb19e 100644
--- a/.reviewdog.yml
+++ b/.reviewdog.yml
@@ -1,38 +1,20 @@
 # reviewdog.yml
-
 runner:
-  golint:
+  golint-by-project-conf:
     cmd: golint $(go list ./... | grep -v /vendor/)
     format: golint
     level: warning
-  govet:
-    cmd: go vet $(go list ./... | grep -v /vendor/)
+  govet-by-project-conf:
+    cmd: go vet
     format: govet
-  errcheck:
-    cmd: errcheck -asserts -ignoretests -blank $(go list ./... | grep -v /vendor/)
+    level: error
+  custom-rdjson:
+    cmd: cat ./_testdata/custom_rdjson.json
+    name: custom-rdjson-by-project-conf
+    format: rdjson
+  test-project-conf:
+    cmd: |
+      echo '.reviewdog.yml:17: test project conf (errorformat)'
+    level: info
     errorformat:
-      - "%f:%l:%c:%m"
-    level: warning
-  staticcheck:
-    cmd: staticcheck $(go list ./... | grep -v /vendor/)
-    errorformat:
-      - "%f:%l:%c: %m"
-  misspell:
-    cmd: misspell $(git ls-files)
-    errorformat:
-      - "%f:%l:%c: %m"
-  unparam:
-    cmd: unparam $(go list ./... | grep -v /vendor/)
-    errorformat:
-      - "%f:%l:%c: %m"
-  revive:
-    cmd: revive -config=.revive.toml $(go list ./... | grep -v /vendor/)
-    format: golint
-    level: warning
-  golangci:
-    cmd: golangci-lint run --out-format=line-number ./...
-    errorformat:
-      - '%E%f:%l:%c: %m'
-      - '%E%f:%l: %m'
-      - '%C%.%#'
-    level: warning
+      - '%f:%l: %m'
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c9b9b33c..838ec1ec 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,8 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - [#1846](https://github.com/reviewdog/reviewdog/issues/1846) Unexpected error
   exit code when using `-reporter=github-[pr-]check` with
   `-fail-on-error=true`. See the below breaking changes section for more details.
-- [#1894](https://github.com/reviewdog/reviewdog/pull/1894) fix 
-error when repository uses multiselect custom-properties [#1881](https://github.com/reviewdog/reviewdog/issues/1881)
+- [#1867](https://github.com/reviewdog/reviewdog/pull/1867) Tool name was incorrect for github-[pr-]check reporters and check API does not work correctly for project config based run
+- [#1867](https://github.com/reviewdog/reviewdog/pull/1867) Deletion of outdated comment didn't work properly with github-pr-review reporter since tool name isn't correctly configured for project config based run
+- [#1894](https://github.com/reviewdog/reviewdog/pull/1894) fix error when repository uses multiselect custom-properties [#1881](https://github.com/reviewdog/reviewdog/issues/1881)
 
 ### :rotating_light: Breaking changes
 - [#1858](https://github.com/reviewdog/reviewdog/pull/1858) Remove original_output from rdjson/rdjsonl reporters' output
diff --git a/cmd/reviewdog/main.go b/cmd/reviewdog/main.go
index f00b9580..85a532bb 100644
--- a/cmd/reviewdog/main.go
+++ b/cmd/reviewdog/main.go
@@ -165,7 +165,7 @@ const (
 			$ export GERRIT_REVISION_ID=ed318bf9a3c
 			$ export GERRIT_BRANCH=master
 			$ export GERRIT_ADDRESS=http://localhost:8080
-	
+
 	"bitbucket-code-report"
 		Create Bitbucket Code Report via Code Insights
 		(https://confluence.atlassian.com/display/BITBUCKET/Code+insights).
@@ -177,7 +177,7 @@ const (
 		- For Basic Auth you need to set following env variables:
 			  BITBUCKET_USER and BITBUCKET_PASSWORD
 		- For AccessToken Auth you need to set BITBUCKET_ACCESS_TOKEN
-		
+
 		To post results to Bitbucket Server specify BITBUCKET_SERVER_URL.
 
 	"gitea-pr-review"
@@ -317,10 +317,12 @@ func run(r io.Reader, w io.Writer, opt *option) error {
 			return runDoghouse(ctx, r, w, opt, isProject)
 		}
 		var err error
-		cs, ds, err = githubCheckService(ctx, opt)
+		checkService, ghDiffService, err := githubCheckService(ctx, opt)
 		if err != nil {
 			return err
 		}
+		ds = ghDiffService
+		cs = reviewdog.MultiCommentService(checkService, cs)
 	case "github-pr-annotations":
 		g, client, err := githubBuildInfoWithClient(ctx)
 		if err != nil {
diff --git a/comment.go b/comment.go
index 5d05c7e4..3d002f54 100644
--- a/comment.go
+++ b/comment.go
@@ -28,6 +28,14 @@ func (m *multiCommentService) Flush(ctx context.Context) error {
 	return nil
 }
 
+func (m *multiCommentService) SetTool(toolName string, level string) {
+	for _, cs := range m.services {
+		if ncs, ok := cs.(NamedCommentService); ok {
+			ncs.SetTool(toolName, level)
+		}
+	}
+}
+
 // MultiCommentService creates a comment service that duplicates its post to
 // all the provided comment services.
 func MultiCommentService(services ...CommentService) CommentService {
diff --git a/project/run.go b/project/run.go
index 7f21e0f8..c8ae47e9 100644
--- a/project/run.go
+++ b/project/run.go
@@ -3,6 +3,7 @@ package project
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -39,7 +40,7 @@ func RunAndParse(ctx context.Context, conf *Config, runners map[string]bool, def
 		}
 		usedRunners = append(usedRunners, runnerName)
 		semaphore <- 1
-		log.Printf("reviewdog: [start]\trunner=%s", runnerName)
+		log.Printf("reviewdog: [start] runner=%s", runnerName)
 		fname := runner.Format
 		if fname == "" && len(runner.Errorformat) == 0 {
 			fname = runnerName
@@ -73,7 +74,7 @@ func RunAndParse(ctx context.Context, conf *Config, runners map[string]bool, def
 				Diagnostics: diagnostics,
 				CmdErr:      cmdErr,
 			})
-			msg := fmt.Sprintf("reviewdog: [finish]\trunner=%s", runnerName)
+			msg := fmt.Sprintf("reviewdog: [finish] runner=%s", runnerName)
 			if cmdErr != nil {
 				msg += fmt.Sprintf("\terror=%v", cmdErr)
 			}
@@ -109,17 +110,20 @@ func Run(ctx context.Context, conf *Config, runners map[string]bool, c reviewdog
 	if err != nil {
 		return err
 	}
-	var g errgroup.Group
+	var errs []error
 	results.Range(func(toolname string, result *reviewdog.Result) {
-		ds := result.Diagnostics
-		g.Go(func() error {
-			if err := result.CheckUnexpectedFailure(); err != nil {
-				return err
-			}
-			return reviewdog.RunFromResult(ctx, c, ds, filediffs, d.Strip(), toolname, filterMode, failLevel)
-		})
+		if err := result.CheckUnexpectedFailure(); err != nil {
+			errs = append(errs, err)
+		}
+		if ncs, ok := c.(reviewdog.NamedCommentService); ok {
+			ncs.SetTool(toolname, result.Level)
+		}
+		// Note: CommentService shouldn't be run concurrently with different tool.
+		if err := reviewdog.RunFromResult(ctx, c, result.Diagnostics, filediffs, d.Strip(), toolname, filterMode, failLevel); err != nil {
+			errs = append(errs, err)
+		}
 	})
-	return g.Wait()
+	return errors.Join(errs...)
 }
 
 var secretEnvs = [...]string{
diff --git a/reviewdog.go b/reviewdog.go
index 18c1c85a..f47875b2 100644
--- a/reviewdog.go
+++ b/reviewdog.go
@@ -55,12 +55,19 @@ type FilteredCommentService interface {
 }
 
 // BulkCommentService posts comments all at once when Flush() is called.
-// Flush() will be called at the end of reviewdog run.
+// Flush() will be called at the end of each reviewdog run.
 type BulkCommentService interface {
 	CommentService
 	Flush(context.Context) error
 }
 
+// NamedCommentService can set tool name and level. Useful for update tool name
+// for each reviewdog run with reviewdog project config.
+type NamedCommentService interface {
+	CommentService
+	SetTool(toolName string, level string)
+}
+
 // DiffService is an interface which get diff.
 type DiffService interface {
 	Diff(context.Context) ([]byte, error)
diff --git a/service/bitbucket/annotator.go b/service/bitbucket/annotator.go
index 027948a9..c4c48e04 100644
--- a/service/bitbucket/annotator.go
+++ b/service/bitbucket/annotator.go
@@ -100,6 +100,7 @@ func (r *ReportAnnotator) Post(_ context.Context, c *reviewdog.Comment) error {
 func (r *ReportAnnotator) Flush(ctx context.Context) error {
 	r.muAnnotations.Lock()
 	defer r.muAnnotations.Unlock()
+	defer func() { r.comments = nil }()
 
 	// create/update/annotate report per tool
 	for tool, comments := range r.comments {
diff --git a/service/gerrit/change_review.go b/service/gerrit/change_review.go
index 16c2f73c..c74df8a1 100644
--- a/service/gerrit/change_review.go
+++ b/service/gerrit/change_review.go
@@ -61,6 +61,7 @@ func (g *ChangeReviewCommenter) Post(_ context.Context, c *reviewdog.Comment) er
 func (g *ChangeReviewCommenter) Flush(ctx context.Context) error {
 	g.muComments.Lock()
 	defer g.muComments.Unlock()
+	defer func() { g.postComments = nil }()
 
 	return g.postAllComments(ctx)
 }
diff --git a/service/gitea/gitea.go b/service/gitea/gitea.go
index d99514b6..e76ebabc 100644
--- a/service/gitea/gitea.go
+++ b/service/gitea/gitea.go
@@ -92,6 +92,7 @@ func (g *PullRequest) Post(_ context.Context, c *reviewdog.Comment) error {
 func (g *PullRequest) Flush(_ context.Context) error {
 	g.muComments.Lock()
 	defer g.muComments.Unlock()
+	defer func() { g.postComments = nil }()
 
 	if err := g.setPostedComment(); err != nil {
 		return err
diff --git a/service/github/check.go b/service/github/check.go
index b79090af..2fac45aa 100644
--- a/service/github/check.go
+++ b/service/github/check.go
@@ -78,9 +78,15 @@ func (ch *Check) GetResult() *CheckResult {
 	return ch.result
 }
 
+func (ch *Check) SetTool(toolName string, level string) {
+	ch.ToolName = toolName
+	ch.Level = level
+}
+
 // Flush actually posts comments.
 func (ch *Check) Flush(ctx context.Context) error {
 	ch.muComments.Lock()
+	defer func() { ch.postComments = nil }()
 	defer ch.muComments.Unlock()
 	check, err := ch.createCheck(ctx)
 	if err != nil {
@@ -119,7 +125,7 @@ func (ch *Check) Flush(ctx context.Context) error {
 
 func (ch *Check) createCheck(ctx context.Context) (*github.CheckRun, error) {
 	opt := github.CreateCheckRunOptions{
-		Name:    ch.ToolName,
+		Name:    ch.checkName(),
 		HeadSHA: ch.SHA,
 		Status:  github.String("in_progress"),
 	}
diff --git a/service/github/github.go b/service/github/github.go
index 8b5b0201..2c96964b 100644
--- a/service/github/github.go
+++ b/service/github/github.go
@@ -116,6 +116,7 @@ func (g *PullRequest) Post(_ context.Context, c *reviewdog.Comment) error {
 func (g *PullRequest) Flush(ctx context.Context) error {
 	g.muComments.Lock()
 	defer g.muComments.Unlock()
+	defer func() { g.postComments = nil }()
 
 	if err := g.setPostedComment(ctx); err != nil {
 		return err
@@ -123,6 +124,11 @@ func (g *PullRequest) Flush(ctx context.Context) error {
 	return g.postAsReviewComment(ctx)
 }
 
+func (g *PullRequest) SetTool(toolName string, level string) {
+	g.toolName = toolName
+	g.logWriter = githubutils.NewGitHubActionLogWriter(level)
+}
+
 func (g *PullRequest) postAsReviewComment(ctx context.Context) error {
 	if g.fallbackToLog {
 		// we don't have permission to post a review comment.
diff --git a/service/gitlab/gitlab_mr_commit.go b/service/gitlab/gitlab_mr_commit.go
index be637c6e..7833fac2 100644
--- a/service/gitlab/gitlab_mr_commit.go
+++ b/service/gitlab/gitlab_mr_commit.go
@@ -70,6 +70,7 @@ func (g *MergeRequestCommitCommenter) Post(_ context.Context, c *reviewdog.Comme
 func (g *MergeRequestCommitCommenter) Flush(ctx context.Context) error {
 	g.muComments.Lock()
 	defer g.muComments.Unlock()
+	defer func() { g.postComments = nil }()
 
 	if err := g.setPostedComment(ctx); err != nil {
 		return err
diff --git a/service/gitlab/gitlab_mr_discussion.go b/service/gitlab/gitlab_mr_discussion.go
index ddd309f0..856c49c5 100644
--- a/service/gitlab/gitlab_mr_discussion.go
+++ b/service/gitlab/gitlab_mr_discussion.go
@@ -72,6 +72,7 @@ func (g *MergeRequestDiscussionCommenter) Post(_ context.Context, c *reviewdog.C
 func (g *MergeRequestDiscussionCommenter) Flush(ctx context.Context) error {
 	g.muComments.Lock()
 	defer g.muComments.Unlock()
+	defer func() { g.postComments = nil }()
 	postedcs, err := g.createPostedComments()
 	if err != nil {
 		return fmt.Errorf("failed to create posted comments: %w", err)


Test patch:
diff --git a/_testdata/custom_rdjson.json b/_testdata/custom_rdjson.json
index 0c4a0894..ed712e56 100644
--- a/_testdata/custom_rdjson.json
+++ b/_testdata/custom_rdjson.json
@@ -90,7 +90,7 @@
       "severity": "INFO"
     },
     {
-      "message": "severity test 2 (INFO)",
+      "message": "severity test 2 (INFO enum value should be discarded)",
       "location": {
         "path": "_testdata/custom.txt",
         "range": {
diff --git a/service/github/check_test.go b/service/github/check_test.go
index a08b1f9d..0a649189 100644
--- a/service/github/check_test.go
+++ b/service/github/check_test.go
@@ -44,116 +44,6 @@ func TestCheck_OK(t *testing.T) {
 				},
 				ShouldReport: true,
 			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message: "test message outside diff",
-					Location: &rdf.Location{
-						Path: "sample.new.txt",
-						Range: &rdf.Range{
-							Start: &rdf.Position{Line: 14},
-						},
-					},
-					OriginalOutput: "raw test message outside diff",
-				},
-				ShouldReport: false,
-			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message: "test multiline",
-					Location: &rdf.Location{
-						Path: "sample.new.txt",
-						Range: &rdf.Range{
-							Start: &rdf.Position{Line: 2},
-							End:   &rdf.Position{Line: 3},
-						},
-					},
-				},
-				ShouldReport: true,
-			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message: "test multiline with column",
-					Location: &rdf.Location{
-						Path: "sample.new.txt",
-						Range: &rdf.Range{
-							Start: &rdf.Position{Line: 2, Column: 1},
-							End:   &rdf.Position{Line: 3, Column: 5},
-						},
-					},
-				},
-				ShouldReport: true,
-			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message: "test range comment",
-					Location: &rdf.Location{
-						Path: "sample.new.txt",
-						Range: &rdf.Range{
-							Start: &rdf.Position{Line: 2, Column: 1},
-							End:   &rdf.Position{Line: 2, Column: 5},
-						},
-					},
-				},
-				ShouldReport: true,
-			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message:  "test severity override",
-					Severity: rdf.Severity_ERROR,
-					Location: &rdf.Location{
-						Path: "sample.new.txt",
-						Range: &rdf.Range{
-							Start: &rdf.Position{Line: 2},
-						},
-					},
-				},
-				ShouldReport: true,
-			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message: "source test",
-					Source: &rdf.Source{
-						Name: "awesome-linter",
-					},
-					Location: &rdf.Location{
-						Path: "sample.new.txt",
-						Range: &rdf.Range{
-							Start: &rdf.Position{Line: 2},
-						},
-					},
-				},
-				ShouldReport: true,
-			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message: "code test w/o URL",
-					Location: &rdf.Location{
-						Path:  "sample.new.txt",
-						Range: &rdf.Range{Start: &rdf.Position{Line: 2}},
-					},
-					Code: &rdf.Code{Value: "CODE14"},
-				},
-				ShouldReport: true,
-			},
-		}, {
-			Result: &filter.FilteredDiagnostic{
-				Diagnostic: &rdf.Diagnostic{
-					Message: "code test w/ URL",
-					Location: &rdf.Location{
-						Path:  "sample.new.txt",
-						Range: &rdf.Range{Start: &rdf.Position{Line: 2}},
-					},
-					Code: &rdf.Code{Value: "CODE14", Url: "https://github.com/reviewdog#CODE14"},
-				},
-				ShouldReport: true,
-			},
 		},
 	}
 
@@ -169,107 +59,10 @@ func TestCheck_OK(t *testing.T) {
 			t.Error(err)
 		}
 
-		wantAnnotations := []*github.CheckRunAnnotation{
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(2),
-				AnnotationLevel: github.String("warning"),
-				Message:         github.String("test message"),
-				Title:           github.String("[haya14busa-linter] sample.new.txt#L2"),
-				RawDetails:      github.String("raw test message"),
-			},
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(3),
-				AnnotationLevel: github.String("warning"),
-				Message:         github.String("test multiline"),
-				Title:           github.String("[haya14busa-linter] sample.new.txt#L2-L3"),
-			},
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(3),
-				AnnotationLevel: github.String("warning"),
-				Message:         github.String("test multiline with column"),
-				Title:           github.String("[haya14busa-linter] sample.new.txt#L2-L3"),
-			},
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(2),
-				StartColumn:     github.Int(1),
-				EndColumn:       github.Int(5),
-				AnnotationLevel: github.String("warning"),
-				Message:         github.String("test range comment"),
-				Title:           github.String("[haya14busa-linter] sample.new.txt#L2"),
-			},
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(2),
-				AnnotationLevel: github.String("failure"),
-				Message:         github.String("test severity override"),
-				Title:           github.String("[haya14busa-linter] sample.new.txt#L2"),
-			},
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(2),
-				AnnotationLevel: github.String("warning"),
-				Message:         github.String("source test"),
-				Title:           github.String("[awesome-linter] sample.new.txt#L2"),
-			},
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(2),
-				AnnotationLevel: github.String("warning"),
-				Message:         github.String("code test w/o URL"),
-				Title:           github.String("[haya14busa-linter] sample.new.txt#L2 <CODE14>"),
-			},
-			{
-				Path:            github.String("sample.new.txt"),
-				StartLine:       github.Int(2),
-				EndLine:         github.Int(2),
-				AnnotationLevel: github.String("warning"),
-				Message:         github.String("code test w/ URL"),
-				Title:           github.String("[haya14busa-linter] sample.new.txt#L2 <CODE14>(https://github.com/reviewdog#CODE14)"),
-			},
-		}
-
 		if req.GetStatus() == "completed" {
-			if want := "neutral"; req.GetConclusion() != want {
-				t.Errorf("conclusion = %s, want %s", req.GetConclusion(), want)
-			}
 			if wantTitle := "reviewdog [haya14busa-linter] report"; req.GetOutput().GetTitle() != wantTitle {
 				t.Errorf("title = %s, want %s", req.GetOutput().GetTitle(), wantTitle)
 			}
-			if wantSummary := `reported by [reviewdog](https://github.com/reviewdog/reviewdog) :dog:
-<details>
-<summary>Findings (8)</summary>
-
-[sample.new.txt|2 col 1|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) test message
-[sample.new.txt|2|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) test multiline
-[sample.new.txt|2 col 1|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) test multiline with column
-[sample.new.txt|2 col 1|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) test range comment
-[sample.new.txt|2|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) test severity override
-[sample.new.txt|2|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) source test
-[sample.new.txt|2|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) code test w/o URL
-[sample.new.txt|2|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L2) code test w/ URL
-</details>
-<details>
-<summary>Filtered Findings (1)</summary>
-
-[sample.new.txt|14|](https://github.com/haya14busa/reviewdog/blob/1414/sample.new.txt#L14) test message outside diff
-</details>`; req.GetOutput().GetSummary() != wantSummary {
-				t.Errorf("summary =\n%s\n\nwant\n%s", req.GetOutput().GetSummary(), wantSummary)
-			}
-		} else {
-			if d := cmp.Diff(req.Output.Annotations, wantAnnotations); d != "" {
-				t.Errorf("Annotation diff found:\n%s", d)
-			}
 		}
 	})
 
@@ -438,3 +231,158 @@ func TestCheck_summary_too_many_findings_cut_off_correctly(t *testing.T) {
 		t.Error("summary text was not cut off correctly")
 	}
 }
+
+func TestCheck_setToolNameForEachRun(t *testing.T) {
+	const (
+		owner       = "haya14busa"
+		repo        = "reviewdog"
+		prNum       = 14
+		sha         = "1414"
+		reportURL   = "http://example.com/report_url"
+		conclusion  = "neutral"
+		wantCheckID = 1414
+		toolName1   = "toolName1"
+		level1      = "warning"
+		toolName2   = "toolName2"
+		level2      = ""
+	)
+
+	mux := http.NewServeMux()
+	checkRunNum := 0
+	mux.HandleFunc("/repos/haya14busa/reviewdog/check-runs", func(w http.ResponseWriter, r *http.Request) {
+		checkRunNum++
+		var req github.CreateCheckRunOptions
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			t.Error(err)
+		}
+		switch checkRunNum {
+		case 1:
+			if req.Name != toolName1 {
+				t.Errorf("toolName = %s, want %s", req.Name, toolName1)
+			}
+		case 2:
+			if req.Name != toolName2 {
+				t.Errorf("toolName = %s, want %s", req.Name, toolName2)
+			}
+		}
+		if err := json.NewEncoder(w).Encode(&github.CheckRun{ID: github.Int64(wantCheckID)}); err != nil {
+			t.Fatal(err)
+		}
+	})
+	updateCheckRunNum := 0
+	mux.HandleFunc("/repos/haya14busa/reviewdog/check-runs/1414", func(w http.ResponseWriter, r *http.Request) {
+		var req github.CheckRun
+		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+			t.Error(err)
+		}
+		switch updateCheckRunNum {
+		case 0:
+			if req.GetName() != toolName1 {
+				t.Errorf("toolName = %s, want %s", req.GetName(), toolName1)
+			}
+			wantAnnotations := []*github.CheckRunAnnotation{
+				{
+					Path:            github.String("sample.new.txt"),
+					StartLine:       github.Int(2),
+					EndLine:         github.Int(2),
+					AnnotationLevel: github.String("warning"),
+					Message:         github.String("comment 1"),
+					Title:           github.String("[toolName1] sample.new.txt#L2"),
+				},
+			}
+			if req.GetStatus() == "completed" {
+				if wantTitle := "reviewdog [toolName1] report"; req.GetOutput().GetTitle() != wantTitle {
+					t.Errorf("title = %s, want %s", req.GetOutput().GetTitle(), wantTitle)
+				}
+			} else {
+				if d := cmp.Diff(req.Output.Annotations, wantAnnotations); d != "" {
+					t.Errorf("Annotation diff found:\n%s", d)
+				}
+			}
+		case 1:
+			if req.GetName() != toolName2 {
+				t.Errorf("toolName = %s, want %s", req.GetName(), toolName2)
+			}
+			wantAnnotations := []*github.CheckRunAnnotation{
+				{
+					Path:            github.String("sample.new.txt"),
+					StartLine:       github.Int(2),
+					EndLine:         github.Int(2),
+					AnnotationLevel: github.String("failure"), // default
+					Message:         github.String("comment 2"),
+					Title:           github.String("[toolName2] sample.new.txt#L2"),
+				},
+			}
+			if req.GetStatus() == "completed" {
+				if wantTitle := "reviewdog [toolName2] report"; req.GetOutput().GetTitle() != wantTitle {
+					t.Errorf("title = %s, want %s", req.GetOutput().GetTitle(), wantTitle)
+				}
+			} else {
+				if d := cmp.Diff(req.Output.Annotations, wantAnnotations); d != "" {
+					t.Errorf("Annotation diff found:\n%s", d)
+				}
+			}
+		}
+		if req.GetStatus() == "completed" {
+			updateCheckRunNum++
+		}
+	})
+
+	ts := httptest.NewServer(mux)
+	defer ts.Close()
+
+	cli := github.NewClient(nil)
+	cli.BaseURL, _ = url.Parse(ts.URL + "/")
+
+	check := &Check{
+		CLI:      cli,
+		Owner:    owner,
+		Repo:     repo,
+		PR:       prNum,
+		SHA:      sha,
+		ToolName: "", // empty
+		Level:    "", // empty
+	}
+
+	check.SetTool(toolName1, level1)
+	if err := check.Post(context.Background(), &reviewdog.Comment{
+		Result: &filter.FilteredDiagnostic{
+			Diagnostic: &rdf.Diagnostic{
+				Message: "comment 1",
+				Location: &rdf.Location{
+					Path: "sample.new.txt",
+					Range: &rdf.Range{
+						Start: &rdf.Position{Line: 2, Column: 1},
+					},
+				},
+			},
+			ShouldReport: true,
+		},
+	}); err != nil {
+		t.Errorf("failed to post: %v", err)
+	}
+	if err := check.Flush(context.Background()); err != nil {
+		t.Error(err)
+	}
+
+	check.SetTool(toolName2, level2)
+	if err := check.Post(context.Background(), &reviewdog.Comment{
+		Result: &filter.FilteredDiagnostic{
+			Diagnostic: &rdf.Diagnostic{
+				Message: "comment 2",
+				Location: &rdf.Location{
+					Path: "sample.new.txt",
+					Range: &rdf.Range{
+						Start: &rdf.Position{Line: 2, Column: 1},
+					},
+				},
+			},
+			ShouldReport: true,
+		},
+	}); err != nil {
+		t.Errorf("failed to post: %v", err)
+	}
+	if err := check.Flush(context.Background()); err != nil {
+		t.Error(err)
+	}
+}