Repository: reviewdog/reviewdog
Pull request #674 (2020-07-20 16:26:07)

Support rdjsonl as input format
- [ ] Updated Unreleased section in [CHANGELOG](https://github.com/reviewdog/reviewdog/blob/master/CHANGELOG.md) or it's not notable changes.




Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 059bd385..7a6e493d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,10 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### :book: Release Note <!-- optional -->
-<!-- TODO(haya14busa): Write about Reviewdog Diagnostic Format -->
 
 ### :rocket: Enhancements
-- [#629](https://github.com/reviewdog/reviewdog/pull/629) Introduced Reviewdog Diagnostic Format. ([@haya14busa])
 - ...
 
 ### :bug: Fixes
diff --git a/cmd/reviewdog/main.go b/cmd/reviewdog/main.go
index b8b78076..a57a75ec 100644
--- a/cmd/reviewdog/main.go
+++ b/cmd/reviewdog/main.go
@@ -19,7 +19,7 @@ import (
 	"golang.org/x/net/context" // "context"
 	"golang.org/x/oauth2"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 	"github.com/mattn/go-shellwords"
 	"github.com/reviewdog/errorformat/fmts"
 	"github.com/xanzy/go-gitlab"
@@ -368,6 +368,7 @@ func runList(w io.Writer) error {
 		fmt.Fprintf(tabw, "%s\t%s\t- %s\n", f.Name, f.Description, f.URL)
 	}
 	fmt.Fprintf(tabw, "%s\t%s\t- %s\n", "checkstyle", "checkstyle XML format", "http://checkstyle.sourceforge.net/")
+	fmt.Fprintf(tabw, "%s\t%s\t- %s\n", "rdjsonl", "Reviewdog Diagnostic JSONL Format", "https://github.com/reviewdog/reviewdog")
 	return tabw.Flush()
 }
 
diff --git a/doghouse/appengine/github.go b/doghouse/appengine/github.go
index 969c6151..38fd51f8 100644
--- a/doghouse/appengine/github.go
+++ b/doghouse/appengine/github.go
@@ -14,7 +14,7 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 	"github.com/justinas/nosurf"
 	"golang.org/x/oauth2"
 
diff --git a/doghouse/appengine/github_webhook.go b/doghouse/appengine/github_webhook.go
index 1ba7eb44..4803ba5e 100644
--- a/doghouse/appengine/github_webhook.go
+++ b/doghouse/appengine/github_webhook.go
@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 
 	"github.com/reviewdog/reviewdog/doghouse/server/storage"
 )
diff --git a/doghouse/client/github_client.go b/doghouse/client/github_client.go
index 660a6ad3..5158c6ce 100644
--- a/doghouse/client/github_client.go
+++ b/doghouse/client/github_client.go
@@ -3,7 +3,7 @@ package client
 import (
 	"context"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 
 	"github.com/reviewdog/reviewdog/doghouse"
 	"github.com/reviewdog/reviewdog/doghouse/server"
diff --git a/doghouse/server/doghouse.go b/doghouse/server/doghouse.go
index 7e45a032..51edbcab 100644
--- a/doghouse/server/doghouse.go
+++ b/doghouse/server/doghouse.go
@@ -8,7 +8,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 
 	"github.com/reviewdog/reviewdog"
 	"github.com/reviewdog/reviewdog/diff"
diff --git a/doghouse/server/github.go b/doghouse/server/github.go
index ee032c11..08503930 100644
--- a/doghouse/server/github.go
+++ b/doghouse/server/github.go
@@ -6,7 +6,7 @@ import (
 	"net/http"
 
 	"github.com/bradleyfalzon/ghinstallation"
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 )
 
 type NewGitHubClientOption struct {
diff --git a/doghouse/server/github_checker.go b/doghouse/server/github_checker.go
index 88dd28ed..1c55ecb5 100644
--- a/doghouse/server/github_checker.go
+++ b/doghouse/server/github_checker.go
@@ -3,7 +3,7 @@ package server
 import (
 	"context"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 )
 
 type checkerGitHubClientInterface interface {
diff --git a/go.mod b/go.mod
index 739e9c1b..4aa897cf 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/bradleyfalzon/ghinstallation v1.1.1
 	github.com/golang/protobuf v1.4.2
 	github.com/google/go-cmp v0.5.0
-	github.com/google/go-github/v32 v32.1.0
+	github.com/google/go-github/v31 v31.0.0
 	github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01
 	github.com/haya14busa/secretbox v0.0.0-20180525171038-07c7ecf409f5
 	github.com/justinas/nosurf v1.1.0
diff --git a/go.sum b/go.sum
index ea898553..5860189c 100644
--- a/go.sum
+++ b/go.sum
@@ -127,8 +127,8 @@ github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4r
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU8N8Pts=
 github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E=
-github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II=
-github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
+github.com/google/go-github/v31 v31.0.0 h1:JJUxlP9lFK+ziXKimTCprajMApV1ecWD4NB6CCb0plo=
+github.com/google/go-github/v31 v31.0.0/go.mod h1:NQPZol8/1sMoWYGN2yaALIBytu17gAWfhbweiEed3pM=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
diff --git a/parser.go b/parser.go
index b4e41bbf..78e1c9aa 100644
--- a/parser.go
+++ b/parser.go
@@ -1,6 +1,8 @@
 package reviewdog
 
 import (
+	"bufio"
+	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
@@ -26,9 +28,13 @@ func NewParser(opt *ParserOpt) (Parser, error) {
 		return nil, errors.New("you cannot specify both format name and errorformat at the same time")
 	}
 
-	if name == "checkstyle" {
+	switch name {
+	case "checkstyle":
 		return NewCheckStyleParser(), nil
+	case "rdjsonl":
+		return NewRDJSONLParser(), nil
 	}
+
 	// use defined errorformat
 	if name != "" {
 		efm, ok := fmts.DefinedFmts()[name]
@@ -158,3 +164,23 @@ type CheckStyleError struct {
 	Severity string `xml:"severity,attr,omitempty"`
 	Source   string `xml:"source,attr,omitempty"`
 }
+
+// RDJSONLParser is parser for rdjsonl format.
+type RDJSONLParser struct{}
+
+func NewRDJSONLParser() *RDJSONLParser {
+	return &RDJSONLParser{}
+}
+
+func (p *RDJSONLParser) Parse(r io.Reader) ([]*CheckResult, error) {
+	var results []*CheckResult
+	s := bufio.NewScanner(r)
+	for s.Scan() {
+		d := new(rdf.Diagnostic)
+		if err := json.Unmarshal(s.Bytes(), d); err != nil {
+			return nil, err
+		}
+		results = append(results, &CheckResult{Diagnostic: d, Lines: []string{s.Text()}})
+	}
+	return results, nil
+}
diff --git a/proto/rdf/README.md b/proto/rdf/README.md
index e55d2f75..22d78d9c 100644
--- a/proto/rdf/README.md
+++ b/proto/rdf/README.md
@@ -116,9 +116,6 @@ Example:
 
 ## Background: Still No Good Standard Diagnostic Format Out There in 2020
 
-Update: Found *The Static Analysis Results Interchange Format (SARIF)* as a
-potential good standard format.
-
 As of writing (2020), most diagnostic tools such as linters or compilers output
 results with their own format. Some tools support machine-readable structured
 format like their own JSON format, and other tools just support unstructured
@@ -160,45 +157,6 @@ format, but not everyone wants to support it.
 There are altenative solutions out there (which are not used by reviewdog) as
 well.
 
-### The Static Analysis Results Interchange Format (SARIF)
-[The Static Analysis Results Interchange Format (SARIF)](https://sarifweb.azurewebsites.net/)
-has been approved as an OASIS standard.
-
-Although, there are not many usages of SARIF as of writing (2020 July, 21),
-it can be good standard format.
-A promising usage example is [GitHub Code Scanning](https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/about-code-scanning#about-third-party-code-scanning-tools)
-(beta), which uses SARIF to support third party code scanning tools.
-Other examples: [spotbugs](https://github.com/spotbugs/discuss/issues/95).
-
-Problems:
-- No stream output support and static analysis tools cannot output each diagnostic result one by one.
-- `columnKind` doesn't support byte count. https://github.com/oasis-tcs/sarif-spec/issues/466
-- The spec is too big and complex ([SARIF v2.1.0 PDF](https://docs.oasis-open.org/sarif/sarif/v2.1.0/sarif-v2.1.0.pdf) is 227 pages!)
-  for developer tools as consumer of SARIF (e.g.  reviewdog). Probably most
-  tools end up with supporting SARIF partially.  GitHub Code Scanning feature
-  actually doesn't support a whole spec
-  ([doc](https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/sarif-support-for-code-scanning))
-  for example.
-- The spec is too big and complex for static analysis tools as provider of
-  SARIF. They can just support partial and minimum SARIF support as result
-  output format but it's still not simple and the output still needs to pass
-  SARIF validatiotor.
-- Not all languages have good tools to generate code from JSON Schema.
-  To create Go SARIF package [haya14busa/go-sarif](https://github.com/haya14busa/go-sarif),
-  I needed to try 3+ [Go](https://github.com/atombender/go-jsonschema) [JSON Schema](https://github.com/idubinskiy/schematyper)
-  [Code Generator](https://github.com/aaharu/schemarshal)
-  tools but all of them didn't work for the complex SARIF JSON Schema.
-  I ended up using [quicktype](https://github.com/quicktype/quicktype) and it
-  worked but I still needed to send [a Pull Request](https://github.com/quicktype/quicktype/pull/1513)...
-- SARIF SDK and related tools are written in C# (and TypeScript), which means we need dotnet runtime.
-  SARIF is general and standard format while the related tools requires dotnet runtime.
-
-There are some problems as above but SARIF should be still good to support
-considering it has been already approved as an OASIS standard and GitHub Code
-Scanning uses it.
-Reviewdog Diagnostic Format can be used as simpler format and we can create
-converters between RD Format and SARIF.
-
 ### *Problem Matcher*
 [VSCode](https://vscode-docs.readthedocs.io/en/stable/editor/tasks/#defining-a-problem-matcher)
 and [GitHub Actions](https://github.com/actions/toolkit/blob/master/docs/problem-matchers.md)
@@ -304,8 +262,17 @@ depend on the CLI, so providing libraries for their implementation languages
 should be useful to format results natively by each diagnostic tool.
 
 ## Open Questions
+- Naming:
+  - Is "Reviewdog Diagnostic Protocol" good naming?
+  - Should we really include *reviewdog* in the name? It's true that the
+    primary motivation is for the reviewdog CLI, but it can be just a standard
+    diagnostic format and it can be used outside reviwedog.
+  - Is "Protocol" good word? Should we use scheme/format/specification/etc...
+    instead?
+  - RDP is not unique acronym and [Remote Desktop Protocol](https://en.wikipedia.org/wiki/Remote_Desktop_Protocol)
+    is already using it.
 - Protocol Version Representation and Backward/Future Compatibility
-  - Should we add version or some capability data in RD Format?
-  - RD Format should be stable, but there are still a possibility to extend it with
+  - Should we add version or some capability data in RDP?
+  - RPD should be stable, but there are still a possibility to extend it with
     backward incompatible way. e.g. We **may** want to add byte offset field in
     Position message as an alternative of line and column.
diff --git a/service/github/github.go b/service/github/github.go
index 8e00e58a..8eb1d64f 100644
--- a/service/github/github.go
+++ b/service/github/github.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 
 	"github.com/reviewdog/reviewdog"
 	"github.com/reviewdog/reviewdog/cienv"


Test patch:
diff --git a/doghouse/server/doghouse_test.go b/doghouse/server/doghouse_test.go
index d28fd4ed..0a4bbf32 100644
--- a/doghouse/server/doghouse_test.go
+++ b/doghouse/server/doghouse_test.go
@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 
 	"github.com/reviewdog/reviewdog/difffilter"
 	"github.com/reviewdog/reviewdog/doghouse"
diff --git a/parser_test.go b/parser_test.go
index e0dc043a..b0a1016a 100644
--- a/parser_test.go
+++ b/parser_test.go
@@ -18,6 +18,12 @@ func TestNewParser(t *testing.T) {
 			},
 			typ: &CheckStyleParser{},
 		},
+		{
+			in: &ParserOpt{
+				FormatName: "rdjsonl",
+			},
+			typ: &RDJSONLParser{},
+		},
 		{
 			in: &ParserOpt{
 				FormatName: "golint",
@@ -103,3 +109,22 @@ func TestCheckStyleParser(t *testing.T) {
 		}
 	}
 }
+
+func TestRDJSONLParser(t *testing.T) {
+	const sample = `{"source":{"name":"deadcode"},"message":"'unused' is unused","location":{"path":"testdata/main.go","range":{"start":{"line":18,"column":6}}}}
+{"source":{"name":"deadcode"},"message":"'unused2' is unused","location":{"path":"testdata/main.go","range":{"start":{"line":24,"column":6}}}}
+{"source":{"name":"errcheck"},"message":"Error return value of 'os.Open' is not checked","location":{"path":"testdata/main.go","range":{"start":{"line":15,"column":9}}}}
+{"source":{"name":"ineffassign"},"message":"ineffectual assignment to 'x'","location":{"path":"testdata/main.go","range":{"start":{"line":12,"column":2}}}}
+{"source":{"name":"govet"},"message":"printf: Sprintf format %d reads arg #1, but call has 0 args","location":{"path":"testdata/main.go","range":{"start":{"line":13,"column":2}}}}`
+	sampleLines := strings.Split(sample, "\n")
+	p := NewRDJSONLParser()
+	crs, err := p.Parse(strings.NewReader(sample))
+	if err != nil {
+		t.Error(err)
+	}
+	for i, cr := range crs {
+		if got, want := cr.Lines[0], sampleLines[i]; got != want {
+			t.Errorf("%d: got %v, want %v", i, got, want)
+		}
+	}
+}
diff --git a/service/github/github_test.go b/service/github/github_test.go
index 1f868b4c..73319c45 100644
--- a/service/github/github_test.go
+++ b/service/github/github_test.go
@@ -10,7 +10,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/google/go-github/v32/github"
+	"github.com/google/go-github/v31/github"
 	"github.com/kylelemons/godebug/pretty"
 	"golang.org/x/oauth2"