nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: gabriel-vasile/mimetype
Pull request #47 (2019-09-25 13:44:09)

Detect CRLF line endings in vcard and vcalendar
.ics files generated by Google Calendar contain CRLF (dos) line
endings which conforms to the RFC.

See:
vcalendar(.ics): https://tools.ietf.org/html/rfc5545#section-3.4
vcard(.vcf): https://tools.ietf.org/html/rfc6350#section-3.3

Bonus (unrelated):
Removes execute permissions from following sample files in testdata directory:
```
testdata/exe.exe
testdata/ln
testdata/php.php
testdata/tcl.tcl
testdata/wasm.wasm
```


Patch:
diff --git a/internal/matchers/text.go b/internal/matchers/text.go
index 2775ef1..f31472e 100644
--- a/internal/matchers/text.go
+++ b/internal/matchers/text.go
@@ -69,9 +69,11 @@ var (
 	}
 	vCardSigs = []sig{
 		ciSig("BEGIN:VCARD\n"),
+		ciSig("BEGIN:VCARD\r\n"),
 	}
 	iCalSigs = []sig{
 		ciSig("BEGIN:VCALENDAR\n"),
+		ciSig("BEGIN:VCALENDAR\r\n"),
 	}
 	phpSigs = []sig{
 		ciSig("<?PHP"),


Test patch:
diff --git a/mime_test.go b/mime_test.go
index facd8a4..c9c8654 100644
--- a/mime_test.go
+++ b/mime_test.go
@@ -109,7 +109,9 @@ var files = map[string]*node{
 	"py.py":             python,
 	"tcl.tcl":           tcl,
 	"vCard.vCard":       vCard,
+	"vCard.dos.vCard":   vCard,
 	"ics.ics":           iCalendar,
+	"ics.dos.ics":       iCalendar,
 
 	// binary
 	"class.class": class,
diff --git a/testdata/exe.exe b/testdata/exe.exe
old mode 100755
new mode 100644
diff --git a/testdata/ics.dos.ics b/testdata/ics.dos.ics
new file mode 100644
index 0000000..c6e1f8d
--- /dev/null
+++ b/testdata/ics.dos.ics
@@ -0,0 +1,13 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//hacksw/handcal//NONSGML v1.0//EN
+BEGIN:VEVENT
+UID:uid1@example.com
+DTSTAMP:19970714T170000Z
+ORGANIZER;CN=John Doe:MAILTO:john.doe@example.com
+DTSTART:19970714T170000Z
+DTEND:19970715T035959Z
+SUMMARY:Bastille Day Party
+GEO:48.85299;2.36885
+END:VEVENT
+END:VCALENDAR
diff --git a/testdata/ln b/testdata/ln
old mode 100755
new mode 100644
diff --git a/testdata/php.php b/testdata/php.php
old mode 100755
new mode 100644
diff --git a/testdata/tcl.tcl b/testdata/tcl.tcl
old mode 100755
new mode 100644
diff --git a/testdata/vCard.dos.vCard b/testdata/vCard.dos.vCard
new file mode 100644
index 0000000..293c49d
--- /dev/null
+++ b/testdata/vCard.dos.vCard
@@ -0,0 +1,15 @@
+BEGIN:VCARD
+VERSION:4.0
+N:Gump;Forrest;;Mr.;
+FN:Forrest Gump
+ORG:Bubba Gump Shrimp Co.
+TITLE:Shrimp Man
+PHOTO;MEDIATYPE=image/gif:http://www.example.com/dir_photos/my_photo.gif
+TEL;TYPE=work,voice;VALUE=uri:tel:+1-111-555-1212
+TEL;TYPE=home,voice;VALUE=uri:tel:+1-404-555-1212
+ADR;TYPE=WORK;PREF=1;LABEL="100 Waters Edge\nBaytown\, LA 30314\nUnited States of America":;;100 Waters Edge;Baytown;LA;30314;United States of America
+ADR;TYPE=HOME;LABEL="42 Plantation St.\nBaytown\, LA 30314\nUnited States of America":;;42 Plantation St.;Baytown;LA;30314;United States of America
+EMAIL:forrestgump@example.com
+REV:20080424T195243Z
+x-qq:21588891
+END:VCARD
Repository: gabriel-vasile/mimetype
Pull request #59 (2019-10-14 22:33:36)

Trim whitespace from start of HTML/XML files
Problem:
- HTML+XML files with whitespace before the opening tags were being identified as `text/plain`.

Solution:
- Trim whitespace from beginning using trimLWS (similar to geojson matcher)


Patch:
diff --git a/internal/matchers/text.go b/internal/matchers/text.go
index 71ea83c..2c391c4 100644
--- a/internal/matchers/text.go
+++ b/internal/matchers/text.go
@@ -136,11 +136,19 @@ func Txt(in []byte) bool {
 
 // Html matches a Hypertext Markup Language file.
 func Html(in []byte) bool {
+	in = trimLWS(in)
+	if len(in) == 0 {
+		return false
+	}
 	return detect(in, htmlSigs)
 }
 
 // Xml matches an Extensible Markup Language file.
 func Xml(in []byte) bool {
+	in = trimLWS(in)
+	if len(in) == 0 {
+		return false
+	}
 	return detect(in, xmlSigs)
 }
 


Test patch:
diff --git a/mime_test.go b/mime_test.go
index 1e99b79..d0726de 100644
--- a/mime_test.go
+++ b/mime_test.go
@@ -97,6 +97,7 @@ var files = map[string]*node{
 
 	// source code
 	"html.html":         html,
+	"html.withbr.html":  html,
 	"svg.svg":           svg,
 	"svg.1.svg":         svg,
 	"txt.txt":           txt,
@@ -138,18 +139,18 @@ var files = map[string]*node{
 	"eot.eot":     eot,
 
 	// XML and subtypes of XML
-	"xml.xml":   xml,
-	"kml.kml":   kml,
-	"xlf.xlf":   xliff,
-	"dae.dae":   collada,
-	"gml.gml":   gml,
-	"gpx.gpx":   gpx,
-	"tcx.tcx":   tcx,
-	"x3d.x3d":   x3d,
-	"amf.amf":   amf,
-	"3mf.3mf":   threemf,
-	"rss.rss":   rss,
-	"atom.atom": atom,
+	"xml.withbr.xml": xml,
+	"kml.kml":        kml,
+	"xlf.xlf":        xliff,
+	"dae.dae":        collada,
+	"gml.gml":        gml,
+	"gpx.gpx":        gpx,
+	"tcx.tcx":        tcx,
+	"x3d.x3d":        x3d,
+	"amf.amf":        amf,
+	"3mf.3mf":        threemf,
+	"rss.rss":        rss,
+	"atom.atom":      atom,
 
 	"shp.shp": shp,
 	"shx.shx": shx,
diff --git a/testdata/html.withbr.html b/testdata/html.withbr.html
new file mode 100644
index 0000000..db24d84
--- /dev/null
+++ b/testdata/html.withbr.html
@@ -0,0 +1,14 @@
+
+<!DOCTYPE html>
+<html>
+  <head><!--[if lt IE 9]><script language="javascript" type="text/javascript" src="//html5shim.googlecode.com/svn/trunk/html5.js"></script><![endif]-->
+    <meta charset="UTF-8"><style>/*
+     </style>
+    <link rel="stylesheet" href="css/animation.css"><!--[if IE 7]><link rel="stylesheet" href="css/" + font.fontname + "-ie7.css"><![endif]-->
+    <script>
+    </script>
+  </head>
+  <body>
+    <div class="container footer"></div>
+  </body>
+</html>
diff --git a/testdata/xml.withbr.xml b/testdata/xml.withbr.xml
new file mode 100644
index 0000000..9706f55
--- /dev/null
+++ b/testdata/xml.withbr.xml
@@ -0,0 +1,8 @@
+
+<?xml version="1.0" encoding="UTF-8"?>
+<note>
+  <to>Tove</to>
+  <from>Jani</from>
+  <heading>Reminder</heading>
+  <body>Don't forget me this weekend!</body>
+</note>
Repository: gabriel-vasile/mimetype
Pull request #91 (2020-03-12 16:26:02)

DetectReader should read much
https://play.golang.org/p/cIGB9tHLPf1

```go
package main

import (
	"fmt"

	"github.com/gabriel-vasile/mimetype"
)

func main() {
	const src = "<html><head><title>html</title></head><body>abcdefg</body></html>"
	// breakReader breaks the string every breakSize characters.
	// It is like:
	//   <htm
	//   l><h
	//   ead>
	//   <tit
	//   le>h
	//   tml<
	//   ...
	r := breakReader(src)
	m, err := mimetype.DetectReader(&r)
	fmt.Println(m, err)
}

type breakReader string

func (s *breakReader) Read(p []byte) (int, error) {
	const breakSize = 4
	var t string
	n := min(breakSize, len(*s))
	*s, t = (*s)[n:], string((*s)[:n])
	copy(p, t)
	return n, nil
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}
```

Currently, DetectReader reads from a reader just once. As the above example shows, DetectReader may not be able to detect mime type.




Patch:
diff --git a/mimetype.go b/mimetype.go
index c0a9f4e..672e222 100644
--- a/mimetype.go
+++ b/mimetype.go
@@ -36,8 +36,8 @@ func Detect(in []byte) (mime *MIME) {
 // matchers.ReadLimit bytes from the reader.
 func DetectReader(r io.Reader) (mime *MIME, err error) {
 	in := make([]byte, matchers.ReadLimit)
-	n, err := r.Read(in)
-	if err != nil && err != io.EOF {
+	n, err := io.ReadFull(r, in)
+	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 		return root, err
 	}
 	in = in[:n]


Test patch:
diff --git a/mimetype_test.go b/mimetype_test.go
index 82a1e1e..8cbbe17 100644
--- a/mimetype_test.go
+++ b/mimetype_test.go
@@ -216,6 +216,41 @@ func TestDetect(t *testing.T) {
 	}
 }
 
+func TestDetectReader(t *testing.T) {
+	errStr := "File: %s; Mime: %s != DetectedMime: %s; err: %v"
+	for fName, node := range files {
+		fileName := filepath.Join(testDataDir, fName)
+		f, err := os.Open(fileName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		r := breakReader{
+			r:         f,
+			breakSize: 3,
+		}
+		if mime, err := DetectReader(&r); mime.String() != node.mime {
+			t.Errorf(errStr, fName, node.mime, mime.String(), err)
+		}
+		f.Close()
+	}
+}
+
+type breakReader struct {
+	r         io.Reader
+	breakSize int
+}
+
+func (b *breakReader) Read(p []byte) (int, error) {
+	if len(p) > b.breakSize {
+		p = p[:b.breakSize]
+	}
+	n, err := io.ReadFull(b.r, p)
+	if err == io.EOF && err == io.ErrUnexpectedEOF {
+		return n, io.EOF
+	}
+	return n, err
+}
+
 func TestFaultyInput(t *testing.T) {
 	inexistent := "inexistent.file"
 	if _, err := DetectFile(inexistent); err == nil {
Repository: gabriel-vasile/mimetype
Pull request #199 (2021-11-11 21:54:03)

Fix JSON returned for partial documents when readLimit=0


Patch:
diff --git a/internal/magic/text.go b/internal/magic/text.go
index 026c6aa..40bfb19 100644
--- a/internal/magic/text.go
+++ b/internal/magic/text.go
@@ -155,12 +155,14 @@ func Php(raw []byte, limit uint32) bool {
 // JSON matches a JavaScript Object Notation file.
 func JSON(raw []byte, limit uint32) bool {
 	raw = trimLWS(raw)
-	if len(raw) == 0 || (raw[0] != '[' && raw[0] != '{') {
+	// #175 A single JSON string, number or bool is not considered JSON.
+	// JSON objects and arrays are reported as JSON.
+	if len(raw) < 2 || (raw[0] != '[' && raw[0] != '{') {
 		return false
 	}
 	parsed, err := json.Scan(raw)
 	// If the full file content was provided, check there is no error.
-	if len(raw) < int(limit) {
+	if limit == 0 || len(raw) < int(limit) {
 		return err == nil
 	}
 


Test patch:
diff --git a/internal/magic/magic_test.go b/internal/magic/magic_test.go
new file mode 100644
index 0000000..f89aff8
--- /dev/null
+++ b/internal/magic/magic_test.go
@@ -0,0 +1,20 @@
+package magic
+
+import "testing"
+
+var magicTests = []struct {
+	raw      string
+	limit    uint32
+	res      bool
+	detector Detector
+}{
+	{`["an incomplete JSON array`, 0, false, JSON},
+}
+
+func TestMagic(t *testing.T) {
+	for i, tt := range magicTests {
+		if got := tt.detector([]byte(tt.raw), tt.limit); got != tt.res {
+			t.Errorf("Detector %d error: expected: %t; got: %t", i, tt.res, got)
+		}
+	}
+}
Repository: gabriel-vasile/mimetype
Pull request #241 (2022-01-30 21:43:57)

Fix json detected as dbf when it starts with low ascii values
Byte 0x09 represents ninth month in dbf header, but it is also the ascii
value of < TAB >. This made dbf detection pass for json and txt files
happening to contain low ascii values at the start.
This commit improves dbf detection to check for several null bytes in
the input. json and txt files never contain null bytes.


Patch:
diff --git a/internal/magic/binary.go b/internal/magic/binary.go
index feca467..83ba2d0 100644
--- a/internal/magic/binary.go
+++ b/internal/magic/binary.go
@@ -64,16 +64,27 @@ func MachO(raw []byte, limit uint32) bool {
 // Dbf matches a dBase file.
 // https://www.dbase.com/Knowledgebase/INT/db7_file_fmt.htm
 func Dbf(raw []byte, limit uint32) bool {
-	if len(raw) < 4 {
+	if len(raw) < 68 {
 		return false
 	}
 
-	// 3rd and 4th bytes contain the last update month and day of month
+	// 3rd and 4th bytes contain the last update month and day of month.
 	if !(0 < raw[2] && raw[2] < 13 && 0 < raw[3] && raw[3] < 32) {
 		return false
 	}
 
-	// dbf type is dictated by the first byte
+	// 12, 13, 30, 31 are reserved bytes and always filled with 0x00.
+	if raw[12] != 0x00 || raw[13] != 0x00 || raw[30] != 0x00 || raw[31] != 0x00 {
+		return false
+	}
+	// Production MDX flag;
+	// 0x01 if a production .MDX file exists for this table;
+	// 0x00 if no .MDX file exists.
+	if raw[28] > 0x01 {
+		return false
+	}
+
+	// dbf type is dictated by the first byte.
 	dbfTypes := []byte{
 		0x02, 0x03, 0x04, 0x05, 0x30, 0x31, 0x32, 0x42, 0x62, 0x7B, 0x82,
 		0x83, 0x87, 0x8A, 0x8B, 0x8E, 0xB3, 0xCB, 0xE5, 0xF5, 0xF4, 0xFB,


Test patch:
diff --git a/mimetype_test.go b/mimetype_test.go
index 7bb6aea..0f6fe12 100644
--- a/mimetype_test.go
+++ b/mimetype_test.go
@@ -94,6 +94,7 @@ var files = map[string]string{
 	"xpm.xpm":            "image/x-xpixmap",
 	"js.js":              "application/javascript",
 	"json.json":          "application/json",
+	"json.lowascii.json": "application/json",
 	// json.{int,float,string}.txt contain a single JSON value. They are valid JSON
 	// documents, but they should not be detected as application/json. This mimics
 	// the behaviour of the file utility and seems the correct thing to do.
diff --git a/testdata/json.lowascii.json b/testdata/json.lowascii.json
new file mode 100644
index 0000000..36f2f4c
--- /dev/null
+++ b/testdata/json.lowascii.json
@@ -0,0 +1,2 @@
+{
+		"fixture for issue #": 239}
Repository: gabriel-vasile/mimetype
Pull request #531 (2024-05-22 16:16:21)

add defaultLimit and use it when resetting back


Patch:
diff --git a/mimetype.go b/mimetype.go
index 2d31ae6..b7163dc 100644
--- a/mimetype.go
+++ b/mimetype.go
@@ -13,8 +13,10 @@ import (
 	"sync/atomic"
 )
 
+var defaultLimit uint32 = 3072
+
 // readLimit is the maximum number of bytes from the input used when detecting.
-var readLimit uint32 = 3072
+var readLimit uint32 = defaultLimit
 
 // Detect returns the MIME type found from the provided byte slice.
 //


Test patch:
diff --git a/mimetype_test.go b/mimetype_test.go
index 0673f7e..445b1ed 100644
--- a/mimetype_test.go
+++ b/mimetype_test.go
@@ -444,7 +444,7 @@ func TestConcurrent(t *testing.T) {
 
 	wg.Wait()
 	// Reset to original limit for benchmarks.
-	SetLimit(3072)
+	SetLimit(defaultLimit)
 }
 
 // For #162.
@@ -469,7 +469,7 @@ func TestEmptyInput(t *testing.T) {
 	if !mtype.Is(plain) {
 		t.Fatalf("0 limit, empty reader detection; expected: %s, got: %s", plain, mtype)
 	}
-	SetLimit(3072)
+	SetLimit(defaultLimit)
 }
 
 // Benchmarking a random slice of bytes is as close as possible to the real
@@ -480,7 +480,7 @@ func TestEmptyInput(t *testing.T) {
 // might be tested for zip, gzip, etc., before it is identified.
 func BenchmarkSliceRand(b *testing.B) {
 	r := rand.New(rand.NewSource(0))
-	data := make([]byte, 3072)
+	data := make([]byte, defaultLimit)
 	if _, err := io.ReadFull(r, data); err != io.ErrUnexpectedEOF && err != nil {
 		b.Fatal(err)
 	}
@@ -497,7 +497,7 @@ func BenchmarkSliceRand(b *testing.B) {
 
 func BenchmarkAll(b *testing.B) {
 	r := rand.New(rand.NewSource(0))
-	data := make([]byte, 3072)
+	data := make([]byte, defaultLimit)
 	if _, err := io.ReadFull(r, data); err != io.ErrUnexpectedEOF && err != nil {
 		b.Fatal(err)
 	}
@@ -510,7 +510,6 @@ func BenchmarkAll(b *testing.B) {
 			}
 		})
 	}
-
 }
 
 func BenchmarkCommon(b *testing.B) {