nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: netlify/build
Pull request #1852 (2020-09-22 18:44:21)

Fix telemetry crash when required programmatically
The two main ways `@netlify/build` is used (called `mode`) are the buildbot and the CLI. There is a third way: programmatically with `require()`. That third way is only useful in some specific cases, such as when testing a Build plugin.

Our telemetry logic does not make much sense in that mode, since the code could potentially be run a lot of times if the user is doing stress testing. Also, we found that this code is sometimes crashing. Therefore, it is safer to disable telemetry by default when `mode` is `require`, which this PR does.


Patch:
diff --git a/packages/build/src/core/normalize_flags.js b/packages/build/src/core/normalize_flags.js
index 1f6c41674..ee10617f0 100644
--- a/packages/build/src/core/normalize_flags.js
+++ b/packages/build/src/core/normalize_flags.js
@@ -24,18 +24,18 @@ const normalizeFlags = function(flags, logs) {
 }
 
 // Default values of CLI flags
-const getDefaultFlags = function({ env: envOpt = {} }) {
+const getDefaultFlags = function({ env: envOpt = {}, mode = REQUIRE_MODE }) {
   const combinedEnv = { ...env, ...envOpt }
   return {
     env: envOpt,
     nodePath: execPath,
     token: combinedEnv.NETLIFY_AUTH_TOKEN,
-    mode: 'require',
+    mode: REQUIRE_MODE,
     functionsDistDir: DEFAULT_FUNCTIONS_DIST,
     deployId: combinedEnv.DEPLOY_ID,
     debug: Boolean(combinedEnv.NETLIFY_BUILD_DEBUG),
     bugsnagKey: combinedEnv.BUGSNAG_KEY,
-    telemetry: !combinedEnv.BUILD_TELEMETRY_DISABLED,
+    telemetry: !combinedEnv.BUILD_TELEMETRY_DISABLED && mode !== REQUIRE_MODE,
     sendStatus: false,
     testOpts: {},
     featureFlags: DEFAULT_FEATURE_FLAGS,
@@ -43,6 +43,7 @@ const getDefaultFlags = function({ env: envOpt = {} }) {
   }
 }
 
+const REQUIRE_MODE = 'require'
 const DEFAULT_FUNCTIONS_DIST = '.netlify/functions/'
 const DEFAULT_STATSD_PORT = 8125
 


Test patch:
diff --git a/packages/build/tests/core/tests.js b/packages/build/tests/core/tests.js
index 2a5913dc2..42ac0950d 100644
--- a/packages/build/tests/core/tests.js
+++ b/packages/build/tests/core/tests.js
@@ -263,6 +263,16 @@ test('Telemetry disabled with flag', async t => {
   t.is(requests.length, 0)
 })
 
+test('Telemetry disabled with mode', async t => {
+  const { scheme, host, requests, stopServer } = await startServer(TELEMETRY_PATH)
+  await runFixture(t, 'success', {
+    flags: { siteId: 'test', testOpts: { telemetryOrigin: `${scheme}://${host}` }, telemetry: undefined },
+    snapshot: false,
+  })
+  await stopServer()
+  t.is(requests.length, 0)
+})
+
 test('Telemetry error', async t => {
   const { stopServer } = await startServer(TELEMETRY_PATH)
   await runFixture(t, 'success', {
Repository: netlify/build
Pull request #1890 (2020-09-29 18:40:01)

Expose `constants.NETLIFY_API_TOKEN`
Part of https://github.com/netlify/pod-the-builder/issues/33

This exposes `constants.NETLIFY_API_TOKEN` to all plugins, not only user plugins.
This constant will be undocumented for the moment since we only expose three scopes (which might not be useful to most plugins): `deploy:cancel` (limited only to the current deploy), `deploy:edgefns` (limited only to the current deploy) and `plugin_run:create` (limited only to the current build/deploy).


Patch:
diff --git a/packages/build/src/plugins/load.js b/packages/build/src/plugins/load.js
index 05aceee0f..74c4d9b07 100644
--- a/packages/build/src/plugins/load.js
+++ b/packages/build/src/plugins/load.js
@@ -49,13 +49,13 @@ const loadPlugin = async function(
   }
 }
 
-// Only core plugins can use `constants.NETLIFY_API_TOKEN` at the moment
-const cleanConstants = function({ NETLIFY_API_TOKEN, BUILDBOT_SERVER_SOCKET, ...constants }, loadedFrom) {
+// Only core plugins can use `constants.BUILDBOT_SERVER_SOCKET` at the moment
+const cleanConstants = function({ BUILDBOT_SERVER_SOCKET, ...constants }, loadedFrom) {
   if (loadedFrom !== 'core') {
     return constants
   }
 
-  return { ...constants, NETLIFY_API_TOKEN, BUILDBOT_SERVER_SOCKET }
+  return { ...constants, BUILDBOT_SERVER_SOCKET }
 }
 
 module.exports = { loadPlugins }


Test patch:
diff --git a/packages/build/tests/plugins/snapshots/tests.js.md b/packages/build/tests/plugins/snapshots/tests.js.md
index 6ee702bc4..c6c5bb336 100644
--- a/packages/build/tests/plugins/snapshots/tests.js.md
+++ b/packages/build/tests/plugins/snapshots/tests.js.md
@@ -2703,7 +2703,7 @@ Generated by [AVA](https://ava.li).
     │ 1. onPreBuild command from /file/path │␊
     └─────────────────────────────────────┘␊
     ␊
-    none␊
+    test␊
     ␊
     (/file/path onPreBuild completed in 1ms)␊
     ␊
Repository: Khan/KaTeX
Pull request #960 (2017-11-12 03:42:29)

Append ApplyFunction to math functions
As described at: https://www.w3.org/TR/REC-MathML/chap3_2.html#sec3.2.4

Character U+2061 is the same as &ApplyFunction;. Ref: https://www.w3schools.com/charsets/ref_html_entities_a.asp


Patch:
diff --git a/src/functions/op.js b/src/functions/op.js
index 4f8ff499..285c279d 100644
--- a/src/functions/op.js
+++ b/src/functions/op.js
@@ -209,8 +209,12 @@ const mathmlBuilder = (group, options) => {
         node = new mathMLTree.MathNode(
             "mi", [new mathMLTree.TextNode(group.value.body.slice(1))]);
 
-        // TODO(ron): Append an <mo>&ApplyFunction;</mo> as in \operatorname
-        // ref: https://www.w3.org/TR/REC-MathML/chap3_2.html#sec3.2.2
+        // Append an <mo>&ApplyFunction;</mo>.
+        // ref: https://www.w3.org/TR/REC-MathML/chap3_2.html#sec3.2.4
+        const operator = new mathMLTree.MathNode("mo",
+            [mml.makeText("\u2061", "text")]);
+
+        return new domTree.documentFragment([node, operator]);
     }
 
     return node;
diff --git a/src/functions/operatorname.js b/src/functions/operatorname.js
index ef47d682..1a15b272 100644
--- a/src/functions/operatorname.js
+++ b/src/functions/operatorname.js
@@ -68,8 +68,10 @@ defineFunction({
         const identifier = new mathMLTree.MathNode("mi", output);
         identifier.setAttribute("mathvariant", "normal");
 
+        // \u2061 is the same as &ApplyFunction;
+        // ref: https://www.w3schools.com/charsets/ref_html_entities_a.asp
         const operator = new mathMLTree.MathNode("mo",
-            [mml.makeText("&ApplyFunction;", "text")]);
+            [mml.makeText("\u2061", "text")]);
 
         return new domTree.documentFragment([identifier, operator]);
     },


Test patch:
diff --git a/test/__snapshots__/mathml-spec.js.snap b/test/__snapshots__/mathml-spec.js.snap
index ad4b36d7..3b378772 100644
--- a/test/__snapshots__/mathml-spec.js.snap
+++ b/test/__snapshots__/mathml-spec.js.snap
@@ -27,6 +27,9 @@ exports[`A MathML builder should generate the right types of nodes 1`] = `
       <mi>
         sin
       </mi>
+      <mo>
+        ⁡
+      </mo>
       <mrow>
         <mi>
           x
Repository: Khan/KaTeX
Pull request #991 (2017-11-25 17:46:45)

Support Unicode ∴ and ∵
Map ∴ and ∵ to `\therefore` and `\because`.

ronkok: Unicode work will now proceed much more slowly than in the bulk phase. The PRs will be small and each one will explain the choices made.

In this PR, it's obvious which aliases are the correct ones. [unicode-math](https://github.com/wspr/unicode-math/blob/master/unicode-math-table.tex) agrees that these two characters map to `\therefore` and `\because`.

I’ve withheld these two characters until now because there is a conflict as to the proper type of atom. KaTeX `symbols.js` says they are `rel` atoms. **unicode-math** says they are `\mathord`. I think KaTeX got this one right. They come from the [amssymb](http://ctan.math.utah.edu/ctan/tex-archive/fonts/amsfonts/doc/amssymb.pdf) package, which contains: `\DeclareMathSymbol{\therefore} {\mathrel}{AMSa}{"29}`.

So this PR merely sets the flag in `symbols.js`, without changing the atom type.
ronkok: Why not? I'll edit the comments.
edemaine: Great!  Does anyone else want to weigh in on the atom types before this gets merged?
kevinbarabash: Let's go with the AMS.

Patch:
diff --git a/src/symbols.js b/src/symbols.js
index a8313774..3ac0c135 100644
--- a/src/symbols.js
+++ b/src/symbols.js
@@ -355,10 +355,14 @@ defineSymbol(math, ams, rel, "\u226c", "\\between", true);
 defineSymbol(math, ams, rel, "\u22d4", "\\pitchfork", true);
 defineSymbol(math, ams, rel, "\u221d", "\\varpropto");
 defineSymbol(math, ams, rel, "\u25c0", "\\blacktriangleleft");
-defineSymbol(math, ams, rel, "\u2234", "\\therefore");
+// unicode-math says that \therefore is a mathord atom.
+// We kept the amssymb atom type, which is rel.
+defineSymbol(math, ams, rel, "\u2234", "\\therefore", true);
 defineSymbol(math, ams, rel, "\u220d", "\\backepsilon");
 defineSymbol(math, ams, rel, "\u25b6", "\\blacktriangleright");
-defineSymbol(math, ams, rel, "\u2235", "\\because");
+// unicode-math says that \because is a mathord atom.
+// We kept the amssymb atom type, which is rel.
+defineSymbol(math, ams, rel, "\u2235", "\\because", true);
 defineSymbol(math, ams, rel, "\u22d8", "\\llless");
 defineSymbol(math, ams, rel, "\u22d9", "\\gggtr");
 defineSymbol(math, ams, bin, "\u22b2", "\\lhd");


Test patch:
diff --git a/test/katex-spec.js b/test/katex-spec.js
index c8d89ce8..b49abe68 100644
--- a/test/katex-spec.js
+++ b/test/katex-spec.js
@@ -2694,7 +2694,7 @@ describe("Unicode", function() {
     });
 
     it("should parse relations", function() {
-        expect("∈∋∝∼∽≂≃≅≈≊≍≎≏≐≑≒≓≖≗≜≡≤≥≦≧≫≬≳≷≺≻≼≽≾≿").toParse();
+        expect("∈∋∝∼∽≂≃≅≈≊≍≎≏≐≑≒≓≖≗≜≡≤≥≦≧≫≬≳≷≺≻≼≽≾≿∴∵").toParse();
     });
 
     it("should parse big operators", function() {
Repository: Khan/KaTeX
Pull request #993 (2017-11-26 19:06:01)

Support Unicode \mid
Map ∣, U+2223, to `\mid`

ronkok: This character is U+2223, DIVIDES. I’ve withheld it until now only because it looks just like `|`, U+007C. So the question here is do we include any confusables at all. If we do, then the path forward is reasonably clear. [unicode-math](https://github.com/wspr/unicode-math/blob/master/unicode-math-table.tex) maps this character to `\mid`, the vertical line with `rel` spacing, useful for set builder notation. [John Cook](https://www.johndcook.com/unicode_latex.html) maps it the same way, as does Microsoft Word.

Patch:
diff --git a/src/symbols.js b/src/symbols.js
index 3ac0c135..05bdc02b 100644
--- a/src/symbols.js
+++ b/src/symbols.js
@@ -82,7 +82,7 @@ defineSymbol(math, main, rel, "\u22a5", "\\perp");
 defineSymbol(math, main, rel, "\u2aaf", "\\preceq", true);
 defineSymbol(math, main, rel, "\u2ab0", "\\succeq", true);
 defineSymbol(math, main, rel, "\u2243", "\\simeq", true);
-defineSymbol(math, main, rel, "\u2223", "\\mid");
+defineSymbol(math, main, rel, "\u2223", "\\mid", true);
 defineSymbol(math, main, rel, "\u226a", "\\ll");
 defineSymbol(math, main, rel, "\u226b", "\\gg", true);
 defineSymbol(math, main, rel, "\u224d", "\\asymp", true);


Test patch:
diff --git a/test/katex-spec.js b/test/katex-spec.js
index b49abe68..64ff2f21 100644
--- a/test/katex-spec.js
+++ b/test/katex-spec.js
@@ -2694,7 +2694,7 @@ describe("Unicode", function() {
     });
 
     it("should parse relations", function() {
-        expect("∈∋∝∼∽≂≃≅≈≊≍≎≏≐≑≒≓≖≗≜≡≤≥≦≧≫≬≳≷≺≻≼≽≾≿∴∵").toParse();
+        expect("∈∋∝∼∽≂≃≅≈≊≍≎≏≐≑≒≓≖≗≜≡≤≥≦≧≫≬≳≷≺≻≼≽≾≿∴∵∣").toParse();
     });
 
     it("should parse big operators", function() {
Repository: Khan/KaTeX
Pull request #1029 (2017-12-22 18:35:01)

add \i and \j for text mode
<img width="209" alt="screen shot 2017-12-22 at 11 31 00 am" src="https://user-images.githubusercontent.com/1044413/34308780-23caa864-e70c-11e7-8529-b974877176a2.png">


kevinbarabash: @rrandallcainc I added a test to verify that `\\text{\\i\\j}` parses.  A screenshot test is probably overkill for this as there a lots of symbols we don't have screenshot tests for.

Patch:
diff --git a/src/symbols.js b/src/symbols.js
index f626adcd..5fe2787d 100644
--- a/src/symbols.js
+++ b/src/symbols.js
@@ -646,6 +646,8 @@ defineSymbol(math, main, accent, "\u20d7", "\\vec");
 defineSymbol(math, main, accent, "\u02d9", "\\dot");
 defineSymbol(math, main, mathord, "\u0131", "\\imath");
 defineSymbol(math, main, mathord, "\u0237", "\\jmath");
+defineSymbol(text, main, textord, "\u0131", "\\i");
+defineSymbol(text, main, textord, "\u0237", "\\j");
 defineSymbol(text, main, accent, "\u02ca", "\\'"); // acute
 defineSymbol(text, main, accent, "\u02cb", "\\`"); // grave
 defineSymbol(text, main, accent, "\u02c6", "\\^"); // circumflex


Test patch:
diff --git a/test/katex-spec.js b/test/katex-spec.js
index fd0ff4cd..448ae480 100644
--- a/test/katex-spec.js
+++ b/test/katex-spec.js
@@ -2838,3 +2838,9 @@ describe("The \\mathchoice function", function() {
         expect(built).toEqual(plain);
     });
 });
+
+describe("Symbols", function() {
+    it("should parse \\text{\\i\\j}", () => {
+        expect("\\text{\\i\\j}").toParse();
+    });
+});
Repository: Khan/KaTeX
Pull request #1069 (2018-01-16 15:10:17)

Add support for \aa, \AA, \lq, \rq, \lbrack, \rbrack
The lack of `\aa` and `\AA` was reported in #1066.  Added these via simple macros, with supporting tests.

Also add `\lq`, `\rq`, `\lbrack`, `\rbrack` as aliases for "\`", `'`, `[`, `]` which were in the same area of latex.ltx.


Patch:
diff --git a/src/macros.js b/src/macros.js
index b59a88df..152c90ac 100644
--- a/src/macros.js
+++ b/src/macros.js
@@ -96,12 +96,27 @@ defineMacro("\\TextOrMath", function(context) {
 });
 
 //////////////////////////////////////////////////////////////////////
-// basics
+// Grouping
+// \let\bgroup={ \let\egroup=}
 defineMacro("\\bgroup", "{");
 defineMacro("\\egroup", "}");
 defineMacro("\\begingroup", "{");
 defineMacro("\\endgroup", "}");
 
+// Symbols from latex.ltx:
+// \def\lq{`}
+// \def\rq{'}
+// \def\lbrack{[}
+// \def\rbrack{]}
+// \def \aa {\r a}
+// \def \AA {\r A}
+defineMacro("\\lq", "`");
+defineMacro("\\rq", "'");
+defineMacro("\\lbrack", "[");
+defineMacro("\\rbrack", "]");
+defineMacro("\\aa", "\\r a");
+defineMacro("\\AA", "\\r A");
+
 // Unicode double-struck letters
 defineMacro("\u2102", "\\mathbb{C}");
 defineMacro("\u210D", "\\mathbb{H}");


Test patch:
diff --git a/test/katex-spec.js b/test/katex-spec.js
index f9ed9b5d..0f86b6d7 100644
--- a/test/katex-spec.js
+++ b/test/katex-spec.js
@@ -2798,6 +2798,12 @@ describe("Unicode accents", function() {
             "\\'y\\\"y}");
     });
 
+    it("should support \\aa in text mode", function() {
+        expect("\\text{\\aa\\AA}").toParseLike("\\text{\\r a\\r A}");
+        expect("\\aa").toNotParse();
+        expect("\\Aa").toNotParse();
+    });
+
     it("should parse combining characters", function() {
         expect("A\u0301C\u0301").toParseLike("Á\\acute C");
         expect("\\text{A\u0301C\u0301}").toParseLike("\\text{Á\\'C}");
Repository: frictionlessdata/tabulator-py
Pull request #119 (2016-10-27 18:26:34)

Add support for not keyed NDJSON sources
It fixes https://github.com/frictionlessdata/tabulator-py/issues/115


Patch:
diff --git a/tabulator/parsers/ndjson.py b/tabulator/parsers/ndjson.py
index 7080dcd..d7731b2 100644
--- a/tabulator/parsers/ndjson.py
+++ b/tabulator/parsers/ndjson.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 
 import jsonlines
 
+from .. import exceptions
 from .. import helpers
 from . import api
 
@@ -56,5 +57,12 @@ class NDJSONParser(api.Parser):
     def __iter_extended_rows(self):
         rows = jsonlines.Reader(self.__chars)
         for number, row in enumerate(rows, start=1):
-            keys, values = zip(*sorted(row.items()))
-            yield number, list(keys), list(values)
+            if isinstance(row, (tuple, list)):
+                yield number, None, list(row)
+            elif isinstance(row, dict):
+                keys, values = zip(*sorted(row.items()))
+                yield number, list(keys), list(values)
+            else:
+                raise exceptions.SourceError(
+                    "JSON item has to be list or dict"
+                )


Test patch:
diff --git a/tests/parsers/test_ndjson.py b/tests/parsers/test_ndjson.py
index 3ae272c..f82d37d 100644
--- a/tests/parsers/test_ndjson.py
+++ b/tests/parsers/test_ndjson.py
@@ -5,8 +5,11 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import io
+import pytest
+from six import StringIO
 from mock import Mock
-from tabulator import Stream
+
+from tabulator import exceptions, Stream
 from tabulator.parsers.ndjson import NDJSONParser
 
 
@@ -43,3 +46,33 @@ def test_stream_ndjson():
         assert stream.read(keyed=True) == [
             {'id': 1, 'name': 'english'},
             {'id': 2, 'name': '中国人'}]
+
+
+def test_ndjson_list():
+    stream = StringIO(
+        '[1, 2, 3]\n'
+        '[4, 5, 6]\n'
+    )
+
+    parser = NDJSONParser()
+    loader = Mock(load=Mock(return_value=stream))
+    parser.open(None, None, loader)
+
+    assert list(parser.extended_rows) == [
+        (1, None, [1, 2, 3]),
+        (2, None, [4, 5, 6]),
+    ]
+
+
+def test_ndjson_scalar():
+    stream = StringIO(
+        '1\n'
+        '2\n'
+    )
+
+    parser = NDJSONParser()
+    loader = Mock(load=Mock(return_value=stream))
+    parser.open(None, None, loader)
+
+    with pytest.raises(exceptions.SourceError):
+        list(parser.extended_rows)
Repository: frictionlessdata/tabulator-py
Pull request #120 (2016-10-27 19:04:04)

Improve html detection algorithm
Replace html detection using BeautifulSoup with a simple regex.

New implementatio tries to detect only most like HTML candidates.

It fixes https://github.com/frictionlessdata/tabulator-py/issues/101


Patch:
diff --git a/setup.py b/setup.py
index fdd8af3..34ccf0f 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,6 @@ INSTALL_REQUIRES = [
     'chardet>=2.0,<3.0a',
     'openpyxl>=2.4,<3.0a',
     'requests>=2.8,<3.0a',
-    'beautifulsoup4>=4.4,<5.0a',
     'linear-tsv>=1.0,<2.0a',
     'unicodecsv>=0.14,<1.0a',
     'jsonlines>=1.1,<1.2',
diff --git a/tabulator/helpers.py b/tabulator/helpers.py
index 7bcda39..4f9f5f6 100644
--- a/tabulator/helpers.py
+++ b/tabulator/helpers.py
@@ -89,12 +89,13 @@ def detect_encoding(bytes, encoding=None):
     return encoding
 
 
+detect_html_re = re.compile('\s*<(!doctype|html)', re.IGNORECASE)
+
+
 def detect_html(text):
     """Detect if text is HTML.
     """
-    # To reduce tabulator import time
-    from bs4 import BeautifulSoup
-    return bool(BeautifulSoup(text, 'html.parser').find())
+    return bool(detect_html_re.match(text))
 
 
 def reset_stream(stream):


Test patch:
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 6f22df6..bca3de2 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -96,3 +96,14 @@ def test_extract_options():
     extracted_options = helpers.extract_options(options, names)
     assert options == {'opt3': 3}
     assert extracted_options == {'opt1': 1, 'opt2': 2}
+
+
+@pytest.mark.parametrize('sample', [
+    ('\n\n\t <html>', True),
+    ('<!DOCTYPE html>', True),
+    ('col1,col2\nval1,<html>', False),
+    ('val1,<html>', False),
+])
+def test_detect_html(sample):
+    text, is_html = sample
+    assert helpers.detect_html(text) is is_html
Repository: frictionlessdata/tabulator-py
Pull request #150 (2017-02-21 06:19:34)

Added gsheet gid test, fixed export url


Patch:
diff --git a/tabulator/parsers/gsheet.py b/tabulator/parsers/gsheet.py
index 1c67c0e..b1bbba8 100644
--- a/tabulator/parsers/gsheet.py
+++ b/tabulator/parsers/gsheet.py
@@ -28,13 +28,15 @@ class GsheetParser(api.Parser):
 
     def open(self, source, encoding, loader):
         self.close()
-        url = 'https://docs.google.com/spreadsheets/d/%s/export?format=csv&id=%s&gid=%s'
+        url = 'https://docs.google.com/spreadsheets/d/%s/export?format=csv&id=%s'
         match = re.search(r'.*/d/(?P<key>[^/]+)/.*?(?:gid=(?P<gid>\d+))?$', source)
         key, gid = '', ''
         if match:
             key = match.group('key')
-            gid = match.group('gid') or '0'
-        url = url % (key, key, gid)
+            gid = match.group('gid')
+        url = url % (key, key)
+        if gid:
+            url = '%s&gid=%s' % (url, gid)
         self.__stream = Stream(url, format='csv', encoding=encoding).open()
         self.__extended_rows = self.__stream.iter(extended=True)
 


Test patch:
diff --git a/tests/test_stream.py b/tests/test_stream.py
index 3e8f3a5..001fa9e 100644
--- a/tests/test_stream.py
+++ b/tests/test_stream.py
@@ -63,15 +63,15 @@ def test_stream_xlsx_remote():
 # Tests [format:gsheet]
 
 def test_stream_gsheet():
-    source = 'https://docs.google.com/spreadsheets/d/1HXCBuWlekSd359yvHimvMJiD2CC_2og75WlL4EFAcmg/edit?usp=sharing'
+    source = 'https://docs.google.com/spreadsheets/d/1mHIWnDvW9cALRMq9OdNfRwjAthCUFUOACPp0Lkyl7b4/edit?usp=sharing'
     with Stream(source) as stream:
         assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]
 
 
 def test_stream_gsheet_with_gid():
-    source = 'https://docs.google.com/spreadsheets/d/1HXCBuWlekSd359yvHimvMJiD2CC_2og75WlL4EFAcmg/edit#gid=1137640675'
+    source = 'https://docs.google.com/spreadsheets/d/1mHIWnDvW9cALRMq9OdNfRwjAthCUFUOACPp0Lkyl7b4/edit#gid=960698813'
     with Stream(source) as stream:
-        assert stream.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]
+        assert stream.read() == [['id', 'name'], ['2', '中国人'], ['3', 'german']]
 
 
 def test_stream_gsheet_bad_url():
Repository: frictionlessdata/tabulator-py
Pull request #185 (2017-08-02 10:18:53)

Autodetect common csv delimiters
Add common delimiters for tabular data to the csv dialect Sniffer
arguments. This means more inputs can be parsed without needing to
pass an explicit delimiter to Stream.

Also fixes an existing duplicated test name.


Patch:
diff --git a/tabulator/parsers/csv.py b/tabulator/parsers/csv.py
index 935e81c..a279f6e 100644
--- a/tabulator/parsers/csv.py
+++ b/tabulator/parsers/csv.py
@@ -107,7 +107,7 @@ class CSVParser(Parser):
         # Get dialect
         try:
             separator = b'' if six.PY2 else ''
-            delimiter = self.__options.get('delimiter', ',')
+            delimiter = self.__options.get('delimiter', ',\t;|')
             dialect = csv.Sniffer().sniff(separator.join(sample), delimiter)
             if not dialect.escapechar:
                 dialect.doublequote = True


Test patch:
diff --git a/tests/formats/test_csv.py b/tests/formats/test_csv.py
index 681c5d5..7f9ce58 100644
--- a/tests/formats/test_csv.py
+++ b/tests/formats/test_csv.py
@@ -113,12 +113,30 @@ def test_stream_csv_quotechar():
         assert stream.read() == [['value1,value2'], ['value3,value4']]
 
 
-def test_stream_csv_quotechar():
+def test_stream_csv_skipinitialspace():
     source = 'value1, value2\nvalue3, value4'
     with Stream(source, scheme='text', format='csv', skipinitialspace=True) as stream:
         assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]
 
 
+def test_stream_csv_detect_delimiter_tab():
+    source = 'a1\tb1\tc1A,c1B\na2\tb2\tc2\n'
+    with Stream(source, scheme='text', format='csv') as stream:
+        assert stream.read() == [['a1', 'b1', 'c1A,c1B'], ['a2', 'b2', 'c2']]
+
+
+def test_stream_csv_detect_delimiter_semicolon():
+    source = 'a1;b1\na2;b2\n'
+    with Stream(source, scheme='text', format='csv') as stream:
+        assert stream.read() == [['a1', 'b1'], ['a2', 'b2']]
+
+
+def test_stream_csv_detect_delimiter_pipe():
+    source = 'a1|b1\na2|b2\n'
+    with Stream(source, scheme='text', format='csv') as stream:
+        assert stream.read() == [['a1', 'b1'], ['a2', 'b2']]
+
+
 # Parser
 
 def test_parser_csv():