Repository: scrapy/protego

Issue:
Accept robots.txt as bytes
```python
>>> robots_txt = b"User-Agent: *\nDisallow: /\n"
>>> robots_txt_parser = Protego.parse(robots_txt)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/adrian/temporal/venv/lib/python3.9/site-packages/protego.py", line 310, in parse
    o._parse_robotstxt(content)
  File "/home/adrian/temporal/venv/lib/python3.9/site-packages/protego.py", line 327, in _parse_robotstxt
    hash_pos = line.find('#')
TypeError: argument should be integer or bytes-like object, not 'str'
>>> robots_txt = "User-Agent: *\nDisallow: /\n"
>>> robots_txt_parser = Protego.parse(robots_txt)
>>>
```

Discussion:

Patch:
diff --git a/src/protego.py b/src/protego.py
index f859e0c..205e5d5 100644
--- a/src/protego.py
+++ b/src/protego.py
@@ -299,6 +299,8 @@ class Protego(object):
     @classmethod
     def parse(cls, content):
         o = cls()
+        if not isinstance(content, str):
+            raise ValueError(f"Protego.parse expects str, got {type(content).__name__}")
         o._parse_robotstxt(content)
         return o
 


Test patch:
diff --git a/tests/test_protego.py b/tests/test_protego.py
index 97ffdfe..6389b02 100644
--- a/tests/test_protego.py
+++ b/tests/test_protego.py
@@ -1059,3 +1059,10 @@ class TestProtego(TestCase):
         rp = Protego.parse(content)
         self.assertFalse(rp.can_fetch("http://example.com/", "FooBot"))
         self.assertFalse(rp.can_fetch("http://example.com", "FooBot"))
+
+    def test_bytestrings(self):
+        content = b"User-Agent: FootBot\nDisallow: /something"
+        with self.assertRaises(ValueError) as context:
+            Protego.parse(content=content)
+
+        self.assertEqual("Protego.parse expects str, got bytes", str(context.exception))