Add equal-sign to parse safe characters

scrapy · Mar 24, 2024 · b716391 · b716391
1 parent 94ebfa3
commit b716391
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 1 deletion.
diff --git a/src/protego.py b/src/protego.py
@@ -184,7 +184,7 @@ def _quote_pattern(self, pattern):
 
         parts = urlparse(pattern)
         pattern = self._unquote(parts.path, ignore="/*$%")
-        pattern = quote(pattern, safe="/*%")
+        pattern = quote(pattern, safe="/*%=")
 
         parts = ParseResult(
             "", "", pattern + last_char, parts.params, parts.query, parts.fragment

diff --git a/tests/test_protego.py b/tests/test_protego.py
@@ -1141,6 +1141,16 @@ def test_parse_time_period(self):
         self.assertEqual(start_time, time(5, 0))
         self.assertEqual(end_time, time(6, 0))
 
+    def test_disallow_query_wildcard(self):
+        content = (
+            "User-agent: * \n"
+            "Disallow: /*s="
+        )
+        rp = Protego.parse(content=content)
+        self.assertTrue(rp.can_fetch("https://www.site.local/", "*"))
+        self.assertTrue(rp.can_fetch("https://www.site.local/s/", "*"))
+        self.assertFalse(rp.can_fetch("https://www.site.local/?s=asd", "*"))
+
 
 @pytest.mark.parametrize(
     "allow,disallow,url,allowed",