Make safe_url_string safer

.bandit.yml

@@ -1,2 +1,3 @@
 skips:
+- B101  # Use of assert; we use them for mypy
 - B107

.coveragerc

@@ -1,3 +1,2 @@
 [run]
 branch = true
-include = w3lib/*

MANIFEST.in

@@ -10,3 +10,4 @@ include NEWS
 include README.rst
 include pytest.ini
 include tox.ini
+include w3lib/idna.txt

setup.py

@@ -17,6 +17,9 @@
     zip_safe=False,
     platforms=["Any"],
     python_requires=">=3.7",
+    install_requires=[
+        "idna",
+    ],
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "License :: OSI Approved :: BSD License",

tests/percent-encoding-test-data.json

@@ -0,0 +1,50 @@
+[
+    "Source: https://github.com/web-platform-tests/wpt/blob/77d95d52351ebf51bc80ee21eea455de31a5e356/url/resources/percent-encoding.json",
+
+    "Tests for percent-encoding.",
+    {
+      "input": "\u2020",
+      "output": {
+        "big5": "%26%238224%3B",
+        "euc-kr": "%A2%D3",
+        "utf-8": "%E2%80%A0",
+        "windows-1252": "%86"
+      }
+    },
+    "This uses a trailing A to prevent the URL parser from trimming the C0 control.",
+    {
+      "input": "\u000EA",
+      "output": {
+        "big5": "%0EA",
+        "iso-2022-jp": "%26%2365533%3BA",
+        "utf-8": "%0EA"
+      }
+    },
+    {
+      "input": "\u203E\u005C",
+      "output": {
+        "iso-2022-jp": "%1B(J~%1B(B\\",
+        "utf-8": "%E2%80%BE\\"
+      }
+    },
+    {
+      "input": "\uE5E5",
+      "output": {
+        "gb18030": "%26%2358853%3B",
+        "utf-8": "%EE%97%A5"
+      }
+    },
+    {
+      "input": "\u2212",
+      "output": {
+        "shift_jis": "%81|",
+        "utf-8": "%E2%88%92"
+      }
+    },
+    {
+      "input": "á|",
+      "output": {
+        "utf-8": "%C3%A1|"
+      }
+    }
+  ]

tests/test_encoding.py

@@ -1,6 +1,9 @@
 import codecs
 import unittest
 
+import pytest
+
+from w3lib._encoding import _get_encoding, _LABEL_ENCODINGS
 from w3lib.encoding import (
     html_body_declared_encoding,
     http_content_type_encoding,
@@ -10,6 +13,26 @@
     to_unicode,
 )
 
+# Encodings from the spec that Python does not support.
+_UNSUPPORTED_ENCODINGS = {
+    "iso-8859-8-i",  # https://bugs.python.org/msg213772
+    "replacement",  # Not an actual encoding
+    # Not supported.
+    # We could bring support to it with the webencodings package.
+    "x-user-defined",
+}
+
+
+@pytest.mark.parametrize(
+    "label,name", tuple((label, name) for label, name in _LABEL_ENCODINGS.items())
+)
+def test_get_encoding_python(label, name):
+    """The encodings that _get_encoding can return must work as encoding
+    aliases in Python."""
+    assert _get_encoding(label) == name
+    if name not in _UNSUPPORTED_ENCODINGS:
+        codecs.lookup(name)  # Raises LookupError if not found.
+
 
 class RequestEncodingTests(unittest.TestCase):
     utf8_fragments = [