Merge branch 'pr-crashfix_compat_urllib_unquote' of https://github.com/atomicdryad/youtube-dl into atomicdryad-pr-crashfix_compat_urllib_unquote
Sergey M․
commit
89f691e141
|
|
@ -14,6 +14,7 @@ from youtube_dl.utils import get_filesystem_encoding
|
|||
from youtube_dl.compat import (
|
||||
compat_getenv,
|
||||
compat_expanduser,
|
||||
compat_urllib_parse_unquote,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -42,5 +43,24 @@ class TestCompat(unittest.TestCase):
|
|||
dir(youtube_dl.compat))) - set(['unicode_literals'])
|
||||
self.assertEqual(all_names, sorted(present_names))
|
||||
|
||||
def test_compat_urllib_parse_unquote(self):
|
||||
test_strings = [
|
||||
['''''', ''''''],
|
||||
['''津波''', '''%E6%B4%A5%E6%B3%A2'''],
|
||||
['''津波''', str('%E6%B4%A5%E6%B3%A2')],
|
||||
['''<meta property="og:description" content="▁▂▃▄%▅▆▇█" />
|
||||
%<a href="https://ar.wikipedia.org/wiki/تسونامي">%a''',
|
||||
'''<meta property="og:description" content="%E2%96%81%E2%96%82%E2%96%83%E2%96%84%25%E2%96%85%E2%96%86%E2%96%87%E2%96%88" />
|
||||
%<a href="https://ar.wikipedia.org/wiki/%D8%AA%D8%B3%D9%88%D9%86%D8%A7%D9%85%D9%8A">%a'''],
|
||||
['''(^◣_◢^)っ︻デ═一 ⇀ ⇀ ⇀ ⇀ ⇀ ↶%I%Break%Things%''',
|
||||
'''%28%5E%E2%97%A3_%E2%97%A2%5E%29%E3%81%A3%EF%B8%BB%E3%83%87%E2%95%90%E4%B8%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%87%80 %E2%86%B6%I%Break%25Things%''']
|
||||
]
|
||||
for test in test_strings:
|
||||
strutf = test[0]
|
||||
strurlenc = test[1]
|
||||
strurldec = compat_urllib_parse_unquote(strurlenc)
|
||||
self.assertEqual(strutf, strurldec)
|
||||
self.assertEqual(strutf, compat_urllib_parse_unquote(strurlenc))
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -77,40 +77,73 @@ except ImportError:
|
|||
try:
|
||||
from urllib.parse import unquote as compat_urllib_parse_unquote
|
||||
except ImportError:
|
||||
def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
|
||||
if string == '':
|
||||
def compat_urllib_parse_unquote_to_bytes(string):
|
||||
"""unquote_to_bytes('abc%20def') -> b'abc def'."""
|
||||
# Note: strings are encoded as UTF-8. This is only an issue if it contains
|
||||
# unescaped non-ASCII characters, which URIs should not.
|
||||
if not string:
|
||||
# Is it a string-like object?
|
||||
string.split
|
||||
return b''
|
||||
if isinstance(string, str):
|
||||
string = string.encode('utf-8')
|
||||
# string = encode('utf-8')
|
||||
|
||||
# python3 -> 2: must implicitly convert to bits
|
||||
bits = bytes(string).split(b'%')
|
||||
|
||||
if len(bits) == 1:
|
||||
return string
|
||||
res = string.split('%')
|
||||
if len(res) == 1:
|
||||
res = [bits[0]]
|
||||
append = res.append
|
||||
|
||||
for item in bits[1:]:
|
||||
if item == '':
|
||||
append(b'%')
|
||||
continue
|
||||
try:
|
||||
append(item[:2].decode('hex'))
|
||||
append(item[2:])
|
||||
except:
|
||||
append(b'%')
|
||||
append(item)
|
||||
return b''.join(res)
|
||||
|
||||
compat_urllib_parse_asciire = re.compile('([\x00-\x7f]+)')
|
||||
|
||||
def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
|
||||
"""Replace %xx escapes by their single-character equivalent. The optional
|
||||
encoding and errors parameters specify how to decode percent-encoded
|
||||
sequences into Unicode characters, as accepted by the bytes.decode()
|
||||
method.
|
||||
By default, percent-encoded sequences are decoded with UTF-8, and invalid
|
||||
sequences are replaced by a placeholder character.
|
||||
|
||||
unquote('abc%20def') -> 'abc def'.
|
||||
"""
|
||||
|
||||
if '%' not in string:
|
||||
string.split
|
||||
return string
|
||||
if encoding is None:
|
||||
encoding = 'utf-8'
|
||||
if errors is None:
|
||||
errors = 'replace'
|
||||
# pct_sequence: contiguous sequence of percent-encoded bytes, decoded
|
||||
pct_sequence = b''
|
||||
string = res[0]
|
||||
for item in res[1:]:
|
||||
try:
|
||||
if not item:
|
||||
raise ValueError
|
||||
pct_sequence += item[:2].decode('hex')
|
||||
rest = item[2:]
|
||||
if not rest:
|
||||
# This segment was just a single percent-encoded character.
|
||||
# May be part of a sequence of code units, so delay decoding.
|
||||
# (Stored in pct_sequence).
|
||||
continue
|
||||
except ValueError:
|
||||
rest = '%' + item
|
||||
# Encountered non-percent-encoded characters. Flush the current
|
||||
# pct_sequence.
|
||||
string += pct_sequence.decode(encoding, errors) + rest
|
||||
pct_sequence = b''
|
||||
if pct_sequence:
|
||||
# Flush the final pct_sequence
|
||||
string += pct_sequence.decode(encoding, errors)
|
||||
return string
|
||||
|
||||
bits = compat_urllib_parse_asciire.split(string)
|
||||
res = [bits[0]]
|
||||
append = res.append
|
||||
for i in range(1, len(bits), 2):
|
||||
foo = compat_urllib_parse_unquote_to_bytes(bits[i])
|
||||
foo = foo.decode(encoding, errors)
|
||||
append(foo)
|
||||
|
||||
if bits[i + 1]:
|
||||
bar = bits[i + 1]
|
||||
if not isinstance(bar, unicode):
|
||||
bar = bar.decode('utf-8')
|
||||
append(bar)
|
||||
return ''.join(res)
|
||||
|
||||
try:
|
||||
compat_str = unicode # Python 2
|
||||
|
|
@ -421,7 +454,9 @@ __all__ = [
|
|||
'compat_subprocess_get_DEVNULL',
|
||||
'compat_urllib_error',
|
||||
'compat_urllib_parse',
|
||||
'compat_urllib_parse_asciire',
|
||||
'compat_urllib_parse_unquote',
|
||||
'compat_urllib_parse_unquote_to_bytes',
|
||||
'compat_urllib_parse_urlparse',
|
||||
'compat_urllib_request',
|
||||
'compat_urlparse',
|
||||
|
|
|
|||
|
|
@ -1115,7 +1115,7 @@ class GenericIE(InfoExtractor):
|
|||
# Sometimes embedded video player is hidden behind percent encoding
|
||||
# (e.g. https://github.com/rg3/youtube-dl/issues/2448)
|
||||
# Unescaping the whole page allows to handle those cases in a generic way
|
||||
webpage = compat_urllib_parse.unquote(webpage)
|
||||
webpage = compat_urllib_parse_unquote(webpage)
|
||||
|
||||
# it's tempting to parse this further, but you would
|
||||
# have to take into account all the variations like
|
||||
|
|
|
|||
Loading…
Reference in New Issue