Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
C
cython
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
0
Merge Requests
0
Analytics
Analytics
Repository
Value Stream
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Commits
Open sidebar
nexedi
cython
Commits
297d7c2b
Commit
297d7c2b
authored
Apr 23, 2019
by
Stefan Behnel
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Fix a compiler crash when non-ASCII characters appear in unprefixed strings in "3str" parsing mode.
parent
3b3bcbfc
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
106 additions
and
2 deletions
+106
-2
Cython/Compiler/Parsing.py
Cython/Compiler/Parsing.py
+2
-1
tests/run/cython3.pyx
tests/run/cython3.pyx
+41
-1
tests/run/cython3_no_unicode_literals.pyx
tests/run/cython3_no_unicode_literals.pyx
+63
-0
No files found.
Cython/Compiler/Parsing.py
View file @
297d7c2b
...
...
@@ -958,7 +958,8 @@ def p_string_literal(s, kind_override=None):
error
(
pos
,
u"invalid character literal: %r"
%
bytes_value
)
else
:
bytes_value
,
unicode_value
=
chars
.
getstrings
()
if
is_python3_source
and
has_non_ascii_literal_characters
:
if
(
has_non_ascii_literal_characters
and
is_python3_source
and
Future
.
unicode_literals
in
s
.
context
.
future_directives
):
# Python 3 forbids literal non-ASCII characters in byte strings
if
kind
==
'b'
:
s
.
error
(
"bytes can only contain ASCII literal characters."
,
pos
=
pos
)
...
...
tests/run/cython3.pyx
View file @
297d7c2b
...
...
@@ -21,7 +21,8 @@ True
"""
import
sys
if
sys
.
version_info
[
0
]
>=
3
:
IS_PY2
=
sys
.
version_info
[
0
]
<
3
if
not
IS_PY2
:
__doc__
=
__doc__
.
replace
(
" u'"
,
" '"
)
def
locals_function
(
a
,
b
=
2
):
...
...
@@ -312,6 +313,45 @@ def unicode_literals():
return
ustring
def
non_ascii_unprefixed_str
():
u"""
>>> s = non_ascii_unprefixed_str()
>>> isinstance(s, bytes)
False
>>> len(s)
3
"""
s
=
'ø
\
x20
\
u0020
'
assert
isinstance
(
s
,
unicode
)
return
s
def
non_ascii_raw_str
():
u"""
>>> s = non_ascii_raw_str()
>>> isinstance(s, bytes)
False
>>> len(s)
11
"""
s
=
r'ø\x20\u0020'
assert
isinstance
(
s
,
unicode
)
return
s
def
non_ascii_raw_prefixed_unicode
():
u"""
>>> s = non_ascii_raw_prefixed_unicode()
>>> isinstance(s, bytes)
False
>>> len(s)
11
"""
s
=
ru'ø\x20\u0020'
assert
isinstance
(
s
,
unicode
)
return
s
def
str_type_is_unicode
():
"""
>>> str_type, s = str_type_is_unicode()
...
...
tests/run/cython3_no_unicode_literals.pyx
View file @
297d7c2b
...
...
@@ -13,6 +13,10 @@ b = 2
x = 'abc'
"""
import
sys
IS_PY2
=
sys
.
version_info
[
0
]
<
3
def
locals_function
(
a
,
b
=
2
):
x
=
'abc'
return
locals
()
...
...
@@ -64,6 +68,65 @@ def no_unicode_literals():
return
str_string
def
non_ascii_str
():
u"""
>>> s = 'ø
\
\
x20
\
\
u0020'
>>> isinstance(s, str)
True
>>> print(not IS_PY2 or len(s) == 9 or len(s)) # first is 2-char bytes in Py2, hex escape is resolved
True
>>> print(IS_PY2 or len(s) == 3 or len(s)) # 3 unicode characters in Py3
True
>>> s = non_ascii_str()
>>> isinstance(s, str)
True
>>> print(not IS_PY2 or len(s) == 9 or len(s)) # first is 2-char bytes in Py2, hex escape is resolved
True
>>> print(IS_PY2 or len(s) == 3 or len(s)) # 3 unicode characters in Py3
True
"""
s
=
'ø
\
x20
\
u0020
'
assert
isinstance
(
s
,
str
)
assert
(
IS_PY2
and
isinstance
(
s
,
bytes
))
or
(
not
IS_PY2
and
isinstance
(
s
,
unicode
))
return
s
def
non_ascii_raw_str
():
u"""
>>> s = r'ø
\
\
x20
\
\
u0020'
>>> print(not IS_PY2 or len(s) == 12 or len(s)) # Py2 (first character is two bytes)
True
>>> print(IS_PY2 or len(s) == 11 or len(s)) # Py3 (unicode string)
True
>>> s = non_ascii_raw_str()
>>> isinstance(s, str)
True
>>> print(not IS_PY2 or len(s) == 12 or len(s)) # Py2 (first character is two bytes)
True
>>> print(IS_PY2 or len(s) == 11 or len(s)) # Py3 (unicode string)
True
"""
s
=
r'ø\x20\u0020'
assert
isinstance
(
s
,
str
)
assert
(
IS_PY2
and
isinstance
(
s
,
bytes
))
or
(
not
IS_PY2
and
isinstance
(
s
,
unicode
))
return
s
def
non_ascii_raw_unicode
():
u"""
>>> s = non_ascii_raw_unicode()
>>> isinstance(s, bytes)
False
>>> len(s)
11
"""
s
=
ru'ø\x20\u0020'
assert
isinstance
(
s
,
unicode
)
return
s
def
str_type_is_str
():
"""
>>> str_type, s = str_type_is_str()
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment