Skip to content

Commit 41a863c

Browse files
author
Victor Stinner
committed
Issue #13706: Fix format(int, "n") for locale with non-ASCII thousands separator
 * Decode thousands separator and decimal point using PyUnicode_DecodeLocale() (from the locale encoding), instead of decoding them implicitly from latin1 * Remove _PyUnicode_InsertThousandsGroupingLocale(), it was not used * Change _PyUnicode_InsertThousandsGrouping() API to return the maximum character if unicode is NULL * Replace MIN/MAX macros by Py_MIN/Py_MAX * stringlib/undef.h undefines STRINGLIB_IS_UNICODE * stringlib/localeutil.h only supports Unicode
1 parent dcb30cf commit 41a863c

File tree

12 files changed

+188
-151
lines changed

12 files changed

+188
-151
lines changed

‎Include/unicodeobject.h‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1936,32 +1936,20 @@ PyAPI_FUNC(PyObject *) _PyUnicode_XStrip(
19361936
);
19371937
#endif
19381938

1939-
/* Using the current locale, insert the thousands grouping
1940-
into the string pointed to by buffer. For the argument descriptions,
1941-
see Objects/stringlib/localeutil.h */
1942-
1943-
#ifndef Py_LIMITED_API
1944-
PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGroupingLocale(Py_UNICODE *buffer,
1945-
Py_ssize_t n_buffer,
1946-
Py_UNICODE *digits,
1947-
Py_ssize_t n_digits,
1948-
Py_ssize_t min_width);
1949-
#endif
1950-
19511939
/* Using explicit passed-in values, insert the thousands grouping
19521940
into the string pointed to by buffer. For the argument descriptions,
19531941
see Objects/stringlib/localeutil.h */
19541942
#ifndef Py_LIMITED_API
19551943
PyAPI_FUNC(Py_ssize_t) _PyUnicode_InsertThousandsGrouping(
19561944
PyObject *unicode,
1957-
int kind,
1958-
void *buffer,
1945+
Py_ssize_t index,
19591946
Py_ssize_t n_buffer,
19601947
void *digits,
19611948
Py_ssize_t n_digits,
19621949
Py_ssize_t min_width,
19631950
const char *grouping,
1964-
const char *thousands_sep);
1951+
PyObject *thousands_sep,
1952+
Py_UCS4 *maxchar);
19651953
#endif
19661954
/* === Characters Type APIs =============================================== */
19671955

‎Lib/test/test_format.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from test.support import verbose, TestFailed
2+
import locale
23
import sys
34
import test.support as support
45
import unittest
@@ -282,6 +283,20 @@ def test_non_ascii(self):
282283
self.assertEqual(format(1+2j, "\u2007^8"), "\u2007(1+2j)\u2007")
283284
self.assertEqual(format(0j, "\u2007^4"), "\u20070j\u2007")
284285

286+
def test_locale(self):
287+
try:
288+
oldloc = locale.setlocale(locale.LC_ALL, '')
289+
except locale.Error as err:
290+
self.skipTest("Cannot set locale: {}".format(err))
291+
try:
292+
sep = locale.localeconv()['thousands_sep']
293+
text = format(123456789, "n")
294+
self.assertIn(sep, text)
295+
self.assertEqual(text.replace(sep, ''), '123456789')
296+
finally:
297+
locale.setlocale(locale.LC_ALL, oldloc)
298+
299+
285300

286301
def test_main():
287302
support.run_unittest(FormatTest)

‎Objects/stringlib/asciilib.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,9 @@
2121
#define STRINGLIB_RESIZE not_supported
2222
#define STRINGLIB_CHECK PyUnicode_Check
2323
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
24-
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
25-
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
2624

2725
#define STRINGLIB_TOSTR PyObject_Str
2826
#define STRINGLIB_TOASCII PyObject_ASCII
2927

3028
#define _Py_InsertThousandsGrouping _PyUnicode_ascii_InsertThousandsGrouping
31-
#define _Py_InsertThousandsGroupingLocale _PyUnicode_ascii_InsertThousandsGroupingLocale
3229

‎Objects/stringlib/localeutil.h‎

Lines changed: 22 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22

33
#include <locale.h>
44

5-
#define MAX(x, y) ((x) < (y) ? (y) : (x))
6-
#define MIN(x, y) ((x) < (y) ? (x) : (y))
5+
#ifndef STRINGLIB_IS_UNICODE
6+
# error "localeutil is specific to Unicode"
7+
#endif
78

89
typedef struct {
910
const char *grouping;
@@ -46,7 +47,7 @@ STRINGLIB(GroupGenerator_next)(STRINGLIB(GroupGenerator) *self)
4647
are optional, depending on when we're called. */
4748
static void
4849
STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
49-
Py_ssize_t n_chars, Py_ssize_t n_zeros, const char* thousands_sep,
50+
Py_ssize_t n_chars, Py_ssize_t n_zeros, STRINGLIB_CHAR* thousands_sep,
5051
Py_ssize_t thousands_sep_len)
5152
{
5253
Py_ssize_t i;
@@ -55,15 +56,8 @@ STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
5556
*buffer_end -= thousands_sep_len;
5657

5758
/* Copy the thousands_sep chars into the buffer. */
58-
#if STRINGLIB_IS_UNICODE
59-
/* Convert from the char's of the thousands_sep from
60-
the locale into unicode. */
61-
for (i = 0; i < thousands_sep_len; ++i)
62-
(*buffer_end)[i] = thousands_sep[i];
63-
#else
64-
/* No conversion, just memcpy the thousands_sep. */
65-
memcpy(*buffer_end, thousands_sep, thousands_sep_len);
66-
#endif
59+
memcpy(*buffer_end, thousands_sep,
60+
thousands_sep_len * STRINGLIB_SIZEOF_CHAR);
6761
}
6862

6963
*buffer_end -= n_chars;
@@ -76,7 +70,7 @@ STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
7670
}
7771

7872
/**
79-
* _Py_InsertThousandsGrouping:
73+
* InsertThousandsGrouping:
8074
* @buffer: A pointer to the start of a string.
8175
* @n_buffer: Number of characters in @buffer.
8276
* @digits: A pointer to the digits we're reading from. If count
@@ -106,13 +100,15 @@ STRINGLIB(fill)(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
106100
_insert_thousands_sep().
107101
**/
108102
Py_ssize_t
109-
_Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
110-
Py_ssize_t n_buffer,
111-
STRINGLIB_CHAR *digits,
112-
Py_ssize_t n_digits,
113-
Py_ssize_t min_width,
114-
const char *grouping,
115-
const char *thousands_sep)
103+
STRINGLIB(InsertThousandsGrouping)(
104+
STRINGLIB_CHAR *buffer,
105+
Py_ssize_t n_buffer,
106+
STRINGLIB_CHAR *digits,
107+
Py_ssize_t n_digits,
108+
Py_ssize_t min_width,
109+
const char *grouping,
110+
STRINGLIB_CHAR *thousands_sep,
111+
Py_ssize_t thousands_sep_len)
116112
{
117113
Py_ssize_t count = 0;
118114
Py_ssize_t n_zeros;
@@ -124,7 +120,6 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
124120
STRINGLIB_CHAR *digits_end = NULL;
125121
Py_ssize_t l;
126122
Py_ssize_t n_chars;
127-
Py_ssize_t thousands_sep_len = strlen(thousands_sep);
128123
Py_ssize_t remaining = n_digits; /* Number of chars remaining to
129124
be looked at */
130125
/* A generator that returns all of the grouping widths, until it
@@ -138,9 +133,9 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
138133
}
139134

140135
while ((l = STRINGLIB(GroupGenerator_next)(&groupgen)) > 0) {
141-
l = MIN(l, MAX(MAX(remaining, min_width), 1));
142-
n_zeros = MAX(0, l - remaining);
143-
n_chars = MAX(0, MIN(remaining, l));
136+
l = Py_MIN(l, Py_MAX(Py_MAX(remaining, min_width), 1));
137+
n_zeros = Py_MAX(0, l - remaining);
138+
n_chars = Py_MAX(0, Py_MIN(remaining, l));
144139

145140
/* Use n_zero zero's and n_chars chars */
146141

@@ -168,9 +163,9 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
168163
if (!loop_broken) {
169164
/* We left the loop without using a break statement. */
170165

171-
l = MAX(MAX(remaining, min_width), 1);
172-
n_zeros = MAX(0, l - remaining);
173-
n_chars = MAX(0, MIN(remaining, l));
166+
l = Py_MAX(Py_MAX(remaining, min_width), 1);
167+
n_zeros = Py_MAX(0, l - remaining);
168+
n_chars = Py_MAX(0, Py_MIN(remaining, l));
174169

175170
/* Use n_zero zero's and n_chars chars */
176171
count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars;
@@ -183,25 +178,3 @@ _Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
183178
return count;
184179
}
185180

186-
/**
187-
* _Py_InsertThousandsGroupingLocale:
188-
* @buffer: A pointer to the start of a string.
189-
* @n_digits: The number of digits in the string, in which we want
190-
* to put the grouping chars.
191-
*
192-
* Reads thee current locale and calls _Py_InsertThousandsGrouping().
193-
**/
194-
Py_ssize_t
195-
_Py_InsertThousandsGroupingLocale(STRINGLIB_CHAR *buffer,
196-
Py_ssize_t n_buffer,
197-
STRINGLIB_CHAR *digits,
198-
Py_ssize_t n_digits,
199-
Py_ssize_t min_width)
200-
{
201-
struct lconv *locale_data = localeconv();
202-
const char *grouping = locale_data->grouping;
203-
const char *thousands_sep = locale_data->thousands_sep;
204-
205-
return _Py_InsertThousandsGrouping(buffer, n_buffer, digits, n_digits,
206-
min_width, grouping, thousands_sep);
207-
}

‎Objects/stringlib/stringdefs.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,5 @@
2525
#define STRINGLIB_CHECK PyBytes_Check
2626
#define STRINGLIB_CHECK_EXACT PyBytes_CheckExact
2727
#define STRINGLIB_TOSTR PyObject_Str
28-
#define STRINGLIB_GROUPING _PyBytes_InsertThousandsGrouping
29-
#define STRINGLIB_GROUPING_LOCALE _PyBytes_InsertThousandsGroupingLocale
3028
#define STRINGLIB_TOASCII PyObject_Repr
3129
#endif /* !STRINGLIB_STRINGDEFS_H */

‎Objects/stringlib/ucs1lib.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,10 @@
2121
#define STRINGLIB_RESIZE not_supported
2222
#define STRINGLIB_CHECK PyUnicode_Check
2323
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
24-
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
25-
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
2624

2725
#define STRINGLIB_TOSTR PyObject_Str
2826
#define STRINGLIB_TOASCII PyObject_ASCII
2927

3028
#define _Py_InsertThousandsGrouping _PyUnicode_ucs1_InsertThousandsGrouping
31-
#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs1_InsertThousandsGroupingLocale
3229

3330

‎Objects/stringlib/ucs2lib.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,9 @@
2121
#define STRINGLIB_RESIZE not_supported
2222
#define STRINGLIB_CHECK PyUnicode_Check
2323
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
24-
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
25-
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
2624

2725
#define STRINGLIB_TOSTR PyObject_Str
2826
#define STRINGLIB_TOASCII PyObject_ASCII
2927

3028
#define _Py_InsertThousandsGrouping _PyUnicode_ucs2_InsertThousandsGrouping
31-
#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs2_InsertThousandsGroupingLocale
3229

‎Objects/stringlib/ucs4lib.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,9 @@
2121
#define STRINGLIB_RESIZE not_supported
2222
#define STRINGLIB_CHECK PyUnicode_Check
2323
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
24-
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
25-
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
2624

2725
#define STRINGLIB_TOSTR PyObject_Str
2826
#define STRINGLIB_TOASCII PyObject_ASCII
2927

3028
#define _Py_InsertThousandsGrouping _PyUnicode_ucs4_InsertThousandsGrouping
31-
#define _Py_InsertThousandsGroupingLocale _PyUnicode_ucs4_InsertThousandsGroupingLocale
3229

‎Objects/stringlib/undef.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@
77
#undef STRINGLIB_NEW
88
#undef STRINGLIB_RESIZE
99
#undef _Py_InsertThousandsGrouping
10-
#undef _Py_InsertThousandsGroupingLocale
10+
#undef STRINGLIB_IS_UNICODE
1111

‎Objects/stringlib/unicodedefs.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@
2424
#define STRINGLIB_RESIZE PyUnicode_Resize
2525
#define STRINGLIB_CHECK PyUnicode_Check
2626
#define STRINGLIB_CHECK_EXACT PyUnicode_CheckExact
27-
#define STRINGLIB_GROUPING _PyUnicode_InsertThousandsGrouping
28-
#define STRINGLIB_GROUPING_LOCALE _PyUnicode_InsertThousandsGroupingLocale
2927

3028
#if PY_VERSION_HEX < 0x03000000
3129
#define STRINGLIB_TOSTR PyObject_Unicode

0 commit comments

Comments
 (0)