From 974742a6f568fea84615ffa77edeb0e843aff2eb Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Mon, 16 Nov 2026 19:23:51 -0800 Subject: [PATCH] gh-101703: Replace sprintf with _PyLong_FormatBytesWriter in bytesobject Format %d, %i, %u, %ld, %lu, %zd, %zu and %x with _PyLong_FormatBytesWriter(), the code bytes % int uses, which writes straight into the PyBytesWriter instead of going through a stack buffer. Output is unchanged. This leaves the current %p as it has some platform specific variation which changing requires more validation and a NEWS. Each row is 1000 calls made from a C extension module (below), so 1 us is 1 ns per call. Linux x86-64, clang PGO+LTO, pyperf, upstream/main b90572af8f9 against this change. The %s and %p rows are not changed. | Benchmark | main | PR | |------------------------------|:-------:|:---------------------:| | run(b'%x', 123456789, 1000) | 36.9 us | 26.9 us: 1.37x faster | | run(b'%d', 5, 1000) | 28.1 us | 20.6 us: 1.37x faster | | run(b'%d.%d.%d', 42, 1000) | 74.7 us | 55.0 us: 1.36x faster | | run(b'%i', 42, 1000) | 34.1 us | 26.0 us: 1.31x faster | | run(b'%d', 42, 1000) | 33.9 us | 26.3 us: 1.29x faster | | run(b'%d', -123456789, 1000) | 40.5 us | 35.1 us: 1.16x faster | | run(b'%d', 123456789, 1000) | 40.0 us | 34.6 us: 1.16x faster | | run(b'%u', 123456789, 1000) | 39.7 us | 34.6 us: 1.15x faster | | run(b'%ld', 10**18, 1000) | 50.5 us | 46.0 us: 1.10x faster | | run(b'%lu', 2**64 - 1, 1000) | 50.1 us | 47.3 us: 1.06x faster | | run(b'%zu', 2**64 - 1, 1000) | 49.2 us | 47.7 us: 1.03x faster | | run(b'%s', 0, 1000) | 19.4 us | 18.8 us: 1.03x faster | | run(b'%p', 0, 1000) | 35.8 us | 34.8 us: 1.03x faster | Not significant: run(b'%p', 2**47 - 4096, 1000), run(b'%zd', 2**62, 1000). Benchmark script (bench.py): # PyBytes_FromFormat() integer formatting, called from C by # fromformat_bench.c (build it first, see its header). Each row is 1000 # calls, so 1 us is 1 ns per call. The %s and %p rows are formats the PR # does not change. # python bench.py -o main.json (main build) # python bench.py -o pr.json (PR build) # python -m pyperf compare_to main.json pr.json --table import pyperf runner = pyperf.Runner() for stmt in [ "run(b'%d', 5, 1000)", "run(b'%d', 42, 1000)", "run(b'%d', 123456789, 1000)", "run(b'%d', -123456789, 1000)", "run(b'%i', 42, 1000)", "run(b'%u', 123456789, 1000)", "run(b'%ld', 10**18, 1000)", "run(b'%lu', 2**64 - 1, 1000)", "run(b'%zd', 2**62, 1000)", "run(b'%zu', 2**64 - 1, 1000)", "run(b'%x', 123456789, 1000)", "run(b'%d.%d.%d', 42, 1000)", "run(b'%p', 2**47 - 4096, 1000)", "run(b'%p', 0, 1000)", "run(b'%s', 0, 1000)", ]: runner.timeit(stmt, stmt, setup="from fromformat_bench import run") C extension module (fromformat_bench.c): // run(format, value, loops): call PyBytes_FromFormat(format, value, value, // value) loops times. The first conversion in format picks the C type of // value; %s gets "abc". // // cc -shared -fPIC -O2 $(python3-config --includes) fromformat_bench.c // -o fromformat_bench.abi3.so #define Py_LIMITED_API 0x030c0000 #include #include static PyObject * run(PyObject *self, PyObject *args) { const char *format; unsigned long long value; Py_ssize_t loops; if (!PyArg_ParseTuple(args, "yKn", &format, &value, &loops)) { return NULL; } const char *conv = strchr(format, '%'); if (conv == NULL) { PyErr_SetString(PyExc_ValueError, "no conversion in format"); return NULL; } conv++; for (Py_ssize_t i = 0; i < loops; i++) { PyObject *bytes; if (*conv == 'l') { long v = (long)value; bytes = PyBytes_FromFormat(format, v, v, v); } else if (*conv == 'z') { size_t v = (size_t)value; bytes = PyBytes_FromFormat(format, v, v, v); } else if (*conv == 'p') { void *v = (void *)(uintptr_t)value; bytes = PyBytes_FromFormat(format, v, v, v); } else if (*conv == 's') { bytes = PyBytes_FromFormat(format, "abc", "abc", "abc"); } else { int v = (int)value; bytes = PyBytes_FromFormat(format, v, v, v); } if (bytes == NULL) { return NULL; } Py_DECREF(bytes); } Py_RETURN_NONE; } static PyMethodDef methods[] = { {"run", run, METH_VARARGS, NULL}, {NULL, NULL, 0, NULL}, }; static struct PyModuleDef module = { PyModuleDef_HEAD_INIT, "fromformat_bench", NULL, 0, methods, }; PyMODINIT_FUNC PyInit_fromformat_bench(void) { return PyModule_Create(&module); } Co-Authored-By: Claude Fable 5.2 --- Objects/bytesobject.c | 49 ++++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index e9264f765a08e6d..9a807b59a78210a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -237,12 +237,7 @@ static char* bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, const char *format, va_list vargs) { - /* Longest 64-bit formatted numbers: - - "18446744073709551615\0" (21 bytes) - - "-9223372036854775808\0" (21 bytes) - Decimal takes the most space (it isn't enough for octal.) - - Longest 64-bit pointer representation: + /* Longest 64-bit pointer representation: "0xffffffffffffffff\0" (19 bytes). Longest 64-bit "%p" with "0x" prefix: len(hex(2**64-1)+'\0') = 19. @@ -265,6 +260,20 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, s += _len; \ } while (0) + /* Format an integer with the same code as bytes % int. */ +#define WRITE_INT(obj_expr, base, alternate) \ + do { \ + PyObject *_obj = (obj_expr); \ + if (_obj == NULL) { \ + return NULL; \ + } \ + s = _PyLong_FormatBytesWriter(writer, s, _obj, (base), (alternate)); \ + Py_DECREF(_obj); \ + if (s == NULL) { \ + return NULL; \ + } \ + } while (0) + for (const char *f = format; *f; f++) { if (*f != '%') { *s++ = *f; @@ -324,42 +333,37 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, case 'd': if (longflag) { - len = sprintf(buffer, "%ld", va_arg(vargs, long)); + WRITE_INT(PyLong_FromLong(va_arg(vargs, long)), 10, 0); } else if (size_tflag) { - len = sprintf(buffer, "%zd", va_arg(vargs, Py_ssize_t)); + WRITE_INT(PyLong_FromSsize_t(va_arg(vargs, Py_ssize_t)), 10, 0); } else { - len = sprintf(buffer, "%d", va_arg(vargs, int)); + WRITE_INT(PyLong_FromLong(va_arg(vargs, int)), 10, 0); } - assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); - WRITE_BYTES(buffer, len); break; case 'u': if (longflag) { - len = sprintf(buffer, "%lu", va_arg(vargs, unsigned long)); + WRITE_INT(PyLong_FromUnsignedLong(va_arg(vargs, unsigned long)), + 10, 0); } else if (size_tflag) { - len = sprintf(buffer, "%zu", va_arg(vargs, size_t)); + WRITE_INT(PyLong_FromSize_t(va_arg(vargs, size_t)), 10, 0); } else { - len = sprintf(buffer, "%u", va_arg(vargs, unsigned int)); + WRITE_INT(PyLong_FromUnsignedLong(va_arg(vargs, unsigned int)), + 10, 0); } - assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); - WRITE_BYTES(buffer, len); break; case 'i': - len = sprintf(buffer, "%i", va_arg(vargs, int)); - assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); - WRITE_BYTES(buffer, len); + WRITE_INT(PyLong_FromLong(va_arg(vargs, int)), 10, 0); break; case 'x': - len = sprintf(buffer, "%x", va_arg(vargs, int)); - assert(1 <= len && len < (Py_ssize_t)sizeof(buffer)); - WRITE_BYTES(buffer, len); + WRITE_INT(PyLong_FromUnsignedLong((unsigned int)va_arg(vargs, int)), + 16, 0); break; case 's': @@ -414,6 +418,7 @@ bytes_fromformat(PyBytesWriter *writer, Py_ssize_t writer_pos, return s; +#undef WRITE_INT #undef WRITE_BYTES }