Skip to content

gh-101703: Replace sprintf with _PyLong_FormatBytesWriter in bytesobject for integer formats - #162148

Open
cmaloney wants to merge 1 commit into
python:mainfrom
cmaloney:bytes_fromformat_pylong
Open

cmaloney wants to merge 1 commit into
python:mainfrom
cmaloney:bytes_fromformat_pylong

Conversation

@cmaloney

@cmaloney cmaloney commented Nov 17, 2026 •

Copy link
Copy Markdown
Contributor

Handle %d, %i, %u, %ld, %lu, %zd, %zu and %x with existing implementation in _PyLong_FormatBytesWriter(). That writes straight into the PyBytesWriter instead of going through a local stack buffer.

This leaves the current %p as it has some platform specific variation which changing requires more validation and a NEWS. When that case is changed the stack buffer goes away. The temporary buffer could be shrunk by 2 bytes here but I think leaving the existing code is less risky short term.

Performance: Slightly faster but not majorly.

Code to help validate output didn't change
# Check the integer output of PyBytes_FromFormat(). Run it with a main build
# and with a PR build: both must print 0 mismatches and the same sha256.
import hashlib
import random
from ctypes import (pythonapi, py_object, c_int, c_uint, c_long, c_ulong,
                    c_ssize_t, c_size_t, c_void_p)

f = pythonapi.PyBytes_FromFormat
f.restype = py_object
rng = random.Random(0)
# Around every power of two and of ten, then random values of every bit length.
values = [sign * (2**bits + d)
          for bits in range(66) for d in (-1, 0, 1) for sign in (1, -1)]
values += [sign * (10**digits + d)
           for digits in range(21) for d in (-1, 0, 1) for sign in (1, -1)]
values += [rng.choice((1, -1)) * rng.getrandbits(rng.randint(0, 64))
           for _ in range(100_000)]

sha = hashlib.sha256()
checks = mismatches = 0
for v in values:
    # ctypes truncates v to the C type; arg.value is what C receives.
    for fmt, arg in [
        (b'%d', c_int(v)), (b'%i', c_int(v)), (b'%u', c_uint(v)),
        (b'%ld', c_long(v)), (b'%lu', c_ulong(v)),
        (b'%zd', c_ssize_t(v)), (b'%zu', c_size_t(v)),
    ]:
        got = f(fmt, arg)
        sha.update(got)
        checks += 1
        if got != b'%d' % arg.value:
            mismatches += 1
            print('MISMATCH', fmt, arg, got)
        # Width, precision and surrounding text: compared between builds only.
        sha.update(f(b'a' + fmt + b'|%5' + fmt[1:] + b'|%.3' + fmt[1:] + b'z%%',
                     arg, arg, arg))
    got = f(b'%x', c_int(v))
    sha.update(got)
    checks += 1
    if got != b'%x' % c_uint(v).value:
        mismatches += 1
        print('MISMATCH', b'%x', c_int(v), got)
    # %p is platform specific: compared between builds only.
    sha.update(f(b'%p', c_void_p(v)))
print(f'{checks} checks, {mismatches} mismatches, sha256 {sha.hexdigest()}')
Benchmarking results, code, c module

Each row is 1000 calls made from a C extension module (below), so 1 us is 1 ns per call. Linux x86-64, clang PGO+LTO, pyperf, upstream/main b90572a against this change. The %s and %p rows are not changed.

Benchmark main PR
run(b'%x', 123456789, 1000) 36.9 us 26.9 us: 1.37x faster
run(b'%d', 5, 1000) 28.1 us 20.6 us: 1.37x faster
run(b'%d.%d.%d', 42, 1000) 74.7 us 55.0 us: 1.36x faster
run(b'%i', 42, 1000) 34.1 us 26.0 us: 1.31x faster
run(b'%d', 42, 1000) 33.9 us 26.3 us: 1.29x faster
run(b'%d', -123456789, 1000) 40.5 us 35.1 us: 1.16x faster
run(b'%d', 123456789, 1000) 40.0 us 34.6 us: 1.16x faster
run(b'%u', 123456789, 1000) 39.7 us 34.6 us: 1.15x faster
run(b'%ld', 10**18, 1000) 50.5 us 46.0 us: 1.10x faster
run(b'%lu', 2**64 - 1, 1000) 50.1 us 47.3 us: 1.06x faster
run(b'%zu', 2**64 - 1, 1000) 49.2 us 47.7 us: 1.03x faster
run(b'%s', 0, 1000) 19.4 us 18.8 us: 1.03x faster
run(b'%p', 0, 1000) 35.8 us 34.8 us: 1.03x faster

Not significant: run(b'%p', 247 - 4096, 1000), run(b'%zd', 262, 1000).

Benchmark script (bench.py):

# PyBytes_FromFormat() integer formatting, called from C by
# fromformat_bench.c (build it first, see its header). Each row is 1000
# calls, so 1 us is 1 ns per call. The %s and %p rows are formats the PR
# does not change.
#   python bench.py -o main.json    (main build)
#   python bench.py -o pr.json      (PR build)
#   python -m pyperf compare_to main.json pr.json --table
import pyperf

runner = pyperf.Runner()
for stmt in [
    "run(b'%d', 5, 1000)",
    "run(b'%d', 42, 1000)",
    "run(b'%d', 123456789, 1000)",
    "run(b'%d', -123456789, 1000)",
    "run(b'%i', 42, 1000)",
    "run(b'%u', 123456789, 1000)",
    "run(b'%ld', 10**18, 1000)",
    "run(b'%lu', 2**64 - 1, 1000)",
    "run(b'%zd', 2**62, 1000)",
    "run(b'%zu', 2**64 - 1, 1000)",
    "run(b'%x', 123456789, 1000)",
    "run(b'%d.%d.%d', 42, 1000)",
    "run(b'%p', 2**47 - 4096, 1000)",
    "run(b'%p', 0, 1000)",
    "run(b'%s', 0, 1000)",
]:
    runner.timeit(stmt, stmt, setup="from fromformat_bench import run")

C extension module (fromformat_bench.c):

    // run(format, value, loops): call PyBytes_FromFormat(format, value, value,
    // value) loops times. The first conversion in format picks the C type of
    // value; %s gets "abc".
    //
    //   cc -shared -fPIC -O2 $(python3-config --includes) fromformat_bench.c
    //       -o fromformat_bench.abi3.so
    #define Py_LIMITED_API 0x030c0000
    #include <Python.h>
    #include <string.h>

    static PyObject *
    run(PyObject *self, PyObject *args)
    {
        const char *format;
        unsigned long long value;
        Py_ssize_t loops;
        if (!PyArg_ParseTuple(args, "yKn", &format, &value, &loops)) {
            return NULL;
        }
        const char *conv = strchr(format, '%');
        if (conv == NULL) {
            PyErr_SetString(PyExc_ValueError, "no conversion in format");
            return NULL;
        }
        conv++;
        for (Py_ssize_t i = 0; i < loops; i++) {
            PyObject *bytes;
            if (*conv == 'l') {
                long v = (long)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            else if (*conv == 'z') {
                size_t v = (size_t)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            else if (*conv == 'p') {
                void *v = (void *)(uintptr_t)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            else if (*conv == 's') {
                bytes = PyBytes_FromFormat(format, "abc", "abc", "abc");
            }
            else {
                int v = (int)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            if (bytes == NULL) {
                return NULL;
            }
            Py_DECREF(bytes);
        }
        Py_RETURN_NONE;
    }

    static PyMethodDef methods[] = {
        {"run", run, METH_VARARGS, NULL},
        {NULL, NULL, 0, NULL},
    };

    static struct PyModuleDef module = {
        PyModuleDef_HEAD_INIT, "fromformat_bench", NULL, 0, methods,
    };

    PyMODINIT_FUNC
    PyInit_fromformat_bench(void)
    {
        return PyModule_Create(&module);
    }

Another option for gh-161994 (does all but %p which would be a separate PR)

…tesobject

Format %d, %i, %u, %ld, %lu, %zd, %zu and %x with
_PyLong_FormatBytesWriter(), the code bytes % int uses, which writes
straight into the PyBytesWriter instead of going through a stack buffer.
Output is unchanged.

This leaves the current %p as it has some platform specific variation
which changing requires more validation and a NEWS.

Each row is 1000 calls made from a C extension module (below), so 1 us is
1 ns per call. Linux x86-64, clang PGO+LTO, pyperf, upstream/main
b90572a against this change. The %s and %p rows are not changed.

| Benchmark                    | main    | PR                    |
|------------------------------|:-------:|:---------------------:|
| run(b'%x', 123456789, 1000)  | 36.9 us | 26.9 us: 1.37x faster |
| run(b'%d', 5, 1000)          | 28.1 us | 20.6 us: 1.37x faster |
| run(b'%d.%d.%d', 42, 1000)   | 74.7 us | 55.0 us: 1.36x faster |
| run(b'%i', 42, 1000)         | 34.1 us | 26.0 us: 1.31x faster |
| run(b'%d', 42, 1000)         | 33.9 us | 26.3 us: 1.29x faster |
| run(b'%d', -123456789, 1000) | 40.5 us | 35.1 us: 1.16x faster |
| run(b'%d', 123456789, 1000)  | 40.0 us | 34.6 us: 1.16x faster |
| run(b'%u', 123456789, 1000)  | 39.7 us | 34.6 us: 1.15x faster |
| run(b'%ld', 10**18, 1000)    | 50.5 us | 46.0 us: 1.10x faster |
| run(b'%lu', 2**64 - 1, 1000) | 50.1 us | 47.3 us: 1.06x faster |
| run(b'%zu', 2**64 - 1, 1000) | 49.2 us | 47.7 us: 1.03x faster |
| run(b'%s', 0, 1000)          | 19.4 us | 18.8 us: 1.03x faster |
| run(b'%p', 0, 1000)          | 35.8 us | 34.8 us: 1.03x faster |

Not significant: run(b'%p', 2**47 - 4096, 1000), run(b'%zd', 2**62, 1000).

Benchmark script (bench.py):

    # PyBytes_FromFormat() integer formatting, called from C by
    # fromformat_bench.c (build it first, see its header). Each row is 1000
    # calls, so 1 us is 1 ns per call. The %s and %p rows are formats the PR
    # does not change.
    #   python bench.py -o main.json    (main build)
    #   python bench.py -o pr.json      (PR build)
    #   python -m pyperf compare_to main.json pr.json --table
    import pyperf

    runner = pyperf.Runner()
    for stmt in [
        "run(b'%d', 5, 1000)",
        "run(b'%d', 42, 1000)",
        "run(b'%d', 123456789, 1000)",
        "run(b'%d', -123456789, 1000)",
        "run(b'%i', 42, 1000)",
        "run(b'%u', 123456789, 1000)",
        "run(b'%ld', 10**18, 1000)",
        "run(b'%lu', 2**64 - 1, 1000)",
        "run(b'%zd', 2**62, 1000)",
        "run(b'%zu', 2**64 - 1, 1000)",
        "run(b'%x', 123456789, 1000)",
        "run(b'%d.%d.%d', 42, 1000)",
        "run(b'%p', 2**47 - 4096, 1000)",
        "run(b'%p', 0, 1000)",
        "run(b'%s', 0, 1000)",
    ]:
        runner.timeit(stmt, stmt, setup="from fromformat_bench import run")

C extension module (fromformat_bench.c):

    // run(format, value, loops): call PyBytes_FromFormat(format, value, value,
    // value) loops times. The first conversion in format picks the C type of
    // value; %s gets "abc".
    //
    //   cc -shared -fPIC -O2 $(python3-config --includes) fromformat_bench.c
    //       -o fromformat_bench.abi3.so
    #define Py_LIMITED_API 0x030c0000
    #include <Python.h>
    #include <string.h>

    static PyObject *
    run(PyObject *self, PyObject *args)
    {
        const char *format;
        unsigned long long value;
        Py_ssize_t loops;
        if (!PyArg_ParseTuple(args, "yKn", &format, &value, &loops)) {
            return NULL;
        }
        const char *conv = strchr(format, '%');
        if (conv == NULL) {
            PyErr_SetString(PyExc_ValueError, "no conversion in format");
            return NULL;
        }
        conv++;
        for (Py_ssize_t i = 0; i < loops; i++) {
            PyObject *bytes;
            if (*conv == 'l') {
                long v = (long)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            else if (*conv == 'z') {
                size_t v = (size_t)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            else if (*conv == 'p') {
                void *v = (void *)(uintptr_t)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            else if (*conv == 's') {
                bytes = PyBytes_FromFormat(format, "abc", "abc", "abc");
            }
            else {
                int v = (int)value;
                bytes = PyBytes_FromFormat(format, v, v, v);
            }
            if (bytes == NULL) {
                return NULL;
            }
            Py_DECREF(bytes);
        }
        Py_RETURN_NONE;
    }

    static PyMethodDef methods[] = {
        {"run", run, METH_VARARGS, NULL},
        {NULL, NULL, 0, NULL},
    };

    static struct PyModuleDef module = {
        PyModuleDef_HEAD_INIT, "fromformat_bench", NULL, 0, methods,
    };

    PyMODINIT_FUNC
    PyInit_fromformat_bench(void)
    {
        return PyModule_Create(&module);
    }

Co-Authored-By: Claude Fable 5.2 <[email protected]>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Projects

None yet

Development

Successfully merging this pull request may close these issues.

1 participant