fix: optimize GRPO trainer, ignore checkpoints and binary libs

Dockerfile

@@ -1,15 +1,33 @@
-FROM python:3.10-slim
+FROM python:3.13-slim
 
-# System dependencies for bitsandbytes
+# Install system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     && rm -rf /var/lib/apt/lists/*
 
-WORKDIR /app
+# Hugging Face Spaces requires running as a non-root user (UID 1000)
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+    PATH=/home/user/.local/bin:$PATH
 
-COPY requirements.txt .
+WORKDIR $HOME/app
+
+# Copy requirements
+COPY --chown=user requirements.txt .
+
+# Install PyTorch 2.11 with cu126 (matching our local battle-tested environment)
+RUN pip install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu126 --upgrade
+
+# Install remaining requirements and Triton explicitly for Linux cloud environment
 RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir triton
+
+# Copy application files
+COPY --chown=user . .
 
-COPY . .
+# Expose the mandatory port required by Hugging Face Spaces
+EXPOSE 7860
 
-CMD ["bash"]
+# Launch the core FastAPI application on the expected HF Spaces port
+CMD ["python", "-m", "uvicorn", "src.api.server:app", "--host", "0.0.0.0", "--port", "7860"]

Include/Python.h

@@ -0,0 +1,140 @@
+// Entry point of the Python C API.
+// C extensions should only #include <Python.h>, and not include directly
+// the other Python header files included by <Python.h>.
+
+#ifndef Py_PYTHON_H
+#define Py_PYTHON_H
+
+// Since this is a "meta-include" file, "#ifdef __cplusplus / extern "C" {"
+// is not needed.
+
+
+// Include Python header files
+#include "patchlevel.h"
+#include "pyconfig.h"
+#include "pymacconfig.h"
+
+
+// Include standard header files
+// When changing these files, remember to update Doc/extending/extending.rst.
+#include <assert.h>               // assert()
+#include <inttypes.h>             // uintptr_t
+#include <limits.h>               // INT_MAX
+#include <math.h>                 // HUGE_VAL
+#include <stdarg.h>               // va_list
+#include <wchar.h>                // wchar_t
+#ifdef HAVE_SYS_TYPES_H
+#  include <sys/types.h>          // ssize_t
+#endif
+
+// <errno.h>, <stdio.h>, <stdlib.h> and <string.h> headers are no longer used
+// by Python, but kept for the backward compatibility of existing third party C
+// extensions. They are not included by limited C API version 3.11 and newer.
+//
+// The <ctype.h> and <unistd.h> headers are not included by limited C API
+// version 3.13 and newer.
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000
+#  include <errno.h>              // errno
+#  include <stdio.h>              // FILE*
+#  include <stdlib.h>             // getenv()
+#  include <string.h>             // memcpy()
+#endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030d0000
+#  include <ctype.h>              // tolower()
+#  ifndef MS_WINDOWS
+#    include <unistd.h>           // close()
+#  endif
+#endif
+
+// gh-111506: The free-threaded build is not compatible with the limited API
+// or the stable ABI.
+#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED)
+#  error "The limited API is not currently supported in the free-threaded build"
+#endif
+
+#if defined(Py_GIL_DISABLED) && defined(_MSC_VER)
+#  include <intrin.h>             // __readgsqword()
+#endif
+
+#if defined(Py_GIL_DISABLED) && defined(__MINGW32__)
+#  include <intrin.h>             // __readgsqword()
+#endif
+
+// Include Python header files
+#include "pyport.h"
+#include "pymacro.h"
+#include "pymath.h"
+#include "pymem.h"
+#include "pytypedefs.h"
+#include "pybuffer.h"
+#include "pystats.h"
+#include "pyatomic.h"
+#include "lock.h"
+#include "object.h"
+#include "objimpl.h"
+#include "typeslots.h"
+#include "pyhash.h"
+#include "cpython/pydebug.h"
+#include "bytearrayobject.h"
+#include "bytesobject.h"
+#include "unicodeobject.h"
+#include "pyerrors.h"
+#include "longobject.h"
+#include "cpython/longintrepr.h"
+#include "boolobject.h"
+#include "floatobject.h"
+#include "complexobject.h"
+#include "rangeobject.h"
+#include "memoryobject.h"
+#include "tupleobject.h"
+#include "listobject.h"
+#include "dictobject.h"
+#include "cpython/odictobject.h"
+#include "enumobject.h"
+#include "setobject.h"
+#include "methodobject.h"
+#include "moduleobject.h"
+#include "monitoring.h"
+#include "cpython/funcobject.h"
+#include "cpython/classobject.h"
+#include "fileobject.h"
+#include "pycapsule.h"
+#include "cpython/code.h"
+#include "pyframe.h"
+#include "traceback.h"
+#include "sliceobject.h"
+#include "cpython/cellobject.h"
+#include "iterobject.h"
+#include "cpython/initconfig.h"
+#include "pystate.h"
+#include "cpython/genobject.h"
+#include "descrobject.h"
+#include "genericaliasobject.h"
+#include "warnings.h"
+#include "weakrefobject.h"
+#include "structseq.h"
+#include "cpython/picklebufobject.h"
+#include "cpython/pytime.h"
+#include "codecs.h"
+#include "pythread.h"
+#include "cpython/context.h"
+#include "modsupport.h"
+#include "compile.h"
+#include "pythonrun.h"
+#include "pylifecycle.h"
+#include "ceval.h"
+#include "sysmodule.h"
+#include "osmodule.h"
+#include "intrcheck.h"
+#include "import.h"
+#include "abstract.h"
+#include "bltinmodule.h"
+#include "critical_section.h"
+#include "cpython/pyctype.h"
+#include "pystrtod.h"
+#include "pystrcmp.h"
+#include "fileutils.h"
+#include "cpython/pyfpe.h"
+#include "cpython/tracemalloc.h"
+
+#endif /* !Py_PYTHON_H */

Include/abstract.h

@@ -0,0 +1,921 @@
+/* Abstract Object Interface (many thanks to Jim Fulton) */
+
+#ifndef Py_ABSTRACTOBJECT_H
+#define Py_ABSTRACTOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* === Object Protocol ================================================== */
+
+/* Implemented elsewhere:
+
+   int PyObject_Print(PyObject *o, FILE *fp, int flags);
+
+   Print an object 'o' on file 'fp'.  Returns -1 on error. The flags argument
+   is used to enable certain printing options. The only option currently
+   supported is Py_PRINT_RAW. By default (flags=0), PyObject_Print() formats
+   the object by calling PyObject_Repr(). If flags equals to Py_PRINT_RAW, it
+   formats the object by calling PyObject_Str(). */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_HasAttrString(PyObject *o, const char *attr_name);
+
+   Returns 1 if object 'o' has the attribute attr_name, and 0 otherwise.
+
+   This is equivalent to the Python expression: hasattr(o,attr_name).
+
+   This function always succeeds. */
+
+
+/* Implemented elsewhere:
+
+   PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name);
+
+   Retrieve an attributed named attr_name form object o.
+   Returns the attribute value on success, or NULL on failure.
+
+   This is the equivalent of the Python expression: o.attr_name. */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_HasAttr(PyObject *o, PyObject *attr_name);
+
+   Returns 1 if o has the attribute attr_name, and 0 otherwise.
+
+   This is equivalent to the Python expression: hasattr(o,attr_name).
+
+   This function always succeeds. */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_HasAttrStringWithError(PyObject *o, const char *attr_name);
+
+   Returns 1 if object 'o' has the attribute attr_name, and 0 otherwise.
+   This is equivalent to the Python expression: hasattr(o,attr_name).
+   Returns -1 on failure. */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name);
+
+   Returns 1 if o has the attribute attr_name, and 0 otherwise.
+   This is equivalent to the Python expression: hasattr(o,attr_name).
+   Returns -1 on failure. */
+
+
+/* Implemented elsewhere:
+
+   PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name);
+
+   Retrieve an attributed named 'attr_name' form object 'o'.
+   Returns the attribute value on success, or NULL on failure.
+
+   This is the equivalent of the Python expression: o.attr_name. */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_GetOptionalAttr(PyObject *obj, PyObject *attr_name, PyObject **result);
+
+   Variant of PyObject_GetAttr() which doesn't raise AttributeError
+   if the attribute is not found.
+
+   If the attribute is found, return 1 and set *result to a new strong
+   reference to the attribute.
+   If the attribute is not found, return 0 and set *result to NULL;
+   the AttributeError is silenced.
+   If an error other than AttributeError is raised, return -1 and
+   set *result to NULL.
+*/
+
+
+/* Implemented elsewhere:
+
+   int PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result);
+
+   Variant of PyObject_GetAttrString() which doesn't raise AttributeError
+   if the attribute is not found.
+
+   If the attribute is found, return 1 and set *result to a new strong
+   reference to the attribute.
+   If the attribute is not found, return 0 and set *result to NULL;
+   the AttributeError is silenced.
+   If an error other than AttributeError is raised, return -1 and
+   set *result to NULL.
+*/
+
+
+/* Implemented elsewhere:
+
+   int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v);
+
+   Set the value of the attribute named attr_name, for object 'o',
+   to the value 'v'. Raise an exception and return -1 on failure; return 0 on
+   success.
+
+   This is the equivalent of the Python statement o.attr_name=v. */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_SetAttr(PyObject *o, PyObject *attr_name, PyObject *v);
+
+   Set the value of the attribute named attr_name, for object 'o', to the value
+   'v'. an exception and return -1 on failure; return 0 on success.
+
+   This is the equivalent of the Python statement o.attr_name=v. */
+
+/* Implemented elsewhere:
+
+   int PyObject_DelAttrString(PyObject *o, const char *attr_name);
+
+   Delete attribute named attr_name, for object o. Returns
+   -1 on failure.
+
+   This is the equivalent of the Python statement: del o.attr_name.
+
+   Implemented as a macro in the limited C API 3.12 and older. */
+#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000
+#  define PyObject_DelAttrString(O, A) PyObject_SetAttrString((O), (A), NULL)
+#endif
+
+
+/* Implemented elsewhere:
+
+   int PyObject_DelAttr(PyObject *o, PyObject *attr_name);
+
+   Delete attribute named attr_name, for object o. Returns -1
+   on failure.  This is the equivalent of the Python
+   statement: del o.attr_name.
+
+   Implemented as a macro in the limited C API 3.12 and older. */
+#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000
+#  define PyObject_DelAttr(O, A) PyObject_SetAttr((O), (A), NULL)
+#endif
+
+
+/* Implemented elsewhere:
+
+   PyObject *PyObject_Repr(PyObject *o);
+
+   Compute the string representation of object 'o'.  Returns the
+   string representation on success, NULL on failure.
+
+   This is the equivalent of the Python expression: repr(o).
+
+   Called by the repr() built-in function. */
+
+
+/* Implemented elsewhere:
+
+   PyObject *PyObject_Str(PyObject *o);
+
+   Compute the string representation of object, o.  Returns the
+   string representation on success, NULL on failure.
+
+   This is the equivalent of the Python expression: str(o).
+
+   Called by the str() and print() built-in functions. */
+
+
+/* Declared elsewhere
+
+   PyAPI_FUNC(int) PyCallable_Check(PyObject *o);
+
+   Determine if the object, o, is callable.  Return 1 if the object is callable
+   and 0 otherwise.
+
+   This function always succeeds. */
+
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
+/* Call a callable Python object without any arguments */
+PyAPI_FUNC(PyObject *) PyObject_CallNoArgs(PyObject *func);
+#endif
+
+
+/* Call a callable Python object 'callable' with arguments given by the
+   tuple 'args' and keywords arguments given by the dictionary 'kwargs'.
+
+   'args' must not be NULL, use an empty tuple if no arguments are
+   needed. If no named arguments are needed, 'kwargs' can be NULL.
+
+   This is the equivalent of the Python expression:
+   callable(*args, **kwargs). */
+PyAPI_FUNC(PyObject *) PyObject_Call(PyObject *callable,
+                                     PyObject *args, PyObject *kwargs);
+
+
+/* Call a callable Python object 'callable', with arguments given by the
+   tuple 'args'.  If no arguments are needed, then 'args' can be NULL.
+
+   Returns the result of the call on success, or NULL on failure.
+
+   This is the equivalent of the Python expression:
+   callable(*args). */
+PyAPI_FUNC(PyObject *) PyObject_CallObject(PyObject *callable,
+                                           PyObject *args);
+
+/* Call a callable Python object, callable, with a variable number of C
+   arguments. The C arguments are described using a mkvalue-style format
+   string.
+
+   The format may be NULL, indicating that no arguments are provided.
+
+   Returns the result of the call on success, or NULL on failure.
+
+   This is the equivalent of the Python expression:
+   callable(arg1, arg2, ...). */
+PyAPI_FUNC(PyObject *) PyObject_CallFunction(PyObject *callable,
+                                             const char *format, ...);
+
+/* Call the method named 'name' of object 'obj' with a variable number of
+   C arguments.  The C arguments are described by a mkvalue format string.
+
+   The format can be NULL, indicating that no arguments are provided.
+
+   Returns the result of the call on success, or NULL on failure.
+
+   This is the equivalent of the Python expression:
+   obj.name(arg1, arg2, ...). */
+PyAPI_FUNC(PyObject *) PyObject_CallMethod(PyObject *obj,
+                                           const char *name,
+                                           const char *format, ...);
+
+/* Call a callable Python object 'callable' with a variable number of C
+   arguments. The C arguments are provided as PyObject* values, terminated
+   by a NULL.
+
+   Returns the result of the call on success, or NULL on failure.
+
+   This is the equivalent of the Python expression:
+   callable(arg1, arg2, ...). */
+PyAPI_FUNC(PyObject *) PyObject_CallFunctionObjArgs(PyObject *callable,
+                                                    ...);
+
+/* Call the method named 'name' of object 'obj' with a variable number of
+   C arguments.  The C arguments are provided as PyObject* values, terminated
+   by NULL.
+
+   Returns the result of the call on success, or NULL on failure.
+
+   This is the equivalent of the Python expression: obj.name(*args). */
+
+PyAPI_FUNC(PyObject *) PyObject_CallMethodObjArgs(
+    PyObject *obj,
+    PyObject *name,
+    ...);
+
+/* Given a vectorcall nargsf argument, return the actual number of arguments.
+ * (For use outside the limited API, this is re-defined as a static inline
+ * function in cpython/abstract.h)
+ */
+PyAPI_FUNC(Py_ssize_t) PyVectorcall_NARGS(size_t nargsf);
+
+/* Call "callable" (which must support vectorcall) with positional arguments
+   "tuple" and keyword arguments "dict". "dict" may also be NULL */
+PyAPI_FUNC(PyObject *) PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict);
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000
+#define PY_VECTORCALL_ARGUMENTS_OFFSET \
+    (_Py_STATIC_CAST(size_t, 1) << (8 * sizeof(size_t) - 1))
+
+/* Perform a PEP 590-style vector call on 'callable' */
+PyAPI_FUNC(PyObject *) PyObject_Vectorcall(
+    PyObject *callable,
+    PyObject *const *args,
+    size_t nargsf,
+    PyObject *kwnames);
+
+/* Call the method 'name' on args[0] with arguments in args[1..nargsf-1]. */
+PyAPI_FUNC(PyObject *) PyObject_VectorcallMethod(
+    PyObject *name, PyObject *const *args,
+    size_t nargsf, PyObject *kwnames);
+#endif
+
+/* Implemented elsewhere:
+
+   Py_hash_t PyObject_Hash(PyObject *o);
+
+   Compute and return the hash, hash_value, of an object, o.  On
+   failure, return -1.
+
+   This is the equivalent of the Python expression: hash(o). */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_IsTrue(PyObject *o);
+
+   Returns 1 if the object, o, is considered to be true, 0 if o is
+   considered to be false and -1 on failure.
+
+   This is equivalent to the Python expression: not not o. */
+
+
+/* Implemented elsewhere:
+
+   int PyObject_Not(PyObject *o);
+
+   Returns 0 if the object, o, is considered to be true, 1 if o is
+   considered to be false and -1 on failure.
+
+   This is equivalent to the Python expression: not o. */
+
+
+/* Get the type of an object.
+
+   On success, returns a type object corresponding to the object type of object
+   'o'. On failure, returns NULL.
+
+   This is equivalent to the Python expression: type(o) */
+PyAPI_FUNC(PyObject *) PyObject_Type(PyObject *o);
+
+
+/* Return the size of object 'o'.  If the object 'o' provides both sequence and
+   mapping protocols, the sequence size is returned.
+
+   On error, -1 is returned.
+
+   This is the equivalent to the Python expression: len(o) */
+PyAPI_FUNC(Py_ssize_t) PyObject_Size(PyObject *o);
+
+
+/* For DLL compatibility */
+#undef PyObject_Length
+PyAPI_FUNC(Py_ssize_t) PyObject_Length(PyObject *o);
+#define PyObject_Length PyObject_Size
+
+/* Return element of 'o' corresponding to the object 'key'. Return NULL
+  on failure.
+
+  This is the equivalent of the Python expression: o[key] */
+PyAPI_FUNC(PyObject *) PyObject_GetItem(PyObject *o, PyObject *key);
+
+
+/* Map the object 'key' to the value 'v' into 'o'.
+
+   Raise an exception and return -1 on failure; return 0 on success.
+
+   This is the equivalent of the Python statement: o[key]=v. */
+PyAPI_FUNC(int) PyObject_SetItem(PyObject *o, PyObject *key, PyObject *v);
+
+/* Remove the mapping for the string 'key' from the object 'o'.
+   Returns -1 on failure.
+
+   This is equivalent to the Python statement: del o[key]. */
+PyAPI_FUNC(int) PyObject_DelItemString(PyObject *o, const char *key);
+
+/* Delete the mapping for the object 'key' from the object 'o'.
+   Returns -1 on failure.
+
+   This is the equivalent of the Python statement: del o[key]. */
+PyAPI_FUNC(int) PyObject_DelItem(PyObject *o, PyObject *key);
+
+
+/* Takes an arbitrary object and returns the result of calling
+   obj.__format__(format_spec). */
+PyAPI_FUNC(PyObject *) PyObject_Format(PyObject *obj,
+                                       PyObject *format_spec);
+
+
+/* ==== Iterators ================================================ */
+
+/* Takes an object and returns an iterator for it.
+   This is typically a new iterator but if the argument is an iterator, this
+   returns itself. */
+PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *);
+
+/* Takes an AsyncIterable object and returns an AsyncIterator for it.
+   This is typically a new iterator but if the argument is an AsyncIterator,
+   this returns itself. */
+PyAPI_FUNC(PyObject *) PyObject_GetAIter(PyObject *);
+
+/* Returns non-zero if the object 'obj' provides iterator protocols, and 0 otherwise.
+
+   This function always succeeds. */
+PyAPI_FUNC(int) PyIter_Check(PyObject *);
+
+/* Returns non-zero if the object 'obj' provides AsyncIterator protocols, and 0 otherwise.
+
+   This function always succeeds. */
+PyAPI_FUNC(int) PyAIter_Check(PyObject *);
+
+/* Takes an iterator object and calls its tp_iternext slot,
+   returning the next value.
+
+   If the iterator is exhausted, this returns NULL without setting an
+   exception.
+
+   NULL with an exception means an error occurred. */
+PyAPI_FUNC(PyObject *) PyIter_Next(PyObject *);
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000
+
+/* Takes generator, coroutine or iterator object and sends the value into it.
+   Returns:
+   - PYGEN_RETURN (0) if generator has returned.
+     'result' parameter is filled with return value
+   - PYGEN_ERROR (-1) if exception was raised.
+     'result' parameter is NULL
+   - PYGEN_NEXT (1) if generator has yielded.
+     'result' parameter is filled with yielded value. */
+PyAPI_FUNC(PySendResult) PyIter_Send(PyObject *, PyObject *, PyObject **);
+#endif
+
+
+/* === Number Protocol ================================================== */
+
+/* Returns 1 if the object 'o' provides numeric protocols, and 0 otherwise.
+
+   This function always succeeds. */
+PyAPI_FUNC(int) PyNumber_Check(PyObject *o);
+
+/* Returns the result of adding o1 and o2, or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 + o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Add(PyObject *o1, PyObject *o2);
+
+/* Returns the result of subtracting o2 from o1, or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 - o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Subtract(PyObject *o1, PyObject *o2);
+
+/* Returns the result of multiplying o1 and o2, or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 * o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Multiply(PyObject *o1, PyObject *o2);
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* This is the equivalent of the Python expression: o1 @ o2. */
+PyAPI_FUNC(PyObject *) PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2);
+#endif
+
+/* Returns the result of dividing o1 by o2 giving an integral result,
+   or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 // o2. */
+PyAPI_FUNC(PyObject *) PyNumber_FloorDivide(PyObject *o1, PyObject *o2);
+
+/* Returns the result of dividing o1 by o2 giving a float result, or NULL on
+   failure.
+
+   This is the equivalent of the Python expression: o1 / o2. */
+PyAPI_FUNC(PyObject *) PyNumber_TrueDivide(PyObject *o1, PyObject *o2);
+
+/* Returns the remainder of dividing o1 by o2, or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 % o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Remainder(PyObject *o1, PyObject *o2);
+
+/* See the built-in function divmod.
+
+   Returns NULL on failure.
+
+   This is the equivalent of the Python expression: divmod(o1, o2). */
+PyAPI_FUNC(PyObject *) PyNumber_Divmod(PyObject *o1, PyObject *o2);
+
+/* See the built-in function pow. Returns NULL on failure.
+
+   This is the equivalent of the Python expression: pow(o1, o2, o3),
+   where o3 is optional. */
+PyAPI_FUNC(PyObject *) PyNumber_Power(PyObject *o1, PyObject *o2,
+                                      PyObject *o3);
+
+/* Returns the negation of o on success, or NULL on failure.
+
+ This is the equivalent of the Python expression: -o. */
+PyAPI_FUNC(PyObject *) PyNumber_Negative(PyObject *o);
+
+/* Returns the positive of o on success, or NULL on failure.
+
+   This is the equivalent of the Python expression: +o. */
+PyAPI_FUNC(PyObject *) PyNumber_Positive(PyObject *o);
+
+/* Returns the absolute value of 'o', or NULL on failure.
+
+   This is the equivalent of the Python expression: abs(o). */
+PyAPI_FUNC(PyObject *) PyNumber_Absolute(PyObject *o);
+
+/* Returns the bitwise negation of 'o' on success, or NULL on failure.
+
+   This is the equivalent of the Python expression: ~o. */
+PyAPI_FUNC(PyObject *) PyNumber_Invert(PyObject *o);
+
+/* Returns the result of left shifting o1 by o2 on success, or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 << o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Lshift(PyObject *o1, PyObject *o2);
+
+/* Returns the result of right shifting o1 by o2 on success, or NULL on
+   failure.
+
+   This is the equivalent of the Python expression: o1 >> o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Rshift(PyObject *o1, PyObject *o2);
+
+/* Returns the result of bitwise and of o1 and o2 on success, or NULL on
+   failure.
+
+   This is the equivalent of the Python expression: o1 & o2. */
+PyAPI_FUNC(PyObject *) PyNumber_And(PyObject *o1, PyObject *o2);
+
+/* Returns the bitwise exclusive or of o1 by o2 on success, or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 ^ o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Xor(PyObject *o1, PyObject *o2);
+
+/* Returns the result of bitwise or on o1 and o2 on success, or NULL on
+   failure.
+
+   This is the equivalent of the Python expression: o1 | o2. */
+PyAPI_FUNC(PyObject *) PyNumber_Or(PyObject *o1, PyObject *o2);
+
+/* Returns 1 if obj is an index integer (has the nb_index slot of the
+   tp_as_number structure filled in), and 0 otherwise. */
+PyAPI_FUNC(int) PyIndex_Check(PyObject *);
+
+/* Returns the object 'o' converted to a Python int, or NULL with an exception
+   raised on failure. */
+PyAPI_FUNC(PyObject *) PyNumber_Index(PyObject *o);
+
+/* Returns the object 'o' converted to Py_ssize_t by going through
+   PyNumber_Index() first.
+
+   If an overflow error occurs while converting the int to Py_ssize_t, then the
+   second argument 'exc' is the error-type to return.  If it is NULL, then the
+   overflow error is cleared and the value is clipped. */
+PyAPI_FUNC(Py_ssize_t) PyNumber_AsSsize_t(PyObject *o, PyObject *exc);
+
+/* Returns the object 'o' converted to an integer object on success, or NULL
+   on failure.
+
+   This is the equivalent of the Python expression: int(o). */
+PyAPI_FUNC(PyObject *) PyNumber_Long(PyObject *o);
+
+/* Returns the object 'o' converted to a float object on success, or NULL
+  on failure.
+
+  This is the equivalent of the Python expression: float(o). */
+PyAPI_FUNC(PyObject *) PyNumber_Float(PyObject *o);
+
+
+/* --- In-place variants of (some of) the above number protocol functions -- */
+
+/* Returns the result of adding o2 to o1, possibly in-place, or NULL
+   on failure.
+
+   This is the equivalent of the Python expression: o1 += o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceAdd(PyObject *o1, PyObject *o2);
+
+/* Returns the result of subtracting o2 from o1, possibly in-place or
+   NULL on failure.
+
+   This is the equivalent of the Python expression: o1 -= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceSubtract(PyObject *o1, PyObject *o2);
+
+/* Returns the result of multiplying o1 by o2, possibly in-place, or NULL on
+   failure.
+
+   This is the equivalent of the Python expression: o1 *= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceMultiply(PyObject *o1, PyObject *o2);
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* This is the equivalent of the Python expression: o1 @= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2);
+#endif
+
+/* Returns the result of dividing o1 by o2 giving an integral result, possibly
+   in-place, or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 /= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceFloorDivide(PyObject *o1,
+                                                   PyObject *o2);
+
+/* Returns the result of dividing o1 by o2 giving a float result, possibly
+   in-place, or null on failure.
+
+   This is the equivalent of the Python expression: o1 /= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceTrueDivide(PyObject *o1,
+                                                  PyObject *o2);
+
+/* Returns the remainder of dividing o1 by o2, possibly in-place, or NULL on
+   failure.
+
+   This is the equivalent of the Python expression: o1 %= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceRemainder(PyObject *o1, PyObject *o2);
+
+/* Returns the result of raising o1 to the power of o2, possibly in-place,
+   or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 **= o2,
+   or o1 = pow(o1, o2, o3) if o3 is present. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlacePower(PyObject *o1, PyObject *o2,
+                                             PyObject *o3);
+
+/* Returns the result of left shifting o1 by o2, possibly in-place, or NULL
+   on failure.
+
+   This is the equivalent of the Python expression: o1 <<= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceLshift(PyObject *o1, PyObject *o2);
+
+/* Returns the result of right shifting o1 by o2, possibly in-place or NULL
+   on failure.
+
+   This is the equivalent of the Python expression: o1 >>= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceRshift(PyObject *o1, PyObject *o2);
+
+/* Returns the result of bitwise and of o1 and o2, possibly in-place, or NULL
+   on failure.
+
+   This is the equivalent of the Python expression: o1 &= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceAnd(PyObject *o1, PyObject *o2);
+
+/* Returns the bitwise exclusive or of o1 by o2, possibly in-place, or NULL
+   on failure.
+
+   This is the equivalent of the Python expression: o1 ^= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceXor(PyObject *o1, PyObject *o2);
+
+/* Returns the result of bitwise or of o1 and o2, possibly in-place,
+   or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 |= o2. */
+PyAPI_FUNC(PyObject *) PyNumber_InPlaceOr(PyObject *o1, PyObject *o2);
+
+/* Returns the integer n converted to a string with a base, with a base
+   marker of 0b, 0o or 0x prefixed if applicable.
+
+   If n is not an int object, it is converted with PyNumber_Index first. */
+PyAPI_FUNC(PyObject *) PyNumber_ToBase(PyObject *n, int base);
+
+
+/* === Sequence protocol ================================================ */
+
+/* Return 1 if the object provides sequence protocol, and zero
+   otherwise.
+
+   This function always succeeds. */
+PyAPI_FUNC(int) PySequence_Check(PyObject *o);
+
+/* Return the size of sequence object o, or -1 on failure. */
+PyAPI_FUNC(Py_ssize_t) PySequence_Size(PyObject *o);
+
+/* For DLL compatibility */
+#undef PySequence_Length
+PyAPI_FUNC(Py_ssize_t) PySequence_Length(PyObject *o);
+#define PySequence_Length PySequence_Size
+
+
+/* Return the concatenation of o1 and o2 on success, and NULL on failure.
+
+   This is the equivalent of the Python expression: o1 + o2. */
+PyAPI_FUNC(PyObject *) PySequence_Concat(PyObject *o1, PyObject *o2);
+
+/* Return the result of repeating sequence object 'o' 'count' times,
+  or NULL on failure.
+
+  This is the equivalent of the Python expression: o * count. */
+PyAPI_FUNC(PyObject *) PySequence_Repeat(PyObject *o, Py_ssize_t count);
+
+/* Return the ith element of o, or NULL on failure.
+
+   This is the equivalent of the Python expression: o[i]. */
+PyAPI_FUNC(PyObject *) PySequence_GetItem(PyObject *o, Py_ssize_t i);
+
+/* Return the slice of sequence object o between i1 and i2, or NULL on failure.
+
+   This is the equivalent of the Python expression: o[i1:i2]. */
+PyAPI_FUNC(PyObject *) PySequence_GetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2);
+
+/* Assign object 'v' to the ith element of the sequence 'o'. Raise an exception
+   and return -1 on failure; return 0 on success.
+
+   This is the equivalent of the Python statement o[i] = v. */
+PyAPI_FUNC(int) PySequence_SetItem(PyObject *o, Py_ssize_t i, PyObject *v);
+
+/* Delete the 'i'-th element of the sequence 'v'. Returns -1 on failure.
+
+   This is the equivalent of the Python statement: del o[i]. */
+PyAPI_FUNC(int) PySequence_DelItem(PyObject *o, Py_ssize_t i);
+
+/* Assign the sequence object 'v' to the slice in sequence object 'o',
+   from 'i1' to 'i2'. Returns -1 on failure.
+
+   This is the equivalent of the Python statement: o[i1:i2] = v. */
+PyAPI_FUNC(int) PySequence_SetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2,
+                                    PyObject *v);
+
+/* Delete the slice in sequence object 'o' from 'i1' to 'i2'.
+   Returns -1 on failure.
+
+   This is the equivalent of the Python statement: del o[i1:i2]. */
+PyAPI_FUNC(int) PySequence_DelSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2);
+
+/* Returns the sequence 'o' as a tuple on success, and NULL on failure.
+
+   This is equivalent to the Python expression: tuple(o). */
+PyAPI_FUNC(PyObject *) PySequence_Tuple(PyObject *o);
+
+/* Returns the sequence 'o' as a list on success, and NULL on failure.
+   This is equivalent to the Python expression: list(o) */
+PyAPI_FUNC(PyObject *) PySequence_List(PyObject *o);
+
+/* Return the sequence 'o' as a list, unless it's already a tuple or list.
+
+   Use PySequence_Fast_GET_ITEM to access the members of this list, and
+   PySequence_Fast_GET_SIZE to get its length.
+
+   Returns NULL on failure.  If the object does not support iteration, raises a
+   TypeError exception with 'm' as the message text. */
+PyAPI_FUNC(PyObject *) PySequence_Fast(PyObject *o, const char* m);
+
+/* Return the size of the sequence 'o', assuming that 'o' was returned by
+   PySequence_Fast and is not NULL. */
+#define PySequence_Fast_GET_SIZE(o) \
+    (PyList_Check(o) ? PyList_GET_SIZE(o) : PyTuple_GET_SIZE(o))
+
+/* Return the 'i'-th element of the sequence 'o', assuming that o was returned
+   by PySequence_Fast, and that i is within bounds. */
+#define PySequence_Fast_GET_ITEM(o, i)\
+     (PyList_Check(o) ? PyList_GET_ITEM((o), (i)) : PyTuple_GET_ITEM((o), (i)))
+
+/* Return a pointer to the underlying item array for
+   an object returned by PySequence_Fast */
+#define PySequence_Fast_ITEMS(sf) \
+    (PyList_Check(sf) ? ((PyListObject *)(sf))->ob_item \
+                      : ((PyTupleObject *)(sf))->ob_item)
+
+/* Return the number of occurrences on value on 'o', that is, return
+   the number of keys for which o[key] == value.
+
+   On failure, return -1.  This is equivalent to the Python expression:
+   o.count(value). */
+PyAPI_FUNC(Py_ssize_t) PySequence_Count(PyObject *o, PyObject *value);
+
+/* Return 1 if 'ob' is in the sequence 'seq'; 0 if 'ob' is not in the sequence
+   'seq'; -1 on error.
+
+   Use __contains__ if possible, else _PySequence_IterSearch(). */
+PyAPI_FUNC(int) PySequence_Contains(PyObject *seq, PyObject *ob);
+
+/* For DLL-level backwards compatibility */
+#undef PySequence_In
+/* Determine if the sequence 'o' contains 'value'. If an item in 'o' is equal
+   to 'value', return 1, otherwise return 0. On error, return -1.
+
+   This is equivalent to the Python expression: value in o. */
+PyAPI_FUNC(int) PySequence_In(PyObject *o, PyObject *value);
+
+/* For source-level backwards compatibility */
+#define PySequence_In PySequence_Contains
+
+
+/* Return the first index for which o[i] == value.
+   On error, return -1.
+
+   This is equivalent to the Python expression: o.index(value). */
+PyAPI_FUNC(Py_ssize_t) PySequence_Index(PyObject *o, PyObject *value);
+
+
+/* --- In-place versions of some of the above Sequence functions --- */
+
+/* Append sequence 'o2' to sequence 'o1', in-place when possible. Return the
+   resulting object, which could be 'o1', or NULL on failure.
+
+  This is the equivalent of the Python expression: o1 += o2. */
+PyAPI_FUNC(PyObject *) PySequence_InPlaceConcat(PyObject *o1, PyObject *o2);
+
+/* Repeat sequence 'o' by 'count', in-place when possible. Return the resulting
+   object, which could be 'o', or NULL on failure.
+
+   This is the equivalent of the Python expression: o1 *= count.  */
+PyAPI_FUNC(PyObject *) PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count);
+
+
+/* === Mapping protocol ================================================= */
+
+/* Return 1 if the object provides mapping protocol, and 0 otherwise.
+
+   This function always succeeds. */
+PyAPI_FUNC(int) PyMapping_Check(PyObject *o);
+
+/* Returns the number of keys in mapping object 'o' on success, and -1 on
+  failure. This is equivalent to the Python expression: len(o). */
+PyAPI_FUNC(Py_ssize_t) PyMapping_Size(PyObject *o);
+
+/* For DLL compatibility */
+#undef PyMapping_Length
+PyAPI_FUNC(Py_ssize_t) PyMapping_Length(PyObject *o);
+#define PyMapping_Length PyMapping_Size
+
+
+/* Implemented as a macro:
+
+   int PyMapping_DelItemString(PyObject *o, const char *key);
+
+   Remove the mapping for the string 'key' from the mapping 'o'. Returns -1 on
+   failure.
+
+   This is equivalent to the Python statement: del o[key]. */
+#define PyMapping_DelItemString(O, K) PyObject_DelItemString((O), (K))
+
+/* Implemented as a macro:
+
+   int PyMapping_DelItem(PyObject *o, PyObject *key);
+
+   Remove the mapping for the object 'key' from the mapping object 'o'.
+   Returns -1 on failure.
+
+   This is equivalent to the Python statement: del o[key]. */
+#define PyMapping_DelItem(O, K) PyObject_DelItem((O), (K))
+
+/* On success, return 1 if the mapping object 'o' has the key 'key',
+   and 0 otherwise.
+
+   This is equivalent to the Python expression: key in o.
+
+   This function always succeeds. */
+PyAPI_FUNC(int) PyMapping_HasKeyString(PyObject *o, const char *key);
+
+/* Return 1 if the mapping object has the key 'key', and 0 otherwise.
+
+   This is equivalent to the Python expression: key in o.
+
+   This function always succeeds. */
+PyAPI_FUNC(int) PyMapping_HasKey(PyObject *o, PyObject *key);
+
+/* Return 1 if the mapping object has the key 'key', and 0 otherwise.
+   This is equivalent to the Python expression: key in o.
+   On failure, return -1. */
+
+PyAPI_FUNC(int) PyMapping_HasKeyWithError(PyObject *o, PyObject *key);
+
+/* Return 1 if the mapping object has the key 'key', and 0 otherwise.
+   This is equivalent to the Python expression: key in o.
+   On failure, return -1. */
+
+PyAPI_FUNC(int) PyMapping_HasKeyStringWithError(PyObject *o, const char *key);
+
+/* On success, return a list or tuple of the keys in mapping object 'o'.
+   On failure, return NULL. */
+PyAPI_FUNC(PyObject *) PyMapping_Keys(PyObject *o);
+
+/* On success, return a list or tuple of the values in mapping object 'o'.
+   On failure, return NULL. */
+PyAPI_FUNC(PyObject *) PyMapping_Values(PyObject *o);
+
+/* On success, return a list or tuple of the items in mapping object 'o',
+   where each item is a tuple containing a key-value pair. On failure, return
+   NULL. */
+PyAPI_FUNC(PyObject *) PyMapping_Items(PyObject *o);
+
+/* Return element of 'o' corresponding to the string 'key' or NULL on failure.
+
+   This is the equivalent of the Python expression: o[key]. */
+PyAPI_FUNC(PyObject *) PyMapping_GetItemString(PyObject *o,
+                                               const char *key);
+
+/* Variants of PyObject_GetItem() and PyMapping_GetItemString() which don't
+   raise KeyError if the key is not found.
+
+   If the key is found, return 1 and set *result to a new strong
+   reference to the corresponding value.
+   If the key is not found, return 0 and set *result to NULL;
+   the KeyError is silenced.
+   If an error other than KeyError is raised, return -1 and
+   set *result to NULL.
+*/
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000
+PyAPI_FUNC(int) PyMapping_GetOptionalItem(PyObject *, PyObject *, PyObject **);
+PyAPI_FUNC(int) PyMapping_GetOptionalItemString(PyObject *, const char *, PyObject **);
+#endif
+
+/* Map the string 'key' to the value 'v' in the mapping 'o'.
+   Returns -1 on failure.
+
+   This is the equivalent of the Python statement: o[key]=v. */
+PyAPI_FUNC(int) PyMapping_SetItemString(PyObject *o, const char *key,
+                                        PyObject *value);
+
+/* isinstance(object, typeorclass) */
+PyAPI_FUNC(int) PyObject_IsInstance(PyObject *object, PyObject *typeorclass);
+
+/* issubclass(object, typeorclass) */
+PyAPI_FUNC(int) PyObject_IsSubclass(PyObject *object, PyObject *typeorclass);
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_ABSTRACTOBJECT_H
+#  include "cpython/abstract.h"
+#  undef Py_CPYTHON_ABSTRACTOBJECT_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* Py_ABSTRACTOBJECT_H */

Include/bltinmodule.h

@@ -0,0 +1,14 @@
+#ifndef Py_BLTINMODULE_H
+#define Py_BLTINMODULE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_DATA(PyTypeObject) PyFilter_Type;
+PyAPI_DATA(PyTypeObject) PyMap_Type;
+PyAPI_DATA(PyTypeObject) PyZip_Type;
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_BLTINMODULE_H */

Include/boolobject.h

@@ -0,0 +1,54 @@
+/* Boolean object interface */
+
+#ifndef Py_BOOLOBJECT_H
+#define Py_BOOLOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+// PyBool_Type is declared by object.h
+
+#define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type)
+
+/* Py_False and Py_True are the only two bools in existence. */
+
+/* Don't use these directly */
+PyAPI_DATA(PyLongObject) _Py_FalseStruct;
+PyAPI_DATA(PyLongObject) _Py_TrueStruct;
+
+/* Use these macros */
+#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030D0000
+#  define Py_False Py_GetConstantBorrowed(Py_CONSTANT_FALSE)
+#  define Py_True Py_GetConstantBorrowed(Py_CONSTANT_TRUE)
+#else
+#  define Py_False _PyObject_CAST(&_Py_FalseStruct)
+#  define Py_True _PyObject_CAST(&_Py_TrueStruct)
+#endif
+
+// Test if an object is the True singleton, the same as "x is True" in Python.
+PyAPI_FUNC(int) Py_IsTrue(PyObject *x);
+#define Py_IsTrue(x) Py_Is((x), Py_True)
+
+// Test if an object is the False singleton, the same as "x is False" in Python.
+PyAPI_FUNC(int) Py_IsFalse(PyObject *x);
+#define Py_IsFalse(x) Py_Is((x), Py_False)
+
+/* Macros for returning Py_True or Py_False, respectively.
+ * Only treat Py_True and Py_False as immortal in the limited C API 3.12
+ * and newer. */
+#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000
+#  define Py_RETURN_TRUE return Py_NewRef(Py_True)
+#  define Py_RETURN_FALSE return Py_NewRef(Py_False)
+#else
+#  define Py_RETURN_TRUE return Py_True
+#  define Py_RETURN_FALSE return Py_False
+#endif
+
+/* Function to return a bool from a C long */
+PyAPI_FUNC(PyObject *) PyBool_FromLong(long);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_BOOLOBJECT_H */

Include/bytearrayobject.h

@@ -0,0 +1,44 @@
+/* ByteArray object interface */
+
+#ifndef Py_BYTEARRAYOBJECT_H
+#define Py_BYTEARRAYOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Type PyByteArrayObject represents a mutable array of bytes.
+ * The Python API is that of a sequence;
+ * the bytes are mapped to ints in [0, 256).
+ * Bytes are not characters; they may be used to encode characters.
+ * The only way to go between bytes and str/unicode is via encoding
+ * and decoding.
+ * For the convenience of C programmers, the bytes type is considered
+ * to contain a char pointer, not an unsigned char pointer.
+ */
+
+/* Type object */
+PyAPI_DATA(PyTypeObject) PyByteArray_Type;
+PyAPI_DATA(PyTypeObject) PyByteArrayIter_Type;
+
+/* Type check macros */
+#define PyByteArray_Check(self) PyObject_TypeCheck((self), &PyByteArray_Type)
+#define PyByteArray_CheckExact(self) Py_IS_TYPE((self), &PyByteArray_Type)
+
+/* Direct API functions */
+PyAPI_FUNC(PyObject *) PyByteArray_FromObject(PyObject *);
+PyAPI_FUNC(PyObject *) PyByteArray_Concat(PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyByteArray_FromStringAndSize(const char *, Py_ssize_t);
+PyAPI_FUNC(Py_ssize_t) PyByteArray_Size(PyObject *);
+PyAPI_FUNC(char *) PyByteArray_AsString(PyObject *);
+PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t);
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_BYTEARRAYOBJECT_H
+#  include "cpython/bytearrayobject.h"
+#  undef Py_CPYTHON_BYTEARRAYOBJECT_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_BYTEARRAYOBJECT_H */

Include/bytesobject.h

@@ -0,0 +1,66 @@
+// Bytes object interface
+
+#ifndef Py_BYTESOBJECT_H
+#define Py_BYTESOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+Type PyBytesObject represents a byte string.  An extra zero byte is
+reserved at the end to ensure it is zero-terminated, but a size is
+present so strings with null bytes in them can be represented.  This
+is an immutable object type.
+
+There are functions to create new bytes objects, to test
+an object for bytes-ness, and to get the
+byte string value.  The latter function returns a null pointer
+if the object is not of the proper type.
+There is a variant that takes an explicit size as well as a
+variant that assumes a zero-terminated string.  Note that none of the
+functions should be applied to NULL pointer.
+*/
+
+PyAPI_DATA(PyTypeObject) PyBytes_Type;
+PyAPI_DATA(PyTypeObject) PyBytesIter_Type;
+
+#define PyBytes_Check(op) \
+                 PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_BYTES_SUBCLASS)
+#define PyBytes_CheckExact(op) Py_IS_TYPE((op), &PyBytes_Type)
+
+PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t);
+PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *);
+PyAPI_FUNC(PyObject *) PyBytes_FromObject(PyObject *);
+PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list)
+                                Py_GCC_ATTRIBUTE((format(printf, 1, 0)));
+PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...)
+                                Py_GCC_ATTRIBUTE((format(printf, 1, 2)));
+PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *);
+PyAPI_FUNC(char *) PyBytes_AsString(PyObject *);
+PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int);
+PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *);
+PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *);
+PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t,
+                                            const char *, Py_ssize_t,
+                                            const char *);
+
+/* Provides access to the internal data buffer and size of a bytes object.
+   Passing NULL as len parameter will force the string buffer to be
+   0-terminated (passing a string with embedded NUL characters will
+   cause an exception).  */
+PyAPI_FUNC(int) PyBytes_AsStringAndSize(
+    PyObject *obj,      /* bytes object */
+    char **s,           /* pointer to buffer variable */
+    Py_ssize_t *len     /* pointer to length variable or NULL */
+    );
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_BYTESOBJECT_H
+#  include "cpython/bytesobject.h"
+#  undef Py_CPYTHON_BYTESOBJECT_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_BYTESOBJECT_H */

Include/ceval.h

@@ -0,0 +1,145 @@
+/* Interface to random parts in ceval.c */
+
+#ifndef Py_CEVAL_H
+#define Py_CEVAL_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+PyAPI_FUNC(PyObject *) PyEval_EvalCode(PyObject *, PyObject *, PyObject *);
+
+PyAPI_FUNC(PyObject *) PyEval_EvalCodeEx(PyObject *co,
+                                         PyObject *globals,
+                                         PyObject *locals,
+                                         PyObject *const *args, int argc,
+                                         PyObject *const *kwds, int kwdc,
+                                         PyObject *const *defs, int defc,
+                                         PyObject *kwdefs, PyObject *closure);
+
+PyAPI_FUNC(PyObject *) PyEval_GetBuiltins(void);
+PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void);
+PyAPI_FUNC(PyObject *) PyEval_GetLocals(void);
+PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void);
+
+PyAPI_FUNC(PyObject *) PyEval_GetFrameBuiltins(void);
+PyAPI_FUNC(PyObject *) PyEval_GetFrameGlobals(void);
+PyAPI_FUNC(PyObject *) PyEval_GetFrameLocals(void);
+
+PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg);
+PyAPI_FUNC(int) Py_MakePendingCalls(void);
+
+/* Protection against deeply nested recursive calls
+
+   In Python 3.0, this protection has two levels:
+   * normal anti-recursion protection is triggered when the recursion level
+     exceeds the current recursion limit. It raises a RecursionError, and sets
+     the "overflowed" flag in the thread state structure. This flag
+     temporarily *disables* the normal protection; this allows cleanup code
+     to potentially outgrow the recursion limit while processing the
+     RecursionError.
+   * "last chance" anti-recursion protection is triggered when the recursion
+     level exceeds "current recursion limit + 50". By construction, this
+     protection can only be triggered when the "overflowed" flag is set. It
+     means the cleanup code has itself gone into an infinite loop, or the
+     RecursionError has been mistakingly ignored. When this protection is
+     triggered, the interpreter aborts with a Fatal Error.
+
+   In addition, the "overflowed" flag is automatically reset when the
+   recursion level drops below "current recursion limit - 50". This heuristic
+   is meant to ensure that the normal anti-recursion protection doesn't get
+   disabled too long.
+
+   Please note: this scheme has its own limitations. See:
+   http://mail.python.org/pipermail/python-dev/2008-August/082106.html
+   for some observations.
+*/
+PyAPI_FUNC(void) Py_SetRecursionLimit(int);
+PyAPI_FUNC(int) Py_GetRecursionLimit(void);
+
+PyAPI_FUNC(int) Py_EnterRecursiveCall(const char *where);
+PyAPI_FUNC(void) Py_LeaveRecursiveCall(void);
+
+PyAPI_FUNC(const char *) PyEval_GetFuncName(PyObject *);
+PyAPI_FUNC(const char *) PyEval_GetFuncDesc(PyObject *);
+
+PyAPI_FUNC(PyObject *) PyEval_EvalFrame(PyFrameObject *);
+PyAPI_FUNC(PyObject *) PyEval_EvalFrameEx(PyFrameObject *f, int exc);
+
+/* Interface for threads.
+
+   A module that plans to do a blocking system call (or something else
+   that lasts a long time and doesn't touch Python data) can allow other
+   threads to run as follows:
+
+    ...preparations here...
+    Py_BEGIN_ALLOW_THREADS
+    ...blocking system call here...
+    Py_END_ALLOW_THREADS
+    ...interpret result here...
+
+   The Py_BEGIN_ALLOW_THREADS/Py_END_ALLOW_THREADS pair expands to a
+   {}-surrounded block.
+   To leave the block in the middle (e.g., with return), you must insert
+   a line containing Py_BLOCK_THREADS before the return, e.g.
+
+    if (...premature_exit...) {
+        Py_BLOCK_THREADS
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+   An alternative is:
+
+    Py_BLOCK_THREADS
+    if (...premature_exit...) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+    Py_UNBLOCK_THREADS
+
+   For convenience, that the value of 'errno' is restored across
+   Py_END_ALLOW_THREADS and Py_BLOCK_THREADS.
+
+   WARNING: NEVER NEST CALLS TO Py_BEGIN_ALLOW_THREADS AND
+   Py_END_ALLOW_THREADS!!!
+
+   Note that not yet all candidates have been converted to use this
+   mechanism!
+*/
+
+PyAPI_FUNC(PyThreadState *) PyEval_SaveThread(void);
+PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *);
+
+Py_DEPRECATED(3.9) PyAPI_FUNC(void) PyEval_InitThreads(void);
+
+PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate);
+PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate);
+
+#define Py_BEGIN_ALLOW_THREADS { \
+                        PyThreadState *_save; \
+                        _save = PyEval_SaveThread();
+#define Py_BLOCK_THREADS        PyEval_RestoreThread(_save);
+#define Py_UNBLOCK_THREADS      _save = PyEval_SaveThread();
+#define Py_END_ALLOW_THREADS    PyEval_RestoreThread(_save); \
+                 }
+
+/* Masks and values used by FORMAT_VALUE opcode. */
+#define FVC_MASK      0x3
+#define FVC_NONE      0x0
+#define FVC_STR       0x1
+#define FVC_REPR      0x2
+#define FVC_ASCII     0x3
+#define FVS_MASK      0x4
+#define FVS_HAVE_SPEC 0x4
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_CEVAL_H
+#  include "cpython/ceval.h"
+#  undef Py_CPYTHON_CEVAL_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CEVAL_H */

Include/codecs.h

@@ -0,0 +1,176 @@
+#ifndef Py_CODECREGISTRY_H
+#define Py_CODECREGISTRY_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ------------------------------------------------------------------------
+
+   Python Codec Registry and support functions
+
+
+Written by Marc-Andre Lemburg (mal@lemburg.com).
+
+Copyright (c) Corporation for National Research Initiatives.
+
+   ------------------------------------------------------------------------ */
+
+/* Register a new codec search function.
+
+   As side effect, this tries to load the encodings package, if not
+   yet done, to make sure that it is always first in the list of
+   search functions.
+
+   The search_function's refcount is incremented by this function. */
+
+PyAPI_FUNC(int) PyCodec_Register(
+       PyObject *search_function
+       );
+
+/* Unregister a codec search function and clear the registry's cache.
+   If the search function is not registered, do nothing.
+   Return 0 on success. Raise an exception and return -1 on error. */
+
+PyAPI_FUNC(int) PyCodec_Unregister(
+       PyObject *search_function
+       );
+
+/* Codec registry encoding check API.
+
+   Returns 1/0 depending on whether there is a registered codec for
+   the given encoding.
+
+*/
+
+PyAPI_FUNC(int) PyCodec_KnownEncoding(
+       const char *encoding
+       );
+
+/* Generic codec based encoding API.
+
+   object is passed through the encoder function found for the given
+   encoding using the error handling method defined by errors. errors
+   may be NULL to use the default method defined for the codec.
+
+   Raises a LookupError in case no encoder can be found.
+
+ */
+
+PyAPI_FUNC(PyObject *) PyCodec_Encode(
+       PyObject *object,
+       const char *encoding,
+       const char *errors
+       );
+
+/* Generic codec based decoding API.
+
+   object is passed through the decoder function found for the given
+   encoding using the error handling method defined by errors. errors
+   may be NULL to use the default method defined for the codec.
+
+   Raises a LookupError in case no encoder can be found.
+
+ */
+
+PyAPI_FUNC(PyObject *) PyCodec_Decode(
+       PyObject *object,
+       const char *encoding,
+       const char *errors
+       );
+
+// --- Codec Lookup APIs --------------------------------------------------
+
+/* Codec registry lookup API.
+
+   Looks up the given encoding and returns a CodecInfo object with
+   function attributes which implement the different aspects of
+   processing the encoding.
+
+   The encoding string is looked up converted to all lower-case
+   characters. This makes encodings looked up through this mechanism
+   effectively case-insensitive.
+
+   If no codec is found, a KeyError is set and NULL returned.
+
+   As side effect, this tries to load the encodings package, if not
+   yet done. This is part of the lazy load strategy for the encodings
+   package.
+ */
+
+/* Get an encoder function for the given encoding. */
+
+PyAPI_FUNC(PyObject *) PyCodec_Encoder(const char *encoding);
+
+/* Get a decoder function for the given encoding. */
+
+PyAPI_FUNC(PyObject *) PyCodec_Decoder(const char *encoding);
+
+/* Get an IncrementalEncoder object for the given encoding. */
+
+PyAPI_FUNC(PyObject *) PyCodec_IncrementalEncoder(
+   const char *encoding,
+   const char *errors);
+
+/* Get an IncrementalDecoder object function for the given encoding. */
+
+PyAPI_FUNC(PyObject *) PyCodec_IncrementalDecoder(
+   const char *encoding,
+   const char *errors);
+
+/* Get a StreamReader factory function for the given encoding. */
+
+PyAPI_FUNC(PyObject *) PyCodec_StreamReader(
+   const char *encoding,
+   PyObject *stream,
+   const char *errors);
+
+/* Get a StreamWriter factory function for the given encoding. */
+
+PyAPI_FUNC(PyObject *) PyCodec_StreamWriter(
+   const char *encoding,
+   PyObject *stream,
+   const char *errors);
+
+/* Unicode encoding error handling callback registry API */
+
+/* Register the error handling callback function error under the given
+   name. This function will be called by the codec when it encounters
+   unencodable characters/undecodable bytes and doesn't know the
+   callback name, when name is specified as the error parameter
+   in the call to the encode/decode function.
+   Return 0 on success, -1 on error */
+PyAPI_FUNC(int) PyCodec_RegisterError(const char *name, PyObject *error);
+
+/* Lookup the error handling callback function registered under the given
+   name. As a special case NULL can be passed, in which case
+   the error handling callback for "strict" will be returned. */
+PyAPI_FUNC(PyObject *) PyCodec_LookupError(const char *name);
+
+/* raise exc as an exception */
+PyAPI_FUNC(PyObject *) PyCodec_StrictErrors(PyObject *exc);
+
+/* ignore the unicode error, skipping the faulty input */
+PyAPI_FUNC(PyObject *) PyCodec_IgnoreErrors(PyObject *exc);
+
+/* replace the unicode encode error with ? or U+FFFD */
+PyAPI_FUNC(PyObject *) PyCodec_ReplaceErrors(PyObject *exc);
+
+/* replace the unicode encode error with XML character references */
+PyAPI_FUNC(PyObject *) PyCodec_XMLCharRefReplaceErrors(PyObject *exc);
+
+/* replace the unicode encode error with backslash escapes (\x, \u and \U) */
+PyAPI_FUNC(PyObject *) PyCodec_BackslashReplaceErrors(PyObject *exc);
+
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* replace the unicode encode error with backslash escapes (\N, \x, \u and \U) */
+PyAPI_FUNC(PyObject *) PyCodec_NameReplaceErrors(PyObject *exc);
+#endif
+
+#ifndef Py_LIMITED_API
+PyAPI_DATA(const char *) Py_hexdigits;
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CODECREGISTRY_H */

Include/compile.h

@@ -0,0 +1,22 @@
+#ifndef Py_COMPILE_H
+#define Py_COMPILE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* These definitions must match corresponding definitions in graminit.h. */
+#define Py_single_input 256
+#define Py_file_input 257
+#define Py_eval_input 258
+#define Py_func_type_input 345
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_COMPILE_H
+#  include "cpython/compile.h"
+#  undef Py_CPYTHON_COMPILE_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_COMPILE_H */

Include/complexobject.h

@@ -0,0 +1,30 @@
+/* Complex number structure */
+
+#ifndef Py_COMPLEXOBJECT_H
+#define Py_COMPLEXOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Complex object interface */
+
+PyAPI_DATA(PyTypeObject) PyComplex_Type;
+
+#define PyComplex_Check(op) PyObject_TypeCheck((op), &PyComplex_Type)
+#define PyComplex_CheckExact(op) Py_IS_TYPE((op), &PyComplex_Type)
+
+PyAPI_FUNC(PyObject *) PyComplex_FromDoubles(double real, double imag);
+
+PyAPI_FUNC(double) PyComplex_RealAsDouble(PyObject *op);
+PyAPI_FUNC(double) PyComplex_ImagAsDouble(PyObject *op);
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_COMPLEXOBJECT_H
+#  include "cpython/complexobject.h"
+#  undef Py_CPYTHON_COMPLEXOBJECT_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_COMPLEXOBJECT_H */

Include/cpython/abstract.h

@@ -0,0 +1,87 @@
+#ifndef Py_CPYTHON_ABSTRACTOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+/* === Object Protocol ================================================== */
+
+/* Like PyObject_CallMethod(), but expect a _Py_Identifier*
+   as the method name. */
+PyAPI_FUNC(PyObject*) _PyObject_CallMethodId(
+    PyObject *obj,
+    _Py_Identifier *name,
+    const char *format, ...);
+
+/* Convert keyword arguments from the FASTCALL (stack: C array, kwnames: tuple)
+   format to a Python dictionary ("kwargs" dict).
+
+   The type of kwnames keys is not checked. The final function getting
+   arguments is responsible to check if all keys are strings, for example using
+   PyArg_ParseTupleAndKeywords() or PyArg_ValidateKeywordArguments().
+
+   Duplicate keys are merged using the last value. If duplicate keys must raise
+   an exception, the caller is responsible to implement an explicit keys on
+   kwnames. */
+PyAPI_FUNC(PyObject*) _PyStack_AsDict(PyObject *const *values, PyObject *kwnames);
+
+
+/* === Vectorcall protocol (PEP 590) ============================= */
+
+// PyVectorcall_NARGS() is exported as a function for the stable ABI.
+// Here (when we are not using the stable ABI), the name is overridden to
+// call a static inline function for best performance.
+static inline Py_ssize_t
+_PyVectorcall_NARGS(size_t n)
+{
+    return n & ~PY_VECTORCALL_ARGUMENTS_OFFSET;
+}
+#define PyVectorcall_NARGS(n) _PyVectorcall_NARGS(n)
+
+PyAPI_FUNC(vectorcallfunc) PyVectorcall_Function(PyObject *callable);
+
+// Backwards compatibility aliases (PEP 590) for API that was provisional
+// in Python 3.8
+#define _PyObject_Vectorcall PyObject_Vectorcall
+#define _PyObject_VectorcallMethod PyObject_VectorcallMethod
+#define _PyObject_FastCallDict PyObject_VectorcallDict
+#define _PyVectorcall_Function PyVectorcall_Function
+#define _PyObject_CallOneArg PyObject_CallOneArg
+#define _PyObject_CallMethodNoArgs PyObject_CallMethodNoArgs
+#define _PyObject_CallMethodOneArg PyObject_CallMethodOneArg
+
+/* Same as PyObject_Vectorcall except that keyword arguments are passed as
+   dict, which may be NULL if there are no keyword arguments. */
+PyAPI_FUNC(PyObject *) PyObject_VectorcallDict(
+    PyObject *callable,
+    PyObject *const *args,
+    size_t nargsf,
+    PyObject *kwargs);
+
+PyAPI_FUNC(PyObject *) PyObject_CallOneArg(PyObject *func, PyObject *arg);
+
+static inline PyObject *
+PyObject_CallMethodNoArgs(PyObject *self, PyObject *name)
+{
+    size_t nargsf = 1 | PY_VECTORCALL_ARGUMENTS_OFFSET;
+    return PyObject_VectorcallMethod(name, &self, nargsf, _Py_NULL);
+}
+
+static inline PyObject *
+PyObject_CallMethodOneArg(PyObject *self, PyObject *name, PyObject *arg)
+{
+    PyObject *args[2] = {self, arg};
+    size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET;
+    assert(arg != NULL);
+    return PyObject_VectorcallMethod(name, args, nargsf, _Py_NULL);
+}
+
+/* Guess the size of object 'o' using len(o) or o.__length_hint__().
+   If neither of those return a non-negative value, then return the default
+   value.  If one of the calls fails, this function returns -1. */
+PyAPI_FUNC(Py_ssize_t) PyObject_LengthHint(PyObject *o, Py_ssize_t);
+
+/* === Sequence protocol ================================================ */
+
+/* Assume tp_as_sequence and sq_item exist and that 'i' does not
+   need to be corrected for a negative index. */
+#define PySequence_ITEM(o, i)\
+    ( Py_TYPE(o)->tp_as_sequence->sq_item((o), (i)) )

Include/cpython/bytearrayobject.h

@@ -0,0 +1,34 @@
+#ifndef Py_CPYTHON_BYTEARRAYOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Object layout */
+typedef struct {
+    PyObject_VAR_HEAD
+    Py_ssize_t ob_alloc;   /* How many bytes allocated in ob_bytes */
+    char *ob_bytes;        /* Physical backing buffer */
+    char *ob_start;        /* Logical start inside ob_bytes */
+    Py_ssize_t ob_exports; /* How many buffer exports */
+} PyByteArrayObject;
+
+PyAPI_DATA(char) _PyByteArray_empty_string[];
+
+/* Macros and static inline functions, trading safety for speed */
+#define _PyByteArray_CAST(op) \
+    (assert(PyByteArray_Check(op)), _Py_CAST(PyByteArrayObject*, op))
+
+static inline char* PyByteArray_AS_STRING(PyObject *op)
+{
+    PyByteArrayObject *self = _PyByteArray_CAST(op);
+    if (Py_SIZE(self)) {
+        return self->ob_start;
+    }
+    return _PyByteArray_empty_string;
+}
+#define PyByteArray_AS_STRING(self) PyByteArray_AS_STRING(_PyObject_CAST(self))
+
+static inline Py_ssize_t PyByteArray_GET_SIZE(PyObject *op) {
+    PyByteArrayObject *self = _PyByteArray_CAST(op);
+    return Py_SIZE(self);
+}
+#define PyByteArray_GET_SIZE(self) PyByteArray_GET_SIZE(_PyObject_CAST(self))

Include/cpython/bytesobject.h

@@ -0,0 +1,37 @@
+#ifndef Py_CPYTHON_BYTESOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+typedef struct {
+    PyObject_VAR_HEAD
+    Py_DEPRECATED(3.11) Py_hash_t ob_shash;
+    char ob_sval[1];
+
+    /* Invariants:
+     *     ob_sval contains space for 'ob_size+1' elements.
+     *     ob_sval[ob_size] == 0.
+     *     ob_shash is the hash of the byte string or -1 if not computed yet.
+     */
+} PyBytesObject;
+
+PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t);
+
+/* Macros and static inline functions, trading safety for speed */
+#define _PyBytes_CAST(op) \
+    (assert(PyBytes_Check(op)), _Py_CAST(PyBytesObject*, op))
+
+static inline char* PyBytes_AS_STRING(PyObject *op)
+{
+    return _PyBytes_CAST(op)->ob_sval;
+}
+#define PyBytes_AS_STRING(op) PyBytes_AS_STRING(_PyObject_CAST(op))
+
+static inline Py_ssize_t PyBytes_GET_SIZE(PyObject *op) {
+    PyBytesObject *self = _PyBytes_CAST(op);
+    return Py_SIZE(self);
+}
+#define PyBytes_GET_SIZE(self) PyBytes_GET_SIZE(_PyObject_CAST(self))
+
+/* _PyBytes_Join(sep, x) is like sep.join(x).  sep must be PyBytesObject*,
+   x must be an iterable object. */
+PyAPI_FUNC(PyObject*) _PyBytes_Join(PyObject *sep, PyObject *x);

Include/cpython/cellobject.h

@@ -0,0 +1,44 @@
+/* Cell object interface */
+
+#ifndef Py_LIMITED_API
+#ifndef Py_CELLOBJECT_H
+#define Py_CELLOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    /* Content of the cell or NULL when empty */
+    PyObject *ob_ref;
+} PyCellObject;
+
+PyAPI_DATA(PyTypeObject) PyCell_Type;
+
+#define PyCell_Check(op) Py_IS_TYPE((op), &PyCell_Type)
+
+PyAPI_FUNC(PyObject *) PyCell_New(PyObject *);
+PyAPI_FUNC(PyObject *) PyCell_Get(PyObject *);
+PyAPI_FUNC(int) PyCell_Set(PyObject *, PyObject *);
+
+static inline PyObject* PyCell_GET(PyObject *op) {
+    PyCellObject *cell;
+    assert(PyCell_Check(op));
+    cell = _Py_CAST(PyCellObject*, op);
+    return cell->ob_ref;
+}
+#define PyCell_GET(op) PyCell_GET(_PyObject_CAST(op))
+
+static inline void PyCell_SET(PyObject *op, PyObject *value) {
+    PyCellObject *cell;
+    assert(PyCell_Check(op));
+    cell = _Py_CAST(PyCellObject*, op);
+    cell->ob_ref = value;
+}
+#define PyCell_SET(op, value) PyCell_SET(_PyObject_CAST(op), (value))
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_TUPLEOBJECT_H */
+#endif /* Py_LIMITED_API */

Include/cpython/ceval.h

@@ -0,0 +1,25 @@
+#ifndef Py_CPYTHON_CEVAL_H
+#  error "this header file must not be included directly"
+#endif
+
+PyAPI_FUNC(void) PyEval_SetProfile(Py_tracefunc, PyObject *);
+PyAPI_FUNC(void) PyEval_SetProfileAllThreads(Py_tracefunc, PyObject *);
+PyAPI_FUNC(void) PyEval_SetTrace(Py_tracefunc, PyObject *);
+PyAPI_FUNC(void) PyEval_SetTraceAllThreads(Py_tracefunc, PyObject *);
+
+/* Look at the current frame's (if any) code's co_flags, and turn on
+   the corresponding compiler flags in cf->cf_flags.  Return 1 if any
+   flag was set, else return 0. */
+PyAPI_FUNC(int) PyEval_MergeCompilerFlags(PyCompilerFlags *cf);
+
+PyAPI_FUNC(PyObject *) _PyEval_EvalFrameDefault(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc);
+
+PyAPI_FUNC(Py_ssize_t) PyUnstable_Eval_RequestCodeExtraIndex(freefunc);
+// Old name -- remove when this API changes:
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline Py_ssize_t
+_PyEval_RequestCodeExtraIndex(freefunc f) {
+    return PyUnstable_Eval_RequestCodeExtraIndex(f);
+}
+
+PyAPI_FUNC(int) _PyEval_SliceIndex(PyObject *, Py_ssize_t *);
+PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *);

Include/cpython/classobject.h

@@ -0,0 +1,71 @@
+/* Former class object interface -- now only bound methods are here  */
+
+/* Revealing some structures (not for general use) */
+
+#ifndef Py_LIMITED_API
+#ifndef Py_CLASSOBJECT_H
+#define Py_CLASSOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *im_func;   /* The callable object implementing the method */
+    PyObject *im_self;   /* The instance it is bound to */
+    PyObject *im_weakreflist; /* List of weak references */
+    vectorcallfunc vectorcall;
+} PyMethodObject;
+
+PyAPI_DATA(PyTypeObject) PyMethod_Type;
+
+#define PyMethod_Check(op) Py_IS_TYPE((op), &PyMethod_Type)
+
+PyAPI_FUNC(PyObject *) PyMethod_New(PyObject *, PyObject *);
+
+PyAPI_FUNC(PyObject *) PyMethod_Function(PyObject *);
+PyAPI_FUNC(PyObject *) PyMethod_Self(PyObject *);
+
+#define _PyMethod_CAST(meth) \
+    (assert(PyMethod_Check(meth)), _Py_CAST(PyMethodObject*, meth))
+
+/* Static inline functions for direct access to these values.
+   Type checks are *not* done, so use with care. */
+static inline PyObject* PyMethod_GET_FUNCTION(PyObject *meth) {
+    return _PyMethod_CAST(meth)->im_func;
+}
+#define PyMethod_GET_FUNCTION(meth) PyMethod_GET_FUNCTION(_PyObject_CAST(meth))
+
+static inline PyObject* PyMethod_GET_SELF(PyObject *meth) {
+    return _PyMethod_CAST(meth)->im_self;
+}
+#define PyMethod_GET_SELF(meth) PyMethod_GET_SELF(_PyObject_CAST(meth))
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *func;
+} PyInstanceMethodObject;
+
+PyAPI_DATA(PyTypeObject) PyInstanceMethod_Type;
+
+#define PyInstanceMethod_Check(op) Py_IS_TYPE((op), &PyInstanceMethod_Type)
+
+PyAPI_FUNC(PyObject *) PyInstanceMethod_New(PyObject *);
+PyAPI_FUNC(PyObject *) PyInstanceMethod_Function(PyObject *);
+
+#define _PyInstanceMethod_CAST(meth) \
+    (assert(PyInstanceMethod_Check(meth)), \
+     _Py_CAST(PyInstanceMethodObject*, meth))
+
+/* Static inline function for direct access to these values.
+   Type checks are *not* done, so use with care. */
+static inline PyObject* PyInstanceMethod_GET_FUNCTION(PyObject *meth) {
+    return _PyInstanceMethod_CAST(meth)->func;
+}
+#define PyInstanceMethod_GET_FUNCTION(meth) PyInstanceMethod_GET_FUNCTION(_PyObject_CAST(meth))
+
+#ifdef __cplusplus
+}
+#endif
+#endif   // !Py_CLASSOBJECT_H
+#endif   // !Py_LIMITED_API

Include/cpython/code.h

@@ -0,0 +1,358 @@
+/* Definitions for bytecode */
+
+#ifndef Py_LIMITED_API
+#ifndef Py_CODE_H
+#define Py_CODE_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Count of all local monitoring events */
+#define  _PY_MONITORING_LOCAL_EVENTS 10
+/* Count of all "real" monitoring events (not derived from other events) */
+#define _PY_MONITORING_UNGROUPED_EVENTS 15
+/* Count of all  monitoring events */
+#define _PY_MONITORING_EVENTS 17
+
+/* Tables of which tools are active for each monitored event. */
+typedef struct _Py_LocalMonitors {
+    uint8_t tools[_PY_MONITORING_LOCAL_EVENTS];
+} _Py_LocalMonitors;
+
+typedef struct _Py_GlobalMonitors {
+    uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS];
+} _Py_GlobalMonitors;
+
+
+typedef struct {
+    PyObject *_co_code;
+    PyObject *_co_varnames;
+    PyObject *_co_cellvars;
+    PyObject *_co_freevars;
+} _PyCoCached;
+
+/* Ancillary data structure used for instrumentation.
+   Line instrumentation creates this with sufficient
+   space for one entry per code unit. The total size
+   of the data will be `bytes_per_entry * Py_SIZE(code)` */
+typedef struct {
+    uint8_t bytes_per_entry;
+    uint8_t data[1];
+} _PyCoLineInstrumentationData;
+
+
+typedef struct {
+    int size;
+    int capacity;
+    struct _PyExecutorObject *executors[1];
+} _PyExecutorArray;
+
+/* Main data structure used for instrumentation.
+ * This is allocated when needed for instrumentation
+ */
+typedef struct {
+    /* Monitoring specific to this code object */
+    _Py_LocalMonitors local_monitors;
+    /* Monitoring that is active on this code object */
+    _Py_LocalMonitors active_monitors;
+    /* The tools that are to be notified for events for the matching code unit */
+    uint8_t *tools;
+    /* Information to support line events */
+    _PyCoLineInstrumentationData *lines;
+    /* The tools that are to be notified for line events for the matching code unit */
+    uint8_t *line_tools;
+    /* Information to support instruction events */
+    /* The underlying instructions, which can themselves be instrumented */
+    uint8_t *per_instruction_opcodes;
+    /* The tools that are to be notified for instruction events for the matching code unit */
+    uint8_t *per_instruction_tools;
+} _PyCoMonitoringData;
+
+// To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are
+// defined in this macro:
+#define _PyCode_DEF(SIZE) {                                                    \
+    PyObject_VAR_HEAD                                                          \
+                                                                               \
+    /* Note only the following fields are used in hash and/or comparisons      \
+     *                                                                         \
+     * - co_name                                                               \
+     * - co_argcount                                                           \
+     * - co_posonlyargcount                                                    \
+     * - co_kwonlyargcount                                                     \
+     * - co_nlocals                                                            \
+     * - co_stacksize                                                          \
+     * - co_flags                                                              \
+     * - co_firstlineno                                                        \
+     * - co_consts                                                             \
+     * - co_names                                                              \
+     * - co_localsplusnames                                                    \
+     * This is done to preserve the name and line number for tracebacks        \
+     * and debuggers; otherwise, constant de-duplication would collapse        \
+     * identical functions/lambdas defined on different lines.                 \
+     */                                                                        \
+                                                                               \
+    /* These fields are set with provided values on new code objects. */       \
+                                                                               \
+    /* The hottest fields (in the eval loop) are grouped here at the top. */   \
+    PyObject *co_consts;           /* list (constants used) */                 \
+    PyObject *co_names;            /* list of strings (names used) */          \
+    PyObject *co_exceptiontable;   /* Byte string encoding exception handling  \
+                                      table */                                 \
+    int co_flags;                  /* CO_..., see below */                     \
+                                                                               \
+    /* The rest are not so impactful on performance. */                        \
+    int co_argcount;              /* #arguments, except *args */               \
+    int co_posonlyargcount;       /* #positional only arguments */             \
+    int co_kwonlyargcount;        /* #keyword only arguments */                \
+    int co_stacksize;             /* #entries needed for evaluation stack */   \
+    int co_firstlineno;           /* first source line number */               \
+                                                                               \
+    /* redundant values (derived from co_localsplusnames and                   \
+       co_localspluskinds) */                                                  \
+    int co_nlocalsplus;           /* number of local + cell + free variables */ \
+    int co_framesize;             /* Size of frame in words */                 \
+    int co_nlocals;               /* number of local variables */              \
+    int co_ncellvars;             /* total number of cell variables */         \
+    int co_nfreevars;             /* number of free variables */               \
+    uint32_t co_version;          /* version number */                         \
+                                                                               \
+    PyObject *co_localsplusnames; /* tuple mapping offsets to names */         \
+    PyObject *co_localspluskinds; /* Bytes mapping to local kinds (one byte    \
+                                     per variable) */                          \
+    PyObject *co_filename;        /* unicode (where it was loaded from) */     \
+    PyObject *co_name;            /* unicode (name, for reference) */          \
+    PyObject *co_qualname;        /* unicode (qualname, for reference) */      \
+    PyObject *co_linetable;       /* bytes object that holds location info */  \
+    PyObject *co_weakreflist;     /* to support weakrefs to code objects */    \
+    _PyExecutorArray *co_executors;      /* executors from optimizer */        \
+    _PyCoCached *_co_cached;      /* cached co_* attributes */                 \
+    uintptr_t _co_instrumentation_version; /* current instrumentation version */ \
+    _PyCoMonitoringData *_co_monitoring; /* Monitoring data */                 \
+    int _co_firsttraceable;       /* index of first traceable instruction */   \
+    /* Scratch space for extra data relating to the code object.               \
+       Type is a void* to keep the format private in codeobject.c to force     \
+       people to go through the proper APIs. */                                \
+    void *co_extra;                                                            \
+    char co_code_adaptive[(SIZE)];                                             \
+}
+
+/* Bytecode object */
+struct PyCodeObject _PyCode_DEF(1);
+
+/* Masks for co_flags above */
+#define CO_OPTIMIZED    0x0001
+#define CO_NEWLOCALS    0x0002
+#define CO_VARARGS      0x0004
+#define CO_VARKEYWORDS  0x0008
+#define CO_NESTED       0x0010
+#define CO_GENERATOR    0x0020
+
+/* The CO_COROUTINE flag is set for coroutine functions (defined with
+   ``async def`` keywords) */
+#define CO_COROUTINE            0x0080
+#define CO_ITERABLE_COROUTINE   0x0100
+#define CO_ASYNC_GENERATOR      0x0200
+
+/* bpo-39562: These constant values are changed in Python 3.9
+   to prevent collision with compiler flags. CO_FUTURE_ and PyCF_
+   constants must be kept unique. PyCF_ constants can use bits from
+   0x0100 to 0x10000. CO_FUTURE_ constants use bits starting at 0x20000. */
+#define CO_FUTURE_DIVISION      0x20000
+#define CO_FUTURE_ABSOLUTE_IMPORT 0x40000 /* do absolute imports by default */
+#define CO_FUTURE_WITH_STATEMENT  0x80000
+#define CO_FUTURE_PRINT_FUNCTION  0x100000
+#define CO_FUTURE_UNICODE_LITERALS 0x200000
+
+#define CO_FUTURE_BARRY_AS_BDFL  0x400000
+#define CO_FUTURE_GENERATOR_STOP  0x800000
+#define CO_FUTURE_ANNOTATIONS    0x1000000
+
+#define CO_NO_MONITORING_EVENTS 0x2000000
+
+/* This should be defined if a future statement modifies the syntax.
+   For example, when a keyword is added.
+*/
+#define PY_PARSER_REQUIRES_FUTURE_KEYWORD
+
+#define CO_MAXBLOCKS 21 /* Max static block nesting within a function */
+
+PyAPI_DATA(PyTypeObject) PyCode_Type;
+
+#define PyCode_Check(op) Py_IS_TYPE((op), &PyCode_Type)
+
+static inline Py_ssize_t PyCode_GetNumFree(PyCodeObject *op) {
+    assert(PyCode_Check(op));
+    return op->co_nfreevars;
+}
+
+static inline int PyUnstable_Code_GetFirstFree(PyCodeObject *op) {
+    assert(PyCode_Check(op));
+    return op->co_nlocalsplus - op->co_nfreevars;
+}
+
+Py_DEPRECATED(3.13) static inline int PyCode_GetFirstFree(PyCodeObject *op) {
+    return PyUnstable_Code_GetFirstFree(op);
+}
+
+/* Unstable public interface */
+PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_New(
+        int, int, int, int, int, PyObject *, PyObject *,
+        PyObject *, PyObject *, PyObject *, PyObject *,
+        PyObject *, PyObject *, PyObject *, int, PyObject *,
+        PyObject *);
+
+PyAPI_FUNC(PyCodeObject *) PyUnstable_Code_NewWithPosOnlyArgs(
+        int, int, int, int, int, int, PyObject *, PyObject *,
+        PyObject *, PyObject *, PyObject *, PyObject *,
+        PyObject *, PyObject *, PyObject *, int, PyObject *,
+        PyObject *);
+        /* same as struct above */
+// Old names -- remove when this API changes:
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline PyCodeObject *
+PyCode_New(
+        int a, int b, int c, int d, int e, PyObject *f, PyObject *g,
+        PyObject *h, PyObject *i, PyObject *j, PyObject *k,
+        PyObject *l, PyObject *m, PyObject *n, int o, PyObject *p,
+        PyObject *q)
+{
+    return PyUnstable_Code_New(
+        a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q);
+}
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline PyCodeObject *
+PyCode_NewWithPosOnlyArgs(
+        int a, int poac, int b, int c, int d, int e, PyObject *f, PyObject *g,
+        PyObject *h, PyObject *i, PyObject *j, PyObject *k,
+        PyObject *l, PyObject *m, PyObject *n, int o, PyObject *p,
+        PyObject *q)
+{
+    return PyUnstable_Code_NewWithPosOnlyArgs(
+        a, poac, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q);
+}
+
+/* Creates a new empty code object with the specified source location. */
+PyAPI_FUNC(PyCodeObject *)
+PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno);
+
+/* Return the line number associated with the specified bytecode index
+   in this code object.  If you just need the line number of a frame,
+   use PyFrame_GetLineNumber() instead. */
+PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
+
+PyAPI_FUNC(int) PyCode_Addr2Location(PyCodeObject *, int, int *, int *, int *, int *);
+
+#define PY_FOREACH_CODE_EVENT(V) \
+    V(CREATE)                 \
+    V(DESTROY)
+
+typedef enum {
+    #define PY_DEF_EVENT(op) PY_CODE_EVENT_##op,
+    PY_FOREACH_CODE_EVENT(PY_DEF_EVENT)
+    #undef PY_DEF_EVENT
+} PyCodeEvent;
+
+
+/*
+ * A callback that is invoked for different events in a code object's lifecycle.
+ *
+ * The callback is invoked with a borrowed reference to co, after it is
+ * created and before it is destroyed.
+ *
+ * If the callback sets an exception, it must return -1. Otherwise
+ * it should return 0.
+ */
+typedef int (*PyCode_WatchCallback)(
+  PyCodeEvent event,
+  PyCodeObject* co);
+
+/*
+ * Register a per-interpreter callback that will be invoked for code object
+ * lifecycle events.
+ *
+ * Returns a handle that may be passed to PyCode_ClearWatcher on success,
+ * or -1 and sets an error if no more handles are available.
+ */
+PyAPI_FUNC(int) PyCode_AddWatcher(PyCode_WatchCallback callback);
+
+/*
+ * Clear the watcher associated with the watcher_id handle.
+ *
+ * Returns 0 on success or -1 if no watcher exists for the provided id.
+ */
+PyAPI_FUNC(int) PyCode_ClearWatcher(int watcher_id);
+
+/* for internal use only */
+struct _opaque {
+    int computed_line;
+    const uint8_t *lo_next;
+    const uint8_t *limit;
+};
+
+typedef struct _line_offsets {
+    int ar_start;
+    int ar_end;
+    int ar_line;
+    struct _opaque opaque;
+} PyCodeAddressRange;
+
+/* Update *bounds to describe the first and one-past-the-last instructions in the
+   same line as lasti.  Return the number of that line.
+*/
+PyAPI_FUNC(int) _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds);
+
+/* Create a comparable key used to compare constants taking in account the
+ * object type. It is used to make sure types are not coerced (e.g., float and
+ * complex) _and_ to distinguish 0.0 from -0.0 e.g. on IEEE platforms
+ *
+ * Return (type(obj), obj, ...): a tuple with variable size (at least 2 items)
+ * depending on the type and the value. The type is the first item to not
+ * compare bytes and str which can raise a BytesWarning exception. */
+PyAPI_FUNC(PyObject*) _PyCode_ConstantKey(PyObject *obj);
+
+PyAPI_FUNC(PyObject*) PyCode_Optimize(PyObject *code, PyObject* consts,
+                                      PyObject *names, PyObject *lnotab);
+
+PyAPI_FUNC(int) PyUnstable_Code_GetExtra(
+    PyObject *code, Py_ssize_t index, void **extra);
+PyAPI_FUNC(int) PyUnstable_Code_SetExtra(
+    PyObject *code, Py_ssize_t index, void *extra);
+// Old names -- remove when this API changes:
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline int
+_PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra)
+{
+    return PyUnstable_Code_GetExtra(code, index, extra);
+}
+_Py_DEPRECATED_EXTERNALLY(3.12) static inline int
+_PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra)
+{
+    return PyUnstable_Code_SetExtra(code, index, extra);
+}
+
+/* Equivalent to getattr(code, 'co_code') in Python.
+   Returns a strong reference to a bytes object. */
+PyAPI_FUNC(PyObject *) PyCode_GetCode(PyCodeObject *code);
+/* Equivalent to getattr(code, 'co_varnames') in Python. */
+PyAPI_FUNC(PyObject *) PyCode_GetVarnames(PyCodeObject *code);
+/* Equivalent to getattr(code, 'co_cellvars') in Python. */
+PyAPI_FUNC(PyObject *) PyCode_GetCellvars(PyCodeObject *code);
+/* Equivalent to getattr(code, 'co_freevars') in Python. */
+PyAPI_FUNC(PyObject *) PyCode_GetFreevars(PyCodeObject *code);
+
+typedef enum _PyCodeLocationInfoKind {
+    /* short forms are 0 to 9 */
+    PY_CODE_LOCATION_INFO_SHORT0 = 0,
+    /* one lineforms are 10 to 12 */
+    PY_CODE_LOCATION_INFO_ONE_LINE0 = 10,
+    PY_CODE_LOCATION_INFO_ONE_LINE1 = 11,
+    PY_CODE_LOCATION_INFO_ONE_LINE2 = 12,
+
+    PY_CODE_LOCATION_INFO_NO_COLUMNS = 13,
+    PY_CODE_LOCATION_INFO_LONG = 14,
+    PY_CODE_LOCATION_INFO_NONE = 15
+} _PyCodeLocationInfoKind;
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // !Py_CODE_H
+#endif  // !Py_LIMITED_API

Include/cpython/compile.h

@@ -0,0 +1,50 @@
+#ifndef Py_CPYTHON_COMPILE_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Public interface */
+#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
+                   CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
+                   CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
+                   CO_FUTURE_GENERATOR_STOP | CO_FUTURE_ANNOTATIONS)
+#define PyCF_MASK_OBSOLETE (CO_NESTED)
+
+/* bpo-39562: CO_FUTURE_ and PyCF_ constants must be kept unique.
+   PyCF_ constants can use bits from 0x0100 to 0x10000.
+   CO_FUTURE_ constants use bits starting at 0x20000. */
+#define PyCF_SOURCE_IS_UTF8  0x0100
+#define PyCF_DONT_IMPLY_DEDENT 0x0200
+#define PyCF_ONLY_AST 0x0400
+#define PyCF_IGNORE_COOKIE 0x0800
+#define PyCF_TYPE_COMMENTS 0x1000
+#define PyCF_ALLOW_TOP_LEVEL_AWAIT 0x2000
+#define PyCF_ALLOW_INCOMPLETE_INPUT 0x4000
+#define PyCF_OPTIMIZED_AST (0x8000 | PyCF_ONLY_AST)
+#define PyCF_COMPILE_MASK (PyCF_ONLY_AST | PyCF_ALLOW_TOP_LEVEL_AWAIT | \
+                           PyCF_TYPE_COMMENTS | PyCF_DONT_IMPLY_DEDENT | \
+                           PyCF_ALLOW_INCOMPLETE_INPUT | PyCF_OPTIMIZED_AST)
+
+typedef struct {
+    int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
+    int cf_feature_version;  /* minor Python version (PyCF_ONLY_AST) */
+} PyCompilerFlags;
+
+#define _PyCompilerFlags_INIT \
+    (PyCompilerFlags){.cf_flags = 0, .cf_feature_version = PY_MINOR_VERSION}
+
+/* Future feature support */
+
+#define FUTURE_NESTED_SCOPES "nested_scopes"
+#define FUTURE_GENERATORS "generators"
+#define FUTURE_DIVISION "division"
+#define FUTURE_ABSOLUTE_IMPORT "absolute_import"
+#define FUTURE_WITH_STATEMENT "with_statement"
+#define FUTURE_PRINT_FUNCTION "print_function"
+#define FUTURE_UNICODE_LITERALS "unicode_literals"
+#define FUTURE_BARRY_AS_BDFL "barry_as_FLUFL"
+#define FUTURE_GENERATOR_STOP "generator_stop"
+#define FUTURE_ANNOTATIONS "annotations"
+
+#define PY_INVALID_STACK_EFFECT INT_MAX
+PyAPI_FUNC(int) PyCompile_OpcodeStackEffect(int opcode, int oparg);
+PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump);

Include/cpython/complexobject.h

@@ -0,0 +1,33 @@
+#ifndef Py_CPYTHON_COMPLEXOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+typedef struct {
+    double real;
+    double imag;
+} Py_complex;
+
+// Operations on complex numbers.
+PyAPI_FUNC(Py_complex) _Py_c_sum(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_diff(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_neg(Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_prod(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_quot(Py_complex, Py_complex);
+PyAPI_FUNC(Py_complex) _Py_c_pow(Py_complex, Py_complex);
+PyAPI_FUNC(double) _Py_c_abs(Py_complex);
+
+
+/* Complex object interface */
+
+/*
+PyComplexObject represents a complex number with double-precision
+real and imaginary parts.
+*/
+typedef struct {
+    PyObject_HEAD
+    Py_complex cval;
+} PyComplexObject;
+
+PyAPI_FUNC(PyObject *) PyComplex_FromCComplex(Py_complex);
+
+PyAPI_FUNC(Py_complex) PyComplex_AsCComplex(PyObject *op);

Include/cpython/context.h

@@ -0,0 +1,74 @@
+#ifndef Py_LIMITED_API
+#ifndef Py_CONTEXT_H
+#define Py_CONTEXT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+PyAPI_DATA(PyTypeObject) PyContext_Type;
+typedef struct _pycontextobject PyContext;
+
+PyAPI_DATA(PyTypeObject) PyContextVar_Type;
+typedef struct _pycontextvarobject PyContextVar;
+
+PyAPI_DATA(PyTypeObject) PyContextToken_Type;
+typedef struct _pycontexttokenobject PyContextToken;
+
+
+#define PyContext_CheckExact(o) Py_IS_TYPE((o), &PyContext_Type)
+#define PyContextVar_CheckExact(o) Py_IS_TYPE((o), &PyContextVar_Type)
+#define PyContextToken_CheckExact(o) Py_IS_TYPE((o), &PyContextToken_Type)
+
+
+PyAPI_FUNC(PyObject *) PyContext_New(void);
+PyAPI_FUNC(PyObject *) PyContext_Copy(PyObject *);
+PyAPI_FUNC(PyObject *) PyContext_CopyCurrent(void);
+
+PyAPI_FUNC(int) PyContext_Enter(PyObject *);
+PyAPI_FUNC(int) PyContext_Exit(PyObject *);
+
+
+/* Create a new context variable.
+
+   default_value can be NULL.
+*/
+PyAPI_FUNC(PyObject *) PyContextVar_New(
+    const char *name, PyObject *default_value);
+
+
+/* Get a value for the variable.
+
+   Returns -1 if an error occurred during lookup.
+
+   Returns 0 if value either was or was not found.
+
+   If value was found, *value will point to it.
+   If not, it will point to:
+
+   - default_value, if not NULL;
+   - the default value of "var", if not NULL;
+   - NULL.
+
+   '*value' will be a new ref, if not NULL.
+*/
+PyAPI_FUNC(int) PyContextVar_Get(
+    PyObject *var, PyObject *default_value, PyObject **value);
+
+
+/* Set a new value for the variable.
+   Returns NULL if an error occurs.
+*/
+PyAPI_FUNC(PyObject *) PyContextVar_Set(PyObject *var, PyObject *value);
+
+
+/* Reset a variable to its previous value.
+   Returns 0 on success, -1 on error.
+*/
+PyAPI_FUNC(int) PyContextVar_Reset(PyObject *var, PyObject *token);
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_CONTEXT_H */
+#endif /* !Py_LIMITED_API */

Include/cpython/critical_section.h

@@ -0,0 +1,134 @@
+#ifndef Py_CPYTHON_CRITICAL_SECTION_H
+#  error "this header file must not be included directly"
+#endif
+
+// Python critical sections
+//
+// Conceptually, critical sections are a deadlock avoidance layer on top of
+// per-object locks. These helpers, in combination with those locks, replace
+// our usage of the global interpreter lock to provide thread-safety for
+// otherwise thread-unsafe objects, such as dict.
+//
+// NOTE: These APIs are no-ops in non-free-threaded builds.
+//
+// Straightforward per-object locking could introduce deadlocks that were not
+// present when running with the GIL. Threads may hold locks for multiple
+// objects simultaneously because Python operations can nest. If threads were
+// to acquire the same locks in different orders, they would deadlock.
+//
+// One way to avoid deadlocks is to allow threads to hold only the lock (or
+// locks) for a single operation at a time (typically a single lock, but some
+// operations involve two locks). When a thread begins a nested operation it
+// could suspend the locks for any outer operation: before beginning the nested
+// operation, the locks for the outer operation are released and when the
+// nested operation completes, the locks for the outer operation are
+// reacquired.
+//
+// To improve performance, this API uses a variation of the above scheme.
+// Instead of immediately suspending locks any time a nested operation begins,
+// locks are only suspended if the thread would block. This reduces the number
+// of lock acquisitions and releases for nested operations, while still
+// avoiding deadlocks.
+//
+// Additionally, the locks for any active operation are suspended around
+// other potentially blocking operations, such as I/O. This is because the
+// interaction between locks and blocking operations can lead to deadlocks in
+// the same way as the interaction between multiple locks.
+//
+// Each thread's critical sections and their corresponding locks are tracked in
+// a stack in `PyThreadState.critical_section`. When a thread calls
+// `_PyThreadState_Detach()`, such as before a blocking I/O operation or when
+// waiting to acquire a lock, the thread suspends all of its active critical
+// sections, temporarily releasing the associated locks. When the thread calls
+// `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent)
+// critical section by reacquiring the associated lock or locks.  See
+// `_PyCriticalSection_Resume()`.
+//
+// NOTE: Only the top-most critical section is guaranteed to be active.
+// Operations that need to lock two objects at once must use
+// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections
+// to lock more than one object at once, because the inner critical section
+// may  suspend the outer critical sections. This API does not provide a way
+// to lock more than two objects at once (though it could be added later
+// if actually needed).
+//
+// NOTE: Critical sections implicitly behave like reentrant locks because
+// attempting to acquire the same lock will suspend any outer (earlier)
+// critical sections. However, they are less efficient for this use case than
+// purposefully designed reentrant locks.
+//
+// Example usage:
+//  Py_BEGIN_CRITICAL_SECTION(op);
+//  ...
+//  Py_END_CRITICAL_SECTION();
+//
+// To lock two objects at once:
+//  Py_BEGIN_CRITICAL_SECTION2(op1, op2);
+//  ...
+//  Py_END_CRITICAL_SECTION2();
+
+typedef struct PyCriticalSection PyCriticalSection;
+typedef struct PyCriticalSection2 PyCriticalSection2;
+
+PyAPI_FUNC(void)
+PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op);
+
+PyAPI_FUNC(void)
+PyCriticalSection_End(PyCriticalSection *c);
+
+PyAPI_FUNC(void)
+PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b);
+
+PyAPI_FUNC(void)
+PyCriticalSection2_End(PyCriticalSection2 *c);
+
+#ifndef Py_GIL_DISABLED
+# define Py_BEGIN_CRITICAL_SECTION(op)      \
+    {
+# define Py_END_CRITICAL_SECTION()          \
+    }
+# define Py_BEGIN_CRITICAL_SECTION2(a, b)   \
+    {
+# define Py_END_CRITICAL_SECTION2()         \
+    }
+#else /* !Py_GIL_DISABLED */
+
+// NOTE: the contents of this struct are private and may change betweeen
+// Python releases without a deprecation period.
+struct PyCriticalSection {
+    // Tagged pointer to an outer active critical section (or 0).
+    uintptr_t _cs_prev;
+
+    // Mutex used to protect critical section
+    PyMutex *_cs_mutex;
+};
+
+// A critical section protected by two mutexes. Use
+// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2.
+// NOTE: the contents of this struct are private and may change betweeen
+// Python releases without a deprecation period.
+struct PyCriticalSection2 {
+    PyCriticalSection _cs_base;
+
+    PyMutex *_cs_mutex2;
+};
+
+# define Py_BEGIN_CRITICAL_SECTION(op)                                  \
+    {                                                                   \
+        PyCriticalSection _py_cs;                                       \
+        PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op))
+
+# define Py_END_CRITICAL_SECTION()                                      \
+        PyCriticalSection_End(&_py_cs);                                 \
+    }
+
+# define Py_BEGIN_CRITICAL_SECTION2(a, b)                               \
+    {                                                                   \
+        PyCriticalSection2 _py_cs2;                                     \
+        PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b))
+
+# define Py_END_CRITICAL_SECTION2()                                     \
+        PyCriticalSection2_End(&_py_cs2);                               \
+    }
+
+#endif

Include/cpython/descrobject.h

@@ -0,0 +1,62 @@
+#ifndef Py_CPYTHON_DESCROBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+typedef PyObject *(*wrapperfunc)(PyObject *self, PyObject *args,
+                                 void *wrapped);
+
+typedef PyObject *(*wrapperfunc_kwds)(PyObject *self, PyObject *args,
+                                      void *wrapped, PyObject *kwds);
+
+struct wrapperbase {
+    const char *name;
+    int offset;
+    void *function;
+    wrapperfunc wrapper;
+    const char *doc;
+    int flags;
+    PyObject *name_strobj;
+};
+
+/* Flags for above struct */
+#define PyWrapperFlag_KEYWORDS 1 /* wrapper function takes keyword args */
+
+/* Various kinds of descriptor objects */
+
+typedef struct {
+    PyObject_HEAD
+    PyTypeObject *d_type;
+    PyObject *d_name;
+    PyObject *d_qualname;
+} PyDescrObject;
+
+#define PyDescr_COMMON PyDescrObject d_common
+
+#define PyDescr_TYPE(x) (((PyDescrObject *)(x))->d_type)
+#define PyDescr_NAME(x) (((PyDescrObject *)(x))->d_name)
+
+typedef struct {
+    PyDescr_COMMON;
+    PyMethodDef *d_method;
+    vectorcallfunc vectorcall;
+} PyMethodDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    PyMemberDef *d_member;
+} PyMemberDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    PyGetSetDef *d_getset;
+} PyGetSetDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    struct wrapperbase *d_base;
+    void *d_wrapped; /* This can be any function pointer */
+} PyWrapperDescrObject;
+
+PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *,
+                                                struct wrapperbase *, void *);
+PyAPI_FUNC(int) PyDescr_IsData(PyObject *);

Include/cpython/dictobject.h

@@ -0,0 +1,102 @@
+#ifndef Py_CPYTHON_DICTOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+typedef struct _dictkeysobject PyDictKeysObject;
+typedef struct _dictvalues PyDictValues;
+
+/* The ma_values pointer is NULL for a combined table
+ * or points to an array of PyObject* for a split table
+ */
+typedef struct {
+    PyObject_HEAD
+
+    /* Number of items in the dictionary */
+    Py_ssize_t ma_used;
+
+    /* Dictionary version: globally unique, value change each time
+       the dictionary is modified */
+#ifdef Py_BUILD_CORE
+    /* Bits 0-7 are for dict watchers.
+     * Bits 8-11 are for the watched mutation counter (used by tier2 optimization)
+     * The remaining bits (12-63) are the actual version tag. */
+    uint64_t ma_version_tag;
+#else
+    Py_DEPRECATED(3.12) uint64_t ma_version_tag;
+#endif
+
+    PyDictKeysObject *ma_keys;
+
+    /* If ma_values is NULL, the table is "combined": keys and values
+       are stored in ma_keys.
+
+       If ma_values is not NULL, the table is split:
+       keys are stored in ma_keys and values are stored in ma_values */
+    PyDictValues *ma_values;
+} PyDictObject;
+
+PyAPI_FUNC(PyObject *) _PyDict_GetItem_KnownHash(PyObject *mp, PyObject *key,
+                                                 Py_hash_t hash);
+PyAPI_FUNC(PyObject *) _PyDict_GetItemStringWithError(PyObject *, const char *);
+PyAPI_FUNC(PyObject *) PyDict_SetDefault(
+    PyObject *mp, PyObject *key, PyObject *defaultobj);
+
+// Inserts `key` with a value `default_value`, if `key` is not already present
+// in the dictionary.  If `result` is not NULL, then the value associated
+// with `key` is returned in `*result` (either the existing value, or the now
+// inserted `default_value`).
+// Returns:
+//   -1 on error
+//    0 if `key` was not present and `default_value` was inserted
+//    1 if `key` was present and `default_value` was not inserted
+PyAPI_FUNC(int) PyDict_SetDefaultRef(PyObject *mp, PyObject *key, PyObject *default_value, PyObject **result);
+
+/* Get the number of items of a dictionary. */
+static inline Py_ssize_t PyDict_GET_SIZE(PyObject *op) {
+    PyDictObject *mp;
+    assert(PyDict_Check(op));
+    mp = _Py_CAST(PyDictObject*, op);
+#ifdef Py_GIL_DISABLED
+    return _Py_atomic_load_ssize_relaxed(&mp->ma_used);
+#else
+    return mp->ma_used;
+#endif
+}
+#define PyDict_GET_SIZE(op) PyDict_GET_SIZE(_PyObject_CAST(op))
+
+PyAPI_FUNC(int) PyDict_ContainsString(PyObject *mp, const char *key);
+
+PyAPI_FUNC(PyObject *) _PyDict_NewPresized(Py_ssize_t minused);
+
+PyAPI_FUNC(int) PyDict_Pop(PyObject *dict, PyObject *key, PyObject **result);
+PyAPI_FUNC(int) PyDict_PopString(PyObject *dict, const char *key, PyObject **result);
+PyAPI_FUNC(PyObject *) _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value);
+
+/* Dictionary watchers */
+
+#define PY_FOREACH_DICT_EVENT(V) \
+    V(ADDED)                     \
+    V(MODIFIED)                  \
+    V(DELETED)                   \
+    V(CLONED)                    \
+    V(CLEARED)                   \
+    V(DEALLOCATED)
+
+typedef enum {
+    #define PY_DEF_EVENT(EVENT) PyDict_EVENT_##EVENT,
+    PY_FOREACH_DICT_EVENT(PY_DEF_EVENT)
+    #undef PY_DEF_EVENT
+} PyDict_WatchEvent;
+
+// Callback to be invoked when a watched dict is cleared, dealloced, or modified.
+// In clear/dealloc case, key and new_value will be NULL. Otherwise, new_value will be the
+// new value for key, NULL if key is being deleted.
+typedef int(*PyDict_WatchCallback)(PyDict_WatchEvent event, PyObject* dict, PyObject* key, PyObject* new_value);
+
+// Register/unregister a dict-watcher callback
+PyAPI_FUNC(int) PyDict_AddWatcher(PyDict_WatchCallback callback);
+PyAPI_FUNC(int) PyDict_ClearWatcher(int watcher_id);
+
+// Mark given dictionary as "watched" (callback will be called if it is modified)
+PyAPI_FUNC(int) PyDict_Watch(int watcher_id, PyObject* dict);
+PyAPI_FUNC(int) PyDict_Unwatch(int watcher_id, PyObject* dict);

Include/cpython/fileobject.h

@@ -0,0 +1,16 @@
+#ifndef Py_CPYTHON_FILEOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *);
+
+/* The std printer acts as a preliminary sys.stderr until the new io
+   infrastructure is in place. */
+PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int);
+PyAPI_DATA(PyTypeObject) PyStdPrinter_Type;
+
+typedef PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *);
+
+PyAPI_FUNC(PyObject *) PyFile_OpenCode(const char *utf8path);
+PyAPI_FUNC(PyObject *) PyFile_OpenCodeObject(PyObject *path);
+PyAPI_FUNC(int) PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction hook, void *userData);

Include/cpython/fileutils.h

@@ -0,0 +1,8 @@
+#ifndef Py_CPYTHON_FILEUTILS_H
+#  error "this header file must not be included directly"
+#endif
+
+// Used by _testcapi which must not use the internal C API
+PyAPI_FUNC(FILE*) _Py_fopen_obj(
+    PyObject *path,
+    const char *mode);

Include/cpython/floatobject.h

@@ -0,0 +1,27 @@
+#ifndef Py_CPYTHON_FLOATOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    double ob_fval;
+} PyFloatObject;
+
+#define _PyFloat_CAST(op) \
+    (assert(PyFloat_Check(op)), _Py_CAST(PyFloatObject*, op))
+
+// Static inline version of PyFloat_AsDouble() trading safety for speed.
+// It doesn't check if op is a double object.
+static inline double PyFloat_AS_DOUBLE(PyObject *op) {
+    return _PyFloat_CAST(op)->ob_fval;
+}
+#define PyFloat_AS_DOUBLE(op) PyFloat_AS_DOUBLE(_PyObject_CAST(op))
+
+
+PyAPI_FUNC(int) PyFloat_Pack2(double x, char *p, int le);
+PyAPI_FUNC(int) PyFloat_Pack4(double x, char *p, int le);
+PyAPI_FUNC(int) PyFloat_Pack8(double x, char *p, int le);
+
+PyAPI_FUNC(double) PyFloat_Unpack2(const char *p, int le);
+PyAPI_FUNC(double) PyFloat_Unpack4(const char *p, int le);
+PyAPI_FUNC(double) PyFloat_Unpack8(const char *p, int le);

Include/cpython/frameobject.h

@@ -0,0 +1,35 @@
+/* Frame object interface */
+
+#ifndef Py_CPYTHON_FRAMEOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Standard object interface */
+
+PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *,
+                                        PyObject *, PyObject *);
+
+/* The rest of the interface is specific for frame objects */
+
+/* Conversions between "fast locals" and locals in dictionary */
+
+PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int);
+
+/* -- Caveat emptor --
+ * The concept of entry frames is an implementation detail of the CPython
+ * interpreter. This API is considered unstable and is provided for the
+ * convenience of debuggers, profilers and state-inspecting tools. Notice that
+ * this API can be changed in future minor versions if the underlying frame
+ * mechanism change or the concept of an 'entry frame' or its semantics becomes
+ * obsolete or outdated. */
+
+PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame);
+
+PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f);
+PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *);
+
+
+typedef struct {
+    PyObject_HEAD
+    PyFrameObject* frame;
+} PyFrameLocalsProxyObject;

Include/cpython/funcobject.h

@@ -0,0 +1,184 @@
+/* Function object interface */
+
+#ifndef Py_LIMITED_API
+#ifndef Py_FUNCOBJECT_H
+#define Py_FUNCOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#define _Py_COMMON_FIELDS(PREFIX) \
+    PyObject *PREFIX ## globals; \
+    PyObject *PREFIX ## builtins; \
+    PyObject *PREFIX ## name; \
+    PyObject *PREFIX ## qualname; \
+    PyObject *PREFIX ## code;        /* A code object, the __code__ attribute */ \
+    PyObject *PREFIX ## defaults;    /* NULL or a tuple */ \
+    PyObject *PREFIX ## kwdefaults;  /* NULL or a dict */ \
+    PyObject *PREFIX ## closure;     /* NULL or a tuple of cell objects */
+
+typedef struct {
+    _Py_COMMON_FIELDS(fc_)
+} PyFrameConstructor;
+
+/* Function objects and code objects should not be confused with each other:
+ *
+ * Function objects are created by the execution of the 'def' statement.
+ * They reference a code object in their __code__ attribute, which is a
+ * purely syntactic object, i.e. nothing more than a compiled version of some
+ * source code lines.  There is one code object per source code "fragment",
+ * but each code object can be referenced by zero or many function objects
+ * depending only on how many times the 'def' statement in the source was
+ * executed so far.
+ */
+
+typedef struct {
+    PyObject_HEAD
+    _Py_COMMON_FIELDS(func_)
+    PyObject *func_doc;         /* The __doc__ attribute, can be anything */
+    PyObject *func_dict;        /* The __dict__ attribute, a dict or NULL */
+    PyObject *func_weakreflist; /* List of weak references */
+    PyObject *func_module;      /* The __module__ attribute, can be anything */
+    PyObject *func_annotations; /* Annotations, a dict or NULL */
+    PyObject *func_typeparams;  /* Tuple of active type variables or NULL */
+    vectorcallfunc vectorcall;
+    /* Version number for use by specializer.
+     * Can set to non-zero when we want to specialize.
+     * Will be set to zero if any of these change:
+     *     defaults
+     *     kwdefaults (only if the object changes, not the contents of the dict)
+     *     code
+     *     annotations
+     *     vectorcall function pointer */
+    uint32_t func_version;
+
+    /* Invariant:
+     *     func_closure contains the bindings for func_code->co_freevars, so
+     *     PyTuple_Size(func_closure) == PyCode_GetNumFree(func_code)
+     *     (func_closure may be NULL if PyCode_GetNumFree(func_code) == 0).
+     */
+} PyFunctionObject;
+
+#undef _Py_COMMON_FIELDS
+
+PyAPI_DATA(PyTypeObject) PyFunction_Type;
+
+#define PyFunction_Check(op) Py_IS_TYPE((op), &PyFunction_Type)
+
+PyAPI_FUNC(PyObject *) PyFunction_New(PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyFunction_NewWithQualName(PyObject *, PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyFunction_GetCode(PyObject *);
+PyAPI_FUNC(PyObject *) PyFunction_GetGlobals(PyObject *);
+PyAPI_FUNC(PyObject *) PyFunction_GetModule(PyObject *);
+PyAPI_FUNC(PyObject *) PyFunction_GetDefaults(PyObject *);
+PyAPI_FUNC(int) PyFunction_SetDefaults(PyObject *, PyObject *);
+PyAPI_FUNC(void) PyFunction_SetVectorcall(PyFunctionObject *, vectorcallfunc);
+PyAPI_FUNC(PyObject *) PyFunction_GetKwDefaults(PyObject *);
+PyAPI_FUNC(int) PyFunction_SetKwDefaults(PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyFunction_GetClosure(PyObject *);
+PyAPI_FUNC(int) PyFunction_SetClosure(PyObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *);
+PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *);
+
+#define _PyFunction_CAST(func) \
+    (assert(PyFunction_Check(func)), _Py_CAST(PyFunctionObject*, func))
+
+/* Static inline functions for direct access to these values.
+   Type checks are *not* done, so use with care. */
+static inline PyObject* PyFunction_GET_CODE(PyObject *func) {
+    return _PyFunction_CAST(func)->func_code;
+}
+#define PyFunction_GET_CODE(func) PyFunction_GET_CODE(_PyObject_CAST(func))
+
+static inline PyObject* PyFunction_GET_GLOBALS(PyObject *func) {
+    return _PyFunction_CAST(func)->func_globals;
+}
+#define PyFunction_GET_GLOBALS(func) PyFunction_GET_GLOBALS(_PyObject_CAST(func))
+
+static inline PyObject* PyFunction_GET_MODULE(PyObject *func) {
+    return _PyFunction_CAST(func)->func_module;
+}
+#define PyFunction_GET_MODULE(func) PyFunction_GET_MODULE(_PyObject_CAST(func))
+
+static inline PyObject* PyFunction_GET_DEFAULTS(PyObject *func) {
+    return _PyFunction_CAST(func)->func_defaults;
+}
+#define PyFunction_GET_DEFAULTS(func) PyFunction_GET_DEFAULTS(_PyObject_CAST(func))
+
+static inline PyObject* PyFunction_GET_KW_DEFAULTS(PyObject *func) {
+    return _PyFunction_CAST(func)->func_kwdefaults;
+}
+#define PyFunction_GET_KW_DEFAULTS(func) PyFunction_GET_KW_DEFAULTS(_PyObject_CAST(func))
+
+static inline PyObject* PyFunction_GET_CLOSURE(PyObject *func) {
+    return _PyFunction_CAST(func)->func_closure;
+}
+#define PyFunction_GET_CLOSURE(func) PyFunction_GET_CLOSURE(_PyObject_CAST(func))
+
+static inline PyObject* PyFunction_GET_ANNOTATIONS(PyObject *func) {
+    return _PyFunction_CAST(func)->func_annotations;
+}
+#define PyFunction_GET_ANNOTATIONS(func) PyFunction_GET_ANNOTATIONS(_PyObject_CAST(func))
+
+/* The classmethod and staticmethod types lives here, too */
+PyAPI_DATA(PyTypeObject) PyClassMethod_Type;
+PyAPI_DATA(PyTypeObject) PyStaticMethod_Type;
+
+PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *);
+PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *);
+
+#define PY_FOREACH_FUNC_EVENT(V) \
+    V(CREATE)                    \
+    V(DESTROY)                   \
+    V(MODIFY_CODE)               \
+    V(MODIFY_DEFAULTS)           \
+    V(MODIFY_KWDEFAULTS)
+
+typedef enum {
+    #define PY_DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT,
+    PY_FOREACH_FUNC_EVENT(PY_DEF_EVENT)
+    #undef PY_DEF_EVENT
+} PyFunction_WatchEvent;
+
+/*
+ * A callback that is invoked for different events in a function's lifecycle.
+ *
+ * The callback is invoked with a borrowed reference to func, after it is
+ * created and before it is modified or destroyed. The callback should not
+ * modify func.
+ *
+ * When a function's code object, defaults, or kwdefaults are modified the
+ * callback will be invoked with the respective event and new_value will
+ * contain a borrowed reference to the new value that is about to be stored in
+ * the function. Otherwise the third argument is NULL.
+ *
+ * If the callback returns with an exception set, it must return -1. Otherwise
+ * it should return 0.
+ */
+typedef int (*PyFunction_WatchCallback)(
+  PyFunction_WatchEvent event,
+  PyFunctionObject *func,
+  PyObject *new_value);
+
+/*
+ * Register a per-interpreter callback that will be invoked for function lifecycle
+ * events.
+ *
+ * Returns a handle that may be passed to PyFunction_ClearWatcher on success,
+ * or -1 and sets an error if no more handles are available.
+ */
+PyAPI_FUNC(int) PyFunction_AddWatcher(PyFunction_WatchCallback callback);
+
+/*
+ * Clear the watcher associated with the watcher_id handle.
+ *
+ * Returns 0 on success or -1 if no watcher exists for the supplied id.
+ */
+PyAPI_FUNC(int) PyFunction_ClearWatcher(int watcher_id);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_FUNCOBJECT_H */
+#endif /* Py_LIMITED_API */

Include/cpython/genobject.h

@@ -0,0 +1,83 @@
+/* Generator object interface */
+
+#ifndef Py_LIMITED_API
+#ifndef Py_GENOBJECT_H
+#define Py_GENOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* --- Generators --------------------------------------------------------- */
+
+/* _PyGenObject_HEAD defines the initial segment of generator
+   and coroutine objects. */
+#define _PyGenObject_HEAD(prefix)                                           \
+    PyObject_HEAD                                                           \
+    /* List of weak reference. */                                           \
+    PyObject *prefix##_weakreflist;                                         \
+    /* Name of the generator. */                                            \
+    PyObject *prefix##_name;                                                \
+    /* Qualified name of the generator. */                                  \
+    PyObject *prefix##_qualname;                                            \
+    _PyErr_StackItem prefix##_exc_state;                                    \
+    PyObject *prefix##_origin_or_finalizer;                                 \
+    char prefix##_hooks_inited;                                             \
+    char prefix##_closed;                                                   \
+    char prefix##_running_async;                                            \
+    /* The frame */                                                         \
+    int8_t prefix##_frame_state;                                            \
+    PyObject *prefix##_iframe[1];                                           \
+
+typedef struct {
+    /* The gi_ prefix is intended to remind of generator-iterator. */
+    _PyGenObject_HEAD(gi)
+} PyGenObject;
+
+PyAPI_DATA(PyTypeObject) PyGen_Type;
+
+#define PyGen_Check(op) PyObject_TypeCheck((op), &PyGen_Type)
+#define PyGen_CheckExact(op) Py_IS_TYPE((op), &PyGen_Type)
+
+PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *);
+PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *,
+    PyObject *name, PyObject *qualname);
+PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen);
+
+
+/* --- PyCoroObject ------------------------------------------------------- */
+
+typedef struct {
+    _PyGenObject_HEAD(cr)
+} PyCoroObject;
+
+PyAPI_DATA(PyTypeObject) PyCoro_Type;
+
+#define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type)
+PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *,
+    PyObject *name, PyObject *qualname);
+
+
+/* --- Asynchronous Generators -------------------------------------------- */
+
+typedef struct {
+    _PyGenObject_HEAD(ag)
+} PyAsyncGenObject;
+
+PyAPI_DATA(PyTypeObject) PyAsyncGen_Type;
+PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type;
+
+PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *,
+    PyObject *name, PyObject *qualname);
+
+#define PyAsyncGen_CheckExact(op) Py_IS_TYPE((op), &PyAsyncGen_Type)
+
+#define PyAsyncGenASend_CheckExact(op) Py_IS_TYPE((op), &_PyAsyncGenASend_Type)
+
+
+#undef _PyGenObject_HEAD
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_GENOBJECT_H */
+#endif /* Py_LIMITED_API */

Include/cpython/import.h

@@ -0,0 +1,25 @@
+#ifndef Py_CPYTHON_IMPORT_H
+#  error "this header file must not be included directly"
+#endif
+
+PyMODINIT_FUNC PyInit__imp(void);
+
+struct _inittab {
+    const char *name;           /* ASCII encoded string */
+    PyObject* (*initfunc)(void);
+};
+// This is not used after Py_Initialize() is called.
+PyAPI_DATA(struct _inittab *) PyImport_Inittab;
+PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab);
+
+struct _frozen {
+    const char *name;                 /* ASCII encoded string */
+    const unsigned char *code;
+    int size;
+    int is_package;
+};
+
+/* Embedding apps may change this pointer to point to their favorite
+   collection of frozen modules: */
+
+PyAPI_DATA(const struct _frozen *) PyImport_FrozenModules;

Include/cpython/initconfig.h

@@ -0,0 +1,274 @@
+#ifndef Py_PYCORECONFIG_H
+#define Py_PYCORECONFIG_H
+#ifndef Py_LIMITED_API
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* --- PyStatus ----------------------------------------------- */
+
+typedef struct {
+    enum {
+        _PyStatus_TYPE_OK=0,
+        _PyStatus_TYPE_ERROR=1,
+        _PyStatus_TYPE_EXIT=2
+    } _type;
+    const char *func;
+    const char *err_msg;
+    int exitcode;
+} PyStatus;
+
+PyAPI_FUNC(PyStatus) PyStatus_Ok(void);
+PyAPI_FUNC(PyStatus) PyStatus_Error(const char *err_msg);
+PyAPI_FUNC(PyStatus) PyStatus_NoMemory(void);
+PyAPI_FUNC(PyStatus) PyStatus_Exit(int exitcode);
+PyAPI_FUNC(int) PyStatus_IsError(PyStatus err);
+PyAPI_FUNC(int) PyStatus_IsExit(PyStatus err);
+PyAPI_FUNC(int) PyStatus_Exception(PyStatus err);
+
+/* --- PyWideStringList ------------------------------------------------ */
+
+typedef struct {
+    /* If length is greater than zero, items must be non-NULL
+       and all items strings must be non-NULL */
+    Py_ssize_t length;
+    wchar_t **items;
+} PyWideStringList;
+
+PyAPI_FUNC(PyStatus) PyWideStringList_Append(PyWideStringList *list,
+    const wchar_t *item);
+PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list,
+    Py_ssize_t index,
+    const wchar_t *item);
+
+
+/* --- PyPreConfig ----------------------------------------------- */
+
+typedef struct PyPreConfig {
+    int _config_init;     /* _PyConfigInitEnum value */
+
+    /* Parse Py_PreInitializeFromBytesArgs() arguments?
+       See PyConfig.parse_argv */
+    int parse_argv;
+
+    /* If greater than 0, enable isolated mode: sys.path contains
+       neither the script's directory nor the user's site-packages directory.
+
+       Set to 1 by the -I command line option. If set to -1 (default), inherit
+       Py_IsolatedFlag value. */
+    int isolated;
+
+    /* If greater than 0: use environment variables.
+       Set to 0 by -E command line option. If set to -1 (default), it is
+       set to !Py_IgnoreEnvironmentFlag. */
+    int use_environment;
+
+    /* Set the LC_CTYPE locale to the user preferred locale? If equals to 0,
+       set coerce_c_locale and coerce_c_locale_warn to 0. */
+    int configure_locale;
+
+    /* Coerce the LC_CTYPE locale if it's equal to "C"? (PEP 538)
+
+       Set to 0 by PYTHONCOERCECLOCALE=0. Set to 1 by PYTHONCOERCECLOCALE=1.
+       Set to 2 if the user preferred LC_CTYPE locale is "C".
+
+       If it is equal to 1, LC_CTYPE locale is read to decide if it should be
+       coerced or not (ex: PYTHONCOERCECLOCALE=1). Internally, it is set to 2
+       if the LC_CTYPE locale must be coerced.
+
+       Disable by default (set to 0). Set it to -1 to let Python decide if it
+       should be enabled or not. */
+    int coerce_c_locale;
+
+    /* Emit a warning if the LC_CTYPE locale is coerced?
+
+       Set to 1 by PYTHONCOERCECLOCALE=warn.
+
+       Disable by default (set to 0). Set it to -1 to let Python decide if it
+       should be enabled or not. */
+    int coerce_c_locale_warn;
+
+#ifdef MS_WINDOWS
+    /* If greater than 1, use the "mbcs" encoding instead of the UTF-8
+       encoding for the filesystem encoding.
+
+       Set to 1 if the PYTHONLEGACYWINDOWSFSENCODING environment variable is
+       set to a non-empty string. If set to -1 (default), inherit
+       Py_LegacyWindowsFSEncodingFlag value.
+
+       See PEP 529 for more details. */
+    int legacy_windows_fs_encoding;
+#endif
+
+    /* Enable UTF-8 mode? (PEP 540)
+
+       Disabled by default (equals to 0).
+
+       Set to 1 by "-X utf8" and "-X utf8=1" command line options.
+       Set to 1 by PYTHONUTF8=1 environment variable.
+
+       Set to 0 by "-X utf8=0" and PYTHONUTF8=0.
+
+       If equals to -1, it is set to 1 if the LC_CTYPE locale is "C" or
+       "POSIX", otherwise it is set to 0. Inherit Py_UTF8Mode value value. */
+    int utf8_mode;
+
+    /* If non-zero, enable the Python Development Mode.
+
+       Set to 1 by the -X dev command line option. Set by the PYTHONDEVMODE
+       environment variable. */
+    int dev_mode;
+
+    /* Memory allocator: PYTHONMALLOC env var.
+       See PyMemAllocatorName for valid values. */
+    int allocator;
+} PyPreConfig;
+
+PyAPI_FUNC(void) PyPreConfig_InitPythonConfig(PyPreConfig *config);
+PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config);
+
+
+/* --- PyConfig ---------------------------------------------- */
+
+/* This structure is best documented in the Doc/c-api/init_config.rst file. */
+typedef struct PyConfig {
+    int _config_init;     /* _PyConfigInitEnum value */
+
+    int isolated;
+    int use_environment;
+    int dev_mode;
+    int install_signal_handlers;
+    int use_hash_seed;
+    unsigned long hash_seed;
+    int faulthandler;
+    int tracemalloc;
+    int perf_profiling;
+    int import_time;
+    int code_debug_ranges;
+    int show_ref_count;
+    int dump_refs;
+    wchar_t *dump_refs_file;
+    int malloc_stats;
+    wchar_t *filesystem_encoding;
+    wchar_t *filesystem_errors;
+    wchar_t *pycache_prefix;
+    int parse_argv;
+    PyWideStringList orig_argv;
+    PyWideStringList argv;
+    PyWideStringList xoptions;
+    PyWideStringList warnoptions;
+    int site_import;
+    int bytes_warning;
+    int warn_default_encoding;
+    int inspect;
+    int interactive;
+    int optimization_level;
+    int parser_debug;
+    int write_bytecode;
+    int verbose;
+    int quiet;
+    int user_site_directory;
+    int configure_c_stdio;
+    int buffered_stdio;
+    wchar_t *stdio_encoding;
+    wchar_t *stdio_errors;
+#ifdef MS_WINDOWS
+    int legacy_windows_stdio;
+#endif
+    wchar_t *check_hash_pycs_mode;
+    int use_frozen_modules;
+    int safe_path;
+    int int_max_str_digits;
+
+    int cpu_count;
+#ifdef Py_GIL_DISABLED
+    int enable_gil;
+#endif
+
+    /* --- Path configuration inputs ------------ */
+    int pathconfig_warnings;
+    wchar_t *program_name;
+    wchar_t *pythonpath_env;
+    wchar_t *home;
+    wchar_t *platlibdir;
+
+    /* --- Path configuration outputs ----------- */
+    int module_search_paths_set;
+    PyWideStringList module_search_paths;
+    wchar_t *stdlib_dir;
+    wchar_t *executable;
+    wchar_t *base_executable;
+    wchar_t *prefix;
+    wchar_t *base_prefix;
+    wchar_t *exec_prefix;
+    wchar_t *base_exec_prefix;
+
+    /* --- Parameter only used by Py_Main() ---------- */
+    int skip_source_first_line;
+    wchar_t *run_command;
+    wchar_t *run_module;
+    wchar_t *run_filename;
+
+    /* --- Set by Py_Main() -------------------------- */
+    wchar_t *sys_path_0;
+
+    /* --- Private fields ---------------------------- */
+
+    // Install importlib? If equals to 0, importlib is not initialized at all.
+    // Needed by freeze_importlib.
+    int _install_importlib;
+
+    // If equal to 0, stop Python initialization before the "main" phase.
+    int _init_main;
+
+    // If non-zero, we believe we're running from a source tree.
+    int _is_python_build;
+
+#ifdef Py_STATS
+    // If non-zero, turns on statistics gathering.
+    int _pystats;
+#endif
+
+#ifdef Py_DEBUG
+    // If not empty, import a non-__main__ module before site.py is executed.
+    // PYTHON_PRESITE=package.module or -X presite=package.module
+    wchar_t *run_presite;
+#endif
+} PyConfig;
+
+PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);
+PyAPI_FUNC(void) PyConfig_InitIsolatedConfig(PyConfig *config);
+PyAPI_FUNC(void) PyConfig_Clear(PyConfig *);
+PyAPI_FUNC(PyStatus) PyConfig_SetString(
+    PyConfig *config,
+    wchar_t **config_str,
+    const wchar_t *str);
+PyAPI_FUNC(PyStatus) PyConfig_SetBytesString(
+    PyConfig *config,
+    wchar_t **config_str,
+    const char *str);
+PyAPI_FUNC(PyStatus) PyConfig_Read(PyConfig *config);
+PyAPI_FUNC(PyStatus) PyConfig_SetBytesArgv(
+    PyConfig *config,
+    Py_ssize_t argc,
+    char * const *argv);
+PyAPI_FUNC(PyStatus) PyConfig_SetArgv(PyConfig *config,
+    Py_ssize_t argc,
+    wchar_t * const *argv);
+PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config,
+    PyWideStringList *list,
+    Py_ssize_t length, wchar_t **items);
+
+
+/* --- Helper functions --------------------------------------- */
+
+/* Get the original command line arguments, before Python modified them.
+
+   See also PyConfig.orig_argv. */
+PyAPI_FUNC(void) Py_GetArgcArgv(int *argc, wchar_t ***argv);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_LIMITED_API */
+#endif /* !Py_PYCORECONFIG_H */

Include/cpython/listobject.h

@@ -0,0 +1,53 @@
+#ifndef Py_CPYTHON_LISTOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+typedef struct {
+    PyObject_VAR_HEAD
+    /* Vector of pointers to list elements.  list[0] is ob_item[0], etc. */
+    PyObject **ob_item;
+
+    /* ob_item contains space for 'allocated' elements.  The number
+     * currently in use is ob_size.
+     * Invariants:
+     *     0 <= ob_size <= allocated
+     *     len(list) == ob_size
+     *     ob_item == NULL implies ob_size == allocated == 0
+     * list.sort() temporarily sets allocated to -1 to detect mutations.
+     *
+     * Items must normally not be NULL, except during construction when
+     * the list is not yet visible outside the function that builds it.
+     */
+    Py_ssize_t allocated;
+} PyListObject;
+
+/* Cast argument to PyListObject* type. */
+#define _PyList_CAST(op) \
+    (assert(PyList_Check(op)), _Py_CAST(PyListObject*, (op)))
+
+// Macros and static inline functions, trading safety for speed
+
+static inline Py_ssize_t PyList_GET_SIZE(PyObject *op) {
+    PyListObject *list = _PyList_CAST(op);
+#ifdef Py_GIL_DISABLED
+    return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(list)->ob_size));
+#else
+    return Py_SIZE(list);
+#endif
+}
+#define PyList_GET_SIZE(op) PyList_GET_SIZE(_PyObject_CAST(op))
+
+#define PyList_GET_ITEM(op, index) (_PyList_CAST(op)->ob_item[(index)])
+
+static inline void
+PyList_SET_ITEM(PyObject *op, Py_ssize_t index, PyObject *value) {
+    PyListObject *list = _PyList_CAST(op);
+    assert(0 <= index);
+    assert(index < list->allocated);
+    list->ob_item[index] = value;
+}
+#define PyList_SET_ITEM(op, index, value) \
+    PyList_SET_ITEM(_PyObject_CAST(op), (index), _PyObject_CAST(value))
+
+PyAPI_FUNC(int) PyList_Extend(PyObject *self, PyObject *iterable);
+PyAPI_FUNC(int) PyList_Clear(PyObject *self);

Include/cpython/lock.h

@@ -0,0 +1,63 @@
+#ifndef Py_CPYTHON_LOCK_H
+#  error "this header file must not be included directly"
+#endif
+
+#define _Py_UNLOCKED    0
+#define _Py_LOCKED      1
+
+// A mutex that occupies one byte. The lock can be zero initialized to
+// represent the unlocked state.
+//
+// Typical initialization:
+//   PyMutex m = (PyMutex){0};
+//
+// Or initialize as global variables:
+//   static PyMutex m;
+//
+// Typical usage:
+//   PyMutex_Lock(&m);
+//   ...
+//   PyMutex_Unlock(&m);
+//
+// The contents of the PyMutex are not part of the public API, but are
+// described to aid in understanding the implementation and debugging. Only
+// the two least significant bits are used. The remaining bits are always zero:
+// 0b00: unlocked
+// 0b01: locked
+// 0b10: unlocked and has parked threads
+// 0b11: locked and has parked threads
+typedef struct PyMutex {
+    uint8_t _bits;  // (private)
+} PyMutex;
+
+// exported function for locking the mutex
+PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m);
+
+// exported function for unlocking the mutex
+PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m);
+
+// Locks the mutex.
+//
+// If the mutex is currently locked, the calling thread will be parked until
+// the mutex is unlocked. If the current thread holds the GIL, then the GIL
+// will be released while the thread is parked.
+static inline void
+_PyMutex_Lock(PyMutex *m)
+{
+    uint8_t expected = _Py_UNLOCKED;
+    if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_LOCKED)) {
+        PyMutex_Lock(m);
+    }
+}
+#define PyMutex_Lock _PyMutex_Lock
+
+// Unlocks the mutex.
+static inline void
+_PyMutex_Unlock(PyMutex *m)
+{
+    uint8_t expected = _Py_LOCKED;
+    if (!_Py_atomic_compare_exchange_uint8(&m->_bits, &expected, _Py_UNLOCKED)) {
+        PyMutex_Unlock(m);
+    }
+}
+#define PyMutex_Unlock _PyMutex_Unlock

Include/cpython/longintrepr.h

@@ -0,0 +1,146 @@
+#ifndef Py_LIMITED_API
+#ifndef Py_LONGINTREPR_H
+#define Py_LONGINTREPR_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* This is published for the benefit of "friends" marshal.c and _decimal.c. */
+
+/* Parameters of the integer representation.  There are two different
+   sets of parameters: one set for 30-bit digits, stored in an unsigned 32-bit
+   integer type, and one set for 15-bit digits with each digit stored in an
+   unsigned short.  The value of PYLONG_BITS_IN_DIGIT, defined either at
+   configure time or in pyport.h, is used to decide which digit size to use.
+
+   Type 'digit' should be able to hold 2*PyLong_BASE-1, and type 'twodigits'
+   should be an unsigned integer type able to hold all integers up to
+   PyLong_BASE*PyLong_BASE-1.  x_sub assumes that 'digit' is an unsigned type,
+   and that overflow is handled by taking the result modulo 2**N for some N >
+   PyLong_SHIFT.  The majority of the code doesn't care about the precise
+   value of PyLong_SHIFT, but there are some notable exceptions:
+
+   - PyLong_{As,From}ByteArray require that PyLong_SHIFT be at least 8
+
+   - long_hash() requires that PyLong_SHIFT is *strictly* less than the number
+     of bits in an unsigned long, as do the PyLong <-> long (or unsigned long)
+     conversion functions
+
+   - the Python int <-> size_t/Py_ssize_t conversion functions expect that
+     PyLong_SHIFT is strictly less than the number of bits in a size_t
+
+   - the marshal code currently expects that PyLong_SHIFT is a multiple of 15
+
+   - NSMALLNEGINTS and NSMALLPOSINTS should be small enough to fit in a single
+     digit; with the current values this forces PyLong_SHIFT >= 9
+
+  The values 15 and 30 should fit all of the above requirements, on any
+  platform.
+*/
+
+#if PYLONG_BITS_IN_DIGIT == 30
+typedef uint32_t digit;
+typedef int32_t sdigit; /* signed variant of digit */
+typedef uint64_t twodigits;
+typedef int64_t stwodigits; /* signed variant of twodigits */
+#define PyLong_SHIFT    30
+#define _PyLong_DECIMAL_SHIFT   9 /* max(e such that 10**e fits in a digit) */
+#define _PyLong_DECIMAL_BASE    ((digit)1000000000) /* 10 ** DECIMAL_SHIFT */
+#elif PYLONG_BITS_IN_DIGIT == 15
+typedef unsigned short digit;
+typedef short sdigit; /* signed variant of digit */
+typedef unsigned long twodigits;
+typedef long stwodigits; /* signed variant of twodigits */
+#define PyLong_SHIFT    15
+#define _PyLong_DECIMAL_SHIFT   4 /* max(e such that 10**e fits in a digit) */
+#define _PyLong_DECIMAL_BASE    ((digit)10000) /* 10 ** DECIMAL_SHIFT */
+#else
+#error "PYLONG_BITS_IN_DIGIT should be 15 or 30"
+#endif
+#define PyLong_BASE     ((digit)1 << PyLong_SHIFT)
+#define PyLong_MASK     ((digit)(PyLong_BASE - 1))
+
+/* Long integer representation.
+
+   Long integers are made up of a number of 30- or 15-bit digits, depending on
+   the platform. The number of digits (ndigits) is stored in the high bits of
+   the lv_tag field (lvtag >> _PyLong_NON_SIZE_BITS).
+
+   The absolute value of a number is equal to
+        SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i)
+
+   The sign of the value is stored in the lower 2 bits of lv_tag.
+
+    - 0: Positive
+    - 1: Zero
+    - 2: Negative
+
+   The third lowest bit of lv_tag is reserved for an immortality flag, but is
+   not currently used.
+
+   In a normalized number, ob_digit[ndigits-1] (the most significant
+   digit) is never zero.  Also, in all cases, for all valid i,
+        0 <= ob_digit[i] <= PyLong_MASK.
+
+   The allocation function takes care of allocating extra memory
+   so that ob_digit[0] ... ob_digit[ndigits-1] are actually available.
+   We always allocate memory for at least one digit, so accessing ob_digit[0]
+   is always safe. However, in the case ndigits == 0, the contents of
+   ob_digit[0] may be undefined.
+*/
+
+typedef struct _PyLongValue {
+    uintptr_t lv_tag; /* Number of digits, sign and flags */
+    digit ob_digit[1];
+} _PyLongValue;
+
+struct _longobject {
+    PyObject_HEAD
+    _PyLongValue long_value;
+};
+
+PyAPI_FUNC(PyLongObject*) _PyLong_New(Py_ssize_t);
+
+// Return a copy of src.
+PyAPI_FUNC(PyObject*) _PyLong_Copy(PyLongObject *src);
+
+PyAPI_FUNC(PyLongObject*) _PyLong_FromDigits(
+    int negative,
+    Py_ssize_t digit_count,
+    digit *digits);
+
+
+/* Inline some internals for speed. These should be in pycore_long.h
+ * if user code didn't need them inlined. */
+
+#define _PyLong_SIGN_MASK 3
+#define _PyLong_NON_SIZE_BITS 3
+
+
+static inline int
+_PyLong_IsCompact(const PyLongObject* op) {
+    assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS));
+    return op->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS);
+}
+
+#define PyUnstable_Long_IsCompact _PyLong_IsCompact
+
+static inline Py_ssize_t
+_PyLong_CompactValue(const PyLongObject *op)
+{
+    Py_ssize_t sign;
+    assert(PyType_HasFeature((op)->ob_base.ob_type, Py_TPFLAGS_LONG_SUBCLASS));
+    assert(PyUnstable_Long_IsCompact(op));
+    sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK);
+    return sign * (Py_ssize_t)op->long_value.ob_digit[0];
+}
+
+#define PyUnstable_Long_CompactValue _PyLong_CompactValue
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_LONGINTREPR_H */
+#endif /* Py_LIMITED_API */

Include/cpython/longobject.h

@@ -0,0 +1,116 @@
+#ifndef Py_CPYTHON_LONGOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+PyAPI_FUNC(PyObject*) PyLong_FromUnicodeObject(PyObject *u, int base);
+
+#define Py_ASNATIVEBYTES_DEFAULTS -1
+#define Py_ASNATIVEBYTES_BIG_ENDIAN 0
+#define Py_ASNATIVEBYTES_LITTLE_ENDIAN 1
+#define Py_ASNATIVEBYTES_NATIVE_ENDIAN 3
+#define Py_ASNATIVEBYTES_UNSIGNED_BUFFER 4
+#define Py_ASNATIVEBYTES_REJECT_NEGATIVE 8
+#define Py_ASNATIVEBYTES_ALLOW_INDEX 16
+
+/* PyLong_AsNativeBytes: Copy the integer value to a native variable.
+   buffer points to the first byte of the variable.
+   n_bytes is the number of bytes available in the buffer. Pass 0 to request
+   the required size for the value.
+   flags is a bitfield of the following flags:
+   * 1 - little endian
+   * 2 - native endian
+   * 4 - unsigned destination (e.g. don't reject copying 255 into one byte)
+   * 8 - raise an exception for negative inputs
+   * 16 - call __index__ on non-int types
+   If flags is -1 (all bits set), native endian is used, value truncation
+   behaves most like C (allows negative inputs and allow MSB set), and non-int
+   objects will raise a TypeError.
+   Big endian mode will write the most significant byte into the address
+   directly referenced by buffer; little endian will write the least significant
+   byte into that address.
+
+   If an exception is raised, returns a negative value.
+   Otherwise, returns the number of bytes that are required to store the value.
+   To check that the full value is represented, ensure that the return value is
+   equal or less than n_bytes.
+   All n_bytes are guaranteed to be written (unless an exception occurs), and
+   so ignoring a positive return value is the equivalent of a downcast in C.
+   In cases where the full value could not be represented, the returned value
+   may be larger than necessary - this function is not an accurate way to
+   calculate the bit length of an integer object.
+   */
+PyAPI_FUNC(Py_ssize_t) PyLong_AsNativeBytes(PyObject* v, void* buffer,
+    Py_ssize_t n_bytes, int flags);
+
+/* PyLong_FromNativeBytes: Create an int value from a native integer
+   n_bytes is the number of bytes to read from the buffer. Passing 0 will
+   always produce the zero int.
+   PyLong_FromUnsignedNativeBytes always produces a non-negative int.
+   flags is the same as for PyLong_AsNativeBytes, but only supports selecting
+   the endianness or forcing an unsigned buffer.
+
+   Returns the int object, or NULL with an exception set. */
+PyAPI_FUNC(PyObject*) PyLong_FromNativeBytes(const void* buffer, size_t n_bytes,
+    int flags);
+PyAPI_FUNC(PyObject*) PyLong_FromUnsignedNativeBytes(const void* buffer,
+    size_t n_bytes, int flags);
+
+PyAPI_FUNC(int) PyUnstable_Long_IsCompact(const PyLongObject* op);
+PyAPI_FUNC(Py_ssize_t) PyUnstable_Long_CompactValue(const PyLongObject* op);
+
+// _PyLong_Sign.  Return 0 if v is 0, -1 if v < 0, +1 if v > 0.
+// v must not be NULL, and must be a normalized long.
+// There are no error cases.
+PyAPI_FUNC(int) _PyLong_Sign(PyObject *v);
+
+/* _PyLong_NumBits.  Return the number of bits needed to represent the
+   absolute value of a long.  For example, this returns 1 for 1 and -1, 2
+   for 2 and -2, and 2 for 3 and -3.  It returns 0 for 0.
+   v must not be NULL, and must be a normalized long.
+   (size_t)-1 is returned and OverflowError set if the true result doesn't
+   fit in a size_t.
+*/
+PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v);
+
+/* _PyLong_FromByteArray:  View the n unsigned bytes as a binary integer in
+   base 256, and return a Python int with the same numeric value.
+   If n is 0, the integer is 0.  Else:
+   If little_endian is 1/true, bytes[n-1] is the MSB and bytes[0] the LSB;
+   else (little_endian is 0/false) bytes[0] is the MSB and bytes[n-1] the
+   LSB.
+   If is_signed is 0/false, view the bytes as a non-negative integer.
+   If is_signed is 1/true, view the bytes as a 2's-complement integer,
+   non-negative if bit 0x80 of the MSB is clear, negative if set.
+   Error returns:
+   + Return NULL with the appropriate exception set if there's not
+     enough memory to create the Python int.
+*/
+PyAPI_FUNC(PyObject *) _PyLong_FromByteArray(
+    const unsigned char* bytes, size_t n,
+    int little_endian, int is_signed);
+
+/* _PyLong_AsByteArray: Convert the least-significant 8*n bits of long
+   v to a base-256 integer, stored in array bytes.  Normally return 0,
+   return -1 on error.
+   If little_endian is 1/true, store the MSB at bytes[n-1] and the LSB at
+   bytes[0]; else (little_endian is 0/false) store the MSB at bytes[0] and
+   the LSB at bytes[n-1].
+   If is_signed is 0/false, it's an error if v < 0; else (v >= 0) n bytes
+   are filled and there's nothing special about bit 0x80 of the MSB.
+   If is_signed is 1/true, bytes is filled with the 2's-complement
+   representation of v's value.  Bit 0x80 of the MSB is the sign bit.
+   Error returns (-1):
+   + is_signed is 0 and v < 0.  TypeError is set in this case, and bytes
+     isn't altered.
+   + n isn't big enough to hold the full mathematical value of v.  For
+     example, if is_signed is 0 and there are more digits in the v than
+     fit in n; or if is_signed is 1, v < 0, and n is just 1 bit shy of
+     being large enough to hold a sign bit.  OverflowError is set in this
+     case, but bytes holds the least-significant n bytes of the true value.
+*/
+PyAPI_FUNC(int) _PyLong_AsByteArray(PyLongObject* v,
+    unsigned char* bytes, size_t n,
+    int little_endian, int is_signed, int with_exceptions);
+
+/* For use by the gcd function in mathmodule.c */
+PyAPI_FUNC(PyObject *) _PyLong_GCD(PyObject *, PyObject *);

Include/cpython/memoryobject.h

@@ -0,0 +1,50 @@
+#ifndef Py_CPYTHON_MEMORYOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+/* The structs are declared here so that macros can work, but they shouldn't
+   be considered public. Don't access their fields directly, use the macros
+   and functions instead! */
+#define _Py_MANAGED_BUFFER_RELEASED    0x001  /* access to exporter blocked */
+#define _Py_MANAGED_BUFFER_FREE_FORMAT 0x002  /* free format */
+
+typedef struct {
+    PyObject_HEAD
+    int flags;          /* state flags */
+    Py_ssize_t exports; /* number of direct memoryview exports */
+    Py_buffer master; /* snapshot buffer obtained from the original exporter */
+} _PyManagedBufferObject;
+
+
+/* memoryview state flags */
+#define _Py_MEMORYVIEW_RELEASED    0x001  /* access to master buffer blocked */
+#define _Py_MEMORYVIEW_C           0x002  /* C-contiguous layout */
+#define _Py_MEMORYVIEW_FORTRAN     0x004  /* Fortran contiguous layout */
+#define _Py_MEMORYVIEW_SCALAR      0x008  /* scalar: ndim = 0 */
+#define _Py_MEMORYVIEW_PIL         0x010  /* PIL-style layout */
+#define _Py_MEMORYVIEW_RESTRICTED  0x020  /* Disallow new references to the memoryview's buffer */
+
+typedef struct {
+    PyObject_VAR_HEAD
+    _PyManagedBufferObject *mbuf; /* managed buffer */
+    Py_hash_t hash;               /* hash value for read-only views */
+    int flags;                    /* state flags */
+    Py_ssize_t exports;           /* number of buffer re-exports */
+    Py_buffer view;               /* private copy of the exporter's view */
+    PyObject *weakreflist;
+    Py_ssize_t ob_array[1];       /* shape, strides, suboffsets */
+} PyMemoryViewObject;
+
+#define _PyMemoryView_CAST(op) _Py_CAST(PyMemoryViewObject*, op)
+
+/* Get a pointer to the memoryview's private copy of the exporter's buffer. */
+static inline Py_buffer* PyMemoryView_GET_BUFFER(PyObject *op) {
+    return (&_PyMemoryView_CAST(op)->view);
+}
+#define PyMemoryView_GET_BUFFER(op) PyMemoryView_GET_BUFFER(_PyObject_CAST(op))
+
+/* Get a pointer to the exporting object (this may be NULL!). */
+static inline PyObject* PyMemoryView_GET_BASE(PyObject *op) {
+    return _PyMemoryView_CAST(op)->view.obj;
+}
+#define PyMemoryView_GET_BASE(op) PyMemoryView_GET_BASE(_PyObject_CAST(op))

Include/cpython/methodobject.h

@@ -0,0 +1,66 @@
+#ifndef Py_CPYTHON_METHODOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+// PyCFunctionObject structure
+
+typedef struct {
+    PyObject_HEAD
+    PyMethodDef *m_ml; /* Description of the C function to call */
+    PyObject    *m_self; /* Passed as 'self' arg to the C func, can be NULL */
+    PyObject    *m_module; /* The __module__ attribute, can be anything */
+    PyObject    *m_weakreflist; /* List of weak references */
+    vectorcallfunc vectorcall;
+} PyCFunctionObject;
+
+#define _PyCFunctionObject_CAST(func) \
+    (assert(PyCFunction_Check(func)), \
+     _Py_CAST(PyCFunctionObject*, (func)))
+
+
+// PyCMethodObject structure
+
+typedef struct {
+    PyCFunctionObject func;
+    PyTypeObject *mm_class; /* Class that defines this method */
+} PyCMethodObject;
+
+#define _PyCMethodObject_CAST(func) \
+    (assert(PyCMethod_Check(func)), \
+     _Py_CAST(PyCMethodObject*, (func)))
+
+PyAPI_DATA(PyTypeObject) PyCMethod_Type;
+
+#define PyCMethod_CheckExact(op) Py_IS_TYPE((op), &PyCMethod_Type)
+#define PyCMethod_Check(op) PyObject_TypeCheck((op), &PyCMethod_Type)
+
+
+/* Static inline functions for direct access to these values.
+   Type checks are *not* done, so use with care. */
+static inline PyCFunction PyCFunction_GET_FUNCTION(PyObject *func) {
+    return _PyCFunctionObject_CAST(func)->m_ml->ml_meth;
+}
+#define PyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(_PyObject_CAST(func))
+
+static inline PyObject* PyCFunction_GET_SELF(PyObject *func_obj) {
+    PyCFunctionObject *func = _PyCFunctionObject_CAST(func_obj);
+    if (func->m_ml->ml_flags & METH_STATIC) {
+        return _Py_NULL;
+    }
+    return func->m_self;
+}
+#define PyCFunction_GET_SELF(func) PyCFunction_GET_SELF(_PyObject_CAST(func))
+
+static inline int PyCFunction_GET_FLAGS(PyObject *func) {
+    return _PyCFunctionObject_CAST(func)->m_ml->ml_flags;
+}
+#define PyCFunction_GET_FLAGS(func) PyCFunction_GET_FLAGS(_PyObject_CAST(func))
+
+static inline PyTypeObject* PyCFunction_GET_CLASS(PyObject *func_obj) {
+    PyCFunctionObject *func = _PyCFunctionObject_CAST(func_obj);
+    if (func->m_ml->ml_flags & METH_METHOD) {
+        return _PyCMethodObject_CAST(func)->mm_class;
+    }
+    return _Py_NULL;
+}
+#define PyCFunction_GET_CLASS(func) PyCFunction_GET_CLASS(_PyObject_CAST(func))

Include/cpython/modsupport.h

@@ -0,0 +1,26 @@
+#ifndef Py_CPYTHON_MODSUPPORT_H
+#  error "this header file must not be included directly"
+#endif
+
+// A data structure that can be used to run initialization code once in a
+// thread-safe manner. The C++11 equivalent is std::call_once.
+typedef struct {
+    uint8_t v;
+} _PyOnceFlag;
+
+typedef struct _PyArg_Parser {
+    const char *format;
+    const char * const *keywords;
+    const char *fname;
+    const char *custom_msg;
+    _PyOnceFlag once;       /* atomic one-time initialization flag */
+    int is_kwtuple_owned;   /* does this parser own the kwtuple object? */
+    int pos;                /* number of positional-only arguments */
+    int min;                /* minimal number of arguments */
+    int max;                /* maximal number of positional arguments */
+    PyObject *kwtuple;      /* tuple of keyword parameter names */
+    struct _PyArg_Parser *next;
+} _PyArg_Parser;
+
+PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *,
+                                                 struct _PyArg_Parser *, ...);

Include/cpython/monitoring.h

@@ -0,0 +1,250 @@
+#ifndef Py_CPYTHON_MONITORING_H
+#  error "this header file must not be included directly"
+#endif
+
+/* Local events.
+ * These require bytecode instrumentation */
+
+#define PY_MONITORING_EVENT_PY_START 0
+#define PY_MONITORING_EVENT_PY_RESUME 1
+#define PY_MONITORING_EVENT_PY_RETURN 2
+#define PY_MONITORING_EVENT_PY_YIELD 3
+#define PY_MONITORING_EVENT_CALL 4
+#define PY_MONITORING_EVENT_LINE 5
+#define PY_MONITORING_EVENT_INSTRUCTION 6
+#define PY_MONITORING_EVENT_JUMP 7
+#define PY_MONITORING_EVENT_BRANCH 8
+#define PY_MONITORING_EVENT_STOP_ITERATION 9
+
+#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \
+    ((ev) < _PY_MONITORING_LOCAL_EVENTS)
+
+/* Other events, mainly exceptions */
+
+#define PY_MONITORING_EVENT_RAISE 10
+#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11
+#define PY_MONITORING_EVENT_PY_UNWIND 12
+#define PY_MONITORING_EVENT_PY_THROW 13
+#define PY_MONITORING_EVENT_RERAISE 14
+
+
+/* Ancillary events */
+
+#define PY_MONITORING_EVENT_C_RETURN 15
+#define PY_MONITORING_EVENT_C_RAISE 16
+
+
+typedef struct _PyMonitoringState {
+    uint8_t active;
+    uint8_t opaque;
+} PyMonitoringState;
+
+
+PyAPI_FUNC(int)
+PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version,
+                         const uint8_t *event_types, Py_ssize_t length);
+
+PyAPI_FUNC(int)
+PyMonitoring_ExitScope(void);
+
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                                PyObject *retval);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject *retval);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            PyObject* callable, PyObject *arg0);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            int lineno);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                            PyObject *target_offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                              PyObject *target_offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject *retval);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset);
+
+PyAPI_FUNC(int)
+_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value);
+
+
+#define _PYMONITORING_IF_ACTIVE(STATE, X)  \
+    if ((STATE)->active) { \
+        return (X); \
+    } \
+    else { \
+        return 0; \
+    }
+
+static inline int
+PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyStartEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyResumeEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                               PyObject *retval)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval));
+}
+
+static inline int
+PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                              PyObject *retval)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval));
+}
+
+static inline int
+PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                           PyObject* callable, PyObject *arg0)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0));
+}
+
+static inline int
+PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                           int lineno)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireLineEvent(state, codelike, offset, lineno));
+}
+
+static inline int
+PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                           PyObject *target_offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset));
+}
+
+static inline int
+PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                             PyObject *target_offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset));
+}
+
+static inline int
+PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset,
+                              PyObject *retval)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval));
+}
+
+static inline int
+PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyThrowEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireRaiseEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireReraiseEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireCRaiseEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FirePyUnwindEvent(state, codelike, offset));
+}
+
+static inline int
+PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value)
+{
+    _PYMONITORING_IF_ACTIVE(
+        state,
+        _PyMonitoring_FireStopIterationEvent(state, codelike, offset, value));
+}
+
+#undef _PYMONITORING_IF_ACTIVE

Include/cpython/object.h

@@ -0,0 +1,525 @@
+#ifndef Py_CPYTHON_OBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+PyAPI_FUNC(void) _Py_NewReference(PyObject *op);
+PyAPI_FUNC(void) _Py_NewReferenceNoTotal(PyObject *op);
+PyAPI_FUNC(void) _Py_ResurrectReference(PyObject *op);
+
+#ifdef Py_REF_DEBUG
+/* These are useful as debugging aids when chasing down refleaks. */
+PyAPI_FUNC(Py_ssize_t) _Py_GetGlobalRefTotal(void);
+#  define _Py_GetRefTotal() _Py_GetGlobalRefTotal()
+PyAPI_FUNC(Py_ssize_t) _Py_GetLegacyRefTotal(void);
+PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GetRefTotal(PyInterpreterState *);
+#endif
+
+
+/********************* String Literals ****************************************/
+/* This structure helps managing static strings. The basic usage goes like this:
+   Instead of doing
+
+       r = PyObject_CallMethod(o, "foo", "args", ...);
+
+   do
+
+       _Py_IDENTIFIER(foo);
+       ...
+       r = _PyObject_CallMethodId(o, &PyId_foo, "args", ...);
+
+   PyId_foo is a static variable, either on block level or file level. On first
+   usage, the string "foo" is interned, and the structures are linked. On interpreter
+   shutdown, all strings are released.
+
+   Alternatively, _Py_static_string allows choosing the variable name.
+   _PyUnicode_FromId returns a borrowed reference to the interned string.
+   _PyObject_{Get,Set,Has}AttrId are __getattr__ versions using _Py_Identifier*.
+*/
+typedef struct _Py_Identifier {
+    const char* string;
+    // Index in PyInterpreterState.unicode.ids.array. It is process-wide
+    // unique and must be initialized to -1.
+    Py_ssize_t index;
+    // Hidden PyMutex struct for non free-threaded build.
+    struct {
+        uint8_t v;
+    } mutex;
+} _Py_Identifier;
+
+#ifndef Py_BUILD_CORE
+// For now we are keeping _Py_IDENTIFIER for continued use
+// in non-builtin extensions (and naughty PyPI modules).
+
+#define _Py_static_string_init(value) { .string = (value), .index = -1 }
+#define _Py_static_string(varname, value)  static _Py_Identifier varname = _Py_static_string_init(value)
+#define _Py_IDENTIFIER(varname) _Py_static_string(PyId_##varname, #varname)
+
+#endif /* !Py_BUILD_CORE */
+
+
+typedef struct {
+    /* Number implementations must check *both*
+       arguments for proper type and implement the necessary conversions
+       in the slot functions themselves. */
+
+    binaryfunc nb_add;
+    binaryfunc nb_subtract;
+    binaryfunc nb_multiply;
+    binaryfunc nb_remainder;
+    binaryfunc nb_divmod;
+    ternaryfunc nb_power;
+    unaryfunc nb_negative;
+    unaryfunc nb_positive;
+    unaryfunc nb_absolute;
+    inquiry nb_bool;
+    unaryfunc nb_invert;
+    binaryfunc nb_lshift;
+    binaryfunc nb_rshift;
+    binaryfunc nb_and;
+    binaryfunc nb_xor;
+    binaryfunc nb_or;
+    unaryfunc nb_int;
+    void *nb_reserved;  /* the slot formerly known as nb_long */
+    unaryfunc nb_float;
+
+    binaryfunc nb_inplace_add;
+    binaryfunc nb_inplace_subtract;
+    binaryfunc nb_inplace_multiply;
+    binaryfunc nb_inplace_remainder;
+    ternaryfunc nb_inplace_power;
+    binaryfunc nb_inplace_lshift;
+    binaryfunc nb_inplace_rshift;
+    binaryfunc nb_inplace_and;
+    binaryfunc nb_inplace_xor;
+    binaryfunc nb_inplace_or;
+
+    binaryfunc nb_floor_divide;
+    binaryfunc nb_true_divide;
+    binaryfunc nb_inplace_floor_divide;
+    binaryfunc nb_inplace_true_divide;
+
+    unaryfunc nb_index;
+
+    binaryfunc nb_matrix_multiply;
+    binaryfunc nb_inplace_matrix_multiply;
+} PyNumberMethods;
+
+typedef struct {
+    lenfunc sq_length;
+    binaryfunc sq_concat;
+    ssizeargfunc sq_repeat;
+    ssizeargfunc sq_item;
+    void *was_sq_slice;
+    ssizeobjargproc sq_ass_item;
+    void *was_sq_ass_slice;
+    objobjproc sq_contains;
+
+    binaryfunc sq_inplace_concat;
+    ssizeargfunc sq_inplace_repeat;
+} PySequenceMethods;
+
+typedef struct {
+    lenfunc mp_length;
+    binaryfunc mp_subscript;
+    objobjargproc mp_ass_subscript;
+} PyMappingMethods;
+
+typedef PySendResult (*sendfunc)(PyObject *iter, PyObject *value, PyObject **result);
+
+typedef struct {
+    unaryfunc am_await;
+    unaryfunc am_aiter;
+    unaryfunc am_anext;
+    sendfunc am_send;
+} PyAsyncMethods;
+
+typedef struct {
+     getbufferproc bf_getbuffer;
+     releasebufferproc bf_releasebuffer;
+} PyBufferProcs;
+
+/* Allow printfunc in the tp_vectorcall_offset slot for
+ * backwards-compatibility */
+typedef Py_ssize_t printfunc;
+
+// If this structure is modified, Doc/includes/typestruct.h should be updated
+// as well.
+struct _typeobject {
+    PyObject_VAR_HEAD
+    const char *tp_name; /* For printing, in format "<module>.<name>" */
+    Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */
+
+    /* Methods to implement standard operations */
+
+    destructor tp_dealloc;
+    Py_ssize_t tp_vectorcall_offset;
+    getattrfunc tp_getattr;
+    setattrfunc tp_setattr;
+    PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2)
+                                    or tp_reserved (Python 3) */
+    reprfunc tp_repr;
+
+    /* Method suites for standard classes */
+
+    PyNumberMethods *tp_as_number;
+    PySequenceMethods *tp_as_sequence;
+    PyMappingMethods *tp_as_mapping;
+
+    /* More standard operations (here for binary compatibility) */
+
+    hashfunc tp_hash;
+    ternaryfunc tp_call;
+    reprfunc tp_str;
+    getattrofunc tp_getattro;
+    setattrofunc tp_setattro;
+
+    /* Functions to access object as input/output buffer */
+    PyBufferProcs *tp_as_buffer;
+
+    /* Flags to define presence of optional/expanded features */
+    unsigned long tp_flags;
+
+    const char *tp_doc; /* Documentation string */
+
+    /* Assigned meaning in release 2.0 */
+    /* call function for all accessible objects */
+    traverseproc tp_traverse;
+
+    /* delete references to contained objects */
+    inquiry tp_clear;
+
+    /* Assigned meaning in release 2.1 */
+    /* rich comparisons */
+    richcmpfunc tp_richcompare;
+
+    /* weak reference enabler */
+    Py_ssize_t tp_weaklistoffset;
+
+    /* Iterators */
+    getiterfunc tp_iter;
+    iternextfunc tp_iternext;
+
+    /* Attribute descriptor and subclassing stuff */
+    PyMethodDef *tp_methods;
+    PyMemberDef *tp_members;
+    PyGetSetDef *tp_getset;
+    // Strong reference on a heap type, borrowed reference on a static type
+    PyTypeObject *tp_base;
+    PyObject *tp_dict;
+    descrgetfunc tp_descr_get;
+    descrsetfunc tp_descr_set;
+    Py_ssize_t tp_dictoffset;
+    initproc tp_init;
+    allocfunc tp_alloc;
+    newfunc tp_new;
+    freefunc tp_free; /* Low-level free-memory routine */
+    inquiry tp_is_gc; /* For PyObject_IS_GC */
+    PyObject *tp_bases;
+    PyObject *tp_mro; /* method resolution order */
+    PyObject *tp_cache; /* no longer used */
+    void *tp_subclasses;  /* for static builtin types this is an index */
+    PyObject *tp_weaklist; /* not used for static builtin types */
+    destructor tp_del;
+
+    /* Type attribute cache version tag. Added in version 2.6 */
+    unsigned int tp_version_tag;
+
+    destructor tp_finalize;
+    vectorcallfunc tp_vectorcall;
+
+    /* bitset of which type-watchers care about this type */
+    unsigned char tp_watched;
+    uint16_t tp_versions_used;
+};
+
+/* This struct is used by the specializer
+ * It should be treated as an opaque blob
+ * by code other than the specializer and interpreter. */
+struct _specialization_cache {
+    // In order to avoid bloating the bytecode with lots of inline caches, the
+    // members of this structure have a somewhat unique contract. They are set
+    // by the specialization machinery, and are invalidated by PyType_Modified.
+    // The rules for using them are as follows:
+    // - If getitem is non-NULL, then it is the same Python function that
+    //   PyType_Lookup(cls, "__getitem__") would return.
+    // - If getitem is NULL, then getitem_version is meaningless.
+    // - If getitem->func_version == getitem_version, then getitem can be called
+    //   with two positional arguments and no keyword arguments, and has neither
+    //   *args nor **kwargs (as required by BINARY_SUBSCR_GETITEM):
+    PyObject *getitem;
+    uint32_t getitem_version;
+    PyObject *init;
+};
+
+/* The *real* layout of a type object when allocated on the heap */
+typedef struct _heaptypeobject {
+    /* Note: there's a dependency on the order of these members
+       in slotptr() in typeobject.c . */
+    PyTypeObject ht_type;
+    PyAsyncMethods as_async;
+    PyNumberMethods as_number;
+    PyMappingMethods as_mapping;
+    PySequenceMethods as_sequence; /* as_sequence comes after as_mapping,
+                                      so that the mapping wins when both
+                                      the mapping and the sequence define
+                                      a given operator (e.g. __getitem__).
+                                      see add_operators() in typeobject.c . */
+    PyBufferProcs as_buffer;
+    PyObject *ht_name, *ht_slots, *ht_qualname;
+    struct _dictkeysobject *ht_cached_keys;
+    PyObject *ht_module;
+    char *_ht_tpname;  // Storage for "tp_name"; see PyType_FromModuleAndSpec
+    struct _specialization_cache _spec_cache; // For use by the specializer.
+    /* here are optional user slots, followed by the members. */
+} PyHeapTypeObject;
+
+PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *);
+PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *);
+PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *);
+PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *);
+
+PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int);
+PyAPI_FUNC(void) _Py_BreakPoint(void);
+PyAPI_FUNC(void) _PyObject_Dump(PyObject *);
+
+PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Identifier *);
+
+PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *);
+PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *);
+PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *);
+
+PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *);
+
+/* Same as PyObject_Generic{Get,Set}Attr, but passing the attributes
+   dict as the last parameter. */
+PyAPI_FUNC(PyObject *)
+_PyObject_GenericGetAttrWithDict(PyObject *, PyObject *, PyObject *, int);
+PyAPI_FUNC(int)
+_PyObject_GenericSetAttrWithDict(PyObject *, PyObject *,
+                                 PyObject *, PyObject *);
+
+PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *);
+
+/* Safely decref `dst` and set `dst` to `src`.
+ *
+ * As in case of Py_CLEAR "the obvious" code can be deadly:
+ *
+ *     Py_DECREF(dst);
+ *     dst = src;
+ *
+ * The safe way is:
+ *
+ *      Py_SETREF(dst, src);
+ *
+ * That arranges to set `dst` to `src` _before_ decref'ing, so that any code
+ * triggered as a side-effect of `dst` getting torn down no longer believes
+ * `dst` points to a valid object.
+ *
+ * Temporary variables are used to only evalutate macro arguments once and so
+ * avoid the duplication of side effects. _Py_TYPEOF() or memcpy() is used to
+ * avoid a miscompilation caused by type punning. See Py_CLEAR() comment for
+ * implementation details about type punning.
+ *
+ * The memcpy() implementation does not emit a compiler warning if 'src' has
+ * not the same type than 'src': any pointer type is accepted for 'src'.
+ */
+#ifdef _Py_TYPEOF
+#define Py_SETREF(dst, src) \
+    do { \
+        _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \
+        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_DECREF(_tmp_old_dst); \
+    } while (0)
+#else
+#define Py_SETREF(dst, src) \
+    do { \
+        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
+        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
+        PyObject *_tmp_src = _PyObject_CAST(src); \
+        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
+        Py_DECREF(_tmp_old_dst); \
+    } while (0)
+#endif
+
+/* Py_XSETREF() is a variant of Py_SETREF() that uses Py_XDECREF() instead of
+ * Py_DECREF().
+ */
+#ifdef _Py_TYPEOF
+#define Py_XSETREF(dst, src) \
+    do { \
+        _Py_TYPEOF(dst)* _tmp_dst_ptr = &(dst); \
+        _Py_TYPEOF(dst) _tmp_old_dst = (*_tmp_dst_ptr); \
+        *_tmp_dst_ptr = (src); \
+        Py_XDECREF(_tmp_old_dst); \
+    } while (0)
+#else
+#define Py_XSETREF(dst, src) \
+    do { \
+        PyObject **_tmp_dst_ptr = _Py_CAST(PyObject**, &(dst)); \
+        PyObject *_tmp_old_dst = (*_tmp_dst_ptr); \
+        PyObject *_tmp_src = _PyObject_CAST(src); \
+        memcpy(_tmp_dst_ptr, &_tmp_src, sizeof(PyObject*)); \
+        Py_XDECREF(_tmp_old_dst); \
+    } while (0)
+#endif
+
+
+/* Define a pair of assertion macros:
+   _PyObject_ASSERT_FROM(), _PyObject_ASSERT_WITH_MSG() and _PyObject_ASSERT().
+
+   These work like the regular C assert(), in that they will abort the
+   process with a message on stderr if the given condition fails to hold,
+   but compile away to nothing if NDEBUG is defined.
+
+   However, before aborting, Python will also try to call _PyObject_Dump() on
+   the given object.  This may be of use when investigating bugs in which a
+   particular object is corrupt (e.g. buggy a tp_visit method in an extension
+   module breaking the garbage collector), to help locate the broken objects.
+
+   The WITH_MSG variant allows you to supply an additional message that Python
+   will attempt to print to stderr, after the object dump. */
+#ifdef NDEBUG
+   /* No debugging: compile away the assertions: */
+#  define _PyObject_ASSERT_FROM(obj, expr, msg, filename, lineno, func) \
+    ((void)0)
+#else
+   /* With debugging: generate checks: */
+#  define _PyObject_ASSERT_FROM(obj, expr, msg, filename, lineno, func) \
+    ((expr) \
+      ? (void)(0) \
+      : _PyObject_AssertFailed((obj), Py_STRINGIFY(expr), \
+                               (msg), (filename), (lineno), (func)))
+#endif
+
+#define _PyObject_ASSERT_WITH_MSG(obj, expr, msg) \
+    _PyObject_ASSERT_FROM((obj), expr, (msg), __FILE__, __LINE__, __func__)
+#define _PyObject_ASSERT(obj, expr) \
+    _PyObject_ASSERT_WITH_MSG((obj), expr, NULL)
+
+#define _PyObject_ASSERT_FAILED_MSG(obj, msg) \
+    _PyObject_AssertFailed((obj), NULL, (msg), __FILE__, __LINE__, __func__)
+
+/* Declare and define _PyObject_AssertFailed() even when NDEBUG is defined,
+   to avoid causing compiler/linker errors when building extensions without
+   NDEBUG against a Python built with NDEBUG defined.
+
+   msg, expr and function can be NULL. */
+PyAPI_FUNC(void) _Py_NO_RETURN _PyObject_AssertFailed(
+    PyObject *obj,
+    const char *expr,
+    const char *msg,
+    const char *file,
+    int line,
+    const char *function);
+
+
+/* Trashcan mechanism, thanks to Christian Tismer.
+
+When deallocating a container object, it's possible to trigger an unbounded
+chain of deallocations, as each Py_DECREF in turn drops the refcount on "the
+next" object in the chain to 0.  This can easily lead to stack overflows,
+especially in threads (which typically have less stack space to work with).
+
+A container object can avoid this by bracketing the body of its tp_dealloc
+function with a pair of macros:
+
+static void
+mytype_dealloc(mytype *p)
+{
+    ... declarations go here ...
+
+    PyObject_GC_UnTrack(p);        // must untrack first
+    Py_TRASHCAN_BEGIN(p, mytype_dealloc)
+    ... The body of the deallocator goes here, including all calls ...
+    ... to Py_DECREF on contained objects.                         ...
+    Py_TRASHCAN_END                // there should be no code after this
+}
+
+CAUTION:  Never return from the middle of the body!  If the body needs to
+"get out early", put a label immediately before the Py_TRASHCAN_END
+call, and goto it.  Else the call-depth counter (see below) will stay
+above 0 forever, and the trashcan will never get emptied.
+
+How it works:  The BEGIN macro increments a call-depth counter.  So long
+as this counter is small, the body of the deallocator is run directly without
+further ado.  But if the counter gets large, it instead adds p to a list of
+objects to be deallocated later, skips the body of the deallocator, and
+resumes execution after the END macro.  The tp_dealloc routine then returns
+without deallocating anything (and so unbounded call-stack depth is avoided).
+
+When the call stack finishes unwinding again, code generated by the END macro
+notices this, and calls another routine to deallocate all the objects that
+may have been added to the list of deferred deallocations.  In effect, a
+chain of N deallocations is broken into (N-1)/(Py_TRASHCAN_HEADROOM-1) pieces,
+with the call stack never exceeding a depth of Py_TRASHCAN_HEADROOM.
+
+Since the tp_dealloc of a subclass typically calls the tp_dealloc of the base
+class, we need to ensure that the trashcan is only triggered on the tp_dealloc
+of the actual class being deallocated. Otherwise we might end up with a
+partially-deallocated object. To check this, the tp_dealloc function must be
+passed as second argument to Py_TRASHCAN_BEGIN().
+*/
+
+/* Python 3.9 private API, invoked by the macros below. */
+PyAPI_FUNC(int) _PyTrash_begin(PyThreadState *tstate, PyObject *op);
+PyAPI_FUNC(void) _PyTrash_end(PyThreadState *tstate);
+
+PyAPI_FUNC(void) _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op);
+PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(PyThreadState *tstate);
+
+
+/* Python 3.10 private API, invoked by the Py_TRASHCAN_BEGIN(). */
+
+/* To avoid raising recursion errors during dealloc trigger trashcan before we reach
+ * recursion limit. To avoid trashing, we don't attempt to empty the trashcan until
+ * we have headroom above the trigger limit */
+#define Py_TRASHCAN_HEADROOM 50
+
+#define Py_TRASHCAN_BEGIN(op, dealloc) \
+do { \
+    PyThreadState *tstate = PyThreadState_Get(); \
+    if (tstate->c_recursion_remaining <= Py_TRASHCAN_HEADROOM && Py_TYPE(op)->tp_dealloc == (destructor)dealloc) { \
+        _PyTrash_thread_deposit_object(tstate, (PyObject *)op); \
+        break; \
+    } \
+    tstate->c_recursion_remaining--;
+    /* The body of the deallocator is here. */
+#define Py_TRASHCAN_END \
+    tstate->c_recursion_remaining++; \
+    if (tstate->delete_later && tstate->c_recursion_remaining > (Py_TRASHCAN_HEADROOM*2)) { \
+        _PyTrash_thread_destroy_chain(tstate); \
+    } \
+} while (0);
+
+
+PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj);
+
+PyAPI_FUNC(int) PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg);
+PyAPI_FUNC(int) _PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict);
+PyAPI_FUNC(void) PyObject_ClearManagedDict(PyObject *obj);
+
+#define TYPE_MAX_WATCHERS 8
+
+typedef int(*PyType_WatchCallback)(PyTypeObject *);
+PyAPI_FUNC(int) PyType_AddWatcher(PyType_WatchCallback callback);
+PyAPI_FUNC(int) PyType_ClearWatcher(int watcher_id);
+PyAPI_FUNC(int) PyType_Watch(int watcher_id, PyObject *type);
+PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type);
+
+/* Attempt to assign a version tag to the given type.
+ *
+ * Returns 1 if the type already had a valid version tag or a new one was
+ * assigned, or 0 if a new tag could not be assigned.
+ */
+PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type);
+
+
+typedef enum {
+    PyRefTracer_CREATE = 0,
+    PyRefTracer_DESTROY = 1,
+} PyRefTracerEvent;
+
+typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *);
+PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data);
+PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**);

Include/cpython/objimpl.h

@@ -0,0 +1,104 @@
+#ifndef Py_CPYTHON_OBJIMPL_H
+#  error "this header file must not be included directly"
+#endif
+
+static inline size_t _PyObject_SIZE(PyTypeObject *type) {
+    return _Py_STATIC_CAST(size_t, type->tp_basicsize);
+}
+
+/* _PyObject_VAR_SIZE returns the number of bytes (as size_t) allocated for a
+   vrbl-size object with nitems items, exclusive of gc overhead (if any).  The
+   value is rounded up to the closest multiple of sizeof(void *), in order to
+   ensure that pointer fields at the end of the object are correctly aligned
+   for the platform (this is of special importance for subclasses of, e.g.,
+   str or int, so that pointers can be stored after the embedded data).
+
+   Note that there's no memory wastage in doing this, as malloc has to
+   return (at worst) pointer-aligned memory anyway.
+*/
+#if ((SIZEOF_VOID_P - 1) & SIZEOF_VOID_P) != 0
+#   error "_PyObject_VAR_SIZE requires SIZEOF_VOID_P be a power of 2"
+#endif
+
+static inline size_t _PyObject_VAR_SIZE(PyTypeObject *type, Py_ssize_t nitems) {
+    size_t size = _Py_STATIC_CAST(size_t, type->tp_basicsize);
+    size += _Py_STATIC_CAST(size_t, nitems) * _Py_STATIC_CAST(size_t, type->tp_itemsize);
+    return _Py_SIZE_ROUND_UP(size, SIZEOF_VOID_P);
+}
+
+
+/* This example code implements an object constructor with a custom
+   allocator, where PyObject_New is inlined, and shows the important
+   distinction between two steps (at least):
+       1) the actual allocation of the object storage;
+       2) the initialization of the Python specific fields
+      in this storage with PyObject_{Init, InitVar}.
+
+   PyObject *
+   YourObject_New(...)
+   {
+       PyObject *op;
+
+       op = (PyObject *) Your_Allocator(_PyObject_SIZE(YourTypeStruct));
+       if (op == NULL) {
+           return PyErr_NoMemory();
+       }
+
+       PyObject_Init(op, &YourTypeStruct);
+
+       op->ob_field = value;
+       ...
+       return op;
+   }
+
+   Note that in C++, the use of the new operator usually implies that
+   the 1st step is performed automatically for you, so in a C++ class
+   constructor you would start directly with PyObject_Init/InitVar. */
+
+
+typedef struct {
+    /* user context passed as the first argument to the 2 functions */
+    void *ctx;
+
+    /* allocate an arena of size bytes */
+    void* (*alloc) (void *ctx, size_t size);
+
+    /* free an arena */
+    void (*free) (void *ctx, void *ptr, size_t size);
+} PyObjectArenaAllocator;
+
+/* Get the arena allocator. */
+PyAPI_FUNC(void) PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator);
+
+/* Set the arena allocator. */
+PyAPI_FUNC(void) PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator);
+
+
+/* Test if an object implements the garbage collector protocol */
+PyAPI_FUNC(int) PyObject_IS_GC(PyObject *obj);
+
+
+// Test if a type supports weak references
+PyAPI_FUNC(int) PyType_SUPPORTS_WEAKREFS(PyTypeObject *type);
+
+PyAPI_FUNC(PyObject **) PyObject_GET_WEAKREFS_LISTPTR(PyObject *op);
+
+PyAPI_FUNC(PyObject *) PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *,
+                                                             size_t);
+
+
+/* Visit all live GC-capable objects, similar to gc.get_objects(None). The
+ * supplied callback is called on every such object with the void* arg set
+ * to the supplied arg. Returning 0 from the callback ends iteration, returning
+ * 1 allows iteration to continue. Returning any other value may result in
+ * undefined behaviour.
+ *
+ * If new objects are (de)allocated by the callback it is undefined if they
+ * will be visited.
+
+ * Garbage collection is disabled during operation. Explicitly running a
+ * collection in the callback may lead to undefined behaviour e.g. visiting the
+ * same objects multiple times or not at all.
+ */
+typedef int (*gcvisitobjects_t)(PyObject*, void*);
+PyAPI_FUNC(void) PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void* arg);

Include/cpython/odictobject.h

@@ -0,0 +1,43 @@
+#ifndef Py_ODICTOBJECT_H
+#define Py_ODICTOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+/* OrderedDict */
+/* This API is optional and mostly redundant. */
+
+#ifndef Py_LIMITED_API
+
+typedef struct _odictobject PyODictObject;
+
+PyAPI_DATA(PyTypeObject) PyODict_Type;
+PyAPI_DATA(PyTypeObject) PyODictIter_Type;
+PyAPI_DATA(PyTypeObject) PyODictKeys_Type;
+PyAPI_DATA(PyTypeObject) PyODictItems_Type;
+PyAPI_DATA(PyTypeObject) PyODictValues_Type;
+
+#define PyODict_Check(op) PyObject_TypeCheck((op), &PyODict_Type)
+#define PyODict_CheckExact(op) Py_IS_TYPE((op), &PyODict_Type)
+#define PyODict_SIZE(op) PyDict_GET_SIZE((op))
+
+PyAPI_FUNC(PyObject *) PyODict_New(void);
+PyAPI_FUNC(int) PyODict_SetItem(PyObject *od, PyObject *key, PyObject *item);
+PyAPI_FUNC(int) PyODict_DelItem(PyObject *od, PyObject *key);
+
+/* wrappers around PyDict* functions */
+#define PyODict_GetItem(od, key) PyDict_GetItem(_PyObject_CAST(od), (key))
+#define PyODict_GetItemWithError(od, key) \
+    PyDict_GetItemWithError(_PyObject_CAST(od), (key))
+#define PyODict_Contains(od, key) PyDict_Contains(_PyObject_CAST(od), (key))
+#define PyODict_Size(od) PyDict_Size(_PyObject_CAST(od))
+#define PyODict_GetItemString(od, key) \
+    PyDict_GetItemString(_PyObject_CAST(od), (key))
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_ODICTOBJECT_H */

Include/cpython/picklebufobject.h

@@ -0,0 +1,31 @@
+/* PickleBuffer object. This is built-in for ease of use from third-party
+ * C extensions.
+ */
+
+#ifndef Py_PICKLEBUFOBJECT_H
+#define Py_PICKLEBUFOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_LIMITED_API
+
+PyAPI_DATA(PyTypeObject) PyPickleBuffer_Type;
+
+#define PyPickleBuffer_Check(op) Py_IS_TYPE((op), &PyPickleBuffer_Type)
+
+/* Create a PickleBuffer redirecting to the given buffer-enabled object */
+PyAPI_FUNC(PyObject *) PyPickleBuffer_FromObject(PyObject *);
+/* Get the PickleBuffer's underlying view to the original object
+ * (NULL if released)
+ */
+PyAPI_FUNC(const Py_buffer *) PyPickleBuffer_GetBuffer(PyObject *);
+/* Release the PickleBuffer.  Returns 0 on success, -1 on error. */
+PyAPI_FUNC(int) PyPickleBuffer_Release(PyObject *);
+
+#endif /* !Py_LIMITED_API */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PICKLEBUFOBJECT_H */

Include/cpython/pthread_stubs.h

@@ -0,0 +1,105 @@
+#ifndef Py_CPYTHON_PTRHEAD_STUBS_H
+#define Py_CPYTHON_PTRHEAD_STUBS_H
+
+#if !defined(HAVE_PTHREAD_STUBS)
+#  error "this header file requires stubbed pthreads."
+#endif
+
+#ifndef _POSIX_THREADS
+#  define _POSIX_THREADS 1
+#endif
+
+/* Minimal pthread stubs for CPython.
+ *
+ * The stubs implement the minimum pthread API for CPython.
+ * - pthread_create() fails.
+ * - pthread_exit() calls exit(0).
+ * - pthread_key_*() functions implement minimal TSS without destructor.
+ * - all other functions do nothing and return 0.
+ */
+
+#ifdef __wasi__
+// WASI's bits/alltypes.h provides type definitions when __NEED_ is set.
+// The header file can be included multiple times.
+//
+// <sys/types.h> may also define these macros.
+#  ifndef __NEED_pthread_cond_t
+#    define __NEED_pthread_cond_t 1
+#  endif
+#  ifndef __NEED_pthread_condattr_t
+#    define __NEED_pthread_condattr_t 1
+#  endif
+#  ifndef __NEED_pthread_mutex_t
+#    define __NEED_pthread_mutex_t 1
+#  endif
+#  ifndef __NEED_pthread_mutexattr_t
+#    define __NEED_pthread_mutexattr_t 1
+#  endif
+#  ifndef __NEED_pthread_key_t
+#    define __NEED_pthread_key_t 1
+#  endif
+#  ifndef __NEED_pthread_t
+#    define __NEED_pthread_t 1
+#  endif
+#  ifndef __NEED_pthread_attr_t
+#    define __NEED_pthread_attr_t 1
+#  endif
+#  include <bits/alltypes.h>
+#else
+typedef struct { void *__x; } pthread_cond_t;
+typedef struct { unsigned __attr; } pthread_condattr_t;
+typedef struct { void *__x; } pthread_mutex_t;
+typedef struct { unsigned __attr; } pthread_mutexattr_t;
+typedef unsigned pthread_key_t;
+typedef unsigned pthread_t;
+typedef struct { unsigned __attr; } pthread_attr_t;
+#endif
+
+// mutex
+PyAPI_FUNC(int) pthread_mutex_init(pthread_mutex_t *restrict mutex,
+                                   const pthread_mutexattr_t *restrict attr);
+PyAPI_FUNC(int) pthread_mutex_destroy(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_trylock(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_lock(pthread_mutex_t *mutex);
+PyAPI_FUNC(int) pthread_mutex_unlock(pthread_mutex_t *mutex);
+
+// condition
+PyAPI_FUNC(int) pthread_cond_init(pthread_cond_t *restrict cond,
+                                  const pthread_condattr_t *restrict attr);
+PyAPI_FUNC(int) pthread_cond_destroy(pthread_cond_t *cond);
+PyAPI_FUNC(int) pthread_cond_wait(pthread_cond_t *restrict cond,
+                                  pthread_mutex_t *restrict mutex);
+PyAPI_FUNC(int) pthread_cond_timedwait(pthread_cond_t *restrict cond,
+                                       pthread_mutex_t *restrict mutex,
+                                       const struct timespec *restrict abstime);
+PyAPI_FUNC(int) pthread_cond_signal(pthread_cond_t *cond);
+PyAPI_FUNC(int) pthread_condattr_init(pthread_condattr_t *attr);
+PyAPI_FUNC(int) pthread_condattr_setclock(
+    pthread_condattr_t *attr, clockid_t clock_id);
+
+// pthread
+PyAPI_FUNC(int) pthread_create(pthread_t *restrict thread,
+                               const pthread_attr_t *restrict attr,
+                               void *(*start_routine)(void *),
+                               void *restrict arg);
+PyAPI_FUNC(int) pthread_detach(pthread_t thread);
+PyAPI_FUNC(int) pthread_join(pthread_t thread, void** value_ptr);
+PyAPI_FUNC(pthread_t) pthread_self(void);
+PyAPI_FUNC(int) pthread_exit(void *retval) __attribute__ ((__noreturn__));
+PyAPI_FUNC(int) pthread_attr_init(pthread_attr_t *attr);
+PyAPI_FUNC(int) pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize);
+PyAPI_FUNC(int) pthread_attr_destroy(pthread_attr_t *attr);
+
+
+// pthread_key
+#ifndef PTHREAD_KEYS_MAX
+#  define PTHREAD_KEYS_MAX 128
+#endif
+
+PyAPI_FUNC(int) pthread_key_create(pthread_key_t *key,
+                                   void (*destr_function)(void *));
+PyAPI_FUNC(int) pthread_key_delete(pthread_key_t key);
+PyAPI_FUNC(void *) pthread_getspecific(pthread_key_t key);
+PyAPI_FUNC(int) pthread_setspecific(pthread_key_t key, const void *value);
+
+#endif // Py_CPYTHON_PTRHEAD_STUBS_H

Include/cpython/pyatomic.h

@@ -0,0 +1,569 @@
+// This header provides cross-platform low-level atomic operations
+// similar to C11 atomics.
+//
+// Operations are sequentially consistent unless they have a suffix indicating
+// otherwise. If in doubt, prefer the sequentially consistent operations.
+//
+// The "_relaxed" suffix for load and store operations indicates the "relaxed"
+// memory order. They don't provide synchronization, but (roughly speaking)
+// guarantee somewhat sane behavior for races instead of undefined behavior.
+// In practice, they correspond to "normal" hardware load and store
+// instructions, so they are almost as inexpensive as plain loads and stores
+// in C.
+//
+// Note that atomic read-modify-write operations like _Py_atomic_add_* return
+// the previous value of the atomic variable, not the new value.
+//
+// See https://en.cppreference.com/w/c/atomic for more information on C11
+// atomics.
+// See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf
+// "A Relaxed Guide to memory_order_relaxed" for discussion of and common usage
+// or relaxed atomics.
+//
+// Functions with pseudo Python code:
+//
+//   def _Py_atomic_load(obj):
+//        return obj  # sequential consistency
+//
+//   def _Py_atomic_load_relaxed(obj):
+//       return obj  # relaxed consistency
+//
+//   def _Py_atomic_store(obj, value):
+//       obj = value  # sequential consistency
+//
+//   def _Py_atomic_store_relaxed(obj, value):
+//       obj = value  # relaxed consistency
+//
+//   def _Py_atomic_exchange(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj = value
+//       return old_obj
+//
+//   def _Py_atomic_compare_exchange(obj, expected, desired):
+//       # sequential consistency
+//       if obj == expected:
+//           obj = desired
+//           return True
+//       else:
+//           expected = obj
+//           return False
+//
+//   def _Py_atomic_add(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj += value
+//       return old_obj
+//
+//   def _Py_atomic_and(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj &= value
+//       return old_obj
+//
+//   def _Py_atomic_or(obj, value):
+//       # sequential consistency
+//       old_obj = obj
+//       obj |= value
+//       return old_obj
+//
+// Other functions:
+//
+//   def _Py_atomic_load_ptr_acquire(obj):
+//       return obj  # acquire
+//
+//   def _Py_atomic_store_ptr_release(obj):
+//       return obj  # release
+//
+//   def _Py_atomic_fence_seq_cst():
+//       # sequential consistency
+//       ...
+//
+//   def _Py_atomic_fence_release():
+//       # release
+//       ...
+
+#ifndef Py_CPYTHON_ATOMIC_H
+#  error "this header file must not be included directly"
+#endif
+
+// --- _Py_atomic_add --------------------------------------------------------
+// Atomically adds `value` to `obj` and returns the previous value
+
+static inline int
+_Py_atomic_add_int(int *obj, int value);
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value);
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value);
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value);
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value);
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value);
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value);
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value);
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value);
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+// Performs an atomic compare-and-exchange.
+//
+// - If `*obj` and `*expected` are equal, store `desired` into `*obj`
+//   and return 1 (success).
+// - Otherwise, store the `*obj` current value into `*expected`
+//   and return 0 (failure).
+//
+// These correspond to the C11 atomic_compare_exchange_strong() function.
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired);
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired);
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired);
+
+// NOTE: `obj` and `expected` are logically `void**` types, but we use `void*`
+// so that we can pass types like `PyObject**` without a cast.
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value);
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+// Atomically replaces `*obj` with `value` and returns the previous value of `*obj`.
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value);
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value);
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value);
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value);
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value);
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value);
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value);
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value);
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value);
+
+static inline void *
+_Py_atomic_exchange_ptr(void *obj, void *value);
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+// Performs `*obj &= value` atomically and returns the previous value of `*obj`.
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value);
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+// Performs `*obj |= value` atomically and returns the previous value of `*obj`.
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value);
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value);
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value);
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value);
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value);
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+// Atomically loads `*obj` (sequential consistency)
+
+static inline int
+_Py_atomic_load_int(const int *obj);
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj);
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj);
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj);
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj);
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj);
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj);
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj);
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj);
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj);
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj);
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj);
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj);
+
+static inline void *
+_Py_atomic_load_ptr(const void *obj);
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+// Loads `*obj` (relaxed consistency, i.e., no ordering)
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj);
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj);
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj);
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj);
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj);
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj);
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj);
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj);
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj);
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj);
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj);
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj);
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj);
+
+static inline void *
+_Py_atomic_load_ptr_relaxed(const void *obj);
+
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj);
+
+// --- _Py_atomic_store ------------------------------------------------------
+// Atomically performs `*obj = value` (sequential consistency)
+
+static inline void
+_Py_atomic_store_int(int *obj, int value);
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value);
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value);
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value);
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value);
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value);
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value);
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value);
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value);
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value);
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value);
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value);
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value);
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t* obj, Py_ssize_t value);
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+// Stores `*obj = value` (relaxed consistency, i.e., no ordering)
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value);
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value);
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value);
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value);
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value);
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value);
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t* obj, uint8_t value);
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value);
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value);
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value);
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value);
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value);
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void *value);
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value);
+
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+                                unsigned long long value);
+
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+// Loads `*obj` (acquire operation)
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj);
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj);
+
+// Stores `*obj = value` (release operation)
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value);
+
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value);
+
+static inline void
+_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value);
+
+static inline void
+_Py_atomic_store_int_release(int *obj, int value);
+
+static inline int
+_Py_atomic_load_int_acquire(const int *obj);
+
+static inline void
+_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value);
+
+static inline void
+_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value);
+
+static inline uint64_t
+_Py_atomic_load_uint64_acquire(const uint64_t *obj);
+
+static inline uint32_t
+_Py_atomic_load_uint32_acquire(const uint32_t *obj);
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj);
+
+
+
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+// Sequential consistency fence. C11 fences have complex semantics. When
+// possible, use the atomic operations on variables defined above, which
+// generally do not require explicit use of a fence.
+// See https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence
+static inline void _Py_atomic_fence_seq_cst(void);
+
+// Acquire fence
+static inline void _Py_atomic_fence_acquire(void);
+
+// Release fence
+static inline void _Py_atomic_fence_release(void);
+
+
+#ifndef _Py_USE_GCC_BUILTIN_ATOMICS
+#  if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))
+#    define _Py_USE_GCC_BUILTIN_ATOMICS 1
+#  elif defined(__clang__)
+#    if __has_builtin(__atomic_load)
+#      define _Py_USE_GCC_BUILTIN_ATOMICS 1
+#    endif
+#  endif
+#endif
+
+#if _Py_USE_GCC_BUILTIN_ATOMICS
+#  define Py_ATOMIC_GCC_H
+#  include "pyatomic_gcc.h"
+#  undef Py_ATOMIC_GCC_H
+#elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)
+#  define Py_ATOMIC_STD_H
+#  include "pyatomic_std.h"
+#  undef Py_ATOMIC_STD_H
+#elif defined(_MSC_VER)
+#  define Py_ATOMIC_MSC_H
+#  include "pyatomic_msc.h"
+#  undef Py_ATOMIC_MSC_H
+#else
+#  error "no available pyatomic implementation for this platform/compiler"
+#endif
+
+
+// --- aliases ---------------------------------------------------------------
+
+#if SIZEOF_LONG == 8
+# define _Py_atomic_load_ulong(p) \
+    _Py_atomic_load_uint64((uint64_t *)p)
+# define _Py_atomic_load_ulong_relaxed(p) \
+    _Py_atomic_load_uint64_relaxed((uint64_t *)p)
+# define _Py_atomic_store_ulong(p, v) \
+    _Py_atomic_store_uint64((uint64_t *)p, v)
+# define _Py_atomic_store_ulong_relaxed(p, v) \
+    _Py_atomic_store_uint64_relaxed((uint64_t *)p, v)
+#elif SIZEOF_LONG == 4
+# define _Py_atomic_load_ulong(p) \
+    _Py_atomic_load_uint32((uint32_t *)p)
+# define _Py_atomic_load_ulong_relaxed(p) \
+    _Py_atomic_load_uint32_relaxed((uint32_t *)p)
+# define _Py_atomic_store_ulong(p, v) \
+    _Py_atomic_store_uint32((uint32_t *)p, v)
+# define _Py_atomic_store_ulong_relaxed(p, v) \
+    _Py_atomic_store_uint32_relaxed((uint32_t *)p, v)
+#else
+# error "long must be 4 or 8 bytes in size"
+#endif  // SIZEOF_LONG

Include/cpython/pyatomic_gcc.h

@@ -0,0 +1,551 @@
+// This is the implementation of Python atomic operations using GCC's built-in
+// functions that match the C+11 memory model. This implementation is preferred
+// for GCC compatible compilers, such as Clang. These functions are available
+// in GCC 4.8+ without needing to compile with --std=c11 or --std=gnu11.
+
+#ifndef Py_ATOMIC_GCC_H
+#  error "this header file must not be included directly"
+#endif
+
+
+// --- _Py_atomic_add --------------------------------------------------------
+
+static inline int
+_Py_atomic_add_int(int *obj, int value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{ return __atomic_fetch_add(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired)
+{ return __atomic_compare_exchange_n(obj, expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired)
+{ return __atomic_compare_exchange_n((void **)obj, (void **)expected, desired, 0,
+                                     __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{ return __atomic_exchange_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void *
+_Py_atomic_exchange_ptr(void *obj, void *value)
+{ return __atomic_exchange_n((void **)obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_fetch_and(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
+{ return __atomic_fetch_or(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+
+static inline int
+_Py_atomic_load_int(const int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_SEQ_CST); }
+
+static inline void *
+_Py_atomic_load_ptr(const void *obj)
+{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+static inline void *
+_Py_atomic_load_ptr_relaxed(const void *obj)
+{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_RELAXED); }
+
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
+{ return __atomic_load_n(obj, __ATOMIC_RELAXED); }
+
+
+// --- _Py_atomic_store ------------------------------------------------------
+
+static inline void
+_Py_atomic_store_int(int *obj, int value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value)
+{ __atomic_store_n((void **)obj, value, __ATOMIC_SEQ_CST); }
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_SEQ_CST); }
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void *value)
+{ __atomic_store_n((void **)obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+                                unsigned long long value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); }
+
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj)
+{ return (void *)__atomic_load_n((void * const *)obj, __ATOMIC_ACQUIRE); }
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
+{ return (uintptr_t)__atomic_load_n(obj, __ATOMIC_ACQUIRE); }
+
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value)
+{ __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); }
+
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
+static inline void
+_Py_atomic_store_int_release(int *obj, int value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
+static inline void
+_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
+static inline int
+_Py_atomic_load_int_acquire(const int *obj)
+{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); }
+
+static inline void
+_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
+static inline void
+_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
+static inline uint64_t
+_Py_atomic_load_uint64_acquire(const uint64_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); }
+
+static inline uint32_t
+_Py_atomic_load_uint32_acquire(const uint32_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); }
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)
+{ return __atomic_load_n(obj, __ATOMIC_ACQUIRE); }
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+static inline void
+_Py_atomic_fence_seq_cst(void)
+{ __atomic_thread_fence(__ATOMIC_SEQ_CST); }
+
+ static inline void
+_Py_atomic_fence_acquire(void)
+{ __atomic_thread_fence(__ATOMIC_ACQUIRE); }
+
+ static inline void
+_Py_atomic_fence_release(void)
+{ __atomic_thread_fence(__ATOMIC_RELEASE); }

Include/cpython/pyatomic_msc.h

@@ -0,0 +1,1095 @@
+// This is the implementation of Python atomic operations for MSVC if the
+// compiler does not support C11 or C++11 atomics.
+//
+// MSVC intrinsics are defined on char, short, long, __int64, and pointer
+// types. Note that long and int are both 32-bits even on 64-bit Windows,
+// so operations on int are cast to long.
+//
+// The volatile keyword has additional memory ordering semantics on MSVC. On
+// x86 and x86-64, volatile accesses have acquire-release semantics. On ARM64,
+// volatile accesses behave like C11's memory_order_relaxed.
+
+#ifndef Py_ATOMIC_MSC_H
+#  error "this header file must not be included directly"
+#endif
+
+#include <intrin.h>
+
+#define _Py_atomic_ASSERT_ARG_TYPE(TYPE) \
+    Py_BUILD_ASSERT(sizeof(*obj) == sizeof(TYPE))
+
+
+// --- _Py_atomic_add --------------------------------------------------------
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (int8_t)_InterlockedExchangeAdd8((volatile char *)obj, (char)value);
+}
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (int16_t)_InterlockedExchangeAdd16((volatile short *)obj, (short)value);
+}
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (int32_t)_InterlockedExchangeAdd((volatile long *)obj, (long)value);
+}
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (int64_t)_InterlockedExchangeAdd64((volatile __int64 *)obj, (__int64)value);
+#else
+    int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
+    for (;;) {
+        int64_t new_value = old_value + value;
+        if (_Py_atomic_compare_exchange_int64(obj, &old_value, new_value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
+{
+    return (uint8_t)_Py_atomic_add_int8((int8_t *)obj, (int8_t)value);
+}
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
+{
+    return (uint16_t)_Py_atomic_add_int16((int16_t *)obj, (int16_t)value);
+}
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
+{
+    return (uint32_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+}
+
+static inline int
+_Py_atomic_add_int(int *obj, int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+}
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (unsigned int)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+}
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
+{
+    return (uint64_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
+}
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
+{
+#if SIZEOF_VOID_P == 8
+    _Py_atomic_ASSERT_ARG_TYPE(int64_t);
+    return (intptr_t)_Py_atomic_add_int64((int64_t *)obj, (int64_t)value);
+#else
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (intptr_t)_Py_atomic_add_int32((int32_t *)obj, (int32_t)value);
+#endif
+}
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
+    return (uintptr_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(intptr_t);
+    return (Py_ssize_t)_Py_atomic_add_intptr((intptr_t *)obj, (intptr_t)value);
+}
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    int8_t initial = (int8_t)_InterlockedCompareExchange8(
+                                       (volatile char *)obj,
+                                       (char)value,
+                                       (char)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    int16_t initial = (int16_t)_InterlockedCompareExchange16(
+                                       (volatile short *)obj,
+                                       (short)value,
+                                       (short)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    int32_t initial = (int32_t)_InterlockedCompareExchange(
+                                       (volatile long *)obj,
+                                       (long)value,
+                                       (long)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    int64_t initial = (int64_t)_InterlockedCompareExchange64(
+                                       (volatile __int64 *)obj,
+                                       (__int64)value,
+                                       (__int64)*expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+}
+
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *value)
+{
+    void *initial = _InterlockedCompareExchangePointer(
+                                       (void**)obj,
+                                       value,
+                                       *(void**)expected);
+    if (initial == *(void**)expected) {
+        return 1;
+    }
+    *(void**)expected = initial;
+    return 0;
+}
+
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t value)
+{
+    return _Py_atomic_compare_exchange_int8((int8_t *)obj,
+                                            (int8_t *)expected,
+                                            (int8_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t value)
+{
+    return _Py_atomic_compare_exchange_int16((int16_t *)obj,
+                                             (int16_t *)expected,
+                                             (int16_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t value)
+{
+    return _Py_atomic_compare_exchange_int32((int32_t *)obj,
+                                             (int32_t *)expected,
+                                             (int32_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return _Py_atomic_compare_exchange_int32((int32_t *)obj,
+                                             (int32_t *)expected,
+                                             (int32_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return _Py_atomic_compare_exchange_int32((int32_t *)obj,
+                                             (int32_t *)expected,
+                                             (int32_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t value)
+{
+    return _Py_atomic_compare_exchange_int64((int64_t *)obj,
+                                             (int64_t *)expected,
+                                             (int64_t)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return _Py_atomic_compare_exchange_ptr((void**)obj,
+                                           (void**)expected,
+                                           (void*)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return _Py_atomic_compare_exchange_ptr((void**)obj,
+                                           (void**)expected,
+                                           (void*)value);
+}
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return _Py_atomic_compare_exchange_ptr((void**)obj,
+                                           (void**)expected,
+                                           (void*)value);
+}
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (int8_t)_InterlockedExchange8((volatile char *)obj, (char)value);
+}
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (int16_t)_InterlockedExchange16((volatile short *)obj, (short)value);
+}
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (int32_t)_InterlockedExchange((volatile long *)obj, (long)value);
+}
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (int64_t)_InterlockedExchange64((volatile __int64 *)obj, (__int64)value);
+#else
+    int64_t old_value = _Py_atomic_load_int64_relaxed(obj);
+    for (;;) {
+        if (_Py_atomic_compare_exchange_int64(obj, &old_value, value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+static inline void*
+_Py_atomic_exchange_ptr(void *obj, void *value)
+{
+    return (void*)_InterlockedExchangePointer((void * volatile *)obj, (void *)value);
+}
+
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
+{
+    return (uint8_t)_Py_atomic_exchange_int8((int8_t *)obj,
+                                             (int8_t)value);
+}
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
+{
+    return (uint16_t)_Py_atomic_exchange_int16((int16_t *)obj,
+                                               (int16_t)value);
+}
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
+{
+    return (uint32_t)_Py_atomic_exchange_int32((int32_t *)obj,
+                                               (int32_t)value);
+}
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (int)_Py_atomic_exchange_int32((int32_t *)obj,
+                                           (int32_t)value);
+}
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(int32_t);
+    return (unsigned int)_Py_atomic_exchange_int32((int32_t *)obj,
+                                                   (int32_t)value);
+}
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
+{
+    return (uint64_t)_Py_atomic_exchange_int64((int64_t *)obj,
+                                               (int64_t)value);
+}
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (intptr_t)_Py_atomic_exchange_ptr((void**)obj,
+                                             (void*)value);
+}
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (uintptr_t)_Py_atomic_exchange_ptr((void**)obj,
+                                              (void*)value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (Py_ssize_t)_Py_atomic_exchange_ptr((void**)obj,
+                                               (void*)value);
+}
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (uint8_t)_InterlockedAnd8((volatile char *)obj, (char)value);
+}
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (uint16_t)_InterlockedAnd16((volatile short *)obj, (short)value);
+}
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (uint32_t)_InterlockedAnd((volatile long *)obj, (long)value);
+}
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (uint64_t)_InterlockedAnd64((volatile __int64 *)obj, (__int64)value);
+#else
+    uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
+    for (;;) {
+        uint64_t new_value = old_value & value;
+        if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
+{
+#if SIZEOF_VOID_P == 8
+    _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
+    return (uintptr_t)_Py_atomic_and_uint64((uint64_t *)obj,
+                                            (uint64_t)value);
+#else
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (uintptr_t)_Py_atomic_and_uint32((uint32_t *)obj,
+                                            (uint32_t)value);
+#endif
+}
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(char);
+    return (uint8_t)_InterlockedOr8((volatile char *)obj, (char)value);
+}
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(short);
+    return (uint16_t)_InterlockedOr16((volatile short *)obj, (short)value);
+}
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(long);
+    return (uint32_t)_InterlockedOr((volatile long *)obj, (long)value);
+}
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
+{
+#if defined(_M_X64) || defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (uint64_t)_InterlockedOr64((volatile __int64 *)obj, (__int64)value);
+#else
+    uint64_t old_value = _Py_atomic_load_uint64_relaxed(obj);
+    for (;;) {
+        uint64_t new_value = old_value | value;
+        if (_Py_atomic_compare_exchange_uint64(obj, &old_value, new_value)) {
+            return old_value;
+        }
+    }
+#endif
+}
+
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
+{
+#if SIZEOF_VOID_P == 8
+    _Py_atomic_ASSERT_ARG_TYPE(uint64_t);
+    return (uintptr_t)_Py_atomic_or_uint64((uint64_t *)obj,
+                                           (uint64_t)value);
+#else
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (uintptr_t)_Py_atomic_or_uint32((uint32_t *)obj,
+                                           (uint32_t)value);
+#endif
+}
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint8_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint8_t)__ldar8((unsigned __int8 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint8"
+#endif
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint16_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint16_t)__ldar16((unsigned __int16 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint16"
+#endif
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint32_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint32_t)__ldar32((unsigned __int32 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint32"
+#endif
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile uint64_t *)obj;
+#elif defined(_M_ARM64)
+    return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint64"
+#endif
+}
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj)
+{
+    return (int8_t)_Py_atomic_load_uint8((const uint8_t *)obj);
+}
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj)
+{
+    return (int16_t)_Py_atomic_load_uint16((const uint16_t *)obj);
+}
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj)
+{
+    return (int32_t)_Py_atomic_load_uint32((const uint32_t *)obj);
+}
+
+static inline int
+_Py_atomic_load_int(const int *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (int)_Py_atomic_load_uint32((uint32_t *)obj);
+}
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(uint32_t);
+    return (unsigned int)_Py_atomic_load_uint32((uint32_t *)obj);
+}
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj)
+{
+    return (int64_t)_Py_atomic_load_uint64((const uint64_t *)obj);
+}
+
+static inline void*
+_Py_atomic_load_ptr(const void *obj)
+{
+#if SIZEOF_VOID_P == 8
+    return (void*)_Py_atomic_load_uint64((const uint64_t *)obj);
+#else
+    return (void*)_Py_atomic_load_uint32((const uint32_t *)obj);
+#endif
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (intptr_t)_Py_atomic_load_ptr((void*)obj);
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (uintptr_t)_Py_atomic_load_ptr((void*)obj);
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj)
+{
+    _Py_atomic_ASSERT_ARG_TYPE(void*);
+    return (Py_ssize_t)_Py_atomic_load_ptr((void*)obj);
+}
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj)
+{
+    return *(volatile int *)obj;
+}
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj)
+{
+    return *(volatile int8_t *)obj;
+}
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj)
+{
+    return *(volatile int16_t *)obj;
+}
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj)
+{
+    return *(volatile int32_t *)obj;
+}
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj)
+{
+    return *(volatile int64_t *)obj;
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj)
+{
+    return *(volatile intptr_t *)obj;
+}
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj)
+{
+    return *(volatile uint8_t *)obj;
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj)
+{
+    return *(volatile uint16_t *)obj;
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj)
+{
+    return *(volatile uint32_t *)obj;
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj)
+{
+    return *(volatile uint64_t *)obj;
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
+{
+    return *(volatile uintptr_t *)obj;
+}
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj)
+{
+    return *(volatile unsigned int *)obj;
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
+{
+    return *(volatile Py_ssize_t *)obj;
+}
+
+static inline void*
+_Py_atomic_load_ptr_relaxed(const void *obj)
+{
+    return *(void * volatile *)obj;
+}
+
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
+{
+    return *(volatile unsigned long long *)obj;
+}
+
+
+// --- _Py_atomic_store ------------------------------------------------------
+
+static inline void
+_Py_atomic_store_int(int *obj, int value)
+{
+    (void)_Py_atomic_exchange_int(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value)
+{
+    (void)_Py_atomic_exchange_int8(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value)
+{
+    (void)_Py_atomic_exchange_int16(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value)
+{
+    (void)_Py_atomic_exchange_int32(obj, value);
+}
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value)
+{
+    (void)_Py_atomic_exchange_int64(obj, value);
+}
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
+{
+    (void)_Py_atomic_exchange_intptr(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
+{
+    (void)_Py_atomic_exchange_uint8(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
+{
+    (void)_Py_atomic_exchange_uint16(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
+{
+    (void)_Py_atomic_exchange_uint32(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
+{
+    (void)_Py_atomic_exchange_uint64(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    (void)_Py_atomic_exchange_uintptr(obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value)
+{
+    (void)_Py_atomic_exchange_uint(obj, value);
+}
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value)
+{
+    (void)_Py_atomic_exchange_ptr(obj, value);
+}
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    (void)_Py_atomic_exchange_ssize(obj, value);
+}
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value)
+{
+    *(volatile int *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
+{
+    *(volatile int8_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
+{
+    *(volatile int16_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
+{
+    *(volatile int32_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
+{
+    *(volatile int64_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
+{
+    *(volatile intptr_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
+{
+    *(volatile uint8_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
+{
+    *(volatile uint16_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
+{
+    *(volatile uint32_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
+{
+    *(volatile uint64_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
+{
+    *(volatile uintptr_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
+{
+    *(volatile unsigned int *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void* value)
+{
+    *(void * volatile *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
+{
+    *(volatile Py_ssize_t *)obj = value;
+}
+
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+                                unsigned long long value)
+{
+    *(volatile unsigned long long *)obj = value;
+}
+
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(void * volatile *)obj;
+#elif defined(_M_ARM64)
+    return (void *)__ldar64((unsigned __int64 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_ptr_acquire"
+#endif
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(uintptr_t volatile *)obj;
+#elif defined(_M_ARM64)
+    return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uintptr_acquire"
+#endif
+}
+
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    *(void * volatile *)obj = value;
+#elif defined(_M_ARM64)
+    __stlr64((unsigned __int64 volatile *)obj, (uintptr_t)value);
+#else
+#  error "no implementation of _Py_atomic_store_ptr_release"
+#endif
+}
+
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    *(uintptr_t volatile *)obj = value;
+#elif defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
+    __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
+#else
+#  error "no implementation of _Py_atomic_store_uintptr_release"
+#endif
+}
+
+static inline void
+_Py_atomic_store_int_release(int *obj, int value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    *(int volatile *)obj = value;
+#elif defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
+    __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);
+#else
+#  error "no implementation of _Py_atomic_store_int_release"
+#endif
+}
+
+static inline void
+_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    *(Py_ssize_t volatile *)obj = value;
+#elif defined(_M_ARM64)
+    __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
+#else
+#  error "no implementation of _Py_atomic_store_ssize_release"
+#endif
+}
+
+static inline int
+_Py_atomic_load_int_acquire(const int *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(int volatile *)obj;
+#elif defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
+    return (int)__ldar32((unsigned __int32 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_int_acquire"
+#endif
+}
+
+static inline void
+_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    *(uint32_t volatile *)obj = value;
+#elif defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32);
+    __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value);
+#else
+#  error "no implementation of _Py_atomic_store_uint32_release"
+#endif
+}
+
+static inline void
+_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    *(uint64_t volatile *)obj = value;
+#elif defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
+    __stlr64((unsigned __int64 volatile *)obj, (unsigned __int64)value);
+#else
+#  error "no implementation of _Py_atomic_store_uint64_release"
+#endif
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64_acquire(const uint64_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(uint64_t volatile *)obj;
+#elif defined(_M_ARM64)
+    _Py_atomic_ASSERT_ARG_TYPE(__int64);
+    return (uint64_t)__ldar64((unsigned __int64 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint64_acquire"
+#endif
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32_acquire(const uint32_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(uint32_t volatile *)obj;
+#elif defined(_M_ARM64)
+    return (uint32_t)__ldar32((uint32_t volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_uint32_acquire"
+#endif
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(Py_ssize_t volatile *)obj;
+#elif defined(_M_ARM64)
+    return (Py_ssize_t)__ldar64((unsigned __int64 volatile *)obj);
+#else
+#  error "no implementation of _Py_atomic_load_ssize_acquire"
+#endif
+}
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+ static inline void
+_Py_atomic_fence_seq_cst(void)
+{
+#if defined(_M_ARM64)
+    __dmb(_ARM64_BARRIER_ISH);
+#elif defined(_M_X64)
+    __faststorefence();
+#elif defined(_M_IX86)
+    _mm_mfence();
+#else
+#  error "no implementation of _Py_atomic_fence_seq_cst"
+#endif
+}
+
+ static inline void
+_Py_atomic_fence_acquire(void)
+{
+#if defined(_M_ARM64)
+    __dmb(_ARM64_BARRIER_ISHLD);
+#elif defined(_M_X64) || defined(_M_IX86)
+    _ReadBarrier();
+#else
+#  error "no implementation of _Py_atomic_fence_acquire"
+#endif
+}
+
+ static inline void
+_Py_atomic_fence_release(void)
+{
+#if defined(_M_ARM64)
+    __dmb(_ARM64_BARRIER_ISH);
+#elif defined(_M_X64) || defined(_M_IX86)
+    _ReadWriteBarrier();
+#else
+#  error "no implementation of _Py_atomic_fence_release"
+#endif
+}
+
+#undef _Py_atomic_ASSERT_ARG_TYPE

Include/cpython/pyatomic_std.h

@@ -0,0 +1,976 @@
+// This is the implementation of Python atomic operations using C++11 or C11
+// atomics. Note that the pyatomic_gcc.h implementation is preferred for GCC
+// compatible compilers, even if they support C++11 atomics.
+
+#ifndef Py_ATOMIC_STD_H
+#  error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C++" {
+#  include <atomic>
+}
+#  define _Py_USING_STD using namespace std
+#  define _Atomic(tp) atomic<tp>
+#else
+#  define  _Py_USING_STD
+#  include <stdatomic.h>
+#endif
+
+
+// --- _Py_atomic_add --------------------------------------------------------
+
+static inline int
+_Py_atomic_add_int(int *obj, int value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int)*)obj, value);
+}
+
+static inline int8_t
+_Py_atomic_add_int8(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int8_t)*)obj, value);
+}
+
+static inline int16_t
+_Py_atomic_add_int16(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int16_t)*)obj, value);
+}
+
+static inline int32_t
+_Py_atomic_add_int32(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int32_t)*)obj, value);
+}
+
+static inline int64_t
+_Py_atomic_add_int64(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(int64_t)*)obj, value);
+}
+
+static inline intptr_t
+_Py_atomic_add_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(intptr_t)*)obj, value);
+}
+
+static inline unsigned int
+_Py_atomic_add_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(unsigned int)*)obj, value);
+}
+
+static inline uint8_t
+_Py_atomic_add_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_add_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_add_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_add_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_add_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(uintptr_t)*)obj, value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_add_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_add((_Atomic(Py_ssize_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_compare_exchange -------------------------------------------
+
+static inline int
+_Py_atomic_compare_exchange_int(int *obj, int *expected, int desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int8(int8_t *obj, int8_t *expected, int8_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int16(int16_t *obj, int16_t *expected, int16_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int16_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int32(int32_t *obj, int32_t *expected, int32_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int32_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_int64(int64_t *obj, int64_t *expected, int64_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(int64_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_intptr(intptr_t *obj, intptr_t *expected, intptr_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(intptr_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint(unsigned int *obj, unsigned int *expected, unsigned int desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(unsigned int)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint8(uint8_t *obj, uint8_t *expected, uint8_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint8_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint16(uint16_t *obj, uint16_t *expected, uint16_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint16_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint32(uint32_t *obj, uint32_t *expected, uint32_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint32_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uint64(uint64_t *obj, uint64_t *expected, uint64_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uint64_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_uintptr(uintptr_t *obj, uintptr_t *expected, uintptr_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(uintptr_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_ssize(Py_ssize_t *obj, Py_ssize_t *expected, Py_ssize_t desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(Py_ssize_t)*)obj,
+                                          expected, desired);
+}
+
+static inline int
+_Py_atomic_compare_exchange_ptr(void *obj, void *expected, void *desired)
+{
+    _Py_USING_STD;
+    return atomic_compare_exchange_strong((_Atomic(void *)*)obj,
+                                          (void **)expected, desired);
+}
+
+
+// --- _Py_atomic_exchange ---------------------------------------------------
+
+static inline int
+_Py_atomic_exchange_int(int *obj, int value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int)*)obj, value);
+}
+
+static inline int8_t
+_Py_atomic_exchange_int8(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int8_t)*)obj, value);
+}
+
+static inline int16_t
+_Py_atomic_exchange_int16(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int16_t)*)obj, value);
+}
+
+static inline int32_t
+_Py_atomic_exchange_int32(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int32_t)*)obj, value);
+}
+
+static inline int64_t
+_Py_atomic_exchange_int64(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(int64_t)*)obj, value);
+}
+
+static inline intptr_t
+_Py_atomic_exchange_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(intptr_t)*)obj, value);
+}
+
+static inline unsigned int
+_Py_atomic_exchange_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(unsigned int)*)obj, value);
+}
+
+static inline uint8_t
+_Py_atomic_exchange_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_exchange_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_exchange_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_exchange_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_exchange_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(uintptr_t)*)obj, value);
+}
+
+static inline Py_ssize_t
+_Py_atomic_exchange_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(Py_ssize_t)*)obj, value);
+}
+
+static inline void*
+_Py_atomic_exchange_ptr(void *obj, void *value)
+{
+    _Py_USING_STD;
+    return atomic_exchange((_Atomic(void *)*)obj, value);
+}
+
+
+// --- _Py_atomic_and --------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_and_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_and_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_and_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_and_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_and_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_and((_Atomic(uintptr_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_or ---------------------------------------------------------
+
+static inline uint8_t
+_Py_atomic_or_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline uint16_t
+_Py_atomic_or_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline uint32_t
+_Py_atomic_or_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline uint64_t
+_Py_atomic_or_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline uintptr_t
+_Py_atomic_or_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    return atomic_fetch_or((_Atomic(uintptr_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_load -------------------------------------------------------
+
+static inline int
+_Py_atomic_load_int(const int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int)*)obj);
+}
+
+static inline int8_t
+_Py_atomic_load_int8(const int8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int8_t)*)obj);
+}
+
+static inline int16_t
+_Py_atomic_load_int16(const int16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int16_t)*)obj);
+}
+
+static inline int32_t
+_Py_atomic_load_int32(const int32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int32_t)*)obj);
+}
+
+static inline int64_t
+_Py_atomic_load_int64(const int64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(int64_t)*)obj);
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr(const intptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(intptr_t)*)obj);
+}
+
+static inline uint8_t
+_Py_atomic_load_uint8(const uint8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint8_t)*)obj);
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16(const uint16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint32_t)*)obj);
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32(const uint32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint32_t)*)obj);
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64(const uint64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uint64_t)*)obj);
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr(const uintptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(uintptr_t)*)obj);
+}
+
+static inline unsigned int
+_Py_atomic_load_uint(const unsigned int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(unsigned int)*)obj);
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize(const Py_ssize_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(Py_ssize_t)*)obj);
+}
+
+static inline void*
+_Py_atomic_load_ptr(const void *obj)
+{
+    _Py_USING_STD;
+    return atomic_load((const _Atomic(void*)*)obj);
+}
+
+
+// --- _Py_atomic_load_relaxed -----------------------------------------------
+
+static inline int
+_Py_atomic_load_int_relaxed(const int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int8_t
+_Py_atomic_load_int8_relaxed(const int8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int8_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int16_t
+_Py_atomic_load_int16_relaxed(const int16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int16_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int32_t
+_Py_atomic_load_int32_relaxed(const int32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int32_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline int64_t
+_Py_atomic_load_int64_relaxed(const int64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int64_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline intptr_t
+_Py_atomic_load_intptr_relaxed(const intptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(intptr_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint8_t
+_Py_atomic_load_uint8_relaxed(const uint8_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint8_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint16_t
+_Py_atomic_load_uint16_relaxed(const uint16_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint16_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32_relaxed(const uint32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint32_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64_relaxed(const uint64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint64_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_relaxed(const uintptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline unsigned int
+_Py_atomic_load_uint_relaxed(const unsigned int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(unsigned int)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_relaxed(const Py_ssize_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline void*
+_Py_atomic_load_ptr_relaxed(const void *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(void*)*)obj,
+                                memory_order_relaxed);
+}
+
+static inline unsigned long long
+_Py_atomic_load_ullong_relaxed(const unsigned long long *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(unsigned long long)*)obj,
+                                memory_order_relaxed);
+}
+
+
+// --- _Py_atomic_store ------------------------------------------------------
+
+static inline void
+_Py_atomic_store_int(int *obj, int value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int8(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int8_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int16(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int16_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int32(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int32_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_int64(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(int64_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_intptr(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(intptr_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint8(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint8_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint16(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint16_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint32(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint32_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint64(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uint64_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uintptr(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(uintptr_t)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_uint(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(unsigned int)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_ptr(void *obj, void *value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(void*)*)obj, value);
+}
+
+static inline void
+_Py_atomic_store_ssize(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    atomic_store((_Atomic(Py_ssize_t)*)obj, value);
+}
+
+
+// --- _Py_atomic_store_relaxed ----------------------------------------------
+
+static inline void
+_Py_atomic_store_int_relaxed(int *obj, int value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int8_relaxed(int8_t *obj, int8_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int8_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int16_relaxed(int16_t *obj, int16_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int16_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int32_relaxed(int32_t *obj, int32_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int32_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_int64_relaxed(int64_t *obj, int64_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int64_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_intptr_relaxed(intptr_t *obj, intptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(intptr_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint8_relaxed(uint8_t *obj, uint8_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint8_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint16_relaxed(uint16_t *obj, uint16_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint16_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint32_relaxed(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint32_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint64_relaxed(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint64_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uintptr_relaxed(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_uint_relaxed(unsigned int *obj, unsigned int value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(unsigned int)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_ptr_relaxed(void *obj, void *value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(void*)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_ssize_relaxed(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value,
+                          memory_order_relaxed);
+}
+
+static inline void
+_Py_atomic_store_ullong_relaxed(unsigned long long *obj,
+                                unsigned long long value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(unsigned long long)*)obj, value,
+                          memory_order_relaxed);
+}
+
+
+// --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------
+
+static inline void *
+_Py_atomic_load_ptr_acquire(const void *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(void*)*)obj,
+                                memory_order_acquire);
+}
+
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
+                                memory_order_acquire);
+}
+
+static inline void
+_Py_atomic_store_ptr_release(void *obj, void *value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(void*)*)obj, value,
+                          memory_order_release);
+}
+
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
+                          memory_order_release);
+}
+
+static inline void
+_Py_atomic_store_int_release(int *obj, int value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(int)*)obj, value,
+                          memory_order_release);
+}
+
+static inline void
+_Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(Py_ssize_t)*)obj, value,
+                          memory_order_release);
+}
+
+static inline int
+_Py_atomic_load_int_acquire(const int *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(int)*)obj,
+                                memory_order_acquire);
+}
+
+static inline void
+_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint32_t)*)obj, value,
+                          memory_order_release);
+}
+
+static inline void
+_Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value)
+{
+    _Py_USING_STD;
+    atomic_store_explicit((_Atomic(uint64_t)*)obj, value,
+                          memory_order_release);
+}
+
+static inline uint64_t
+_Py_atomic_load_uint64_acquire(const uint64_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint64_t)*)obj,
+                                memory_order_acquire);
+}
+
+static inline uint32_t
+_Py_atomic_load_uint32_acquire(const uint32_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(uint32_t)*)obj,
+                                memory_order_acquire);
+}
+
+static inline Py_ssize_t
+_Py_atomic_load_ssize_acquire(const Py_ssize_t *obj)
+{
+    _Py_USING_STD;
+    return atomic_load_explicit((const _Atomic(Py_ssize_t)*)obj,
+                                memory_order_acquire);
+}
+
+
+// --- _Py_atomic_fence ------------------------------------------------------
+
+ static inline void
+_Py_atomic_fence_seq_cst(void)
+{
+    _Py_USING_STD;
+    atomic_thread_fence(memory_order_seq_cst);
+}
+
+ static inline void
+_Py_atomic_fence_acquire(void)
+{
+    _Py_USING_STD;
+    atomic_thread_fence(memory_order_acquire);
+}
+
+ static inline void
+_Py_atomic_fence_release(void)
+{
+    _Py_USING_STD;
+    atomic_thread_fence(memory_order_release);
+}