Headline
CVE-2022-31116: Merge pull request #555 from JustAnotherArchivist/fix-decode-surrogat… · ultrajson/ultrajson@67ec071
UltraJSON is a fast JSON encoder and decoder written in pure C with bindings for Python 3.7+. Affected versions were found to improperly decode certain characters. JSON strings that contain escaped surrogate characters not part of a proper surrogate pair were decoded incorrectly. Besides corrupting strings, this allowed for potential key confusion and value overwriting in dictionaries. All users parsing JSON from untrusted sources are vulnerable. From version 5.4.0, UltraJSON decodes lone surrogates in the same way as the standard library’s json module does, preserving them in the parsed output. Users are advised to upgrade. There are no known workarounds for this issue.
@@ -41,7 +41,6 @@ Numeric decoder derived from from TCL library #include <assert.h> #include <string.h> #include <limits.h> #include <wchar.h> #include <stdlib.h> #include <errno.h> #include <stdint.h> @@ -58,8 +57,8 @@ struct DecoderState { char *start; char *end; wchar_t *escStart; wchar_t *escEnd; JSUINT32 *escStart; JSUINT32 *escEnd; int escHeap; int lastType; JSUINT32 objDepth; @@ -361,14 +360,12 @@ static const JSUINT8 g_decoderLookup[256] = static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds) { int index; wchar_t *escOffset; wchar_t *escStart; JSUINT32 *escOffset; JSUINT32 *escStart; size_t escLen = (ds->escEnd - ds->escStart); JSUINT8 *inputOffset; JSUTF16 ch = 0; #if WCHAR_MAX >= 0x10FFFF JSUINT8 *lastHighSurrogate = NULL; #endif JSUINT8 oct; JSUTF32 ucs; ds->lastType = JT_INVALID; @@ -380,11 +377,11 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds
if (ds->escHeap) { if (newSize > (SIZE_MAX / sizeof(wchar_t))) if (newSize > (SIZE_MAX / sizeof(JSUINT32))) { return SetError(ds, -1, “Could not reserve memory block”); } escStart = (wchar_t *)ds->dec->realloc(ds->escStart, newSize * sizeof(wchar_t)); escStart = (JSUINT32 *)ds->dec->realloc(ds->escStart, newSize * sizeof(JSUINT32)); if (!escStart) { ds->dec->free(ds->escStart); @@ -394,18 +391,18 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds } else { wchar_t *oldStart = ds->escStart; if (newSize > (SIZE_MAX / sizeof(wchar_t))) JSUINT32 *oldStart = ds->escStart; if (newSize > (SIZE_MAX / sizeof(JSUINT32))) { return SetError(ds, -1, “Could not reserve memory block”); } ds->escStart = (wchar_t *) ds->dec->malloc(newSize * sizeof(wchar_t)); ds->escStart = (JSUINT32 *) ds->dec->malloc(newSize * sizeof(JSUINT32)); if (!ds->escStart) { return SetError(ds, -1, “Could not reserve memory block”); } ds->escHeap = 1; memcpy(ds->escStart, oldStart, escLen * sizeof(wchar_t)); memcpy(ds->escStart, oldStart, escLen * sizeof(JSUINT32)); }
ds->escEnd = ds->escStart + newSize; @@ -438,14 +435,14 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds inputOffset ++; switch (*inputOffset) { case '\\’: *(escOffset++) = L’\\’; inputOffset++; continue; case '\"’: *(escOffset++) = L’\"’; inputOffset++; continue; case '/’: *(escOffset++) = L’/’; inputOffset++; continue; case 'b’: *(escOffset++) = L’\b’; inputOffset++; continue; case 'f’: *(escOffset++) = L’\f’; inputOffset++; continue; case 'n’: *(escOffset++) = L’\n’; inputOffset++; continue; case 'r’: *(escOffset++) = L’\r’; inputOffset++; continue; case 't’: *(escOffset++) = L’\t’; inputOffset++; continue; case '\\’: *(escOffset++) = '\\’; inputOffset++; continue; case '\"’: *(escOffset++) = '\"’; inputOffset++; continue; case '/’: *(escOffset++) = '/’; inputOffset++; continue; case 'b’: *(escOffset++) = '\b’; inputOffset++; continue; case 'f’: *(escOffset++) = '\f’; inputOffset++; continue; case 'n’: *(escOffset++) = '\n’; inputOffset++; continue; case 'r’: *(escOffset++) = '\r’; inputOffset++; continue; case 't’: *(escOffset++) = '\t’; inputOffset++; continue;
case 'u’: { @@ -494,24 +491,20 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds inputOffset ++; }
#if WCHAR_MAX >= 0x10FFFF if ((ch & 0xfc00) == 0xdc00 && lastHighSurrogate == inputOffset - 6 * sizeof(*inputOffset)) { // Low surrogate immediately following a high surrogate // Overwrite existing high surrogate with combined character *(escOffset-1) = (((*(escOffset-1) - 0xd800) <<10) | (ch - 0xdc00)) + 0x10000; } else #endif { *(escOffset++) = (wchar_t) ch; *(escOffset++) = (JSUINT32) ch; } #if WCHAR_MAX >= 0x10FFFF if ((ch & 0xfc00) == 0xd800) { lastHighSurrogate = inputOffset; } #endif break; }
@@ -523,7 +516,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds
case 1: { *(escOffset++) = (wchar_t) (*inputOffset++); *(escOffset++) = (JSUINT32) (*inputOffset++); break; }
@@ -537,7 +530,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds } ucs |= (*inputOffset++) & 0x3f; if (ucs < 0x80) return SetError (ds, -1, “Overlong 2 byte UTF-8 sequence detected when decoding 'string’”); *(escOffset++) = (wchar_t) ucs; *(escOffset++) = (JSUINT32) ucs; break; }
@@ -560,7 +553,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds }
if (ucs < 0x800) return SetError (ds, -1, “Overlong 3 byte UTF-8 sequence detected when encoding string”); *(escOffset++) = (wchar_t) ucs; *(escOffset++) = (JSUINT32) ucs; break; }
@@ -584,20 +577,7 @@ static FASTCALL_ATTR JSOBJ FASTCALL_MSVC decode_string ( struct DecoderState *ds
if (ucs < 0x10000) return SetError (ds, -1, “Overlong 4 byte UTF-8 sequence detected when decoding 'string’”);
#if WCHAR_MAX == 0xffff if (ucs >= 0x10000) { ucs -= 0x10000; *(escOffset++) = (wchar_t) (ucs >> 10) + 0xd800; *(escOffset++) = (wchar_t) (ucs & 0x3ff) + 0xdc00; } else { *(escOffset++) = (wchar_t) ucs; } #else *(escOffset++) = (wchar_t) ucs; #endif *(escOffset++) = (JSUINT32) ucs; break; } } @@ -810,14 +790,14 @@ JSOBJ JSON_DecodeObject(JSONObjectDecoder *dec, const char *buffer, size_t cbBuf /* FIXME: Base the size of escBuffer of that of cbBuffer so that the unicode escaping doesn’t run into the wall each time */ struct DecoderState ds; wchar_t escBuffer[(JSON_MAX_STACK_BUFFER_SIZE / sizeof(wchar_t))]; JSUINT32 escBuffer[(JSON_MAX_STACK_BUFFER_SIZE / sizeof(JSUINT32))]; JSOBJ ret;
ds.start = (char *) buffer; ds.end = ds.start + cbBuffer;
ds.escStart = escBuffer; ds.escEnd = ds.escStart + (JSON_MAX_STACK_BUFFER_SIZE / sizeof(wchar_t)); ds.escEnd = ds.escStart + (JSON_MAX_STACK_BUFFER_SIZE / sizeof(JSUINT32)); ds.escHeap = 0; ds.prv = dec->prv; ds.dec = dec;
Related news
Gentoo Linux Security Advisory 202403-3 - Multiple vulnerabilities have been discovered in UltraJSON, the worst of which could lead to key confusion and value overwriting. Versions greater than or equal to 5.4.0 are affected.
Ubuntu Security Notice 6629-3 - USN-6629-1 fixed vulnerabilities in UltraJSON. This update provides the corresponding updates for Ubuntu 20.04 LTS. It was discovered that UltraJSON incorrectly handled certain input with a large amount of indentation. An attacker could possibly use this issue to crash the program, resulting in a denial of service. Jake Miller discovered that UltraJSON incorrectly decoded certain characters. An attacker could possibly use this issue to cause key confusion and overwrite values in dictionaries. It was discovered that UltraJSON incorrectly handled an error when reallocating a buffer for string decoding. An attacker could possibly use this issue to corrupt memory.
Ubuntu Security Notice 6629-2 - USN-6629-1 fixed vulnerabilities in UltraJSON. This update provides the corresponding updates for Ubuntu 20.04 LTS. It was discovered that UltraJSON incorrectly handled certain input with a large amount of indentation. An attacker could possibly use this issue to crash the program, resulting in a denial of service. Jake Miller discovered that UltraJSON incorrectly decoded certain characters. An attacker could possibly use this issue to cause key confusion and overwrite values in dictionaries. It was discovered that UltraJSON incorrectly handled an error when reallocating a buffer for string decoding. An attacker could possibly use this issue to corrupt memory.
Ubuntu Security Notice 6629-1 - It was discovered that UltraJSON incorrectly handled certain input with a large amount of indentation. An attacker could possibly use this issue to crash the program, resulting in a denial of service. Jake Miller discovered that UltraJSON incorrectly decoded certain characters. An attacker could possibly use this issue to cause key confusion and overwrite values in dictionaries. It was discovered that UltraJSON incorrectly handled an error when reallocating a buffer for string decoding. An attacker could possibly use this issue to corrupt memory.
Red Hat Security Advisory 2022-8864-01 - UltraJSON is an ultra fast JSON encoder and decoder. Issues addressed include a double free vulnerability.
An update for python-ujson is now available for Red Hat OpenStack Platform 16.1.9 (Train) for Red Hat Enterprise Linux (RHEL) 8.2. Red Hat Product Security has rated this update as having a security impact of Moderate. A Common Vulnerability Scoring System (CVSS) base score, which gives a detailed severity rating, is available for each vulnerability from the CVE link(s) in the References section.This content is licensed under the Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). If you distribute this content, or a modified version of it, you must provide attribution to Red Hat Inc. and provide a link to the original. Related CVEs: * CVE-2022-31116: python-ujson: improper decoding of escaped surrogate characters may lead to string corruption, key confusion or value overwriting * CVE-2022-31117: python-ujson: Potential double free of buffer during string decoding
An update for python-ujson is now available for Red Hat OpenStack Platform 16.2.4 (Train) for Red Hat Enterprise Linux (RHEL) 8.4. Red Hat Product Security has rated this update as having a security impact of Moderate. A Common Vulnerability Scoring System (CVSS) base score, which gives a detailed severity rating, is available for each vulnerability from the CVE link(s) in the References section.This content is licensed under the Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). If you distribute this content, or a modified version of it, you must provide attribution to Red Hat Inc. and provide a link to the original. Related CVEs: * CVE-2022-31116: python-ujson: improper decoding of escaped surrogate characters may lead to string corruption, key confusion or value overwriting * CVE-2022-31117: python-ujson: Potential double free of buffer during string decoding
### Impact _What kind of vulnerability is it? Who is impacted?_ Anyone parsing JSON from an untrusted source is vulnerable. JSON strings that contain escaped surrogate characters not part of a proper surrogate pair were decoded incorrectly. Besides corrupting strings, this allowed for potential key confusion and value overwriting in dictionaries. Examples: ```python # An unpaired high surrogate character is ignored. >>> ujson.loads(r'"\uD800"') '' >>> ujson.loads(r'"\uD800hello"') 'hello' # An unpaired low surrogate character is preserved. >>> ujson.loads(r'"\uDC00"') '\udc00' # A pair of surrogates with additional non surrogate characters pair up in spite of being invalid. >>> ujson.loads(r'"\uD800foo bar\uDC00"') 'foo bar𐀀' ``` ### Patches _Has the problem been patched? What versions should users upgrade to?_ Users should upgrade to UltraJSON 5.4.0. From version 5.4.0, UltraJSON decodes lone surrogates in the same way as the standard library's `json` module does, preserving...