Home /Modules/test-suites/reg_utf16.c

reg_utf16.c

test-suites/regression/c/reg_utf16.c · 410 line(s) · c

/*
 * reg_utf16.c -- regression suite for impl/core/sleela_utf16.c
 *
 * Contract (impl/core/sleela_utf16.h): input is length-delimited UTF-8;
 * return 0 success, -1 invalid argument, -2 malformed UTF-8, -3 UTF-16
 * index out of range.  "Malformed" is the Unicode well-formedness set
 * (Table 3-7): no overlongs, no surrogates, nothing above U+10FFFF.
 *
 * The oracle below is written independently of the implementation (a
 * byte-range table instead of arithmetic range checks) and the test drives it
 * exhaustively over every 1-, 2- and 3-byte input, plus every code point and
 * a deterministic random sample of longer inputs.
 */
#include "../../../impl/core/sleela_utf16.h"
#include "regression.h"

#include <stdlib.h>
#include <string.h>

/* ---- independent oracle ------------------------------------------------ */

/* Returns the byte length (1..4) of the well-formed sequence at s, or 0. */
static size_t oracle_seq(const unsigned char *s, size_t n, uint32_t *cp) {
    unsigned char a;
    if (n == 0) return 0;
    a = s[0];
    if (a <= 0x7F) { *cp = a; return 1; }
    if (a >= 0xC2 && a <= 0xDF) {
        if (n < 2 || s[1] < 0x80 || s[1] > 0xBF) return 0;
        *cp = ((uint32_t)(a & 0x1Fu) << 6) | (uint32_t)(s[1] & 0x3Fu);
        return 2;
    }
    if (a >= 0xE0 && a <= 0xEF) {
        unsigned char lo = 0x80, hi = 0xBF;
        if (a == 0xE0) lo = 0xA0;
        if (a == 0xED) hi = 0x9F;
        if (n < 3 || s[1] < lo || s[1] > hi || s[2] < 0x80 || s[2] > 0xBF) return 0;
        *cp = ((uint32_t)(a & 0x0Fu) << 12) | ((uint32_t)(s[1] & 0x3Fu) << 6) |
              (uint32_t)(s[2] & 0x3Fu);
        return 3;
    }
    if (a >= 0xF0 && a <= 0xF4) {
        unsigned char lo = 0x80, hi = 0xBF;
        if (a == 0xF0) lo = 0x90;
        if (a == 0xF4) hi = 0x8F;
        if (n < 4 || s[1] < lo || s[1] > hi || s[2] < 0x80 || s[2] > 0xBF ||
            s[3] < 0x80 || s[3] > 0xBF) return 0;
        *cp = ((uint32_t)(a & 0x07u) << 18) | ((uint32_t)(s[1] & 0x3Fu) << 12) |
              ((uint32_t)(s[2] & 0x3Fu) << 6) | (uint32_t)(s[3] & 0x3Fu);
        return 4;
    }
    return 0;
}

/* Whole-buffer oracle. Returns 1 and fills units[] (capacity cap) if valid. */
static int oracle_decode(const unsigned char *s, size_t n, uint16_t *units,
                         size_t cap, size_t *count) {
    size_t off = 0, used = 0;
    while (off < n) {
        uint32_t cp; size_t k = oracle_seq(s + off, n - off, &cp);
        if (!k) return 0;
        if (cp < 0x10000u) { if (used >= cap) return 0; units[used++] = (uint16_t)cp; }
        else {
            if (used + 2 > cap) return 0;
            cp -= 0x10000u;
            units[used++] = (uint16_t)(0xD800u + (cp >> 10));
            units[used++] = (uint16_t)(0xDC00u + (cp & 0x3FFu));
        }
        off += k;
    }
    *count = used;
    return 1;
}

static size_t encode_utf8(uint32_t cp, unsigned char *o) {
    if (cp < 0x80u) { o[0] = (unsigned char)cp; return 1; }
    if (cp < 0x800u) {
        o[0] = (unsigned char)(0xC0u | (cp >> 6));
        o[1] = (unsigned char)(0x80u | (cp & 0x3Fu)); return 2;
    }
    if (cp < 0x10000u) {
        o[0] = (unsigned char)(0xE0u | (cp >> 12));
        o[1] = (unsigned char)(0x80u | ((cp >> 6) & 0x3Fu));
        o[2] = (unsigned char)(0x80u | (cp & 0x3Fu)); return 3;
    }
    o[0] = (unsigned char)(0xF0u | (cp >> 18));
    o[1] = (unsigned char)(0x80u | ((cp >> 12) & 0x3Fu));
    o[2] = (unsigned char)(0x80u | ((cp >> 6) & 0x3Fu));
    o[3] = (unsigned char)(0x80u | (cp & 0x3Fu)); return 4;
}

/* Compare the implementation against the oracle for one buffer. */
static void compare_one(const unsigned char *buf, size_t n) {
    uint16_t want[256]; size_t want_n = 0;
    int valid = oracle_decode(buf, n, want, 256, &want_n);
    int64_t len = -77;
    int rc = sleela_utf8_utf16_length((const char *)buf, n, &len);
    if (valid) {
        uint16_t *got = NULL; size_t got_n = 0;
        reg_note_check();
        if (rc != 0 || len != (int64_t)want_n) {
            reg_fail(__FILE__, __LINE__, "valid input rejected/miscounted (rc=%d len=%lld want=%zu, first byte 0x%02X)",
                     rc, (long long)len, want_n, buf[0]);
            return;
        }
        rc = sleela_utf8_to_utf16((const char *)buf, n, &got, &got_n);
        reg_note_check();
        if (rc != 0 || got_n != want_n || memcmp(got, want, want_n * sizeof(uint16_t)) != 0)
            reg_fail(__FILE__, __LINE__, "to_utf16 disagrees with oracle (first byte 0x%02X)", buf[0]);
        free(got);
        for (size_t i = 0; i < want_n; ++i) {
            uint16_t u = 0;
            int r2 = sleela_utf8_utf16_char_at((const char *)buf, n, (int64_t)i, &u);
            reg_note_check();
            if (r2 != 0 || u != want[i])
                reg_fail(__FILE__, __LINE__, "char_at(%zu) disagrees with oracle", i);
        }
    } else {
        reg_note_check();
        if (rc != -2)
            reg_fail(__FILE__, __LINE__, "malformed input accepted/misclassified (rc=%d, bytes %02X %02X %02X)",
                     rc, buf[0], n > 1 ? buf[1] : 0, n > 2 ? buf[2] : 0);
    }
}

/* ---- cases -------------------------------------------------------------- */

/* Contract: documented argument validation (sleela_utf16.h). */
static void t_argument_contract(void) {
    int64_t n = 123; uint16_t *u = (uint16_t *)0x1; size_t k = 9; uint16_t c = 0;
    REG_EQ_I(0, sleela_utf8_utf16_length(NULL, 0, &n)); REG_EQ_I(0, n);
    REG_EQ_I(0, sleela_utf8_utf16_length("", 0, &n)); REG_EQ_I(0, n);
    REG_EQ_I(-1, sleela_utf8_utf16_length(NULL, 1, &n));
    REG_EQ_I(-1, sleela_utf8_utf16_length("a", 1, NULL));

    REG_EQ_I(0, sleela_utf8_to_utf16(NULL, 0, &u, &k));
    REG_CHECK(u == NULL); REG_EQ_U(0, k);
    REG_EQ_I(-1, sleela_utf8_to_utf16(NULL, 1, &u, &k));
    REG_EQ_I(-1, sleela_utf8_to_utf16("a", 1, NULL, &k));
    REG_EQ_I(-1, sleela_utf8_to_utf16("a", 1, &u, NULL));

    REG_EQ_I(-1, sleela_utf8_utf16_char_at("a", 1, -1, &c));
    REG_EQ_I(-1, sleela_utf8_utf16_char_at("a", 1, 0, NULL));
    REG_EQ_I(-1, sleela_utf8_utf16_char_at(NULL, 1, 0, &c));
    REG_EQ_I(-3, sleela_utf8_utf16_char_at("", 0, 0, &c));
    REG_EQ_I(-3, sleela_utf8_utf16_char_at("a", 1, 1, &c));
    REG_EQ_I(-3, sleela_utf8_utf16_char_at("a", 1, INT64_MAX, &c));
}

/* Contract: output parameters are reset before any failure (no stale pointer). */
static void t_outputs_reset_on_failure(void) {
    uint16_t *u = (uint16_t *)0x1; size_t k = 9;
    REG_EQ_I(-2, sleela_utf8_to_utf16("\xFF", 1, &u, &k));
    REG_CHECK(u == NULL); REG_EQ_U(0, k);
    u = (uint16_t *)0x1; k = 9;
    REG_EQ_I(-2, sleela_utf8_to_utf16("ok\xC3", 3, &u, &k));
    REG_CHECK(u == NULL); REG_EQ_U(0, k);
}

/* Contract: input is length-delimited; embedded NUL is data, not a terminator. */
static void t_embedded_nul(void) {
    static const char s[] = {'a', '\0', 'b'};
    int64_t n = 0; uint16_t *u = NULL; size_t k = 0;
    REG_EQ_I(0, sleela_utf8_utf16_length(s, 3, &n)); REG_EQ_I(3, n);
    REG_EQ_I(0, sleela_utf8_to_utf16(s, 3, &u, &k)); REG_EQ_U(3, k);
    REG_REQUIRE(u != NULL);
    REG_EQ_U(0x61, u[0]); REG_EQ_U(0, u[1]); REG_EQ_U(0x62, u[2]);
    free(u);
    /* a NUL-terminated view of the same bytes must not be consulted */
    REG_EQ_I(0, sleela_utf8_utf16_length("ab\0\xFF", 2, &n)); REG_EQ_I(2, n);
}

/* Contract: well-known encodings, including the BMP/plane boundaries. */
static void t_known_vectors(void) {
    struct { const char *bytes; size_t nb; uint16_t u[2]; size_t nu; } v[] = {
        {"\xC3\xA9", 2, {0x00E9, 0}, 1},
        {"\xE2\x82\xAC", 3, {0x20AC, 0}, 1},
        {"\xF0\x9F\x98\x80", 4, {0xD83D, 0xDE00}, 2},
        {"\xF4\x8F\xBF\xBF", 4, {0xDBFF, 0xDFFF}, 2},
        {"\xF0\x90\x80\x80", 4, {0xD800, 0xDC00}, 2},
        {"\xEF\xBF\xBF", 3, {0xFFFF, 0}, 1},
        {"\xED\x9F\xBF", 3, {0xD7FF, 0}, 1},
        {"\xEE\x80\x80", 3, {0xE000, 0}, 1},
        {"\xEF\xBB\xBF", 3, {0xFEFF, 0}, 1}, /* BOM is preserved, never stripped */
        {"\xC2\x80", 2, {0x0080, 0}, 1},
        {"\xDF\xBF", 2, {0x07FF, 0}, 1},
        {"\xE0\xA0\x80", 3, {0x0800, 0}, 1},
    };
    for (size_t i = 0; i < sizeof v / sizeof v[0]; ++i) {
        uint16_t *u = NULL; size_t k = 0;
        REG_EQ_I(0, sleela_utf8_to_utf16(v[i].bytes, v[i].nb, &u, &k));
        REG_EQ_U(v[i].nu, k);
        if (u) REG_EQ_MEM(v[i].u, u, v[i].nu * sizeof(uint16_t));
        free(u);
    }
}

/* Contract: every ill-formed class is rejected with -2 by every entry point. */
static void t_malformed_vectors(void) {
    static const struct { const char *b; size_t n; const char *why; } bad[] = {
        {"\x80", 1, "lone continuation"}, {"\xBF", 1, "lone continuation"},
        {"\xC0\x80", 2, "overlong NUL"}, {"\xC1\xBF", 2, "overlong ASCII"},
        {"\xE0\x80\x80", 3, "overlong 3-byte"}, {"\xE0\x9F\xBF", 3, "overlong 3-byte max"},
        {"\xF0\x80\x80\x80", 4, "overlong 4-byte"}, {"\xF0\x8F\xBF\xBF", 4, "overlong 4-byte max"},
        {"\xED\xA0\x80", 3, "U+D800 surrogate"}, {"\xED\xBF\xBF", 3, "U+DFFF surrogate"},
        {"\xED\xA0\xBD\xED\xB8\x80", 6, "CESU-8 surrogate pair"},
        {"\xF4\x90\x80\x80", 4, "above U+10FFFF"}, {"\xF5\x80\x80\x80", 4, "F5 lead"},
        {"\xF8\x88\x80\x80\x80", 5, "5-byte form"}, {"\xFC\x84\x80\x80\x80\x80", 6, "6-byte form"},
        {"\xFE", 1, "FE"}, {"\xFF", 1, "FF"},
        {"\xC3", 1, "truncated 2-byte"}, {"\xE2\x82", 2, "truncated 3-byte"},
        {"\xF0\x9F\x98", 3, "truncated 4-byte"},
        {"\xC3(", 2, "bad continuation"}, {"\xE2\x28\xA1", 3, "bad continuation 2"},
        {"ok\xFF" "ok", 5, "bad byte mid-string"},
    };
    for (size_t i = 0; i < sizeof bad / sizeof bad[0]; ++i) {
        int64_t n = 0; uint16_t *u = NULL; size_t k = 0;
        reg_note_check();
        if (sleela_utf8_utf16_length(bad[i].b, bad[i].n, &n) != -2)
            reg_fail(__FILE__, __LINE__, "length accepted: %s", bad[i].why);
        reg_note_check();
        if (sleela_utf8_to_utf16(bad[i].b, bad[i].n, &u, &k) != -2 || u != NULL)
            reg_fail(__FILE__, __LINE__, "to_utf16 accepted: %s", bad[i].why);
    }
}

/* Exhaustive: every 1-byte and 2-byte input (65,792 buffers). */
static void t_exhaustive_1_and_2_bytes(void) {
    unsigned char b[2];
    for (unsigned x = 0; x < 256; ++x) { b[0] = (unsigned char)x; compare_one(b, 1); }
    for (unsigned x = 0; x < 256; ++x)
        for (unsigned y = 0; y < 256; ++y) {
            b[0] = (unsigned char)x; b[1] = (unsigned char)y;
            compare_one(b, 2);
        }
}

/* Exhaustive: every 3-byte input (16,777,216 buffers). */
static void t_exhaustive_3_bytes(void) {
    unsigned char b[3];
    for (unsigned x = 0; x < 256; ++x)
        for (unsigned y = 0; y < 256; ++y)
            for (unsigned z = 0; z < 256; ++z) {
                int64_t len = 0; uint16_t want[4]; size_t wn = 0;
                int valid, rc;
                b[0] = (unsigned char)x; b[1] = (unsigned char)y; b[2] = (unsigned char)z;
                valid = oracle_decode(b, 3, want, 4, &wn);
                rc = sleela_utf8_utf16_length((const char *)b, 3, &len);
                if (valid ? (rc != 0 || len != (int64_t)wn) : (rc != -2)) {
                    reg_note_check();
                    reg_fail(__FILE__, __LINE__, "3-byte disagreement at %02X %02X %02X (rc=%d valid=%d)",
                             x, y, z, rc, valid);
                    return;
                }
                /* heavier cross-check on a stride so the whole space stays fast */
                if (valid && ((x + y + z) & 15u) == 0) compare_one(b, 3);
            }
    REG_CHECK(1);
}

/* 4-byte inputs: every combination drawn from the boundary bytes of each
 * position (lead F0..F4 and every edge value of the continuation ranges). */
static void t_four_byte_boundaries(void) {
    static const unsigned char edge[] = {0x00, 0x7F, 0x80, 0x8F, 0x90, 0x9F, 0xA0, 0xBF, 0xC0, 0xFF};
    unsigned char b[4];
    for (unsigned lead = 0xEF; lead <= 0xF8; ++lead)
        for (size_t i = 0; i < sizeof edge; ++i)
            for (size_t j = 0; j < sizeof edge; ++j)
                for (size_t k = 0; k < sizeof edge; ++k) {
                    b[0] = (unsigned char)lead; b[1] = edge[i]; b[2] = edge[j]; b[3] = edge[k];
                    compare_one(b, 4);
                }
}

/* Every scalar value round-trips through an independent encoder. */
static void t_every_code_point(void) {
    for (uint32_t cp = 0; cp <= 0x10FFFFu; ++cp) {
        unsigned char b[4]; size_t nb; int64_t len = 0; uint16_t u = 0, hi = 0, lo = 0;
        int rc;
        if (cp >= 0xD800u && cp <= 0xDFFFu) continue;
        nb = encode_utf8(cp, b);
        rc = sleela_utf8_utf16_length((const char *)b, nb, &len);
        if (rc != 0 || len != (cp > 0xFFFFu ? 2 : 1)) {
            reg_note_check();
            reg_fail(__FILE__, __LINE__, "U+%04X length rc=%d len=%lld", cp, rc, (long long)len);
            return;
        }
        if (cp <= 0xFFFFu) {
            rc = sleela_utf8_utf16_char_at((const char *)b, nb, 0, &u);
            if (rc != 0 || u != (uint16_t)cp) {
                reg_note_check();
                reg_fail(__FILE__, __LINE__, "U+%04X char_at rc=%d u=%04X", cp, rc, u);
                return;
            }
        } else {
            uint32_t v = cp - 0x10000u;
            rc = sleela_utf8_utf16_char_at((const char *)b, nb, 0, &hi);
            rc |= sleela_utf8_utf16_char_at((const char *)b, nb, 1, &lo);
            if (rc != 0 || hi != (uint16_t)(0xD800u + (v >> 10)) ||
                lo != (uint16_t)(0xDC00u + (v & 0x3FFu))) {
                reg_note_check();
                reg_fail(__FILE__, __LINE__, "U+%04X surrogate pair wrong", cp);
                return;
            }
        }
    }
    REG_CHECK(1);
}

/* Deterministic random strings: valid ones must match; one-byte mutations
 * must be accepted/rejected exactly as the oracle says. */
static void t_random_strings_and_mutations(void) {
    reg_seed(0x5EE1EA0001ull);
    for (int iter = 0; iter < 4000; ++iter) {
        unsigned char buf[4 * 40]; size_t n = 0, cps = reg_below(41);
        for (size_t i = 0; i < cps; ++i) {
            uint32_t cp;
            switch (reg_below(5)) {
                case 0: cp = reg_below(0x80); break;
                case 1: cp = 0x80u + reg_below(0x780); break;
                case 2: cp = 0x800u + reg_below(0xD800 - 0x800); break;
                case 3: cp = 0xE000u + reg_below(0x10000 - 0xE000); break;
                default: cp = 0x10000u + reg_below(0x100000); break;
            }
            n += encode_utf8(cp, buf + n);
        }
        if (n == 0) continue;
        compare_one(buf, n);
        for (int m = 0; m < 6; ++m) {
            unsigned char mut[sizeof buf];
            memcpy(mut, buf, n);
            mut[reg_below((uint32_t)n)] = (unsigned char)reg_below(256);
            compare_one(mut, n);
        }
    }
}

/* CHARACTERIZATION of current behaviour: char_at decodes lazily from the front.
 * Malformed bytes AFTER the addressed unit do not affect the answer; malformed
 * bytes BEFORE (or at) it yield -2; an index past the end of valid text is -3. */
static void t_char_at_is_lazy(void) {
    uint16_t c = 0;
    REG_EQ_I(0, sleela_utf8_utf16_char_at("A\xFF", 2, 0, &c)); REG_EQ_U(0x41, c);
    REG_EQ_I(-2, sleela_utf8_utf16_char_at("A\xFF", 2, 1, &c));
    REG_EQ_I(-2, sleela_utf8_utf16_char_at("A\xFF", 2, 5, &c));
    REG_EQ_I(-3, sleela_utf8_utf16_char_at("A", 1, 1, &c));
    /* but the strict whole-string entry points reject it */
    { int64_t n; REG_EQ_I(-2, sleela_utf8_utf16_length("A\xFF", 2, &n)); }
    /* surrogate pair addressing: index 0 = high, 1 = low, 2 = out of range */
    REG_EQ_I(0, sleela_utf8_utf16_char_at("\xF0\x9F\x98\x80", 4, 0, &c)); REG_EQ_U(0xD83D, c);
    REG_EQ_I(0, sleela_utf8_utf16_char_at("\xF0\x9F\x98\x80", 4, 1, &c)); REG_EQ_U(0xDE00, c);
    REG_EQ_I(-3, sleela_utf8_utf16_char_at("\xF0\x9F\x98\x80", 4, 2, &c));
}

/* Out-of-memory: fail each allocation in turn; no leak, no stale output. */
static void t_out_of_memory(void) {
    for (unsigned long nth = 1; nth <= 3; ++nth) {
        uint16_t *u = (uint16_t *)0x1; size_t k = 9; int rc;
        reg_alloc_fail_nth(nth);
        rc = sleela_utf8_to_utf16("h\xC3\xA9llo", 6, &u, &k);
        if (reg_alloc_injected()) {
            REG_EQ_I(-1, rc); REG_CHECK(u == NULL); REG_EQ_U(0, k);
        } else {
            REG_EQ_I(0, rc); REG_EQ_U(5, k); free(u);
        }
        reg_alloc_fail_nth(0);
    }
    /* the very first allocation is the only one the call makes */
    {
        uint16_t *u = (uint16_t *)0x1; size_t k = 9;
        reg_alloc_fail_nth(1);
        REG_EQ_I(-1, sleela_utf8_to_utf16("abc", 3, &u, &k));
        REG_CHECK(reg_alloc_injected()); REG_CHECK(u == NULL); REG_EQ_U(0, k);
    }
}

/* Scale: a 4 MiB all-supplementary string (1 Mi code points, 2 Mi units). */
static void t_large_input(void) {
    const size_t cps = 1u << 20; const size_t nb = cps * 4u;
    unsigned char *b = (unsigned char *)malloc(nb);
    int64_t len = 0; uint16_t *u = NULL; size_t k = 0;
    REG_REQUIRE(b != NULL);
    for (size_t i = 0; i < cps; ++i) memcpy(b + i * 4u, "\xF0\x9F\x98\x80", 4);
    REG_EQ_I(0, sleela_utf8_utf16_length((const char *)b, nb, &len));
    REG_EQ_I((long long)(cps * 2u), len);
    REG_EQ_I(0, sleela_utf8_to_utf16((const char *)b, nb, &u, &k));
    REG_EQ_U(cps * 2u, k);
    if (u) { REG_EQ_U(0xD83D, u[0]); REG_EQ_U(0xDE00, u[k - 1]); }
    free(u);
    b[nb - 1] = 0x20; /* corrupt the final continuation byte */
    REG_EQ_I(-2, sleela_utf8_utf16_length((const char *)b, nb, &len));
    free(b);
}

int main(void) {
    REG_RUN(t_argument_contract);
    REG_RUN(t_outputs_reset_on_failure);
    REG_RUN(t_embedded_nul);
    REG_RUN(t_known_vectors);
    REG_RUN(t_malformed_vectors);
    REG_RUN(t_exhaustive_1_and_2_bytes);
    REG_RUN(t_exhaustive_3_bytes);
    REG_RUN(t_four_byte_boundaries);
    REG_RUN(t_every_code_point);
    REG_RUN(t_random_strings_and_mutations);
    REG_RUN(t_char_at_is_lazy);
    REG_RUN(t_out_of_memory);
    REG_RUN(t_large_input);
    return reg_finish("utf16");
}