Home /Modules/impl/read_utf8_pieces.sleela

read_utf8_pieces.sleela

impl/examples/read_utf8_pieces.sleela · 64 line(s) · sleela

#sleela 3.0
material;
< read_utf8_pieces.sleela -- regression check: read(handle) never splits a UTF-8 character. >
< read returns a file in pieces of at most 4,095 bytes. It used to cut wherever the >
< 4,095 bytes ended, so a two, three or four byte character could be split between >
< two pieces; each half is malformed UTF-8 and javaStringFromUtf8 stopped the VM on it >
< (lib/java/converter skipped every Java file with a non-ASCII byte for that reason). >
< This example writes a file whose multi-byte characters straddle the first three >
< piece boundaries, reads it back piece by piece, converts every piece, and checks the >
< text is complete. A wrong answer throws, so make test-sleela fails. >
class ReadUtf8Pieces {
  int checks = 0;

  void expect(boolean ok, String what) {
    checks = checks + 1;
    if (!ok) {
      print("FAIL: " + what);
      throw "read_utf8_pieces failed: " + what;
    }
  }

  < n copies of the letter a >
  String pad(int n) {
    String s = "";
    int i = 0;
    while (i < n) { s = s + "a"; i = i + 1; }
    return s;
  }

  void main() {
    < byte 4094 starts a 2-byte character, so the first piece would end inside it; >
    < the next two boundaries fall inside a 3-byte and a 4-byte character >
    String text = pad(4094) + "é" + pad(4092) + "€" + pad(4091) + "😀" + "end";
    String path = osTempDir() + "/sleela-read-utf8-" + osProcessId() + ".txt";
    int w = openFile(path, "w");
    expect(w >= 0, "the test file opens for writing");
    write(w, text);
    close(w);

    int h = openFile(path, "r");
    expect(h >= 0, "the test file opens for reading");
    int pieces = 0;
    int units = 0;
    String back = "";
    String piece = read(h);
    while (piece != "") {
      < this conversion is what stopped the VM when a piece ended inside a character >
      String u = javaStringFromUtf8(piece);
      units = units + javaStringLength(u);
      back = back + piece;
      pieces = pieces + 1;
      piece = read(h);
    }
    close(h);
    int rc = osRun("rm -f '" + path + "'");

    expect(pieces >= 4, "the file comes back in several pieces (" + pieces + ")");
    < 4094 + 1 + 4092 + 1 + 4091 + 2 (a surrogate pair) + 3 = 12284 UTF-16 code units >
    expect(units == 12284, "every character arrives whole: 12284 code units (" + units + ")");
    expect(back == text, "the pieces join to the text that was written");
    print("read_utf8_pieces: " + pieces + " pieces, " + units + " code units, " + checks + " checks passed");
  }
}