Truncate text without breaking emoji

The same thing in your own code

Each code block has been compiled and verified. Provided under the MIT license to illustrate disarm. The C example counts rather than truncates: the C ABI exposes disarm_grapheme_len and disarm_terminal_width but no split or truncate. disarm on GitHub →

# Count what a reader sees, and cut without splitting it.
#   pip install disarm
from disarm import grapheme_len, grapheme_truncate, terminal_width

# Four people joined by three zero-width joiners. Every one of these numbers is
# correct; they answer different questions.
FAMILY = "👨‍👩‍👧‍👦"
assert len(FAMILY.encode()) == 25   # bytes
assert len(FAMILY) == 7             # codepoints — what len() gives you
assert grapheme_len(FAMILY) == 1    # characters — what a reader counts
assert terminal_width(FAMILY) == 2  # columns

# Cutting by codepoints keeps the father and discards his family. Asserting on
# the output is the point: it is valid text that says the wrong thing.
text = FAMILY + "🎉"
assert text[:1] == "👨"
assert grapheme_truncate(text, 1) == FAMILY

# A skin tone modifier is a separate codepoint inside the same cluster, so a
# naive cut silently changes who is depicted.
assert "👋🏽 hi"[:1] == "👋"
assert grapheme_truncate("👋🏽 hi", 1) == "👋🏽"

print(f"ok: {len(FAMILY.encode())} bytes, {len(FAMILY)} codepoints, "
      f"{grapheme_len(FAMILY)} character, {terminal_width(FAMILY)} columns")

// Count what a reader sees, and cut without splitting it.
//   cargo add disarm
use disarm::api::{grapheme_len, grapheme_truncate, terminal_width};

fn main() {
    // Four people joined by three zero-width joiners. Every one of these
    // numbers is correct; they answer different questions.
    let family = "👨‍👩‍👧‍👦";
    assert_eq!(family.len(), 25); // bytes
    assert_eq!(family.chars().count(), 7); // codepoints
    assert_eq!(grapheme_len(family), 1); // characters
    assert_eq!(terminal_width(family, false), 2); // columns

    // Cutting by codepoints keeps the father and discards his family.
    let text = format!("{family}🎉");
    let naive: String = text.chars().take(1).collect();
    assert_eq!(naive, "👨");
    assert_eq!(grapheme_truncate(&text, 1), family);

    // A skin tone modifier is a separate codepoint inside the same cluster.
    assert_eq!("👋🏽 hi".chars().take(1).collect::<String>(), "👋");
    assert_eq!(grapheme_truncate("👋🏽 hi", 1), "👋🏽");

    println!(
        "ok: {} bytes, {} codepoints, {} character, {} columns",
        family.len(),
        family.chars().count(),
        grapheme_len(family),
        terminal_width(family, false)
    );
}

// Count what a reader sees, and cut without splitting it.
//   npm install disarm
const { graphemeLen, graphemeTruncate, terminalWidth } = require("disarm");

// Four people joined by three zero-width joiners. Every one of these numbers is
// correct; they answer different questions. Note that JavaScript's own .length
// is a third answer again — UTF-16 code units, not codepoints.
const FAMILY = "👨‍👩‍👧‍👦";
const bytes = Buffer.byteLength(FAMILY, "utf8");
const cps = [...FAMILY].length;

if (bytes !== 25) throw new Error("expected 25 bytes");
if (cps !== 7) throw new Error("expected 7 codepoints");
if (FAMILY.length !== 11) throw new Error("expected 11 UTF-16 code units");
if (graphemeLen(FAMILY) !== 1) throw new Error("expected 1 character");
if (terminalWidth(FAMILY) !== 2) throw new Error("expected 2 columns");

// Cutting by codepoints keeps the father and discards his family.
const text = FAMILY + "🎉";
if ([...text].slice(0, 1).join("") !== "👨") throw new Error("expected a lone father");
if (graphemeTruncate(text, 1) !== FAMILY) throw new Error("expected the whole family");

// A skin tone modifier is a separate codepoint inside the same cluster.
if ([..."👋🏽 hi"].slice(0, 1).join("") !== "👋") throw new Error("expected a bare hand");
if (graphemeTruncate("👋🏽 hi", 1) !== "👋🏽") throw new Error("expected the modifier kept");

console.log(
  `ok: ${bytes} bytes, ${cps} codepoints, ${graphemeLen(FAMILY)} character, ${terminalWidth(FAMILY)} columns`,
);

# Count what a reader sees, and cut without splitting it.
#   gem install disarm
require "disarm"

# Four people joined by three zero-width joiners. Every one of these numbers is
# correct; they answer different questions.
FAMILY = "👨‍👩‍👧‍👦"
raise "expected 25 bytes"      unless FAMILY.bytesize == 25
raise "expected 7 codepoints"  unless FAMILY.length == 7
raise "expected 1 character"   unless Disarm.grapheme_len(FAMILY) == 1
raise "expected 2 columns"     unless Disarm.terminal_width(FAMILY) == 2

# Cutting by codepoints keeps the father and discards his family.
text = FAMILY + "🎉"
raise "expected a lone father"    unless text[0, 1] == "👨"
raise "expected the whole family" unless Disarm.grapheme_truncate(text, 1) == FAMILY

# A skin tone modifier is a separate codepoint inside the same cluster.
raise "expected a bare hand"       unless "👋🏽 hi"[0, 1] == "👋"
raise "expected the modifier kept" unless Disarm.grapheme_truncate("👋🏽 hi", 1) == "👋🏽"

puts "ok: #{FAMILY.bytesize} bytes, #{FAMILY.length} codepoints, " \
     "#{Disarm.grapheme_len(FAMILY)} character, #{Disarm.terminal_width(FAMILY)} columns"

// Count what a reader sees, and cut without splitting it.
//   implementation("dev.disarm:disarm:0.14.1")
import dev.disarm.Disarm;
import java.nio.charset.StandardCharsets;

public class CountGraphemes {
    public static void main(String[] args) {
        // Four people joined by three zero-width joiners. Every one of these
        // numbers is correct; they answer different questions. Java's own
        // String.length() is a fourth answer: UTF-16 code units.
        String family = "👨‍👩‍👧‍👦";
        int bytes = family.getBytes(StandardCharsets.UTF_8).length;
        int cps = family.codePointCount(0, family.length());

        // Java's `assert` is disabled unless the JVM is started with -ea.
        if (bytes != 25) throw new IllegalStateException("expected 25 bytes");
        if (cps != 7) throw new IllegalStateException("expected 7 codepoints");
        if (family.length() != 11) throw new IllegalStateException("expected 11 UTF-16 units");
        if (Disarm.graphemeLen(family) != 1) throw new IllegalStateException("expected 1 character");
        if (Disarm.terminalWidth(family) != 2) throw new IllegalStateException("expected 2 columns");

        // Cutting by codepoints keeps the father and discards his family.
        String text = family + "🎉";
        String naive = new String(text.codePoints().limit(1).toArray(), 0, 1);
        if (!naive.equals("👨")) throw new IllegalStateException("expected a lone father");
        if (!Disarm.graphemeTruncate(text, 1).equals(family)) {
            throw new IllegalStateException("expected the whole family");
        }

        // A skin tone modifier is a separate codepoint inside the same cluster.
        if (!Disarm.graphemeTruncate("👋🏽 hi", 1).equals("👋🏽")) {
            throw new IllegalStateException("expected the modifier kept");
        }

        System.out.printf("ok: %d bytes, %d codepoints, %d character, %d columns%n",
            bytes, cps, Disarm.graphemeLen(family), Disarm.terminalWidth(family));
    }
}

// Count what a reader sees, and cut without splitting it.
//   implementation("dev.disarm:disarm-kotlin:0.14.1")
import dev.disarm.Disarm

fun main() {
    // Four people joined by three zero-width joiners. Every one of these
    // numbers is correct; they answer different questions.
    val family = "👨‍👩‍👧‍👦"
    val bytes = family.toByteArray(Charsets.UTF_8).size
    val cps = family.codePointCount(0, family.length)

    check(bytes == 25) { "expected 25 bytes" }
    check(cps == 7) { "expected 7 codepoints" }
    check(family.length == 11) { "expected 11 UTF-16 units" }
    check(Disarm.graphemeLen(family) == 1L) { "expected 1 character" }
    check(Disarm.terminalWidth(family) == 2L) { "expected 2 columns" }

    // Cutting by codepoints keeps the father and discards his family.
    val text = family + "🎉"
    val naive = String(text.codePoints().limit(1).toArray(), 0, 1)
    check(naive == "👨") { "expected a lone father" }
    check(Disarm.graphemeTruncate(text, 1) == family) { "expected the whole family" }

    // A skin tone modifier is a separate codepoint inside the same cluster.
    check(Disarm.graphemeTruncate("👋🏽 hi", 1) == "👋🏽") { "expected the modifier kept" }

    println("ok: $bytes bytes, $cps codepoints, ${Disarm.graphemeLen(family)} character, " +
            "${Disarm.terminalWidth(family)} columns")
}

/* Count what a reader sees.
 *
 * The C ABI exposes disarm_grapheme_len and disarm_terminal_width but neither
 * grapheme_split nor grapheme_truncate, so this example counts rather than
 * truncates. The counting is the argument anyway: four measures, four answers.
 *
 * Links against a cdylib built from bindings/cabi in the disarm repository.
 */
#include <inttypes.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include "disarm.h"

/* Four people joined by three zero-width joiners. */
static const char *FAMILY = "👨‍👩‍👧‍👦";

/* Codepoints are the UTF-8 bytes that are not continuation bytes. */
static uint64_t codepoints(const char *s) {
    uint64_t n = 0;
    for (const unsigned char *p = (const unsigned char *)s; *p; p++) {
        if ((*p & 0xC0) != 0x80) n++;
    }
    return n;
}

int main(void) {
    uint64_t bytes = (uint64_t)strlen(FAMILY);
    uint64_t cps = codepoints(FAMILY);
    uint64_t graphemes = disarm_grapheme_len(FAMILY);
    /* false: ambiguous East Asian width counts as one column, which is what a
     * modern UTF-8 terminal does. */
    uint64_t columns = disarm_terminal_width(FAMILY, false);

    if (bytes != 25 || cps != 7 || graphemes != 1 || columns != 2) {
        fprintf(stderr, "unexpected measurements\n");
        return 1;
    }

    printf("ok: %" PRIu64 " bytes, %" PRIu64 " codepoints, %" PRIu64 " character,"
           " %" PRIu64 " columns\n", bytes, cps, graphemes, columns);
    return 0;
}