Normalize Unicode whitespace

The same thing in your own code

Each code block has been compiled and verified. Provided under the MIT license to illustrate disarm. disarm on GitHub →

# Fold every blank-rendering character to an ordinary space.
#   pip install disarm
from disarm import collapse_whitespace

# Four characters that all render as a gap. The last two are not whitespace by
# any standard-library test, which is the whole point of this example.
NBSP, IDEO, BRAILLE, FILLER = " ", "　", "⠀", "ㅤ"
BLANKS = (NBSP, IDEO, BRAILLE, FILLER)

text = f"Total{NBSP}due:{IDEO}1,240{BRAILLE}INV{FILLER}0117"

# Printing folded text proves nothing — a gap looks like a gap either way —
# so assert the codepoints are absent instead.
cleaned = collapse_whitespace(text)
for ch in BLANKS:
    assert ch not in cleaned, f"U+{ord(ch):04X} survived"

missed = [c for c in (BRAILLE, FILLER) if not c.isspace()]
assert len(missed) == 2, "expected str.isspace to miss both"

print(f"ok: {len(BLANKS)} blanks folded, {len(missed)} of them invisible to this language's whitespace test")

// Fold every blank-rendering character to an ordinary space.
//   cargo add disarm
use disarm::api::collapse_whitespace;

fn main() {
    // Four characters that all render as a gap. The last two are not whitespace
    // by any standard-library test, which is the whole point of this example.
    const NBSP: char = '\u{00a0}';
    const IDEO: char = '\u{3000}';
    const BRAILLE: char = '\u{2800}';
    const FILLER: char = '\u{3164}';
    let blanks = [NBSP, IDEO, BRAILLE, FILLER];

    let text = format!("Total{NBSP}due:{IDEO}1,240{BRAILLE}INV{FILLER}0117");

    // Printing folded text proves nothing — a gap looks like a gap either way —
    // so assert the codepoints are absent instead.
    let cleaned = collapse_whitespace(&text);
    for ch in blanks {
        assert!(!cleaned.contains(ch), "U+{:04X} survived", ch as u32);
    }

    let missed = [BRAILLE, FILLER].iter().filter(|c| !c.is_whitespace()).count();
    assert_eq!(missed, 2, "expected char::is_whitespace to miss both");

    println!(
        "ok: {} blanks folded, {} of them invisible to this language's whitespace test",
        blanks.len(),
        missed
    );
}

// Fold every blank-rendering character to an ordinary space.
//   npm install disarm
const { collapseWhitespace } = require("disarm");

// Four characters that all render as a gap. The last two are not whitespace by
// any standard-library test, which is the whole point of this example.
const NBSP = " ";
const IDEO = "　";
const BRAILLE = "⠀";
const FILLER = "ㅤ";
const BLANKS = [NBSP, IDEO, BRAILLE, FILLER];

const text = `Total${NBSP}due:${IDEO}1,240${BRAILLE}INV${FILLER}0117`;

// Printing folded text proves nothing — a gap looks like a gap either way —
// so assert the codepoints are absent instead.
const cleaned = collapseWhitespace(text);
for (const ch of BLANKS) {
  if (cleaned.includes(ch)) {
    throw new Error(`U+${ch.codePointAt(0).toString(16).toUpperCase()} survived`);
  }
}

const missed = [BRAILLE, FILLER].filter((c) => !/\s/.test(c));
if (missed.length !== 2) throw new Error("expected \\s to miss both");

console.log(
  `ok: ${BLANKS.length} blanks folded, ${missed.length} of them invisible to this language's whitespace test`,
);

# Fold every blank-rendering character to an ordinary space.
#   gem install disarm
require "disarm"

# Four characters that all render as a gap. The last two are not whitespace by
# any standard-library test, which is the whole point of this example.
NBSP    = " "
IDEO    = "　"
BRAILLE = "⠀"
FILLER  = "ㅤ"
BLANKS  = [NBSP, IDEO, BRAILLE, FILLER].freeze

text = "Total#{NBSP}due:#{IDEO}1,240#{BRAILLE}INV#{FILLER}0117"

# Printing folded text proves nothing — a gap looks like a gap either way —
# so assert the codepoints are absent instead.
cleaned = Disarm.collapse_whitespace(text)
BLANKS.each do |ch|
  raise format("U+%04X survived", ch.ord) if cleaned.include?(ch)
end

missed = [BRAILLE, FILLER].reject { |c| c.match?(/\s/) }
raise "expected /\\s/ to miss both" unless missed.size == 2

puts "ok: #{BLANKS.size} blanks folded, #{missed.size} of them invisible to this language's whitespace test"

// Fold every blank-rendering character to an ordinary space.
//   implementation("dev.disarm:disarm:0.14.1")
import dev.disarm.Disarm;

public class CollapseWhitespace {
    public static void main(String[] args) {
        // Four characters that all render as a gap: no-break space, ideographic
        // space, Braille Pattern Blank and Hangul Filler. The last two are not
        // whitespace by any standard-library test, which is the point here.
        // Written as codepoints rather than escapes, because javac resolves a
        // backslash-u sequence before it tokenises — even inside a comment.
        int[] blanks = { 0x00A0, 0x3000, 0x2800, 0x3164 };

        String text = "Total" + Character.toString(blanks[0])
            + "due:" + Character.toString(blanks[1])
            + "1,240" + Character.toString(blanks[2])
            + "INV" + Character.toString(blanks[3]) + "0117";

        // Printing folded text proves nothing — a gap looks like a gap either
        // way — so check the codepoints are absent instead. Java's `assert` is
        // disabled unless the JVM is started with -ea, so this throws.
        String cleaned = Disarm.collapseWhitespace(text);
        for (int cp : blanks) {
            if (cleaned.indexOf(cp) >= 0) {
                throw new IllegalStateException(String.format("U+%04X survived", cp));
            }
        }

        int missed = 0;
        for (int cp : new int[] { 0x2800, 0x3164 }) {
            if (!Character.isWhitespace(cp)) missed++;
        }
        if (missed != 2) {
            throw new IllegalStateException("expected Character.isWhitespace to miss both");
        }

        System.out.printf(
            "ok: %d blanks folded, %d of them invisible to this language's whitespace test%n",
            blanks.length, missed);
    }
}

// Fold every blank-rendering character to an ordinary space.
//   implementation("dev.disarm:disarm-kotlin:0.14.1")
import dev.disarm.Disarm

fun main() {
    // Four characters that all render as a gap: no-break space, ideographic
    // space, Braille Pattern Blank and Hangul Filler. The last two are not
    // whitespace by any standard-library test, which is the point here.
    val blanks = intArrayOf(0x00A0, 0x3000, 0x2800, 0x3164)
    val (nbsp, ideo, braille, filler) =
        blanks.map { String(Character.toChars(it)) }.let {
            listOf(it[0], it[1], it[2], it[3])
        }

    val text = "Total${nbsp}due:${ideo}1,240${braille}INV${filler}0117"

    // Printing folded text proves nothing — a gap looks like a gap either way —
    // so check the codepoints are absent instead.
    val cleaned = Disarm.collapseWhitespace(text)
    for (cp in blanks) {
        check(cleaned.indexOf(cp) < 0) { "U+%04X survived".format(cp) }
    }

    val missed = intArrayOf(0x2800, 0x3164).count { !Character.isWhitespace(it) }
    check(missed == 2) { "expected Character.isWhitespace to miss both" }

    println("ok: ${blanks.size} blanks folded, $missed of them invisible to this language's whitespace test")
}

/* Fold every blank-rendering character to an ordinary space.
 *
 * The C ABI is not published to any registry, so this links against a cdylib
 * built from bindings/cabi in the disarm repository. Every disarm_* function
 * here returns an owned string the caller frees with disarm_string_free.
 */
#include <ctype.h>
#include <stdio.h>
#include <string.h>
#include "disarm.h"

/* Four characters that all render as a gap, as UTF-8: no-break space,
 * ideographic space, Braille Pattern Blank and Hangul Filler. */
static const char *BLANKS[] = { " ", "　", "⠀", "ㅤ" };
#define BLANKS_LEN (sizeof BLANKS / sizeof *BLANKS)

/* C has no Unicode whitespace predicate in the standard library at all:
 * isspace() classifies single bytes, and every byte of these two characters is
 * non-ASCII, so it reports false for all of them. */
static int missed_by_isspace(const char *s) {
    for (const unsigned char *p = (const unsigned char *)s; *p; p++) {
        if (isspace(*p)) return 0;
    }
    return 1;
}

int main(void) {
    const char *input =
        "Total due:　1,240⠀INVㅤ0117";

    /* Printing folded text proves nothing — a gap looks like a gap either way —
     * so search for the codepoints instead. */
    char *cleaned = disarm_collapse_whitespace(input);

    for (size_t i = 0; i < BLANKS_LEN; i++) {
        if (strstr(cleaned, BLANKS[i]) != NULL) {
            fprintf(stderr, "a blank survived\n");
            disarm_string_free(cleaned);
            return 1;
        }
    }

    int missed = missed_by_isspace(BLANKS[2]) + missed_by_isspace(BLANKS[3]);
    if (missed != 2) {
        fprintf(stderr, "expected isspace to miss both\n");
        disarm_string_free(cleaned);
        return 1;
    }

    printf("ok: %zu blanks folded, %d of them invisible to this language's"
           " whitespace test\n", BLANKS_LEN, missed);
    disarm_string_free(cleaned);
    return 0;
}