Remove invisible characters from text

The same thing in your own code

Each block is a file CI compiles and runs, so none can quietly stop working, and all seven print the same line. disarm on GitHub →

# Remove every class of invisible character disarm exposes in all bindings.
#   pip install disarm
from disarm import (
    strip_zero_width_chars, strip_bidi, strip_tags,
    strip_variation_selectors, strip_noncharacters,
    strip_pua, strip_control_chars,
)

# The characters hidden in the sample below. Printing stripped text proves
# nothing — it looks the same either way — so assert they are gone instead.
HOSTILE = "​‌⁠‮‬️\U000e0041"

text = "pay​pal‌ ad⁠min ‮reversed‬ text️\U000e0041"

for step in (strip_zero_width_chars, strip_bidi, strip_tags,
             strip_variation_selectors, strip_noncharacters,
             strip_pua, strip_control_chars):
    text = step(text)

for ch in HOSTILE:
    assert ch not in text, f"U+{ord(ch):04X} survived"

print(f'ok: {len(HOSTILE)} hostile codepoints absent from "{text}"')

// Remove every class of invisible character disarm exposes in all bindings.
//   cargo add disarm
use disarm::api::*;

fn main() {
    // The characters hidden in the sample below. Printing stripped text proves
    // nothing — it looks the same either way — so assert they are gone instead.
    const HOSTILE: [char; 8] = [
        '\u{200b}', '\u{200c}', '\u{feff}', '\u{2060}',
        '\u{202e}', '\u{202c}', '\u{fe0f}', '\u{e0041}',
    ];

    let text = "pay\u{200b}pal\u{200c}\u{feff} ad\u{2060}min \u{202e}reversed\u{202c} text\u{fe0f}\u{e0041}";

    let text = strip_zero_width_chars(text);
    let text = strip_bidi(&text);
    let text = strip_tags(&text);
    let text = strip_variation_selectors(&text);
    let text = strip_noncharacters(&text);
    let text = strip_pua(&text);
    let text = strip_control_chars(&text);

    for ch in HOSTILE {
        assert!(!text.contains(ch), "U+{:04X} survived", ch as u32);
    }

    println!(r#"ok: {} hostile codepoints absent from "{}""#, HOSTILE.len(), text);
}

// Remove every class of invisible character disarm exposes in all bindings.
//   npm i disarm
const assert = require("node:assert/strict");
const {
  stripZeroWidthChars, stripBidi, stripTags,
  stripVariationSelectors, stripNoncharacters,
  stripPua, stripControlChars,
} = require("disarm");

// The characters hidden in the sample below. Printing stripped text proves
// nothing — it looks the same either way — so assert they are gone instead.
const HOSTILE = [..."​‌⁠‮‬️\u{E0041}"];

let text = "pay​pal‌ ad⁠min ‮reversed‬ text️\u{E0041}";

for (const step of [stripZeroWidthChars, stripBidi, stripTags,
                    stripVariationSelectors, stripNoncharacters,
                    stripPua, stripControlChars]) {
  text = step(text);
}

for (const ch of HOSTILE) {
  assert.ok(!text.includes(ch), `U+${ch.codePointAt(0).toString(16).toUpperCase()} survived`);
}

console.log(`ok: ${HOSTILE.length} hostile codepoints absent from "${text}"`);

# Remove every class of invisible character disarm exposes in all bindings.
#   gem install disarm
require "disarm"

# The characters hidden in the sample below. Printing stripped text proves
# nothing — it looks the same either way — so assert they are gone instead.
HOSTILE = "​‌⁠‮‬️\u{E0041}".chars

text = "pay​pal‌ ad⁠min ‮reversed‬ text️\u{E0041}"

%i[strip_zero_width_chars strip_bidi strip_tags
   strip_variation_selectors strip_noncharacters
   strip_pua strip_control_chars].each do |step|
  text = Disarm.public_send(step, text)
end

HOSTILE.each do |ch|
  raise format("U+%04X survived", ch.ord) if text.include?(ch)
end

puts %(ok: #{HOSTILE.length} hostile codepoints absent from "#{text}")

// Remove every class of invisible character disarm exposes in all bindings.
//   implementation("dev.disarm:disarm:0.14.1")
import dev.disarm.Disarm;

public class StripInvisibles {
    public static void main(String[] args) {
        // The characters hidden in the sample below. Printing stripped text
        // proves nothing — it looks the same either way — so check they are
        // gone instead. Java's `assert` is disabled unless the JVM is started
        // with -ea, so this throws rather than asserting.
        int[] hostile = {
            0x200B, 0x200C, 0xFEFF, 0x2060,
            0x202E, 0x202C, 0xFE0F, 0xE0041,
        };

        // U+E0041 is above the BMP, so Java spells it as a surrogate pair.
        String text = "pay​pal‌ ad⁠min ‮reversed‬ text️󠁁";

        text = Disarm.stripZeroWidthChars(text);
        text = Disarm.stripBidi(text);
        text = Disarm.stripTags(text);
        text = Disarm.stripVariationSelectors(text);
        text = Disarm.stripNoncharacters(text);
        text = Disarm.stripPua(text);
        text = Disarm.stripControlChars(text);

        for (int cp : hostile) {
            if (text.codePoints().anyMatch(c -> c == cp)) {
                throw new AssertionError(String.format("U+%04X survived", cp));
            }
        }

        System.out.printf("ok: %d hostile codepoints absent from \"%s\"%n",
                          hostile.length, text);
    }
}

// Remove every class of invisible character disarm exposes in all bindings.
//   implementation("dev.disarm:disarm-kotlin:0.14.1")
import dev.disarm.kotlin.*

fun main() {
    // The characters hidden in the sample below. Printing stripped text proves
    // nothing — it looks the same either way — so check they are gone instead.
    val hostile = listOf(
        0x200B, 0x200C, 0xFEFF, 0x2060,
        0x202E, 0x202C, 0xFE0F, 0xE0041,
    )

    // U+E0041 is above the BMP, so it is spelled as a surrogate pair.
    val text = "pay​pal‌ ad⁠min ‮reversed‬ text️󠁁"
        .stripZeroWidthChars()
        .stripBidi()
        .stripTags()
        .stripVariationSelectors()
        .stripNoncharacters()
        .stripPua()
        .stripControlChars()

    val survivors = text.codePoints().toArray().toSet()
    hostile.forEach { cp ->
        check(cp !in survivors) { "U+%04X survived".format(cp) }
    }

    println("ok: ${hostile.size} hostile codepoints absent from \"$text\"")
}

/* Remove every class of invisible character disarm exposes in all bindings.
 *
 * The C ABI is not published to any registry, so this links against a cdylib
 * built from bindings/cabi in the disarm repository. Every disarm_* function
 * here returns an owned string the caller frees with disarm_string_free.
 */
#include <stdio.h>
#include <string.h>
#include "disarm.h"

/* The characters hidden in the sample below, as UTF-8. Printing stripped text
 * proves nothing — it looks the same either way — so search for them instead. */
static const char *HOSTILE[] = {
    "​", "‌", "", "⁠",
    "‮", "‬", "️", "\U000E0041",
};
#define HOSTILE_LEN (sizeof HOSTILE / sizeof *HOSTILE)

int main(void) {
    const char *input =
        "pay​pal‌ ad⁠min ‮reversed‬ text️\U000E0041";

    char *(*steps[])(const char *) = {
        disarm_strip_zero_width_chars, disarm_strip_bidi, disarm_strip_tags,
        disarm_strip_variation_selectors, disarm_strip_noncharacters,
        disarm_strip_pua, disarm_strip_control_chars,
    };

    char *text = disarm_strip_zero_width_chars(input);
    for (size_t i = 1; i < sizeof steps / sizeof *steps; i++) {
        char *next = steps[i](text);
        disarm_string_free(text);
        text = next;
    }

    for (size_t i = 0; i < HOSTILE_LEN; i++) {
        if (strstr(text, HOSTILE[i]) != NULL) {
            fprintf(stderr, "a hostile codepoint survived\n");
            disarm_string_free(text);
            return 1;
        }
    }

    printf("ok: %zu hostile codepoints absent from \"%s\"\n", HOSTILE_LEN, text);
    disarm_string_free(text);
    return 0;
}