Detect zalgo text without breaking Vietnamese

The same thing in your own code

Each block is a file CI compiles and runs, so none can quietly stop working, and all six print the same line. There is no C here: the C ABI exposes no is_zalgo or strip_zalgo, so there is nothing to call. disarm on GitHub →

# Cap combining-mark stacking without corrupting Vietnamese.
#   pip install disarm
from disarm import is_zalgo, strip_zalgo

# Vietnamese puts two marks on one base: ế is U+0065 U+0302 U+0301.
VIETNAMESE = "Tiếng Việt"
# Five marks on one base, which no writing system uses.
ZALGO = "Hͤͥͦͧͨ"

# The naive rule — at most one mark — rejects an ordinary Vietnamese word.
assert is_zalgo(VIETNAMESE, threshold=1), "a threshold of 1 rejects Vietnamese"
assert not is_zalgo(VIETNAMESE, threshold=3), "the default does not"
assert is_zalgo(ZALGO, threshold=3), "and still catches sprawling zalgo"

# Capping at two is what leaves Vietnamese untouched.
assert strip_zalgo(VIETNAMESE, max_marks=2) == VIETNAMESE
assert strip_zalgo(ZALGO, max_marks=2) != ZALGO

print("ok: threshold 1 rejects Vietnamese, threshold 3 does not, zalgo caught either way")

// Cap combining-mark stacking without corrupting Vietnamese.
//   cargo add disarm
use disarm::api::{is_zalgo, strip_zalgo};

fn main() {
    // Vietnamese puts two marks on one base: ế is U+0065 U+0302 U+0301.
    let vietnamese = "Ti\u{1ebf}ng Vi\u{1ec7}t";
    // Five marks on one base, which no writing system uses.
    let zalgo = "H\u{0364}\u{0365}\u{0366}\u{0367}\u{0368}";

    // The naive rule — at most one mark — rejects an ordinary Vietnamese word.
    assert!(is_zalgo(vietnamese, 1), "a threshold of 1 rejects Vietnamese");
    assert!(!is_zalgo(vietnamese, 3), "the default does not");
    assert!(is_zalgo(zalgo, 3), "and still catches sprawling zalgo");

    // Capping at two is what leaves Vietnamese untouched.
    assert_eq!(strip_zalgo(vietnamese, 2), vietnamese);
    assert_ne!(strip_zalgo(zalgo, 2), zalgo);

    println!("ok: threshold 1 rejects Vietnamese, threshold 3 does not, zalgo caught either way");
}

// Cap combining-mark stacking without corrupting Vietnamese.
//   npm i disarm
const assert = require("node:assert/strict");
const { isZalgo, stripZalgo } = require("disarm");

// Vietnamese puts two marks on one base: ế is U+0065 U+0302 U+0301.
const VIETNAMESE = "Tiếng Việt";
// Five marks on one base, which no writing system uses.
const ZALGO = "Hͤͥͦͧͨ";

// The naive rule — at most one mark — rejects an ordinary Vietnamese word.
assert.ok(isZalgo(VIETNAMESE, { threshold: 1 }), "a threshold of 1 rejects Vietnamese");
assert.ok(!isZalgo(VIETNAMESE, { threshold: 3 }), "the default does not");
assert.ok(isZalgo(ZALGO, { threshold: 3 }), "and still catches sprawling zalgo");

// Capping at two is what leaves Vietnamese untouched.
assert.equal(stripZalgo(VIETNAMESE, { maxMarks: 2 }), VIETNAMESE);
assert.notEqual(stripZalgo(ZALGO, { maxMarks: 2 }), ZALGO);

console.log("ok: threshold 1 rejects Vietnamese, threshold 3 does not, zalgo caught either way");

# Cap combining-mark stacking without corrupting Vietnamese.
#   gem install disarm
require "disarm"

# Vietnamese puts two marks on one base: ế is U+0065 U+0302 U+0301.
VIETNAMESE = "Ti\u{1EBF}ng Vi\u{1EC7}t"
# Five marks on one base, which no writing system uses.
ZALGO = "H\u{0364}\u{0365}\u{0366}\u{0367}\u{0368}"

# The naive rule — at most one mark — rejects an ordinary Vietnamese word.
raise "a threshold of 1 rejects Vietnamese" unless Disarm.zalgo?(VIETNAMESE, threshold: 1)
raise "the default does not" if Disarm.zalgo?(VIETNAMESE, threshold: 3)
raise "and still catches sprawling zalgo" unless Disarm.zalgo?(ZALGO, threshold: 3)

# Capping at two is what leaves Vietnamese untouched.
raise "cap 2 must leave Vietnamese alone" unless Disarm.strip_zalgo(VIETNAMESE, max_marks: 2) == VIETNAMESE
raise "cap 2 must change zalgo" if Disarm.strip_zalgo(ZALGO, max_marks: 2) == ZALGO

puts "ok: threshold 1 rejects Vietnamese, threshold 3 does not, zalgo caught either way"

// Cap combining-mark stacking without corrupting Vietnamese.
//   implementation("dev.disarm:disarm:0.14.1")
import dev.disarm.Disarm;

public class DetectZalgo {
    public static void main(String[] args) {
        // Vietnamese puts two marks on one base: ế is U+0065 U+0302 U+0301.
        String vietnamese = "Tiếng Việt";
        // Five marks on one base, which no writing system uses.
        String zalgo = "Hͤͥͦͧͨ";

        // The naive rule — at most one mark — rejects an ordinary Vietnamese word.
        if (!Disarm.isZalgo(vietnamese, 1)) throw new AssertionError("a threshold of 1 rejects Vietnamese");
        if (Disarm.isZalgo(vietnamese, 3)) throw new AssertionError("the default does not");
        if (!Disarm.isZalgo(zalgo, 3)) throw new AssertionError("and still catches sprawling zalgo");

        // Capping at two is what leaves Vietnamese untouched.
        if (!Disarm.stripZalgo(vietnamese, 2).equals(vietnamese)) throw new AssertionError("cap 2 must leave Vietnamese alone");
        if (Disarm.stripZalgo(zalgo, 2).equals(zalgo)) throw new AssertionError("cap 2 must change zalgo");

        System.out.println("ok: threshold 1 rejects Vietnamese, threshold 3 does not, zalgo caught either way");
    }
}

// Cap combining-mark stacking without corrupting Vietnamese.
//   implementation("dev.disarm:disarm-kotlin:0.14.1")
import dev.disarm.kotlin.*

fun main() {
    // Vietnamese puts two marks on one base: ế is U+0065 U+0302 U+0301.
    val vietnamese = "Tiếng Việt"
    // Five marks on one base, which no writing system uses.
    val zalgo = "Hͤͥͦͧͨ"

    // The naive rule — at most one mark — rejects an ordinary Vietnamese word.
    check(vietnamese.isZalgo(1)) { "a threshold of 1 rejects Vietnamese" }
    check(!vietnamese.isZalgo(3)) { "the default does not" }
    check(zalgo.isZalgo(3)) { "and still catches sprawling zalgo" }

    // Capping at two is what leaves Vietnamese untouched.
    check(vietnamese.stripZalgo(2) == vietnamese) { "cap 2 must leave Vietnamese alone" }
    check(zalgo.stripZalgo(2) != zalgo) { "cap 2 must change zalgo" }

    println("ok: threshold 1 rejects Vietnamese, threshold 3 does not, zalgo caught either way")
}