source: trunk/gsdl/perllib/cnseg.pm@ 241

# this package segments a chinese UTF-8 encoded Unicode
# string into words.

package cnseg;

use unicode;


# 'segment' takes a UTF-8 encoded Unicode Chinese-language
# string and places U-200B between words -- the ZERO
# WIDTH SPACE. Each line is treated as a separate 
# paragraph, so lines in one paragraph should
# be joined before using this method (normally a single
# word might span more than one line within a paragraph).
#
# 'segment' is currently written in Perl, however, I (Rodger)
# plan to use C++ (via pipes) once a more complex (and useful!)
# algorithm is being used. Currently, each Chinese character
# is treated as a seperate word.

sub segment {
    my ($in) = @_;
    my ($c);

    my $uniin = &unicode::utf82unicode($in);
    my $out = [];

    my $space = 1; # start doesn't need a space
    foreach $c (@$uniin) {
    if (($c >= 0x4e00 && $c <= 0x9fa5) ||
        ($c >= 0xf900 && $c <= 0xfa2d)) {
        # Chinese character
        push (@$out, 0x200b) unless $space;
        push (@$out, $c);
        push (@$out, 0x200b);
        $space = 1;

    } else {
        # non-Chinese character
        push (@$out, $c);
        $space = 0;
    }
    }

    return &unicode::unicode2utf8($out);
}

1;