| <?php |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| if (!defined('IN_PHPBB')) |
| { |
| exit; |
| } |
| |
| |
| |
| |
| |
| |
| define('UTF8_REPLACEMENT', "\xEF\xBF\xBD"); |
| define('UTF8_MAX', "\xF4\x8F\xBF\xBF"); |
| define('UTF8_FFFE', "\xEF\xBF\xBE"); |
| define('UTF8_FFFF', "\xEF\xBF\xBF"); |
| define('UTF8_SURROGATE_FIRST', "\xED\xA0\x80"); |
| define('UTF8_SURROGATE_LAST', "\xED\xBF\xBF"); |
| define('UTF8_HANGUL_FIRST', "\xEA\xB0\x80"); |
| define('UTF8_HANGUL_LAST', "\xED\x9E\xA3"); |
| |
| define('UTF8_CJK_FIRST', "\xE4\xB8\x80"); |
| define('UTF8_CJK_LAST', "\xE9\xBE\xBB"); |
| define('UTF8_CJK_B_FIRST', "\xF0\xA0\x80\x80"); |
| define('UTF8_CJK_B_LAST', "\xF0\xAA\x9B\x96"); |
| |
| |
| unset($GLOBALS['utf_jamo_index'], $GLOBALS['utf_jamo_type'], $GLOBALS['utf_nfc_qc'], $GLOBALS['utf_combining_class'], $GLOBALS['utf_canonical_comp'], $GLOBALS['utf_canonical_decomp'], $GLOBALS['utf_nfkc_qc'], $GLOBALS['utf_compatibility_decomp']); |
| |
| |
| define('UNICODE_QC_MAYBE', 0); |
| define('UNICODE_QC_NO', 1); |
| |
| |
| define('UTF8_ASCII_RANGE', "\x20\x65\x69\x61\x73\x6E\x74\x72\x6F\x6C\x75\x64\x5D\x5B\x63\x6D\x70\x27\x0A\x67\x7C\x68\x76\x2E\x66\x62\x2C\x3A\x3D\x2D\x71\x31\x30\x43\x32\x2A\x79\x78\x29\x28\x4C\x39\x41\x53\x2F\x50\x22\x45\x6A\x4D\x49\x6B\x33\x3E\x35\x54\x3C\x44\x34\x7D\x42\x7B\x38\x46\x77\x52\x36\x37\x55\x47\x4E\x3B\x4A\x7A\x56\x23\x48\x4F\x57\x5F\x26\x21\x4B\x3F\x58\x51\x25\x59\x5C\x09\x5A\x2B\x7E\x5E\x24\x40\x60\x7F\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0D\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F"); |
| |
| |
| define('UTF8_TRAILING_BYTES', "\xA9\xA0\xA8\x80\xAA\x99\xA7\xBB\xAB\x89\x94\x82\xB4\xA2\xAE\x83\xB0\xB9\xB8\x93\xAF\xBC\xB3\x81\xA4\xB2\x9C\xA1\xB5\xBE\xBD\xBA\x98\xAD\xB1\x84\x95\xA6\xB6\x88\x8D\x90\xB7\xBF\x92\x85\xA5\x97\x8C\x86\xA3\x8E\x9F\x8F\x87\x91\x9D\xAC\x9E\x8B\x96\x9B\x8A\x9A"); |
| |
| |
| define('UNICODE_HANGUL_SBASE', 0xAC00); |
| define('UNICODE_HANGUL_LBASE', 0x1100); |
| define('UNICODE_HANGUL_VBASE', 0x1161); |
| define('UNICODE_HANGUL_TBASE', 0x11A7); |
| define('UNICODE_HANGUL_SCOUNT', 11172); |
| define('UNICODE_HANGUL_LCOUNT', 19); |
| define('UNICODE_HANGUL_VCOUNT', 21); |
| define('UNICODE_HANGUL_TCOUNT', 28); |
| define('UNICODE_HANGUL_NCOUNT', 588); |
| define('UNICODE_JAMO_L', 0); |
| define('UNICODE_JAMO_V', 1); |
| define('UNICODE_JAMO_T', 2); |
| |
| |
| |
| |
| |
| |
| class utf_normalizer |
| { |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| function cleanup(&$str) |
| { |
| |
| $pos = strspn($str, "\x20\x65\x69\x61\x73\x6E\x74\x72\x6F\x6C\x75\x64\x5D\x5B\x63\x6D\x70\x27\x0A\x67\x7C\x68\x76\x2E\x66\x62\x2C\x3A\x3D\x2D\x71\x31\x30\x43\x32\x2A\x79\x78\x29\x28\x4C\x39\x41\x53\x2F\x50\x22\x45\x6A\x4D\x49\x6B\x33\x3E\x35\x54\x3C\x44\x34\x7D\x42\x7B\x38\x46\x77\x52\x36\x37\x55\x47\x4E\x3B\x4A\x7A\x56\x23\x48\x4F\x57\x5F\x26\x21\x4B\x3F\x58\x51\x25\x59\x5C\x09\x5A\x2B\x7E\x5E\x24\x40\x60\x7F\x0D"); |
| $len = strlen($str); |
| |
| if ($pos == $len) |
| { |
| |
| return; |
| } |
| |
| |
| if (!isset($GLOBALS['utf_nfc_qc'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_nfc_qc.' . $phpEx); |
| } |
| |
| if (!isset($GLOBALS['utf_canonical_decomp'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx); |
| } |
| |
| |
| |
| $str = strtr( |
| $str, |
| "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F", |
| "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" |
| ); |
| |
| $str = utf_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfc_qc'], $GLOBALS['utf_canonical_decomp']); |
| } |
| |
| |
| |
| |
| |
| |
| |
| function nfc(&$str) |
| { |
| $pos = strspn($str, UTF8_ASCII_RANGE); |
| $len = strlen($str); |
| |
| if ($pos == $len) |
| { |
| |
| return; |
| } |
| |
| if (!isset($GLOBALS['utf_nfc_qc'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_nfc_qc.' . $phpEx); |
| } |
| |
| if (!isset($GLOBALS['utf_canonical_decomp'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx); |
| } |
| |
| $str = utf_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfc_qc'], $GLOBALS['utf_canonical_decomp']); |
| } |
| |
| |
| |
| |
| |
| |
| |
| function nfkc(&$str) |
| { |
| $pos = strspn($str, UTF8_ASCII_RANGE); |
| $len = strlen($str); |
| |
| if ($pos == $len) |
| { |
| |
| return; |
| } |
| |
| if (!isset($GLOBALS['utf_nfkc_qc'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_nfkc_qc.' . $phpEx); |
| } |
| |
| if (!isset($GLOBALS['utf_compatibility_decomp'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_compatibility_decomp.' . $phpEx); |
| } |
| |
| $str = utf_normalizer::recompose($str, $pos, $len, $GLOBALS['utf_nfkc_qc'], $GLOBALS['utf_compatibility_decomp']); |
| } |
| |
| |
| |
| |
| |
| |
| |
| function nfd(&$str) |
| { |
| $pos = strspn($str, UTF8_ASCII_RANGE); |
| $len = strlen($str); |
| |
| if ($pos == $len) |
| { |
| |
| return; |
| } |
| |
| if (!isset($GLOBALS['utf_canonical_decomp'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_canonical_decomp.' . $phpEx); |
| } |
| |
| $str = utf_normalizer::decompose($str, $pos, $len, $GLOBALS['utf_canonical_decomp']); |
| } |
| |
| |
| |
| |
| |
| |
| |
| function nfkd(&$str) |
| { |
| $pos = strspn($str, UTF8_ASCII_RANGE); |
| $len = strlen($str); |
| |
| if ($pos == $len) |
| { |
| |
| return; |
| } |
| |
| if (!isset($GLOBALS['utf_compatibility_decomp'])) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_compatibility_decomp.' . $phpEx); |
| } |
| |
| $str = utf_normalizer::decompose($str, $pos, $len, $GLOBALS['utf_compatibility_decomp']); |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| function recompose($str, $pos, $len, &$qc, &$decomp_map) |
| { |
| global $utf_combining_class, $utf_canonical_comp, $utf_jamo_type, $utf_jamo_index; |
| |
| |
| if (!isset($utf_jamo_index, $utf_jamo_type, $utf_combining_class)) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_normalizer_common.' . $phpEx); |
| } |
| |
| |
| if (!isset($utf_canonical_comp)) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_canonical_comp.' . $phpEx); |
| } |
| |
| |
| $tmp = ''; |
| $i = $tmp_pos = $last_cc = 0; |
| |
| $buffer = ($pos) ? array(++$i => $str[$pos - 1]) : array(); |
| |
| |
| |
| |
| |
| |
| |
| $utf_len_mask = array( |
| |
| "\xC0" => 2, "\xD0" => 2, "\xE0" => 3, "\xF0" => 4, |
| |
| "\x80" => 0, "\x90" => 0, "\xA0" => 0, "\xB0" => 0 |
| ); |
| |
| $extra_check = array( |
| "\xED" => 1, "\xEF" => 1, "\xC0" => 1, "\xC1" => 1, "\xE0" => 1, "\xF0" => 1, |
| "\xF4" => 1, "\xF5" => 1, "\xF6" => 1, "\xF7" => 1, "\xF8" => 1, "\xF9" => 1, |
| "\xFA" => 1, "\xFB" => 1, "\xFC" => 1, "\xFD" => 1, "\xFE" => 1, "\xFF" => 1 |
| ); |
| |
| $utf_validation_mask = array( |
| 2 => "\xE0\xC0", |
| 3 => "\xF0\xC0\xC0", |
| 4 => "\xF8\xC0\xC0\xC0" |
| ); |
| |
| $utf_validation_check = array( |
| 2 => "\xC0\x80", |
| 3 => "\xE0\x80\x80", |
| 4 => "\xF0\x80\x80\x80" |
| ); |
| |
| |
| do |
| { |
| |
| $c = $str[$pos]; |
| $c_mask = $c & "\xF0"; |
| |
| if (isset($utf_len_mask[$c_mask])) |
| { |
| |
| if ($utf_len = $utf_len_mask[$c_mask]) |
| { |
| |
| $buffer[++$i & 7] = $utf_char = substr($str, $pos, $utf_len); |
| |
| |
| if (isset($qc[$utf_char])) |
| { |
| |
| } |
| else if (isset($utf_combining_class[$utf_char])) |
| { |
| if ($utf_combining_class[$utf_char] < $last_cc) |
| { |
| |
| } |
| else |
| { |
| |
| $last_cc = $utf_combining_class[$utf_char]; |
| |
| $pos += $utf_len; |
| continue; |
| } |
| } |
| else |
| { |
| |
| |
| $last_cc = 0; |
| |
| |
| if (($utf_char & $utf_validation_mask[$utf_len]) != $utf_validation_check[$utf_len]) |
| { |
| |
| |
| if ($utf_char[0] >= "\xF8") |
| { |
| if ($utf_char[0] < "\xFC") |
| { |
| $trailing_bytes = 4; |
| } |
| else if ($utf_char[0] > "\xFD") |
| { |
| $trailing_bytes = 0; |
| } |
| else |
| { |
| $trailing_bytes = 5; |
| } |
| } |
| else |
| { |
| $trailing_bytes = $utf_len - 1; |
| } |
| |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . UTF8_REPLACEMENT; |
| $pos += strspn($str, UTF8_TRAILING_BYTES, ++$pos, $trailing_bytes); |
| $tmp_pos = $pos; |
| |
| continue; |
| } |
| |
| if (isset($extra_check[$c])) |
| { |
| switch ($c) |
| { |
| |
| case "\xED": |
| if ($utf_char >= "\xED\xA0\x80") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . UTF8_REPLACEMENT; |
| $pos += $utf_len; |
| $tmp_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| |
| case "\xEF": |
| if ($utf_char == "\xEF\xBF\xBE" || $utf_char == "\xEF\xBF\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . UTF8_REPLACEMENT; |
| $pos += $utf_len; |
| $tmp_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| case "\xC0": |
| case "\xC1": |
| if ($utf_char <= "\xC1\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . UTF8_REPLACEMENT; |
| $pos += $utf_len; |
| $tmp_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| case "\xE0": |
| if ($utf_char <= "\xE0\x9F\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . UTF8_REPLACEMENT; |
| $pos += $utf_len; |
| $tmp_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| case "\xF0": |
| if ($utf_char <= "\xF0\x8F\xBF\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . UTF8_REPLACEMENT; |
| $pos += $utf_len; |
| $tmp_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| default: |
| |
| if ($utf_char > UTF8_MAX) |
| { |
| |
| if ($utf_char[0] < "\xF8") |
| { |
| $trailing_bytes = 3; |
| } |
| else if ($utf_char[0] < "\xFC") |
| { |
| $trailing_bytes = 4; |
| } |
| else if ($utf_char[0] > "\xFD") |
| { |
| $trailing_bytes = 0; |
| } |
| else |
| { |
| $trailing_bytes = 5; |
| } |
| |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . UTF8_REPLACEMENT; |
| $pos += strspn($str, UTF8_TRAILING_BYTES, ++$pos, $trailing_bytes); |
| $tmp_pos = $pos; |
| continue 2; |
| } |
| break; |
| } |
| } |
| |
| |
| $pos += $utf_len; |
| continue; |
| } |
| } |
| else |
| { |
| |
| |
| $spn = strspn($str, UTF8_TRAILING_BYTES, $pos); |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . str_repeat(UTF8_REPLACEMENT, $spn); |
| |
| $pos += $spn; |
| $tmp_pos = $pos; |
| continue; |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| $utf_seq = array(); |
| $last_cc = 0; |
| $lpos = $pos; |
| $pos += $utf_len; |
| |
| if (isset($decomp_map[$utf_char])) |
| { |
| $_pos = 0; |
| $_len = strlen($decomp_map[$utf_char]); |
| |
| do |
| { |
| $_utf_len =& $utf_len_mask[$decomp_map[$utf_char][$_pos] & "\xF0"]; |
| |
| if (isset($_utf_len)) |
| { |
| $utf_seq[] = substr($decomp_map[$utf_char], $_pos, $_utf_len); |
| $_pos += $_utf_len; |
| } |
| else |
| { |
| $utf_seq[] = $decomp_map[$utf_char][$_pos]; |
| ++$_pos; |
| } |
| } |
| while ($_pos < $_len); |
| } |
| else |
| { |
| |
| $utf_seq = array($utf_char); |
| } |
| |
| |
| |
| |
| |
| $k = 0; |
| if (isset($utf_combining_class[$utf_seq[0]]) || $qc[$utf_char] == UNICODE_QC_MAYBE) |
| { |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| $starter_found = 0; |
| $j_min = max(1, $i - 7); |
| |
| for ($j = $i - 1; $j >= $j_min && $lpos > $tmp_pos; --$j) |
| { |
| $utf_char = $buffer[$j & 7]; |
| $lpos -= strlen($utf_char); |
| |
| if (isset($decomp_map[$utf_char])) |
| { |
| |
| $decomp_seq = array(); |
| $_pos = 0; |
| $_len = strlen($decomp_map[$utf_char]); |
| |
| do |
| { |
| $c = $decomp_map[$utf_char][$_pos]; |
| $_utf_len =& $utf_len_mask[$c & "\xF0"]; |
| |
| if (isset($_utf_len)) |
| { |
| $decomp_seq[] = substr($decomp_map[$utf_char], $_pos, $_utf_len); |
| $_pos += $_utf_len; |
| } |
| else |
| { |
| $decomp_seq[] = $c; |
| ++$_pos; |
| } |
| } |
| while ($_pos < $_len); |
| |
| |
| if (isset($decomp_seq[1])) |
| { |
| |
| $decomp_cnt = sizeof($decomp_seq); |
| |
| foreach ($decomp_seq as $decomp_i => $decomp_char) |
| { |
| $utf_seq[$k + $decomp_i - $decomp_cnt] = $decomp_char; |
| } |
| $k -= $decomp_cnt; |
| } |
| else |
| { |
| |
| $utf_seq[--$k] = $decomp_seq[0]; |
| } |
| } |
| else |
| { |
| $utf_seq[--$k] = $utf_char; |
| } |
| |
| if (!isset($utf_combining_class[$utf_seq[$k]])) |
| { |
| |
| $starter_found = 1; |
| break; |
| } |
| } |
| |
| if (!$starter_found && $lpos > $tmp_pos) |
| { |
| |
| do |
| { |
| |
| $c = $str[--$lpos]; |
| $c_mask = $c & "\xF0"; |
| |
| if (isset($utf_len_mask[$c_mask])) |
| { |
| |
| if ($utf_len = $utf_len_mask[$c_mask]) |
| { |
| |
| $utf_char = substr($str, $lpos, $utf_len); |
| |
| if (isset($decomp_map[$utf_char])) |
| { |
| |
| $decomp_seq = array(); |
| $_pos = 0; |
| $_len = strlen($decomp_map[$utf_char]); |
| |
| do |
| { |
| $c = $decomp_map[$utf_char][$_pos]; |
| $_utf_len =& $utf_len_mask[$c & "\xF0"]; |
| |
| if (isset($_utf_len)) |
| { |
| $decomp_seq[] = substr($decomp_map[$utf_char], $_pos, $_utf_len); |
| $_pos += $_utf_len; |
| } |
| else |
| { |
| $decomp_seq[] = $c; |
| ++$_pos; |
| } |
| } |
| while ($_pos < $_len); |
| |
| |
| if (isset($decomp_seq[1])) |
| { |
| |
| $decomp_cnt = sizeof($decomp_seq); |
| foreach ($decomp_seq as $decomp_i => $utf_char) |
| { |
| $utf_seq[$k + $decomp_i - $decomp_cnt] = $utf_char; |
| } |
| $k -= $decomp_cnt; |
| } |
| else |
| { |
| |
| $utf_seq[--$k] = $decomp_seq[0]; |
| } |
| } |
| else |
| { |
| $utf_seq[--$k] = $utf_char; |
| } |
| } |
| } |
| else |
| { |
| |
| $utf_seq[--$k] = $c; |
| } |
| } |
| while ($lpos > $tmp_pos); |
| } |
| } |
| |
| |
| |
| |
| while ($pos < $len) |
| { |
| $c_mask = $str[$pos] & "\xF0"; |
| |
| if (isset($utf_len_mask[$c_mask])) |
| { |
| if ($utf_len = $utf_len_mask[$c_mask]) |
| { |
| $utf_char = substr($str, $pos, $utf_len); |
| } |
| else |
| { |
| |
| |
| |
| break; |
| } |
| |
| if (isset($utf_combining_class[$utf_char]) || isset($qc[$utf_char])) |
| { |
| |
| if (isset($decomp_map[$utf_char])) |
| { |
| |
| $_pos = 0; |
| $_len = strlen($decomp_map[$utf_char]); |
| |
| do |
| { |
| $c = $decomp_map[$utf_char][$_pos]; |
| $_utf_len =& $utf_len_mask[$c & "\xF0"]; |
| |
| if (isset($_utf_len)) |
| { |
| $utf_seq[] = substr($decomp_map[$utf_char], $_pos, $_utf_len); |
| $_pos += $_utf_len; |
| } |
| else |
| { |
| $utf_seq[] = $c; |
| ++$_pos; |
| } |
| } |
| while ($_pos < $_len); |
| } |
| else |
| { |
| $utf_seq[] = $utf_char; |
| } |
| |
| $pos += $utf_len; |
| } |
| else |
| { |
| |
| |
| break; |
| } |
| } |
| else |
| { |
| |
| break; |
| } |
| } |
| |
| |
| |
| |
| |
| $k_max = $k + sizeof($utf_seq); |
| |
| if (!$k && $k_max == 1) |
| { |
| |
| |
| |
| |
| $tmp .= substr($str, $tmp_pos, $lpos - $tmp_pos) . $utf_seq[0]; |
| $tmp_pos = $pos; |
| |
| |
| continue; |
| } |
| |
| |
| if (isset($utf_combining_class[$utf_seq[$k]])) |
| { |
| $starter = $nf_seq = ''; |
| } |
| else |
| { |
| $starter = $utf_seq[$k++]; |
| $nf_seq = ''; |
| } |
| $utf_sort = array(); |
| |
| |
| |
| $utf_seq[] = ''; |
| |
| do |
| { |
| $utf_char = $utf_seq[$k++]; |
| |
| if (isset($utf_combining_class[$utf_char])) |
| { |
| $utf_sort[$utf_combining_class[$utf_char]][] = $utf_char; |
| } |
| else |
| { |
| if (empty($utf_sort)) |
| { |
| |
| if (isset($utf_canonical_comp[$starter . $utf_char])) |
| { |
| |
| $starter = $utf_canonical_comp[$starter . $utf_char]; |
| } |
| else if (isset($utf_jamo_type[$utf_char])) |
| { |
| |
| if (isset($utf_jamo_type[$starter]) && $utf_jamo_type[$starter] == UNICODE_JAMO_L && $utf_jamo_type[$utf_char] == UNICODE_JAMO_V) |
| { |
| |
| if (isset($utf_jamo_type[$utf_seq[$k]]) && $utf_jamo_type[$utf_seq[$k]] == UNICODE_JAMO_T) |
| { |
| |
| $cp = $utf_jamo_index[$starter] + $utf_jamo_index[$utf_char] + $utf_jamo_index[$utf_seq[$k]]; |
| ++$k; |
| } |
| else |
| { |
| |
| $cp = $utf_jamo_index[$starter] + $utf_jamo_index[$utf_char]; |
| } |
| |
| $starter = chr(0xE0 | ($cp >> 12)) . chr(0x80 | (($cp >> 6) & 0x3F)) . chr(0x80 | ($cp & 0x3F)); |
| } |
| else |
| { |
| |
| $nf_seq .= $starter; |
| $starter = $utf_char; |
| } |
| } |
| else |
| { |
| |
| $nf_seq .= $starter; |
| $starter = $utf_char; |
| } |
| } |
| else |
| { |
| ksort($utf_sort); |
| |
| |
| foreach ($utf_sort as $cc => $utf_chars) |
| { |
| $j = 0; |
| |
| do |
| { |
| |
| if (isset($utf_canonical_comp[$starter . $utf_chars[$j]])) |
| { |
| |
| $starter = $utf_canonical_comp[$starter . $utf_chars[$j]]; |
| unset($utf_sort[$cc][$j]); |
| } |
| else |
| { |
| |
| break; |
| } |
| } |
| while (isset($utf_sort[$cc][++$j])); |
| } |
| |
| |
| $nf_seq .= $starter; |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| if (!empty($utf_chars)) |
| { |
| $nf_seq .= implode('', $utf_chars); |
| } |
| } |
| |
| |
| $utf_sort = array(); |
| $starter = $utf_char; |
| } |
| } |
| } |
| while ($k <= $k_max); |
| |
| $tmp .= substr($str, $tmp_pos, $lpos - $tmp_pos) . $nf_seq; |
| $tmp_pos = $pos; |
| } |
| else |
| { |
| |
| |
| |
| |
| |
| |
| if (++$pos != $len) |
| { |
| if ($str[$pos] < "\x80") |
| { |
| $pos += strspn($str, UTF8_ASCII_RANGE, ++$pos); |
| $buffer[++$i & 7] = $str[$pos - 1]; |
| } |
| else |
| { |
| $buffer[++$i & 7] = $c; |
| } |
| } |
| } |
| } |
| while ($pos < $len); |
| |
| |
| if ($tmp_pos) |
| { |
| |
| if ($tmp_pos == $len) |
| { |
| |
| return $tmp; |
| } |
| else |
| { |
| |
| return $tmp . substr($str, $tmp_pos); |
| } |
| } |
| |
| |
| return $str; |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| function decompose($str, $pos, $len, &$decomp_map) |
| { |
| global $utf_combining_class; |
| |
| |
| if (!isset($utf_combining_class)) |
| { |
| global $phpbb_root_path, $phpEx; |
| include($phpbb_root_path . 'includes/utf/data/utf_normalizer_common.' . $phpEx); |
| } |
| |
| |
| $utf_len_mask = array( |
| |
| "\xC0" => 2, "\xD0" => 2, "\xE0" => 3, "\xF0" => 4, |
| |
| "\x80" => 0, "\x90" => 0, "\xA0" => 0, "\xB0" => 0 |
| ); |
| |
| |
| $extra_check = array( |
| "\xED" => 1, "\xEF" => 1, "\xC0" => 1, "\xC1" => 1, "\xE0" => 1, "\xF0" => 1, |
| "\xF4" => 1, "\xF5" => 1, "\xF6" => 1, "\xF7" => 1, "\xF8" => 1, "\xF9" => 1, |
| "\xFA" => 1, "\xFB" => 1, "\xFC" => 1, "\xFD" => 1, "\xFE" => 1, "\xFF" => 1 |
| ); |
| |
| |
| |
| |
| |
| |
| $utf_validation_mask = array( |
| 2 => "\xE0\xC0", |
| 3 => "\xF0\xC0\xC0", |
| 4 => "\xF8\xC0\xC0\xC0" |
| ); |
| |
| $utf_validation_check = array( |
| 2 => "\xC0\x80", |
| 3 => "\xE0\x80\x80", |
| 4 => "\xF0\x80\x80\x80" |
| ); |
| |
| $tmp = ''; |
| $starter_pos = $pos; |
| $tmp_pos = $last_cc = $sort = $dump = 0; |
| $utf_sort = array(); |
| |
| |
| |
| do |
| { |
| |
| |
| $cur_mask = $str[$pos] & "\xF0"; |
| if (isset($utf_len_mask[$cur_mask])) |
| { |
| if ($utf_len = $utf_len_mask[$cur_mask]) |
| { |
| |
| $utf_char = substr($str, $pos, $utf_len); |
| $pos += $utf_len; |
| } |
| else |
| { |
| |
| |
| $spn = strspn($str, UTF8_TRAILING_BYTES, $pos); |
| |
| if ($dump) |
| { |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| |
| if (!empty($utf_sort)) |
| { |
| if ($sort) |
| { |
| ksort($utf_sort); |
| } |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| } |
| |
| $tmp .= str_repeat(UTF8_REPLACEMENT, $spn); |
| $dump = $sort = 0; |
| } |
| else |
| { |
| $tmp .= substr($str, $tmp_pos, $pos - $tmp_pos) . str_repeat(UTF8_REPLACEMENT, $spn); |
| } |
| |
| $pos += $spn; |
| $tmp_pos = $starter_pos = $pos; |
| |
| $utf_sort = array(); |
| $last_cc = 0; |
| |
| continue; |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| if (isset($decomp_map[$utf_char])) |
| { |
| |
| $_pos = 0; |
| $_len = strlen($decomp_map[$utf_char]); |
| |
| do |
| { |
| $c = $decomp_map[$utf_char][$_pos]; |
| $_utf_len =& $utf_len_mask[$c & "\xF0"]; |
| |
| if (isset($_utf_len)) |
| { |
| $_utf_char = substr($decomp_map[$utf_char], $_pos, $_utf_len); |
| $_pos += $_utf_len; |
| |
| if (isset($utf_combining_class[$_utf_char])) |
| { |
| |
| $utf_sort[$utf_combining_class[$_utf_char]][] = $_utf_char; |
| |
| if ($utf_combining_class[$_utf_char] < $last_cc) |
| { |
| |
| $sort = $dump = 1; |
| } |
| else |
| { |
| $dump = 1; |
| $last_cc = $utf_combining_class[$_utf_char]; |
| } |
| } |
| else |
| { |
| |
| if ($dump) |
| { |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| |
| if (!empty($utf_sort)) |
| { |
| if ($sort) |
| { |
| ksort($utf_sort); |
| } |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| } |
| |
| $tmp .= $_utf_char; |
| $dump = $sort = 0; |
| } |
| else |
| { |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos) . $_utf_char; |
| } |
| |
| $tmp_pos = $starter_pos = $pos; |
| $utf_sort = array(); |
| $last_cc = 0; |
| } |
| } |
| else |
| { |
| |
| ++$_pos; |
| |
| if ($dump) |
| { |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| |
| if (!empty($utf_sort)) |
| { |
| if ($sort) |
| { |
| ksort($utf_sort); |
| } |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| } |
| |
| $tmp .= $c; |
| $dump = $sort = 0; |
| } |
| else |
| { |
| $tmp .= substr($str, $tmp_pos, $pos - $utf_len - $tmp_pos) . $c; |
| } |
| |
| $tmp_pos = $starter_pos = $pos; |
| $utf_sort = array(); |
| $last_cc = 0; |
| } |
| } |
| while ($_pos < $_len); |
| } |
| else if (isset($utf_combining_class[$utf_char])) |
| { |
| |
| if ($utf_combining_class[$utf_char] < $last_cc) |
| { |
| |
| $sort = $dump = 1; |
| } |
| else |
| { |
| $last_cc = $utf_combining_class[$utf_char]; |
| } |
| |
| $utf_sort[$utf_combining_class[$utf_char]][] = $utf_char; |
| } |
| else |
| { |
| |
| if ($utf_char < UTF8_HANGUL_FIRST || $utf_char > UTF8_HANGUL_LAST) |
| { |
| |
| if (($utf_char & $utf_validation_mask[$utf_len]) != $utf_validation_check[$utf_len]) |
| { |
| |
| |
| |
| $pos -= $utf_len; |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| if (!empty($utf_sort)) |
| { |
| ksort($utf_sort); |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| $utf_sort = array(); |
| } |
| |
| |
| |
| |
| $spn = strspn($str, UTF8_TRAILING_BYTES, ++$pos); |
| $tmp .= str_repeat(UTF8_REPLACEMENT, $spn + 1); |
| |
| $dump = $sort = 0; |
| |
| $pos += $spn; |
| $tmp_pos = $pos; |
| continue; |
| } |
| |
| if (isset($extra_check[$utf_char[0]])) |
| { |
| switch ($utf_char[0]) |
| { |
| |
| case "\xED": |
| if ($utf_char >= "\xED\xA0\x80") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| if (!empty($utf_sort)) |
| { |
| ksort($utf_sort); |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| $utf_sort = array(); |
| } |
| |
| $tmp .= UTF8_REPLACEMENT; |
| $dump = $sort = 0; |
| |
| $tmp_pos = $starter_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| |
| case "\xEF": |
| if ($utf_char == "\xEF\xBF\xBE" || $utf_char == "\xEF\xBF\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| if (!empty($utf_sort)) |
| { |
| ksort($utf_sort); |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| $utf_sort = array(); |
| } |
| |
| $tmp .= UTF8_REPLACEMENT; |
| $dump = $sort = 0; |
| |
| $tmp_pos = $starter_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| case "\xC0": |
| case "\xC1": |
| if ($utf_char <= "\xC1\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| if (!empty($utf_sort)) |
| { |
| ksort($utf_sort); |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| $utf_sort = array(); |
| } |
| |
| $tmp .= UTF8_REPLACEMENT; |
| $dump = $sort = 0; |
| |
| $tmp_pos = $starter_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| case "\xE0": |
| if ($utf_char <= "\xE0\x9F\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| if (!empty($utf_sort)) |
| { |
| ksort($utf_sort); |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| $utf_sort = array(); |
| } |
| |
| $tmp .= UTF8_REPLACEMENT; |
| $dump = $sort = 0; |
| |
| $tmp_pos = $starter_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| case "\xF0": |
| if ($utf_char <= "\xF0\x8F\xBF\xBF") |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| if (!empty($utf_sort)) |
| { |
| ksort($utf_sort); |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| $utf_sort = array(); |
| } |
| |
| $tmp .= UTF8_REPLACEMENT; |
| $dump = $sort = 0; |
| |
| $tmp_pos = $starter_pos = $pos; |
| continue 2; |
| } |
| break; |
| |
| default: |
| if ($utf_char > UTF8_MAX) |
| { |
| |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| if (!empty($utf_sort)) |
| { |
| ksort($utf_sort); |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| $utf_sort = array(); |
| } |
| |
| $tmp .= UTF8_REPLACEMENT; |
| $dump = $sort = 0; |
| |
| $tmp_pos = $starter_pos = $pos; |
| continue 2; |
| } |
| break; |
| } |
| } |
| } |
| else |
| { |
| |
| $idx = (((ord($utf_char[0]) & 0x0F) << 12) | ((ord($utf_char[1]) & 0x3F) << 6) | (ord($utf_char[2]) & 0x3F)) - UNICODE_HANGUL_SBASE; |
| |
| |
| |
| |
| if ($t_index = $idx % UNICODE_HANGUL_TCOUNT) |
| { |
| if ($t_index < 25) |
| { |
| $utf_char = "\xE1\x84\x00\xE1\x85\x00\xE1\x86\x00"; |
| $utf_char[8] = chr(0xA7 + $t_index); |
| } |
| else |
| { |
| $utf_char = "\xE1\x84\x00\xE1\x85\x00\xE1\x87\x00"; |
| $utf_char[8] = chr(0x67 + $t_index); |
| } |
| } |
| else |
| { |
| $utf_char = "\xE1\x84\x00\xE1\x85\x00"; |
| } |
| |
| $utf_char[2] = chr(0x80 + (int) ($idx / UNICODE_HANGUL_NCOUNT)); |
| $utf_char[5] = chr(0xA1 + (int) (($idx % UNICODE_HANGUL_NCOUNT) / UNICODE_HANGUL_TCOUNT)); |
| |
| |
| $dump = 1; |
| } |
| |
| |
| if ($dump) |
| { |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| |
| if (!empty($utf_sort)) |
| { |
| if ($sort) |
| { |
| ksort($utf_sort); |
| } |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| } |
| |
| $tmp .= $utf_char; |
| $dump = $sort = 0; |
| $tmp_pos = $pos; |
| } |
| |
| $last_cc = 0; |
| $utf_sort = array(); |
| $starter_pos = $pos; |
| } |
| } |
| else |
| { |
| |
| if ($dump) |
| { |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| |
| if (!empty($utf_sort)) |
| { |
| if ($sort) |
| { |
| ksort($utf_sort); |
| } |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| } |
| |
| $tmp .= $str[$pos]; |
| $dump = $sort = 0; |
| $tmp_pos = ++$pos; |
| |
| $pos += strspn($str, UTF8_ASCII_RANGE, $pos); |
| } |
| else |
| { |
| $pos += strspn($str, UTF8_ASCII_RANGE, ++$pos); |
| } |
| |
| $last_cc = 0; |
| $utf_sort = array(); |
| $starter_pos = $pos; |
| } |
| } |
| while ($pos < $len); |
| |
| |
| if ($dump) |
| { |
| $tmp .= substr($str, $tmp_pos, $starter_pos - $tmp_pos); |
| |
| |
| if (!empty($utf_sort)) |
| { |
| if ($sort) |
| { |
| ksort($utf_sort); |
| } |
| |
| foreach ($utf_sort as $utf_chars) |
| { |
| $tmp .= implode('', $utf_chars); |
| } |
| } |
| |
| return $tmp; |
| } |
| else if ($tmp_pos) |
| { |
| |
| if ($tmp_pos == $len) |
| { |
| |
| return $tmp; |
| } |
| else |
| { |
| |
| return $tmp . substr($str, $tmp_pos); |
| } |
| } |
| |
| |
| return $str; |
| } |
| } |
| |
| ?> |