To: vim-dev@vim.org

Subject: Patch 7.2.330

Fcc: outbox

From: Bram Moolenaar <Bram@moolenaar.net>

Mime-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

------------

Patch 7.2.330

Problem:    Tables for Unicode case operators are outdated.

Solution:   Add a Vim script for generating the tables.  Include tables for

	    Unicode 5.2.

Files:	    runtime/tools/README.txt, runtime/tools/unicode.vim, src/mbyte.c

*** ../vim-7.2.329/runtime/tools/README.txt	2005-12-02 01:47:43.000000000 +0100

--- runtime/tools/README.txt	2010-01-12 19:43:13.000000000 +0100

***************

*** 32,35 ****

--- 32,37 ----

  xcmdsrv_client.c:  Example for a client program that communicates with a Vim

  		   server through the X-Windows interface.

+ unicode.vim	Vim script to generate tables for src/mbyte.c.

+ 

  [xxd (and tee for OS/2) can be found in the src directory]

*** ../vim-7.2.329/runtime/tools/unicode.vim	2010-01-12 19:47:33.000000000 +0100

--- runtime/tools/unicode.vim	2010-01-12 19:42:14.000000000 +0100

***************

*** 0 ****

--- 1,280 ----

+ " Script to extract tables from Unicode .txt files, to be used in src/mbyte.c.

+ " The format of the UnicodeData.txt file is explained here:

+ " http://www.unicode.org/Public/5.1.0/ucd/UCD.html

+ " For the other files see the header.

+ "

+ " Usage: Vim -S <this-file>

+ "

+ " Author: Bram Moolenaar

+ " Last Update: 2010 Jan 12

+ 

+ " Parse lines of UnicodeData.txt.  Creates a list of lists in s:dataprops.

+ func! ParseDataToProps()

+   let s:dataprops = []

+   let lnum = 1

+   while lnum <= line('$')

+     let l = split(getline(lnum), '\s*;\s*', 1)

+     if len(l) != 15

+       echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 15'

+       return

+     endif

+     call add(s:dataprops, l)

+     let lnum += 1

+   endwhile

+ endfunc

+ 

+ " Parse lines of CaseFolding.txt.  Creates a list of lists in s:foldprops.

+ func! ParseFoldProps()

+   let s:foldprops = []

+   let lnum = 1

+   while lnum <= line('$')

+     let line = getline(lnum)

+     if line !~ '^#' && line !~ '^\s*$'

+       let l = split(line, '\s*;\s*', 1)

+       if len(l) != 4

+ 	echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 4'

+ 	return

+       endif

+       call add(s:foldprops, l)

+     endif

+     let lnum += 1

+   endwhile

+ endfunc

+ 

+ " Parse lines of EastAsianWidth.txt.  Creates a list of lists in s:widthprops.

+ func! ParseWidthProps()

+   let s:widthprops = []

+   let lnum = 1

+   while lnum <= line('$')

+     let line = getline(lnum)

+     if line !~ '^#' && line !~ '^\s*$'

+       let l = split(line, '\s*;\s*', 1)

+       if len(l) != 2

+ 	echoerr 'Found ' . len(l) . ' items in line ' . lnum . ', expected 2'

+ 	return

+       endif

+       call add(s:widthprops, l)

+     endif

+     let lnum += 1

+   endwhile

+ endfunc

+ 

+ " Build the toLower or toUpper table in a new buffer.

+ " Uses s:dataprops.

+ func! BuildCaseTable(name, index)

+   let start = -1

+   let end = -1

+   let step = 0

+   let add = -1

+   let ranges = []

+   for p in s:dataprops

+     if p[a:index] != ''

+       let n = ('0x' . p[0]) + 0

+       let nl = ('0x' . p[a:index]) + 0

+       if start >= 0 && add == nl - n && (step == 0 || n - end == step)

+ 	" continue with same range.

+ 	let step = n - end

+ 	let end = n

+       else

+ 	if start >= 0

+ 	  " produce previous range

+ 	  call Range(ranges, start, end, step, add)

+ 	endif

+ 	let start = n

+ 	let end = n

+ 	let step = 0

+ 	let add = nl - n

+       endif

+     endif

+   endfor

+   if start >= 0

+     call Range(ranges, start, end, step, add)

+   endif

+ 

+   " New buffer to put the result in.

+   new

+   exe "file to" . a:name

+   call setline(1, "static convertStruct to" . a:name . "[] =")

+   call setline(2, "{")

+   call append('$', ranges)

+   call setline('$', getline('$')[:-2])  " remove last comma

+   call setline(line('$') + 1, "};")

+   wincmd p

+ endfunc

+ 

+ " Build the foldCase table in a new buffer.

+ " Uses s:foldprops.

+ func! BuildFoldTable()

+   let start = -1

+   let end = -1

+   let step = 0

+   let add = -1

+   let ranges = []

+   for p in s:foldprops

+     if p[1] == 'C' || p[1] == 'S'

+       let n = ('0x' . p[0]) + 0

+       let nl = ('0x' . p[2]) + 0

+       if start >= 0 && add == nl - n && (step == 0 || n - end == step)

+ 	" continue with same range.

+ 	let step = n - end

+ 	let end = n

+       else

+ 	if start >= 0

+ 	  " produce previous range

+ 	  call Range(ranges, start, end, step, add)

+ 	endif

+ 	let start = n

+ 	let end = n

+ 	let step = 0

+ 	let add = nl - n

+       endif

+     endif

+   endfor

+   if start >= 0

+     call Range(ranges, start, end, step, add)

+   endif

+ 

+   " New buffer to put the result in.

+   new

+   file foldCase

+   call setline(1, "static convertStruct foldCase[] =")

+   call setline(2, "{")

+   call append('$', ranges)

+   call setline('$', getline('$')[:-2])  " remove last comma

+   call setline(line('$') + 1, "};")

+   wincmd p

+ endfunc

+ 

+ func! Range(ranges, start, end, step, add)

+   let s = printf("\t{0x%x,0x%x,%d,%d},", a:start, a:end, a:step == 0 ? -1 : a:step, a:add)

+   call add(a:ranges, s)

+ endfunc

+ 

+ " Build the combining table.

+ " Uses s:dataprops.

+ func! BuildCombiningTable()

+   let start = -1

+   let end = -1

+   let ranges = []

+   for p in s:dataprops

+     if p[2] == 'Mn' || p[2] == 'Mc' || p[2] == 'Me'

+       let n = ('0x' . p[0]) + 0

+       if start >= 0 && end + 1 == n

+ 	" continue with same range.

+ 	let end = n

+       else

+ 	if start >= 0

+ 	  " produce previous range

+ 	  call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))

+ 	endif

+ 	let start = n

+ 	let end = n

+       endif

+     endif

+   endfor

+   if start >= 0

+     call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))

+   endif

+ 

+   " New buffer to put the result in.

+   new

+   file combining

+   call setline(1, "    static struct interval combining[] =")

+   call setline(2, "    {")

+   call append('$', ranges)

+   call setline('$', getline('$')[:-2])  " remove last comma

+   call setline(line('$') + 1, "    };")

+   wincmd p

+ endfunc

+ 

+ " Build the ambiguous table in a new buffer.

+ " Uses s:widthprops and s:dataprops.

+ func! BuildAmbiguousTable()

+   let start = -1

+   let end = -1

+   let ranges = []

+   let dataidx = 0

+   for p in s:widthprops

+     if p[1][0] == 'A'

+       let n = ('0x' . p[0]) + 0

+       " Find this char in the data table.

+       while 1

+ 	let dn = ('0x' . s:dataprops[dataidx][0]) + 0

+ 	if dn >= n

+ 	  break

+ 	endif

+ 	let dataidx += 1

+       endwhile

+       if dn != n

+ 	echoerr "Cannot find character " . n . " in data table"

+       endif

+       " Only use the char when it's not a composing char.

+       let dp = s:dataprops[dataidx]

+       if dp[2] != 'Mn' && dp[2] != 'Mc' && dp[2] != 'Me'

+ 	if start >= 0 && end + 1 == n

+ 	  " continue with same range.

+ 	  let end = n

+ 	else

+ 	  if start >= 0

+ 	    " produce previous range

+ 	    call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))

+ 	  endif

+ 	  let start = n

+ 	  if p[0] =~ '\.\.'

+ 	    let end = ('0x' . substitute(p[0], '.*\.\.', '', '')) + 0

+ 	  else

+ 	    let end = n

+ 	  endif

+ 	endif

+       endif

+     endif

+   endfor

+   if start >= 0

+     call add(ranges, printf("\t{0x%04x, 0x%04x},", start, end))

+   endif

+ 

+   " New buffer to put the result in.

+   new

+   file ambiguous

+   call setline(1, "    static struct interval ambiguous[] =")

+   call setline(2, "    {")

+   call append('$', ranges)

+   call setline('$', getline('$')[:-2])  " remove last comma

+   call setline(line('$') + 1, "    };")

+   wincmd p

+ endfunc

+ 

+ 

+ 

+ " Edit the Unicode text file.  Requires the netrw plugin.

+ edit http://unicode.org/Public/UNIDATA/UnicodeData.txt

+ 

+ " Parse each line, create a list of lists.

+ call ParseDataToProps()

+ 

+ " Build the toLower table.

+ call BuildCaseTable("Lower", 13)

+ 

+ " Build the toUpper table.

+ call BuildCaseTable("Upper", 12)

+ 

+ " Build the ranges of composing chars.

+ call BuildCombiningTable()

+ 

+ " Edit the case folding text file.  Requires the netrw plugin.

+ edit http://www.unicode.org/Public/UNIDATA/CaseFolding.txt

+ 

+ " Parse each line, create a list of lists.

+ call ParseFoldProps()

+ 

+ " Build the foldCase table.

+ call BuildFoldTable()

+ 

+ " Edit the width text file.  Requires the netrw plugin.

+ edit http://www.unicode.org/Public/UNIDATA/EastAsianWidth.txt

+ 

+ " Parse each line, create a list of lists.

+ call ParseWidthProps()

+ 

+ " Build the ambiguous table.

+ call BuildAmbiguousTable()

*** ../vim-7.2.329/src/mbyte.c	2009-12-02 15:03:24.000000000 +0100

--- src/mbyte.c	2010-01-12 19:35:49.000000000 +0100

***************

*** 26,32 ****

   *		    Recognizing bytes is easy: 0xxx.xxxx is a single-byte

   *		    char, 10xx.xxxx is a trailing byte, 11xx.xxxx is a leading

   *		    byte of a multi-byte character.

!  *		    To make things complicated, up to two composing characters

   *		    are allowed.  These are drawn on top of the first char.

   *		    For most editing the sequence of bytes with composing

   *		    characters included is considered to be one character.

--- 26,32 ----

   *		    Recognizing bytes is easy: 0xxx.xxxx is a single-byte

   *		    char, 10xx.xxxx is a trailing byte, 11xx.xxxx is a leading

   *		    byte of a multi-byte character.

!  *		    To make things complicated, up to six composing characters

   *		    are allowed.  These are drawn on top of the first char.

   *		    For most editing the sequence of bytes with composing

   *		    characters included is considered to be one character.

***************

*** 1153,1160 ****

  struct interval

  {

!     unsigned short first;

!     unsigned short last;

  };

  static int intable __ARGS((struct interval *table, size_t size, int c));

--- 1153,1160 ----

  struct interval

  {

!     long first;

!     long last;

  };

  static int intable __ARGS((struct interval *table, size_t size, int c));

***************

*** 1200,1261 ****

  utf_char2cells(c)

      int		c;

  {

!     /* sorted list of non-overlapping intervals of East Asian Ambiguous

!      * characters, generated with:

!      * "uniset +WIDTH-A -cat=Me -cat=Mn -cat=Cf c" */

!     static struct interval ambiguous[] = {

! 	{0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8},

! 	{0x00AA, 0x00AA}, {0x00AE, 0x00AE}, {0x00B0, 0x00B4},

! 	{0x00B6, 0x00BA}, {0x00BC, 0x00BF}, {0x00C6, 0x00C6},

! 	{0x00D0, 0x00D0}, {0x00D7, 0x00D8}, {0x00DE, 0x00E1},

! 	{0x00E6, 0x00E6}, {0x00E8, 0x00EA}, {0x00EC, 0x00ED},

! 	{0x00F0, 0x00F0}, {0x00F2, 0x00F3}, {0x00F7, 0x00FA},

! 	{0x00FC, 0x00FC}, {0x00FE, 0x00FE}, {0x0101, 0x0101},

! 	{0x0111, 0x0111}, {0x0113, 0x0113}, {0x011B, 0x011B},

! 	{0x0126, 0x0127}, {0x012B, 0x012B}, {0x0131, 0x0133},

! 	{0x0138, 0x0138}, {0x013F, 0x0142}, {0x0144, 0x0144},

! 	{0x0148, 0x014B}, {0x014D, 0x014D}, {0x0152, 0x0153},

! 	{0x0166, 0x0167}, {0x016B, 0x016B}, {0x01CE, 0x01CE},

! 	{0x01D0, 0x01D0}, {0x01D2, 0x01D2}, {0x01D4, 0x01D4},

! 	{0x01D6, 0x01D6}, {0x01D8, 0x01D8}, {0x01DA, 0x01DA},

! 	{0x01DC, 0x01DC}, {0x0251, 0x0251}, {0x0261, 0x0261},

! 	{0x02C4, 0x02C4}, {0x02C7, 0x02C7}, {0x02C9, 0x02CB},

! 	{0x02CD, 0x02CD}, {0x02D0, 0x02D0}, {0x02D8, 0x02DB},

! 	{0x02DD, 0x02DD}, {0x02DF, 0x02DF}, {0x0391, 0x03A1},

! 	{0x03A3, 0x03A9}, {0x03B1, 0x03C1}, {0x03C3, 0x03C9},

! 	{0x0401, 0x0401}, {0x0410, 0x044F}, {0x0451, 0x0451},

! 	{0x2010, 0x2010}, {0x2013, 0x2016}, {0x2018, 0x2019},

! 	{0x201C, 0x201D}, {0x2020, 0x2022}, {0x2024, 0x2027},

! 	{0x2030, 0x2030}, {0x2032, 0x2033}, {0x2035, 0x2035},

! 	{0x203B, 0x203B}, {0x203E, 0x203E}, {0x2074, 0x2074},

! 	{0x207F, 0x207F}, {0x2081, 0x2084}, {0x20AC, 0x20AC},

! 	{0x2103, 0x2103}, {0x2105, 0x2105}, {0x2109, 0x2109},

! 	{0x2113, 0x2113}, {0x2116, 0x2116}, {0x2121, 0x2122},

! 	{0x2126, 0x2126}, {0x212B, 0x212B}, {0x2153, 0x2154},

! 	{0x215B, 0x215E}, {0x2160, 0x216B}, {0x2170, 0x2179},

! 	{0x2190, 0x2199}, {0x21B8, 0x21B9}, {0x21D2, 0x21D2},

! 	{0x21D4, 0x21D4}, {0x21E7, 0x21E7}, {0x2200, 0x2200},

! 	{0x2202, 0x2203}, {0x2207, 0x2208}, {0x220B, 0x220B},

! 	{0x220F, 0x220F}, {0x2211, 0x2211}, {0x2215, 0x2215},

! 	{0x221A, 0x221A}, {0x221D, 0x2220}, {0x2223, 0x2223},

! 	{0x2225, 0x2225}, {0x2227, 0x222C}, {0x222E, 0x222E},

! 	{0x2234, 0x2237}, {0x223C, 0x223D}, {0x2248, 0x2248},

! 	{0x224C, 0x224C}, {0x2252, 0x2252}, {0x2260, 0x2261},

! 	{0x2264, 0x2267}, {0x226A, 0x226B}, {0x226E, 0x226F},

! 	{0x2282, 0x2283}, {0x2286, 0x2287}, {0x2295, 0x2295},

! 	{0x2299, 0x2299}, {0x22A5, 0x22A5}, {0x22BF, 0x22BF},

! 	{0x2312, 0x2312}, {0x2460, 0x24E9}, {0x24EB, 0x254B},

! 	{0x2550, 0x2573}, {0x2580, 0x258F}, {0x2592, 0x2595},

! 	{0x25A0, 0x25A1}, {0x25A3, 0x25A9}, {0x25B2, 0x25B3},

! 	{0x25B6, 0x25B7}, {0x25BC, 0x25BD}, {0x25C0, 0x25C1},

! 	{0x25C6, 0x25C8}, {0x25CB, 0x25CB}, {0x25CE, 0x25D1},

! 	{0x25E2, 0x25E5}, {0x25EF, 0x25EF}, {0x2605, 0x2606},

! 	{0x2609, 0x2609}, {0x260E, 0x260F}, {0x2614, 0x2615},

! 	{0x261C, 0x261C}, {0x261E, 0x261E}, {0x2640, 0x2640},

! 	{0x2642, 0x2642}, {0x2660, 0x2661}, {0x2663, 0x2665},

! 	{0x2667, 0x266A}, {0x266C, 0x266D}, {0x266F, 0x266F},

! 	{0x273D, 0x273D}, {0x2776, 0x277F}, {0xE000, 0xF8FF},

! 	{0xFFFD, 0xFFFD}, /* {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD} */

      };

      if (c >= 0x100)

--- 1200,1390 ----

  utf_char2cells(c)

      int		c;

  {

!     /* Sorted list of non-overlapping intervals of East Asian Ambiguous

!      * characters, generated with ../runtime/tools/unicode.vim. */

!     static struct interval ambiguous[] =

!     {

! 	{0x00a1, 0x00a1},

! 	{0x00a4, 0x00a4},

! 	{0x00a7, 0x00a8},

! 	{0x00aa, 0x00aa},

! 	{0x00ad, 0x00ae},

! 	{0x00b0, 0x00b4},

! 	{0x00b6, 0x00ba},

! 	{0x00bc, 0x00bf},

! 	{0x00c6, 0x00c6},

! 	{0x00d0, 0x00d0},

! 	{0x00d7, 0x00d8},

! 	{0x00de, 0x00e1},

! 	{0x00e6, 0x00e6},

! 	{0x00e8, 0x00ea},

! 	{0x00ec, 0x00ed},

! 	{0x00f0, 0x00f0},

! 	{0x00f2, 0x00f3},

! 	{0x00f7, 0x00fa},

! 	{0x00fc, 0x00fc},

! 	{0x00fe, 0x00fe},

! 	{0x0101, 0x0101},

! 	{0x0111, 0x0111},

! 	{0x0113, 0x0113},

! 	{0x011b, 0x011b},

! 	{0x0126, 0x0127},

! 	{0x012b, 0x012b},

! 	{0x0131, 0x0133},

! 	{0x0138, 0x0138},

! 	{0x013f, 0x0142},

! 	{0x0144, 0x0144},

! 	{0x0148, 0x014b},

! 	{0x014d, 0x014d},

! 	{0x0152, 0x0153},

! 	{0x0166, 0x0167},

! 	{0x016b, 0x016b},

! 	{0x01ce, 0x01ce},

! 	{0x01d0, 0x01d0},

! 	{0x01d2, 0x01d2},

! 	{0x01d4, 0x01d4},

! 	{0x01d6, 0x01d6},

! 	{0x01d8, 0x01d8},

! 	{0x01da, 0x01da},

! 	{0x01dc, 0x01dc},

! 	{0x0251, 0x0251},

! 	{0x0261, 0x0261},

! 	{0x02c4, 0x02c4},

! 	{0x02c7, 0x02c7},

! 	{0x02c9, 0x02cb},

! 	{0x02cd, 0x02cd},

! 	{0x02d0, 0x02d0},

! 	{0x02d8, 0x02db},

! 	{0x02dd, 0x02dd},

! 	{0x02df, 0x02df},

! 	{0x0391, 0x03a1},

! 	{0x03a3, 0x03a9},

! 	{0x03b1, 0x03c1},

! 	{0x03c3, 0x03c9},

! 	{0x0401, 0x0401},

! 	{0x0410, 0x044f},

! 	{0x0451, 0x0451},

! 	{0x2010, 0x2010},

! 	{0x2013, 0x2016},

! 	{0x2018, 0x2019},

! 	{0x201c, 0x201d},

! 	{0x2020, 0x2022},

! 	{0x2024, 0x2027},

! 	{0x2030, 0x2030},

! 	{0x2032, 0x2033},

! 	{0x2035, 0x2035},

! 	{0x203b, 0x203b},

! 	{0x203e, 0x203e},

! 	{0x2074, 0x2074},

! 	{0x207f, 0x207f},

! 	{0x2081, 0x2084},

! 	{0x20ac, 0x20ac},

! 	{0x2103, 0x2103},

! 	{0x2105, 0x2105},

! 	{0x2109, 0x2109},

! 	{0x2113, 0x2113},

! 	{0x2116, 0x2116},

! 	{0x2121, 0x2122},

! 	{0x2126, 0x2126},

! 	{0x212b, 0x212b},

! 	{0x2153, 0x2154},

! 	{0x215b, 0x215e},

! 	{0x2160, 0x216b},

! 	{0x2170, 0x2179},

! 	{0x2189, 0x2189},

! 	{0x2190, 0x2199},

! 	{0x21b8, 0x21b9},

! 	{0x21d2, 0x21d2},

! 	{0x21d4, 0x21d4},

! 	{0x21e7, 0x21e7},

! 	{0x2200, 0x2200},

! 	{0x2202, 0x2203},

! 	{0x2207, 0x2208},

! 	{0x220b, 0x220b},

! 	{0x220f, 0x220f},

! 	{0x2211, 0x2211},

! 	{0x2215, 0x2215},

! 	{0x221a, 0x221a},

! 	{0x221d, 0x2220},

! 	{0x2223, 0x2223},

! 	{0x2225, 0x2225},

! 	{0x2227, 0x222c},

! 	{0x222e, 0x222e},

! 	{0x2234, 0x2237},

! 	{0x223c, 0x223d},

! 	{0x2248, 0x2248},

! 	{0x224c, 0x224c},

! 	{0x2252, 0x2252},

! 	{0x2260, 0x2261},

! 	{0x2264, 0x2267},

! 	{0x226a, 0x226b},

! 	{0x226e, 0x226f},

! 	{0x2282, 0x2283},

! 	{0x2286, 0x2287},

! 	{0x2295, 0x2295},

! 	{0x2299, 0x2299},

! 	{0x22a5, 0x22a5},

! 	{0x22bf, 0x22bf},

! 	{0x2312, 0x2312},

! 	{0x2460, 0x24e9},

! 	{0x24eb, 0x254b},

! 	{0x2550, 0x2573},

! 	{0x2580, 0x258f},

! 	{0x2592, 0x2595},

! 	{0x25a0, 0x25a1},

! 	{0x25a3, 0x25a9},

! 	{0x25b2, 0x25b3},

! 	{0x25b6, 0x25b7},

! 	{0x25bc, 0x25bd},

! 	{0x25c0, 0x25c1},

! 	{0x25c6, 0x25c8},

! 	{0x25cb, 0x25cb},

! 	{0x25ce, 0x25d1},

! 	{0x25e2, 0x25e5},

! 	{0x25ef, 0x25ef},

! 	{0x2605, 0x2606},

! 	{0x2609, 0x2609},

! 	{0x260e, 0x260f},

! 	{0x2614, 0x2615},

! 	{0x261c, 0x261c},

! 	{0x261e, 0x261e},

! 	{0x2640, 0x2640},

! 	{0x2642, 0x2642},

! 	{0x2660, 0x2661},

! 	{0x2663, 0x2665},

! 	{0x2667, 0x266a},

! 	{0x266c, 0x266d},

! 	{0x266f, 0x266f},

! 	{0x269e, 0x269f},

! 	{0x26be, 0x26bf},

! 	{0x26c4, 0x26cd},

! 	{0x26cf, 0x26e1},

! 	{0x26e3, 0x26e3},

! 	{0x26e8, 0x26ff},

! 	{0x273d, 0x273d},

! 	{0x2757, 0x2757},

! 	{0x2776, 0x277f},

! 	{0x2b55, 0x2b59},

! 	{0x3248, 0x324f},

! 	{0xe000, 0xf8ff},

! 	{0xfffd, 0xfffd},

! 	{0x1f100, 0x1f10a},

! 	{0x1f110, 0x1f12d},

! 	{0x1f131, 0x1f131},

! 	{0x1f13d, 0x1f13d},

! 	{0x1f13f, 0x1f13f},

! 	{0x1f142, 0x1f142},

! 	{0x1f146, 0x1f146},

! 	{0x1f14a, 0x1f14e},

! 	{0x1f157, 0x1f157},

! 	{0x1f15f, 0x1f15f},

! 	{0x1f179, 0x1f179},

! 	{0x1f17b, 0x1f17c},

! 	{0x1f17f, 0x1f17f},

! 	{0x1f18a, 0x1f18d},

! 	{0x1f190, 0x1f190},

! 	{0xf0000, 0xffffd},

! 	{0x100000, 0x10fffd}

      };

      if (c >= 0x100)

***************

*** 1807,1813 ****

  	return 1;

      /*

!      * Check for composing characters.  We can handle only the first two, but

       * skip all of them (otherwise the cursor would get stuck).

       */

  #ifdef FEAT_ARABIC

--- 1936,1942 ----

  	return 1;

      /*

!      * Check for composing characters.  We can handle only the first six, but

       * skip all of them (otherwise the cursor would get stuck).

       */

  #ifdef FEAT_ARABIC

***************

*** 1855,1861 ****

  	return 1;

      /*

!      * Check for composing characters.  We can handle only the first two, but

       * skip all of them (otherwise the cursor would get stuck).

       */

  #ifdef FEAT_ARABIC

--- 1984,1990 ----

  	return 1;

      /*

!      * Check for composing characters.  We can handle only the first six, but

       * skip all of them (otherwise the cursor would get stuck).

       */

  #ifdef FEAT_ARABIC

***************

*** 1973,2010 ****

  utf_iscomposing(c)

      int		c;

  {

!     /* sorted list of non-overlapping intervals */

      static struct interval combining[] =

      {

! 	{0x0300, 0x034f}, {0x0360, 0x036f}, {0x0483, 0x0486}, {0x0488, 0x0489},

! 	{0x0591, 0x05a1}, {0x05a3, 0x05b9}, {0x05bb, 0x05bd}, {0x05bf, 0x05bf},

! 	{0x05c1, 0x05c2}, {0x05c4, 0x05c4}, {0x0610, 0x0615}, {0x064b, 0x0658},

! 	{0x0670, 0x0670}, {0x06d6, 0x06dc}, {0x06de, 0x06e4}, {0x06e7, 0x06e8},

! 	{0x06ea, 0x06ed}, {0x0711, 0x0711}, {0x0730, 0x074a}, {0x07a6, 0x07b0},

! 	{0x0901, 0x0903}, {0x093c, 0x093c}, {0x093e, 0x094d}, {0x0951, 0x0954},

! 	{0x0962, 0x0963}, {0x0981, 0x0983}, {0x09bc, 0x09bc}, {0x09be, 0x09c4},

! 	{0x09c7, 0x09c8}, {0x09cb, 0x09cd}, {0x09d7, 0x09d7}, {0x09e2, 0x09e3},

! 	{0x0a01, 0x0a03}, {0x0a3c, 0x0a3c}, {0x0a3e, 0x0a42}, {0x0a47, 0x0a48},

! 	{0x0a4b, 0x0a4d}, {0x0a70, 0x0a71}, {0x0a81, 0x0a83}, {0x0abc, 0x0abc},

! 	{0x0abe, 0x0ac5}, {0x0ac7, 0x0ac9}, {0x0acb, 0x0acd}, {0x0ae2, 0x0ae3},

! 	{0x0b01, 0x0b03}, {0x0b3c, 0x0b3c}, {0x0b3e, 0x0b43}, {0x0b47, 0x0b48},

! 	{0x0b4b, 0x0b4d}, {0x0b56, 0x0b57}, {0x0b82, 0x0b82}, {0x0bbe, 0x0bc2},

! 	{0x0bc6, 0x0bc8}, {0x0bca, 0x0bcd}, {0x0bd7, 0x0bd7}, {0x0c01, 0x0c03},

! 	{0x0c3e, 0x0c44}, {0x0c46, 0x0c48}, {0x0c4a, 0x0c4d}, {0x0c55, 0x0c56},

! 	{0x0c82, 0x0c83}, {0x0cbc, 0x0cbc}, {0x0cbe, 0x0cc4}, {0x0cc6, 0x0cc8},

! 	{0x0cca, 0x0ccd}, {0x0cd5, 0x0cd6}, {0x0d02, 0x0d03}, {0x0d3e, 0x0d43},

! 	{0x0d46, 0x0d48}, {0x0d4a, 0x0d4d}, {0x0d57, 0x0d57}, {0x0d82, 0x0d83},

! 	{0x0dca, 0x0dca}, {0x0dcf, 0x0dd4}, {0x0dd6, 0x0dd6}, {0x0dd8, 0x0ddf},

! 	{0x0df2, 0x0df3}, {0x0e31, 0x0e31}, {0x0e34, 0x0e3a}, {0x0e47, 0x0e4e},

! 	{0x0eb1, 0x0eb1}, {0x0eb4, 0x0eb9}, {0x0ebb, 0x0ebc}, {0x0ec8, 0x0ecd},

! 	{0x0f18, 0x0f19}, {0x0f35, 0x0f35}, {0x0f37, 0x0f37}, {0x0f39, 0x0f39},

! 	{0x0f3e, 0x0f3f}, {0x0f71, 0x0f84}, {0x0f86, 0x0f87}, {0x0f90, 0x0f97},

! 	{0x0f99, 0x0fbc}, {0x0fc6, 0x0fc6}, {0x102c, 0x1032}, {0x1036, 0x1039},

! 	{0x1056, 0x1059}, {0x1712, 0x1714}, {0x1732, 0x1734}, {0x1752, 0x1753},

! 	{0x1772, 0x1773}, {0x17b6, 0x17d3}, {0x17dd, 0x17dd}, {0x180b, 0x180d},

! 	{0x18a9, 0x18a9}, {0x1920, 0x192b}, {0x1930, 0x193b}, {0x20d0, 0x20ea},

! 	{0x302a, 0x302f}, {0x3099, 0x309a}, {0xfb1e, 0xfb1e}, {0xfe00, 0xfe0f},

! 	{0xfe20, 0xfe23},

      };

      return intable(combining, sizeof(combining), c);

--- 2102,2299 ----

  utf_iscomposing(c)

      int		c;

  {

!     /* Sorted list of non-overlapping intervals.

!      * Generated by ../runtime/tools/unicode.vim. */

      static struct interval combining[] =

      {

! 	{0x0300, 0x036f},

! 	{0x0483, 0x0489},

! 	{0x0591, 0x05bd},

! 	{0x05bf, 0x05bf},

! 	{0x05c1, 0x05c2},

! 	{0x05c4, 0x05c5},

! 	{0x05c7, 0x05c7},

! 	{0x0610, 0x061a},

! 	{0x064b, 0x065e},

! 	{0x0670, 0x0670},

! 	{0x06d6, 0x06dc},

! 	{0x06de, 0x06e4},

! 	{0x06e7, 0x06e8},

! 	{0x06ea, 0x06ed},

! 	{0x0711, 0x0711},

! 	{0x0730, 0x074a},

! 	{0x07a6, 0x07b0},

! 	{0x07eb, 0x07f3},

! 	{0x0816, 0x0819},

! 	{0x081b, 0x0823},

! 	{0x0825, 0x0827},

! 	{0x0829, 0x082d},

! 	{0x0900, 0x0903},

! 	{0x093c, 0x093c},

! 	{0x093e, 0x094e},

! 	{0x0951, 0x0955},

! 	{0x0962, 0x0963},

! 	{0x0981, 0x0983},

! 	{0x09bc, 0x09bc},

! 	{0x09be, 0x09c4},

! 	{0x09c7, 0x09c8},

! 	{0x09cb, 0x09cd},

! 	{0x09d7, 0x09d7},

! 	{0x09e2, 0x09e3},

! 	{0x0a01, 0x0a03},

! 	{0x0a3c, 0x0a3c},

! 	{0x0a3e, 0x0a42},

! 	{0x0a47, 0x0a48},

! 	{0x0a4b, 0x0a4d},

! 	{0x0a51, 0x0a51},

! 	{0x0a70, 0x0a71},

! 	{0x0a75, 0x0a75},

! 	{0x0a81, 0x0a83},

! 	{0x0abc, 0x0abc},

! 	{0x0abe, 0x0ac5},

! 	{0x0ac7, 0x0ac9},

! 	{0x0acb, 0x0acd},

! 	{0x0ae2, 0x0ae3},

! 	{0x0b01, 0x0b03},

! 	{0x0b3c, 0x0b3c},

! 	{0x0b3e, 0x0b44},

! 	{0x0b47, 0x0b48},

! 	{0x0b4b, 0x0b4d},

! 	{0x0b56, 0x0b57},

! 	{0x0b62, 0x0b63},

! 	{0x0b82, 0x0b82},

! 	{0x0bbe, 0x0bc2},

! 	{0x0bc6, 0x0bc8},

! 	{0x0bca, 0x0bcd},

! 	{0x0bd7, 0x0bd7},

! 	{0x0c01, 0x0c03},

! 	{0x0c3e, 0x0c44},

! 	{0x0c46, 0x0c48},

! 	{0x0c4a, 0x0c4d},

! 	{0x0c55, 0x0c56},

! 	{0x0c62, 0x0c63},

! 	{0x0c82, 0x0c83},

! 	{0x0cbc, 0x0cbc},

! 	{0x0cbe, 0x0cc4},

! 	{0x0cc6, 0x0cc8},

! 	{0x0cca, 0x0ccd},

! 	{0x0cd5, 0x0cd6},

! 	{0x0ce2, 0x0ce3},

! 	{0x0d02, 0x0d03},

! 	{0x0d3e, 0x0d44},

! 	{0x0d46, 0x0d48},

! 	{0x0d4a, 0x0d4d},

! 	{0x0d57, 0x0d57},

! 	{0x0d62, 0x0d63},

! 	{0x0d82, 0x0d83},

! 	{0x0dca, 0x0dca},

! 	{0x0dcf, 0x0dd4},

! 	{0x0dd6, 0x0dd6},

! 	{0x0dd8, 0x0ddf},

! 	{0x0df2, 0x0df3},

! 	{0x0e31, 0x0e31},

! 	{0x0e34, 0x0e3a},

! 	{0x0e47, 0x0e4e},

! 	{0x0eb1, 0x0eb1},

! 	{0x0eb4, 0x0eb9},

! 	{0x0ebb, 0x0ebc},

! 	{0x0ec8, 0x0ecd},

! 	{0x0f18, 0x0f19},

! 	{0x0f35, 0x0f35},

! 	{0x0f37, 0x0f37},

! 	{0x0f39, 0x0f39},

! 	{0x0f3e, 0x0f3f},

! 	{0x0f71, 0x0f84},

! 	{0x0f86, 0x0f87},

! 	{0x0f90, 0x0f97},

! 	{0x0f99, 0x0fbc},

! 	{0x0fc6, 0x0fc6},

! 	{0x102b, 0x103e},

! 	{0x1056, 0x1059},

! 	{0x105e, 0x1060},

! 	{0x1062, 0x1064},

! 	{0x1067, 0x106d},

! 	{0x1071, 0x1074},

! 	{0x1082, 0x108d},

! 	{0x108f, 0x108f},

! 	{0x109a, 0x109d},

! 	{0x135f, 0x135f},

! 	{0x1712, 0x1714},

! 	{0x1732, 0x1734},

! 	{0x1752, 0x1753},

! 	{0x1772, 0x1773},

! 	{0x17b6, 0x17d3},

! 	{0x17dd, 0x17dd},

! 	{0x180b, 0x180d},

! 	{0x18a9, 0x18a9},

! 	{0x1920, 0x192b},

! 	{0x1930, 0x193b},

! 	{0x19b0, 0x19c0},

! 	{0x19c8, 0x19c9},

! 	{0x1a17, 0x1a1b},

! 	{0x1a55, 0x1a5e},

! 	{0x1a60, 0x1a7c},

! 	{0x1a7f, 0x1a7f},

! 	{0x1b00, 0x1b04},

! 	{0x1b34, 0x1b44},

! 	{0x1b6b, 0x1b73},

! 	{0x1b80, 0x1b82},

! 	{0x1ba1, 0x1baa},

! 	{0x1c24, 0x1c37},

! 	{0x1cd0, 0x1cd2},

! 	{0x1cd4, 0x1ce8},

! 	{0x1ced, 0x1ced},

! 	{0x1cf2, 0x1cf2},

! 	{0x1dc0, 0x1de6},

! 	{0x1dfd, 0x1dff},

! 	{0x20d0, 0x20f0},

! 	{0x2cef, 0x2cf1},

! 	{0x2de0, 0x2dff},

! 	{0x302a, 0x302f},

! 	{0x3099, 0x309a},

! 	{0xa66f, 0xa672},

! 	{0xa67c, 0xa67d},

! 	{0xa6f0, 0xa6f1},

! 	{0xa802, 0xa802},

! 	{0xa806, 0xa806},

! 	{0xa80b, 0xa80b},

! 	{0xa823, 0xa827},

! 	{0xa880, 0xa881},

! 	{0xa8b4, 0xa8c4},

! 	{0xa8e0, 0xa8f1},

! 	{0xa926, 0xa92d},

! 	{0xa947, 0xa953},

! 	{0xa980, 0xa983},

! 	{0xa9b3, 0xa9c0},

! 	{0xaa29, 0xaa36},

! 	{0xaa43, 0xaa43},

! 	{0xaa4c, 0xaa4d},

! 	{0xaa7b, 0xaa7b},

! 	{0xaab0, 0xaab0},

! 	{0xaab2, 0xaab4},

! 	{0xaab7, 0xaab8},

! 	{0xaabe, 0xaabf},

! 	{0xaac1, 0xaac1},

! 	{0xabe3, 0xabea},

! 	{0xabec, 0xabed},

! 	{0xfb1e, 0xfb1e},

! 	{0xfe00, 0xfe0f},

! 	{0xfe20, 0xfe26},

! 	{0x101fd, 0x101fd},

! 	{0x10a01, 0x10a03},

! 	{0x10a05, 0x10a06},

! 	{0x10a0c, 0x10a0f},

! 	{0x10a38, 0x10a3a},

! 	{0x10a3f, 0x10a3f},

! 	{0x11080, 0x11082},

! 	{0x110b0, 0x110ba},

! 	{0x1d165, 0x1d169},

! 	{0x1d16d, 0x1d172},

! 	{0x1d17b, 0x1d182},

! 	{0x1d185, 0x1d18b},

! 	{0x1d1aa, 0x1d1ad},

! 	{0x1d242, 0x1d244},

! 	{0xe0100, 0xe01ef}

      };

      return intable(combining, sizeof(combining), c);