To: vim_dev@googlegroups.com
Subject: Patch 7.4.852
Fcc: outbox
From: Bram Moolenaar <Bram@moolenaar.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
------------
 
Patch 7.4.852
Problem:    On MS-Windows console Vim uses ANSI APIs for keyboard input and
            console output, it cannot input/output Unicode characters.
Solution:   Use Unicode APIs for console I/O. (Ken Takata, Yasuhiro Matsumoto)
Files:      src/os_win32.c, src/ui.c, runtime/doc/options.txt
 
 
*** ../vim-7.4.851/src/os_win32.c	2015-09-01 20:23:30.408603580 +0200
--- src/os_win32.c	2015-09-01 20:28:43.193363546 +0200
***************
*** 213,220 ****
  static void standend(void);
  static void visual_bell(void);
  static void cursor_visible(BOOL fVisible);
! static BOOL write_chars(LPCSTR pchBuf, DWORD cchToWrite);
! static char_u tgetch(int *pmodifiers, char_u *pch2);
  static void create_conin(void);
  static int s_cursor_visible = TRUE;
  static int did_create_conin = FALSE;
--- 213,220 ----
  static void standend(void);
  static void visual_bell(void);
  static void cursor_visible(BOOL fVisible);
! static DWORD write_chars(char_u *pchBuf, DWORD cbToWrite);
! static WCHAR tgetch(int *pmodifiers, WCHAR *pch2);
  static void create_conin(void);
  static int s_cursor_visible = TRUE;
  static int did_create_conin = FALSE;
***************
*** 265,279 ****
      if (!win8_or_later)
      {
  	if (nLength == -1)
! 	    return PeekConsoleInput(hInput, lpBuffer, 1, lpEvents);
! 	return ReadConsoleInput(hInput, lpBuffer, 1, &dwEvents);
      }
  
      if (s_dwMax == 0)
      {
  	if (nLength == -1)
! 	    return PeekConsoleInput(hInput, lpBuffer, 1, lpEvents);
! 	if (!ReadConsoleInput(hInput, s_irCache, IRSIZE, &dwEvents))
  	    return FALSE;
  	s_dwIndex = 0;
  	s_dwMax = dwEvents;
--- 265,279 ----
      if (!win8_or_later)
      {
  	if (nLength == -1)
! 	    return PeekConsoleInputW(hInput, lpBuffer, 1, lpEvents);
! 	return ReadConsoleInputW(hInput, lpBuffer, 1, &dwEvents);
      }
  
      if (s_dwMax == 0)
      {
  	if (nLength == -1)
! 	    return PeekConsoleInputW(hInput, lpBuffer, 1, lpEvents);
! 	if (!ReadConsoleInputW(hInput, s_irCache, IRSIZE, &dwEvents))
  	    return FALSE;
  	s_dwIndex = 0;
  	s_dwMax = dwEvents;
***************
*** 868,876 ****
  #endif
  
  #if defined(__GNUC__) && !defined(__MINGW32__)  && !defined(__CYGWIN__)
! # define AChar AsciiChar
  #else
! # define AChar uChar.AsciiChar
  #endif
  
  /* The return code indicates key code size. */
--- 868,876 ----
  #endif
  
  #if defined(__GNUC__) && !defined(__MINGW32__)  && !defined(__CYGWIN__)
! # define UChar UnicodeChar
  #else
! # define UChar uChar.UnicodeChar
  #endif
  
  /* The return code indicates key code size. */
***************
*** 889,900 ****
  
      if (s_iIsDead == 2)
      {
! 	pker->AChar = (CHAR) awAnsiCode[1];
  	s_iIsDead = 0;
  	return 1;
      }
  
!     if (pker->AChar != 0)
  	return 1;
  
      vim_memset(abKeystate, 0, sizeof (abKeystate));
--- 889,900 ----
  
      if (s_iIsDead == 2)
      {
! 	pker->UChar = (WCHAR) awAnsiCode[1];
  	s_iIsDead = 0;
  	return 1;
      }
  
!     if (pker->UChar != 0)
  	return 1;
  
      vim_memset(abKeystate, 0, sizeof (abKeystate));
***************
*** 909,915 ****
      }
  
      /* Clear any pending dead keys */
!     ToAscii(VK_SPACE, MapVirtualKey(VK_SPACE, 0), abKeystate, awAnsiCode, 0);
  
      if (uMods & SHIFT_PRESSED)
  	abKeystate[VK_SHIFT] = 0x80;
--- 909,915 ----
      }
  
      /* Clear any pending dead keys */
!     ToUnicode(VK_SPACE, MapVirtualKey(VK_SPACE, 0), abKeystate, awAnsiCode, 2, 0);
  
      if (uMods & SHIFT_PRESSED)
  	abKeystate[VK_SHIFT] = 0x80;
***************
*** 922,932 ****
  	    abKeystate[VK_MENU] = abKeystate[VK_RMENU] = 0x80;
      }
  
!     s_iIsDead = ToAscii(pker->wVirtualKeyCode, pker->wVirtualScanCode,
! 			abKeystate, awAnsiCode, 0);
  
      if (s_iIsDead > 0)
! 	pker->AChar = (CHAR) awAnsiCode[0];
  
      return s_iIsDead;
  }
--- 922,932 ----
  	    abKeystate[VK_MENU] = abKeystate[VK_RMENU] = 0x80;
      }
  
!     s_iIsDead = ToUnicode(pker->wVirtualKeyCode, pker->wVirtualScanCode,
! 			abKeystate, awAnsiCode, 2, 0);
  
      if (s_iIsDead > 0)
! 	pker->UChar = (WCHAR) awAnsiCode[0];
  
      return s_iIsDead;
  }
***************
*** 953,960 ****
      static BOOL
  decode_key_event(
      KEY_EVENT_RECORD	*pker,
!     char_u		*pch,
!     char_u		*pch2,
      int			*pmodifiers,
      BOOL		fDoPost)
  {
--- 953,960 ----
      static BOOL
  decode_key_event(
      KEY_EVENT_RECORD	*pker,
!     WCHAR		*pch,
!     WCHAR		*pch2,
      int			*pmodifiers,
      BOOL		fDoPost)
  {
***************
*** 982,988 ****
      }
  
      /* special cases */
!     if ((nModifs & CTRL) != 0 && (nModifs & ~CTRL) == 0 && pker->AChar == NUL)
      {
  	/* Ctrl-6 is Ctrl-^ */
  	if (pker->wVirtualKeyCode == '6')
--- 982,988 ----
      }
  
      /* special cases */
!     if ((nModifs & CTRL) != 0 && (nModifs & ~CTRL) == 0 && pker->UChar == NUL)
      {
  	/* Ctrl-6 is Ctrl-^ */
  	if (pker->wVirtualKeyCode == '6')
***************
*** 1044,1050 ****
  	*pch = NUL;
      else
      {
! 	*pch = (i > 0) ? pker->AChar : NUL;
  
  	if (pmodifiers != NULL)
  	{
--- 1044,1050 ----
  	*pch = NUL;
      else
      {
! 	*pch = (i > 0) ? pker->UChar : NUL;
  
  	if (pmodifiers != NULL)
  	{
***************
*** 1436,1442 ****
      DWORD	    dwNow = 0, dwEndTime = 0;
      INPUT_RECORD    ir;
      DWORD	    cRecords;
!     char_u	    ch, ch2;
  
      if (msec > 0)
  	/* Wait until the specified time has elapsed. */
--- 1436,1442 ----
      DWORD	    dwNow = 0, dwEndTime = 0;
      INPUT_RECORD    ir;
      DWORD	    cRecords;
!     WCHAR	    ch, ch2;
  
      if (msec > 0)
  	/* Wait until the specified time has elapsed. */
***************
*** 1523,1529 ****
  #ifdef FEAT_MBYTE_IME
  		/* Windows IME sends two '\n's with only one 'ENTER'.  First:
  		 * wVirtualKeyCode == 13. second: wVirtualKeyCode == 0 */
! 		if (ir.Event.KeyEvent.uChar.UnicodeChar == 0
  			&& ir.Event.KeyEvent.wVirtualKeyCode == 13)
  		{
  		    read_console_input(g_hConIn, &ir, 1, &cRecords);
--- 1523,1529 ----
  #ifdef FEAT_MBYTE_IME
  		/* Windows IME sends two '\n's with only one 'ENTER'.  First:
  		 * wVirtualKeyCode == 13. second: wVirtualKeyCode == 0 */
! 		if (ir.Event.KeyEvent.UChar == 0
  			&& ir.Event.KeyEvent.wVirtualKeyCode == 13)
  		{
  		    read_console_input(g_hConIn, &ir, 1, &cRecords);
***************
*** 1586,1595 ****
  /*
   * Get a keystroke or a mouse event
   */
!     static char_u
! tgetch(int *pmodifiers, char_u *pch2)
  {
!     char_u ch;
  
      for (;;)
      {
--- 1586,1595 ----
  /*
   * Get a keystroke or a mouse event
   */
!     static WCHAR
! tgetch(int *pmodifiers, WCHAR *pch2)
  {
!     WCHAR ch;
  
      for (;;)
      {
***************
*** 1658,1668 ****
  #define TYPEAHEADLEN 20
      static char_u   typeahead[TYPEAHEADLEN];	/* previously typed bytes. */
      static int	    typeaheadlen = 0;
- #ifdef FEAT_MBYTE
-     static char_u   *rest = NULL;	/* unconverted rest of previous read */
-     static int	    restlen = 0;
-     int		    unconverted;
- #endif
  
      /* First use any typeahead that was kept because "buf" was too small. */
      if (typeaheadlen > 0)
--- 1658,1663 ----
***************
*** 1761,1798 ****
  	else
  #endif
  	{
! 	    char_u	ch2 = NUL;
  	    int		modifiers = 0;
  
  	    c = tgetch(&modifiers, &ch2);
  
- #ifdef FEAT_MBYTE
- 	    /* stolen from fill_input_buf() in ui.c */
- 	    if (rest != NULL)
- 	    {
- 		/* Use remainder of previous call, starts with an invalid
- 		 * character that may become valid when reading more. */
- 		if (restlen > TYPEAHEADLEN - typeaheadlen)
- 		    unconverted = TYPEAHEADLEN - typeaheadlen;
- 		else
- 		    unconverted = restlen;
- 		mch_memmove(typeahead + typeaheadlen, rest, unconverted);
- 		if (unconverted == restlen)
- 		{
- 		    vim_free(rest);
- 		    rest = NULL;
- 		}
- 		else
- 		{
- 		    restlen -= unconverted;
- 		    mch_memmove(rest, rest + unconverted, restlen);
- 		}
- 		typeaheadlen += unconverted;
- 	    }
- 	    else
- 		unconverted = 0;
- #endif
- 
  	    if (typebuf_changed(tb_change_cnt))
  	    {
  		/* "buf" may be invalid now if a client put something in the
--- 1756,1766 ----
  	else
  #endif
  	{
! 	    WCHAR	ch2 = NUL;
  	    int		modifiers = 0;
  
  	    c = tgetch(&modifiers, &ch2);
  
  	    if (typebuf_changed(tb_change_cnt))
  	    {
  		/* "buf" may be invalid now if a client put something in the
***************
*** 1816,1842 ****
  		int	n = 1;
  		int     conv = FALSE;
  
- 		typeahead[typeaheadlen] = c;
- 		if (ch2 != NUL)
- 		{
- 		    typeahead[typeaheadlen + 1] = 3;
- 		    typeahead[typeaheadlen + 2] = ch2;
- 		    n += 2;
- 		}
  #ifdef FEAT_MBYTE
! 		/* Only convert normal characters, not special keys.  Need to
! 		 * convert before applying ALT, otherwise mapping <M-x> breaks
! 		 * when 'tenc' is set. */
! 		if (input_conv.vc_type != CONV_NONE
! 						&& (ch2 == NUL || c != K_NUL))
  		{
! 		    conv = TRUE;
! 		    typeaheadlen -= unconverted;
! 		    n = convert_input_safe(typeahead + typeaheadlen,
! 				n + unconverted, TYPEAHEADLEN - typeaheadlen,
! 				rest == NULL ? &rest : NULL, &restlen);
  		}
  #endif
  
  		if (conv)
  		{
--- 1784,1819 ----
  		int	n = 1;
  		int     conv = FALSE;
  
  #ifdef FEAT_MBYTE
! 		if (ch2 == NUL)
  		{
! 		    int	    i;
! 		    char_u  *p;
! 		    WCHAR   ch[2];
! 
! 		    ch[0] = c;
! 		    if (c >= 0xD800 && c <= 0xDBFF)	/* High surrogate */
! 		    {
! 			ch[1] = tgetch(&modifiers, &ch2);
! 			n++;
! 		    }
! 		    p = utf16_to_enc(ch, &n);
! 		    if (p != NULL)
! 		    {
! 			for (i = 0; i < n; i++)
! 			    typeahead[typeaheadlen + i] = p[i];
! 			vim_free(p);
! 		    }
  		}
+ 		else
  #endif
+ 		    typeahead[typeaheadlen] = c;
+ 		if (ch2 != NUL)
+ 		{
+ 		    typeahead[typeaheadlen + n] = 3;
+ 		    typeahead[typeaheadlen + n + 1] = (char_u)ch2;
+ 		    n += 2;
+ 		}
  
  		if (conv)
  		{
***************
*** 5366,5392 ****
  
  
  /*
!  * write `cchToWrite' characters in `pchBuf' to the screen
!  * Returns the number of characters actually written (at least one).
   */
!     static BOOL
  write_chars(
!     LPCSTR pchBuf,
!     DWORD  cchToWrite)
  {
      COORD coord = g_coord;
      DWORD written;
  
!     FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cchToWrite,
! 				coord, &written);
!     /* When writing fails or didn't write a single character, pretend one
!      * character was written, otherwise we get stuck. */
!     if (WriteConsoleOutputCharacter(g_hConOut, pchBuf, cchToWrite,
! 				coord, &written) == 0
! 	    || written == 0)
! 	written = 1;
  
!     g_coord.X += (SHORT) written;
  
      while (g_coord.X > g_srScrollRegion.Right)
      {
--- 5343,5415 ----
  
  
  /*
!  * write `cbToWrite' bytes in `pchBuf' to the screen
!  * Returns the number of bytes actually written (at least one).
   */
!     static DWORD
  write_chars(
!     char_u *pchBuf,
!     DWORD  cbToWrite)
  {
      COORD coord = g_coord;
      DWORD written;
  
! #ifdef FEAT_MBYTE
!     if (enc_codepage >= 0 && (int)GetACP() != enc_codepage)
!     {
! 	static WCHAR	*unicodebuf = NULL;
! 	static int	unibuflen = 0;
! 	int		length;
! 	DWORD		n, cchwritten, cells;
! 
! 	length = MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)pchBuf, cbToWrite, 0, 0);
! 	if (unicodebuf == NULL || length > unibuflen)
! 	{
! 	    vim_free(unicodebuf);
! 	    unicodebuf = (WCHAR *)lalloc(length * sizeof(WCHAR), FALSE);
! 	    unibuflen = length;
! 	}
! 	MultiByteToWideChar(CP_UTF8, 0, (LPCSTR)pchBuf, cbToWrite,
! 			    unicodebuf, unibuflen);
! 
! 	cells = mb_string2cells(pchBuf, cbToWrite);
! 	FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cells,
! 				    coord, &written);
! 	/* When writing fails or didn't write a single character, pretend one
! 	 * character was written, otherwise we get stuck. */
! 	if (WriteConsoleOutputCharacterW(g_hConOut, unicodebuf, length,
! 		    coord, &cchwritten) == 0
! 		|| cchwritten == 0)
! 	    cchwritten = 1;
! 
! 	if (cchwritten == length)
! 	{
! 	    written = cbToWrite;
! 	    g_coord.X += (SHORT)cells;
! 	}
! 	else
! 	{
! 	    char_u *p = pchBuf;
! 	    for (n = 0; n < cchwritten; n++)
! 		mb_cptr_adv(p);
! 	    written = p - pchBuf;
! 	    g_coord.X += (SHORT)mb_string2cells(pchBuf, written);
! 	}
!     }
!     else
! #endif
!     {
! 	FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cbToWrite,
! 				    coord, &written);
! 	/* When writing fails or didn't write a single character, pretend one
! 	 * character was written, otherwise we get stuck. */
! 	if (WriteConsoleOutputCharacter(g_hConOut, (LPCSTR)pchBuf, cbToWrite,
! 		    coord, &written) == 0
! 		|| written == 0)
! 	    written = 1;
  
! 	g_coord.X += (SHORT) written;
!     }
  
      while (g_coord.X > g_srScrollRegion.Right)
      {
*** ../vim-7.4.851/src/ui.c	2015-08-11 19:13:55.146175594 +0200
--- src/ui.c	2015-09-01 20:27:49.069924312 +0200
***************
*** 42,48 ****
      /* Don't output anything in silent mode ("ex -s") unless 'verbose' set */
      if (!(silent_mode && p_verbose == 0))
      {
! #ifdef FEAT_MBYTE
  	char_u	*tofree = NULL;
  
  	if (output_conv.vc_type != CONV_NONE)
--- 42,48 ----
      /* Don't output anything in silent mode ("ex -s") unless 'verbose' set */
      if (!(silent_mode && p_verbose == 0))
      {
! #if defined(FEAT_MBYTE) && !defined(WIN3264)
  	char_u	*tofree = NULL;
  
  	if (output_conv.vc_type != CONV_NONE)
***************
*** 56,62 ****
  
  	mch_write(s, len);
  
! #ifdef FEAT_MBYTE
  	if (output_conv.vc_type != CONV_NONE)
  	    vim_free(tofree);
  #endif
--- 56,62 ----
  
  	mch_write(s, len);
  
! #if defined(FEAT_MBYTE) && !defined(WIN3264)
  	if (output_conv.vc_type != CONV_NONE)
  	    vim_free(tofree);
  #endif
*** ../vim-7.4.851/runtime/doc/options.txt	2015-07-21 17:53:11.573528028 +0200
--- runtime/doc/options.txt	2015-09-01 20:29:21.724964297 +0200
***************
*** 7377,7390 ****
  	the GUI it only applies to the keyboard ( 'encoding' is used for the
  	display).  Except for the Mac when 'macatsui' is off, then
  	'termencoding' should be "macroman".
- 	In the Win32 console version the default value is the console codepage
- 	when it differs from the ANSI codepage.
  								*E617*
  	Note: This does not apply to the GTK+ 2 GUI.  After the GUI has been
  	successfully initialized, 'termencoding' is forcibly set to "utf-8".
  	Any attempts to set a different value will be rejected, and an error
  	message is shown.
! 	For the Win32 GUI 'termencoding' is not used for typed characters,
  	because the Win32 system always passes Unicode characters.
  	When empty, the same encoding is used as for the 'encoding' option.
  	This is the normal value.
--- 7396,7407 ----
  	the GUI it only applies to the keyboard ( 'encoding' is used for the
  	display).  Except for the Mac when 'macatsui' is off, then
  	'termencoding' should be "macroman".
  								*E617*
  	Note: This does not apply to the GTK+ 2 GUI.  After the GUI has been
  	successfully initialized, 'termencoding' is forcibly set to "utf-8".
  	Any attempts to set a different value will be rejected, and an error
  	message is shown.
! 	For the Win32 GUI and console versions 'termencoding' is not used,
  	because the Win32 system always passes Unicode characters.
  	When empty, the same encoding is used as for the 'encoding' option.
  	This is the normal value.
*** ../vim-7.4.851/src/version.c	2015-09-01 20:23:30.408603580 +0200
--- src/version.c	2015-09-01 20:27:43.713979797 +0200
***************
*** 743,744 ****
--- 743,746 ----
  {   /* Add new patch number below this line */
+ /**/
+     852,
  /**/
 
-- 
(letter from Mark to Mike, about the film's probable certificate)
      I would like to get back to the Censor and agree to lose the shits, take
      the odd Jesus Christ out and lose Oh fuck off, but to retain 'fart in
      your general direction', 'castanets of your testicles' and 'oral sex'
      and ask him for an 'A' rating on that basis.
                 "Monty Python and the Holy Grail" PYTHON (MONTY) PICTURES LTD
 
 /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\
///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\  an exciting new programming language -- http://www.Zimbu.org        ///
 \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///