To: vim-dev@vim.org

Subject: Patch 7.1.310

Fcc: outbox

From: Bram Moolenaar <Bram@moolenaar.net>

Mime-Version: 1.0

Content-Type: text/plain; charset=ISO-8859-1

Content-Transfer-Encoding: 8bit

------------

Patch 7.1.310

Problem:    Incomplete utf-8 byte sequence at end of the file is not detected.

	    Accessing memory that wasn't written.

Solution:   Check the last bytes in the buffer for being a valid utf-8

	    character. (mostly by Ben Schmidt)

	    Also fix that the reported line number of the error was wrong.

Files:	    src/fileio.c

*** ../vim-7.1.309/src/fileio.c	Wed May  7 19:05:55 2008

--- src/fileio.c	Wed Jun  4 18:28:48 2008

***************

*** 1288,1299 ****

  #ifdef FEAT_MBYTE

  		    else if (conv_restlen > 0)

  		    {

! 			/* Reached end-of-file but some trailing bytes could

! 			 * not be converted.  Truncated file? */

! 			if (conv_error == 0)

! 			    conv_error = linecnt;

! 			if (bad_char_behavior != BAD_DROP)

  			{

  			    fio_flags = 0;	/* don't convert this */

  # ifdef USE_ICONV

  			    if (iconv_fd != (iconv_t)-1)

--- 1288,1336 ----

  #ifdef FEAT_MBYTE

  		    else if (conv_restlen > 0)

  		    {

! 			/*

! 			 * Reached end-of-file but some trailing bytes could

! 			 * not be converted.  Truncated file?

! 			 */

! 

! 			/* When we did a conversion report an error. */

! 			if (fio_flags != 0

! # ifdef USE_ICONV

! 				|| iconv_fd != (iconv_t)-1

! # endif

! 			   )

  			{

+ 			    if (conv_error == 0)

+ 				conv_error = curbuf->b_ml.ml_line_count

+ 								- linecnt + 1;

+ 			}

+ 			/* Remember the first linenr with an illegal byte */

+ 			else if (illegal_byte == 0)

+ 			    illegal_byte = curbuf->b_ml.ml_line_count

+ 								- linecnt + 1;

+ 			if (bad_char_behavior == BAD_DROP)

+ 			{

+ 			    *(ptr - conv_restlen) = NUL;

+ 			    conv_restlen = 0;

+ 			}

+ 			else

+ 			{

+ 			    /* Replace the trailing bytes with the replacement

+ 			     * character if we were converting; if we weren't,

+ 			     * leave the UTF8 checking code to do it, as it

+ 			     * works slightly differently. */

+ 			    if (bad_char_behavior != BAD_KEEP && (fio_flags != 0

+ # ifdef USE_ICONV

+ 				    || iconv_fd != (iconv_t)-1

+ # endif

+ 			       ))

+ 			    {

+ 				while (conv_restlen > 0)

+ 				{

+ 				    *(--ptr) = bad_char_behavior;

+ 				    --conv_restlen;

+ 				}

+ 			    }

  			    fio_flags = 0;	/* don't convert this */

  # ifdef USE_ICONV

  			    if (iconv_fd != (iconv_t)-1)

***************

*** 1302,1321 ****

  				iconv_fd = (iconv_t)-1;

  			    }

  # endif

- 			    if (bad_char_behavior == BAD_KEEP)

- 			    {

- 				/* Keep the trailing bytes as-is. */

- 				size = conv_restlen;

- 				ptr -= conv_restlen;

- 			    }

- 			    else

- 			    {

- 				/* Replace the trailing bytes with the

- 				 * replacement character. */

- 				size = 1;

- 				*--ptr = bad_char_behavior;

- 			    }

- 			    conv_restlen = 0;

  			}

  		    }

  #endif

--- 1339,1344 ----

***************

*** 1397,1402 ****

--- 1420,1430 ----

  		    goto retry;

  		}

  	    }

+ 

+ 	    /* Include not converted bytes. */

+ 	    ptr -= conv_restlen;

+ 	    size += conv_restlen;

+ 	    conv_restlen = 0;

  #endif

  	    /*

  	     * Break here for a read error or end-of-file.

***************

*** 1406,1416 ****

  #ifdef FEAT_MBYTE

- 	    /* Include not converted bytes. */

- 	    ptr -= conv_restlen;

- 	    size += conv_restlen;

- 	    conv_restlen = 0;

- 

  # ifdef USE_ICONV

  	    if (iconv_fd != (iconv_t)-1)

  	    {

--- 1434,1439 ----

***************

*** 1872,1883 ****

  		size = (long)((ptr + real_size) - dest);

  		ptr = dest;

  	    }

! 	    else if (enc_utf8 && conv_error == 0 && !curbuf->b_p_bin)

  	    {

! 		/* Reading UTF-8: Check if the bytes are valid UTF-8.

! 		 * Need to start before "ptr" when part of the character was

! 		 * read in the previous read() call. */

! 		for (p = ptr - utf_head_off(buffer, ptr); ; ++p)

  		{

  		    int	 todo = (int)((ptr + size) - p);

  		    int	 l;

--- 1895,1906 ----

  		size = (long)((ptr + real_size) - dest);

  		ptr = dest;

  	    }

! 	    else if (enc_utf8 && !curbuf->b_p_bin)

  	    {

! 		int  incomplete_tail = FALSE;

! 

! 		/* Reading UTF-8: Check if the bytes are valid UTF-8. */

! 		for (p = ptr; ; ++p)

  		{

  		    int	 todo = (int)((ptr + size) - p);

  		    int	 l;

***************

*** 1891,1933 ****

  			 * read() will get the next bytes, we'll check it

  			 * then. */

  			l = utf_ptr2len_len(p, todo);

! 			if (l > todo)

  			{

! 			    /* Incomplete byte sequence, the next read()

! 			     * should get them and check the bytes. */

! 			    p += todo;

! 			    break;

  			}

! 			if (l == 1)

  			{

  			    /* Illegal byte.  If we can try another encoding

! 			     * do that. */

! 			    if (can_retry)

  				break;

- 

- 			    /* Remember the first linenr with an illegal byte */

- 			    if (illegal_byte == 0)

- 				illegal_byte = readfile_linenr(linecnt, ptr, p);

  # ifdef USE_ICONV

  			    /* When we did a conversion report an error. */

  			    if (iconv_fd != (iconv_t)-1 && conv_error == 0)

  				conv_error = readfile_linenr(linecnt, ptr, p);

  # endif

  			    /* Drop, keep or replace the bad byte. */

  			    if (bad_char_behavior == BAD_DROP)

  			    {

! 				mch_memmove(p, p+1, todo - 1);

  				--p;

  				--size;

  			    }

  			    else if (bad_char_behavior != BAD_KEEP)

  				*p = bad_char_behavior;

  			}

! 			p += l - 1;

  		    }

  		}

! 		if (p < ptr + size)

  		{

  		    /* Detected a UTF-8 error. */

  rewind_retry:

--- 1914,1969 ----

  			 * read() will get the next bytes, we'll check it

  			 * then. */

  			l = utf_ptr2len_len(p, todo);

! 			if (l > todo && !incomplete_tail)

  			{

! 			    /* Avoid retrying with a different encoding when

! 			     * a truncated file is more likely, or attempting

! 			     * to read the rest of an incomplete sequence when

! 			     * we have already done so. */

! 			    if (p > ptr || filesize > 0)

! 				incomplete_tail = TRUE;

! 			    /* Incomplete byte sequence, move it to conv_rest[]

! 			     * and try to read the rest of it, unless we've

! 			     * already done so. */

! 			    if (p > ptr)

! 			    {

! 				conv_restlen = todo;

! 				mch_memmove(conv_rest, p, conv_restlen);

! 				size -= conv_restlen;

! 				break;

! 			    }

  			}

! 			if (l == 1 || l > todo)

  			{

  			    /* Illegal byte.  If we can try another encoding

! 			     * do that, unless at EOF where a truncated

! 			     * file is more likely than a conversion error. */

! 			    if (can_retry && !incomplete_tail)

  				break;

  # ifdef USE_ICONV

  			    /* When we did a conversion report an error. */

  			    if (iconv_fd != (iconv_t)-1 && conv_error == 0)

  				conv_error = readfile_linenr(linecnt, ptr, p);

  # endif

+ 			    /* Remember the first linenr with an illegal byte */

+ 			    if (conv_error == 0 && illegal_byte == 0)

+ 				illegal_byte = readfile_linenr(linecnt, ptr, p);

  			    /* Drop, keep or replace the bad byte. */

  			    if (bad_char_behavior == BAD_DROP)

  			    {

! 				mch_memmove(p, p + 1, todo - 1);

  				--p;

  				--size;

  			    }

  			    else if (bad_char_behavior != BAD_KEEP)

  				*p = bad_char_behavior;

  			}

! 			else

! 			    p += l - 1;

  		    }

  		}

! 		if (p < ptr + size && !incomplete_tail)

  		{

  		    /* Detected a UTF-8 error. */

  rewind_retry:

*** ../vim-7.1.309/src/version.c	Wed Jun  4 15:27:43 2008

--- src/version.c	Wed Jun  4 19:35:16 2008

***************

*** 668,669 ****

--- 673,676 ----

  {   /* Add new patch number below this line */

+ /**/

+     310,

  /**/

-- 

Normal people believe that if it ain't broke, don't fix it.  Engineers believe

that if it ain't broke, it doesn't have enough features yet.

				(Scott Adams - The Dilbert principle)

 /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\

///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\

\\\        download, build and distribute -- http://www.A-A-P.org        ///

 \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///