To: vim_dev@googlegroups.com

Subject: Patch 7.3.427

Fcc: outbox

From: Bram Moolenaar <Bram@moolenaar.net>

Mime-Version: 1.0

Content-Type: text/plain; charset=UTF-8

Content-Transfer-Encoding: 8bit

------------

Patch 7.3.427

Problem:    readfile() can be slow with long lines.

Solution:   Use realloc() instead of alloc(). (John Little)

Files:      src/eval.c

*** ../vim-7.3.426/src/eval.c	2012-01-26 14:32:26.000000000 +0100

--- src/eval.c	2012-02-05 00:25:39.000000000 +0100

***************

*** 14325,14346 ****

      typval_T	*rettv;

  {

      int		binary = FALSE;

      char_u	*fname;

      FILE	*fd;

!     listitem_T	*li;

! #define FREAD_SIZE 200	    /* optimized for text lines */

!     char_u	buf[FREAD_SIZE];

!     int		readlen;    /* size of last fread() */

!     int		buflen;	    /* nr of valid chars in buf[] */

!     int		filtd;	    /* how much in buf[] was NUL -> '\n' filtered */

!     int		tolist;	    /* first byte in buf[] still to be put in list */

!     int		chop;	    /* how many CR to chop off */

!     char_u	*prev = NULL;	/* previously read bytes, if any */

!     int		prevlen = 0;    /* length of "prev" if not NULL */

!     char_u	*s;

!     int		len;

!     long	maxline = MAXLNUM;

!     long	cnt = 0;

      if (argvars[1].v_type != VAR_UNKNOWN)

      {

--- 14325,14343 ----

      typval_T	*rettv;

  {

      int		binary = FALSE;

+     int		failed = FALSE;

      char_u	*fname;

      FILE	*fd;

!     char_u	buf[(IOSIZE/256)*256];	/* rounded to avoid odd + 1 */

!     int		io_size = sizeof(buf);

!     int		readlen;		/* size of last fread() */

!     char_u	*prev	 = NULL;	/* previously read bytes, if any */

!     long	prevlen  = 0;		/* length of data in prev */

!     long	prevsize = 0;		/* size of prev buffer */

!     long	maxline  = MAXLNUM;

!     long	cnt	 = 0;

!     char_u	*p;			/* position in buf */

!     char_u	*start;			/* start of current line */

      if (argvars[1].v_type != VAR_UNKNOWN)

      {

***************

*** 14362,14410 ****

  	return;

      }

-     filtd = 0;

      while (cnt < maxline || maxline < 0)

      {

! 	readlen = (int)fread(buf + filtd, 1, FREAD_SIZE - filtd, fd);

! 	buflen = filtd + readlen;

! 	tolist = 0;

! 	for ( ; filtd < buflen || readlen <= 0; ++filtd)

! 	{

! 	    if (readlen <= 0 || buf[filtd] == '\n')

! 	    {

! 		/* In binary mode add an empty list item when the last

! 		 * non-empty line ends in a '\n'. */

! 		if (!binary && readlen == 0 && filtd == 0 && prev == NULL)

! 		    break;

! 		/* Found end-of-line or end-of-file: add a text line to the

! 		 * list. */

! 		chop = 0;

! 		if (!binary)

! 		    while (filtd - chop - 1 >= tolist

! 					  && buf[filtd - chop - 1] == '\r')

! 			++chop;

! 		len = filtd - tolist - chop;

! 		if (prev == NULL)

! 		    s = vim_strnsave(buf + tolist, len);

  		else

  		{

! 		    s = alloc((unsigned)(prevlen + len + 1));

! 		    if (s != NULL)

  		    {

! 			mch_memmove(s, prev, prevlen);

! 			vim_free(prev);

! 			prev = NULL;

! 			mch_memmove(s + prevlen, buf + tolist, len);

  			s[prevlen + len] = NUL;

  		    }

  		}

! 		tolist = filtd + 1;

! 		li = listitem_alloc();

! 		if (li == NULL)

  		{

  		    vim_free(s);

  		    break;

  		}

  		li->li_tv.v_type = VAR_STRING;

--- 14359,14419 ----

  	return;

      }

      while (cnt < maxline || maxline < 0)

      {

! 	readlen = (int)fread(buf, 1, io_size, fd);

! 

! 	/* This for loop processes what was read, but is also entered at end

! 	 * of file so that either:

! 	 * - an incomplete line gets written

! 	 * - a "binary" file gets an empty line at the end if it ends in a

! 	 *   newline.  */

! 	for (p = buf, start = buf;

! 		p < buf + readlen || (readlen <= 0 && (prevlen > 0 || binary));

! 		++p)

! 	{

! 	    if (*p == '\n' || readlen <= 0)

! 	    {

! 		listitem_T  *li;

! 		char_u	    *s	= NULL;

! 		long_u	    len = p - start;

! 		/* Finished a line.  Remove CRs before NL. */

! 		if (readlen > 0 && !binary)

! 		{

! 		    while (len > 0 && start[len - 1] == '\r')

! 			--len;

! 		    /* removal may cross back to the "prev" string */

! 		    if (len == 0)

! 			while (prevlen > 0 && prev[prevlen - 1] == '\r')

! 			    --prevlen;

! 		}

! 		if (prevlen == 0)

! 		    s = vim_strnsave(start, len);

  		else

  		{

! 		    /* Change "prev" buffer to be the right size.  This way

! 		     * the bytes are only copied once, and very long lines are

! 		     * allocated only once.  */

! 		    if ((s = vim_realloc(prev, prevlen + len + 1)) != NULL)

  		    {

! 			mch_memmove(s + prevlen, start, len);

  			s[prevlen + len] = NUL;

+ 			prev = NULL; /* the list will own the string */

+ 			prevlen = prevsize = 0;

  		    }

  		}

! 		if (s == NULL)

! 		{

! 		    do_outofmem_msg((long_u) prevlen + len + 1);

! 		    failed = TRUE;

! 		    break;

! 		}

! 		if ((li = listitem_alloc()) == NULL)

  		{

  		    vim_free(s);

+ 		    failed = TRUE;

  		    break;

  		}

  		li->li_tv.v_type = VAR_STRING;

***************

*** 14412,14485 ****

  		li->li_tv.vval.v_string = s;

  		list_append(rettv->vval.v_list, li);

! 		if (++cnt >= maxline && maxline >= 0)

! 		    break;

! 		if (readlen <= 0)

  		    break;

  	    }

! 	    else if (buf[filtd] == NUL)

! 		buf[filtd] = '\n';

  #ifdef FEAT_MBYTE

! 	    else if (buf[filtd] == 0xef

! 		    && enc_utf8

! 		    && filtd + 2 < buflen

! 		    && !binary

! 		    && buf[filtd + 1] == 0xbb

! 		    && buf[filtd + 2] == 0xbf)

! 	    {

! 		/* remove utf-8 byte order mark */

! 		mch_memmove(buf + filtd, buf + filtd + 3, buflen - filtd - 3);

! 		--filtd;

! 		buflen -= 3;

  	    }

  #endif

! 	}

! 	if (readlen <= 0)

! 	    break;

! 	if (tolist == 0)

  	{

! 	    if (buflen >= FREAD_SIZE / 2)

  	    {

! 		/* "buf" is full, need to move text to an allocated buffer */

! 		if (prev == NULL)

  		{

! 		    prev = vim_strnsave(buf, buflen);

! 		    prevlen = buflen;

  		}

! 		else

  		{

! 		    s = alloc((unsigned)(prevlen + buflen));

! 		    if (s != NULL)

! 		    {

! 			mch_memmove(s, prev, prevlen);

! 			mch_memmove(s + prevlen, buf, buflen);

! 			vim_free(prev);

! 			prev = s;

! 			prevlen += buflen;

! 		    }

  		}

! 		filtd = 0;

  	    }

  	}

! 	else

! 	{

! 	    mch_memmove(buf, buf + tolist, buflen - tolist);

! 	    filtd -= tolist;

! 	}

!     }

      /*

       * For a negative line count use only the lines at the end of the file,

       * free the rest.

       */

!     if (maxline < 0)

  	while (cnt > -maxline)

  	{

  	    listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first);

  	    --cnt;

  	}

      vim_free(prev);

      fclose(fd);

  }

--- 14421,14529 ----

  		li->li_tv.vval.v_string = s;

  		list_append(rettv->vval.v_list, li);

! 		start = p + 1; /* step over newline */

! 		if ((++cnt >= maxline && maxline >= 0) || readlen <= 0)

  		    break;

  	    }

! 	    else if (*p == NUL)

! 		*p = '\n';

  #ifdef FEAT_MBYTE

! 	    /* Check for utf8 "bom"; U+FEFF is encoded as EF BB BF.  Do this

! 	     * when finding the BF and check the previous two bytes. */

! 	    else if (*p == 0xbf && enc_utf8 && !binary)

! 	    {

! 		/* Find the two bytes before the 0xbf.	If p is at buf, or buf

! 		 * + 1, these may be in the "prev" string. */

! 		char_u back1 = p >= buf + 1 ? p[-1]

! 				     : prevlen >= 1 ? prev[prevlen - 1] : NUL;

! 		char_u back2 = p >= buf + 2 ? p[-2]

! 			  : p == buf + 1 && prevlen >= 1 ? prev[prevlen - 1]

! 			  : prevlen >= 2 ? prev[prevlen - 2] : NUL;

! 

! 		if (back2 == 0xef && back1 == 0xbb)

! 		{

! 		    char_u *dest = p - 2;

! 

! 		    /* Usually a BOM is at the beginning of a file, and so at

! 		     * the beginning of a line; then we can just step over it.

! 		     */

! 		    if (start == dest)

! 			start = p + 1;

! 		    else

! 		    {

! 			/* have to shuffle buf to close gap */

! 			int adjust_prevlen = 0;

! 

! 			if (dest < buf)

! 			{

! 			    adjust_prevlen = buf - dest; /* must be 1 or 2 */

! 			    dest = buf;

! 			}

! 			if (readlen > p - buf + 1)

! 			    mch_memmove(dest, p + 1, readlen - (p - buf) - 1);

! 			readlen -= 3 - adjust_prevlen;

! 			prevlen -= adjust_prevlen;

! 			p = dest - 1;

! 		    }

! 		}

  	    }

  #endif

! 	} /* for */

! 	if (failed || (cnt >= maxline && maxline >= 0) || readlen <= 0)

! 	    break;

! 	if (start < p)

  	{

! 	    /* There's part of a line in buf, store it in "prev". */

! 	    if (p - start + prevlen >= prevsize)

  	    {

! 		/* need bigger "prev" buffer */

! 		char_u *newprev;

! 

! 		/* A common use case is ordinary text files and "prev" gets a

! 		 * fragment of a line, so the first allocation is made

! 		 * small, to avoid repeatedly 'allocing' large and

! 		 * 'reallocing' small. */

! 		if (prevsize == 0)

! 		    prevsize = p - start;

! 		else

  		{

! 		    long grow50pc = (prevsize * 3) / 2;

! 		    long growmin  = (p - start) * 2 + prevlen;

! 		    prevsize = grow50pc > growmin ? grow50pc : growmin;

  		}

! 		if ((newprev = vim_realloc(prev, prevsize)) == NULL)

  		{

! 		    do_outofmem_msg((long_u)prevsize);

! 		    failed = TRUE;

! 		    break;

  		}

! 		prev = newprev;

  	    }

+ 	    /* Add the line part to end of "prev". */

+ 	    mch_memmove(prev + prevlen, start, p - start);

+ 	    prevlen += p - start;

  	}

!     } /* while */

      /*

       * For a negative line count use only the lines at the end of the file,

       * free the rest.

       */

!     if (!failed && maxline < 0)

  	while (cnt > -maxline)

  	{

  	    listitem_remove(rettv->vval.v_list, rettv->vval.v_list->lv_first);

  	    --cnt;

  	}

+     if (failed)

+     {

+ 	list_free(rettv->vval.v_list, TRUE);

+ 	/* readfile doc says an empty list is returned on error */

+ 	rettv->vval.v_list = list_alloc();

+     }

+ 

      vim_free(prev);

      fclose(fd);

  }

***************

*** 21802,21808 ****

  	fp = HI2UF(hi);

  	if (fp->uf_flags & FC_DICT)

! 	    return ""; /* don't show dict functions */

  	if (STRLEN(fp->uf_name) + 4 >= IOSIZE)

  	    return fp->uf_name;	/* prevents overflow */

--- 21846,21852 ----

  	fp = HI2UF(hi);

  	if (fp->uf_flags & FC_DICT)

! 	    return (char_u *)""; /* don't show dict functions */

  	if (STRLEN(fp->uf_name) + 4 >= IOSIZE)

  	    return fp->uf_name;	/* prevents overflow */

*** ../vim-7.3.426/src/version.c	2012-02-04 23:34:57.000000000 +0100

--- src/version.c	2012-02-05 00:38:34.000000000 +0100

***************

*** 716,717 ****

--- 716,719 ----

  {   /* Add new patch number below this line */

+ /**/

+     427,

  /**/

-- 

One difference between a man and a machine is that a machine is quiet

when well oiled.

 /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\

///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\

\\\  an exciting new programming language -- http://www.Zimbu.org        ///

 \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///