From ee2958f8a13facbc2bddef1b4b84b0fe0b1bc4c9 Mon Sep 17 00:00:00 2001

From: Charalampos Stratakis <cstratak@redhat.com>

Date: Thu, 28 Apr 2016 17:20:09 +0200

Subject: [PATCH] use Py_ssize_t for file offset and length computations in

 iteration

---

 Lib/test/test_file2k.py | 16 +++++++++++++++-

 Objects/fileobject.c    | 15 +++++++--------

 2 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/Lib/test/test_file2k.py b/Lib/test/test_file2k.py

index d1c1e35..b640aeb 100644

--- a/Lib/test/test_file2k.py

+++ b/Lib/test/test_file2k.py

@@ -14,7 +14,7 @@ except ImportError:

     threading = None

 from test import test_support

-from test.test_support import TESTFN, run_unittest

+from test.test_support import TESTFN, run_unittest, requires

 from UserList import UserList

 class AutoFileTests(unittest.TestCase):

@@ -437,6 +437,20 @@ class OtherFileTests(unittest.TestCase):

         finally:

             f.close()

+    @test_support.precisionbigmemtest(2**31, 2.5, dry_run=False)

+    def test_very_long_line(self, size):

+        # Issue #22526

+        requires('largefile')

+        with open(TESTFN, "wb") as fp:

+            fp.seek(size - 1)

+            fp.write("\0")

+        with open(TESTFN, "rb") as fp:

+            for l in fp:

+                pass

+        self.assertEqual(len(l), size)

+        self.assertEqual(l.count("\0"), size)

+        l = None

+

 class FileSubclassTests(unittest.TestCase):

     def testExit(self):

diff --git a/Objects/fileobject.c b/Objects/fileobject.c

index 5594058..55e074b 100644

--- a/Objects/fileobject.c

+++ b/Objects/fileobject.c

@@ -2236,7 +2236,7 @@ drop_readahead(PyFileObject *f)

    (unless at EOF) and no more than bufsize.  Returns negative value on

    error, will set MemoryError if bufsize bytes cannot be allocated. */

 static int

-readahead(PyFileObject *f, int bufsize)

+readahead(PyFileObject *f, Py_ssize_t bufsize)

 {

     Py_ssize_t chunksize;

@@ -2274,7 +2274,7 @@ readahead(PyFileObject *f, int bufsize)

    logarithmic buffer growth to about 50 even when reading a 1gb line. */

 static PyStringObject *

-readahead_get_line_skip(PyFileObject *f, int skip, int bufsize)

+readahead_get_line_skip(PyFileObject *f, Py_ssize_t skip, Py_ssize_t bufsize)

 {

     PyStringObject* s;

     char *bufptr;

@@ -2294,10 +2294,10 @@ readahead_get_line_skip(PyFileObject *f, int skip, int bufsize)

         bufptr++;                               /* Count the '\n' */

         len = bufptr - f->f_bufptr;

         s = (PyStringObject *)

-            PyString_FromStringAndSize(NULL, skip+len);

+            PyString_FromStringAndSize(NULL, skip + len);

         if (s == NULL)

             return NULL;

-        memcpy(PyString_AS_STRING(s)+skip, f->f_bufptr, len);

+        memcpy(PyString_AS_STRING(s) + skip, f->f_bufptr, len);

         f->f_bufptr = bufptr;

         if (bufptr == f->f_bufend)

             drop_readahead(f);

@@ -2305,14 +2305,13 @@ readahead_get_line_skip(PyFileObject *f, int skip, int bufsize)

         bufptr = f->f_bufptr;

         buf = f->f_buf;

         f->f_buf = NULL;                /* Force new readahead buffer */

-        assert(skip+len < INT_MAX);

-        s = readahead_get_line_skip(

-            f, (int)(skip+len), bufsize + (bufsize>>2) );

+        assert(len <= PY_SSIZE_T_MAX - skip);

+        s = readahead_get_line_skip(f, skip + len, bufsize + (bufsize>>2));

         if (s == NULL) {

             PyMem_Free(buf);

             return NULL;

         }

-        memcpy(PyString_AS_STRING(s)+skip, bufptr, len);

+        memcpy(PyString_AS_STRING(s) + skip, bufptr, len);

         PyMem_Free(buf);

     }

     return s;

--

2.5.5