1berry/cpython
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Checkpoint.

Browse Source 
Some cleanup of test_io.py and io.py.
Added seeking to buffered reader and writer, but no tests yet.
This commit is contained in:
Guido van Rossum 2007-04-06 19:10:29 +00:00
parent 186685905c
commit 76c5d4d72d
2 changed files with 111 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
Lib/io.py
View File

@ -8,6 +8,7 @@ See PEP 3116.
XXX need to default buffer size to 1 if isatty() XXX need to default buffer size to 1 if isatty()
XXX need to support 1 meaning line-buffered XXX need to support 1 meaning line-buffered
XXX change behavior of blocking I/O XXX change behavior of blocking I/O
XXX don't use assert to validate input requirements
""" """
__author__ = ("Guido van Rossum <guido@python.org>, " __author__ = ("Guido van Rossum <guido@python.org>, "
@ -265,7 +266,12 @@ class _PyFileIO(RawIOBase):
os.ftruncate(self._fd, pos) os.ftruncate(self._fd, pos)
def close(self): def close(self):
os.close(self._fd) # Must be idempotent
# XXX But what about thread-safe?
fd = self._fd
self._fd = -1
if fd >= 0:
os.close(fd)
def readable(self): def readable(self):
return "r" in self._mode or "+" in self._mode return "r" in self._mode or "+" in self._mode
@ -431,6 +437,9 @@ class BufferedIOBase(RawIOBase):
"""Flush the buffer to the underlying raw IO object.""" """Flush the buffer to the underlying raw IO object."""
raise IOError(".flush() unsupported") raise IOError(".flush() unsupported")
def seekable(self):
return self.raw.seekable()
class BufferedReader(BufferedIOBase): class BufferedReader(BufferedIOBase):
@ -457,10 +466,12 @@ class BufferedReader(BufferedIOBase):
mode. If n is None, read until EOF or until read() would mode. If n is None, read until EOF or until read() would
block. block.
""" """
# XXX n == 0 should return b""? n < 0 should be the same as n is None?
assert n is None or n > 0, '.read(): Bad read size %r' % n assert n is None or n > 0, '.read(): Bad read size %r' % n
nodata_val = b"" nodata_val = b""
while n is None or len(self._read_buf) < n: while n is None or len(self._read_buf) < n:
to_read = None if n is None else max(n, self.buffer_size) to_read = max(self.buffer_size,
n if n is not None else 2*len(self._read_buf))
current = self.raw.read(to_read) current = self.raw.read(to_read)
if current in (b"", None): if current in (b"", None):
@ -486,6 +497,15 @@ class BufferedReader(BufferedIOBase):
# Flush is a no-op # Flush is a no-op
pass pass
def tell(self):
return self.raw.tell() - len(self._read_buf)
def seek(self, pos, whence=0):
if whence == 1:
pos -= len(self._read_buf)
self.raw.seek(pos, whence)
self._read_buf = b""
def close(self): def close(self):
self.raw.close() self.raw.close()
@ -500,7 +520,7 @@ class BufferedWriter(BufferedIOBase):
self.raw = raw self.raw = raw
self.buffer_size = buffer_size self.buffer_size = buffer_size
self.max_buffer_size = max_buffer_size self.max_buffer_size = max_buffer_size
self._write_buf = b'' self._write_buf = b""
def write(self, b): def write(self, b):
# XXX we can implement some more tricks to try and avoid partial writes # XXX we can implement some more tricks to try and avoid partial writes
@ -511,9 +531,10 @@ class BufferedWriter(BufferedIOBase):
self.flush() self.flush()
except BlockingIO as e: except BlockingIO as e:
# We can't accept anything else. # We can't accept anything else.
# XXX Why not just let the exception pass through?
raise BlockingIO(e.errno, e.strerror, 0) raise BlockingIO(e.errno, e.strerror, 0)
self._write_buf += b self._write_buf += b
if (len(self._write_buf) > self.buffer_size): if len(self._write_buf) > self.buffer_size:
try: try:
self.flush() self.flush()
except BlockingIO as e: except BlockingIO as e:
@ -528,24 +549,34 @@ class BufferedWriter(BufferedIOBase):
return True return True
def flush(self): def flush(self):
written = 0
try: try:
while len(self._write_buf): while self._write_buf:
self._write_buf = self._write_buf[ n = self.raw.write(self._write_buf)
self.raw.write(self._write_buf):] del self._write_buf[:n]
written += n
except BlockingIO as e: except BlockingIO as e:
self._write_buf[e.characters_written:] n = e.characters_written
raise del self._write_buf[:n]
written += n
raise BlockingIO(e.errno, e.strerror, written)
def tell(self):
return self.raw.tell() + len(self._write_buf)
def seek(self, pos, whence=0):
self.flush()
self.raw.seek(pos, whence)
def fileno(self): def fileno(self):
return self.raw.fileno() return self.raw.fileno()
def close(self): def close(self):
self.flush()
self.raw.close() self.raw.close()
def __del__(self): def __del__(self):
# XXX flush buffers before dying. Is there a nicer way to do this? self.close()
if self._write_buf:
self.flush()
class BufferedRWPair(BufferedReader, BufferedWriter): class BufferedRWPair(BufferedReader, BufferedWriter):
@ -604,9 +635,6 @@ class BufferedRandom(BufferedReader, BufferedWriter):
BufferedReader.__init__(self, raw) BufferedReader.__init__(self, raw)
BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size) BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size)
def seekable(self):
return self.raw.seekable()
def readable(self): def readable(self):
return self.raw.readable() return self.raw.readable()
@ -615,10 +643,15 @@ class BufferedRandom(BufferedReader, BufferedWriter):
def seek(self, pos, whence=0): def seek(self, pos, whence=0):
self.flush() self.flush()
self._read_buf = b"" # First do the raw seek, then empty the read buffer, so that
# if the raw seek fails, we don't lose buffered data forever.
self.raw.seek(pos, whence) self.raw.seek(pos, whence)
self._read_buf = b""
# XXX I suppose we could implement some magic here to move through the # XXX I suppose we could implement some magic here to move through the
# existing read buffer in the case of seek(<some small +ve number>, 1) # existing read buffer in the case of seek(<some small +ve number>, 1)
# XXX OTOH it might be good to *guarantee* that the buffer is
# empty after a seek or flush; for small relative forward
# seeks one might as well use small reads instead.
def tell(self): def tell(self):
if (self._write_buf): if (self._write_buf):

123
Lib/test/test_io.py
View File

@ -1,25 +1,26 @@
"""Unit tests for io.py.""" """Unit tests for io.py."""
import unittest import unittest
from test import test_support
from itertools import chain from itertools import chain
from test import test_support
import io import io
class MockIO(io.RawIOBase): class MockIO(io.RawIOBase):
def __init__(self, readStack=()): def __init__(self, read_stack=()):
self._readStack = list(readStack) self._read_stack = list(read_stack)
self._writeStack = [] self._write_stack = []
def read(self, n=None): def read(self, n=None):
try: try:
return self._readStack.pop(0) return self._read_stack.pop(0)
except: except:
return b"" return b""
def write(self, b): def write(self, b):
self._writeStack.append(b) self._write_stack.append(b[:])
return len(b) return len(b)
def writable(self): def writable(self):
@ -60,7 +61,7 @@ class MockNonBlockWriterIO(io.RawIOBase):
self._write_stack = [] self._write_stack = []
def write(self, b): def write(self, b):
self._write_stack.append(b) self._write_stack.append(b[:])
n = self.bs.pop(0) n = self.bs.pop(0)
if (n < 0): if (n < 0):
raise io.BlockingIO(0, "test blocking", -n) raise io.BlockingIO(0, "test blocking", -n)
@ -159,7 +160,7 @@ class IOTest(unittest.TestCase):
f.close() f.close()
class MemorySeekTest(unittest.TestCase): class MemorySeekTestMixin:
def testInit(self): def testInit(self):
buf = self.buftype("1234567890") buf = self.buftype("1234567890")
@ -203,13 +204,13 @@ class MemorySeekTest(unittest.TestCase):
self.assertEquals(10000, bytesIo.tell()) self.assertEquals(10000, bytesIo.tell())
class BytesIOTest(MemorySeekTest): class BytesIOTest(MemorySeekTestMixin, unittest.TestCase):
buftype = bytes buftype = bytes
ioclass = io.BytesIO ioclass = io.BytesIO
EOF = b"" EOF = b""
class StringIOTest(MemorySeekTest): class StringIOTest(MemorySeekTestMixin, unittest.TestCase):
buftype = str buftype = str
ioclass = io.StringIO ioclass = io.StringIO
EOF = "" EOF = ""
@ -218,10 +219,10 @@ class StringIOTest(MemorySeekTest):
class BufferedReaderTest(unittest.TestCase): class BufferedReaderTest(unittest.TestCase):
def testRead(self): def testRead(self):
rawIo = MockIO((b"abc", b"d", b"efg")) rawio = MockIO((b"abc", b"d", b"efg"))
bufIo = io.BufferedReader(rawIo) bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcdef", bufIo.read(6)) self.assertEquals(b"abcdef", bufio.read(6))
def testBuffering(self): def testBuffering(self):
data = b"abcdefghi" data = b"abcdefghi"
@ -234,42 +235,42 @@ class BufferedReaderTest(unittest.TestCase):
] ]
for bufsize, buf_read_sizes, raw_read_sizes in tests: for bufsize, buf_read_sizes, raw_read_sizes in tests:
rawIo = MockFileIO(data) rawio = MockFileIO(data)
bufIo = io.BufferedReader(rawIo, buffer_size=bufsize) bufio = io.BufferedReader(rawio, buffer_size=bufsize)
pos = 0 pos = 0
for nbytes in buf_read_sizes: for nbytes in buf_read_sizes:
self.assertEquals(bufIo.read(nbytes), data[pos:pos+nbytes]) self.assertEquals(bufio.read(nbytes), data[pos:pos+nbytes])
pos += nbytes pos += nbytes
self.assertEquals(rawIo.read_history, raw_read_sizes) self.assertEquals(rawio.read_history, raw_read_sizes)
def testReadNonBlocking(self): def testReadNonBlocking(self):
# Inject some None's in there to simulate EWOULDBLOCK # Inject some None's in there to simulate EWOULDBLOCK
rawIo = MockIO((b"abc", b"d", None, b"efg", None, None)) rawio = MockIO((b"abc", b"d", None, b"efg", None, None))
bufIo = io.BufferedReader(rawIo) bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcd", bufIo.read(6)) self.assertEquals(b"abcd", bufio.read(6))
self.assertEquals(b"e", bufIo.read(1)) self.assertEquals(b"e", bufio.read(1))
self.assertEquals(b"fg", bufIo.read()) self.assertEquals(b"fg", bufio.read())
self.assert_(None is bufIo.read()) self.assert_(None is bufio.read())
self.assertEquals(b"", bufIo.read()) self.assertEquals(b"", bufio.read())
def testReadToEof(self): def testReadToEof(self):
rawIo = MockIO((b"abc", b"d", b"efg")) rawio = MockIO((b"abc", b"d", b"efg"))
bufIo = io.BufferedReader(rawIo) bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcdefg", bufIo.read(9000)) self.assertEquals(b"abcdefg", bufio.read(9000))
def testReadNoArgs(self): def testReadNoArgs(self):
rawIo = MockIO((b"abc", b"d", b"efg")) rawio = MockIO((b"abc", b"d", b"efg"))
bufIo = io.BufferedReader(rawIo) bufio = io.BufferedReader(rawio)
self.assertEquals(b"abcdefg", bufIo.read()) self.assertEquals(b"abcdefg", bufio.read())
def testFileno(self): def testFileno(self):
rawIo = MockIO((b"abc", b"d", b"efg")) rawio = MockIO((b"abc", b"d", b"efg"))
bufIo = io.BufferedReader(rawIo) bufio = io.BufferedReader(rawio)
self.assertEquals(42, bufIo.fileno()) self.assertEquals(42, bufio.fileno())
def testFilenoNoFileno(self): def testFilenoNoFileno(self):
# XXX will we always have fileno() function? If so, kill # XXX will we always have fileno() function? If so, kill
@ -282,55 +283,55 @@ class BufferedWriterTest(unittest.TestCase):
def testWrite(self): def testWrite(self):
# Write to the buffered IO but don't overflow the buffer. # Write to the buffered IO but don't overflow the buffer.
writer = MockIO() writer = MockIO()
bufIo = io.BufferedWriter(writer, 8) bufio = io.BufferedWriter(writer, 8)
bufIo.write(b"abc") bufio.write(b"abc")
self.assertFalse(writer._writeStack) self.assertFalse(writer._write_stack)
def testWriteOverflow(self): def testWriteOverflow(self):
writer = MockIO() writer = MockIO()
bufIo = io.BufferedWriter(writer, 8) bufio = io.BufferedWriter(writer, 8)
bufIo.write(b"abc") bufio.write(b"abc")
bufIo.write(b"defghijkl") bufio.write(b"defghijkl")
self.assertEquals(b"abcdefghijkl", writer._writeStack[0]) self.assertEquals(b"abcdefghijkl", writer._write_stack[0])
def testWriteNonBlocking(self): def testWriteNonBlocking(self):
raw = MockNonBlockWriterIO((9, 2, 22, -6, 10, 12, 12)) raw = MockNonBlockWriterIO((9, 2, 22, -6, 10, 12, 12))
bufIo = io.BufferedWriter(raw, 8, 16) bufio = io.BufferedWriter(raw, 8, 16)
bufIo.write(b"asdf") bufio.write(b"asdf")
bufIo.write(b"asdfa") bufio.write(b"asdfa")
self.assertEquals(b"asdfasdfa", raw._write_stack[0]) self.assertEquals(b"asdfasdfa", raw._write_stack[0])
bufIo.write(b"asdfasdfasdf") bufio.write(b"asdfasdfasdf")
self.assertEquals(b"asdfasdfasdf", raw._write_stack[1]) self.assertEquals(b"asdfasdfasdf", raw._write_stack[1])
bufIo.write(b"asdfasdfasdf") bufio.write(b"asdfasdfasdf")
self.assertEquals(b"dfasdfasdf", raw._write_stack[2]) self.assertEquals(b"dfasdfasdf", raw._write_stack[2])
self.assertEquals(b"asdfasdfasdf", raw._write_stack[3]) self.assertEquals(b"asdfasdfasdf", raw._write_stack[3])
bufIo.write(b"asdfasdfasdf") bufio.write(b"asdfasdfasdf")
# XXX I don't like this test. It relies too heavily on how the # XXX I don't like this test. It relies too heavily on how the
# algorithm actually works, which we might change. Refactor # algorithm actually works, which we might change. Refactor
# later. # later.
def testFileno(self): def testFileno(self):
rawIo = MockIO((b"abc", b"d", b"efg")) rawio = MockIO((b"abc", b"d", b"efg"))
bufIo = io.BufferedWriter(rawIo) bufio = io.BufferedWriter(rawio)
self.assertEquals(42, bufIo.fileno()) self.assertEquals(42, bufio.fileno())
def testFlush(self): def testFlush(self):
writer = MockIO() writer = MockIO()
bufIo = io.BufferedWriter(writer, 8) bufio = io.BufferedWriter(writer, 8)
bufIo.write(b"abc") bufio.write(b"abc")
bufIo.flush() bufio.flush()
self.assertEquals(b"abc", writer._writeStack[0]) self.assertEquals(b"abc", writer._write_stack[0])
class BufferedRWPairTest(unittest.TestCase): class BufferedRWPairTest(unittest.TestCase):
@ -352,9 +353,9 @@ class BufferedRandomTest(unittest.TestCase):
self.assertEqual(b"as", rw.read(2)) self.assertEqual(b"as", rw.read(2))
rw.write(b"ddd") rw.write(b"ddd")
rw.write(b"eee") rw.write(b"eee")
self.assertFalse(raw._writeStack) # Buffer writes self.assertFalse(raw._write_stack) # Buffer writes
self.assertEqual(b"ghjk", rw.read()) # This read forces write flush self.assertEqual(b"ghjk", rw.read()) # This read forces write flush
self.assertEquals(b"dddeee", raw._writeStack[0]) self.assertEquals(b"dddeee", raw._write_stack[0])
def testSeekAndTell(self): def testSeekAndTell(self):
raw = io.BytesIO(b"asdfghjkl") raw = io.BytesIO(b"asdfghjkl")
@ -400,19 +401,19 @@ class TextIOWrapperTest(unittest.TestCase):
for newline, exp_line_ends in tests: for newline, exp_line_ends in tests:
exp_lines = [ pad + line for line in exp_line_ends ] exp_lines = [ pad + line for line in exp_line_ends ]
bufIo = io.BufferedReader(io.BytesIO(data)) bufio = io.BufferedReader(io.BytesIO(data))
textIo = io.TextIOWrapper(bufIo, newline=newline, textio = io.TextIOWrapper(bufio, newline=newline,
encoding=encoding) encoding=encoding)
if do_reads: if do_reads:
got_lines = [] got_lines = []
while True: while True:
c2 = textIo.read(2) c2 = textio.read(2)
if c2 == '': if c2 == '':
break break
self.assertEquals(len(c2), 2) self.assertEquals(len(c2), 2)
got_lines.append(c2 + textIo.readline()) got_lines.append(c2 + textio.readline())
else: else:
got_lines = list(textIo) got_lines = list(textio)
for got_line, exp_line in zip(got_lines, exp_lines): for got_line, exp_line in zip(got_lines, exp_lines):
self.assertEquals(got_line, exp_line) self.assertEquals(got_line, exp_line)
@ -427,4 +428,4 @@ def test_main():
BufferedRandomTest, TextIOWrapperTest) BufferedRandomTest, TextIOWrapperTest)
if __name__ == "__main__": if __name__ == "__main__":
test_main() unittest.main()