From 52deb4b16d6b54f6250b0aaef5801fb967bfdd9c Mon Sep 17 00:00:00 2001 From: Ziver Koc Date: Fri, 15 Jul 2016 16:48:47 +0200 Subject: [PATCH] Added mark support to boundary stream and that fixed hasNext() in multipart --- src/zutil/io/BufferedBoundaryInputStream.java | 103 +++++++++++---- src/zutil/io/StringInputStream.java | 3 +- .../net/http/multipart/MultipartParser.java | 15 ++- .../io/BufferedBoundaryInputStreamTest.java | 123 +++++++++++++++--- .../http/multipart/MultipartParserTest.java | 17 ++- 5 files changed, 209 insertions(+), 52 deletions(-) diff --git a/src/zutil/io/BufferedBoundaryInputStream.java b/src/zutil/io/BufferedBoundaryInputStream.java index b63e67e..0ac5d09 100755 --- a/src/zutil/io/BufferedBoundaryInputStream.java +++ b/src/zutil/io/BufferedBoundaryInputStream.java @@ -48,9 +48,13 @@ public class BufferedBoundaryInputStream extends FilterInputStream{ /** The end position of the buffer */ private int buf_end = 0; /** Boundary position, 0< means no boundary found */ - private int buf_bound_pos; + private int buf_bound_pos = -1; /** The boundary (the delimiter) */ private byte[] boundary; + /** The position in the buffer where user has marked, -1 if no mark is set **/ + private int buf_mark = -1; + /** The read limit of the mark, after this limit the mark will be invalidated **/ + private int buf_mark_limit = 0; /** @@ -70,18 +74,15 @@ public class BufferedBoundaryInputStream extends FilterInputStream{ */ public BufferedBoundaryInputStream(InputStream in, int buf_size){ super(in); - buf_pos = 0; - buf_end = 0; - buf_bound_pos = -1; buffer = new byte[buf_size]; } - + /** - * @return the next byte from the stream or -1 if EOF or stream is on a boundary + * @return the next byte from the stream or -1 if EOF or stream has encountered a boundary */ - public final int read() throws IOException{ + public int read() throws IOException{ if (fillBuffer() < 0) return -1; @@ -132,7 +133,7 @@ public class BufferedBoundaryInputStream extends FilterInputStream{ /** * @return if the current position in the buffer is a boundary */ - private boolean isOnBoundary(){ + public boolean isOnBoundary(){ return buf_bound_pos == buf_pos; } @@ -165,17 +166,19 @@ public class BufferedBoundaryInputStream extends FilterInputStream{ if (buf_bound_pos >= 0){ // is boundary in buffer? buf_pos += boundary.length; - searchNextBoundary(); + findNextBoundary(); } } /** * Skips a specific amounts of bytes in the buffer. - * Note that his method does not check for boundaries. + * Note that his method does not check for boundaries + * and it will only skip in the local buffer. * * @param n the number of bytes to be skipped. - * @return the actual number of bytes skipped. + * @return the actual number of bytes skipped, + * 0 if it has reach the end of the buffer but does not mean end of stream. */ public long skip(long n) throws IOException { int leftover = available(); @@ -192,7 +195,7 @@ public class BufferedBoundaryInputStream extends FilterInputStream{ */ public void setBoundary(String b){ this.boundary = b.getBytes(); - searchNextBoundary(); // redo the search with the new boundary + findNextBoundary(); // redo the search with the new boundary } /** @@ -201,48 +204,96 @@ public class BufferedBoundaryInputStream extends FilterInputStream{ public void setBoundary(byte[] b){ boundary = new byte[b.length]; System.arraycopy(b, 0, boundary, 0, b.length); - searchNextBoundary(); // redo the search with the new boundary + findNextBoundary(); // redo the search with the new boundary } /** * @return an estimate of the number of bytes that can be read (or skipped * over) from this buffered input stream without blocking. - * @exception IOException if an I/O error occurs. */ - public int available() { - return buf_end - buf_pos; + public int available() throws IOException { + if (super.available() <= 0) + return buf_end - buf_pos; // return the whole stream as there are no more boundaries + else if (buf_end < boundary.length) + return 0; // we need a safety block in case boundary is split + return buf_end - boundary.length - buf_pos; } - /** - * Tests if this input stream supports the mark and - * reset methods. + + + /** + * @return true for BufferedBoundaryInputStream */ public boolean markSupported(){ - return false; + return true; } + /** + * See {@link InputStream#mark(int)} for details + * + * @param readlimit the amount of data that can be read before + * the mark is invalidated. Note that if the + * readlimit is larger than the buffer size, + * then the buffer size will be used instead. + */ + public void mark(int readlimit) { + buf_mark_limit = readlimit; + buf_mark = buf_pos; + } + + /** + * See {@link InputStream#reset()} for details + * + * @exception IOException if the mark is or has been invalidated + */ + public void reset() throws IOException { + if (buf_mark < 0) + throw new IOException("Resetting to invalid mark"); + buf_mark_limit = 0; + buf_pos = buf_mark; + buf_mark = -1; + findNextBoundary(); + } /** * Checks if the buffer needs to be appended with data. * If so it moves the remaining data to the beginning of the - * buffer and then fills the buffer with data from - * the source stream + * buffer and then fills the buffer with data from the source + * stream. * * @return the number of new bytes read from the source stream, * or -1 if the buffer is empty and it is the end of the stream */ private int fillBuffer() throws IOException { - int leftover = this.available(); // Do we need to fill the buffer if(buf_pos < buf_end-boundary.length) return 0; + int leftover = buf_end - buf_pos; + int tmp_pos = buf_pos; + // Mark handling + if (buf_mark > 0) { + // Mark enabled and it has not already been moved to the beginning of the buffer + leftover = buf_end - buf_mark; + buf_pos = buf_mark; + } else if (buf_pos >= buf_mark_limit || + buf_mark_limit >= buffer.length) { + // we have passed the read limit or read limit is bigger than the buffer, so reset mark + buf_mark = -1; + buf_mark_limit = 0; + } + // Move the end of the buffer to the start to not miss any split boundary if (leftover > 0 && buf_pos != 0) System.arraycopy(buffer, buf_pos, buffer, 0, leftover); - buf_pos = 0; + // Set new positions + if (buf_mark >= 0){ + buf_pos = tmp_pos - buf_mark; + buf_mark = 0; + } else + buf_pos = 0; buf_end = leftover; // Copy in new data from the stream @@ -254,14 +305,14 @@ public class BufferedBoundaryInputStream extends FilterInputStream{ } // Update boundary position - searchNextBoundary(); + findNextBoundary(); return ((leftover > 0 && n < 0) ? 0 : n); } /** * Searches for the nearest boundary from the current buffer position */ - private void searchNextBoundary(){ + private void findNextBoundary(){ // No need to check for boundary if buffer is smaller than the boundary length for (int i = buf_pos; i <= buf_end-boundary.length; i++) { for (int b = 0; b < boundary.length; b++) { diff --git a/src/zutil/io/StringInputStream.java b/src/zutil/io/StringInputStream.java index d1df0e2..d7d755f 100755 --- a/src/zutil/io/StringInputStream.java +++ b/src/zutil/io/StringInputStream.java @@ -25,6 +25,7 @@ package zutil.io; import java.io.InputStream; +import java.nio.charset.StandardCharsets; /** * This class saves all the input data in to an StringBuffer @@ -90,7 +91,7 @@ public class StringInputStream extends InputStream{ if( buffer.length() < len ){ len = buffer.length(); } - byte[] btmp = buffer.substring(0, len).getBytes(); + byte[] btmp = buffer.substring(0, len).getBytes(StandardCharsets.ISO_8859_1); System.arraycopy(btmp, 0, b, off, len); buffer.delete(0, len); return len; diff --git a/src/zutil/net/http/multipart/MultipartParser.java b/src/zutil/net/http/multipart/MultipartParser.java index ef9ba6c..6520e39 100755 --- a/src/zutil/net/http/multipart/MultipartParser.java +++ b/src/zutil/net/http/multipart/MultipartParser.java @@ -107,11 +107,24 @@ public class MultipartParser implements Iterable{ /** - * TODO: there is a bug where this returns true after the last MultiPart as it cannot read ahead. So use next() != null instead + * @return if there is more data after the closest boundary. + * Note that if all data until the next boundary has + * not been read then this method can only estimate + * if there is a next element as the next boundary or + * end of stream might be out of reach. */ @Override public boolean hasNext() { try { + // check is the last characters are "--" + if (boundaryIn.hasNext() && boundaryIn.isOnBoundary()) { + boundaryIn.mark(delimiter.length() + 10); + boundaryIn.next(); + boolean ret = ! ('-' == boundaryIn.read() && '-' == boundaryIn.read()); + boundaryIn.reset(); + return ret; + } + // Or just guess return boundaryIn.hasNext(); } catch (IOException e) { logger.log(Level.SEVERE, null, e); diff --git a/test/zutil/io/BufferedBoundaryInputStreamTest.java b/test/zutil/io/BufferedBoundaryInputStreamTest.java index c4302a5..5c25cda 100755 --- a/test/zutil/io/BufferedBoundaryInputStreamTest.java +++ b/test/zutil/io/BufferedBoundaryInputStreamTest.java @@ -27,22 +27,37 @@ package zutil.io; import org.junit.Test; import java.io.IOException; +import java.io.InputStream; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; @SuppressWarnings("resource") public class BufferedBoundaryInputStreamTest { - private static BufferedBoundaryInputStream getBufferedBoundaryInputStream(String data, String b) { + private void readAndAssertArray(byte[] expected, InputStream in, int readcount) throws IOException { + byte[] buff = new byte[readcount]; + int n = in.read(buff, 0, readcount); + if (n < readcount) + n += in.read(buff, n, readcount-n); + assertEquals(readcount, n); + assertArrayEquals(expected, buff); + } + + private static BufferedBoundaryInputStream getBufferedBoundaryInputStream(String data, String boundary) { + return getBufferedBoundaryInputStream(data, boundary, -1); + } + private static BufferedBoundaryInputStream getBufferedBoundaryInputStream(String data, String boundary, int buffLimit) { StringInputStream inin = new StringInputStream(data); - BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin); - in.setBoundary(b); + BufferedBoundaryInputStream in = buffLimit >= 0 ? + new BufferedBoundaryInputStream(inin, buffLimit) : + new BufferedBoundaryInputStream(inin); + in.setBoundary(boundary); return in; } + + @Test public void read_normal() throws IOException { BufferedBoundaryInputStream in = getBufferedBoundaryInputStream("aaa#a##aaaaaaa#", "#"); @@ -200,10 +215,8 @@ public class BufferedBoundaryInputStreamTest { @Test public void read_largeData() throws IOException { - String data = "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#"; - StringInputStream inin = new StringInputStream(data); - BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin, 10); - in.setBoundary("#"); + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#", "#", 10); assertEquals(-1, in.read()); assertTrue(in.hasNext()); @@ -226,10 +239,8 @@ public class BufferedBoundaryInputStreamTest { } @Test public void read_largeDataOnlyNext() throws IOException { - String data = "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#"; - StringInputStream inin = new StringInputStream(data); - BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin, 10); - in.setBoundary("#"); + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#", "#", 10); in.next(); for (int i=0; i<12; ++i) @@ -247,10 +258,8 @@ public class BufferedBoundaryInputStreamTest { } @Test public void readArr_largeData() throws IOException { - String data = "aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#"; - StringInputStream inin = new StringInputStream(data); - BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin, 10); - in.setBoundary("#"); + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#", "#", 10); byte[] buff = new byte[100]; int n = 0; @@ -270,4 +279,82 @@ public class BufferedBoundaryInputStreamTest { assertEquals(-1, in.read()); assertFalse(in.hasNext()); } + + @Test + public void readArr_splitBoundary() throws IOException { + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "aaaaaaaa####aaaaaaaa", "####", 10); + + byte[] buff = new byte[10]; + assertEquals(4, in.read(buff, 0 , 4)); + int n = 0; + n += in.read(buff, 0 , 10); + n += in.read(buff, 0 , 10); + assertEquals(4, n); + assertEquals(-1, in.read(buff, 0 , 6)); + in.next(); + assertEquals(6, in.read(buff, 0 , 6)); + } + + + @Test + public void read_mark() throws IOException { + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "0123456789#abcdefghijklmn#opqrstuvwxyz", "#"); + + in.read();in.read();in.read(); + assertEquals('3', in.read()); + in.mark(10); // mark '4' + assertEquals('4', in.read()); + in.read();in.read();in.read(); + assertEquals('8', in.read()); + in.reset(); // go back to '4' + assertEquals('4', in.read()); + } + @Test + public void readArr_mark() throws IOException { + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "0123456789#abcdefghijklmn#opqrstuvwxyz", "#"); + + readAndAssertArray(new byte[]{'0','1','2','3'}, in, 4); + in.mark(10); // mark '4' + readAndAssertArray(new byte[]{'4','5','6','7'}, in, 4); + readAndAssertArray(new byte[]{'8','9'}, in, 2); + in.reset(); // go back to '4' + readAndAssertArray(new byte[]{'4','5','6','7'}, in, 4); + } + + @Test + public void readArr_markBoundary() throws IOException { + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "0123456789#abcdefghijklmn#opqrstuvwxyz", "#"); + + readAndAssertArray(new byte[]{'0','1','2','3'}, in, 4); + in.mark(30); // mark '4' + readAndAssertArray(new byte[]{'4','5','6','7','8','9'}, in, 6); + assertEquals(-1, in.read()); // boundary + in.next(); + readAndAssertArray(new byte[]{'a','b'}, in, 2); + in.reset(); // go back to '4' + readAndAssertArray(new byte[]{'4','5','6','7','8','9'}, in, 6); + assertEquals(-1, in.read()); // is boundary still there + } + + @Test + public void readArr_markLargeData() throws IOException { + BufferedBoundaryInputStream in = getBufferedBoundaryInputStream( + "0123456789#abcdefghijklmn#opqrstuvwxyz", "#", 10); + + readAndAssertArray(new byte[]{'0','1','2','3'}, in, 4); + in.mark(30); // mark '4' + readAndAssertArray(new byte[]{'4','5','6','7','8','9'}, in, 6); + assertEquals(-1, in.read()); // boundary + in.next(); + readAndAssertArray(new byte[]{'a','b','c','d','e','f'}, in, 6); + try{ + in.reset(); // Exception we have passed buffer limit + fail(); + } catch (IOException e){} + readAndAssertArray(new byte[]{'g','h'}, in, 2); + } } diff --git a/test/zutil/net/http/multipart/MultipartParserTest.java b/test/zutil/net/http/multipart/MultipartParserTest.java index 05c45a3..f5e9d25 100755 --- a/test/zutil/net/http/multipart/MultipartParserTest.java +++ b/test/zutil/net/http/multipart/MultipartParserTest.java @@ -33,13 +33,15 @@ public class MultipartParserTest { // Assertions Iterator it = parser.iterator(); + assertTrue(it.hasNext()); MultipartField field = it.next(); assertTrue(field instanceof MultipartStringField); MultipartStringField stringField = (MultipartStringField) field; assertEquals("foo", stringField.getName()); assertEquals("bar", stringField.getValue()); - //assertFalse(it.hasNext()); //TODO: does not work, how to solve this? + + assertFalse(it.hasNext()); assertEquals(null, it.next()); assertEquals(null, it.next()); } @@ -54,19 +56,21 @@ public class MultipartParserTest { String input_data = "?PNG\n" + "\n" + - "lkuytrewacvbnmloiuytrewrtyuiol,mnbvdc xswertyuioplm cdsertyuioölkjgf\n" + - "kgkfdgfhgfhgkhgvytvjgxslkyöiyfödgjdjhföjhdlgdgjfhföjhföiyföudlögföudöfö\n" + - "ögdljgdjhöfjhfdfsgdfg ryrt dtrd ytfc uhöiugljfkdkhdjgd\n" + + "lkuytrewacvbnmloiuytrewrtyuiol,mnbvdc xswertyuioplm cdsertyuiojlkjgf\n" + + "kgkfdgfhgfhgkhgvytvjgxslkysiyfedgjdjhfkjhdlgdgjfhfcjhfqiyfyudlmgfeudcfa\n" + + "bgdljgdjhffjhfdfsgdfg ryrt dtrd ytfc uhhiugljfkdkhdjgd\n" + " xx\n" + " kuykutfytdh ytrd trutrd trxxx"; String input_end = "\n------------------------------83ff53821b7c--"; + String input = input_start+input_data+input_end; MultipartParser parser = new MultipartParser( - new StringInputStream(input_start+input_data+input_end), + new StringInputStream(input), "----------------------------83ff53821b7c", 0); // Assertions Iterator it = parser.iterator(); + assertTrue(it.hasNext()); MultipartField field = it.next(); assertTrue(field instanceof MultipartFileField); @@ -75,7 +79,8 @@ public class MultipartParserTest { assertEquals("a.png", fileField.getFilename()); assertEquals("application/octet-stream", fileField.getContentType()); assertEquals(input_data, new String(fileField.getContent())); - //assertFalse(it.hasNext()); //TODO: does not work, how to solve this? + + assertFalse(it.hasNext()); } //Test