Added mark support to boundary stream and that fixed hasNext() in multipart

src/zutil/io/BufferedBoundaryInputStream.java

@@ -48,9 +48,13 @@ public class BufferedBoundaryInputStream extends FilterInputStream{
 	/** The end position of the buffer */
     private int buf_end = 0;
     /** Boundary position, 0< means no boundary found */
-    private int buf_bound_pos;
+    private int buf_bound_pos = -1;
 	/** The boundary (the delimiter)  */
     private byte[] boundary;
+    /** The position in the buffer where user has marked, -1 if no mark is set **/
+    private int buf_mark = -1;
+    /** The read limit of the mark, after this limit the mark will be invalidated **/
+    private int buf_mark_limit = 0;
 
 
 	/**
@@ -70,18 +74,15 @@ public class BufferedBoundaryInputStream extends FilterInputStream{
 	 */
 	public BufferedBoundaryInputStream(InputStream in, int buf_size){
 		super(in);
-        buf_pos = 0;
-		buf_end = 0;
-        buf_bound_pos = -1;
 		buffer = new byte[buf_size];
 	}
 
 
-	
+
 	/**
-     * @return 			the next byte from the stream or -1 if EOF or stream is on a boundary
+     * @return 			the next byte from the stream or -1 if EOF or stream has encountered a boundary
 	 */
-	public final int read() throws IOException{
+	public int read() throws IOException{
         if (fillBuffer() < 0)
             return -1;
 
@@ -132,7 +133,7 @@ public class BufferedBoundaryInputStream extends FilterInputStream{
     /**
      * @return if the current position in the buffer is a boundary
      */
-    private boolean isOnBoundary(){
+    public boolean isOnBoundary(){
         return buf_bound_pos == buf_pos;
     }
 
@@ -165,17 +166,19 @@ public class BufferedBoundaryInputStream extends FilterInputStream{
 
 		if (buf_bound_pos >= 0){ // is boundary in buffer?
 			buf_pos += boundary.length;
-			searchNextBoundary();
+			findNextBoundary();
 		}
 	}
 	
 
 	/**
 	 * Skips a specific amounts of bytes in the buffer.
-	 * Note that his method does not check for boundaries.
+	 * Note that his method does not check for boundaries
+     * and it will only skip in the local buffer.
 	 * 
      * @param	n 	the number of bytes to be skipped.
-     * @return		the actual number of bytes skipped.
+     * @return		the actual number of bytes skipped,
+     *              0 if it has reach the end of the buffer but does not mean end of stream.
 	 */
 	public long skip(long n) throws IOException {
 		int leftover = available();
@@ -192,7 +195,7 @@ public class BufferedBoundaryInputStream extends FilterInputStream{
 	 */
 	public void setBoundary(String b){
 		this.boundary = b.getBytes();
-        searchNextBoundary(); // redo the search with the new boundary
+        findNextBoundary(); // redo the search with the new boundary
 	}
 	
 	/**
@@ -201,48 +204,96 @@ public class BufferedBoundaryInputStream extends FilterInputStream{
 	public void setBoundary(byte[] b){
 		boundary = new byte[b.length];
 		System.arraycopy(b, 0, boundary, 0, b.length);
-        searchNextBoundary(); // redo the search with the new boundary
+        findNextBoundary(); // redo the search with the new boundary
 	}
 
 	/**
 	 * @return     an estimate of the number of bytes that can be read (or skipped
 	 *             over) from this buffered input stream without blocking.
-	 * @exception  IOException  if an I/O error occurs.
 	 */
-	public int available() {
+	public int available() throws IOException {
-		return buf_end - buf_pos;
+	    if (super.available() <= 0)
+	        return buf_end - buf_pos; // return the whole stream as there are no more boundaries
+        else if (buf_end < boundary.length)
+	        return 0; // we need a safety block in case boundary is split
+		return buf_end - boundary.length - buf_pos;
 	}
 
-	/**
+
-     * Tests if this input stream supports the mark and 
+
-     * reset methods.
+    /**
+     * @return true for BufferedBoundaryInputStream
      */
 	public boolean markSupported(){
-		return false;
+		return true;
 	}
 
+    /**
+     * See {@link InputStream#mark(int)} for details
+     *
+     * @param   readlimit   the amount of data that can be read before
+     *                      the mark is invalidated. Note that if the
+     *                      readlimit is larger than the buffer size,
+     *                      then the buffer size will be used instead.
+     */
+    public void mark(int readlimit) {
+        buf_mark_limit = readlimit;
+        buf_mark = buf_pos;
+    }
+
+    /**
+     * See {@link InputStream#reset()} for details
+     *
+     * @exception  IOException if the mark is or has been invalidated
+     */
+    public void reset() throws IOException {
+        if (buf_mark < 0)
+            throw new IOException("Resetting to invalid mark");
+        buf_mark_limit = 0;
+        buf_pos = buf_mark;
+        buf_mark = -1;
+        findNextBoundary();
+    }
 
 
 
     /**
      * Checks if the buffer needs to be appended with data.
      * If so it moves the remaining data to the beginning of the
-     * buffer and then fills the buffer with data from
+     * buffer and then fills the buffer with data from the source
-     * the source stream
+     * stream.
      *
      * @return 			the number of new bytes read from the source stream,
      *                  or -1 if the buffer is empty and it is the end of the stream
      */
     private int fillBuffer() throws IOException {
-        int leftover = this.available();
         // Do we need to fill the buffer
         if(buf_pos < buf_end-boundary.length)
             return 0;
 
+        int leftover = buf_end - buf_pos;
+        int tmp_pos = buf_pos;
+        // Mark handling
+        if (buf_mark > 0) {
+            // Mark enabled and it has not already been moved to the beginning of the buffer
+            leftover = buf_end - buf_mark;
+            buf_pos  = buf_mark;
+        } else if (buf_pos >= buf_mark_limit ||
+                buf_mark_limit >= buffer.length) {
+            // we have passed the read limit or read limit is bigger than the buffer, so reset mark
+            buf_mark = -1;
+            buf_mark_limit = 0;
+        }
+
         // Move the end of the buffer to the start to not miss any split boundary
         if (leftover > 0 && buf_pos != 0)
             System.arraycopy(buffer, buf_pos, buffer, 0, leftover);
-        buf_pos = 0;
+        // Set new positions
+        if (buf_mark >= 0){
+            buf_pos = tmp_pos - buf_mark;
+            buf_mark = 0;
+        } else
+            buf_pos = 0;
         buf_end = leftover;
 
         // Copy in new data from the stream
@@ -254,14 +305,14 @@ public class BufferedBoundaryInputStream extends FilterInputStream{
         }
 
         // Update boundary position
-        searchNextBoundary();
+        findNextBoundary();
         return ((leftover > 0 && n < 0) ? 0 : n);
     }
 
     /**
      * Searches for the nearest boundary from the current buffer position
      */
-    private void searchNextBoundary(){
+    private void findNextBoundary(){
         // No need to check for boundary if buffer is smaller than the boundary length
         for (int i = buf_pos; i <= buf_end-boundary.length; i++) {
             for (int b = 0; b < boundary.length; b++) {

src/zutil/io/StringInputStream.java

@@ -25,6 +25,7 @@
 package zutil.io;
 
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 
 /**
  * This class saves all the input data in to an StringBuffer
@@ -90,7 +91,7 @@ public class StringInputStream extends InputStream{
     	if( buffer.length() < len ){
     		len = buffer.length();
     	}
-    	byte[] btmp = buffer.substring(0, len).getBytes();
+    	byte[] btmp = buffer.substring(0, len).getBytes(StandardCharsets.ISO_8859_1);
     	System.arraycopy(btmp, 0, b, off, len);
     	buffer.delete(0, len);
 		return len;

src/zutil/net/http/multipart/MultipartParser.java

@@ -107,11 +107,24 @@ public class MultipartParser implements Iterable<MultipartField>{
 
 
         /**
-         * TODO: there is a bug where this returns true after the last MultiPart as it cannot read ahead. So use next() != null instead
+         * @return if there is more data after the closest boundary.
+         *          Note that if all data until the next boundary has
+         *          not been read then this method can only estimate
+         *          if there is a next element as the next boundary or
+         *          end of stream might be out of reach.
          */
         @Override
         public boolean hasNext() {
             try {
+                // check is the last characters are "--"
+                if (boundaryIn.hasNext() && boundaryIn.isOnBoundary()) {
+                    boundaryIn.mark(delimiter.length() + 10);
+                    boundaryIn.next();
+                    boolean ret = ! ('-' == boundaryIn.read() && '-' == boundaryIn.read());
+                    boundaryIn.reset();
+                    return ret;
+                }
+                // Or just guess
                 return boundaryIn.hasNext();
             } catch (IOException e) {
                 logger.log(Level.SEVERE, null, e);

test/zutil/io/BufferedBoundaryInputStreamTest.java

@@ -27,22 +27,37 @@ package zutil.io;
 import org.junit.Test;
 
 import java.io.IOException;
+import java.io.InputStream;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
 
 
 @SuppressWarnings("resource")
 public class BufferedBoundaryInputStreamTest {
 
-    private static BufferedBoundaryInputStream getBufferedBoundaryInputStream(String data, String b) {
+    private void readAndAssertArray(byte[] expected, InputStream in, int readcount) throws IOException {
+        byte[] buff = new byte[readcount];
+        int n = in.read(buff, 0, readcount);
+        if (n < readcount)
+            n += in.read(buff, n, readcount-n);
+        assertEquals(readcount, n);
+        assertArrayEquals(expected, buff);
+    }
+
+    private static BufferedBoundaryInputStream getBufferedBoundaryInputStream(String data, String boundary) {
+        return getBufferedBoundaryInputStream(data, boundary, -1);
+    }
+    private static BufferedBoundaryInputStream getBufferedBoundaryInputStream(String data, String boundary, int buffLimit) {
         StringInputStream inin = new StringInputStream(data);
-        BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin);
+        BufferedBoundaryInputStream in = buffLimit >= 0 ?
-        in.setBoundary(b);
+                new BufferedBoundaryInputStream(inin, buffLimit) :
+                new BufferedBoundaryInputStream(inin);
+        in.setBoundary(boundary);
         return in;
     }
 
+
+
 	@Test
 	public void read_normal() throws IOException {
         BufferedBoundaryInputStream in = getBufferedBoundaryInputStream("aaa#a##aaaaaaa#", "#");
@@ -200,10 +215,8 @@ public class BufferedBoundaryInputStreamTest {
 
     @Test
     public void read_largeData() throws IOException {
-        String data = "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#";
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
-        StringInputStream inin = new StringInputStream(data);
+                "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#", "#", 10);
-        BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin, 10);
-        in.setBoundary("#");
 
         assertEquals(-1, in.read());
         assertTrue(in.hasNext());
@@ -226,10 +239,8 @@ public class BufferedBoundaryInputStreamTest {
     }
     @Test
     public void read_largeDataOnlyNext() throws IOException {
-        String data = "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#";
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
-        StringInputStream inin = new StringInputStream(data);
+                "#aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#", "#", 10);
-        BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin, 10);
-        in.setBoundary("#");
 
         in.next();
         for (int i=0; i<12; ++i)
@@ -247,10 +258,8 @@ public class BufferedBoundaryInputStreamTest {
     }
     @Test
     public void readArr_largeData() throws IOException {
-        String data = "aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#";
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
-        StringInputStream inin = new StringInputStream(data);
+                "aaaaaaaaaaaa#aa#aaaaaaaaaaaaaaa#", "#", 10);
-        BufferedBoundaryInputStream in = new BufferedBoundaryInputStream(inin, 10);
-        in.setBoundary("#");
 
         byte[] buff = new byte[100];
         int n = 0;
@@ -270,4 +279,82 @@ public class BufferedBoundaryInputStreamTest {
         assertEquals(-1, in.read());
         assertFalse(in.hasNext());
     }
+
+    @Test
+    public void readArr_splitBoundary() throws IOException {
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
+                "aaaaaaaa####aaaaaaaa", "####", 10);
+
+        byte[] buff = new byte[10];
+        assertEquals(4, in.read(buff, 0 , 4));
+        int n = 0;
+        n += in.read(buff, 0 , 10);
+        n += in.read(buff, 0 , 10);
+        assertEquals(4, n);
+        assertEquals(-1, in.read(buff, 0 , 6));
+        in.next();
+        assertEquals(6, in.read(buff, 0 , 6));
+    }
+
+
+    @Test
+    public void read_mark() throws IOException {
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
+                "0123456789#abcdefghijklmn#opqrstuvwxyz", "#");
+
+        in.read();in.read();in.read();
+        assertEquals('3', in.read());
+        in.mark(10); // mark '4'
+        assertEquals('4', in.read());
+        in.read();in.read();in.read();
+        assertEquals('8', in.read());
+        in.reset(); // go back to '4'
+        assertEquals('4', in.read());
+    }
+    @Test
+    public void readArr_mark() throws IOException {
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
+                "0123456789#abcdefghijklmn#opqrstuvwxyz", "#");
+
+        readAndAssertArray(new byte[]{'0','1','2','3'}, in, 4);
+        in.mark(10); // mark '4'
+        readAndAssertArray(new byte[]{'4','5','6','7'}, in, 4);
+        readAndAssertArray(new byte[]{'8','9'}, in, 2);
+        in.reset(); // go back to '4'
+        readAndAssertArray(new byte[]{'4','5','6','7'}, in, 4);
+    }
+
+    @Test
+    public void readArr_markBoundary() throws IOException {
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
+                "0123456789#abcdefghijklmn#opqrstuvwxyz", "#");
+
+        readAndAssertArray(new byte[]{'0','1','2','3'}, in, 4);
+        in.mark(30); // mark '4'
+        readAndAssertArray(new byte[]{'4','5','6','7','8','9'}, in, 6);
+        assertEquals(-1, in.read()); // boundary
+        in.next();
+        readAndAssertArray(new byte[]{'a','b'}, in, 2);
+        in.reset(); // go back to '4'
+        readAndAssertArray(new byte[]{'4','5','6','7','8','9'}, in, 6);
+        assertEquals(-1, in.read()); // is boundary still there
+    }
+
+    @Test
+    public void readArr_markLargeData() throws IOException {
+        BufferedBoundaryInputStream in = getBufferedBoundaryInputStream(
+                "0123456789#abcdefghijklmn#opqrstuvwxyz", "#", 10);
+
+        readAndAssertArray(new byte[]{'0','1','2','3'}, in, 4);
+        in.mark(30); // mark '4'
+        readAndAssertArray(new byte[]{'4','5','6','7','8','9'}, in, 6);
+        assertEquals(-1, in.read()); // boundary
+        in.next();
+        readAndAssertArray(new byte[]{'a','b','c','d','e','f'}, in, 6);
+        try{
+            in.reset(); // Exception we have passed buffer limit
+            fail();
+        } catch (IOException e){}
+        readAndAssertArray(new byte[]{'g','h'}, in, 2);
+    }
 }

test/zutil/net/http/multipart/MultipartParserTest.java

@@ -33,13 +33,15 @@ public class MultipartParserTest {
 
         // Assertions
         Iterator<MultipartField> it = parser.iterator();
+
         assertTrue(it.hasNext());
         MultipartField field = it.next();
         assertTrue(field instanceof MultipartStringField);
         MultipartStringField stringField = (MultipartStringField) field;
         assertEquals("foo", stringField.getName());
         assertEquals("bar", stringField.getValue());
-        //assertFalse(it.hasNext()); //TODO: does not work, how to solve this?
+
+        assertFalse(it.hasNext());
         assertEquals(null, it.next());
         assertEquals(null, it.next());
     }
@@ -54,19 +56,21 @@ public class MultipartParserTest {
         String input_data =
                 "?PNG\n" +
                 "\n" +
-                "lkuytrewacvbnmloiuytrewrtyuiol,mnbvdc xswertyuioplm cdsertyuioölkjgf\n" +
+                "lkuytrewacvbnmloiuytrewrtyuiol,mnbvdc xswertyuioplm cdsertyuiojlkjgf\n" +
-                "kgkfdgfhgfhgkhgvytvjgxslkyöiyfödgjdjhföjhdlgdgjfhföjhföiyföudlögföudöfö\n" +
+                "kgkfdgfhgfhgkhgvytvjgxslkysiyfedgjdjhfkjhdlgdgjfhfcjhfqiyfyudlmgfeudcfa\n" +
-                "ögdljgdjhöfjhfdfsgdfg  ryrt dtrd ytfc  uhöiugljfkdkhdjgd\n" +
+                "bgdljgdjhffjhfdfsgdfg  ryrt dtrd ytfc  uhhiugljfkdkhdjgd\n" +
                 " xx\n" +
                 "     kuykutfytdh ytrd trutrd trxxx";
         String input_end = "\n------------------------------83ff53821b7c--";
+        String input = input_start+input_data+input_end;
         MultipartParser parser = new MultipartParser(
-                new StringInputStream(input_start+input_data+input_end),
+                new StringInputStream(input),
                 "----------------------------83ff53821b7c",
                 0);
 
         // Assertions
         Iterator<MultipartField> it = parser.iterator();
+
         assertTrue(it.hasNext());
         MultipartField field = it.next();
         assertTrue(field instanceof MultipartFileField);
@@ -75,7 +79,8 @@ public class MultipartParserTest {
         assertEquals("a.png", fileField.getFilename());
         assertEquals("application/octet-stream", fileField.getContentType());
         assertEquals(input_data, new String(fileField.getContent()));
-        //assertFalse(it.hasNext()); //TODO: does not work, how to solve this?
+
+        assertFalse(it.hasNext());
     }
 
     //Test