Added a work CharBuffer

Author: Ioan Eugen Stan

nbactions.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<actions>
+        <action>
+            <actionName>run</actionName>
+            <goals>
+                <goal>process-classes</goal>
+                <goal>org.codehaus.mojo:exec-maven-plugin:1.2:exec</goal>
+            </goals>
+            <properties>
+                <exec.args>-classpath %classpath ro.ieugen.mboxiterator.MboxIteratorExample</exec.args>
+                <exec.executable>java</exec.executable>
+                <exec.classpathScope>runtime</exec.classpathScope>
+            </properties>
+        </action>
+    </actions>

src/main/java/ro/ieugen/mboxiterator/MboxIterator.java

@@ -1,21 +1,13 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one   *
- * or more contributor license agreements.  See the NOTICE file *
- * distributed with this work for additional information        *
- * regarding copyright ownership.  The ASF licenses this file   *
- * to you under the Apache License, Version 2.0 (the            *
- * "License"); you may not use this file except in compliance   *
- * with the License.  You may obtain a copy of the License at   *
- *                                                              *
- *   http://www.apache.org/licenses/LICENSE-2.0                 *
- *                                                              *
- * Unless required by applicable law or agreed to in writing,   *
- * software distributed under the License is distributed on an  *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
- * KIND, either express or implied.  See the License for the    *
- * specific language governing permissions and limitations      *
- * under the License.                                           *
- ****************************************************************/
+/**
+ * **************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you
+ * under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You
+ * may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or
+ * agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and
+ * limitations * under the License. * **************************************************************
+ */
 package ro.ieugen.mboxiterator;
 
 import java.io.*;
@@ -24,6 +16,7 @@
 import java.nio.channels.FileChannel;
 import java.nio.charset.Charset;
 import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CoderResult;
 import java.util.Iterator;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -38,6 +31,7 @@
 public class MboxIterator implements Iterable<CharBuffer>, Closeable {
 
     private static final Logger LOG = LoggerFactory.getLogger(MboxIterator.class);
+    private static final int MAX_MSG_LENGTH = 1024 * 1024 * 10;     // 10Mb of Chars!
     private final FileInputStream fis;
     private final CharBuffer mboxCharBuffer;
     private final Matcher fromLineMathcer;
@@ -56,11 +50,9 @@ private MboxIterator(final File mbox,
         final MappedByteBuffer byteBuffer = fileChannel.map(FileChannel.MapMode.READ_ONLY, 0,
                 fileChannel.size());
         final CharsetDecoder DECODER = Charset.forName(charset).newDecoder();
-        /*TODO: DECODER.decode() this will try to decode the whole file. 
-         * It could be problematic if the file is large (~2gb). 
-         * Improve this by working with chunks.  
-         */
-        mboxCharBuffer = DECODER.decode(byteBuffer);
+        mboxCharBuffer = CharBuffer.allocate(MAX_MSG_LENGTH);
+        CoderResult result = DECODER.decode(byteBuffer, mboxCharBuffer, false);
+        
         final Pattern MESSAGE_START = Pattern.compile(regexpPattern, regexpFlags);
         fromLineMathcer = MESSAGE_START.matcher(mboxCharBuffer);
         hasMore = fromLineMathcer.find();
@@ -95,10 +87,10 @@ public boolean hasNext() {
 
         @Override
         public CharBuffer next() {
-            LOG.info("next() called at offset {}", fromLineMathcer.start());
+            //LOG.info("next() called at offset {}", fromLineMathcer.start());
             final CharBuffer message = mboxCharBuffer.slice();
             message.position(fromLineMathcer.start());
-            logBufferDetails(message);
+            //logBufferDetails(message);
             hasMore = fromLineMathcer.find();
             if (hasMore) {
                 LOG.info("We limit the buffer to {} ?? {}",

src/main/java/ro/ieugen/mboxiterator/MboxIteratorExample.java

@@ -41,10 +41,10 @@ public class MboxIteratorExample {
 
     // simple example of how to split an mbox into individual files
     public static void main(String[] args) throws IOException, FileNotFoundException {
-        final File mbox = new File("/home/ieugen/contracte/firimituri/gmane.test.yahoo/test-utf");
+        final File mbox = new File("/home/estan/gmail2.mbox");
         long start = System.currentTimeMillis();
         int count = 0;
-        for (CharBuffer buf : new MboxIterator.Builder(mbox).build()) {
+        for (CharBuffer buf : new MboxIterator.Builder(mbox).charset("ISO-8859-1").build()) {
             FileOutputStream fout = new FileOutputStream(new File("target/messages/msg-" + count));
             FileChannel fileChannel = fout.getChannel();
             ByteBuffer buf2 = ENCODER.encode(buf);