From 0a8cadc08a7d67e35548506b36f8b03872d6c415 Mon Sep 17 00:00:00 2001 From: Jason Height Date: Wed, 13 Oct 2004 03:37:56 +0000 Subject: [PATCH] PR: 19974 Obtained from: Submitted by: Reviewed by: This patch fixes writeing VERY long > 8228 character strings. Existing SST serialisation had problem with calculation and writing to multiple continue records. Also fixed some test case errors introduced by my last patch for DBCell and IndexRecords git-svn-id: https://svn.apache.org/repos/asf/jakarta/poi/trunk@353607 13f79535-47bb-0310-9956-ffa450edef68 --- src/java/org/apache/poi/hssf/model/Sheet.java | 4 +- .../hssf/record/SSTRecordSizeCalculator.java | 229 +++++------------- .../apache/poi/hssf/record/SSTSerializer.java | 18 +- .../aggregates/TestValueRecordsAggregate.java | 3 +- 4 files changed, 81 insertions(+), 173 deletions(-) diff --git a/src/java/org/apache/poi/hssf/model/Sheet.java b/src/java/org/apache/poi/hssf/model/Sheet.java index 14bf40256d..540b25f56d 100644 --- a/src/java/org/apache/poi/hssf/model/Sheet.java +++ b/src/java/org/apache/poi/hssf/model/Sheet.java @@ -763,7 +763,7 @@ public class Sheet implements Model if (record.getSid() == BOFRecord.sid) { //Can there be more than one BOF for a sheet? If not then we can //remove this guard. So be safe it is left here. - if (!haveSerializedIndex) { + if ((rows != null) && (!haveSerializedIndex)) { haveSerializedIndex = true; pos += serializeIndexRecord(k, pos, data); } @@ -2029,6 +2029,7 @@ public class Sheet implements Model retval += (( Record ) records.get(k)).getRecordSize(); } //Add space for the IndexRecord + if (rows != null) { final int blocks = rows.getRowBlockCount(); retval += IndexRecord.getRecordSizeForBlockCount(blocks); @@ -2043,6 +2044,7 @@ public class Sheet implements Model if (cells.rowHasCells(row.getRowNumber())) retval += 2; } + } return retval; } diff --git a/src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java b/src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java index 611361400e..59f539c6e9 100644 --- a/src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java +++ b/src/java/org/apache/poi/hssf/record/SSTRecordSizeCalculator.java @@ -30,6 +30,7 @@ import java.util.Map; * the SST serialization code needs to be rewritten. * * @author Glen Stampoultzis (glens at apache.org) + * @author Jason Height (jheight at apache.org) */ class SSTRecordSizeCalculator { @@ -52,180 +53,76 @@ class SSTRecordSizeCalculator this.strings = strings; } - /** - * Calculate the size in bytes of the SST record. This will include continue - * records. - * - * @return the size of the SST record. - */ - public int getRecordSize() - { - initVars(); + private boolean canFitStringInRecord(int recordLength) { + return (recordLength+SSTRecord.STRING_MINIMAL_OVERHEAD) < SSTRecord.MAX_RECORD_SIZE; + } - int retval; - int totalStringSpaceRequired = SSTSerializer.calculateUnicodeSize(strings); + public int getRecordSize() { + //Indicates how much of the current base or continue record has + //been written + int continueSize = SSTRecord.SST_RECORD_OVERHEAD; + int recordSize = 0; + for (int i=0; i < strings.size(); i++ ) + { + Integer intunipos = new Integer(i); + UnicodeString unistr = ( (UnicodeString) strings.get(intunipos)); + final int stringLength = unistr.getRecordSize(); + if ((continueSize + stringLength) <= SSTRecord.MAX_RECORD_SIZE) { + //String can fit within the bounds of the current record (SST or Continue) + continueSize += stringLength; + + if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) { + //Start new continueRecord if there is another string + recordLengths.add(new Integer(continueSize)); + recordSize += continueSize; + //Minimum ammount of space for a new continue record. + continueSize = 4; + } + } else { + int stringRemainder = stringLength; + while (stringRemainder != 0) { + if ( (continueSize + stringRemainder) > SSTRecord.MAX_RECORD_SIZE) { + //Determine number of bytes that can be written in the space + //available + int bytesWritten = Math.min((SSTRecord.MAX_RECORD_SIZE - continueSize), stringRemainder); - if ( totalStringSpaceRequired > SSTRecord.MAX_DATA_SPACE ) - { - retval = sizeOverContinuation( totalStringSpaceRequired ); + //Ensure that the Unicode String writes both the high and low + //byte in the one action. Since the string overhead is 3 bytes + //if the bytes that can be written is even, then we need to + //write one less byte to capture both the high and low bytes. + bytesWritten = unistr.maxBrokenLength(bytesWritten); + continueSize += bytesWritten; + stringRemainder -= bytesWritten; + recordLengths.add(new Integer(continueSize)); + recordSize += continueSize; + //Minimum ammount of space for a new continue record. + continueSize = 4; + //Add one to the size of the string that is remaining, since the + //first byte for the next continue record will be compressed unicode indicator + stringRemainder++; + } else { + //Remainder of string can fit within the bounds of the current + //continue record + continueSize += stringRemainder; + stringRemainder = 0; + if ((i < (strings.size()-1)) && !canFitStringInRecord(continueSize)) { + //Start new continueRecord if there is another string + recordLengths.add(new Integer(continueSize)); + recordSize += continueSize; + //Minimum ammount of space for a new continue record. + continueSize = 4; } - else - { - // short data: write one simple SST record - retval = SSTRecord.SST_RECORD_OVERHEAD + totalStringSpaceRequired; - recordLengths.add( new Integer( totalStringSpaceRequired ) ); + } } - return retval; + } + } + recordLengths.add(new Integer(continueSize)); + recordSize += continueSize; + return recordSize; } public List getRecordLengths() { return recordLengths; } - - private int sizeOverContinuation( int totalStringSpaceRequired ) - { - int retval; - - while ( !finished ) - { - recordSize = 0; - pos = 0; - - if ( firstRecord ) - { - addMaxLengthRecordSize(); - } - else - { - - // writing continue record - pos = 0; - int toBeWritten = ( totalStringSpaceRequired - totalBytesWritten ) + ( isRemainingString ? 1 : 0 ); - int size = Math.min( SSTRecord.MAX_RECORD_SIZE - SSTRecord.STD_RECORD_OVERHEAD, toBeWritten ); - - if ( size == toBeWritten ) - { - finished = true; - } - recordSize = size + SSTRecord.STD_RECORD_OVERHEAD; - recordLengths.add( new Integer( size ) ); - pos = 4; - } - if ( isRemainingString ) - { - calcReminant(); - } - calcRemainingStrings(); - totalWritten += recordSize; - } - retval = totalWritten; - - return retval; - } - - private void addMaxLengthRecordSize() - { - // writing SST record - recordSize = SSTRecord.MAX_RECORD_SIZE; - pos = 12; - firstRecord = false; - recordLengths.add( new Integer( recordSize - SSTRecord.STD_RECORD_OVERHEAD ) ); - } - - private void calcRemainingStrings() - { - for ( ; unipos < strings.size(); unipos++ ) - { - int available = SSTRecord.MAX_RECORD_SIZE - pos; - Integer intunipos = new Integer( unipos ); - - unistr = ( (UnicodeString) strings.get( intunipos ) ); - if ( unistr.getRecordSize() <= available ) - { - totalBytesWritten += unistr.getRecordSize(); - pos += unistr.getRecordSize(); - } - else - { - if ( available >= SSTRecord.STRING_MINIMAL_OVERHEAD ) - { - int toBeWritten = - unistr.maxBrokenLength( available ); - - totalBytesWritten += toBeWritten; - stringReminant = - ( unistr.getRecordSize() - toBeWritten ) - + LittleEndianConsts.BYTE_SIZE; - if ( available != toBeWritten ) - { - int shortrecord = recordSize - - ( available - toBeWritten ); - - recordLengths.set( - recordLengths.size() - 1, - new Integer( - shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) ); - recordSize = shortrecord; - } - isRemainingString = true; - unipos++; - } - else - { - int shortrecord = recordSize - available; - - recordLengths.set( recordLengths.size() - 1, - new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) ); - recordSize = shortrecord; - } - break; - } - } - } - - private void calcReminant() - { - int available = SSTRecord.MAX_RECORD_SIZE - pos; - - if ( stringReminant <= available ) - { - - // write reminant - totalBytesWritten += stringReminant - 1; - pos += stringReminant; - isRemainingString = false; - } - else - { - - // write as much of the remnant as possible - int toBeWritten = unistr.maxBrokenLength( available ); - - if ( available != toBeWritten ) - { - int shortrecord = recordSize - ( available - toBeWritten ); - recordLengths.set( recordLengths.size() - 1, - new Integer( shortrecord - SSTRecord.STD_RECORD_OVERHEAD ) ); - recordSize = shortrecord; - } - totalBytesWritten += toBeWritten - 1; - pos += toBeWritten; - stringReminant -= toBeWritten - 1; - isRemainingString = true; - } - } - - private void initVars() - { - unistr = null; - stringReminant = 0; - unipos = 0; - isRemainingString = false; - totalBytesWritten = 0; - finished = false; - firstRecord = true; - totalWritten = 0; - } - } diff --git a/src/java/org/apache/poi/hssf/record/SSTSerializer.java b/src/java/org/apache/poi/hssf/record/SSTSerializer.java index a8f9f89b73..7f2f94b873 100644 --- a/src/java/org/apache/poi/hssf/record/SSTSerializer.java +++ b/src/java/org/apache/poi/hssf/record/SSTSerializer.java @@ -111,7 +111,7 @@ class SSTSerializer private void serializeSingleSSTRecord( byte[] data, int offset, int record_length_index ) { int len = ( (Integer) recordLengths.get( record_length_index ) ).intValue(); - int recordSize = SSTRecord.SST_RECORD_OVERHEAD + len - SSTRecord.STD_RECORD_OVERHEAD; + int recordSize = len - SSTRecord.STD_RECORD_OVERHEAD; sstRecordHeader.writeSSTHeader( data, 0 + offset, recordSize ); int pos = SSTRecord.SST_RECORD_OVERHEAD; @@ -149,13 +149,16 @@ class SSTSerializer while ( totalWritten != record_size ) { - int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue(); + //Total record length, including excel record header and sst/continue header + final int recordLength = ( (Integer) recordLengths.get( record_length_index++ ) ).intValue(); + //Total available data length (minus the excel record header size) + final int recordDataLength = recordLength - 4; RecordProcessor recordProcessor = new RecordProcessor( buffer, - recordLength, numStrings, numUniqueStrings ); + recordDataLength, numStrings, numUniqueStrings ); // write the appropriate header startOfRecord = offset + totalWritten; - recordProcessor.writeRecordHeader( offset, totalWritten, recordLength, first_record ); + recordProcessor.writeRecordHeader( offset, totalWritten, recordDataLength, first_record ); first_record = false; // now, write the rest of the data into the current @@ -166,6 +169,11 @@ class SSTSerializer // the last string in the previous record was not written out completely stringReminant = recordProcessor.writeStringRemainder( lastneedcontinue, stringReminant, offset, totalWritten ); + //Check to see if still not written out completely + if (lastneedcontinue) { + totalWritten += recordLength; + continue; + } } // last string's remnant, if any, is cleaned up as best as can be done ... now let's try and write @@ -204,7 +212,7 @@ class SSTSerializer break; } } - totalWritten += recordLength + SSTRecord.STD_RECORD_OVERHEAD; + totalWritten += recordLength; } } diff --git a/src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java b/src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java index 58c5debfe9..560db0a655 100755 --- a/src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java +++ b/src/testcases/org/apache/poi/hssf/record/aggregates/TestValueRecordsAggregate.java @@ -161,7 +161,8 @@ public class TestValueRecordsAggregate extends TestCase }; List records = testData(); valueRecord.construct( 0, records ); - int bytesWritten = valueRecord.serialize( 0, actualArray ); + int bytesWritten = valueRecord.serializeCellRow(1, 0, actualArray ); + bytesWritten += valueRecord.serializeCellRow(2, bytesWritten, actualArray ); assertEquals( 36, bytesWritten ); for (int i = 0; i < 36; i++) assertEquals( expectedArray[i], actualArray[i] );