fpspreadsheet: Use TBufStream as general-purpose stream if woBufStream is set in Worksheet.WritingOptions. woBufStream replaces woSaveMemory. Results in a significant speed enhancement for biff2 (64000x100 cells -> 31 sec without, 1.7 sec with woBufStream).

git-svn-id: https://svn.code.sf.net/p/lazarus-ccr/svn@3337 8e941d3f-bd1b-0410-a28a-d453659cc2b4

components/fpspreadsheet/examples/other/test_virtualmode.lpr

@@ -65,11 +65,12 @@ begin
 
       { These are the essential commands to activate virtual mode: }
 
-//      workbook.WritingOptions := [woVirtualMode, woSaveMemory];
+      workbook.WritingOptions := [woVirtualMode, woBufStream];
-      workbook.WritingOptions := [woVirtualMode];
+//      workbook.WritingOptions := [woVirtualMode];
-      { woSaveMemory can be omitted, but is essential for large files: it causes
+      { woBufStream can be omitted, but is important for large files: it causes
-        writing temporaray data to a file stream instead of a memory stream.
+        writing temporary data to a buffered file stream instead of a pure
-        woSaveMemory, however, considerably slows down writing of biff files. }
+        memory stream which can overflow memory. The option can slow down the
+        writing process a bit. }
 
       { Next two numbers define the size of virtual spreadsheet.
         In case of a database, VirtualRowCount is the RecordCount, VirtualColCount

components/fpspreadsheet/fpsopendocument.pas

@@ -2239,7 +2239,7 @@ procedure TsSpreadOpenDocWriter.CreateStreams;
 var
   dir: String;
 begin
-  if (woSaveMemory in Workbook.WritingOptions) then begin
+  if (woBufStream in Workbook.WritingOptions) then begin
     dir := IncludeTrailingPathDelimiter(GetTempDir);
     FSMeta := TFileStream.Create(GetTempFileName(dir, 'fpsM'), fmCreate+fmOpenRead);
     FSSettings := TFileStream.Create(GetTempFileName(dir, 'fpsS'), fmCreate+fmOpenRead);

components/fpspreadsheet/fpspreadsheet.pas

@@ -704,10 +704,9 @@ type
     @param  woVirtualMode   If in virtual mode date are not taken from cells
                             when a spreadsheet is written to file, but are
                             provided by means of the event OnNeedCellData.
-    @param  woSaveMemory    When this option is set temporary files are not
+    @param  woBufStream     When this option is set a buffered stream is used
-                            written to memory streams but to file streams using
+                            for writing (a memory stream swapping to disk) }
-                            temporary files. }
+  TsWorkbookWritingOption = (woVirtualMode, woBufStream);
-  TsWorkbookWritingOption = (woVirtualMode, woSaveMemory);
 
   {@@
     Options considered when writing a workbook }
@@ -1076,7 +1075,7 @@ function SameCellBorders(ACell1, ACell2: PCell): Boolean;
 implementation
 
 uses
-  Math, StrUtils, TypInfo, fpsUtils, fpsNumFormatParser, fpsFunc;
+  Math, StrUtils, TypInfo, fpsStreams, fpsUtils, fpsNumFormatParser, fpsFunc;
 
 { Translatable strings }
 resourcestring
@@ -5702,13 +5701,16 @@ end;
 procedure TsCustomSpreadWriter.WriteToFile(const AFileName: string;
   const AOverwriteExisting: Boolean = False);
 var
-  OutputFile: TFileStream;
+  OutputFile: TStream;
   lMode: Word;
 begin
   if AOverwriteExisting then lMode := fmCreate or fmOpenWrite
   else lMode := fmCreate;
 
-  OutputFile := TFileStream.Create(AFileName, lMode);
+  if (woBufStream in Workbook.WritingOptions) then
+    OutputFile := TBufStream.Create(AFileName, lMode)
+  else
+    OutputFile := TFileStream.Create(AFileName, lMode);
   try
     WriteToStream(OutputFile);
   finally

components/fpspreadsheet/fpsstreams.pas

@@ -6,7 +6,7 @@ uses
   SysUtils, Classes;
 
 const
-  DEFAULT_STREAM_BUFFER_SIZE = 1024; // * 1024;
+  DEFAULT_STREAM_BUFFER_SIZE = 1024 * 1024;
 
 type
   { A buffered stream }
@@ -18,11 +18,14 @@ type
     FBufSize: Int64;
     FKeepTmpFile: Boolean;
     FFileName: String;
+    FFileMode: Word;
   protected
     procedure CreateFileStream;
     function GetPosition: Int64; override;
     function GetSize: Int64; override;
   public
+    constructor Create(AFileName: String; AMode: Word;
+      ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
     constructor Create(ATempFile: String; AKeepFile: Boolean = false;
       ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
     constructor Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
@@ -46,7 +49,18 @@ begin
   AStream.Position := 0;
 end;
 
+{@@
+  Constructor of the TBufStream. Creates a memory stream and prepares everything
+  to create also a file stream if the streamsize exceeds ABufSize bytes.
 
+  @param  ATempFile   File name for the file stream. If an empty string is
+                      used a temporary file name is created by calling GetTempFileName.
+  @param  AKeepFile   If true the stream is flushed to file when the stream is
+                      destroyed. If false the file is deleted when the stream
+                      is destroyed.
+  @param  ABufSize    Maximum size of the memory stream before swapping to file
+                      starts. Value is given in bytes.
+}
 constructor TBufStream.Create(ATempFile: String; AKeepFile: Boolean = false;
   ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
 begin
@@ -60,17 +74,45 @@ begin
   // The file stream is only created when needed because of possible conflicts
   // of random file names.
   FBufSize := ABufSize;
+  FFileMode := fmCreate + fmOpenRead;
 end;
 
+{@@
+  Constructor of the TBufStream. Creates a memory stream and prepares everything
+  to create also a file stream if the streamsize exceeds ABufSize bytes. The
+  stream created by this constructor is mainly intended to serve a temporary
+  purpose, it is not stored permanently to file.
+
+  @param  ABufSize    Maximum size of the memory stream before swapping to file
+                      starts. Value is given in bytes.
+}
 constructor TBufStream.Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
 begin
   Create('', false, ABufSize);
 end;
 
+{@@
+  Constructor of the TBufStream. When swapping to file it will create a file
+  stream using the given file mode. This kind of BufStream is considered as a
+  fast replacement of TFileStream.
+
+  @param  AFileName   File name for the file stream. If an empty string is
+                      used a temporary file name is created by calling GetTempFileName.
+  @param  AMode       FileMode for the file stream (fmCreate, fmOpenRead etc.)
+  @param  ABufSize    Maximum size of the memory stream before swapping to file
+                      starts. Value is given in bytes.
+}
+constructor TBufStream.Create(AFileName: String; AMode: Word;
+  ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
+begin
+  Create(AFileName, true, ABufSize);
+  FFileMode := AMode;
+end;
+
 destructor TBufStream.Destroy;
 begin
   // Write current buffer content to file
-  FlushBuffer;
+  if FKeepTmpFile then FlushBuffer;
 
   // Free streams and delete temporary file, if requested
   FreeAndNil(FMemoryStream);
@@ -87,7 +129,7 @@ procedure TBufStream.CreateFileStream;
 begin
   if FFileStream = nil then begin
     if FFileName = '' then FFileName := ChangeFileExt(GetTempFileName, '.~abc');
-    FFileStream := TFileStream.Create(FFileName, fmCreate + fmOpenRead);
+    FFileStream := TFileStream.Create(FFileName, FFileMode);
   end;
 end;
 
@@ -113,6 +155,8 @@ begin
     Result := FFileStream.Position + FMemoryStream.Position;
 end;
 
+{ Returns the size of the stream. Both memory and file streams are considered
+  if needed. }
 function TBufStream.GetSize: Int64;
 var
   n: Int64;
@@ -125,6 +169,15 @@ begin
   Result := Max(n, GetPosition);
 end;
 
+{@@
+  Reads a given number of bytes into a buffer and return the number of bytes
+  read. If the bytes are not in the memory stream they are read from the file
+  stream.
+
+  @param  Buffer  Buffer into which the bytes are read. Sufficient space must
+                  have been allocated for Count bytes
+  @param  Count   Number of bytes to read from the stream
+  @return Number of bytes that were read from the stream.}
 function TBufStream.Read(var Buffer; Count: Longint): Longint;
 begin
   // Case 1: All "Count" bytes are contained in memory stream

components/fpspreadsheet/tests/internaltests.pas

@@ -34,7 +34,7 @@ type
     // Set up expected values:
     procedure SetUp; override;
     procedure TearDown; override;
-    procedure TestVirtualMode(AFormat: TsSpreadsheetFormat; SaveMemoryMode: Boolean);
+    procedure TestVirtualMode(AFormat: TsSpreadsheetFormat; ABufStreamMode: Boolean);
 
   published
     // Tests getting Excel style A1 cell locations from row/column based locations.
@@ -59,10 +59,10 @@ type
     procedure TestVirtualMode_BIFF8;
     procedure TestVirtualMode_OOXML;
 
-    procedure TestVirtualMode_BIFF2_SaveMemory;
+    procedure TestVirtualMode_BIFF2_BufStream;
-    procedure TestVirtualMode_BIFF5_SaveMemory;
+    procedure TestVirtualMode_BIFF5_BufStream;
-    procedure TestVirtualMode_BIFF8_SaveMemory;
+    procedure TestVirtualMode_BIFF8_BufStream;
-    procedure TestVirtualMode_OOXML_SaveMemory;
+    procedure TestVirtualMode_OOXML_BufStream;
   end;
 
 implementation
@@ -295,7 +295,7 @@ begin
 end;
 
 procedure TSpreadInternalTests.TestVirtualMode(AFormat: TsSpreadsheetFormat;
-  SaveMemoryMode: Boolean);
+  ABufStreamMode: Boolean);
 var
   tempFile: String;
   workbook: TsWorkbook;
@@ -308,8 +308,8 @@ begin
   try
     worksheet := workbook.AddWorksheet('VirtualMode');
     workbook.WritingOptions := workbook.WritingOptions + [woVirtualMode];
-    if SaveMemoryMode then
+    if ABufStreamMode then
-      workbook.WritingOptions := workbook.WritingOptions + [woSaveMemory];
+      workbook.WritingOptions := workbook.WritingOptions + [woBufStream];
     workbook.VirtualColCount := 1;
     workbook.VirtualRowCount := Length(SollNumbers) + 4;
     // We'll use only the first 4 SollStrings, the others cause trouble due to utf8 and formatting.
@@ -368,22 +368,22 @@ begin
   TestVirtualMode(sfOOXML, false);
 end;
 
-procedure TSpreadInternalTests.TestVirtualMode_BIFF2_SaveMemory;
+procedure TSpreadInternalTests.TestVirtualMode_BIFF2_BufStream;
 begin
   TestVirtualMode(sfExcel2, True);
 end;
 
-procedure TSpreadInternalTests.TestVirtualMode_BIFF5_SaveMemory;
+procedure TSpreadInternalTests.TestVirtualMode_BIFF5_BufStream;
 begin
   TestVirtualMode(sfExcel5, true);
 end;
 
-procedure TSpreadInternalTests.TestVirtualMode_BIFF8_SaveMemory;
+procedure TSpreadInternalTests.TestVirtualMode_BIFF8_BufStream;
 begin
   TestVirtualMode(sfExcel8, true);
 end;
 
-procedure TSpreadInternalTests.TestVirtualMode_OOXML_SaveMemory;
+procedure TSpreadInternalTests.TestVirtualMode_OOXML_BufStream;
 begin
   TestVirtualMode(sfOOXML, true);
 end;

components/fpspreadsheet/xlsbiff5.pas

@@ -219,6 +219,9 @@ var
 
 implementation
 
+uses
+  fpsStreams;
+
 const
   { Excel record IDs }
   // see: in xlscommon
@@ -330,21 +333,25 @@ end;
 procedure TsSpreadBIFF5Writer.WriteToFile(const AFileName: string;
   const AOverwriteExisting: Boolean);
 var
-  MemStream: TMemoryStream;
+  Stream: TStream;
   OutputStorage: TOLEStorage;
   OLEDocument: TOLEDocument;
 begin
-  MemStream := TMemoryStream.Create;
+  if (woBufStream in Workbook.WritingOptions) then begin
+    Stream := TBufStream.Create
+  end else
+    Stream := TMemoryStream.Create;
+
   OutputStorage := TOLEStorage.Create;
   try
-    WriteToStream(MemStream);
+    WriteToStream(Stream);
 
     // Only one stream is necessary for any number of worksheets
-    OLEDocument.Stream := MemStream;
+    OLEDocument.Stream := Stream;
 
     OutputStorage.WriteOLEFile(AFileName, OLEDocument, AOverwriteExisting);
   finally
-    MemStream.Free;
+    Stream.Free;
     OutputStorage.Free;
   end;
 end;

components/fpspreadsheet/xlsbiff8.pas

@@ -365,7 +365,7 @@ var
   OutputStorage: TOLEStorage;
   OLEDocument: TOLEDocument;
 begin
-  if (woSaveMemory in Workbook.WritingOptions) then begin
+  if (woBufStream in Workbook.WritingOptions) then begin
     Stream := TBufStream.Create
   end else
     Stream := TMemoryStream.Create;

components/fpspreadsheet/xlsxooxml.pas

@@ -883,7 +883,7 @@ begin
   h0 := Workbook.GetDefaultFontSize;  // Point size of default font
 
   // Create the stream
-  if (woSaveMemory in Workbook.WritingOptions) then
+  if (woBufStream in Workbook.WritingOptions) then
     FSSheets[FCurSheetNum] := TBufStream.Create(GetTempFileName('', Format('fpsSH%d', [FCurSheetNum])))
   else
     FSSheets[FCurSheetNum] := TMemoryStream.Create;
@@ -1013,7 +1013,7 @@ end;
   single xlsx file. }
 procedure TsSpreadOOXMLWriter.CreateStreams;
 begin
-  if (woSaveMemory in Workbook.WritingOptions) then begin
+  if (woBufStream in Workbook.WritingOptions) then begin
     FSContentTypes := TBufStream.Create(GetTempFileName('', 'fpsCT'));
     FSRelsRels := TBufStream.Create(GetTempFileName('', 'fpsRR'));
     FSWorkbookRels := TBufStream.Create(GetTempFileName('', 'fpsWBR'));
@@ -1106,14 +1106,17 @@ end;
 procedure TsSpreadOOXMLWriter.WriteToFile(const AFileName: string;
   const AOverwriteExisting: Boolean);
 var
-  lStream: TFileStream;
+  lStream: TStream;
   lMode: word;
 begin
   if AOverwriteExisting
     then lMode := fmCreate or fmOpenWrite
     else lMode := fmCreate;
 
-  lStream:=TFileStream.Create(AFileName, lMode);
+  if (woBufStream in Workbook.WritingOptions) then
+    lStream := TBufStream.Create(AFileName, lMode)
+  else
+    lStream := TFileStream.Create(AFileName, lMode);
   try
     WriteToStream(lStream);
   finally
@@ -1139,7 +1142,7 @@ begin
   WriteGlobalFiles;
   WriteContent;
 
-  // Stream position must be at beginning, it was moved to end during adding of xml strings.
+  // Stream positions must be at beginning, they were moved to end during adding of xml strings.
   ResetStreams;
 
   { Now compress the files }