diff --git a/components/fpspreadsheet/examples/other/test_virtualmode.lpr b/components/fpspreadsheet/examples/other/test_virtualmode.lpr index 21c19de78..bb8b00d2e 100644 --- a/components/fpspreadsheet/examples/other/test_virtualmode.lpr +++ b/components/fpspreadsheet/examples/other/test_virtualmode.lpr @@ -65,11 +65,12 @@ begin { These are the essential commands to activate virtual mode: } -// workbook.WritingOptions := [woVirtualMode, woSaveMemory]; - workbook.WritingOptions := [woVirtualMode]; - { woSaveMemory can be omitted, but is essential for large files: it causes - writing temporaray data to a file stream instead of a memory stream. - woSaveMemory, however, considerably slows down writing of biff files. } + workbook.WritingOptions := [woVirtualMode, woBufStream]; +// workbook.WritingOptions := [woVirtualMode]; + { woBufStream can be omitted, but is important for large files: it causes + writing temporary data to a buffered file stream instead of a pure + memory stream which can overflow memory. The option can slow down the + writing process a bit. } { Next two numbers define the size of virtual spreadsheet. In case of a database, VirtualRowCount is the RecordCount, VirtualColCount diff --git a/components/fpspreadsheet/fpsopendocument.pas b/components/fpspreadsheet/fpsopendocument.pas index e3ac875f6..2490a2b2a 100755 --- a/components/fpspreadsheet/fpsopendocument.pas +++ b/components/fpspreadsheet/fpsopendocument.pas @@ -2239,7 +2239,7 @@ procedure TsSpreadOpenDocWriter.CreateStreams; var dir: String; begin - if (woSaveMemory in Workbook.WritingOptions) then begin + if (woBufStream in Workbook.WritingOptions) then begin dir := IncludeTrailingPathDelimiter(GetTempDir); FSMeta := TFileStream.Create(GetTempFileName(dir, 'fpsM'), fmCreate+fmOpenRead); FSSettings := TFileStream.Create(GetTempFileName(dir, 'fpsS'), fmCreate+fmOpenRead); diff --git a/components/fpspreadsheet/fpspreadsheet.pas b/components/fpspreadsheet/fpspreadsheet.pas index 4f886e322..d6b3e1837 100755 --- a/components/fpspreadsheet/fpspreadsheet.pas +++ b/components/fpspreadsheet/fpspreadsheet.pas @@ -704,10 +704,9 @@ type @param woVirtualMode If in virtual mode date are not taken from cells when a spreadsheet is written to file, but are provided by means of the event OnNeedCellData. - @param woSaveMemory When this option is set temporary files are not - written to memory streams but to file streams using - temporary files. } - TsWorkbookWritingOption = (woVirtualMode, woSaveMemory); + @param woBufStream When this option is set a buffered stream is used + for writing (a memory stream swapping to disk) } + TsWorkbookWritingOption = (woVirtualMode, woBufStream); {@@ Options considered when writing a workbook } @@ -1076,7 +1075,7 @@ function SameCellBorders(ACell1, ACell2: PCell): Boolean; implementation uses - Math, StrUtils, TypInfo, fpsUtils, fpsNumFormatParser, fpsFunc; + Math, StrUtils, TypInfo, fpsStreams, fpsUtils, fpsNumFormatParser, fpsFunc; { Translatable strings } resourcestring @@ -5702,13 +5701,16 @@ end; procedure TsCustomSpreadWriter.WriteToFile(const AFileName: string; const AOverwriteExisting: Boolean = False); var - OutputFile: TFileStream; + OutputFile: TStream; lMode: Word; begin if AOverwriteExisting then lMode := fmCreate or fmOpenWrite else lMode := fmCreate; - OutputFile := TFileStream.Create(AFileName, lMode); + if (woBufStream in Workbook.WritingOptions) then + OutputFile := TBufStream.Create(AFileName, lMode) + else + OutputFile := TFileStream.Create(AFileName, lMode); try WriteToStream(OutputFile); finally diff --git a/components/fpspreadsheet/fpsstreams.pas b/components/fpspreadsheet/fpsstreams.pas index b796cf26d..c458be8a2 100644 --- a/components/fpspreadsheet/fpsstreams.pas +++ b/components/fpspreadsheet/fpsstreams.pas @@ -6,7 +6,7 @@ uses SysUtils, Classes; const - DEFAULT_STREAM_BUFFER_SIZE = 1024; // * 1024; + DEFAULT_STREAM_BUFFER_SIZE = 1024 * 1024; type { A buffered stream } @@ -18,11 +18,14 @@ type FBufSize: Int64; FKeepTmpFile: Boolean; FFileName: String; + FFileMode: Word; protected procedure CreateFileStream; function GetPosition: Int64; override; function GetSize: Int64; override; public + constructor Create(AFileName: String; AMode: Word; + ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload; constructor Create(ATempFile: String; AKeepFile: Boolean = false; ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload; constructor Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload; @@ -46,7 +49,18 @@ begin AStream.Position := 0; end; +{@@ + Constructor of the TBufStream. Creates a memory stream and prepares everything + to create also a file stream if the streamsize exceeds ABufSize bytes. + @param ATempFile File name for the file stream. If an empty string is + used a temporary file name is created by calling GetTempFileName. + @param AKeepFile If true the stream is flushed to file when the stream is + destroyed. If false the file is deleted when the stream + is destroyed. + @param ABufSize Maximum size of the memory stream before swapping to file + starts. Value is given in bytes. +} constructor TBufStream.Create(ATempFile: String; AKeepFile: Boolean = false; ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); begin @@ -60,17 +74,45 @@ begin // The file stream is only created when needed because of possible conflicts // of random file names. FBufSize := ABufSize; + FFileMode := fmCreate + fmOpenRead; end; +{@@ + Constructor of the TBufStream. Creates a memory stream and prepares everything + to create also a file stream if the streamsize exceeds ABufSize bytes. The + stream created by this constructor is mainly intended to serve a temporary + purpose, it is not stored permanently to file. + + @param ABufSize Maximum size of the memory stream before swapping to file + starts. Value is given in bytes. +} constructor TBufStream.Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); begin Create('', false, ABufSize); end; +{@@ + Constructor of the TBufStream. When swapping to file it will create a file + stream using the given file mode. This kind of BufStream is considered as a + fast replacement of TFileStream. + + @param AFileName File name for the file stream. If an empty string is + used a temporary file name is created by calling GetTempFileName. + @param AMode FileMode for the file stream (fmCreate, fmOpenRead etc.) + @param ABufSize Maximum size of the memory stream before swapping to file + starts. Value is given in bytes. +} +constructor TBufStream.Create(AFileName: String; AMode: Word; + ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); +begin + Create(AFileName, true, ABufSize); + FFileMode := AMode; +end; + destructor TBufStream.Destroy; begin // Write current buffer content to file - FlushBuffer; + if FKeepTmpFile then FlushBuffer; // Free streams and delete temporary file, if requested FreeAndNil(FMemoryStream); @@ -87,7 +129,7 @@ procedure TBufStream.CreateFileStream; begin if FFileStream = nil then begin if FFileName = '' then FFileName := ChangeFileExt(GetTempFileName, '.~abc'); - FFileStream := TFileStream.Create(FFileName, fmCreate + fmOpenRead); + FFileStream := TFileStream.Create(FFileName, FFileMode); end; end; @@ -113,6 +155,8 @@ begin Result := FFileStream.Position + FMemoryStream.Position; end; +{ Returns the size of the stream. Both memory and file streams are considered + if needed. } function TBufStream.GetSize: Int64; var n: Int64; @@ -125,6 +169,15 @@ begin Result := Max(n, GetPosition); end; +{@@ + Reads a given number of bytes into a buffer and return the number of bytes + read. If the bytes are not in the memory stream they are read from the file + stream. + + @param Buffer Buffer into which the bytes are read. Sufficient space must + have been allocated for Count bytes + @param Count Number of bytes to read from the stream + @return Number of bytes that were read from the stream.} function TBufStream.Read(var Buffer; Count: Longint): Longint; begin // Case 1: All "Count" bytes are contained in memory stream diff --git a/components/fpspreadsheet/tests/internaltests.pas b/components/fpspreadsheet/tests/internaltests.pas index 906d8faf9..a05aa10ea 100644 --- a/components/fpspreadsheet/tests/internaltests.pas +++ b/components/fpspreadsheet/tests/internaltests.pas @@ -34,7 +34,7 @@ type // Set up expected values: procedure SetUp; override; procedure TearDown; override; - procedure TestVirtualMode(AFormat: TsSpreadsheetFormat; SaveMemoryMode: Boolean); + procedure TestVirtualMode(AFormat: TsSpreadsheetFormat; ABufStreamMode: Boolean); published // Tests getting Excel style A1 cell locations from row/column based locations. @@ -59,10 +59,10 @@ type procedure TestVirtualMode_BIFF8; procedure TestVirtualMode_OOXML; - procedure TestVirtualMode_BIFF2_SaveMemory; - procedure TestVirtualMode_BIFF5_SaveMemory; - procedure TestVirtualMode_BIFF8_SaveMemory; - procedure TestVirtualMode_OOXML_SaveMemory; + procedure TestVirtualMode_BIFF2_BufStream; + procedure TestVirtualMode_BIFF5_BufStream; + procedure TestVirtualMode_BIFF8_BufStream; + procedure TestVirtualMode_OOXML_BufStream; end; implementation @@ -295,7 +295,7 @@ begin end; procedure TSpreadInternalTests.TestVirtualMode(AFormat: TsSpreadsheetFormat; - SaveMemoryMode: Boolean); + ABufStreamMode: Boolean); var tempFile: String; workbook: TsWorkbook; @@ -308,8 +308,8 @@ begin try worksheet := workbook.AddWorksheet('VirtualMode'); workbook.WritingOptions := workbook.WritingOptions + [woVirtualMode]; - if SaveMemoryMode then - workbook.WritingOptions := workbook.WritingOptions + [woSaveMemory]; + if ABufStreamMode then + workbook.WritingOptions := workbook.WritingOptions + [woBufStream]; workbook.VirtualColCount := 1; workbook.VirtualRowCount := Length(SollNumbers) + 4; // We'll use only the first 4 SollStrings, the others cause trouble due to utf8 and formatting. @@ -368,22 +368,22 @@ begin TestVirtualMode(sfOOXML, false); end; -procedure TSpreadInternalTests.TestVirtualMode_BIFF2_SaveMemory; +procedure TSpreadInternalTests.TestVirtualMode_BIFF2_BufStream; begin TestVirtualMode(sfExcel2, True); end; -procedure TSpreadInternalTests.TestVirtualMode_BIFF5_SaveMemory; +procedure TSpreadInternalTests.TestVirtualMode_BIFF5_BufStream; begin TestVirtualMode(sfExcel5, true); end; -procedure TSpreadInternalTests.TestVirtualMode_BIFF8_SaveMemory; +procedure TSpreadInternalTests.TestVirtualMode_BIFF8_BufStream; begin TestVirtualMode(sfExcel8, true); end; -procedure TSpreadInternalTests.TestVirtualMode_OOXML_SaveMemory; +procedure TSpreadInternalTests.TestVirtualMode_OOXML_BufStream; begin TestVirtualMode(sfOOXML, true); end; diff --git a/components/fpspreadsheet/xlsbiff5.pas b/components/fpspreadsheet/xlsbiff5.pas index 14069d8d7..3334a2a0b 100755 --- a/components/fpspreadsheet/xlsbiff5.pas +++ b/components/fpspreadsheet/xlsbiff5.pas @@ -219,6 +219,9 @@ var implementation +uses + fpsStreams; + const { Excel record IDs } // see: in xlscommon @@ -330,21 +333,25 @@ end; procedure TsSpreadBIFF5Writer.WriteToFile(const AFileName: string; const AOverwriteExisting: Boolean); var - MemStream: TMemoryStream; + Stream: TStream; OutputStorage: TOLEStorage; OLEDocument: TOLEDocument; begin - MemStream := TMemoryStream.Create; + if (woBufStream in Workbook.WritingOptions) then begin + Stream := TBufStream.Create + end else + Stream := TMemoryStream.Create; + OutputStorage := TOLEStorage.Create; try - WriteToStream(MemStream); + WriteToStream(Stream); // Only one stream is necessary for any number of worksheets - OLEDocument.Stream := MemStream; + OLEDocument.Stream := Stream; OutputStorage.WriteOLEFile(AFileName, OLEDocument, AOverwriteExisting); finally - MemStream.Free; + Stream.Free; OutputStorage.Free; end; end; diff --git a/components/fpspreadsheet/xlsbiff8.pas b/components/fpspreadsheet/xlsbiff8.pas index 06d1849a8..b1cf9a654 100755 --- a/components/fpspreadsheet/xlsbiff8.pas +++ b/components/fpspreadsheet/xlsbiff8.pas @@ -365,7 +365,7 @@ var OutputStorage: TOLEStorage; OLEDocument: TOLEDocument; begin - if (woSaveMemory in Workbook.WritingOptions) then begin + if (woBufStream in Workbook.WritingOptions) then begin Stream := TBufStream.Create end else Stream := TMemoryStream.Create; diff --git a/components/fpspreadsheet/xlsxooxml.pas b/components/fpspreadsheet/xlsxooxml.pas index e18eeb87b..25b35c972 100755 --- a/components/fpspreadsheet/xlsxooxml.pas +++ b/components/fpspreadsheet/xlsxooxml.pas @@ -883,7 +883,7 @@ begin h0 := Workbook.GetDefaultFontSize; // Point size of default font // Create the stream - if (woSaveMemory in Workbook.WritingOptions) then + if (woBufStream in Workbook.WritingOptions) then FSSheets[FCurSheetNum] := TBufStream.Create(GetTempFileName('', Format('fpsSH%d', [FCurSheetNum]))) else FSSheets[FCurSheetNum] := TMemoryStream.Create; @@ -1013,7 +1013,7 @@ end; single xlsx file. } procedure TsSpreadOOXMLWriter.CreateStreams; begin - if (woSaveMemory in Workbook.WritingOptions) then begin + if (woBufStream in Workbook.WritingOptions) then begin FSContentTypes := TBufStream.Create(GetTempFileName('', 'fpsCT')); FSRelsRels := TBufStream.Create(GetTempFileName('', 'fpsRR')); FSWorkbookRels := TBufStream.Create(GetTempFileName('', 'fpsWBR')); @@ -1106,14 +1106,17 @@ end; procedure TsSpreadOOXMLWriter.WriteToFile(const AFileName: string; const AOverwriteExisting: Boolean); var - lStream: TFileStream; + lStream: TStream; lMode: word; begin if AOverwriteExisting then lMode := fmCreate or fmOpenWrite else lMode := fmCreate; - lStream:=TFileStream.Create(AFileName, lMode); + if (woBufStream in Workbook.WritingOptions) then + lStream := TBufStream.Create(AFileName, lMode) + else + lStream := TFileStream.Create(AFileName, lMode); try WriteToStream(lStream); finally @@ -1139,7 +1142,7 @@ begin WriteGlobalFiles; WriteContent; - // Stream position must be at beginning, it was moved to end during adding of xml strings. + // Stream positions must be at beginning, they were moved to end during adding of xml strings. ResetStreams; { Now compress the files }