From 064dd6aba288975ca94f73077126081f186cd066 Mon Sep 17 00:00:00 2001 From: wp_xxyyzz Date: Tue, 22 Jul 2014 15:59:29 +0000 Subject: [PATCH] fpspreadsheet: Extend TBufStream for reading. Rename workbook's WritingOptions to Options, and the option flags from woXXXX to boXXXX. Use boBufStream (former woBufStream) to activate TBufStream for reading of xls and ods. Speed up reading of biff2 by factor 3 (rel to commit before prev one). Fix pile-up of temporary files when saving ods. git-svn-id: https://svn.code.sf.net/p/lazarus-ccr/svn@3357 8e941d3f-bd1b-0410-a28a-d453659cc2b4 --- .../examples/fpsSpeedTest/mainform.pas | 114 ++++++++------- .../examples/other/test_virtualmode.lpr | 18 +-- components/fpspreadsheet/fpsopendocument.pas | 31 ++++- components/fpspreadsheet/fpspreadsheet.pas | 65 +++++++-- components/fpspreadsheet/fpsstreams.pas | 130 +++++++++++++----- .../fpspreadsheet/tests/internaltests.pas | 94 ++++++++++++- components/fpspreadsheet/xlsbiff2.pas | 2 +- components/fpspreadsheet/xlsbiff5.pas | 4 +- components/fpspreadsheet/xlsbiff8.pas | 4 +- components/fpspreadsheet/xlsxooxml.pas | 8 +- 10 files changed, 343 insertions(+), 127 deletions(-) diff --git a/components/fpspreadsheet/examples/fpsSpeedTest/mainform.pas b/components/fpspreadsheet/examples/fpsSpeedTest/mainform.pas index 300796719..f54b987cb 100644 --- a/components/fpspreadsheet/examples/fpsSpeedTest/mainform.pas +++ b/components/fpspreadsheet/examples/fpsSpeedTest/mainform.pas @@ -45,10 +45,8 @@ type var AValue: Variant; var AStyleCell: PCell); procedure ReadFromIni; procedure WriteToIni; - procedure RunReadTest(Idx: Integer; Log: String; - Options: TsWorkbookWritingOptions); - procedure RunWriteTest(Idx: integer; Rows: integer; Log: string; - Options: TsWorkbookWritingOptions); + procedure RunReadTest(Idx: Integer; Log: String; Options: TsWorkbookOptions); + procedure RunWriteTest(Idx: integer; Rows: integer; Log: string; Options: TsWorkbookOptions); procedure StatusMsg(const AMsg: String); public { public declarations } @@ -80,6 +78,8 @@ const rc100k = 6; CONTENT_PREFIX: array[0..2] of Char = ('S', 'N', 'M'); + CONTENT_TEXT: array[0..2] of string = ('strings only', 'numbers only', '50% strings and 50% numbers'); + FORMAT_EXT: array[0..4] of String = ('.ods', '.xlsx', '.xls', '_b5.xls', '_b2.xls'); SPREAD_FORMAT: array[0..4] of TsSpreadsheetFormat = (sfOpenDocument, sfOOXML, sfExcel8, sfExcel5, sfExcel2); @@ -116,7 +116,7 @@ begin end; procedure TForm1.RunReadTest(Idx: Integer; Log: String; - Options: TsWorkbookWritingOptions); + Options: TsWorkbookOptions); var MyWorkbook: TsWorkbook; MyWorksheet: TsWorksheet; @@ -127,9 +127,7 @@ var ok: Boolean; begin s := Trim(Log); - Log := Log + ' '; - try for i := 0 to CgFormats.Items.Count-1 do begin if FEscape then begin @@ -162,6 +160,7 @@ begin MyWorkbook := TsWorkbook.Create; try Application.ProcessMessages; + MyWorkbook.Options := Options; Tm := GetTickCount; try MyWorkbook.ReadFromFile(fname, SPREAD_FORMAT[i]); @@ -184,7 +183,7 @@ begin end; procedure TForm1.RunWriteTest(Idx: integer; Rows: integer; Log: string; - Options: TsWorkbookWritingOptions); + Options: TsWorkbookOptions); var MyWorkbook: TsWorkbook; MyWorksheet: TsWorksheet; @@ -201,13 +200,13 @@ begin end; MyWorksheet := MyWorkbook.AddWorksheet('Sheet1'); - MyWorkbook.WritingOptions := Options; + MyWorkbook.Options := Options; Application.ProcessMessages; Tm := GetTickCount; try - if woVirtualMode in Options then + if boVirtualMode in Options then begin MyWorkbook.VirtualRowCount := Rows; MyWorkbook.VirtualColCount := COLCOUNT; @@ -222,7 +221,7 @@ begin for ARow := 0 to Rows - 1 do begin if ARow mod 1000 = 0 then begin - StatusMsg(Format('Populating row %d', [ARow])); + StatusMsg(Format('Building row %d...', [ARow])); if FEscape then begin Log := 'Test aborted'; exit; @@ -236,7 +235,8 @@ begin 1: for ACol := 0 to COLCOUNT-1 do MyWorksheet.WriteNumber(ARow, ACol, 1E5*ARow + ACol); 2: for ACol := 0 to COLCOUNT-1 do - if (odd(ARow) and odd(ACol)) or odd(ARow+ACol) then begin + if (odd(ARow) and odd(ACol)) or odd(ARow+ACol) then + begin S := 'Xy' + IntToStr(ARow) + 'x' + IntToStr(ACol); MyWorksheet.WriteUTF8Text(ARow, ACol, S); end else @@ -255,7 +255,8 @@ begin Log := Log + ' ' + format('%5.1f ', [(GetTickCount - Tm) / 1000]); - for k := 0 to CgFormats.Items.Count-1 do begin + for k := 0 to CgFormats.Items.Count-1 do + begin if FEscape then begin Log := 'Test aborted'; exit; @@ -313,34 +314,30 @@ begin FEscape := false; EnableControls(false); - try - Memo.Append ('Running: Reading TsWorkbook from various file formats'); - case RgContent.ItemIndex of - 0: Memo.Append(' Worksheet contains strings only'); - 1: Memo.Append(' Worksheet contains numbers only'); - 2: Memo.Append(' Worksheet contains 50% strings and 50% numbers'); - end; - Memo.Append (' (Times in seconds)'); - //'----------- .ods .xlsx biff8 biff5 biff2'); - //'Rows x Cols W.Options Build Write Write Write Write Write' - s := '-------------------------------- '; - if CgFormats.Checked[fmtODS] then s := s + ' .ods '; - if CgFormats.Checked[fmtXLSX] then s := s + '.xlsx '; - if CgFormats.Checked[fmtXLS8] then s := s + 'biff8 '; - if CgFormats.Checked[fmtXLS5] then s := s + 'biff5 '; - if CgFormats.Checked[fmtXLS2] then s := s + 'biff2'; - Memo.Append(TrimRight(s)); - s := 'Rows x Cols W.Options '; - if CgFormats.Checked[fmtODS] then s := s + ' Read '; - if CgFormats.Checked[fmtXLSX] then s := s + ' Read '; - if CgFormats.Checked[fmtXLS8] then s := s + ' Read '; - if CgFormats.Checked[fmtXLS5] then s := s + ' Read '; - if CgFormats.Checked[fmtXLS2] then s := s + ' Read'; - s := TrimRight(s); - Memo.Append(s); - len := Length(s); - Memo.Append(DupeString('-', len)); + Memo.Append ('Running: Reading TsWorkbook from various file formats'); + Memo.Append (' Worksheet contains ' + CONTENT_TEXT[RgContent.ItemIndex]); + Memo.Append (' (Times in seconds)'); + //'----------- .ods .xlsx biff8 biff5 biff2'); + //'Rows x Cols Options Build Write Write Write Write Write' + s := '-------------------------------- '; + if CgFormats.Checked[fmtODS] then s := s + ' .ods '; + if CgFormats.Checked[fmtXLSX] then s := s + '.xlsx '; + if CgFormats.Checked[fmtXLS8] then s := s + 'biff8 '; + if CgFormats.Checked[fmtXLS5] then s := s + 'biff5 '; + if CgFormats.Checked[fmtXLS2] then s := s + 'biff2'; + Memo.Append(TrimRight(s)); + s := 'Rows x Cols Options '; + if CgFormats.Checked[fmtODS] then s := s + ' Read '; + if CgFormats.Checked[fmtXLSX] then s := s + ' Read '; + if CgFormats.Checked[fmtXLS8] then s := s + ' Read '; + if CgFormats.Checked[fmtXLS5] then s := s + ' Read '; + if CgFormats.Checked[fmtXLS2] then s := s + ' Read'; + s := TrimRight(s); + Memo.Append(s); + len := Length(s); + Memo.Append(DupeString('-', len)); + try for i:=0 to CgRowCount.Items.Count-1 do begin if FEscape then exit; @@ -351,12 +348,16 @@ begin rows := GetRowCount(i); s := Format('%7.0nx%d', [1.0*rows, COLCOUNT]); - RunReadTest(1, s + ' [ ]', []); - (* - RunReadTest(2, s + ' [woVM ]', [woVirtualMode]); - RunReadTest(3, s + ' [ woBS]', [woBufStream]); - RunReadTest(4, s + ' [woVM, woBS]', [woVirtualMode, woBufStream]); - *) + if CbVirtualModeOnly.Checked then begin + //RunReadTest(2, s + ' [boVM ]', [boVirtualMode]); + //RunReadTest(4, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]); + end else begin + RunReadTest(1, s + ' [ ]', []); + //RunReadTest(2, s + ' [boVM ]', [boVirtualMode]); + RunReadTest(3, s + ' [ boBS]', [boBufStream]); + //RunReadTest(4, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]); + end; + Memo.Append(DupeString('-', len)); end; Memo.Append('Ready'); @@ -378,14 +379,10 @@ begin EnableControls(false); Memo.Append ('Running: Building TsWorkbook and writing to different file formats'); - case RgContent.ItemIndex of - 0: Memo.Append(' Worksheet contains strings only'); - 1: Memo.Append(' Worksheet contains numbers only'); - 2: Memo.Append(' Worksheet contains 50% strings and 50% numbers'); - end; + Memo.Append (' Worksheet contains ' + CONTENT_TEXT[RgContent.ItemIndex]); Memo.Append (' (Times in seconds)'); //'----------- .ods .xlsx biff8 biff5 biff2'); - //'Rows x Cols W.Options Build Write Write Write Write Write' + //'Rows x Cols Options Build Write Write Write Write Write' s := '-------------------------------- '; if CgFormats.Checked[fmtODS] then s := s + ' .ods '; if CgFormats.Checked[fmtXLSX] then s := s + '.xlsx '; @@ -393,7 +390,7 @@ begin if CgFormats.Checked[fmtXLS5] then s := s + 'biff5 '; if CgFormats.Checked[fmtXLS2] then s := s + 'biff2'; Memo.Append(TrimRight(s)); - s := 'Rows x Cols W.Options Build '; + s := 'Rows x Cols Options Build '; if CgFormats.Checked[fmtODS] then s := s + 'Write '; if CgFormats.Checked[fmtXLSX] then s := s + 'Write '; if CgFormats.Checked[fmtXLS8] then s := s + 'Write '; @@ -414,13 +411,13 @@ begin Rows := GetRowCount(i); s := Format('%7.0nx%d', [1.0*Rows, COLCOUNT]); if CbVirtualModeOnly.Checked then begin - RunWriteTest(2, Rows, s + ' [woVM ]', [woVirtualMode]); - RunWriteTest(4, Rows, s + ' [woVM, woBS]', [woVirtualMode, woBufStream]); + RunWriteTest(2, Rows, s + ' [boVM ]', [boVirtualMode]); + RunWriteTest(4, Rows, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]); end else begin RunWriteTest(1, Rows, s + ' [ ]', []); - RunWriteTest(2, Rows, s + ' [woVM ]', [woVirtualMode]); - RunWriteTest(3, Rows, s + ' [ woBS]', [woBufStream]); - RunWriteTest(4, Rows, s + ' [woVM, woBS]', [woVirtualMode, woBufStream]); + RunWriteTest(2, Rows, s + ' [boVM ]', [boVirtualMode]); + RunWriteTest(3, Rows, s + ' [ boBS]', [boBufStream]); + RunWriteTest(4, Rows, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]); end; Memo.Append(DupeString('-', len)); end; @@ -450,6 +447,7 @@ begin CgRowCount.Enabled := AEnable; LblCancel.Visible := not AEnable; StatusMsg(''); + Application.ProcessMessages; end; procedure TForm1.FormCreate(Sender: TObject); diff --git a/components/fpspreadsheet/examples/other/test_virtualmode.lpr b/components/fpspreadsheet/examples/other/test_virtualmode.lpr index ca67c8108..3580c3573 100644 --- a/components/fpspreadsheet/examples/other/test_virtualmode.lpr +++ b/components/fpspreadsheet/examples/other/test_virtualmode.lpr @@ -64,17 +64,17 @@ begin { These are the essential commands to activate virtual mode: } -// workbook.WritingOptions := [woVirtualMode, woBufStream]; - workbook.WritingOptions := [woVirtualMode]; - { woBufStream can be omitted, but is important for large files: it causes + workbook.Options := [boVirtualMode, boBufStream]; +// workbook.Options := [boVirtualMode]; + { boBufStream can be omitted, but is important for large files: it causes writing temporary data to a buffered file stream instead of a pure - memory stream which can overflow memory. The option can slow down the - writing process a bit. } + memory stream which can overflow memory. In cases, the option can slow + down the writing process a bit. } { Next two numbers define the size of virtual spreadsheet. In case of a database, VirtualRowCount is the RecordCount, VirtualColCount the number of fields to be written to the spreadsheet file } - workbook.VirtualRowCount := 20000; + workbook.VirtualRowCount := 5000; workbook.VirtualColCount := 100; { The event handler for OnNeedCellData links the workbook to the method @@ -95,8 +95,10 @@ begin { In case of a database, you would open the dataset before calling this: } t := Now; - //workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true); - workbook.WriteToFile('test_virtual.xls', sfExcel8, true); + workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true); + //workbook.WriteToFile('test_virtual.xls', sfExcel8, true); + //workbook.WriteToFile('test_virtual.xls', sfExcel5, true); + //workbook.WriteToFile('test_virtual.xls', sfExcel2, true); t := Now - t; finally diff --git a/components/fpspreadsheet/fpsopendocument.pas b/components/fpspreadsheet/fpsopendocument.pas index 541e96787..6c9ea71bf 100755 --- a/components/fpspreadsheet/fpsopendocument.pas +++ b/components/fpspreadsheet/fpsopendocument.pas @@ -193,11 +193,11 @@ type implementation uses - StrUtils; + StrUtils, fpsStreams; const { OpenDocument general XML constants } - XML_HEADER = ''; + XML_HEADER = ''; { OpenDocument Directory structure constants } OPENDOC_PATH_CONTENT = 'content.xml'; @@ -1191,8 +1191,23 @@ var parser: TDOMParser; src: TXMLInputSource; stream: TStream; +// fstream: TStream; begin - stream := TFileStream.Create(AFileName, fmOpenRead + fmShareDenyWrite); + { + if (boBufStream in Workbook.Options) then begin + fstream := TFileStream.Create(AFilename, fmOpenRead + fmShareDenyWrite); + stream := TMemorystream.Create; + stream.CopyFrom(fstream, fstream.Size); + stream.Position := 0; + fstream.free; + end + } + + if (boBufStream in Workbook.Options) then + stream := TBufStream.Create(AFileName, fmOpenRead + fmShareDenyWrite) + else + stream := TFileStream.Create(AFileName, fmOpenRead + fmShareDenyWrite); + try parser := TDOMParser.Create; try @@ -2239,7 +2254,14 @@ procedure TsSpreadOpenDocWriter.CreateStreams; var dir: String; begin - if (woBufStream in Workbook.WritingOptions) then begin + if (boBufStream in Workbook.Options) then begin + FSMeta := TBufStream.Create(GetTempFileName('', 'fpsM')); + FSSettings := TBufStream.Create(GetTempFileName('', 'fpsS')); + FSStyles := TBufStream.Create(GetTempFileName('', 'fpsSTY')); + FSContent := TBufStream.Create(GetTempFileName('', 'fpsC')); + FSMimeType := TBufStream.Create(GetTempFileName('', 'fpsMT')); + FSMetaInfManifest := TBufStream.Create(GetTempFileName('', 'fpsMIM')); + { dir := IncludeTrailingPathDelimiter(GetTempDir); FSMeta := TFileStream.Create(GetTempFileName(dir, 'fpsM'), fmCreate+fmOpenRead); FSSettings := TFileStream.Create(GetTempFileName(dir, 'fpsS'), fmCreate+fmOpenRead); @@ -2247,6 +2269,7 @@ begin FSContent := TFileStream.Create(GetTempFileName(dir, 'fpsC'), fmCreate+fmOpenRead); FSMimeType := TFileStream.Create(GetTempFileName(dir, 'fpsMT'), fmCreate+fmOpenRead); FSMetaInfManifest := TFileStream.Create(GetTempFileName(dir, 'fpsMIM'), fmCreate+fmOpenRead); + } end else begin; FSMeta := TMemoryStream.Create; FSSettings := TMemoryStream.Create; diff --git a/components/fpspreadsheet/fpspreadsheet.pas b/components/fpspreadsheet/fpspreadsheet.pas index f05069f3f..bd49d9099 100755 --- a/components/fpspreadsheet/fpspreadsheet.pas +++ b/components/fpspreadsheet/fpspreadsheet.pas @@ -699,22 +699,33 @@ type end; {@@ - Options considered when writing a workbook + Option flags for the workbook - @param woVirtualMode If in virtual mode date are not taken from cells + @param boVirtualMode If in virtual mode date are not taken from cells when a spreadsheet is written to file, but are provided by means of the event OnNeedCellData. - @param woBufStream When this option is set a buffered stream is used - for writing (a memory stream swapping to disk) } - TsWorkbookWritingOption = (woVirtualMode, woBufStream); + @param boBufStream When this option is set a buffered stream is used + for writing (a memory stream swapping to disk) or + reading (a file stream pre-reading chunks of data + to memory) } + TsWorkbookOption = (boVirtualMode, boBufStream); {@@ - Options considered when writing a workbook } - TsWorkbookWritingOptions = set of TsWorkbookWritingOption; + Set of options flags for the workbook } + TsWorkbookOptions = set of TsWorkbookOption; + {@@ + Event fired when writing a file in virtual mode. The event handler has to + pass data ("AValue") and formatting ("AStyleCell") to the writer } TsWorkbookNeedCellDataEvent = procedure(Sender: TObject; ARow, ACol: Cardinal; var AValue: variant; var AStyleCell: PCell) of object; + {@@ + Event fired when reading a file in virtual mode. The event handler has to + process the data provided by the read in the "ADataCell". } + TsWorkbookHaveCellDataEvent = procedure(Sender: TObject; ARow, ACol: Cardinal; + const ADataCell: PCell) of object; + {@@ The workbook contains the worksheets and provides methods for reading from and writing to file. @@ -734,8 +745,9 @@ type FVirtualColCount: Cardinal; FVirtualRowCount: Cardinal; FWriting: Boolean; - FWritingOptions: TsWorkbookWritingOptions; + FOptions: TsWorkbookOptions; FOnNeedCellData: TsWorkbookNeedCellDataEvent; + FOnHaveCellData: TsWorkbookHaveCellDataEvent; FFileName: String; { Setter/Getter } @@ -824,11 +836,15 @@ type property ReadFormulas: Boolean read FReadFormulas write FReadFormulas; property VirtualColCount: cardinal read FVirtualColCount write SetVirtualColCount; property VirtualRowCount: cardinal read FVirtualRowCount write SetVirtualRowCount; - property WritingOptions: TsWorkbookWritingOptions read FWritingOptions write FWritingOptions; + property Options: TsWorkbookOptions read FOptions write FOptions; {@@ This event allows to provide external cell data for writing to file, standard cells are ignored. Intended for converting large database files - to s spreadsheet format. Requires WritingOption woVirtualMode to be set. } + to a spreadsheet format. Requires Option boVirtualMode to be set. } property OnNeedCellData: TsWorkbookNeedCellDataEvent read FOnNeedCellData write FOnNeedCellData; + {@@ This event accepts cell data while reading a spreadsheet file. Data are + not encorporated in a spreadsheet, they are just passed through to the + event handler for processing. Requires Optio boVirtualMode to be set. } + property OnHaveCellData: TsWorkbookHaveCellDataEvent read FOnHaveCellData write FOnHaveCellData; end; {@@ Contents of a number format record } @@ -4188,7 +4204,7 @@ var sheet: TsWorksheet; r1,r2, c1,c2: Cardinal; begin - if (woVirtualMode in WritingOptions) then begin + if (boVirtualMode in Options) then begin ALastRow := FVirtualRowCount - 1; ALastCol := FVirtualColCount - 1; end else begin @@ -5313,10 +5329,29 @@ end; @see TsWorkbook } procedure TsCustomSpreadReader.ReadFromFile(AFileName: string; AData: TsWorkbook); +{ var - InputFile: TFileStream; + fs, ms: TStream; begin - InputFile := TFileStream.Create(AFileName, fmOpenRead); + fs := TFileStream.Create(AFileName, fmOpenRead); + ms := TMemoryStream.Create; + try + ms.CopyFrom(fs, fs.Size); + ms.Position := 0; + ReadFromStream(ms, AData); + finally + ms.Free; + fs.Free; + end; +end; + } +var + InputFile: TStream; +begin + if (boBufStream in Workbook.Options) then + InputFile := TBufStream.Create(AFileName, fmOpenRead) + else + InputFile := TFileStream.Create(AFileName, fmOpenRead); try ReadFromStream(InputFile, AData); finally @@ -5495,7 +5530,7 @@ procedure TsCustomSpreadWriter.GetSheetDimensions(AWorksheet: TsWorksheet; begin AFirstRow := 0; AFirstCol := 0; - if (woVirtualMode in AWorksheet.Workbook.WritingOptions) then begin + if (boVirtualMode in AWorksheet.Workbook.Options) then begin ALastRow := AWorksheet.Workbook.VirtualRowCount-1; ALastCol := AWorksheet.Workbook.VirtualColCount-1; end else begin @@ -5751,7 +5786,7 @@ begin if AOverwriteExisting then lMode := fmCreate or fmOpenWrite else lMode := fmCreate; - if (woBufStream in Workbook.WritingOptions) then + if (boBufStream in Workbook.Options) then OutputFile := TBufStream.Create(AFileName, lMode) else OutputFile := TFileStream.Create(AFileName, lMode); diff --git a/components/fpspreadsheet/fpsstreams.pas b/components/fpspreadsheet/fpsstreams.pas index c458be8a2..793dc3a28 100644 --- a/components/fpspreadsheet/fpsstreams.pas +++ b/components/fpspreadsheet/fpsstreams.pas @@ -5,8 +5,8 @@ interface uses SysUtils, Classes; -const - DEFAULT_STREAM_BUFFER_SIZE = 1024 * 1024; +var + DEFAULT_STREAM_BUFFER_SIZE: Integer = 1024 * 1024; // 1 MB type { A buffered stream } @@ -23,13 +23,15 @@ type procedure CreateFileStream; function GetPosition: Int64; override; function GetSize: Int64; override; + function IsWritingMode: Boolean; public constructor Create(AFileName: String; AMode: Word; - ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload; + ABufSize: Cardinal = Cardinal(-1)); overload; constructor Create(ATempFile: String; AKeepFile: Boolean = false; - ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload; - constructor Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload; + ABufSize: Cardinal = Cardinal(-1)); overload; + constructor Create(ABufSize: Cardinal = Cardinal(-1)); overload; destructor Destroy; override; + procedure FillBuffer; procedure FlushBuffer; function Read(var Buffer; Count: Longint): Longint; override; function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override; @@ -51,18 +53,19 @@ end; {@@ Constructor of the TBufStream. Creates a memory stream and prepares everything - to create also a file stream if the streamsize exceeds ABufSize bytes. + to create also a file stream if the stream size exceeds ABufSize bytes. @param ATempFile File name for the file stream. If an empty string is used a temporary file name is created by calling GetTempFileName. - @param AKeepFile If true the stream is flushed to file when the stream is + @param AKeepFile If true and the stream is in WritingMode the stream is + flushed to file when the stream is destroyed. If false the file is deleted when the stream is destroyed. @param ABufSize Maximum size of the memory stream before swapping to file starts. Value is given in bytes. } constructor TBufStream.Create(ATempFile: String; AKeepFile: Boolean = false; - ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); + ABufSize: Cardinal = Cardinal(-1)); begin if ATempFile = '' then ATempFile := ChangeFileExt(GetTempFileName, '.~abc'); @@ -73,7 +76,10 @@ begin FMemoryStream := TMemoryStream.Create; // The file stream is only created when needed because of possible conflicts // of random file names. - FBufSize := ABufSize; + if ABufSize = Cardinal(-1) then + FBufSize := DEFAULT_STREAM_BUFFER_SIZE + else + FBufSize := ABufSize; FFileMode := fmCreate + fmOpenRead; end; @@ -86,7 +92,7 @@ end; @param ABufSize Maximum size of the memory stream before swapping to file starts. Value is given in bytes. } -constructor TBufStream.Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); +constructor TBufStream.Create(ABufSize: Cardinal = Cardinal(-1)); begin Create('', false, ABufSize); end; @@ -103,9 +109,12 @@ end; starts. Value is given in bytes. } constructor TBufStream.Create(AFileName: String; AMode: Word; - ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); + ABufSize: Cardinal = Cardinal(-1)); +var + keep: Boolean; begin - Create(AFileName, true, ABufSize); + keep := AMode and (fmCreate + fmOpenWrite) <> 0; + Create(AFileName, keep, ABufSize); FFileMode := AMode; end; @@ -117,7 +126,8 @@ begin // Free streams and delete temporary file, if requested FreeAndNil(FMemoryStream); FreeAndNil(FFileStream); - if not FKeepTmpFile and (FFileName <> '') then DeleteFile(FFileName); + if not FKeepTmpFile and (FFileName <> '') and IsWritingMode then + DeleteFile(FFileName); inherited Destroy; end; @@ -133,10 +143,25 @@ begin end; end; +{ Reads FBufSize bytes from the stream into the buffer } +procedure TBufStream.FillBuffer; +var + p, n: Int64; +begin + p := GetPosition; + FMemoryStream.Clear; + FMemoryStream.Position := 0; + FFileStream.Position := p; + n := Min(FBufSize, FFileStream.Size - p); + FMemoryStream.CopyFrom(FFileStream, n); + FMemoryStream.Position := 0; + FFileStream.Position := p; +end; + { Flushes the contents of the memory stream to file } procedure TBufStream.FlushBuffer; begin - if (FMemoryStream.Size > 0) and not FBufWritten then begin + if (FMemoryStream.Size > 0) and not FBufWritten and IsWritingMode then begin FMemoryStream.Position := 0; CreateFileStream; FFileStream.CopyFrom(FMemoryStream, FMemoryStream.Size); @@ -161,12 +186,32 @@ function TBufStream.GetSize: Int64; var n: Int64; begin - if FFileStream <> nil then - n := FFileStream.Size - else - n := 0; - if n = 0 then n := FMemoryStream.Size; - Result := Max(n, GetPosition); + if IsWritingMode then begin + if FFileStream <> nil then + n := FFileStream.Size + else + n := 0; + if n = 0 then n := FMemoryStream.Size; + Result := Max(n, GetPosition); + end else begin + CreateFileStream; + Result := FFileStream.Size; + end; +end; + +{@@ + Returns true if the stream is in WritingMode. + "WritingMode" means that the stream is primarily used for writing. The + memory stream is initially empty but fills during writing, it is written to + disk when it is full. + The (unnamend) opposite of "WritingMode" indicates that the stream is used + for reading. The memory stream is initially full, but the stream pointer is at + it start. When data are read the stream pointer advances towards the end. + When the requested data are not contained in the memory stream another + ABufSize of bytes are read into the memory stream. } +function TBufStream.IsWritingMode: Boolean; +begin + Result := FFileMode and (fmCreate + fmOpenWrite) <> 0; end; {@@ @@ -180,28 +225,39 @@ end; @return Number of bytes that were read from the stream.} function TBufStream.Read(var Buffer; Count: Longint): Longint; begin - // Case 1: All "Count" bytes are contained in memory stream + // Case 1: Memory stream is empty + if FMemoryStream.Size = 0 then begin + CreateFileStream; + if IsWritingMode then begin + Result := FFileStream.Read(Buffer, Count); + end else begin + FillBuffer; + Result := FMemoryStream.Read(Buffer, Count); + end; + exit; + end; + + // Case 2: All "Count" bytes are contained in memory stream if FMemoryStream.Position + Count <= FMemoryStream.Size then begin Result := FMemoryStream.Read(Buffer, Count); exit; end; - // Case 2: Memory stream is empty - if FMemoryStream.Size = 0 then begin - CreateFileStream; - Result := FFileStream.Read(Buffer, Count); - exit; - end; - // Case 3: Memory stream is not empty but contains only part of the bytes requested - FlushBuffer; - Result := FFileStream.Read(Buffer, Count); + if IsWritingMode then begin + FlushBuffer; + Result := FFileStream.Read(Buffer, Count); + end else begin + FillBuffer; + Result := FMemoryStream.Read(Buffer, Count); + end; end; function TBufStream.Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; var oldPos: Int64; newPos: Int64; + n: Int64; begin oldPos := GetPosition; case Origin of @@ -226,8 +282,20 @@ begin end; // case #3: New position is outside buffer - FlushBuffer; + if IsWritingMode then + FlushBuffer; FFileStream.Position := newPos; + FMemoryStream.Position := 0; + if not IsWritingMode then begin + FillBuffer; + { + FMemoryStream.Position := 0; + n := Min(FBufSize, FFileStream.Size - newPos); + FMemoryStream.CopyFrom(FFileStream, n); + FFileStream.Position := newPos; + FMemoryStream.Position := 0; + } + end; end; function TBufStream.Write(const ABuffer; ACount: LongInt): LongInt; diff --git a/components/fpspreadsheet/tests/internaltests.pas b/components/fpspreadsheet/tests/internaltests.pas index a05aa10ea..29db396c8 100644 --- a/components/fpspreadsheet/tests/internaltests.pas +++ b/components/fpspreadsheet/tests/internaltests.pas @@ -51,6 +51,7 @@ type // Write out date cell and try to read as UTF8; verify if contents the same procedure ReadDateAsUTF8; // Test buffered stream + procedure TestReadBufStream; procedure TestBufStream; // Virtual mode tests for all file formats @@ -252,6 +253,95 @@ begin end; end; +procedure TSpreadInternalTests.TestReadBufStream; +const + BUF_SIZE = 1024; + FILE_SIZE = 2000; +var + tempFileName: String; + stream: TStream; + writedata: array of Byte; + readdata: array of Byte; + i, n, nread: Integer; +begin + RandSeed := 0; + + // Create a test file + tempFileName := GetTempFileName; + stream := TFileStream.Create(tempFileName, fmCreate); + try + SetLength(writedata, FILE_SIZE); + for i:=0 to High(writedata) do + writedata[i] := random(256); + stream.WriteBuffer(writedata[0], Length(writedata)); + finally + stream.Free; + end; + + // Use a TBufStream to read parts of the file back + stream := TBufStream.Create(tempFilename, fmOpenRead, BUF_SIZE); + try + // Check stream size + CheckEquals(FILE_SIZE, stream.Size, 'Size mismatch'); + + // Read first 100 bytes and compare with data + nread := 100; + SetLength(readdata, nread); + n := stream.Read(readdata[0], nread); + CheckEquals(nread, n, 'Bytes count mismatch'); + for i:=0 to nread-1 do + CheckEquals(writedata[i], readdata[i], Format('Read mismatch at position %d', [i])); + + // Check stream size + CheckEquals(FILE_SIZE, stream.Size, 'Size mismatch'); + + // Read next 100 bytes and compare + stream.ReadBuffer(readdata[0], nread); + for i:=0 to nread-1 do + CheckEquals(writedata[i+nread], readdata[i], Format('Read mismatch at position %d', [i+nread])); + + // Go to position 1000, this is 24 bytes to the end of the buffer, and read + // 100 bytes again - this process will require to refresh the buffer + stream.Position := 1000; + stream.ReadBuffer(readdata[0], nread); + for i:=0 to nread-1 do + CheckEquals(writedata[i+1000], readdata[i], Format('Read mismatch at position %d', [i+1000])); + + // Check stream size + CheckEquals(FILE_SIZE, stream.Size, 'Size mismatch'); + + // Read next 100 bytes + stream.ReadBuffer(readdata[0], nread); + for i:=0 to nread-1 do + CheckEquals(writedata[i+1000+nread], readdata[i], Format('Read mismatch at position %d', [i+1000+nread])); + + // Go back to start and fill the memory stream again with bytes 0..1023 + stream.Position := 0; + stream.ReadBuffer(readdata[0], nread); + + // Now read 100 bytes which are not in the buffer + stream.Position := 1500; // this is past the buffered range + stream.ReadBuffer(readdata[0], 100); + for i:=0 to nread-1 do + CheckEquals(writedata[i+1500], readdata[i], Format('Read mismatch at position %d', [i+1500])); + + // Go back to start and fill the memory stream again with bytes 0..1023 + stream.Position := 0; + stream.ReadBuffer(readdata[0], 100); + + // Read last 100 bytes + stream.Seek(nread, soFromEnd); + stream.ReadBuffer(readdata[0], nread); + for i:=0 to nread-1 do + CheckEquals(writedata[i+FILE_SIZE-nread], readdata[i], + Format('Read mismatch at position %d', [i+FILE_SIZE-nread])); + + finally + stream.Free; + DeleteFile(tempFileName); + end; +end; + procedure TSpreadInternalTests.TestCellString; var r,c: Cardinal; @@ -307,9 +397,9 @@ begin workbook := TsWorkbook.Create; try worksheet := workbook.AddWorksheet('VirtualMode'); - workbook.WritingOptions := workbook.WritingOptions + [woVirtualMode]; + workbook.Options := workbook.Options + [boVirtualMode]; if ABufStreamMode then - workbook.WritingOptions := workbook.WritingOptions + [woBufStream]; + workbook.Options := workbook.Options + [boBufStream]; workbook.VirtualColCount := 1; workbook.VirtualRowCount := Length(SollNumbers) + 4; // We'll use only the first 4 SollStrings, the others cause trouble due to utf8 and formatting. diff --git a/components/fpspreadsheet/xlsbiff2.pas b/components/fpspreadsheet/xlsbiff2.pas index 873b083d8..e2635c217 100755 --- a/components/fpspreadsheet/xlsbiff2.pas +++ b/components/fpspreadsheet/xlsbiff2.pas @@ -1084,7 +1084,7 @@ begin WriteColWidths(AStream); WriteRows(AStream, sheet); - if (woVirtualMode in Workbook.WritingOptions) then + if (boVirtualMode in Workbook.Options) then WriteVirtualCells(AStream) else begin WriteRows(AStream, sheet); diff --git a/components/fpspreadsheet/xlsbiff5.pas b/components/fpspreadsheet/xlsbiff5.pas index 34c16306b..436853dc4 100755 --- a/components/fpspreadsheet/xlsbiff5.pas +++ b/components/fpspreadsheet/xlsbiff5.pas @@ -347,7 +347,7 @@ var OutputStorage: TOLEStorage; OLEDocument: TOLEDocument; begin - if (woBufStream in Workbook.WritingOptions) then begin + if (boBufStream in Workbook.Options) then begin Stream := TBufStream.Create end else Stream := TMemoryStream.Create; @@ -434,7 +434,7 @@ begin WriteSelection(AStream, sheet, pane); WriteRows(AStream, sheet); - if (woVirtualMode in Workbook.WritingOptions) then + if (boVirtualMode in Workbook.Options) then WriteVirtualCells(AStream) else begin WriteRows(AStream, sheet); diff --git a/components/fpspreadsheet/xlsbiff8.pas b/components/fpspreadsheet/xlsbiff8.pas index b1cf9a654..9e61bdc14 100755 --- a/components/fpspreadsheet/xlsbiff8.pas +++ b/components/fpspreadsheet/xlsbiff8.pas @@ -365,7 +365,7 @@ var OutputStorage: TOLEStorage; OLEDocument: TOLEDocument; begin - if (woBufStream in Workbook.WritingOptions) then begin + if (boBufStream in Workbook.Options) then begin Stream := TBufStream.Create end else Stream := TMemoryStream.Create; @@ -446,7 +446,7 @@ begin WriteDimensions(AStream, sheet); //WriteRowAndCellBlock(AStream, sheet); - if (woVirtualMode in Workbook.WritingOptions) then + if (boVirtualMode in Workbook.Options) then WriteVirtualCells(AStream) else begin WriteRows(AStream, sheet); diff --git a/components/fpspreadsheet/xlsxooxml.pas b/components/fpspreadsheet/xlsxooxml.pas index 437fe6179..b8aef0404 100755 --- a/components/fpspreadsheet/xlsxooxml.pas +++ b/components/fpspreadsheet/xlsxooxml.pas @@ -880,7 +880,7 @@ begin h0 := Workbook.GetDefaultFontSize; // Point size of default font // Create the stream - if (woBufStream in Workbook.WritingOptions) then + if (boBufStream in Workbook.Options) then FSSheets[FCurSheetNum] := TBufStream.Create(GetTempFileName('', Format('fpsSH%d', [FCurSheetNum]))) else FSSheets[FCurSheetNum] := TMemoryStream.Create; @@ -902,7 +902,7 @@ begin AppendToStream(FSSheets[FCurSheetNum], ''); - if (woVirtualMode in Workbook.WritingOptions) and Assigned(Workbook.OnNeedCellData) + if (boVirtualMode in Workbook.Options) and Assigned(Workbook.OnNeedCellData) then begin for r := 0 to Workbook.VirtualRowCount-1 do begin row := CurSheet.FindRow(r); @@ -1012,7 +1012,7 @@ end; single xlsx file. } procedure TsSpreadOOXMLWriter.CreateStreams; begin - if (woBufStream in Workbook.WritingOptions) then begin + if (boBufStream in Workbook.Options) then begin FSContentTypes := TBufStream.Create(GetTempFileName('', 'fpsCT')); FSRelsRels := TBufStream.Create(GetTempFileName('', 'fpsRR')); FSWorkbookRels := TBufStream.Create(GetTempFileName('', 'fpsWBR')); @@ -1111,7 +1111,7 @@ begin then lMode := fmCreate or fmOpenWrite else lMode := fmCreate; - if (woBufStream in Workbook.WritingOptions) then + if (boBufStream in Workbook.Options) then lStream := TBufStream.Create(AFileName, lMode) else lStream := TFileStream.Create(AFileName, lMode);