pascal/lazarus-ccr
1
0
Fork 0
Code Issues Releases Activity

fpspreadsheet: Extend TBufStream for reading. Rename workbook's WritingOptions to Options, and the option flags from woXXXX to boXXXX. Use boBufStream (former woBufStream) to activate TBufStream for reading of xls and ods. Speed up reading of biff2 by factor 3 (rel to commit before prev one). Fix pile-up of temporary files when saving ods.

Browse Source 
git-svn-id: https://svn.code.sf.net/p/lazarus-ccr/svn@3357 8e941d3f-bd1b-0410-a28a-d453659cc2b4
This commit is contained in:
wp_xxyyzz
2014-07-22 15:59:29 +00:00
parent 38bb8b4756
commit 064dd6aba2
10 changed files with 343 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
components/fpspreadsheet/examples/fpsSpeedTest/mainform.pas
View File

@ -45,10 +45,8 @@ type
var AValue: Variant; var AStyleCell: PCell);
procedure ReadFromIni;
procedure WriteToIni;
procedure RunReadTest(Idx: Integer; Log: String;
Options: TsWorkbookWritingOptions);
procedure RunWriteTest(Idx: integer; Rows: integer; Log: string;
Options: TsWorkbookWritingOptions);
procedure RunReadTest(Idx: Integer; Log: String; Options: TsWorkbookOptions);
procedure RunWriteTest(Idx: integer; Rows: integer; Log: string; Options: TsWorkbookOptions);
procedure StatusMsg(const AMsg: String);
public
{ public declarations }
@ -80,6 +78,8 @@ const
rc100k = 6;
CONTENT_PREFIX: array[0..2] of Char = ('S', 'N', 'M');
CONTENT_TEXT: array[0..2] of string = ('strings only', 'numbers only', '50% strings and 50% numbers');
FORMAT_EXT: array[0..4] of String = ('.ods', '.xlsx', '.xls', '_b5.xls', '_b2.xls');
SPREAD_FORMAT: array[0..4] of TsSpreadsheetFormat = (sfOpenDocument, sfOOXML, sfExcel8, sfExcel5, sfExcel2);
@ -116,7 +116,7 @@ begin
end;
procedure TForm1.RunReadTest(Idx: Integer; Log: String;
Options: TsWorkbookWritingOptions);
Options: TsWorkbookOptions);
var
MyWorkbook: TsWorkbook;
MyWorksheet: TsWorksheet;
@ -127,9 +127,7 @@ var
ok: Boolean;
begin
s := Trim(Log);
Log := Log + ' ';
try
for i := 0 to CgFormats.Items.Count-1 do begin
if FEscape then begin
@ -162,6 +160,7 @@ begin
MyWorkbook := TsWorkbook.Create;
try
Application.ProcessMessages;
MyWorkbook.Options := Options;
Tm := GetTickCount;
try
MyWorkbook.ReadFromFile(fname, SPREAD_FORMAT[i]);
@ -184,7 +183,7 @@ begin
end;
procedure TForm1.RunWriteTest(Idx: integer; Rows: integer; Log: string;
Options: TsWorkbookWritingOptions);
Options: TsWorkbookOptions);
var
MyWorkbook: TsWorkbook;
MyWorksheet: TsWorksheet;
@ -201,13 +200,13 @@ begin
end;
MyWorksheet := MyWorkbook.AddWorksheet('Sheet1');
MyWorkbook.WritingOptions := Options;
MyWorkbook.Options := Options;
Application.ProcessMessages;
Tm := GetTickCount;
try
if woVirtualMode in Options then
if boVirtualMode in Options then
begin
MyWorkbook.VirtualRowCount := Rows;
MyWorkbook.VirtualColCount := COLCOUNT;
@ -222,7 +221,7 @@ begin
for ARow := 0 to Rows - 1 do
begin
if ARow mod 1000 = 0 then begin
StatusMsg(Format('Populating row %d', [ARow]));
StatusMsg(Format('Building row %d...', [ARow]));
if FEscape then begin
Log := 'Test aborted';
exit;
@ -236,7 +235,8 @@ begin
1: for ACol := 0 to COLCOUNT-1 do
MyWorksheet.WriteNumber(ARow, ACol, 1E5*ARow + ACol);
2: for ACol := 0 to COLCOUNT-1 do
if (odd(ARow) and odd(ACol)) or odd(ARow+ACol) then begin
if (odd(ARow) and odd(ACol)) or odd(ARow+ACol) then
begin
S := 'Xy' + IntToStr(ARow) + 'x' + IntToStr(ACol);
MyWorksheet.WriteUTF8Text(ARow, ACol, S);
end else
@ -255,7 +255,8 @@ begin
Log := Log + ' ' + format('%5.1f ', [(GetTickCount - Tm) / 1000]);
for k := 0 to CgFormats.Items.Count-1 do begin
for k := 0 to CgFormats.Items.Count-1 do
begin
if FEscape then begin
Log := 'Test aborted';
exit;
@ -313,34 +314,30 @@ begin
FEscape := false;
EnableControls(false);
try
Memo.Append ('Running: Reading TsWorkbook from various file formats');
case RgContent.ItemIndex of
0: Memo.Append(' Worksheet contains strings only');
1: Memo.Append(' Worksheet contains numbers only');
2: Memo.Append(' Worksheet contains 50% strings and 50% numbers');
end;
Memo.Append (' (Times in seconds)');
//'----------- .ods .xlsx biff8 biff5 biff2');
//'Rows x Cols W.Options Build Write Write Write Write Write'
s := '-------------------------------- ';
if CgFormats.Checked[fmtODS] then s := s + ' .ods ';
if CgFormats.Checked[fmtXLSX] then s := s + '.xlsx ';
if CgFormats.Checked[fmtXLS8] then s := s + 'biff8 ';
if CgFormats.Checked[fmtXLS5] then s := s + 'biff5 ';
if CgFormats.Checked[fmtXLS2] then s := s + 'biff2';
Memo.Append(TrimRight(s));
s := 'Rows x Cols W.Options ';
if CgFormats.Checked[fmtODS] then s := s + ' Read ';
if CgFormats.Checked[fmtXLSX] then s := s + ' Read ';
if CgFormats.Checked[fmtXLS8] then s := s + ' Read ';
if CgFormats.Checked[fmtXLS5] then s := s + ' Read ';
if CgFormats.Checked[fmtXLS2] then s := s + ' Read';
s := TrimRight(s);
Memo.Append(s);
len := Length(s);
Memo.Append(DupeString('-', len));
Memo.Append ('Running: Reading TsWorkbook from various file formats');
Memo.Append (' Worksheet contains ' + CONTENT_TEXT[RgContent.ItemIndex]);
Memo.Append (' (Times in seconds)');
//'----------- .ods .xlsx biff8 biff5 biff2');
//'Rows x Cols Options Build Write Write Write Write Write'
s := '-------------------------------- ';
if CgFormats.Checked[fmtODS] then s := s + ' .ods ';
if CgFormats.Checked[fmtXLSX] then s := s + '.xlsx ';
if CgFormats.Checked[fmtXLS8] then s := s + 'biff8 ';
if CgFormats.Checked[fmtXLS5] then s := s + 'biff5 ';
if CgFormats.Checked[fmtXLS2] then s := s + 'biff2';
Memo.Append(TrimRight(s));
s := 'Rows x Cols Options ';
if CgFormats.Checked[fmtODS] then s := s + ' Read ';
if CgFormats.Checked[fmtXLSX] then s := s + ' Read ';
if CgFormats.Checked[fmtXLS8] then s := s + ' Read ';
if CgFormats.Checked[fmtXLS5] then s := s + ' Read ';
if CgFormats.Checked[fmtXLS2] then s := s + ' Read';
s := TrimRight(s);
Memo.Append(s);
len := Length(s);
Memo.Append(DupeString('-', len));
try
for i:=0 to CgRowCount.Items.Count-1 do begin
if FEscape then
exit;
@ -351,12 +348,16 @@ begin
rows := GetRowCount(i);
s := Format('%7.0nx%d', [1.0*rows, COLCOUNT]);
RunReadTest(1, s + ' [ ]', []);
(*
RunReadTest(2, s + ' [woVM ]', [woVirtualMode]);
RunReadTest(3, s + ' [ woBS]', [woBufStream]);
RunReadTest(4, s + ' [woVM, woBS]', [woVirtualMode, woBufStream]);
*)
if CbVirtualModeOnly.Checked then begin
//RunReadTest(2, s + ' [boVM ]', [boVirtualMode]);
//RunReadTest(4, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]);
end else begin
RunReadTest(1, s + ' [ ]', []);
//RunReadTest(2, s + ' [boVM ]', [boVirtualMode]);
RunReadTest(3, s + ' [ boBS]', [boBufStream]);
//RunReadTest(4, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]);
end;
Memo.Append(DupeString('-', len));
end;
Memo.Append('Ready');
@ -378,14 +379,10 @@ begin
EnableControls(false);
Memo.Append ('Running: Building TsWorkbook and writing to different file formats');
case RgContent.ItemIndex of
0: Memo.Append(' Worksheet contains strings only');
1: Memo.Append(' Worksheet contains numbers only');
2: Memo.Append(' Worksheet contains 50% strings and 50% numbers');
end;
Memo.Append (' Worksheet contains ' + CONTENT_TEXT[RgContent.ItemIndex]);
Memo.Append (' (Times in seconds)');
//'----------- .ods .xlsx biff8 biff5 biff2');
//'Rows x Cols W.Options Build Write Write Write Write Write'
//'Rows x Cols Options Build Write Write Write Write Write'
s := '-------------------------------- ';
if CgFormats.Checked[fmtODS] then s := s + ' .ods ';
if CgFormats.Checked[fmtXLSX] then s := s + '.xlsx ';
@ -393,7 +390,7 @@ begin
if CgFormats.Checked[fmtXLS5] then s := s + 'biff5 ';
if CgFormats.Checked[fmtXLS2] then s := s + 'biff2';
Memo.Append(TrimRight(s));
s := 'Rows x Cols W.Options Build ';
s := 'Rows x Cols Options Build ';
if CgFormats.Checked[fmtODS] then s := s + 'Write ';
if CgFormats.Checked[fmtXLSX] then s := s + 'Write ';
if CgFormats.Checked[fmtXLS8] then s := s + 'Write ';
@ -414,13 +411,13 @@ begin
Rows := GetRowCount(i);
s := Format('%7.0nx%d', [1.0*Rows, COLCOUNT]);
if CbVirtualModeOnly.Checked then begin
RunWriteTest(2, Rows, s + ' [woVM ]', [woVirtualMode]);
RunWriteTest(4, Rows, s + ' [woVM, woBS]', [woVirtualMode, woBufStream]);
RunWriteTest(2, Rows, s + ' [boVM ]', [boVirtualMode]);
RunWriteTest(4, Rows, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]);
end else begin
RunWriteTest(1, Rows, s + ' [ ]', []);
RunWriteTest(2, Rows, s + ' [woVM ]', [woVirtualMode]);
RunWriteTest(3, Rows, s + ' [ woBS]', [woBufStream]);
RunWriteTest(4, Rows, s + ' [woVM, woBS]', [woVirtualMode, woBufStream]);
RunWriteTest(2, Rows, s + ' [boVM ]', [boVirtualMode]);
RunWriteTest(3, Rows, s + ' [ boBS]', [boBufStream]);
RunWriteTest(4, Rows, s + ' [boVM, boBS]', [boVirtualMode, boBufStream]);
end;
Memo.Append(DupeString('-', len));
end;
@ -450,6 +447,7 @@ begin
CgRowCount.Enabled := AEnable;
LblCancel.Visible := not AEnable;
StatusMsg('');
Application.ProcessMessages;
end;
procedure TForm1.FormCreate(Sender: TObject);

18
components/fpspreadsheet/examples/other/test_virtualmode.lpr
View File

@ -64,17 +64,17 @@ begin
{ These are the essential commands to activate virtual mode: }
// workbook.WritingOptions := [woVirtualMode, woBufStream];
workbook.WritingOptions := [woVirtualMode];
{ woBufStream can be omitted, but is important for large files: it causes
workbook.Options := [boVirtualMode, boBufStream];
// workbook.Options := [boVirtualMode];
{ boBufStream can be omitted, but is important for large files: it causes
writing temporary data to a buffered file stream instead of a pure
memory stream which can overflow memory. The option can slow down the
writing process a bit. }
memory stream which can overflow memory. In cases, the option can slow
down the writing process a bit. }
{ Next two numbers define the size of virtual spreadsheet.
In case of a database, VirtualRowCount is the RecordCount, VirtualColCount
the number of fields to be written to the spreadsheet file }
workbook.VirtualRowCount := 20000;
workbook.VirtualRowCount := 5000;
workbook.VirtualColCount := 100;
{ The event handler for OnNeedCellData links the workbook to the method
@ -95,8 +95,10 @@ begin
{ In case of a database, you would open the dataset before calling this: }
t := Now;
//workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true);
workbook.WriteToFile('test_virtual.xls', sfExcel8, true);
workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true);
//workbook.WriteToFile('test_virtual.xls', sfExcel8, true);
//workbook.WriteToFile('test_virtual.xls', sfExcel5, true);
//workbook.WriteToFile('test_virtual.xls', sfExcel2, true);
t := Now - t;
finally

31
components/fpspreadsheet/fpsopendocument.pas
View File

@ -193,11 +193,11 @@ type
implementation
uses
StrUtils;
StrUtils, fpsStreams;
const
{ OpenDocument general XML constants }
XML_HEADER = '<?xml version="1.0" encoding="utf-8" ?>';
XML_HEADER = '<?xml version="1.0" encoding="utf-8" ?>';
{ OpenDocument Directory structure constants }
OPENDOC_PATH_CONTENT = 'content.xml';
@ -1191,8 +1191,23 @@ var
parser: TDOMParser;
src: TXMLInputSource;
stream: TStream;
// fstream: TStream;
begin
stream := TFileStream.Create(AFileName, fmOpenRead + fmShareDenyWrite);
{
if (boBufStream in Workbook.Options) then begin
fstream := TFileStream.Create(AFilename, fmOpenRead + fmShareDenyWrite);
stream := TMemorystream.Create;
stream.CopyFrom(fstream, fstream.Size);
stream.Position := 0;
fstream.free;
end
}
if (boBufStream in Workbook.Options) then
stream := TBufStream.Create(AFileName, fmOpenRead + fmShareDenyWrite)
else
stream := TFileStream.Create(AFileName, fmOpenRead + fmShareDenyWrite);
try
parser := TDOMParser.Create;
try
@ -2239,7 +2254,14 @@ procedure TsSpreadOpenDocWriter.CreateStreams;
var
dir: String;
begin
if (woBufStream in Workbook.WritingOptions) then begin
if (boBufStream in Workbook.Options) then begin
FSMeta := TBufStream.Create(GetTempFileName('', 'fpsM'));
FSSettings := TBufStream.Create(GetTempFileName('', 'fpsS'));
FSStyles := TBufStream.Create(GetTempFileName('', 'fpsSTY'));
FSContent := TBufStream.Create(GetTempFileName('', 'fpsC'));
FSMimeType := TBufStream.Create(GetTempFileName('', 'fpsMT'));
FSMetaInfManifest := TBufStream.Create(GetTempFileName('', 'fpsMIM'));
{
dir := IncludeTrailingPathDelimiter(GetTempDir);
FSMeta := TFileStream.Create(GetTempFileName(dir, 'fpsM'), fmCreate+fmOpenRead);
FSSettings := TFileStream.Create(GetTempFileName(dir, 'fpsS'), fmCreate+fmOpenRead);
@ -2247,6 +2269,7 @@ begin
FSContent := TFileStream.Create(GetTempFileName(dir, 'fpsC'), fmCreate+fmOpenRead);
FSMimeType := TFileStream.Create(GetTempFileName(dir, 'fpsMT'), fmCreate+fmOpenRead);
FSMetaInfManifest := TFileStream.Create(GetTempFileName(dir, 'fpsMIM'), fmCreate+fmOpenRead);
}
end else begin;
FSMeta := TMemoryStream.Create;
FSSettings := TMemoryStream.Create;

65
components/fpspreadsheet/fpspreadsheet.pas
View File

@ -699,22 +699,33 @@ type
end;
{@@
Options considered when writing a workbook
Option flags for the workbook
@param woVirtualMode If in virtual mode date are not taken from cells
@param boVirtualMode If in virtual mode date are not taken from cells
when a spreadsheet is written to file, but are
provided by means of the event OnNeedCellData.
@param woBufStream When this option is set a buffered stream is used
for writing (a memory stream swapping to disk) }
TsWorkbookWritingOption = (woVirtualMode, woBufStream);
@param boBufStream When this option is set a buffered stream is used
for writing (a memory stream swapping to disk) or
reading (a file stream pre-reading chunks of data
to memory) }
TsWorkbookOption = (boVirtualMode, boBufStream);
{@@
Options considered when writing a workbook }
TsWorkbookWritingOptions = set of TsWorkbookWritingOption;
Set of options flags for the workbook }
TsWorkbookOptions = set of TsWorkbookOption;
{@@
Event fired when writing a file in virtual mode. The event handler has to
pass data ("AValue") and formatting ("AStyleCell") to the writer }
TsWorkbookNeedCellDataEvent = procedure(Sender: TObject; ARow, ACol: Cardinal;
var AValue: variant; var AStyleCell: PCell) of object;
{@@
Event fired when reading a file in virtual mode. The event handler has to
process the data provided by the read in the "ADataCell". }
TsWorkbookHaveCellDataEvent = procedure(Sender: TObject; ARow, ACol: Cardinal;
const ADataCell: PCell) of object;
{@@
The workbook contains the worksheets and provides methods for reading from
and writing to file.
@ -734,8 +745,9 @@ type
FVirtualColCount: Cardinal;
FVirtualRowCount: Cardinal;
FWriting: Boolean;
FWritingOptions: TsWorkbookWritingOptions;
FOptions: TsWorkbookOptions;
FOnNeedCellData: TsWorkbookNeedCellDataEvent;
FOnHaveCellData: TsWorkbookHaveCellDataEvent;
FFileName: String;
{ Setter/Getter }
@ -824,11 +836,15 @@ type
property ReadFormulas: Boolean read FReadFormulas write FReadFormulas;
property VirtualColCount: cardinal read FVirtualColCount write SetVirtualColCount;
property VirtualRowCount: cardinal read FVirtualRowCount write SetVirtualRowCount;
property WritingOptions: TsWorkbookWritingOptions read FWritingOptions write FWritingOptions;
property Options: TsWorkbookOptions read FOptions write FOptions;
{@@ This event allows to provide external cell data for writing to file,
standard cells are ignored. Intended for converting large database files
to s spreadsheet format. Requires WritingOption woVirtualMode to be set. }
to a spreadsheet format. Requires Option boVirtualMode to be set. }
property OnNeedCellData: TsWorkbookNeedCellDataEvent read FOnNeedCellData write FOnNeedCellData;
{@@ This event accepts cell data while reading a spreadsheet file. Data are
not encorporated in a spreadsheet, they are just passed through to the
event handler for processing. Requires Optio boVirtualMode to be set. }
property OnHaveCellData: TsWorkbookHaveCellDataEvent read FOnHaveCellData write FOnHaveCellData;
end;
{@@ Contents of a number format record }
@ -4188,7 +4204,7 @@ var
sheet: TsWorksheet;
r1,r2, c1,c2: Cardinal;
begin
if (woVirtualMode in WritingOptions) then begin
if (boVirtualMode in Options) then begin
ALastRow := FVirtualRowCount - 1;
ALastCol := FVirtualColCount - 1;
end else begin
@ -5313,10 +5329,29 @@ end;
@see TsWorkbook
}
procedure TsCustomSpreadReader.ReadFromFile(AFileName: string; AData: TsWorkbook);
{
var
InputFile: TFileStream;
fs, ms: TStream;
begin
InputFile := TFileStream.Create(AFileName, fmOpenRead);
fs := TFileStream.Create(AFileName, fmOpenRead);
ms := TMemoryStream.Create;
try
ms.CopyFrom(fs, fs.Size);
ms.Position := 0;
ReadFromStream(ms, AData);
finally
ms.Free;
fs.Free;
end;
end;
}
var
InputFile: TStream;
begin
if (boBufStream in Workbook.Options) then
InputFile := TBufStream.Create(AFileName, fmOpenRead)
else
InputFile := TFileStream.Create(AFileName, fmOpenRead);
try
ReadFromStream(InputFile, AData);
finally
@ -5495,7 +5530,7 @@ procedure TsCustomSpreadWriter.GetSheetDimensions(AWorksheet: TsWorksheet;
begin
AFirstRow := 0;
AFirstCol := 0;
if (woVirtualMode in AWorksheet.Workbook.WritingOptions) then begin
if (boVirtualMode in AWorksheet.Workbook.Options) then begin
ALastRow := AWorksheet.Workbook.VirtualRowCount-1;
ALastCol := AWorksheet.Workbook.VirtualColCount-1;
end else begin
@ -5751,7 +5786,7 @@ begin
if AOverwriteExisting then lMode := fmCreate or fmOpenWrite
else lMode := fmCreate;
if (woBufStream in Workbook.WritingOptions) then
if (boBufStream in Workbook.Options) then
OutputFile := TBufStream.Create(AFileName, lMode)
else
OutputFile := TFileStream.Create(AFileName, lMode);

130
components/fpspreadsheet/fpsstreams.pas
View File

@ -5,8 +5,8 @@ interface
uses
SysUtils, Classes;
const
DEFAULT_STREAM_BUFFER_SIZE = 1024 * 1024;
var
DEFAULT_STREAM_BUFFER_SIZE: Integer = 1024 * 1024; // 1 MB
type
{ A buffered stream }
@ -23,13 +23,15 @@ type
procedure CreateFileStream;
function GetPosition: Int64; override;
function GetSize: Int64; override;
function IsWritingMode: Boolean;
public
constructor Create(AFileName: String; AMode: Word;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
ABufSize: Cardinal = Cardinal(-1)); overload;
constructor Create(ATempFile: String; AKeepFile: Boolean = false;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
constructor Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE); overload;
ABufSize: Cardinal = Cardinal(-1)); overload;
constructor Create(ABufSize: Cardinal = Cardinal(-1)); overload;
destructor Destroy; override;
procedure FillBuffer;
procedure FlushBuffer;
function Read(var Buffer; Count: Longint): Longint; override;
function Seek(const Offset: Int64; Origin: TSeekOrigin): Int64; override;
@ -51,18 +53,19 @@ end;
{@@
Constructor of the TBufStream. Creates a memory stream and prepares everything
to create also a file stream if the streamsize exceeds ABufSize bytes.
to create also a file stream if the stream size exceeds ABufSize bytes.
@param ATempFile File name for the file stream. If an empty string is
used a temporary file name is created by calling GetTempFileName.
@param AKeepFile If true the stream is flushed to file when the stream is
@param AKeepFile If true and the stream is in WritingMode the stream is
flushed to file when the stream is
destroyed. If false the file is deleted when the stream
is destroyed.
@param ABufSize Maximum size of the memory stream before swapping to file
starts. Value is given in bytes.
}
constructor TBufStream.Create(ATempFile: String; AKeepFile: Boolean = false;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
ABufSize: Cardinal = Cardinal(-1));
begin
if ATempFile = '' then
ATempFile := ChangeFileExt(GetTempFileName, '.~abc');
@ -73,7 +76,10 @@ begin
FMemoryStream := TMemoryStream.Create;
// The file stream is only created when needed because of possible conflicts
// of random file names.
FBufSize := ABufSize;
if ABufSize = Cardinal(-1) then
FBufSize := DEFAULT_STREAM_BUFFER_SIZE
else
FBufSize := ABufSize;
FFileMode := fmCreate + fmOpenRead;
end;
@ -86,7 +92,7 @@ end;
@param ABufSize Maximum size of the memory stream before swapping to file
starts. Value is given in bytes.
}
constructor TBufStream.Create(ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
constructor TBufStream.Create(ABufSize: Cardinal = Cardinal(-1));
begin
Create('', false, ABufSize);
end;
@ -103,9 +109,12 @@ end;
starts. Value is given in bytes.
}
constructor TBufStream.Create(AFileName: String; AMode: Word;
ABufSize: Cardinal = DEFAULT_STREAM_BUFFER_SIZE);
ABufSize: Cardinal = Cardinal(-1));
var
keep: Boolean;
begin
Create(AFileName, true, ABufSize);
keep := AMode and (fmCreate + fmOpenWrite) <> 0;
Create(AFileName, keep, ABufSize);
FFileMode := AMode;
end;
@ -117,7 +126,8 @@ begin
// Free streams and delete temporary file, if requested
FreeAndNil(FMemoryStream);
FreeAndNil(FFileStream);
if not FKeepTmpFile and (FFileName <> '') then DeleteFile(FFileName);
if not FKeepTmpFile and (FFileName <> '') and IsWritingMode then
DeleteFile(FFileName);
inherited Destroy;
end;
@ -133,10 +143,25 @@ begin
end;
end;
{ Reads FBufSize bytes from the stream into the buffer }
procedure TBufStream.FillBuffer;
var
p, n: Int64;
begin
p := GetPosition;
FMemoryStream.Clear;
FMemoryStream.Position := 0;
FFileStream.Position := p;
n := Min(FBufSize, FFileStream.Size - p);
FMemoryStream.CopyFrom(FFileStream, n);
FMemoryStream.Position := 0;
FFileStream.Position := p;
end;
{ Flushes the contents of the memory stream to file }
procedure TBufStream.FlushBuffer;
begin
if (FMemoryStream.Size > 0) and not FBufWritten then begin
if (FMemoryStream.Size > 0) and not FBufWritten and IsWritingMode then begin
FMemoryStream.Position := 0;
CreateFileStream;
FFileStream.CopyFrom(FMemoryStream, FMemoryStream.Size);
@ -161,12 +186,32 @@ function TBufStream.GetSize: Int64;
var
n: Int64;
begin
if FFileStream <> nil then
n := FFileStream.Size
else
n := 0;
if n = 0 then n := FMemoryStream.Size;
Result := Max(n, GetPosition);
if IsWritingMode then begin
if FFileStream <> nil then
n := FFileStream.Size
else
n := 0;
if n = 0 then n := FMemoryStream.Size;
Result := Max(n, GetPosition);
end else begin
CreateFileStream;
Result := FFileStream.Size;
end;
end;
{@@
Returns true if the stream is in WritingMode.
"WritingMode" means that the stream is primarily used for writing. The
memory stream is initially empty but fills during writing, it is written to
disk when it is full.
The (unnamend) opposite of "WritingMode" indicates that the stream is used
for reading. The memory stream is initially full, but the stream pointer is at
it start. When data are read the stream pointer advances towards the end.
When the requested data are not contained in the memory stream another
ABufSize of bytes are read into the memory stream. }
function TBufStream.IsWritingMode: Boolean;
begin
Result := FFileMode and (fmCreate + fmOpenWrite) <> 0;
end;
{@@
@ -180,28 +225,39 @@ end;
@return Number of bytes that were read from the stream.}
function TBufStream.Read(var Buffer; Count: Longint): Longint;
begin
// Case 1: All "Count" bytes are contained in memory stream
// Case 1: Memory stream is empty
if FMemoryStream.Size = 0 then begin
CreateFileStream;
if IsWritingMode then begin
Result := FFileStream.Read(Buffer, Count);
end else begin
FillBuffer;
Result := FMemoryStream.Read(Buffer, Count);
end;
exit;
end;
// Case 2: All "Count" bytes are contained in memory stream
if FMemoryStream.Position + Count <= FMemoryStream.Size then begin
Result := FMemoryStream.Read(Buffer, Count);
exit;
end;
// Case 2: Memory stream is empty
if FMemoryStream.Size = 0 then begin
CreateFileStream;
Result := FFileStream.Read(Buffer, Count);
exit;
end;
// Case 3: Memory stream is not empty but contains only part of the bytes requested
FlushBuffer;
Result := FFileStream.Read(Buffer, Count);
if IsWritingMode then begin
FlushBuffer;
Result := FFileStream.Read(Buffer, Count);
end else begin
FillBuffer;
Result := FMemoryStream.Read(Buffer, Count);
end;
end;
function TBufStream.Seek(const Offset: Int64; Origin: TSeekOrigin): Int64;
var
oldPos: Int64;
newPos: Int64;
n: Int64;
begin
oldPos := GetPosition;
case Origin of
@ -226,8 +282,20 @@ begin
end;
// case #3: New position is outside buffer
FlushBuffer;
if IsWritingMode then
FlushBuffer;
FFileStream.Position := newPos;
FMemoryStream.Position := 0;
if not IsWritingMode then begin
FillBuffer;
{
FMemoryStream.Position := 0;
n := Min(FBufSize, FFileStream.Size - newPos);
FMemoryStream.CopyFrom(FFileStream, n);
FFileStream.Position := newPos;
FMemoryStream.Position := 0;
}
end;
end;
function TBufStream.Write(const ABuffer; ACount: LongInt): LongInt;

94
components/fpspreadsheet/tests/internaltests.pas
View File

@ -51,6 +51,7 @@ type
// Write out date cell and try to read as UTF8; verify if contents the same
procedure ReadDateAsUTF8;
// Test buffered stream
procedure TestReadBufStream;
procedure TestBufStream;
// Virtual mode tests for all file formats
@ -252,6 +253,95 @@ begin
end;
end;
procedure TSpreadInternalTests.TestReadBufStream;
const
BUF_SIZE = 1024;
FILE_SIZE = 2000;
var
tempFileName: String;
stream: TStream;
writedata: array of Byte;
readdata: array of Byte;
i, n, nread: Integer;
begin
RandSeed := 0;
// Create a test file
tempFileName := GetTempFileName;
stream := TFileStream.Create(tempFileName, fmCreate);
try
SetLength(writedata, FILE_SIZE);
for i:=0 to High(writedata) do
writedata[i] := random(256);
stream.WriteBuffer(writedata[0], Length(writedata));
finally
stream.Free;
end;
// Use a TBufStream to read parts of the file back
stream := TBufStream.Create(tempFilename, fmOpenRead, BUF_SIZE);
try
// Check stream size
CheckEquals(FILE_SIZE, stream.Size, 'Size mismatch');
// Read first 100 bytes and compare with data
nread := 100;
SetLength(readdata, nread);
n := stream.Read(readdata[0], nread);
CheckEquals(nread, n, 'Bytes count mismatch');
for i:=0 to nread-1 do
CheckEquals(writedata[i], readdata[i], Format('Read mismatch at position %d', [i]));
// Check stream size
CheckEquals(FILE_SIZE, stream.Size, 'Size mismatch');
// Read next 100 bytes and compare
stream.ReadBuffer(readdata[0], nread);
for i:=0 to nread-1 do
CheckEquals(writedata[i+nread], readdata[i], Format('Read mismatch at position %d', [i+nread]));
// Go to position 1000, this is 24 bytes to the end of the buffer, and read
// 100 bytes again - this process will require to refresh the buffer
stream.Position := 1000;
stream.ReadBuffer(readdata[0], nread);
for i:=0 to nread-1 do
CheckEquals(writedata[i+1000], readdata[i], Format('Read mismatch at position %d', [i+1000]));
// Check stream size
CheckEquals(FILE_SIZE, stream.Size, 'Size mismatch');
// Read next 100 bytes
stream.ReadBuffer(readdata[0], nread);
for i:=0 to nread-1 do
CheckEquals(writedata[i+1000+nread], readdata[i], Format('Read mismatch at position %d', [i+1000+nread]));
// Go back to start and fill the memory stream again with bytes 0..1023
stream.Position := 0;
stream.ReadBuffer(readdata[0], nread);
// Now read 100 bytes which are not in the buffer
stream.Position := 1500; // this is past the buffered range
stream.ReadBuffer(readdata[0], 100);
for i:=0 to nread-1 do
CheckEquals(writedata[i+1500], readdata[i], Format('Read mismatch at position %d', [i+1500]));
// Go back to start and fill the memory stream again with bytes 0..1023
stream.Position := 0;
stream.ReadBuffer(readdata[0], 100);
// Read last 100 bytes
stream.Seek(nread, soFromEnd);
stream.ReadBuffer(readdata[0], nread);
for i:=0 to nread-1 do
CheckEquals(writedata[i+FILE_SIZE-nread], readdata[i],
Format('Read mismatch at position %d', [i+FILE_SIZE-nread]));
finally
stream.Free;
DeleteFile(tempFileName);
end;
end;
procedure TSpreadInternalTests.TestCellString;
var
r,c: Cardinal;
@ -307,9 +397,9 @@ begin
workbook := TsWorkbook.Create;
try
worksheet := workbook.AddWorksheet('VirtualMode');
workbook.WritingOptions := workbook.WritingOptions + [woVirtualMode];
workbook.Options := workbook.Options + [boVirtualMode];
if ABufStreamMode then
workbook.WritingOptions := workbook.WritingOptions + [woBufStream];
workbook.Options := workbook.Options + [boBufStream];
workbook.VirtualColCount := 1;
workbook.VirtualRowCount := Length(SollNumbers) + 4;
// We'll use only the first 4 SollStrings, the others cause trouble due to utf8 and formatting.

2
components/fpspreadsheet/xlsbiff2.pas
View File

@ -1084,7 +1084,7 @@ begin
WriteColWidths(AStream);
WriteRows(AStream, sheet);
if (woVirtualMode in Workbook.WritingOptions) then
if (boVirtualMode in Workbook.Options) then
WriteVirtualCells(AStream)
else begin
WriteRows(AStream, sheet);

4
components/fpspreadsheet/xlsbiff5.pas
View File

@ -347,7 +347,7 @@ var
OutputStorage: TOLEStorage;
OLEDocument: TOLEDocument;
begin
if (woBufStream in Workbook.WritingOptions) then begin
if (boBufStream in Workbook.Options) then begin
Stream := TBufStream.Create
end else
Stream := TMemoryStream.Create;
@ -434,7 +434,7 @@ begin
WriteSelection(AStream, sheet, pane);
WriteRows(AStream, sheet);
if (woVirtualMode in Workbook.WritingOptions) then
if (boVirtualMode in Workbook.Options) then
WriteVirtualCells(AStream)
else begin
WriteRows(AStream, sheet);

4
components/fpspreadsheet/xlsbiff8.pas
View File

@ -365,7 +365,7 @@ var
OutputStorage: TOLEStorage;
OLEDocument: TOLEDocument;
begin
if (woBufStream in Workbook.WritingOptions) then begin
if (boBufStream in Workbook.Options) then begin
Stream := TBufStream.Create
end else
Stream := TMemoryStream.Create;
@ -446,7 +446,7 @@ begin
WriteDimensions(AStream, sheet);
//WriteRowAndCellBlock(AStream, sheet);
if (woVirtualMode in Workbook.WritingOptions) then
if (boVirtualMode in Workbook.Options) then
WriteVirtualCells(AStream)
else begin
WriteRows(AStream, sheet);

8
components/fpspreadsheet/xlsxooxml.pas
View File

@ -880,7 +880,7 @@ begin
h0 := Workbook.GetDefaultFontSize; // Point size of default font
// Create the stream
if (woBufStream in Workbook.WritingOptions) then
if (boBufStream in Workbook.Options) then
FSSheets[FCurSheetNum] := TBufStream.Create(GetTempFileName('', Format('fpsSH%d', [FCurSheetNum])))
else
FSSheets[FCurSheetNum] := TMemoryStream.Create;
@ -902,7 +902,7 @@ begin
AppendToStream(FSSheets[FCurSheetNum],
'<sheetData>');
if (woVirtualMode in Workbook.WritingOptions) and Assigned(Workbook.OnNeedCellData)
if (boVirtualMode in Workbook.Options) and Assigned(Workbook.OnNeedCellData)
then begin
for r := 0 to Workbook.VirtualRowCount-1 do begin
row := CurSheet.FindRow(r);
@ -1012,7 +1012,7 @@ end;
single xlsx file. }
procedure TsSpreadOOXMLWriter.CreateStreams;
begin
if (woBufStream in Workbook.WritingOptions) then begin
if (boBufStream in Workbook.Options) then begin
FSContentTypes := TBufStream.Create(GetTempFileName('', 'fpsCT'));
FSRelsRels := TBufStream.Create(GetTempFileName('', 'fpsRR'));
FSWorkbookRels := TBufStream.Create(GetTempFileName('', 'fpsWBR'));
@ -1111,7 +1111,7 @@ begin
then lMode := fmCreate or fmOpenWrite
else lMode := fmCreate;
if (woBufStream in Workbook.WritingOptions) then
if (boBufStream in Workbook.Options) then
lStream := TBufStream.Create(AFileName, lMode)
else
lStream := TFileStream.Create(AFileName, lMode);