fpspreadsheet: Implement virtual mode for all biff file formats. Add record limitations with writer-specific max row and col counts: a file is not written to a particular file format if it exceeds the limitations of the file format.

git-svn-id: https://svn.code.sf.net/p/lazarus-ccr/svn@3307 8e941d3f-bd1b-0410-a28a-d453659cc2b4

components/fpspreadsheet/examples/other/test_virtualmode.lpi

@@ -63,7 +63,6 @@
       <Unit0>
         <Filename Value="test_virtualmode.lpr"/>
         <IsPartOfProject Value="True"/>
-        <UnitName Value="test_virtualmode"/>
       </Unit0>
     </Units>
   </ProjectOptions>

components/fpspreadsheet/examples/other/test_virtualmode.lpr

@@ -8,7 +8,7 @@ uses
   {$ENDIF}{$ENDIF}
   Classes, laz_fpspreadsheet,
   { you can add units after this }
-  SysUtils, variants, fpspreadsheet, xlsxooxml;
+  SysUtils, variants, fpspreadsheet, xlsbiff2, xlsbiff5, xlsbiff8, xlsxooxml;
 
 type
   TDataProvider = class
@@ -35,7 +35,7 @@ type
       AData := 10000*ARow + ACol;
 
     // you can use the OnNeedData also to provide feedback on how the process
-    // progresses.
+    // progresses:
     if (ACol = 0) and (ARow mod 1000 = 0) then
       WriteLn('Writing row ', ARow, '...');
   end;
@@ -52,22 +52,30 @@ begin
     workbook := TsWorkbook.Create;
     try
       worksheet := workbook.AddWorksheet('Sheet1');
+      worksheet.WriteFontStyle(0, 1, [fssBold]);
 
       { These are the essential commands to activate virtual mode: }
-      workbook.WritingOptions := [woVirtualMode, woSaveMemory];
+
+//      workbook.WritingOptions := [woVirtualMode, woSaveMemory];
+      workbook.WritingOptions := [woVirtualMode];
+
         // woSaveMemory can be omitted, but is essential for large files: it causes
-        // writing temporaray data to a file stream instead of to a memory stream.
-      workbook.VirtualRowCount := 10000;
+        // writing temporaray data to a file stream instead of a memory stream.
+        // woSaveMemory, however, considerably slows down writing of biff files.
+
+      workbook.VirtualRowCount := 1000;
       workbook.VirtualColCount := 100;
         // These two numbers define the size of virtual spreadsheet.
         // In case of a database, VirtualRowCount is the RecordCount, VirtualColCount
         // the number of fields to be written to the spreadsheet file
+
       workbook.OnNeedCellData := @dataprovider.NeedCellData;
         // This links the worksheet to the method from which it gets the
         // data to write.
 
       // In case of a database, you would open the dataset before calling this:
-      workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true);
+//      workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true);
+      workbook.WriteToFile('test_virtual.xls', sfExcel5, true);
 
     finally
       workbook.Free;

components/fpspreadsheet/fpolebasic.pas

@@ -23,7 +23,8 @@ type
 
   TOLEDocument = record
     // Information about the document
-    Stream: TMemoryStream;
+    Stream: TStream;
+//    Stream: TMemoryStream;
   end;
 
 
@@ -57,7 +58,7 @@ var
 begin
   VLAbsolutePath:='/'+AStreamName; //Virtual layer always use absolute paths.
   if not AOverwriteExisting and FileExists(AFileName) then begin
-      Raise EStreamError.Createfmt('File already exists "%s"',[AFileName]);
+      Raise EStreamError.Createfmt('File "%s" already exists.',[AFileName]);
   end;
   RealFile:=TFileStream.Create(AFileName,fmCreate);
   fsOLE:=TVirtualLayer_OLE.Create(RealFile);
@@ -101,7 +102,7 @@ begin
           if not Assigned(AOLEDocument.Stream) then begin
             AOLEDocument.Stream:=TMemoryStream.Create;
           end else begin
-            AOLEDocument.Stream.Clear;
+            (AOLEDocument.Stream as TMemoryStream).Clear;
           end;
           AOLEDocument.Stream.CopyFrom(OLEStream,OLEStream.Size);
         end;

components/fpspreadsheet/fpsopendocument.pas

@@ -2888,6 +2888,10 @@ begin
 
   FPointSeparatorSettings := SysUtils.DefaultFormatSettings;
   FPointSeparatorSettings.DecimalSeparator:='.';
+
+  // http://en.wikipedia.org/wiki/List_of_spreadsheet_software#Specifications
+  FLimitations.MaxCols := 1024;
+  FLimitations.MaxRows := 1048576;
 end;
 
 destructor TsSpreadOpenDocWriter.Destroy;

components/fpspreadsheet/fpspreadsheet.pas

@@ -21,6 +21,12 @@ type
   TsSpreadsheetFormat = (sfExcel2, {sfExcel3, sfExcel4,} sfExcel5, sfExcel8,
    sfOOXML, sfOpenDocument, sfCSV, sfWikiTable_Pipes, sfWikiTable_WikiMedia);
 
+  {@@ Record collection limitations of a particular file format }
+  TsSpreadsheetFormatLimitations = record
+    MaxRows: Cardinal;
+    MaxCols: Cardinal;
+  end;
+
 const
   { Default extensions }
   STR_EXCEL_EXTENSION = '.xls';
@@ -738,6 +744,7 @@ type
     procedure SetVirtualRowCount(AValue: Cardinal);
 
     { Internal methods }
+    procedure GetLastRowColIndex(out ALastRow, ALastCol: Cardinal);
     procedure PrepareBeforeSaving;
     procedure RemoveWorksheetsCallback(data, arg: pointer);
     procedure UpdateCaches;
@@ -954,14 +961,19 @@ type
     FWorkbook: TsWorkbook;
 
   protected
+    {@@ Limitations for the specific data file format }
+    FLimitations: TsSpreadsheetFormatLimitations;
     {@@ List of number formats found in the workbook. }
     FNumFormatList: TsCustomNumFormatList;
     { Helper routines }
     procedure AddDefaultFormats(); virtual;
+    procedure CheckLimitations;
     procedure CreateNumFormatList; virtual;
     function  ExpandFormula(AFormula: TsFormula): TsExpandedFormula;
     function  FindFormattingInList(AFormat: PCell): Integer;
     procedure FixFormat(ACell: PCell); virtual;
+    procedure GetSheetDimensions(AWorksheet: TsWorksheet;
+      out AFirstRow, ALastRow, AFirstCol, ALastCol: Cardinal); virtual;
     procedure ListAllFormattingStylesCallback(ACell: PCell; AStream: TStream);
     procedure ListAllFormattingStyles; virtual;
     procedure ListAllNumFormatsCallback(ACell: PCell; AStream: TStream);
@@ -992,6 +1004,7 @@ type
     NextXFIndex: Integer;
     constructor Create(AWorkbook: TsWorkbook); virtual; // To allow descendents to override it
     destructor Destroy; override;
+    function Limitations: TsSpreadsheetFormatLimitations;
     { General writing methods }
     procedure IterateThroughCells(AStream: TStream; ACells: TAVLTree; ACallback: TCellsCallback);
     procedure WriteToFile(const AFileName: string; const AOverwriteExisting: Boolean = False); virtual;
@@ -1070,6 +1083,8 @@ resourcestring
   lpUnsupportedWriteFormat = 'Tried to write a spreadsheet using an unsupported format';
   lpNoValidSpreadsheetFile = '"%s" is not a valid spreadsheet file';
   lpUnknownSpreadsheetFormat = 'unknown format';
+  lpMaxRowsExceeded = 'This workbook contains %d rows, but the selected file format does not support more than %d rows.';
+  lpMaxColsExceeded = 'This workbook contains %d columns, but the selected file format does not support more than %d columns.';
   lpInvalidFontIndex = 'Invalid font index';
   lpInvalidNumberFormat = 'Trying to use an incompatible number format.';
   lpInvalidDateTimeFormat = 'Trying to use an incompatible date/time format.';
@@ -4084,6 +4099,30 @@ begin
   if Result = nil then raise Exception.Create(lpUnsupportedWriteFormat);
 end;
 
+{@@
+  Determines the maximum index of used columns and rows in all sheets of this
+  workbook. Respects VirtualMode.
+  Is needed to disable saving when limitations of the format is exceeded. }
+procedure TsWorkbook.GetLastRowColIndex(out ALastRow, ALastCol: Cardinal);
+var
+  i: Integer;
+  sheet: TsWorksheet;
+  r1,r2, c1,c2: Cardinal;
+begin
+  if (woVirtualMode in WritingOptions) then begin
+    ALastRow := FVirtualRowCount - 1;
+    ALastCol := FVirtualColCount - 1;
+  end else begin
+    ALastRow := 0;
+    ALastCol := 0;
+    for i:=0 to GetWorksheetCount-1 do begin
+      sheet := GetWorksheetByIndex(i);
+      ALastRow := Max(ALastRow, sheet.GetLastRowIndex);
+      ALastCol := Max(ALastCol, sheet.GetLastColIndex);
+    end;
+  end;
+end;
+
 {@@
   Reads the document from a file. It is assumed to have a given file format.
 
@@ -4221,6 +4260,7 @@ begin
   AWriter := CreateSpreadWriter(AFormat);
   try
     FFileName := AFileName;
+    AWriter.CheckLimitations;
     FWriting := true;
     PrepareBeforeSaving;
     AWriter.WriteToFile(AFileName, AOverwriteExisting);
@@ -4263,6 +4303,7 @@ var
 begin
   AWriter := CreateSpreadWriter(AFormat);
   try
+    AWriter.CheckLimitations;
     FWriting := true;
     PrepareBeforeSaving;
     AWriter.WriteToStream(AStream);
@@ -5260,6 +5301,10 @@ begin
   inherited Create;
   FWorkbook := AWorkbook;
   CreateNumFormatList;
+  { A good starting point valid for many formats... }
+  FLimitations.MaxCols := 256;
+  FLimitations.MaxRows :=  65536;
+
 //  FNumFormatList.FWorkbook := AWorkbook;
 end;
 
@@ -5347,6 +5392,39 @@ begin
   // to be overridden
 end;
 
+{@@
+  Returns a record containing limitations of the specific file format of the
+  writer.
+}
+function TsCustomSpreadWriter.Limitations: TsSpreadsheetFormatLimitations;
+begin
+  Result := FLimitations;
+end;
+
+{@@
+  Determines the size of the worksheet to be written. VirtualMode is respected.
+  Is called when the writer needs the size for output.
+
+  @param   AWorksheet  Worksheet to be written
+  @param   AFirsRow    Index of first row to be written
+  @param   ALastRow    Index of last row
+  @param   AFirstCol   Index of first column to be written
+  @param   ALastCol    Index of last column to be written
+}
+procedure TsCustomSpreadWriter.GetSheetDimensions(AWorksheet: TsWorksheet;
+  out AFirstRow, ALastRow, AFirstCol, ALastCol: Cardinal);
+begin
+  AFirstRow := 0;
+  AFirstCol := 0;
+  if (woVirtualMode in AWorksheet.Workbook.WritingOptions) then begin
+    ALastRow := AWorksheet.Workbook.VirtualRowCount-1;
+    ALastCol := AWorksheet.Workbook.VirtualColCount-1;
+  end else begin
+    ALastRow := AWorksheet.GetLastRowIndex;
+    ALastCol := AWorksheet.GetLastColIndex;
+  end;
+end;
+
 {@@
   Each descendent should define its own default formats, if any.
   Always add the normal, unformatted style first to speed things up.
@@ -5359,6 +5437,20 @@ begin
   NextXFIndex := 0;
 end;
 
+{@@
+  Checks limitations of the writer, e.g max row/column count
+}
+procedure TsCustomSpreadWriter.CheckLimitations;
+var
+  lastCol, lastRow: Cardinal;
+begin
+  Workbook.GetLastRowColIndex(lastRow, lastCol);
+  if lastRow >= FLimitations.MaxRows then
+    raise Exception.CreateFmt(lpMaxRowsExceeded, [lastRow+1, FLimitations.MaxRows]);
+  if lastCol >= FLimitations.MaxCols then
+    raise Exception.CreateFmt(lpMaxColsExceeded, [lastCol+1, FLimitations.MaxCols]);
+end;
+
 {@@
   Creates an instance of the number format list which contains prototypes of
   all number formats found in the workbook.

components/fpspreadsheet/xlsbiff2.pas

@@ -981,7 +981,13 @@ begin
     WriteXFRecords(AStream);
     WriteColWidths(AStream);
     WriteRows(AStream, sheet);
-    WriteCellsToStream(AStream, sheet.Cells);
+
+    if (woVirtualMode in Workbook.WritingOptions) then
+      WriteVirtualCells(AStream)
+    else begin
+      WriteRows(AStream, sheet);
+      WriteCellsToStream(AStream, sheet.Cells);
+    end;
 
     WriteWindow1(AStream);
     //  { -- currently not working

components/fpspreadsheet/xlsbiff5.pas

@@ -407,6 +407,7 @@ begin
     AStream.Position := CurrentPos;
 
     WriteBOF(AStream, INT_BOF_SHEET);
+
       WriteIndex(AStream);
 //      WritePageSetup(AStream);
       WriteColInfos(AStream, sheet);
@@ -415,7 +416,14 @@ begin
       WritePane(AStream, sheet, true, pane);  // true for "is BIFF5 or BIFF8"
       WriteSelection(AStream, sheet, pane);
       WriteRows(AStream, sheet);
-      WriteCellsToStream(AStream, sheet.Cells);
+
+      if (woVirtualMode in Workbook.WritingOptions) then
+        WriteVirtualCells(AStream)
+      else begin
+        WriteRows(AStream, sheet);
+        WriteCellsToStream(AStream, sheet.Cells);
+      end;
+
     WriteEOF(AStream);
   end;
   

components/fpspreadsheet/xlsbiff8.pas

@@ -358,21 +358,27 @@ end;
 procedure TsSpreadBIFF8Writer.WriteToFile(const AFileName: string;
   const AOverwriteExisting: Boolean);
 var
-  MemStream: TMemoryStream;
+  Stream: TStream;
   OutputStorage: TOLEStorage;
   OLEDocument: TOLEDocument;
+  fn: String;
 begin
-  MemStream := TMemoryStream.Create;
+  if (woSaveMemory in Workbook.WritingOptions) then begin
+    fn := GetTempFileName;
+    Stream := TFileStream.Create(fn, fmCreate + fmOpenRead)
+  end else
+    Stream := TMemoryStream.Create;
+
   OutputStorage := TOLEStorage.Create;
   try
-    WriteToStream(MemStream);
+    WriteToStream(Stream);
 
     // Only one stream is necessary for any number of worksheets
-    OLEDocument.Stream := MemStream;
+    OLEDocument.Stream := Stream;
 
     OutputStorage.WriteOLEFile(AFileName, OLEDocument, AOverwriteExisting, 'Workbook');
   finally
-    MemStream.Free;
+    Stream.Free;
     OutputStorage.Free;
   end;
 end;
@@ -439,8 +445,12 @@ begin
       WriteDimensions(AStream, sheet);
       //WriteRowAndCellBlock(AStream, sheet);
 
-      WriteRows(AStream, sheet);
-      WriteCellsToStream(AStream, sheet.Cells);
+      if (woVirtualMode in Workbook.WritingOptions) then
+        WriteVirtualCells(AStream)
+      else begin
+        WriteRows(AStream, sheet);
+        WriteCellsToStream(AStream, sheet.Cells);
+      end;
 
       WriteWindow2(AStream, sheet);
       WritePane(AStream, sheet, isBIFF8, pane);
@@ -545,26 +555,26 @@ end;
 }
 procedure TsSpreadBIFF8Writer.WriteDimensions(AStream: TStream; AWorksheet: TsWorksheet);
 var
-  lLastCol: Word;
-  lLastRow: Integer;
+  firstRow, lastRow, firstCol, lastCol: Cardinal;
 begin
   { BIFF Record header }
   AStream.WriteWord(WordToLE(INT_EXCEL_ID_DIMENSIONS));
   AStream.WriteWord(WordToLE(14));
 
+  { Determine sheet size }
+  GetSheetDimensions(AWorksheet, firstRow, lastRow, firstCol, lastCol);
+
   { Index to first used row }
-  AStream.WriteDWord(DWordToLE(0));
+  AStream.WriteDWord(DWordToLE(firstRow));
 
   { Index to last used row, increased by 1 }
-  lLastRow := GetLastRowIndex(AWorksheet)+1;
-  AStream.WriteDWord(DWordToLE(lLastRow)); // Old dummy value: 33
+  AStream.WriteDWord(DWordToLE(lastRow+1));
 
   { Index to first used column }
-  AStream.WriteWord(WordToLE(0));
+  AStream.WriteWord(WordToLE(firstCol));
 
   { Index to last used column, increased by 1 }
-  lLastCol := GetLastColIndex(AWorksheet)+1;
-  AStream.WriteWord(WordToLE(lLastCol)); // Old dummy value: 10
+  AStream.WriteWord(WordToLE(lastCol+1));
 
   { Not used }
   AStream.WriteWord(WordToLE(0));

components/fpspreadsheet/xlscommon.pas

@@ -508,6 +508,8 @@ type
     procedure WriteWindow1(AStream: TStream); virtual;
     // Writes the index of the XF record used in the given cell
     procedure WriteXFIndex(AStream: TStream; ACell: PCell);
+    // Writes cell content received by workbook in OnNeedCellData event
+    procedure WriteVirtualCells(AStream: TStream);
 
   public
     constructor Create(AWorkbook: TsWorkbook); override;
@@ -518,7 +520,7 @@ type
 implementation
 
 uses
-  fpsNumFormatParser;
+  Variants, fpsNumFormatParser;
 
 { Helper table for rpn formulas:
   Assignment of FormulaElementKinds (fekXXXX) to EXCEL_TOKEN IDs. }
@@ -2526,5 +2528,45 @@ begin
   AStream.WriteWord(WordToLE(lXFIndex));
 end;
 
+procedure TsSpreadBIFFWriter.WriteVirtualCells(AStream: TStream);
+var
+  r,c: Cardinal;
+  lCell: TCell;
+  value: variant;
+begin
+  FillChar(lCell, SizeOf(lCell), 0);
+  for r := 0 to Workbook.VirtualRowCount-1 do begin
+    for c := 0 to Workbook.VirtualColCount-1 do begin
+      value := varNull;
+      Workbook.OnNeedCellData(Workbook, r, c, value);
+      lCell.Row := r;
+      lCell.Col := c;
+      if VarIsNull(value) then
+        lCell.ContentType := cctEmpty
+      else
+      if VarIsNumeric(value) then begin
+        lCell.ContentType := cctNumber;
+        lCell.NumberValue := value;
+      end else
+      {
+      if VarIsDateTime(value) then begin
+        lCell.ContentType := cctNumber;
+        lCell.DateTimeValue := value;
+      end else
+      }
+      if VarIsStr(value) then begin
+        lCell.ContentType := cctUTF8String;
+        lCell.UTF8StringValue := VarToStrDef(value, '');
+      end else
+      if VarIsBool(value) then begin
+        lCell.ContentType := cctBool;
+        lCell.BoolValue := value <> 0;
+      end else
+        lCell.ContentType := cctEmpty;
+      WriteCellCallback(@lCell, AStream);
+    end;
+  end;
+end;
+
 end.
 

components/fpspreadsheet/xlsxooxml.pas

@@ -58,6 +58,7 @@ type
   { TsSpreadOOXMLWriter }
 
   TsSpreadOOXMLWriter = class(TsCustomSpreadWriter)
+  private
   protected
     FPointSeparatorSettings: TFormatSettings;
     FSharedStringsCount: Integer;
@@ -470,6 +471,10 @@ begin
   inherited Create(AWorkbook);
   FPointSeparatorSettings := DefaultFormatSettings;
   FPointSeparatorSettings.DecimalSeparator := '.';
+
+  // http://en.wikipedia.org/wiki/List_of_spreadsheet_software#Specifications
+  FLimitations.MaxCols := 16384;
+  FLimitations.MaxRows := 1048576;
 end;
 
 procedure TsSpreadOOXMLWriter.CreateNumFormatList;
@@ -486,13 +491,13 @@ var
 begin
   if (woSaveMemory in Workbook.WritingOptions) then begin
     dir := IncludeTrailingPathDelimiter(GetTempDir);
-    FSContentTypes := TFileStream.Create(GetTempFileName(dir, 'fpsCT'), fmCreate);
-    FSRelsRels := TFileStream.Create(GetTempFileName(dir, 'fpsRR'), fmCreate);
-    FSWorkbookRels := TFileStream.Create(GetTempFileName(dir, 'fpsWBR'), fmCreate);
-    FSWorkbook := TFileStream.Create(GetTempFileName(dir, 'fpsWB'), fmCreate);
-    FSStyles := TFileStream.Create(GetTempFileName(dir, 'fpsSTY'), fmCreate);
-    FSSharedStrings := TFileStream.Create(GetTempFileName(dir, 'fpsSST'), fmCreate);
-    FSSharedStrings_complete := TFileStream.Create(GetTempFileName(dir, 'fpsSSTc'), fmCreate);
+    FSContentTypes := TFileStream.Create(GetTempFileName(dir, 'fpsCT'), fmCreate+fmOpenRead);
+    FSRelsRels := TFileStream.Create(GetTempFileName(dir, 'fpsRR'), fmCreate+fmOpenRead);
+    FSWorkbookRels := TFileStream.Create(GetTempFileName(dir, 'fpsWBR'), fmCreate+fmOpenRead);
+    FSWorkbook := TFileStream.Create(GetTempFileName(dir, 'fpsWB'), fmCreate+fmOpenRead);
+    FSStyles := TFileStream.Create(GetTempFileName(dir, 'fpsSTY'), fmCreate+fmOpenRead);
+    FSSharedStrings := TFileStream.Create(GetTempFileName(dir, 'fpsSST'), fmCreate+fmOpenRead);
+    FSSharedStrings_complete := TFileStream.Create(GetTempFileName(dir, 'fpsSSTc'), fmCreate+fmOpenRead);
   end else begin;
     FSContentTypes := TMemoryStream.Create;
     FSRelsRels := TMemoryStream.Create;
@@ -507,8 +512,6 @@ end;
 
 { Destroys the streams that were created by the writer }
 procedure TsSpreadOOXMLWriter.DestroyStreams;
-var
-  i: Integer;
 
   procedure DestroyStream(AStream: TStream);
   var
@@ -521,6 +524,8 @@ var
     AStream.Free;
   end;
 
+var
+  stream: TStream;
 begin
   DestroyStream(FSContentTypes);
   DestroyStream(FSRelsRels);
@@ -529,40 +534,22 @@ begin
   DestroyStream(FSStyles);
   DestroyStream(FSSharedStrings);
   DestroyStream(FSSharedStrings_complete);
-
-  for i := 0 to Length(FSSheets) - 1 do
-    DestroyStream(FSSheets[i]);
+  for stream in FSSheets do DestroyStream(stream);
   SetLength(FSSheets, 0);
 end;
 
-{ Is called before zipping the individual file parts. Rewinds the memory streams,
-  or, if the stream are file streams, the streams are closed and re-opened for
-  reading. }
+{ Is called before zipping the individual file parts. Rewinds the streams. }
 procedure TsSpreadOOXMLWriter.ResetStreams;
 var
-  i: Integer;
-
-  procedure ResetStream(AStream: TStream);
-  var
-    fn: String;
-  begin
-    if AStream is TFileStream then begin
-      fn := TFileStream(AStream).FileName;
-      AStream.Free;
-      AStream := TFileStream.Create(fn, fmOpenRead);
-    end else
-      AStream.Position := 0;
-  end;
-
+  stream: TStream;
 begin
-  ResetStream(FSContentTypes);
-  ResetStream(FSRelsRels);
-  ResetStream(FSWorkbookRels);
-  ResetStream(FSWorkbook);
-  ResetStream(FSStyles);
-  ResetStream(FSSharedStrings_complete);
-  for i:=0 to Length(FSSheets) - 1 do
-    ResetStream(FSSheets[i]);
+  FSContentTypes.Position := 0;
+  FSRelsRels.Position := 0;
+  FSWorkbookRels.Position := 0;
+  FSWorkbook.Position := 0;
+  FSStyles.Position := 0;
+  FSSharedStrings_complete.Position := 0;
+  for stream in FSSheets do stream.Position := 0;
 end;
 
 {