diff --git a/components/fpspreadsheet/examples/other/test_virtualmode.lpi b/components/fpspreadsheet/examples/other/test_virtualmode.lpi index b0e5aa8c3..dae3987f6 100644 --- a/components/fpspreadsheet/examples/other/test_virtualmode.lpi +++ b/components/fpspreadsheet/examples/other/test_virtualmode.lpi @@ -63,7 +63,6 @@ - diff --git a/components/fpspreadsheet/examples/other/test_virtualmode.lpr b/components/fpspreadsheet/examples/other/test_virtualmode.lpr index 205b23caa..550211198 100644 --- a/components/fpspreadsheet/examples/other/test_virtualmode.lpr +++ b/components/fpspreadsheet/examples/other/test_virtualmode.lpr @@ -8,7 +8,7 @@ uses {$ENDIF}{$ENDIF} Classes, laz_fpspreadsheet, { you can add units after this } - SysUtils, variants, fpspreadsheet, xlsxooxml; + SysUtils, variants, fpspreadsheet, xlsbiff2, xlsbiff5, xlsbiff8, xlsxooxml; type TDataProvider = class @@ -35,7 +35,7 @@ type AData := 10000*ARow + ACol; // you can use the OnNeedData also to provide feedback on how the process - // progresses. + // progresses: if (ACol = 0) and (ARow mod 1000 = 0) then WriteLn('Writing row ', ARow, '...'); end; @@ -52,22 +52,30 @@ begin workbook := TsWorkbook.Create; try worksheet := workbook.AddWorksheet('Sheet1'); + worksheet.WriteFontStyle(0, 1, [fssBold]); { These are the essential commands to activate virtual mode: } - workbook.WritingOptions := [woVirtualMode, woSaveMemory]; + +// workbook.WritingOptions := [woVirtualMode, woSaveMemory]; + workbook.WritingOptions := [woVirtualMode]; + // woSaveMemory can be omitted, but is essential for large files: it causes - // writing temporaray data to a file stream instead of to a memory stream. - workbook.VirtualRowCount := 10000; + // writing temporaray data to a file stream instead of a memory stream. + // woSaveMemory, however, considerably slows down writing of biff files. + + workbook.VirtualRowCount := 1000; workbook.VirtualColCount := 100; // These two numbers define the size of virtual spreadsheet. // In case of a database, VirtualRowCount is the RecordCount, VirtualColCount // the number of fields to be written to the spreadsheet file + workbook.OnNeedCellData := @dataprovider.NeedCellData; // This links the worksheet to the method from which it gets the // data to write. // In case of a database, you would open the dataset before calling this: - workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true); +// workbook.WriteToFile('test_virtual.xlsx', sfOOXML, true); + workbook.WriteToFile('test_virtual.xls', sfExcel5, true); finally workbook.Free; diff --git a/components/fpspreadsheet/fpolebasic.pas b/components/fpspreadsheet/fpolebasic.pas index 77a204dac..1996e9d2d 100644 --- a/components/fpspreadsheet/fpolebasic.pas +++ b/components/fpspreadsheet/fpolebasic.pas @@ -23,7 +23,8 @@ type TOLEDocument = record // Information about the document - Stream: TMemoryStream; + Stream: TStream; +// Stream: TMemoryStream; end; @@ -57,7 +58,7 @@ var begin VLAbsolutePath:='/'+AStreamName; //Virtual layer always use absolute paths. if not AOverwriteExisting and FileExists(AFileName) then begin - Raise EStreamError.Createfmt('File already exists "%s"',[AFileName]); + Raise EStreamError.Createfmt('File "%s" already exists.',[AFileName]); end; RealFile:=TFileStream.Create(AFileName,fmCreate); fsOLE:=TVirtualLayer_OLE.Create(RealFile); @@ -101,7 +102,7 @@ begin if not Assigned(AOLEDocument.Stream) then begin AOLEDocument.Stream:=TMemoryStream.Create; end else begin - AOLEDocument.Stream.Clear; + (AOLEDocument.Stream as TMemoryStream).Clear; end; AOLEDocument.Stream.CopyFrom(OLEStream,OLEStream.Size); end; diff --git a/components/fpspreadsheet/fpsopendocument.pas b/components/fpspreadsheet/fpsopendocument.pas index d119d7da8..430a425ab 100755 --- a/components/fpspreadsheet/fpsopendocument.pas +++ b/components/fpspreadsheet/fpsopendocument.pas @@ -2888,6 +2888,10 @@ begin FPointSeparatorSettings := SysUtils.DefaultFormatSettings; FPointSeparatorSettings.DecimalSeparator:='.'; + + // http://en.wikipedia.org/wiki/List_of_spreadsheet_software#Specifications + FLimitations.MaxCols := 1024; + FLimitations.MaxRows := 1048576; end; destructor TsSpreadOpenDocWriter.Destroy; diff --git a/components/fpspreadsheet/fpspreadsheet.pas b/components/fpspreadsheet/fpspreadsheet.pas index 75650d099..b1ed4928d 100755 --- a/components/fpspreadsheet/fpspreadsheet.pas +++ b/components/fpspreadsheet/fpspreadsheet.pas @@ -21,6 +21,12 @@ type TsSpreadsheetFormat = (sfExcel2, {sfExcel3, sfExcel4,} sfExcel5, sfExcel8, sfOOXML, sfOpenDocument, sfCSV, sfWikiTable_Pipes, sfWikiTable_WikiMedia); + {@@ Record collection limitations of a particular file format } + TsSpreadsheetFormatLimitations = record + MaxRows: Cardinal; + MaxCols: Cardinal; + end; + const { Default extensions } STR_EXCEL_EXTENSION = '.xls'; @@ -738,6 +744,7 @@ type procedure SetVirtualRowCount(AValue: Cardinal); { Internal methods } + procedure GetLastRowColIndex(out ALastRow, ALastCol: Cardinal); procedure PrepareBeforeSaving; procedure RemoveWorksheetsCallback(data, arg: pointer); procedure UpdateCaches; @@ -954,14 +961,19 @@ type FWorkbook: TsWorkbook; protected + {@@ Limitations for the specific data file format } + FLimitations: TsSpreadsheetFormatLimitations; {@@ List of number formats found in the workbook. } FNumFormatList: TsCustomNumFormatList; { Helper routines } procedure AddDefaultFormats(); virtual; + procedure CheckLimitations; procedure CreateNumFormatList; virtual; function ExpandFormula(AFormula: TsFormula): TsExpandedFormula; function FindFormattingInList(AFormat: PCell): Integer; procedure FixFormat(ACell: PCell); virtual; + procedure GetSheetDimensions(AWorksheet: TsWorksheet; + out AFirstRow, ALastRow, AFirstCol, ALastCol: Cardinal); virtual; procedure ListAllFormattingStylesCallback(ACell: PCell; AStream: TStream); procedure ListAllFormattingStyles; virtual; procedure ListAllNumFormatsCallback(ACell: PCell; AStream: TStream); @@ -992,6 +1004,7 @@ type NextXFIndex: Integer; constructor Create(AWorkbook: TsWorkbook); virtual; // To allow descendents to override it destructor Destroy; override; + function Limitations: TsSpreadsheetFormatLimitations; { General writing methods } procedure IterateThroughCells(AStream: TStream; ACells: TAVLTree; ACallback: TCellsCallback); procedure WriteToFile(const AFileName: string; const AOverwriteExisting: Boolean = False); virtual; @@ -1070,6 +1083,8 @@ resourcestring lpUnsupportedWriteFormat = 'Tried to write a spreadsheet using an unsupported format'; lpNoValidSpreadsheetFile = '"%s" is not a valid spreadsheet file'; lpUnknownSpreadsheetFormat = 'unknown format'; + lpMaxRowsExceeded = 'This workbook contains %d rows, but the selected file format does not support more than %d rows.'; + lpMaxColsExceeded = 'This workbook contains %d columns, but the selected file format does not support more than %d columns.'; lpInvalidFontIndex = 'Invalid font index'; lpInvalidNumberFormat = 'Trying to use an incompatible number format.'; lpInvalidDateTimeFormat = 'Trying to use an incompatible date/time format.'; @@ -4084,6 +4099,30 @@ begin if Result = nil then raise Exception.Create(lpUnsupportedWriteFormat); end; +{@@ + Determines the maximum index of used columns and rows in all sheets of this + workbook. Respects VirtualMode. + Is needed to disable saving when limitations of the format is exceeded. } +procedure TsWorkbook.GetLastRowColIndex(out ALastRow, ALastCol: Cardinal); +var + i: Integer; + sheet: TsWorksheet; + r1,r2, c1,c2: Cardinal; +begin + if (woVirtualMode in WritingOptions) then begin + ALastRow := FVirtualRowCount - 1; + ALastCol := FVirtualColCount - 1; + end else begin + ALastRow := 0; + ALastCol := 0; + for i:=0 to GetWorksheetCount-1 do begin + sheet := GetWorksheetByIndex(i); + ALastRow := Max(ALastRow, sheet.GetLastRowIndex); + ALastCol := Max(ALastCol, sheet.GetLastColIndex); + end; + end; +end; + {@@ Reads the document from a file. It is assumed to have a given file format. @@ -4221,6 +4260,7 @@ begin AWriter := CreateSpreadWriter(AFormat); try FFileName := AFileName; + AWriter.CheckLimitations; FWriting := true; PrepareBeforeSaving; AWriter.WriteToFile(AFileName, AOverwriteExisting); @@ -4263,6 +4303,7 @@ var begin AWriter := CreateSpreadWriter(AFormat); try + AWriter.CheckLimitations; FWriting := true; PrepareBeforeSaving; AWriter.WriteToStream(AStream); @@ -5260,6 +5301,10 @@ begin inherited Create; FWorkbook := AWorkbook; CreateNumFormatList; + { A good starting point valid for many formats... } + FLimitations.MaxCols := 256; + FLimitations.MaxRows := 65536; + // FNumFormatList.FWorkbook := AWorkbook; end; @@ -5347,6 +5392,39 @@ begin // to be overridden end; +{@@ + Returns a record containing limitations of the specific file format of the + writer. +} +function TsCustomSpreadWriter.Limitations: TsSpreadsheetFormatLimitations; +begin + Result := FLimitations; +end; + +{@@ + Determines the size of the worksheet to be written. VirtualMode is respected. + Is called when the writer needs the size for output. + + @param AWorksheet Worksheet to be written + @param AFirsRow Index of first row to be written + @param ALastRow Index of last row + @param AFirstCol Index of first column to be written + @param ALastCol Index of last column to be written +} +procedure TsCustomSpreadWriter.GetSheetDimensions(AWorksheet: TsWorksheet; + out AFirstRow, ALastRow, AFirstCol, ALastCol: Cardinal); +begin + AFirstRow := 0; + AFirstCol := 0; + if (woVirtualMode in AWorksheet.Workbook.WritingOptions) then begin + ALastRow := AWorksheet.Workbook.VirtualRowCount-1; + ALastCol := AWorksheet.Workbook.VirtualColCount-1; + end else begin + ALastRow := AWorksheet.GetLastRowIndex; + ALastCol := AWorksheet.GetLastColIndex; + end; +end; + {@@ Each descendent should define its own default formats, if any. Always add the normal, unformatted style first to speed things up. @@ -5359,6 +5437,20 @@ begin NextXFIndex := 0; end; +{@@ + Checks limitations of the writer, e.g max row/column count +} +procedure TsCustomSpreadWriter.CheckLimitations; +var + lastCol, lastRow: Cardinal; +begin + Workbook.GetLastRowColIndex(lastRow, lastCol); + if lastRow >= FLimitations.MaxRows then + raise Exception.CreateFmt(lpMaxRowsExceeded, [lastRow+1, FLimitations.MaxRows]); + if lastCol >= FLimitations.MaxCols then + raise Exception.CreateFmt(lpMaxColsExceeded, [lastCol+1, FLimitations.MaxCols]); +end; + {@@ Creates an instance of the number format list which contains prototypes of all number formats found in the workbook. diff --git a/components/fpspreadsheet/xlsbiff2.pas b/components/fpspreadsheet/xlsbiff2.pas index d4035cad0..a590191ef 100755 --- a/components/fpspreadsheet/xlsbiff2.pas +++ b/components/fpspreadsheet/xlsbiff2.pas @@ -981,7 +981,13 @@ begin WriteXFRecords(AStream); WriteColWidths(AStream); WriteRows(AStream, sheet); - WriteCellsToStream(AStream, sheet.Cells); + + if (woVirtualMode in Workbook.WritingOptions) then + WriteVirtualCells(AStream) + else begin + WriteRows(AStream, sheet); + WriteCellsToStream(AStream, sheet.Cells); + end; WriteWindow1(AStream); // { -- currently not working diff --git a/components/fpspreadsheet/xlsbiff5.pas b/components/fpspreadsheet/xlsbiff5.pas index 17adfc5bd..20f6450f0 100755 --- a/components/fpspreadsheet/xlsbiff5.pas +++ b/components/fpspreadsheet/xlsbiff5.pas @@ -407,6 +407,7 @@ begin AStream.Position := CurrentPos; WriteBOF(AStream, INT_BOF_SHEET); + WriteIndex(AStream); // WritePageSetup(AStream); WriteColInfos(AStream, sheet); @@ -415,7 +416,14 @@ begin WritePane(AStream, sheet, true, pane); // true for "is BIFF5 or BIFF8" WriteSelection(AStream, sheet, pane); WriteRows(AStream, sheet); - WriteCellsToStream(AStream, sheet.Cells); + + if (woVirtualMode in Workbook.WritingOptions) then + WriteVirtualCells(AStream) + else begin + WriteRows(AStream, sheet); + WriteCellsToStream(AStream, sheet.Cells); + end; + WriteEOF(AStream); end; diff --git a/components/fpspreadsheet/xlsbiff8.pas b/components/fpspreadsheet/xlsbiff8.pas index 483589bbd..7785109e2 100755 --- a/components/fpspreadsheet/xlsbiff8.pas +++ b/components/fpspreadsheet/xlsbiff8.pas @@ -358,21 +358,27 @@ end; procedure TsSpreadBIFF8Writer.WriteToFile(const AFileName: string; const AOverwriteExisting: Boolean); var - MemStream: TMemoryStream; + Stream: TStream; OutputStorage: TOLEStorage; OLEDocument: TOLEDocument; + fn: String; begin - MemStream := TMemoryStream.Create; + if (woSaveMemory in Workbook.WritingOptions) then begin + fn := GetTempFileName; + Stream := TFileStream.Create(fn, fmCreate + fmOpenRead) + end else + Stream := TMemoryStream.Create; + OutputStorage := TOLEStorage.Create; try - WriteToStream(MemStream); + WriteToStream(Stream); // Only one stream is necessary for any number of worksheets - OLEDocument.Stream := MemStream; + OLEDocument.Stream := Stream; OutputStorage.WriteOLEFile(AFileName, OLEDocument, AOverwriteExisting, 'Workbook'); finally - MemStream.Free; + Stream.Free; OutputStorage.Free; end; end; @@ -439,8 +445,12 @@ begin WriteDimensions(AStream, sheet); //WriteRowAndCellBlock(AStream, sheet); - WriteRows(AStream, sheet); - WriteCellsToStream(AStream, sheet.Cells); + if (woVirtualMode in Workbook.WritingOptions) then + WriteVirtualCells(AStream) + else begin + WriteRows(AStream, sheet); + WriteCellsToStream(AStream, sheet.Cells); + end; WriteWindow2(AStream, sheet); WritePane(AStream, sheet, isBIFF8, pane); @@ -545,26 +555,26 @@ end; } procedure TsSpreadBIFF8Writer.WriteDimensions(AStream: TStream; AWorksheet: TsWorksheet); var - lLastCol: Word; - lLastRow: Integer; + firstRow, lastRow, firstCol, lastCol: Cardinal; begin { BIFF Record header } AStream.WriteWord(WordToLE(INT_EXCEL_ID_DIMENSIONS)); AStream.WriteWord(WordToLE(14)); + { Determine sheet size } + GetSheetDimensions(AWorksheet, firstRow, lastRow, firstCol, lastCol); + { Index to first used row } - AStream.WriteDWord(DWordToLE(0)); + AStream.WriteDWord(DWordToLE(firstRow)); { Index to last used row, increased by 1 } - lLastRow := GetLastRowIndex(AWorksheet)+1; - AStream.WriteDWord(DWordToLE(lLastRow)); // Old dummy value: 33 + AStream.WriteDWord(DWordToLE(lastRow+1)); { Index to first used column } - AStream.WriteWord(WordToLE(0)); + AStream.WriteWord(WordToLE(firstCol)); { Index to last used column, increased by 1 } - lLastCol := GetLastColIndex(AWorksheet)+1; - AStream.WriteWord(WordToLE(lLastCol)); // Old dummy value: 10 + AStream.WriteWord(WordToLE(lastCol+1)); { Not used } AStream.WriteWord(WordToLE(0)); diff --git a/components/fpspreadsheet/xlscommon.pas b/components/fpspreadsheet/xlscommon.pas index b62c91d3e..c534b3fda 100644 --- a/components/fpspreadsheet/xlscommon.pas +++ b/components/fpspreadsheet/xlscommon.pas @@ -508,6 +508,8 @@ type procedure WriteWindow1(AStream: TStream); virtual; // Writes the index of the XF record used in the given cell procedure WriteXFIndex(AStream: TStream; ACell: PCell); + // Writes cell content received by workbook in OnNeedCellData event + procedure WriteVirtualCells(AStream: TStream); public constructor Create(AWorkbook: TsWorkbook); override; @@ -518,7 +520,7 @@ type implementation uses - fpsNumFormatParser; + Variants, fpsNumFormatParser; { Helper table for rpn formulas: Assignment of FormulaElementKinds (fekXXXX) to EXCEL_TOKEN IDs. } @@ -2526,5 +2528,45 @@ begin AStream.WriteWord(WordToLE(lXFIndex)); end; +procedure TsSpreadBIFFWriter.WriteVirtualCells(AStream: TStream); +var + r,c: Cardinal; + lCell: TCell; + value: variant; +begin + FillChar(lCell, SizeOf(lCell), 0); + for r := 0 to Workbook.VirtualRowCount-1 do begin + for c := 0 to Workbook.VirtualColCount-1 do begin + value := varNull; + Workbook.OnNeedCellData(Workbook, r, c, value); + lCell.Row := r; + lCell.Col := c; + if VarIsNull(value) then + lCell.ContentType := cctEmpty + else + if VarIsNumeric(value) then begin + lCell.ContentType := cctNumber; + lCell.NumberValue := value; + end else + { + if VarIsDateTime(value) then begin + lCell.ContentType := cctNumber; + lCell.DateTimeValue := value; + end else + } + if VarIsStr(value) then begin + lCell.ContentType := cctUTF8String; + lCell.UTF8StringValue := VarToStrDef(value, ''); + end else + if VarIsBool(value) then begin + lCell.ContentType := cctBool; + lCell.BoolValue := value <> 0; + end else + lCell.ContentType := cctEmpty; + WriteCellCallback(@lCell, AStream); + end; + end; +end; + end. diff --git a/components/fpspreadsheet/xlsxooxml.pas b/components/fpspreadsheet/xlsxooxml.pas index aeeda4456..7b5d98676 100755 --- a/components/fpspreadsheet/xlsxooxml.pas +++ b/components/fpspreadsheet/xlsxooxml.pas @@ -58,6 +58,7 @@ type { TsSpreadOOXMLWriter } TsSpreadOOXMLWriter = class(TsCustomSpreadWriter) + private protected FPointSeparatorSettings: TFormatSettings; FSharedStringsCount: Integer; @@ -470,6 +471,10 @@ begin inherited Create(AWorkbook); FPointSeparatorSettings := DefaultFormatSettings; FPointSeparatorSettings.DecimalSeparator := '.'; + + // http://en.wikipedia.org/wiki/List_of_spreadsheet_software#Specifications + FLimitations.MaxCols := 16384; + FLimitations.MaxRows := 1048576; end; procedure TsSpreadOOXMLWriter.CreateNumFormatList; @@ -486,13 +491,13 @@ var begin if (woSaveMemory in Workbook.WritingOptions) then begin dir := IncludeTrailingPathDelimiter(GetTempDir); - FSContentTypes := TFileStream.Create(GetTempFileName(dir, 'fpsCT'), fmCreate); - FSRelsRels := TFileStream.Create(GetTempFileName(dir, 'fpsRR'), fmCreate); - FSWorkbookRels := TFileStream.Create(GetTempFileName(dir, 'fpsWBR'), fmCreate); - FSWorkbook := TFileStream.Create(GetTempFileName(dir, 'fpsWB'), fmCreate); - FSStyles := TFileStream.Create(GetTempFileName(dir, 'fpsSTY'), fmCreate); - FSSharedStrings := TFileStream.Create(GetTempFileName(dir, 'fpsSST'), fmCreate); - FSSharedStrings_complete := TFileStream.Create(GetTempFileName(dir, 'fpsSSTc'), fmCreate); + FSContentTypes := TFileStream.Create(GetTempFileName(dir, 'fpsCT'), fmCreate+fmOpenRead); + FSRelsRels := TFileStream.Create(GetTempFileName(dir, 'fpsRR'), fmCreate+fmOpenRead); + FSWorkbookRels := TFileStream.Create(GetTempFileName(dir, 'fpsWBR'), fmCreate+fmOpenRead); + FSWorkbook := TFileStream.Create(GetTempFileName(dir, 'fpsWB'), fmCreate+fmOpenRead); + FSStyles := TFileStream.Create(GetTempFileName(dir, 'fpsSTY'), fmCreate+fmOpenRead); + FSSharedStrings := TFileStream.Create(GetTempFileName(dir, 'fpsSST'), fmCreate+fmOpenRead); + FSSharedStrings_complete := TFileStream.Create(GetTempFileName(dir, 'fpsSSTc'), fmCreate+fmOpenRead); end else begin; FSContentTypes := TMemoryStream.Create; FSRelsRels := TMemoryStream.Create; @@ -507,8 +512,6 @@ end; { Destroys the streams that were created by the writer } procedure TsSpreadOOXMLWriter.DestroyStreams; -var - i: Integer; procedure DestroyStream(AStream: TStream); var @@ -521,6 +524,8 @@ var AStream.Free; end; +var + stream: TStream; begin DestroyStream(FSContentTypes); DestroyStream(FSRelsRels); @@ -529,40 +534,22 @@ begin DestroyStream(FSStyles); DestroyStream(FSSharedStrings); DestroyStream(FSSharedStrings_complete); - - for i := 0 to Length(FSSheets) - 1 do - DestroyStream(FSSheets[i]); + for stream in FSSheets do DestroyStream(stream); SetLength(FSSheets, 0); end; -{ Is called before zipping the individual file parts. Rewinds the memory streams, - or, if the stream are file streams, the streams are closed and re-opened for - reading. } +{ Is called before zipping the individual file parts. Rewinds the streams. } procedure TsSpreadOOXMLWriter.ResetStreams; var - i: Integer; - - procedure ResetStream(AStream: TStream); - var - fn: String; - begin - if AStream is TFileStream then begin - fn := TFileStream(AStream).FileName; - AStream.Free; - AStream := TFileStream.Create(fn, fmOpenRead); - end else - AStream.Position := 0; - end; - + stream: TStream; begin - ResetStream(FSContentTypes); - ResetStream(FSRelsRels); - ResetStream(FSWorkbookRels); - ResetStream(FSWorkbook); - ResetStream(FSStyles); - ResetStream(FSSharedStrings_complete); - for i:=0 to Length(FSSheets) - 1 do - ResetStream(FSSheets[i]); + FSContentTypes.Position := 0; + FSRelsRels.Position := 0; + FSWorkbookRels.Position := 0; + FSWorkbook.Position := 0; + FSStyles.Position := 0; + FSSharedStrings_complete.Position := 0; + for stream in FSSheets do stream.Position := 0; end; {