minimal date for parcing

EventLogLoaderManager/AddPath.Designer.vb

@@ -36,6 +36,9 @@ Partial Class AddPath
         Me.ButtonChoosePath = New System.Windows.Forms.Button()
         Me.Label5 = New System.Windows.Forms.Label()
         Me.ESServerNameTextBox = New System.Windows.Forms.TextBox()
+        Me.FilterByDateDateTimePicker = New System.Windows.Forms.DateTimePicker()
+        Me.Label6 = New System.Windows.Forms.Label()
+        Me.UseFilterByDateCheckBox = New System.Windows.Forms.CheckBox()
         Me.SuspendLayout()
         '
         'IBName
@@ -114,7 +117,7 @@ Partial Class AddPath
         'Button1
         '
         Me.Button1.Font = New System.Drawing.Font("Microsoft Sans Serif", 8.25!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte))
-        Me.Button1.Location = New System.Drawing.Point(12, 141)
+        Me.Button1.Location = New System.Drawing.Point(9, 195)
         Me.Button1.Name = "Button1"
         Me.Button1.Size = New System.Drawing.Size(200, 43)
         Me.Button1.TabIndex = 2
@@ -124,7 +127,7 @@ Partial Class AddPath
         'Button2
         '
         Me.Button2.Font = New System.Drawing.Font("Microsoft Sans Serif", 8.25!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte))
-        Me.Button2.Location = New System.Drawing.Point(218, 141)
+        Me.Button2.Location = New System.Drawing.Point(215, 195)
         Me.Button2.Name = "Button2"
         Me.Button2.Size = New System.Drawing.Size(200, 43)
         Me.Button2.TabIndex = 2
@@ -163,11 +166,40 @@ Partial Class AddPath
         Me.ESServerNameTextBox.Size = New System.Drawing.Size(268, 20)
         Me.ESServerNameTextBox.TabIndex = 4
         '
+        'FilterByDateDateTimePicker
+        '
+        Me.FilterByDateDateTimePicker.Location = New System.Drawing.Point(213, 166)
+        Me.FilterByDateDateTimePicker.Name = "FilterByDateDateTimePicker"
+        Me.FilterByDateDateTimePicker.Size = New System.Drawing.Size(200, 20)
+        Me.FilterByDateDateTimePicker.TabIndex = 6
+        '
+        'Label6
+        '
+        Me.Label6.AutoSize = True
+        Me.Label6.Location = New System.Drawing.Point(12, 144)
+        Me.Label6.Name = "Label6"
+        Me.Label6.Size = New System.Drawing.Size(88, 13)
+        Me.Label6.TabIndex = 7
+        Me.Label6.Text = "Начальная дата"
+        '
+        'UseFilterByDateCheckBox
+        '
+        Me.UseFilterByDateCheckBox.AutoSize = True
+        Me.UseFilterByDateCheckBox.Location = New System.Drawing.Point(145, 144)
+        Me.UseFilterByDateCheckBox.Name = "UseFilterByDateCheckBox"
+        Me.UseFilterByDateCheckBox.Size = New System.Drawing.Size(263, 17)
+        Me.UseFilterByDateCheckBox.TabIndex = 8
+        Me.UseFilterByDateCheckBox.Text = "Загружать события начиная с указанной даты"
+        Me.UseFilterByDateCheckBox.UseVisualStyleBackColor = True
+        '
         'AddPath
         '
         Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
         Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font
-        Me.ClientSize = New System.Drawing.Size(425, 188)
+        Me.ClientSize = New System.Drawing.Size(425, 244)
+        Me.Controls.Add(Me.UseFilterByDateCheckBox)
+        Me.Controls.Add(Me.Label6)
+        Me.Controls.Add(Me.FilterByDateDateTimePicker)
         Me.Controls.Add(Me.Label5)
         Me.Controls.Add(Me.ESServerNameTextBox)
         Me.Controls.Add(Me.ButtonChoosePath)
@@ -202,4 +234,7 @@ Partial Class AddPath
     Friend WithEvents ButtonChoosePath As System.Windows.Forms.Button
     Friend WithEvents Label5 As Label
     Friend WithEvents ESServerNameTextBox As TextBox
+    Friend WithEvents FilterByDateDateTimePicker As DateTimePicker
+    Friend WithEvents Label6 As Label
+    Friend WithEvents UseFilterByDateCheckBox As CheckBox
 End Class

EventLogLoaderManager/Form1.Designer.vb

@@ -51,13 +51,14 @@ Partial Class Form1
         Me.Label5 = New System.Windows.Forms.Label()
         Me.UseSynonymsForFieldsNamesCheckBox = New System.Windows.Forms.CheckBox()
         Me.Label6 = New System.Windows.Forms.Label()
+        Me.StartAt = CType(New System.Windows.Forms.ColumnHeader(), System.Windows.Forms.ColumnHeader)
         Me.GroupBox1.SuspendLayout()
         Me.SuspendLayout()
         '
         'ListView
         '
         Me.ListView.CheckBoxes = True
-        Me.ListView.Columns.AddRange(New System.Windows.Forms.ColumnHeader() {Me.InfobaseName, Me.InfobaseGUID, Me.InfobaseDescription, Me.IBEvLogSize, Me.InfobaseLogPath, Me.ESServerName})
+        Me.ListView.Columns.AddRange(New System.Windows.Forms.ColumnHeader() {Me.InfobaseName, Me.InfobaseGUID, Me.InfobaseDescription, Me.IBEvLogSize, Me.InfobaseLogPath, Me.ESServerName, Me.StartAt})
         Me.ListView.Dock = System.Windows.Forms.DockStyle.Fill
         Me.ListView.FullRowSelect = True
         Me.ListView.GridLines = True
@@ -332,6 +333,11 @@ Partial Class Form1
         Me.Label6.TabIndex = 24
         Me.Label6.Text = "Использовать свои имена полей"
         '
+        'StartAt
+        '
+        Me.StartAt.Text = "Начальная дата"
+        Me.StartAt.Width = 150
+        '
         'Form1
         '
         Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
@@ -395,4 +401,5 @@ Partial Class Form1
     Friend WithEvents Label5 As Label
     Friend WithEvents UseSynonymsForFieldsNamesCheckBox As CheckBox
     Friend WithEvents Label6 As Label
+    Friend WithEvents StartAt As ColumnHeader
 End Class

EventLogLoaderManager/Form1.vb

@@ -150,6 +150,7 @@ Public Class Form1
                 IBSetting.DatabaseName = Item.SubItems(0).Text
                 IBSetting.DatabaseCatalog = Item.SubItems(4).Text
                 IBSetting.ESServerName = Item.SubItems(5).Text
+                IBSetting.StartDate = Item.SubItems(6).Text
 
                 NewConfigSetting.Infobases.Add(IBSetting)
 
@@ -228,7 +229,7 @@ Public Class Form1
 
     Private Sub Form1_Load(sender As System.Object, e As System.EventArgs) Handles MyBase.Load
 
-        Text = My.Application.Info.ProductName + " / " + My.Application.Info.Copyright
+        Text = My.Application.Info.ProductName
 
         LoadConfigSetting()
 
@@ -278,6 +279,7 @@ Public Class Form1
                     item1.SubItems.Add(a.SizeEventLog.ToString)
                     item1.SubItems.Add(a.CatalogEventLog)
                     item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName
+                    item1.SubItems.Add(SavedIB.StartDate) 'StartAt
 
                     ListView.Items.Add(item1)
 
@@ -302,6 +304,7 @@ Public Class Form1
                 item1.SubItems.Add(a.SizeEventLog.ToString)
                 item1.SubItems.Add(a.CatalogEventLog)
                 item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName
+                item1.SubItems.Add(SavedIB.StartDate) 'StartAt
 
                 ListView.Items.Add(item1)
 
@@ -321,6 +324,7 @@ Public Class Form1
                 item1.SubItems.Add(CalcullateFolderSize(IB.DatabaseCatalog))
                 item1.SubItems.Add(IB.DatabaseCatalog)
                 item1.SubItems.Add(IB.ESServerName)
+                item1.SubItems.Add(IB.StartDate) 'StartAt
 
                 ListView.Items.Add(item1)
             End If
@@ -489,6 +493,11 @@ Public Class Form1
             item1.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text))
             item1.SubItems.Add(AddPath.Path.Text)
             item1.SubItems.Add(AddPath.ESServerNameTextBox.Text)
+            If AddPath.UseFilterByDateCheckBox.Checked Then
+                item1.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd"))
+            Else
+                item1.SubItems.Add("")
+            End If
 
             ListView.Items.Add(item1)
 
@@ -511,6 +520,15 @@ Public Class Form1
         AddPath.ESServerNameTextBox.Text = item.SubItems(5).Text
         AddPath.Description.Text = item.SubItems(2).Text
         AddPath.Path.Text = item.SubItems(4).Text
+
+        Dim DateStr = item.SubItems(6).Text
+        If String.IsNullOrEmpty(DateStr) Then
+            AddPath.UseFilterByDateCheckBox.Checked = False
+        Else
+            AddPath.UseFilterByDateCheckBox.Checked = True
+            AddPath.FilterByDateDateTimePicker.Value = DateTime.Parse(DateStr)
+        End If
+
         AddPath.ExistedRow = True
         AddPath.ShowDialog()
 
@@ -524,14 +542,13 @@ Public Class Form1
             item.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text))
             item.SubItems.Add(AddPath.Path.Text)
             item.SubItems.Add(AddPath.ESServerNameTextBox.Text)
-
+            If AddPath.UseFilterByDateCheckBox.Checked Then
+                item.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd"))
+            Else
+                item.SubItems.Add("")
             End If
 
-
+        End If
-
-        'End If
-
-
 
     End Sub
 

EventLogLoaderService/ConfigSettings.vb

@@ -7,6 +7,7 @@ Public Module ConfigSettingsModule
         Public DatabaseID As String = ""
         Public DatabaseName As String = ""
         Public DatabaseCatalog As String = ""
+        Public StartDate As String = ""
         Public Found As Boolean = False
     End Class
 

EventLogLoaderService/EventLogProcessor.vb

@@ -221,6 +221,7 @@ Public Class EventLogProcessor
     Public ItIsMySQL As Boolean = False
     Public ItIsES As Boolean = False
     Public ESUseSynonymsForFieldsNames As Boolean = False
+    Public LoadEventsStartingAt As Date
     Public ESFieldSynonyms As ElasticSearchFieldSynonymsClass = New ElasticSearchFieldSynonymsClass
     Public SleepTime As Integer = 60 * 1000 '1 минута
 
@@ -1238,7 +1239,7 @@ Public Class EventLogProcessor
                                             [eventCode],
                                             [comment],
                                             [metadataCodes],
-                                            [sessionDataSplitCode],--
+                                            [sessionDataSplitCode],
                                             [dataType],
                                             [data],
                                             [dataPresentation],
@@ -1246,11 +1247,15 @@ Public Class EventLogProcessor
                                             [primaryPortCode],
                                             [secondaryPortCode]
                                         FROM [EventLog] 
-                                        WHERE [rowID] > @LastEventNumber83 
+                                        WHERE [rowID] > @LastEventNumber83 AND [date] >= @MinimumDate
                                         ORDER BY 1
                                         LIMIT 1000"
 
                 Command.Parameters.AddWithValue("LastEventNumber83", LastEventNumber83)
+
+                Dim noOfSeconds As Int64 = (LoadEventsStartingAt - New Date).TotalSeconds
+                Command.Parameters.AddWithValue("MinimumDate", noOfSeconds * 10000)
+
                 Dim rs = Command.ExecuteReader
 
                 Dim HasData = rs.HasRows
@@ -1307,23 +1312,17 @@ Public Class EventLogProcessor
                     OneEvent.Transaction = ""
                     OneEvent.EventType = ""
 
-
-
                     EventsList.Add(OneEvent)
 
-                    If EventsList.Count >= 1000 Then
-                        'Console.WriteLine("Выгрузка 1000 событий: " + Now.ToString)
-                        SaveEventsToSQL()
-                    End If
-
                     LastEventNumber83 = OneEvent.RowID
 
                 End While
 
-
                 rs.Close()
 
+                If EventsList.Count > 0 Then
                     SaveEventsToSQL()
+                End If
 
                 If Not HasData Then
                     Exit While
@@ -1332,8 +1331,6 @@ Public Class EventLogProcessor
             End While
 
 
-
-
             Command.Dispose()
             Conn.Close()
             Conn.Dispose()
@@ -1523,6 +1520,10 @@ Public Class EventLogProcessor
         OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider)
         OneEvent.TransactionStatus = Array(1)
 
+        If OneEvent.DateTime < LoadEventsStartingAt Then
+            Exit Sub
+        End If
+
         Dim TransStr = Array(2).ToString.Replace("}", "").Replace("{", "")
         Dim TransDate = From16To10(TransStr.Substring(0, TransStr.IndexOf(",")))
 

EventLogLoaderService/Service1.vb

@@ -66,6 +66,13 @@ Public Class EventLogLoaderService
                     EventLogProcessorObj.ESUseSynonymsForFieldsNames = ConfigSettingObj.ESUseSynonymsForFieldsNames
                     EventLogProcessorObj.ESFieldSynonyms = ConfigSettingObj.ESFieldSynonyms
 
+                    Try
+                        EventLogProcessorObj.LoadEventsStartingAt = Date.Parse(IBConfig.StartDate)
+                    Catch ex As Exception
+                        EventLogProcessorObj.LoadEventsStartingAt = New Date(1900, 1, 1)
+                    End Try
+
+
                     ListOfProcessors.Add(EventLogProcessorObj)
 
                 Next