minimal date for parcing

2024-11-24 08:32:52 +02:00 · 2017-02-20 14:32:07 -08:00 · 2017-02-20 14:32:07 -08:00 · eea782a1f0
commit eea782a1f0
parent d857ca7f12
6 changed files with 92 additions and 24 deletions
--- a/EventLogLoaderManager/AddPath.Designer.vb
+++ b/EventLogLoaderManager/AddPath.Designer.vb
@ -36,6 +36,9 @@ Partial Class AddPath
        Me.ButtonChoosePath = New System.Windows.Forms.Button()
        Me.Label5 = New System.Windows.Forms.Label()
        Me.ESServerNameTextBox = New System.Windows.Forms.TextBox()
+        Me.FilterByDateDateTimePicker = New System.Windows.Forms.DateTimePicker()
+        Me.Label6 = New System.Windows.Forms.Label()
+        Me.UseFilterByDateCheckBox = New System.Windows.Forms.CheckBox()
        Me.SuspendLayout()
        '
        'IBName
@ -114,7 +117,7 @@ Partial Class AddPath
        'Button1
        '
        Me.Button1.Font = New System.Drawing.Font("Microsoft Sans Serif", 8.25!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte))
-        Me.Button1.Location = New System.Drawing.Point(12, 141)
+        Me.Button1.Location = New System.Drawing.Point(9, 195)
        Me.Button1.Name = "Button1"
        Me.Button1.Size = New System.Drawing.Size(200, 43)
        Me.Button1.TabIndex = 2
@ -124,7 +127,7 @@ Partial Class AddPath
        'Button2
        '
        Me.Button2.Font = New System.Drawing.Font("Microsoft Sans Serif", 8.25!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte))
-        Me.Button2.Location = New System.Drawing.Point(218, 141)
+        Me.Button2.Location = New System.Drawing.Point(215, 195)
        Me.Button2.Name = "Button2"
        Me.Button2.Size = New System.Drawing.Size(200, 43)
        Me.Button2.TabIndex = 2
@ -163,11 +166,40 @@ Partial Class AddPath
        Me.ESServerNameTextBox.Size = New System.Drawing.Size(268, 20)
        Me.ESServerNameTextBox.TabIndex = 4
        '
+        'FilterByDateDateTimePicker
+        '
+        Me.FilterByDateDateTimePicker.Location = New System.Drawing.Point(213, 166)
+        Me.FilterByDateDateTimePicker.Name = "FilterByDateDateTimePicker"
+        Me.FilterByDateDateTimePicker.Size = New System.Drawing.Size(200, 20)
+        Me.FilterByDateDateTimePicker.TabIndex = 6
+        '
+        'Label6
+        '
+        Me.Label6.AutoSize = True
+        Me.Label6.Location = New System.Drawing.Point(12, 144)
+        Me.Label6.Name = "Label6"
+        Me.Label6.Size = New System.Drawing.Size(88, 13)
+        Me.Label6.TabIndex = 7
+        Me.Label6.Text = "Начальная дата"
+        '
+        'UseFilterByDateCheckBox
+        '
+        Me.UseFilterByDateCheckBox.AutoSize = True
+        Me.UseFilterByDateCheckBox.Location = New System.Drawing.Point(145, 144)
+        Me.UseFilterByDateCheckBox.Name = "UseFilterByDateCheckBox"
+        Me.UseFilterByDateCheckBox.Size = New System.Drawing.Size(263, 17)
+        Me.UseFilterByDateCheckBox.TabIndex = 8
+        Me.UseFilterByDateCheckBox.Text = "Загружать события начиная с указанной даты"
+        Me.UseFilterByDateCheckBox.UseVisualStyleBackColor = True
+        '
        'AddPath
        '
        Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
        Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font
-        Me.ClientSize = New System.Drawing.Size(425, 188)
+        Me.ClientSize = New System.Drawing.Size(425, 244)
+        Me.Controls.Add(Me.UseFilterByDateCheckBox)
+        Me.Controls.Add(Me.Label6)
+        Me.Controls.Add(Me.FilterByDateDateTimePicker)
        Me.Controls.Add(Me.Label5)
        Me.Controls.Add(Me.ESServerNameTextBox)
        Me.Controls.Add(Me.ButtonChoosePath)
@ -202,4 +234,7 @@ Partial Class AddPath
    Friend WithEvents ButtonChoosePath As System.Windows.Forms.Button
    Friend WithEvents Label5 As Label
    Friend WithEvents ESServerNameTextBox As TextBox
+    Friend WithEvents FilterByDateDateTimePicker As DateTimePicker
+    Friend WithEvents Label6 As Label
+    Friend WithEvents UseFilterByDateCheckBox As CheckBox
 End Class
--- a/EventLogLoaderManager/Form1.Designer.vb
+++ b/EventLogLoaderManager/Form1.Designer.vb
@ -51,13 +51,14 @@ Partial Class Form1
        Me.Label5 = New System.Windows.Forms.Label()
        Me.UseSynonymsForFieldsNamesCheckBox = New System.Windows.Forms.CheckBox()
        Me.Label6 = New System.Windows.Forms.Label()
+        Me.StartAt = CType(New System.Windows.Forms.ColumnHeader(), System.Windows.Forms.ColumnHeader)
        Me.GroupBox1.SuspendLayout()
        Me.SuspendLayout()
        '
        'ListView
        '
        Me.ListView.CheckBoxes = True
-        Me.ListView.Columns.AddRange(New System.Windows.Forms.ColumnHeader() {Me.InfobaseName, Me.InfobaseGUID, Me.InfobaseDescription, Me.IBEvLogSize, Me.InfobaseLogPath, Me.ESServerName})
+        Me.ListView.Columns.AddRange(New System.Windows.Forms.ColumnHeader() {Me.InfobaseName, Me.InfobaseGUID, Me.InfobaseDescription, Me.IBEvLogSize, Me.InfobaseLogPath, Me.ESServerName, Me.StartAt})
        Me.ListView.Dock = System.Windows.Forms.DockStyle.Fill
        Me.ListView.FullRowSelect = True
        Me.ListView.GridLines = True
@ -332,6 +333,11 @@ Partial Class Form1
        Me.Label6.TabIndex = 24
        Me.Label6.Text = "Использовать свои имена полей"
        '
+        'StartAt
+        '
+        Me.StartAt.Text = "Начальная дата"
+        Me.StartAt.Width = 150
+        '
        'Form1
        '
        Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!)
@ -395,4 +401,5 @@ Partial Class Form1
    Friend WithEvents Label5 As Label
    Friend WithEvents UseSynonymsForFieldsNamesCheckBox As CheckBox
    Friend WithEvents Label6 As Label
+    Friend WithEvents StartAt As ColumnHeader
 End Class
--- a/EventLogLoaderManager/Form1.vb
+++ b/EventLogLoaderManager/Form1.vb
@ -150,6 +150,7 @@ Public Class Form1
                IBSetting.DatabaseName = Item.SubItems(0).Text
                IBSetting.DatabaseCatalog = Item.SubItems(4).Text
                IBSetting.ESServerName = Item.SubItems(5).Text
+                IBSetting.StartDate = Item.SubItems(6).Text

                NewConfigSetting.Infobases.Add(IBSetting)

@ -228,7 +229,7 @@ Public Class Form1

    Private Sub Form1_Load(sender As System.Object, e As System.EventArgs) Handles MyBase.Load

-        Text = My.Application.Info.ProductName + " / " + My.Application.Info.Copyright
+        Text = My.Application.Info.ProductName

        LoadConfigSetting()

@ -278,6 +279,7 @@ Public Class Form1
                    item1.SubItems.Add(a.SizeEventLog.ToString)
                    item1.SubItems.Add(a.CatalogEventLog)
                    item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName
+                    item1.SubItems.Add(SavedIB.StartDate) 'StartAt

                    ListView.Items.Add(item1)

@ -302,6 +304,7 @@ Public Class Form1
                item1.SubItems.Add(a.SizeEventLog.ToString)
                item1.SubItems.Add(a.CatalogEventLog)
                item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName
+                item1.SubItems.Add(SavedIB.StartDate) 'StartAt

                ListView.Items.Add(item1)

@ -321,6 +324,7 @@ Public Class Form1
                item1.SubItems.Add(CalcullateFolderSize(IB.DatabaseCatalog))
                item1.SubItems.Add(IB.DatabaseCatalog)
                item1.SubItems.Add(IB.ESServerName)
+                item1.SubItems.Add(IB.StartDate) 'StartAt

                ListView.Items.Add(item1)
            End If
@ -489,6 +493,11 @@ Public Class Form1
            item1.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text))
            item1.SubItems.Add(AddPath.Path.Text)
            item1.SubItems.Add(AddPath.ESServerNameTextBox.Text)
+            If AddPath.UseFilterByDateCheckBox.Checked Then
+                item1.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd"))
+            Else
+                item1.SubItems.Add("")
+            End If

            ListView.Items.Add(item1)

@ -511,6 +520,15 @@ Public Class Form1
        AddPath.ESServerNameTextBox.Text = item.SubItems(5).Text
        AddPath.Description.Text = item.SubItems(2).Text
        AddPath.Path.Text = item.SubItems(4).Text
+
+        Dim DateStr = item.SubItems(6).Text
+        If String.IsNullOrEmpty(DateStr) Then
+            AddPath.UseFilterByDateCheckBox.Checked = False
+        Else
+            AddPath.UseFilterByDateCheckBox.Checked = True
+            AddPath.FilterByDateDateTimePicker.Value = DateTime.Parse(DateStr)
+        End If
+
        AddPath.ExistedRow = True
        AddPath.ShowDialog()

@ -524,14 +542,13 @@ Public Class Form1
            item.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text))
            item.SubItems.Add(AddPath.Path.Text)
            item.SubItems.Add(AddPath.ESServerNameTextBox.Text)
-
+            If AddPath.UseFilterByDateCheckBox.Checked Then
+                item.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd"))
+            Else
+                item.SubItems.Add("")
            End If

-
-
-        'End If
-
-
+        End If

    End Sub

--- a/EventLogLoaderService/ConfigSettings.vb
+++ b/EventLogLoaderService/ConfigSettings.vb
@ -7,6 +7,7 @@ Public Module ConfigSettingsModule
        Public DatabaseID As String = ""
        Public DatabaseName As String = ""
        Public DatabaseCatalog As String = ""
+        Public StartDate As String = ""
        Public Found As Boolean = False
    End Class

--- a/EventLogLoaderService/EventLogProcessor.vb
+++ b/EventLogLoaderService/EventLogProcessor.vb
@ -221,6 +221,7 @@ Public Class EventLogProcessor
    Public ItIsMySQL As Boolean = False
    Public ItIsES As Boolean = False
    Public ESUseSynonymsForFieldsNames As Boolean = False
+    Public LoadEventsStartingAt As Date
    Public ESFieldSynonyms As ElasticSearchFieldSynonymsClass = New ElasticSearchFieldSynonymsClass
    Public SleepTime As Integer = 60 * 1000 '1 минута

@ -1238,7 +1239,7 @@ Public Class EventLogProcessor
                                            [eventCode],
                                            [comment],
                                            [metadataCodes],
-                                            [sessionDataSplitCode],--
+                                            [sessionDataSplitCode],
                                            [dataType],
                                            [data],
                                            [dataPresentation],
@ -1246,11 +1247,15 @@ Public Class EventLogProcessor
                                            [primaryPortCode],
                                            [secondaryPortCode]
                                        FROM [EventLog] 
-                                        WHERE [rowID] > @LastEventNumber83 
+                                        WHERE [rowID] > @LastEventNumber83 AND [date] >= @MinimumDate
                                        ORDER BY 1
                                        LIMIT 1000"

                Command.Parameters.AddWithValue("LastEventNumber83", LastEventNumber83)
+
+                Dim noOfSeconds As Int64 = (LoadEventsStartingAt - New Date).TotalSeconds
+                Command.Parameters.AddWithValue("MinimumDate", noOfSeconds * 10000)
+
                Dim rs = Command.ExecuteReader

                Dim HasData = rs.HasRows
@ -1307,23 +1312,17 @@ Public Class EventLogProcessor
                    OneEvent.Transaction = ""
                    OneEvent.EventType = ""

-
-
                    EventsList.Add(OneEvent)

-                    If EventsList.Count >= 1000 Then
-                        'Console.WriteLine("Выгрузка 1000 событий: " + Now.ToString)
-                        SaveEventsToSQL()
-                    End If
-
                    LastEventNumber83 = OneEvent.RowID

                End While

-
                rs.Close()

+                If EventsList.Count > 0 Then
                    SaveEventsToSQL()
+                End If

                If Not HasData Then
                    Exit While
@ -1332,8 +1331,6 @@ Public Class EventLogProcessor
            End While


-
-
            Command.Dispose()
            Conn.Close()
            Conn.Dispose()
@ -1523,6 +1520,10 @@ Public Class EventLogProcessor
        OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider)
        OneEvent.TransactionStatus = Array(1)

+        If OneEvent.DateTime < LoadEventsStartingAt Then
+            Exit Sub
+        End If
+
        Dim TransStr = Array(2).ToString.Replace("}", "").Replace("{", "")
        Dim TransDate = From16To10(TransStr.Substring(0, TransStr.IndexOf(",")))

--- a/EventLogLoaderService/Service1.vb
+++ b/EventLogLoaderService/Service1.vb
@ -66,6 +66,13 @@ Public Class EventLogLoaderService
                    EventLogProcessorObj.ESUseSynonymsForFieldsNames = ConfigSettingObj.ESUseSynonymsForFieldsNames
                    EventLogProcessorObj.ESFieldSynonyms = ConfigSettingObj.ESFieldSynonyms

+                    Try
+                        EventLogProcessorObj.LoadEventsStartingAt = Date.Parse(IBConfig.StartDate)
+                    Catch ex As Exception
+                        EventLogProcessorObj.LoadEventsStartingAt = New Date(1900, 1, 1)
+                    End Try
+
+
                    ListOfProcessors.Add(EventLogProcessorObj)

                Next