From eea782a1f0cbc9f6776451fc03d9ea98b98aea95 Mon Sep 17 00:00:00 2001 From: Alex Bochkov Date: Mon, 20 Feb 2017 14:32:07 -0800 Subject: [PATCH] minimal date for parcing --- EventLogLoaderManager/AddPath.Designer.vb | 41 ++++++++++++++++++++-- EventLogLoaderManager/Form1.Designer.vb | 9 ++++- EventLogLoaderManager/Form1.vb | 31 ++++++++++++---- EventLogLoaderService/ConfigSettings.vb | 1 + EventLogLoaderService/EventLogProcessor.vb | 27 +++++++------- EventLogLoaderService/Service1.vb | 7 ++++ 6 files changed, 92 insertions(+), 24 deletions(-) diff --git a/EventLogLoaderManager/AddPath.Designer.vb b/EventLogLoaderManager/AddPath.Designer.vb index c6ebb04..ea3b6e9 100644 --- a/EventLogLoaderManager/AddPath.Designer.vb +++ b/EventLogLoaderManager/AddPath.Designer.vb @@ -36,6 +36,9 @@ Partial Class AddPath Me.ButtonChoosePath = New System.Windows.Forms.Button() Me.Label5 = New System.Windows.Forms.Label() Me.ESServerNameTextBox = New System.Windows.Forms.TextBox() + Me.FilterByDateDateTimePicker = New System.Windows.Forms.DateTimePicker() + Me.Label6 = New System.Windows.Forms.Label() + Me.UseFilterByDateCheckBox = New System.Windows.Forms.CheckBox() Me.SuspendLayout() ' 'IBName @@ -114,7 +117,7 @@ Partial Class AddPath 'Button1 ' Me.Button1.Font = New System.Drawing.Font("Microsoft Sans Serif", 8.25!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte)) - Me.Button1.Location = New System.Drawing.Point(12, 141) + Me.Button1.Location = New System.Drawing.Point(9, 195) Me.Button1.Name = "Button1" Me.Button1.Size = New System.Drawing.Size(200, 43) Me.Button1.TabIndex = 2 @@ -124,7 +127,7 @@ Partial Class AddPath 'Button2 ' Me.Button2.Font = New System.Drawing.Font("Microsoft Sans Serif", 8.25!, System.Drawing.FontStyle.Bold, System.Drawing.GraphicsUnit.Point, CType(204, Byte)) - Me.Button2.Location = New System.Drawing.Point(218, 141) + Me.Button2.Location = New System.Drawing.Point(215, 195) Me.Button2.Name = "Button2" Me.Button2.Size = New System.Drawing.Size(200, 43) Me.Button2.TabIndex = 2 @@ -163,11 +166,40 @@ Partial Class AddPath Me.ESServerNameTextBox.Size = New System.Drawing.Size(268, 20) Me.ESServerNameTextBox.TabIndex = 4 ' + 'FilterByDateDateTimePicker + ' + Me.FilterByDateDateTimePicker.Location = New System.Drawing.Point(213, 166) + Me.FilterByDateDateTimePicker.Name = "FilterByDateDateTimePicker" + Me.FilterByDateDateTimePicker.Size = New System.Drawing.Size(200, 20) + Me.FilterByDateDateTimePicker.TabIndex = 6 + ' + 'Label6 + ' + Me.Label6.AutoSize = True + Me.Label6.Location = New System.Drawing.Point(12, 144) + Me.Label6.Name = "Label6" + Me.Label6.Size = New System.Drawing.Size(88, 13) + Me.Label6.TabIndex = 7 + Me.Label6.Text = "Начальная дата" + ' + 'UseFilterByDateCheckBox + ' + Me.UseFilterByDateCheckBox.AutoSize = True + Me.UseFilterByDateCheckBox.Location = New System.Drawing.Point(145, 144) + Me.UseFilterByDateCheckBox.Name = "UseFilterByDateCheckBox" + Me.UseFilterByDateCheckBox.Size = New System.Drawing.Size(263, 17) + Me.UseFilterByDateCheckBox.TabIndex = 8 + Me.UseFilterByDateCheckBox.Text = "Загружать события начиная с указанной даты" + Me.UseFilterByDateCheckBox.UseVisualStyleBackColor = True + ' 'AddPath ' Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!) Me.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font - Me.ClientSize = New System.Drawing.Size(425, 188) + Me.ClientSize = New System.Drawing.Size(425, 244) + Me.Controls.Add(Me.UseFilterByDateCheckBox) + Me.Controls.Add(Me.Label6) + Me.Controls.Add(Me.FilterByDateDateTimePicker) Me.Controls.Add(Me.Label5) Me.Controls.Add(Me.ESServerNameTextBox) Me.Controls.Add(Me.ButtonChoosePath) @@ -202,4 +234,7 @@ Partial Class AddPath Friend WithEvents ButtonChoosePath As System.Windows.Forms.Button Friend WithEvents Label5 As Label Friend WithEvents ESServerNameTextBox As TextBox + Friend WithEvents FilterByDateDateTimePicker As DateTimePicker + Friend WithEvents Label6 As Label + Friend WithEvents UseFilterByDateCheckBox As CheckBox End Class diff --git a/EventLogLoaderManager/Form1.Designer.vb b/EventLogLoaderManager/Form1.Designer.vb index 14363bc..66dc9ad 100644 --- a/EventLogLoaderManager/Form1.Designer.vb +++ b/EventLogLoaderManager/Form1.Designer.vb @@ -51,13 +51,14 @@ Partial Class Form1 Me.Label5 = New System.Windows.Forms.Label() Me.UseSynonymsForFieldsNamesCheckBox = New System.Windows.Forms.CheckBox() Me.Label6 = New System.Windows.Forms.Label() + Me.StartAt = CType(New System.Windows.Forms.ColumnHeader(), System.Windows.Forms.ColumnHeader) Me.GroupBox1.SuspendLayout() Me.SuspendLayout() ' 'ListView ' Me.ListView.CheckBoxes = True - Me.ListView.Columns.AddRange(New System.Windows.Forms.ColumnHeader() {Me.InfobaseName, Me.InfobaseGUID, Me.InfobaseDescription, Me.IBEvLogSize, Me.InfobaseLogPath, Me.ESServerName}) + Me.ListView.Columns.AddRange(New System.Windows.Forms.ColumnHeader() {Me.InfobaseName, Me.InfobaseGUID, Me.InfobaseDescription, Me.IBEvLogSize, Me.InfobaseLogPath, Me.ESServerName, Me.StartAt}) Me.ListView.Dock = System.Windows.Forms.DockStyle.Fill Me.ListView.FullRowSelect = True Me.ListView.GridLines = True @@ -332,6 +333,11 @@ Partial Class Form1 Me.Label6.TabIndex = 24 Me.Label6.Text = "Использовать свои имена полей" ' + 'StartAt + ' + Me.StartAt.Text = "Начальная дата" + Me.StartAt.Width = 150 + ' 'Form1 ' Me.AutoScaleDimensions = New System.Drawing.SizeF(6.0!, 13.0!) @@ -395,4 +401,5 @@ Partial Class Form1 Friend WithEvents Label5 As Label Friend WithEvents UseSynonymsForFieldsNamesCheckBox As CheckBox Friend WithEvents Label6 As Label + Friend WithEvents StartAt As ColumnHeader End Class diff --git a/EventLogLoaderManager/Form1.vb b/EventLogLoaderManager/Form1.vb index b7c952b..a77c9a7 100644 --- a/EventLogLoaderManager/Form1.vb +++ b/EventLogLoaderManager/Form1.vb @@ -150,6 +150,7 @@ Public Class Form1 IBSetting.DatabaseName = Item.SubItems(0).Text IBSetting.DatabaseCatalog = Item.SubItems(4).Text IBSetting.ESServerName = Item.SubItems(5).Text + IBSetting.StartDate = Item.SubItems(6).Text NewConfigSetting.Infobases.Add(IBSetting) @@ -228,7 +229,7 @@ Public Class Form1 Private Sub Form1_Load(sender As System.Object, e As System.EventArgs) Handles MyBase.Load - Text = My.Application.Info.ProductName + " / " + My.Application.Info.Copyright + Text = My.Application.Info.ProductName LoadConfigSetting() @@ -278,6 +279,7 @@ Public Class Form1 item1.SubItems.Add(a.SizeEventLog.ToString) item1.SubItems.Add(a.CatalogEventLog) item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName + item1.SubItems.Add(SavedIB.StartDate) 'StartAt ListView.Items.Add(item1) @@ -302,6 +304,7 @@ Public Class Form1 item1.SubItems.Add(a.SizeEventLog.ToString) item1.SubItems.Add(a.CatalogEventLog) item1.SubItems.Add(SavedIB.ESServerName) 'ESServerName + item1.SubItems.Add(SavedIB.StartDate) 'StartAt ListView.Items.Add(item1) @@ -321,6 +324,7 @@ Public Class Form1 item1.SubItems.Add(CalcullateFolderSize(IB.DatabaseCatalog)) item1.SubItems.Add(IB.DatabaseCatalog) item1.SubItems.Add(IB.ESServerName) + item1.SubItems.Add(IB.StartDate) 'StartAt ListView.Items.Add(item1) End If @@ -489,6 +493,11 @@ Public Class Form1 item1.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text)) item1.SubItems.Add(AddPath.Path.Text) item1.SubItems.Add(AddPath.ESServerNameTextBox.Text) + If AddPath.UseFilterByDateCheckBox.Checked Then + item1.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd")) + Else + item1.SubItems.Add("") + End If ListView.Items.Add(item1) @@ -511,6 +520,15 @@ Public Class Form1 AddPath.ESServerNameTextBox.Text = item.SubItems(5).Text AddPath.Description.Text = item.SubItems(2).Text AddPath.Path.Text = item.SubItems(4).Text + + Dim DateStr = item.SubItems(6).Text + If String.IsNullOrEmpty(DateStr) Then + AddPath.UseFilterByDateCheckBox.Checked = False + Else + AddPath.UseFilterByDateCheckBox.Checked = True + AddPath.FilterByDateDateTimePicker.Value = DateTime.Parse(DateStr) + End If + AddPath.ExistedRow = True AddPath.ShowDialog() @@ -524,15 +542,14 @@ Public Class Form1 item.SubItems.Add(CalcullateFolderSize(AddPath.Path.Text)) item.SubItems.Add(AddPath.Path.Text) item.SubItems.Add(AddPath.ESServerNameTextBox.Text) + If AddPath.UseFilterByDateCheckBox.Checked Then + item.SubItems.Add(AddPath.FilterByDateDateTimePicker.Value.ToString("yyyy-MM-dd")) + Else + item.SubItems.Add("") + End If End If - - - 'End If - - - End Sub Private Sub LinkLabel2_LinkClicked(sender As Object, e As LinkLabelLinkClickedEventArgs) Handles LinkLabel2.LinkClicked diff --git a/EventLogLoaderService/ConfigSettings.vb b/EventLogLoaderService/ConfigSettings.vb index 41de394..7380349 100644 --- a/EventLogLoaderService/ConfigSettings.vb +++ b/EventLogLoaderService/ConfigSettings.vb @@ -7,6 +7,7 @@ Public Module ConfigSettingsModule Public DatabaseID As String = "" Public DatabaseName As String = "" Public DatabaseCatalog As String = "" + Public StartDate As String = "" Public Found As Boolean = False End Class diff --git a/EventLogLoaderService/EventLogProcessor.vb b/EventLogLoaderService/EventLogProcessor.vb index 4804764..4f102c0 100644 --- a/EventLogLoaderService/EventLogProcessor.vb +++ b/EventLogLoaderService/EventLogProcessor.vb @@ -221,6 +221,7 @@ Public Class EventLogProcessor Public ItIsMySQL As Boolean = False Public ItIsES As Boolean = False Public ESUseSynonymsForFieldsNames As Boolean = False + Public LoadEventsStartingAt As Date Public ESFieldSynonyms As ElasticSearchFieldSynonymsClass = New ElasticSearchFieldSynonymsClass Public SleepTime As Integer = 60 * 1000 '1 минута @@ -1238,7 +1239,7 @@ Public Class EventLogProcessor [eventCode], [comment], [metadataCodes], - [sessionDataSplitCode],-- + [sessionDataSplitCode], [dataType], [data], [dataPresentation], @@ -1246,11 +1247,15 @@ Public Class EventLogProcessor [primaryPortCode], [secondaryPortCode] FROM [EventLog] - WHERE [rowID] > @LastEventNumber83 + WHERE [rowID] > @LastEventNumber83 AND [date] >= @MinimumDate ORDER BY 1 LIMIT 1000" Command.Parameters.AddWithValue("LastEventNumber83", LastEventNumber83) + + Dim noOfSeconds As Int64 = (LoadEventsStartingAt - New Date).TotalSeconds + Command.Parameters.AddWithValue("MinimumDate", noOfSeconds * 10000) + Dim rs = Command.ExecuteReader Dim HasData = rs.HasRows @@ -1307,23 +1312,17 @@ Public Class EventLogProcessor OneEvent.Transaction = "" OneEvent.EventType = "" - - EventsList.Add(OneEvent) - If EventsList.Count >= 1000 Then - 'Console.WriteLine("Выгрузка 1000 событий: " + Now.ToString) - SaveEventsToSQL() - End If - LastEventNumber83 = OneEvent.RowID End While - rs.Close() - SaveEventsToSQL() + If EventsList.Count > 0 Then + SaveEventsToSQL() + End If If Not HasData Then Exit While @@ -1332,8 +1331,6 @@ Public Class EventLogProcessor End While - - Command.Dispose() Conn.Close() Conn.Dispose() @@ -1523,6 +1520,10 @@ Public Class EventLogProcessor OneEvent.DateTime = Date.ParseExact(Array(0), "yyyyMMddHHmmss", provider) OneEvent.TransactionStatus = Array(1) + If OneEvent.DateTime < LoadEventsStartingAt Then + Exit Sub + End If + Dim TransStr = Array(2).ToString.Replace("}", "").Replace("{", "") Dim TransDate = From16To10(TransStr.Substring(0, TransStr.IndexOf(","))) diff --git a/EventLogLoaderService/Service1.vb b/EventLogLoaderService/Service1.vb index 675495f..589361e 100644 --- a/EventLogLoaderService/Service1.vb +++ b/EventLogLoaderService/Service1.vb @@ -66,6 +66,13 @@ Public Class EventLogLoaderService EventLogProcessorObj.ESUseSynonymsForFieldsNames = ConfigSettingObj.ESUseSynonymsForFieldsNames EventLogProcessorObj.ESFieldSynonyms = ConfigSettingObj.ESFieldSynonyms + Try + EventLogProcessorObj.LoadEventsStartingAt = Date.Parse(IBConfig.StartDate) + Catch ex As Exception + EventLogProcessorObj.LoadEventsStartingAt = New Date(1900, 1, 1) + End Try + + ListOfProcessors.Add(EventLogProcessorObj) Next