1
0
mirror of https://github.com/romanlryji/EventLogLoader.git synced 2024-11-24 08:32:52 +02:00
This commit is contained in:
Alex Bochkov 2016-12-25 17:15:40 -08:00
parent 0273181cf0
commit f28d00beaf
10 changed files with 114 additions and 72 deletions

View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
@ -14,7 +14,7 @@
<AssemblyName>EventLogLoader</AssemblyName>
<FileAlignment>512</FileAlignment>
<MyType>Console</MyType>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<TargetFrameworkVersion>v4.6</TargetFrameworkVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
@ -85,6 +85,10 @@
<ItemGroup>
<Reference Include="System" />
<Reference Include="System.Data" />
<Reference Include="System.Data.SQLite, Version=1.0.104.0, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.Core.1.0.104.0\lib\net46\System.Data.SQLite.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Deployment" />
<Reference Include="System.ServiceProcess" />
<Reference Include="System.Xml" />
@ -139,6 +143,7 @@
<CustomToolNamespace>My</CustomToolNamespace>
<LastGenOutput>Settings.Designer.vb</LastGenOutput>
</None>
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\EventLogLoaderManager\EventLogLoaderManager.vbproj">
@ -168,6 +173,13 @@
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.VisualBasic.targets" />
<Import Project="..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets" Condition="Exists('..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets'))" />
</Target>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.
<Target Name="BeforeBuild">

View File

@ -1,12 +1,12 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" /></startup>
<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6"/></startup>
<runtime>
<assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
<dependentAssembly>
<assemblyIdentity name="System.Data.SQLite" publicKeyToken="db937bc2d44ff139" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-1.0.94.0" newVersion="1.0.94.0" />
<assemblyIdentity name="System.Data.SQLite" publicKeyToken="db937bc2d44ff139" culture="neutral"/>
<bindingRedirect oldVersion="0.0.0.0-1.0.94.0" newVersion="1.0.94.0"/>
</dependentAssembly>
</assemblyBinding>
</runtime>

View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="System.Data.SQLite.Core" version="1.0.104.0" targetFramework="net46" />
</packages>

View File

@ -1,11 +1,11 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" /></startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6"/></startup>
<system.data>
<DbProviderFactories>
<remove invariant="MySql.Data.MySqlClient" />
<add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient" description=".Net Framework Data Provider for MySQL" type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.9.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d" />
<remove invariant="MySql.Data.MySqlClient"/>
<add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient" description=".Net Framework Data Provider for MySQL" type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.9.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d"/>
</DbProviderFactories>
</system.data></configuration>

View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
@ -13,7 +13,7 @@
<AssemblyName>EventLogLoaderManager</AssemblyName>
<FileAlignment>512</FileAlignment>
<MyType>WindowsForms</MyType>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<TargetFrameworkVersion>v4.6</TargetFrameworkVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>

View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
<Platform Condition=" '$(Platform)' == '' ">x86</Platform>
@ -14,7 +14,7 @@
<AssemblyName>EventLogLoaderService</AssemblyName>
<FileAlignment>512</FileAlignment>
<MyType>Console</MyType>
<TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
<TargetFrameworkVersion>v4.6</TargetFrameworkVersion>
<IsWebBootstrapper>false</IsWebBootstrapper>
<NuGetPackageImportStamp>
</NuGetPackageImportStamp>
@ -73,15 +73,15 @@
</PropertyGroup>
<ItemGroup>
<Reference Include="Elasticsearch.Net, Version=5.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
<HintPath>..\packages\Elasticsearch.Net.5.0.0\lib\net45\Elasticsearch.Net.dll</HintPath>
<HintPath>..\packages\Elasticsearch.Net.5.0.0-rc4\lib\net46\Elasticsearch.Net.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.dll</HintPath>
<HintPath>..\packages\EntityFramework.6.1.3\lib\net45\EntityFramework.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="EntityFramework.SqlServer, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089, processorArchitecture=MSIL">
<HintPath>..\packages\EntityFramework.6.0.0\lib\net45\EntityFramework.SqlServer.dll</HintPath>
<HintPath>..\packages\EntityFramework.6.1.3\lib\net45\EntityFramework.SqlServer.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="MySql.Data, Version=6.9.9.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d, processorArchitecture=MSIL">
@ -89,7 +89,7 @@
<Private>True</Private>
</Reference>
<Reference Include="Nest, Version=5.0.0.0, Culture=neutral, PublicKeyToken=96c599bbe3e70f5d, processorArchitecture=MSIL">
<HintPath>..\packages\NEST.5.0.0\lib\net45\Nest.dll</HintPath>
<HintPath>..\packages\NEST.5.0.0-rc4\lib\net46\Nest.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json, Version=9.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
@ -105,11 +105,7 @@
<Reference Include="System.Configuration.Install" />
<Reference Include="System.Data" />
<Reference Include="System.Data.SQLite, Version=1.0.104.0, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.Core.1.0.104.0\lib\net45\System.Data.SQLite.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Data.SQLite.EF6, Version=1.0.104.0, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
<HintPath>..\packages\System.Data.SQLite.EF6.1.0.104.0\lib\net45\System.Data.SQLite.EF6.dll</HintPath>
<HintPath>..\packages\System.Data.SQLite.Core.1.0.104.0\lib\net46\System.Data.SQLite.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Deployment" />
@ -203,12 +199,12 @@
</BootstrapperPackage>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.VisualBasic.targets" />
<Import Project="..\packages\System.Data.SQLite.Core.1.0.104.0\build\net45\System.Data.SQLite.Core.targets" Condition="Exists('..\packages\System.Data.SQLite.Core.1.0.104.0\build\net45\System.Data.SQLite.Core.targets')" />
<Import Project="..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets" Condition="Exists('..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets')" />
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('..\packages\System.Data.SQLite.Core.1.0.104.0\build\net45\System.Data.SQLite.Core.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\System.Data.SQLite.Core.1.0.104.0\build\net45\System.Data.SQLite.Core.targets'))" />
<Error Condition="!Exists('..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\System.Data.SQLite.Core.1.0.104.0\build\net46\System.Data.SQLite.Core.targets'))" />
</Target>
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.

View File

@ -64,7 +64,7 @@ Public Class EventLogProcessor
Dim Item = New Computer
Item.Code = Code
Item.Name = Name
Item.Name = Name.Replace("""", "")
DictComputers.Add(Code, Item)
@ -74,7 +74,7 @@ Public Class EventLogProcessor
Dim Item = New Application
Item.Code = Code
Item.Name = Name
Item.Name = Name.Replace("""", "")
DictApplications.Add(Code, Item)
@ -105,7 +105,7 @@ Public Class EventLogProcessor
Dim Item = New Server
Item.Code = Code
Item.Name = Name
Item.Name = Name.Replace("""", "")
DictServers.Add(Code, Item)
@ -176,10 +176,20 @@ Public Class EventLogProcessor
Class ESRecord
Public DateTime As Date
Public EventType As EventType
Public EventTypeString As String
Public Computer As Computer
Public Application As Application
Public Metadata As Metadata
Public UserName As User
Public Seance As Integer
Public DataStructure As String
Public DataString As String
Public Comment As String
Public MainPort As Integer
Public SecondPort As Integer
Public Server As String
End Class
Public EventsList As List(Of OneEventRecord) = New List(Of OneEventRecord)
@ -708,7 +718,7 @@ Public Class EventLogProcessor
SaveReadParametersToFile()
Console.WriteLine(Now.ToShortTimeString + " Записано новых событий в базу " + i.ToString)
Console.WriteLine(Now.ToShortTimeString + " New records have been processed " + i.ToString)
command.CommandText = "COMMIT TRANSACTION"
command.Parameters.Clear()
@ -771,7 +781,7 @@ Public Class EventLogProcessor
Next
Console.WriteLine(Now.ToShortTimeString + " Записано новых событий в базу " + i.ToString)
Console.WriteLine(Now.ToShortTimeString + " New records have been processed " + i.ToString)
SaveReadParametersToFile()
@ -796,12 +806,42 @@ Public Class EventLogProcessor
For Each EventRecord In EventsList
Dim ESRecord = New ESRecord
ESRecord.DateTime = EventRecord.DateTime
ESRecord.Seance = EventRecord.Seance
ESRecord.DataStructure = EventRecord.DataStructure
ESRecord.DataString = EventRecord.DataString
ESRecord.Comment = EventRecord.Comment
ESRecord.EventTypeString = EventRecord.EventType
Dim EventObj = New EventType
If DictEvents.TryGetValue(EventRecord.EventID, EventObj) Then
ESRecord.EventType = EventObj
End If
Dim MetadataObj = New Metadata
If DictMetadata.TryGetValue(EventRecord.MetadataID, MetadataObj) Then
ESRecord.Metadata = MetadataObj
End If
Dim ComputerObj = New Computer
If DictComputers.TryGetValue(EventRecord.ComputerName, ComputerObj) Then
ESRecord.Computer = ComputerObj
End If
Dim MainPortObj = New MainPort
If DictMainPorts.TryGetValue(EventRecord.MainPortID, MainPortObj) Then
ESRecord.MainPort = MainPortObj.Name
End If
Dim ServerObj = New Server
If DictServers.TryGetValue(EventRecord.ServerID, ServerObj) Then
ESRecord.Server = ServerObj.Name
End If
Dim SecondPortObj = New SecondPort
If DictSecondPorts.TryGetValue(EventRecord.SecondPortID, SecondPortObj) Then
ESRecord.SecondPort = SecondPortObj.Name
End If
Dim ApplicationObj = New Application
If DictApplications.TryGetValue(EventRecord.AppName, ApplicationObj) Then
ESRecord.Application = ApplicationObj
@ -815,19 +855,10 @@ Public Class EventLogProcessor
NewRecords.Add(ESRecord)
Next
'Dim elasticsearchMappingResolver As IElasticsearchMappingResolver = New ElasticsearchMappingResolver()
'elasticsearchMappingResolver.AddElasticSearchMappingForEntityType(TypeOf (), New ElasticsearchMappingTestDto());
'Dim context = New ElasticsearchContext(ConnectionString, New ElasticsearchSerializerConfiguration(elasticsearchMappingResolver, True, True))
'Dim descriptor = New BulkDescriptor()
'For Each item In Events
' ' descriptor.Index(Of OneEvent)(Function(op) op.Document(item))
'Next
'Dim result = _current.Bulk(descriptor)
Dim Result = _current.IndexMany(NewRecords, Index, "event-record")
Console.WriteLine(Now.ToShortTimeString + " New records have been processed " + NewRecords.Count.ToString)
SaveReadParametersToFile()
End If
@ -899,7 +930,7 @@ Public Class EventLogProcessor
End If
Catch ex As Exception
Log.ErrorException("Ошибка при работе со справочником", ex)
Log.ErrorException("Error occurred while working with reference file", ex)
End Try
@ -909,9 +940,9 @@ Public Class EventLogProcessor
If My.Computer.FileSystem.FileExists(FileName) Then
Dim Conn = New System.Data.SQLite.SQLiteConnection("Data Source=" + FileName)
Dim Conn = New SQLite.SQLiteConnection("Data Source=" + FileName)
Conn.Open()
Dim Command = New System.Data.SQLite.SQLiteCommand
Dim Command = New SQLite.SQLiteCommand
Command.Connection = Conn
Command.CommandText = "SELECT [code], [name] FROM [AppCodes]"
@ -979,7 +1010,7 @@ Public Class EventLogProcessor
End If
Catch ex As Exception
Log.ErrorException("Ошибка при работе со справочником", ex)
Log.ErrorException("Error occurred while working with reference tables", ex)
End Try
End Sub
@ -1057,6 +1088,7 @@ Public Class EventLogProcessor
" FROM [EventLog] WHERE [rowID] > " + LastEventNumber83.ToString + " LIMIT 1000"
Dim rs = Command.ExecuteReader
Dim HasData = rs.HasRows
While rs.Read
Dim OneEvent = New OneEventRecord
@ -1112,17 +1144,24 @@ Public Class EventLogProcessor
EventsList.Add(OneEvent)
If EventsList.Count >= 1000 Then
'Console.WriteLine("Выгрузка 1000 событий: " + Now.ToString)
SaveEventsToSQL()
End If
LastEventNumber83 = OneEvent.RowID
End While
rs.Close()
If EventsList.Count = 0 Then
SaveEventsToSQL()
If Not HasData Then
Exit While
End If
SaveReferenceValuesToDatabase()
End While

View File

@ -116,7 +116,6 @@ Public Class EventLogLoaderService
WorkerThread.SetApartmentState(Threading.ApartmentState.STA)
End Sub
Private Sub DoWork()
@ -257,6 +256,7 @@ Public Class EventLogLoaderService
command.Dispose()
objConn.Close()
objConn.Dispose()
ElseIf ItIsMySQL Then
@ -310,7 +310,7 @@ Public Class EventLogLoaderService
command.Dispose()
objConn.Close()
objConn.Dispose()
End If

View File

@ -1,31 +1,25 @@
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<configSections>
<section name="entityFramework" type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089" requirePermission="false" />
<!-- For more information on Entity Framework configuration, visit http://go.microsoft.com/fwlink/?LinkID=237468 -->
<section name="entityFramework"
type="System.Data.Entity.Internal.ConfigFile.EntityFrameworkSection, EntityFramework, Version=6.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089"
requirePermission="false"/>
</configSections>
<startup>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/>
<supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.6" />
</startup>
<entityFramework>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.SqlConnectionFactory, EntityFramework"/>
<defaultConnectionFactory type="System.Data.Entity.Infrastructure.SqlConnectionFactory, EntityFramework" />
<providers>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer"/>
<provider invariantName="System.Data.SQLite.EF6" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6"/>
<provider invariantName="System.Data.SqlClient" type="System.Data.Entity.SqlServer.SqlProviderServices, EntityFramework.SqlServer" />
<provider invariantName="System.Data.SQLite.EF6" type="System.Data.SQLite.EF6.SQLiteProviderServices, System.Data.SQLite.EF6" />
</providers>
</entityFramework>
<system.data>
<DbProviderFactories>
<remove invariant="MySql.Data.MySqlClient" />
<add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient" description=".Net Framework Data Provider for MySQL" type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.9.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d" />
<remove invariant="MySql.Data.MySqlClient"/><add name="MySQL Data Provider" invariant="MySql.Data.MySqlClient"
description=".Net Framework Data Provider for MySQL"
type="MySql.Data.MySqlClient.MySqlClientFactory, MySql.Data, Version=6.9.9.0, Culture=neutral, PublicKeyToken=c5687fc88969c44d"/><remove invariant="System.Data.SQLite.EF6"/><add name="SQLite Data Provider (Entity Framework 6)"
invariant="System.Data.SQLite.EF6" description=".NET Framework Data Provider for SQLite (Entity Framework 6)"
type="System.Data.SQLite.EF6.SQLiteProviderFactory, System.Data.SQLite.EF6"/></DbProviderFactories>
</DbProviderFactories>
</system.data>
</configuration>
</configuration>

View File

@ -1,12 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Elasticsearch.Net" version="5.0.0" targetFramework="net45" />
<package id="EntityFramework" version="6.0.0" targetFramework="net45" />
<package id="Elasticsearch.Net" version="5.0.0-rc4" targetFramework="net46" />
<package id="EntityFramework" version="6.1.3" targetFramework="net45" />
<package id="MySql.Data" version="6.9.9" targetFramework="net45" />
<package id="NEST" version="5.0.0" targetFramework="net45" />
<package id="NEST" version="5.0.0-rc4" targetFramework="net46" />
<package id="Newtonsoft.Json" version="9.0.1" targetFramework="net45" />
<package id="System.Data.SQLite" version="1.0.104.0" targetFramework="net45" />
<package id="System.Data.SQLite.Core" version="1.0.104.0" targetFramework="net45" />
<package id="System.Data.SQLite.EF6" version="1.0.104.0" targetFramework="net45" />
<package id="System.Data.SQLite.Linq" version="1.0.104.0" targetFramework="net45" />
<package id="System.Data.SQLite.Core" version="1.0.104.0" targetFramework="net46" />
</packages>