From 5a770ded5b7e9eaa42596527aac25be824bba368 Mon Sep 17 00:00:00 2001 From: YPermitin Date: Sun, 24 Jan 2021 23:02:55 +0500 Subject: [PATCH] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B5=20=D0=B1=D0=B8=D0=B1=D0=BB=D0=B8=D0=BE=D1=82?= =?UTF-8?q?=D0=B5=D0=BA=20=D0=B8=20=D0=B0=D0=B4=D0=B0=D0=BF=D1=82=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D1=8F=20=D0=BA=20=D0=BF=D0=BE=D1=81=D0=BB=D0=B5?= =?UTF-8?q?=D0=B4=D0=BD=D0=B5=D0=B9=20=D0=B2=D0=B5=D1=80=D1=81=D0=B8=D0=B8?= =?UTF-8?q?=20ClickHouse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Обновлены служебные библиотеки для тестирования - Обновлен пакет работы с PostgreSQL - Адаптирован код для работы с ClickHouse свежих версий - Немного рефакторинга --- .../ClickHouseContext.cs | 106 +++++------------- ....EventLogExportAssistant.PostgreSQL.csproj | 8 +- ...LogExportAssistant.ClickHouse.Tests.csproj | 2 +- ....EventLogExportAssistant.Core.Tests.csproj | 2 +- ...ExportAssistant.ElasticSearch.Tests.csproj | 2 +- ...EventLogExportAssistant.MySQL.Tests.csproj | 2 +- ...LogExportAssistant.PostgreSQL.Tests.csproj | 2 +- ...tLogExportAssistant.SQLServer.Tests.csproj | 2 +- 8 files changed, 36 insertions(+), 90 deletions(-) diff --git a/Libs/YY.EventLogExportAssistant.ClickHouse/ClickHouseContext.cs b/Libs/YY.EventLogExportAssistant.ClickHouse/ClickHouseContext.cs index 8547e0c..6b855e6 100644 --- a/Libs/YY.EventLogExportAssistant.ClickHouse/ClickHouseContext.cs +++ b/Libs/YY.EventLogExportAssistant.ClickHouse/ClickHouseContext.cs @@ -28,7 +28,7 @@ namespace YY.EventLogExportAssistant.ClickHouse private ClickHouseConnection _connection; private long logFileLastId = -1; - private IExtendedActions _extendedActions; + private readonly IExtendedActions _extendedActions; #endregion @@ -234,86 +234,32 @@ namespace YY.EventLogExportAssistant.ClickHouse } public void SaveLogPosition(InformationSystemsBase system, FileInfo logFileInfo, EventLogPosition position) { - var commandAddLogInfo = _connection.CreateCommand(); - commandAddLogInfo.CommandText = - @"INSERT INTO LogFiles ( - InformationSystem, - Id, - FileName, - CreateDate, - ModificationDate, - LastEventNumber, - LastCurrentFileReferences, - LastCurrentFileData, - LastStreamPosition - ) VALUES ( - {isId:String}, - {newId:Int64}, - {FileName:String}, - {CreateDate:DateTime}, - {ModificationDate:DateTime}, - {LastEventNumber:Int64}, - {LastCurrentFileReferences:String}, - {LastCurrentFileData:String}, - {LastStreamPosition:Int64} - )"; + using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection) + { + DestinationTableName = "LogFiles", + BatchSize = 100000 + }) + { + long logFileNewId = GetLogFileInfoNewId(system); + IEnumerable values = new List() + { + new object[] + { + system.Name, + logFileNewId, + logFileInfo.Name, + logFileInfo.CreationTimeUtc, + logFileInfo.LastWriteTimeUtc, + position.EventNumber, + position.CurrentFileReferences.Replace("\\", "\\\\"), + position.CurrentFileData.Replace("\\", "\\\\"), + position.StreamPosition ?? 0 + } + }.AsEnumerable(); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "isId", - DbType = DbType.Int64, - Value = system.Name - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "newId", - DbType = DbType.Int64, - Value = GetLogFileInfoNewId(system) - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "FileName", - DbType = DbType.AnsiString, - Value = logFileInfo.Name - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "CreateDate", - DbType = DbType.DateTime, - Value = logFileInfo.CreationTimeUtc - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "ModificationDate", - DbType = DbType.DateTime, - Value = logFileInfo.LastWriteTimeUtc - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "LastEventNumber", - DbType = DbType.Int64, - Value = position.EventNumber - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "LastCurrentFileReferences", - DbType = DbType.AnsiString, - Value = position.CurrentFileReferences.Replace("\\", "\\\\") - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "LastCurrentFileData", - DbType = DbType.AnsiString, - Value = position.CurrentFileData.Replace("\\", "\\\\") - }); - commandAddLogInfo.Parameters.Add(new ClickHouseDbParameter - { - ParameterName = "LastStreamPosition", - DbType = DbType.Int64, - Value = position.StreamPosition ?? 0 - }); - - commandAddLogInfo.ExecuteNonQuery(); + var bulkResult = bulkCopyInterface.WriteToServerAsync(values); + bulkResult.Wait(); + } } public long GetLogFileInfoNewId(InformationSystemsBase system) { diff --git a/Libs/YY.EventLogExportAssistant.PostgreSQL/YY.EventLogExportAssistant.PostgreSQL.csproj b/Libs/YY.EventLogExportAssistant.PostgreSQL/YY.EventLogExportAssistant.PostgreSQL.csproj index 2cfd4b7..0afa930 100644 --- a/Libs/YY.EventLogExportAssistant.PostgreSQL/YY.EventLogExportAssistant.PostgreSQL.csproj +++ b/Libs/YY.EventLogExportAssistant.PostgreSQL/YY.EventLogExportAssistant.PostgreSQL.csproj @@ -2,7 +2,7 @@ netstandard2.1 - 1.0.0.47 + 1.0.0.48 YY.EventLogExportAssistant.PostgreSQL Permitin Yuriy Event log export assistant for PostgreSQL @@ -14,9 +14,9 @@ GIT event, log, 1C, enterprise, export, postgres, postgresql true - 1.0.0.47 + 1.0.0.48 Just beginning... - 1.0.0.47 + 1.0.0.48 @@ -32,7 +32,7 @@ - + diff --git a/Tests/YY.EventLogExportAssistant.ClickHouse.Tests/YY.EventLogExportAssistant.ClickHouse.Tests.csproj b/Tests/YY.EventLogExportAssistant.ClickHouse.Tests/YY.EventLogExportAssistant.ClickHouse.Tests.csproj index b029511..2340fe6 100644 --- a/Tests/YY.EventLogExportAssistant.ClickHouse.Tests/YY.EventLogExportAssistant.ClickHouse.Tests.csproj +++ b/Tests/YY.EventLogExportAssistant.ClickHouse.Tests/YY.EventLogExportAssistant.ClickHouse.Tests.csproj @@ -14,7 +14,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/YY.EventLogExportAssistant.Core.Tests/YY.EventLogExportAssistant.Core.Tests.csproj b/Tests/YY.EventLogExportAssistant.Core.Tests/YY.EventLogExportAssistant.Core.Tests.csproj index 8dff231..19138a6 100644 --- a/Tests/YY.EventLogExportAssistant.Core.Tests/YY.EventLogExportAssistant.Core.Tests.csproj +++ b/Tests/YY.EventLogExportAssistant.Core.Tests/YY.EventLogExportAssistant.Core.Tests.csproj @@ -20,7 +20,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/YY.EventLogExportAssistant.ElasticSearch.Tests/YY.EventLogExportAssistant.ElasticSearch.Tests.csproj b/Tests/YY.EventLogExportAssistant.ElasticSearch.Tests/YY.EventLogExportAssistant.ElasticSearch.Tests.csproj index 1a9a853..b4860eb 100644 --- a/Tests/YY.EventLogExportAssistant.ElasticSearch.Tests/YY.EventLogExportAssistant.ElasticSearch.Tests.csproj +++ b/Tests/YY.EventLogExportAssistant.ElasticSearch.Tests/YY.EventLogExportAssistant.ElasticSearch.Tests.csproj @@ -17,7 +17,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/YY.EventLogExportAssistant.MySQL.Tests/YY.EventLogExportAssistant.MySQL.Tests.csproj b/Tests/YY.EventLogExportAssistant.MySQL.Tests/YY.EventLogExportAssistant.MySQL.Tests.csproj index 75ab742..dcb7a0a 100644 --- a/Tests/YY.EventLogExportAssistant.MySQL.Tests/YY.EventLogExportAssistant.MySQL.Tests.csproj +++ b/Tests/YY.EventLogExportAssistant.MySQL.Tests/YY.EventLogExportAssistant.MySQL.Tests.csproj @@ -14,7 +14,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/YY.EventLogExportAssistant.PostgreSQL.Tests/YY.EventLogExportAssistant.PostgreSQL.Tests.csproj b/Tests/YY.EventLogExportAssistant.PostgreSQL.Tests/YY.EventLogExportAssistant.PostgreSQL.Tests.csproj index 3cae971..49473fe 100644 --- a/Tests/YY.EventLogExportAssistant.PostgreSQL.Tests/YY.EventLogExportAssistant.PostgreSQL.Tests.csproj +++ b/Tests/YY.EventLogExportAssistant.PostgreSQL.Tests/YY.EventLogExportAssistant.PostgreSQL.Tests.csproj @@ -15,7 +15,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/YY.EventLogExportAssistant.SQLServer.Tests/YY.EventLogExportAssistant.SQLServer.Tests.csproj b/Tests/YY.EventLogExportAssistant.SQLServer.Tests/YY.EventLogExportAssistant.SQLServer.Tests.csproj index d99d0ea..6f4f205 100644 --- a/Tests/YY.EventLogExportAssistant.SQLServer.Tests/YY.EventLogExportAssistant.SQLServer.Tests.csproj +++ b/Tests/YY.EventLogExportAssistant.SQLServer.Tests/YY.EventLogExportAssistant.SQLServer.Tests.csproj @@ -18,7 +18,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive