Реализован механизм общего буфера и другие улучшения

1. Отказ от таблиц в ClickHouse с движком Buffer из-за проблем со стабильностью работы.
2. Отказ от хранения имени каталога в объекте технологического журнала
3. Улучшены операции массовой вставки данных записей логов
4. Оптимизация количества обращений к ClickHouse при экспорте логов
5. Реализован вспомогательный класс экспорта логов в буфер
6. Реализован механизм общего буфера для передачи записей логов в ClickHouse из различных источников
7. Расширен набор unit-тестов
8. Добавлен пример приложения для работы с общим буфером
9. Обновлены версии библиотек
10. Исправление ошибок и другие небольшие улучшения
This commit is contained in:
YPermitin
2021-04-29 01:34:11 +05:00
parent 98a2266e08
commit e3be595fa3
27 changed files with 1328 additions and 150 deletions
@@ -73,7 +73,6 @@ namespace YY.TechJournalExportAssistantConsoleApp
target.SetInformationSystem(new TechJournalLogBase()
{
Name = techJournalName,
DirectoryName = tjDirectory.DirectoryData.Name,
Description = techJournalDescription
});
exporter.SetTarget(target);
@@ -15,4 +15,6 @@
</None>
</ItemGroup>
<ProjectExtensions><VisualStudio><UserProperties appsettings_1json__JsonSchema="" /></VisualStudio></ProjectExtensions>
</Project>
@@ -0,0 +1,46 @@
using Microsoft.Extensions.Configuration;
using System;
using System.Linq;
using System.Threading.Tasks;
using YY.TechJournalExportAssistant.ClickHouse;
using YY.TechJournalExportAssistant.Core;
using YY.TechJournalExportAssistant.Core.SharedBuffer;
using YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs;
namespace YY.TechJournalExportAssistantWithSharedBufferConsoleApp
{
class Program
{
public static IConfiguration Configuration = new ConfigurationBuilder()
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: false)
.Build();
static async Task Main(string[] args)
{
TechJournalSettings settings = TechJournalSettings.Create(Configuration);
TechJournalExport exportMaster = new TechJournalExport(settings, new TechJournalOnClickHouseTargetBuilder());
exportMaster.OnErrorEvent += OnError;
exportMaster.OnSendLogEvent += OnSend;
await exportMaster.StartExport();
Console.WriteLine("Good luck & bye!");
}
private static void OnError(
TechJournalSettings.LogSourceSettings settings,
OnErrorExportSharedBufferEventArgs args)
{
Console.WriteLine($"Ошибка при экспорте логов ({settings?.Name ?? "<>"}): {args.Exception}");
}
private static void OnSend(
TechJournalSettings.LogSourceSettings settings,
OnSendLogFromSharedBufferEventArgs args)
{
Console.WriteLine($"Отправка данных в хранилище ({settings?.Name ?? "<>"}):\n" +
$"Записей: {args._rows.Select(e => e.Value.Count).Sum()}\n" +
$"Актуальных позиций чтения: {args._positions.Count}");
}
}
}
@@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\Libs\YY.TechJournalExportAssistant.ClickHouse\YY.TechJournalExportAssistant.ClickHouse.csproj" />
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup>
<ProjectExtensions><VisualStudio><UserProperties appsettings_1json__JsonSchema="" /></VisualStudio></ProjectExtensions>
</Project>
@@ -8,6 +8,7 @@ using ClickHouse.Client.Copy;
using YY.TechJournalExportAssistant.ClickHouse.Helpers;
using YY.TechJournalExportAssistant.Core;
using YY.TechJournalExportAssistant.Core.Helpers;
using YY.TechJournalExportAssistant.Core.SharedBuffer;
using YY.TechJournalReaderAssistant;
using YY.TechJournalReaderAssistant.Helpers;
using YY.TechJournalReaderAssistant.Models;
@@ -19,7 +20,6 @@ namespace YY.TechJournalExportAssistant.ClickHouse
#region Private Members
private ClickHouseConnection _connection;
private long logFileLastId = -1;
#endregion
@@ -31,16 +31,12 @@ namespace YY.TechJournalExportAssistant.ClickHouse
_connection = new ClickHouseConnection(connectionSettings);
_connection.Open();
var cmdDDL = _connection.CreateCommand();
cmdDDL.CommandText = Resource.Query_CreateTable_EventDataStorage;
cmdDDL.ExecuteNonQuery();
cmdDDL.CommandText = Resource.Query_CreateTable_LogFilesStorage;
cmdDDL.ExecuteNonQuery();
cmdDDL.CommandText = Resource.Query_CreateTable_EventData;
cmdDDL.ExecuteNonQuery();
cmdDDL.CommandText = Resource.Query_CreateTable_LogFiles;
cmdDDL.ExecuteNonQuery();
}
#endregion
@@ -49,18 +45,19 @@ namespace YY.TechJournalExportAssistant.ClickHouse
#region RowsData
public void SaveRowsData(TechJournalLogBase techJournalLog, List<EventData> eventData, string fileName)
public void SaveRowsData(TechJournalLogBase techJournalLog, List<EventData> eventData, string directoryName, string fileName)
{
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
{
DestinationTableName = "EventData",
BatchSize = 100000
BatchSize = 100000,
MaxDegreeOfParallelism = 4
})
{
var values = eventData.Select(i => new object[]
{
techJournalLog.Name,
techJournalLog.DirectoryName,
directoryName,
fileName,
i.Id,
i.Period,
@@ -73,7 +70,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
i.SessionId ?? 0,
i.ApplicationName ?? string.Empty,
i.ClientId ?? 0,
i.ComputerName ?? string.Empty,
i.ComputerName ?? string.Empty,
i.ConnectionId ?? 0,
i.UserName ?? string.Empty,
i.ApplicationId ?? 0,
@@ -100,7 +97,93 @@ namespace YY.TechJournalExportAssistant.ClickHouse
bulkResult.Wait();
}
}
public DateTime GetRowsDataMaxPeriod(TechJournalLogBase techJournalLog, string fileName)
public void SaveRowsData(TechJournalLogBase techJournalLog, IDictionary<string, List<EventData>> eventData)
{
Dictionary<string, DateTime> maxPeriodByFiles = new Dictionary<string, DateTime>();
List<object[]> rowsForInsert = new List<object[]>();
foreach (var eventInfo in eventData)
{
FileInfo logFileInfo = new FileInfo(eventInfo.Key);
foreach (var eventItem in eventInfo.Value)
{
if (!maxPeriodByFiles.TryGetValue(logFileInfo.Name, out DateTime maxPeriodValue))
{
if (logFileInfo.Directory != null)
maxPeriodValue = GetRowsDataMaxPeriod(
techJournalLog,
logFileInfo.Directory.Name,
logFileInfo.Name
);
maxPeriodByFiles.Add(logFileInfo.Name, maxPeriodValue);
}
else
{
maxPeriodValue = DateTime.MinValue;
}
if (maxPeriodValue != DateTime.MinValue && eventItem.Period <= maxPeriodValue)
if (logFileInfo.Directory != null && RowDataExistOnDatabase(techJournalLog, eventItem, logFileInfo.Directory.Name, logFileInfo.Name))
continue;
if (logFileInfo.Directory != null)
rowsForInsert.Add(new object[]
{
techJournalLog.Name,
logFileInfo.Directory.Name,
eventInfo.Key,
eventItem.Id,
eventItem.Period,
eventItem.Level,
eventItem.Duration,
eventItem.DurationSec,
eventItem.EventName ?? string.Empty,
eventItem.ServerContextName ?? string.Empty,
eventItem.ProcessName ?? string.Empty,
eventItem.SessionId ?? 0,
eventItem.ApplicationName ?? string.Empty,
eventItem.ClientId ?? 0,
eventItem.ComputerName ?? string.Empty,
eventItem.ConnectionId ?? 0,
eventItem.UserName ?? string.Empty,
eventItem.ApplicationId ?? 0,
eventItem.Context ?? string.Empty,
eventItem.ActionType.GetDescription() ?? string.Empty,
eventItem.Database ?? string.Empty,
eventItem.DatabaseCopy ?? string.Empty,
eventItem.DBMS.GetPresentation() ?? string.Empty,
eventItem.DatabasePID ?? string.Empty,
eventItem.PlanSQLText ?? string.Empty,
eventItem.Rows ?? 0,
eventItem.RowsAffected ?? 0,
eventItem.SQLText ?? string.Empty,
eventItem.SQLQueryOnly ?? string.Empty,
eventItem.SQLQueryParametersOnly ?? string.Empty,
eventItem.SQLQueryHash ?? string.Empty,
eventItem.SDBL ?? string.Empty,
eventItem.Description ?? string.Empty,
eventItem.Message ?? string.Empty,
eventItem.GetCustomFieldsAsJSON() ?? string.Empty
});
}
}
if (rowsForInsert.Count == 0)
return;
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
{
DestinationTableName = "EventData",
BatchSize = 100000,
MaxDegreeOfParallelism = 4
})
{
var bulkResult = bulkCopyInterface.WriteToServerAsync(rowsForInsert);
bulkResult.Wait();
rowsForInsert.Clear();
}
}
public DateTime GetRowsDataMaxPeriod(TechJournalLogBase techJournalLog, string directoryName, string fileName)
{
DateTime output = DateTime.MinValue;
@@ -114,7 +197,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
AND DirectoryName = {directoryName:String}
AND FileName = {fileName:String}";
command.AddParameterToCommand("techJournalLog", techJournalLog.Name);
command.AddParameterToCommand("directoryName", techJournalLog.DirectoryName);
command.AddParameterToCommand("directoryName", directoryName);
command.AddParameterToCommand("fileName", fileName);
using (var cmdReader = command.ExecuteReader())
if (cmdReader.Read()) output = cmdReader.GetDateTime(0);
@@ -122,7 +205,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
return output;
}
public bool RowDataExistOnDatabase(TechJournalLogBase techJournalLog, EventData eventData, string fileName)
public bool RowDataExistOnDatabase(TechJournalLogBase techJournalLog, EventData eventData, string directoryName, string fileName)
{
bool output = false;
@@ -136,11 +219,13 @@ namespace YY.TechJournalExportAssistant.ClickHouse
FROM EventData AS RD
WHERE TechJournalLog = {techJournalLog:String}
AND Id = {existId:Int64}
AND DirectoryName = {directoryName:String}
AND FileName = {fileName:String}
AND Period = {existPeriod:DateTime}";
command.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
command.AddParameterToCommand("existId", DbType.Int64, eventData.Id);
command.AddParameterToCommand("existPeriod", DbType.DateTime, eventData.Period);
command.AddParameterToCommand("directoryName", DbType.String, directoryName);
command.AddParameterToCommand("fileName", DbType.String, fileName);
using (var cmdReader = command.ExecuteReader())
if (cmdReader.Read()) output = true;
@@ -153,7 +238,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
#region LogFiles
public TechJournalPosition GetLogFilePosition(TechJournalLogBase techJournalLog)
public TechJournalPosition GetLogFilePosition(TechJournalLogBase techJournalLog, string directoryName)
{
var cmdGetLastLogFileInfo = _connection.CreateCommand();
cmdGetLastLogFileInfo.CommandText =
@@ -172,7 +257,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
AND LF_LAST.DirectoryName = {directoryName:String}
)";
cmdGetLastLogFileInfo.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
cmdGetLastLogFileInfo.AddParameterToCommand("directoryName", DbType.AnsiString, techJournalLog.DirectoryName);
cmdGetLastLogFileInfo.AddParameterToCommand("directoryName", DbType.AnsiString, directoryName);
TechJournalPosition output = null;
using (var cmdReader = cmdGetLastLogFileInfo.ExecuteReader())
@@ -191,76 +276,63 @@ namespace YY.TechJournalExportAssistant.ClickHouse
return output;
}
public void SaveLogPosition(TechJournalLogBase techJournalLog, FileInfo logFileInfo, TechJournalPosition position)
public void SaveLogPosition(TechJournalLogBase techJournalLog, TechJournalPosition position)
{
var dataFileInfo = new FileInfo(position.CurrentFileData);
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
{
DestinationTableName = "LogFiles", BatchSize = 100000
DestinationTableName = "LogFiles",
BatchSize = 1000000
})
{
IEnumerable<object[]> values = new List<object[]>()
if (dataFileInfo.Directory != null)
{
new object[]
IEnumerable<object[]> values = new List<object[]>()
{
techJournalLog.Name,
techJournalLog.DirectoryName,
GetLogFileInfoNewId(techJournalLog),
logFileInfo.Name,
logFileInfo.CreationTimeUtc,
logFileInfo.LastWriteTimeUtc,
position.EventNumber,
position.CurrentFileData.Replace("\\", "\\\\"),
position.StreamPosition ?? 0
}
}.AsEnumerable();
var bulkResult = bulkCopyInterface.WriteToServerAsync(values);
bulkResult.Wait();
}
}
public long GetLogFileInfoNewId(TechJournalLogBase techJournalLog)
{
long output = 0;
if (logFileLastId < 0)
{
using (var command = _connection.CreateCommand())
{
command.CommandText =
@"SELECT
MAX(Id)
FROM LogFiles
WHERE TechJournalLog = {techJournalLog:String}
AND DirectoryName = {directoryName:String}";
command.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
command.AddParameterToCommand("directoryName", DbType.AnsiString, techJournalLog.DirectoryName);
using (var cmdReader = command.ExecuteReader())
if (cmdReader.Read()) output = cmdReader.GetInt64(0);
new object[]
{
techJournalLog.Name,
dataFileInfo.Directory.Name,
DateTime.Now.Ticks,
dataFileInfo.Name,
dataFileInfo.CreationTimeUtc,
dataFileInfo.LastWriteTimeUtc,
position.EventNumber,
position.CurrentFileData.Replace("\\", "\\\\"),
position.StreamPosition ?? 0
}
}.AsEnumerable();
var bulkResult = bulkCopyInterface.WriteToServerAsync(values);
bulkResult.Wait();
}
}
else output = logFileLastId;
}
public IDictionary<string, TechJournalPosition> GetCurrentLogPositions(
TechJournalLogBase techJournalLog,
TechJournalSettings settings,
KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
{
var cmdGetLastLogFileInfo = _connection.CreateCommand();
cmdGetLastLogFileInfo.CommandText = Resource.Query_GetActualPositions;
cmdGetLastLogFileInfo.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
output += 1;
logFileLastId = output;
IDictionary<string, TechJournalPosition> output = new Dictionary<string, TechJournalPosition>();
using (var cmdReader = cmdGetLastLogFileInfo.ExecuteReader())
{
while (cmdReader.Read())
{
string fileName = cmdReader.GetString(7).Replace("\\\\", "\\");
string directoryName = cmdReader.GetString(1);
output.Add(directoryName, new TechJournalPosition(
cmdReader.GetInt64(6),
fileName,
cmdReader.GetInt64(8)
));
}
}
return output;
}
public void RemoveArchiveLogFileRecords(TechJournalLogBase techJournalLog)
{
var commandRemoveArchiveLogInfo = _connection.CreateCommand();
commandRemoveArchiveLogInfo.CommandText =
@"ALTER TABLE LogFilesStorage DELETE
WHERE TechJournalLog = {techJournalLog:String}
AND DirectoryName = {directoryName:String}
AND Id < (
SELECT MAX(Id) AS LastId
FROM LogFilesStorage lf
WHERE TechJournalLog = {techJournalLog:String}
AND DirectoryName = {directoryName:String}
)";
commandRemoveArchiveLogInfo.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
commandRemoveArchiveLogInfo.AddParameterToCommand("directoryName", DbType.AnsiString, techJournalLog.DirectoryName);
commandRemoveArchiveLogInfo.ExecuteNonQuery();
}
#endregion
@@ -0,0 +1,83 @@
using System;
using System.Collections;
namespace YY.TechJournalExportAssistant.ClickHouse
{
public class EventDataBulkInsertObject : IEnumerable
{
public string TechJournalLog { set; get; }
public string DirectoryName { set; get; }
public string FileName { set; get; }
public long Id { set; get; }
public DateTime Period { set; get; }
public long Level { set; get; }
public long Duration { set; get; }
public long DurationSec { set; get; }
public string EventName { set; get; }
public string ServerContextName { set; get; }
public string ProcessName { set; get; }
public long? SessionId { set; get; }
public string ApplicationName { set; get; }
public long? ClientId { set; get; }
public string ComputerName { set; get; }
public long? ConnectionId { set; get; }
public string UserName { set; get; }
public long? ApplicationId { set; get; }
public string Context { set; get; }
public string ActionType { set; get; }
public string Database { set; get; }
public string DatabaseCopy { set; get; }
public string DBMS { set; get; }
public string DatabasePID { set; get; }
public string PlanSQLText { set; get; }
public long? Rows { set; get; }
public long? RowsAffected { set; get; }
public string SQLText { set; get; }
public string SQLQueryOnly { set; get; }
public string SQLQueryParametersOnly { set; get; }
public string SQLQueryHash { set; get; }
public string SDBL { set; get; }
public string Description { set; get; }
public string Message { set; get; }
public string CustomEventData { set; get; }
public IEnumerator GetEnumerator()
{
yield return TechJournalLog;
yield return DirectoryName;
yield return FileName;
yield return Id;
yield return Period;
yield return Level;
yield return Duration;
yield return DurationSec;
yield return EventName;
yield return ServerContextName;
yield return ProcessName;
yield return SessionId;
yield return ApplicationName;
yield return ClientId;
yield return ComputerName;
yield return ConnectionId;
yield return UserName;
yield return ApplicationId;
yield return Context;
yield return ActionType;
yield return Database;
yield return DatabaseCopy;
yield return DBMS;
yield return DatabasePID;
yield return PlanSQLText;
yield return Rows;
yield return RowsAffected;
yield return SQLText;
yield return SQLQueryOnly;
yield return SQLQueryParametersOnly;
yield return SQLQueryHash;
yield return SDBL;
yield return Description;
yield return Message;
yield return CustomEventData;
}
}
}
@@ -61,16 +61,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
}
/// <summary>
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS EventData AS EventDataStorage ENGINE = Buffer(currentDatabase(), EventDataStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000).
/// </summary>
internal static string Query_CreateTable_EventData {
get {
return ResourceManager.GetString("Query_CreateTable_EventData", resourceCulture);
}
}
/// <summary>
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS EventDataStorage
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS EventData
///(
/// TechJournalLog LowCardinality(String),
/// DirectoryName LowCardinality(String),
@@ -82,7 +73,8 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
/// DurationSec Int64 Codec(DoubleDelta, LZ4),
/// EventName LowCardinality(String),
/// ServerContextName LowCardinality(String),
/// ProcessName LowCardinality(String), [остаток строки не уместился]&quot;;.
/// ProcessName LowCardinality(String),
/// S [остаток строки не уместился]&quot;;.
/// </summary>
internal static string Query_CreateTable_EventDataStorage {
get {
@@ -91,20 +83,11 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
}
/// <summary>
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS LogFiles AS LogFilesStorage ENGINE = Buffer(currentDatabase(), LogFilesStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000).
/// </summary>
internal static string Query_CreateTable_LogFiles {
get {
return ResourceManager.GetString("Query_CreateTable_LogFiles", resourceCulture);
}
}
/// <summary>
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS LogFilesStorage
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS LogFiles
///(
/// TechJournalLog LowCardinality(String),
/// DirectoryName LowCardinality(String),
/// Id Int64 Codec(DoubleDelta, LZ4),
/// Id Int64 DEFAULT (toUnixTimestamp64Milli(now64())*1000000 + rowNumberInAllBlocks()) Codec(DoubleDelta, LZ4),
/// FileName LowCardinality(String),
/// CreateDate DateTime Codec(Delta, LZ4),
/// ModificationDate DateTime Codec(Delta, LZ4),
@@ -112,14 +95,47 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
/// LastCurrentFileData LowCardinality(String),
/// LastStreamPosition Int64 Codec(DoubleDelta, LZ4)
///)
///engine = MergeTree()
///PARTITION BY toYYYYMM(CreateDate)
///PRIMARY KEY CreateD [остаток строки не уместился]&quot;;.
///engine = [остаток строки не уместился]&quot;;.
/// </summary>
internal static string Query_CreateTable_LogFilesStorage {
get {
return ResourceManager.GetString("Query_CreateTable_LogFilesStorage", resourceCulture);
}
}
/// <summary>
/// Ищет локализованную строку, похожую на SELECT
/// l.TechJournalLog,
/// l.DirectoryName,
/// l.Id,
/// l.FileName,
/// l.CreateDate,
/// l.ModificationDate,
/// l.LastEventNumber,
/// l.LastCurrentFileData,
/// l.LastStreamPosition
///FROM LogFiles l
///
///INNER JOIN
///
///(
///SELECT
/// TechJournalLog,
/// DirectoryName,
/// MAX(Id) LastId
///FROM LogFiles AS LF_LAST
///WHERE LF_LAST.TechJournalLog = {techJournalLog:String}
///GROUP BY TechJournalLog,
/// DirectoryName
///) lglast
///
///ON l.TechJournalLog = lglast.TechJournalLog
/// AND l.DirectoryNam [остаток строки не уместился]&quot;;.
/// </summary>
internal static string Query_GetActualPositions {
get {
return ResourceManager.GetString("Query_GetActualPositions", resourceCulture);
}
}
}
}
@@ -117,12 +117,8 @@
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="Query_CreateTable_EventData" xml:space="preserve">
<value>CREATE TABLE IF NOT EXISTS EventData AS EventDataStorage ENGINE = Buffer(currentDatabase(), EventDataStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000)</value>
<comment>Создание буфера для новых данных технологического журнала</comment>
</data>
<data name="Query_CreateTable_EventDataStorage" xml:space="preserve">
<value>CREATE TABLE IF NOT EXISTS EventDataStorage
<value>CREATE TABLE IF NOT EXISTS EventData
(
TechJournalLog LowCardinality(String),
DirectoryName LowCardinality(String),
@@ -167,15 +163,12 @@ ORDER BY Period
SETTINGS index_granularity = 8192;</value>
<comment>Создание таблицы для хранения данных технологического журнала</comment>
</data>
<data name="Query_CreateTable_LogFiles" xml:space="preserve">
<value>CREATE TABLE IF NOT EXISTS LogFiles AS LogFilesStorage ENGINE = Buffer(currentDatabase(), LogFilesStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000)</value>
</data>
<data name="Query_CreateTable_LogFilesStorage" xml:space="preserve">
<value>CREATE TABLE IF NOT EXISTS LogFilesStorage
<value>CREATE TABLE IF NOT EXISTS LogFiles
(
TechJournalLog LowCardinality(String),
DirectoryName LowCardinality(String),
Id Int64 Codec(DoubleDelta, LZ4),
Id Int64 DEFAULT (toUnixTimestamp64Milli(now64())*1000000 + rowNumberInAllBlocks()) Codec(DoubleDelta, LZ4),
FileName LowCardinality(String),
CreateDate DateTime Codec(Delta, LZ4),
ModificationDate DateTime Codec(Delta, LZ4),
@@ -190,4 +183,36 @@ ORDER BY CreateDate
SETTINGS index_granularity = 8192;</value>
<comment>Создание таблицы для хранения данных лога обработки файлов</comment>
</data>
<data name="Query_GetActualPositions" xml:space="preserve">
<value>SELECT
l.TechJournalLog,
l.DirectoryName,
l.Id,
l.FileName,
l.CreateDate,
l.ModificationDate,
l.LastEventNumber,
l.LastCurrentFileData,
l.LastStreamPosition
FROM LogFiles l
INNER JOIN
(
SELECT
TechJournalLog,
DirectoryName,
MAX(Id) LastId
FROM LogFiles AS LF_LAST
WHERE LF_LAST.TechJournalLog = {techJournalLog:String}
GROUP BY TechJournalLog,
DirectoryName
) lglast
ON l.TechJournalLog = lglast.TechJournalLog
AND l.DirectoryName = lglast.DirectoryName
AND l.Id = lglast.LastId
ORDER BY l.DirectoryName</value>
<comment>Получение последних событий</comment>
</data>
</root>
@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using Microsoft.Extensions.Configuration;
using YY.TechJournalExportAssistant.Core;
using YY.TechJournalExportAssistant.Core.SharedBuffer;
using YY.TechJournalReaderAssistant;
using EventData = YY.TechJournalReaderAssistant.Models.EventData;
@@ -18,8 +19,6 @@ namespace YY.TechJournalExportAssistant.ClickHouse
private TechJournalLogBase _techJournalLog;
private readonly string _connectionString;
private TechJournalPosition _lastTechJournalFilePosition;
private int _stepsToClearLogFiles = 10000;
private int _currentStepToClearLogFiles;
#endregion
@@ -53,29 +52,23 @@ namespace YY.TechJournalExportAssistant.ClickHouse
#region Public Methods
public override TechJournalPosition GetLastPosition()
public override TechJournalPosition GetLastPosition(string directoryName)
{
if (_lastTechJournalFilePosition != null)
return _lastTechJournalFilePosition;
TechJournalPosition position;
using(var context = new ClickHouseContext(_connectionString))
position = context.GetLogFilePosition(_techJournalLog);
position = context.GetLogFilePosition(_techJournalLog, directoryName);
_lastTechJournalFilePosition = position;
return position;
}
public override void SaveLogPosition(FileInfo logFileInfo, TechJournalPosition position)
public override void SaveLogPosition(TechJournalPosition position)
{
using (var context = new ClickHouseContext(_connectionString))
{
context.SaveLogPosition(_techJournalLog, logFileInfo, position);
if (_currentStepToClearLogFiles == 0 || _currentStepToClearLogFiles >= _stepsToClearLogFiles)
{
context.RemoveArchiveLogFileRecords(_techJournalLog);
_currentStepToClearLogFiles = 0;
}
_currentStepToClearLogFiles += 1;
context.SaveLogPosition(_techJournalLog, position);
}
_lastTechJournalFilePosition = position;
@@ -98,14 +91,17 @@ namespace YY.TechJournalExportAssistant.ClickHouse
if(rowsData.Count == 0)
return;
FileInfo fileInfo = new FileInfo(fileName);
using (var context = new ClickHouseContext(_connectionString))
{
if (_maxPeriodRowData == DateTime.MinValue)
{
_maxPeriodRowData = context.GetRowsDataMaxPeriod(
_techJournalLog,
fileName
);
if (fileInfo.Directory != null)
_maxPeriodRowData = context.GetRowsDataMaxPeriod(
_techJournalLog,
fileInfo.Directory.Name,
fileName
);
}
List<EventData> newEntities = new List<EventData>();
@@ -114,12 +110,15 @@ namespace YY.TechJournalExportAssistant.ClickHouse
if (itemRow == null)
continue;
if (_maxPeriodRowData != DateTime.MinValue && itemRow.Period <= _maxPeriodRowData)
if (context.RowDataExistOnDatabase(_techJournalLog, itemRow, fileName))
if (fileInfo.Directory != null
&& context.RowDataExistOnDatabase(_techJournalLog, itemRow, fileInfo.Directory.Name, fileInfo.Name))
continue;
newEntities.Add(itemRow);
}
context.SaveRowsData(_techJournalLog, newEntities, fileName);
if (fileInfo.Directory != null)
context.SaveRowsData(_techJournalLog, newEntities, fileInfo.Directory.Name, fileName);
}
}
public override void SetInformationSystem(TechJournalLogBase techJournalLog)
@@ -127,6 +126,29 @@ namespace YY.TechJournalExportAssistant.ClickHouse
_techJournalLog = techJournalLog;
}
public override void Save(IDictionary<string, List<EventData>> rowsData)
{
if (rowsData.Count == 0)
return;
using (var context = new ClickHouseContext(_connectionString))
{
context.SaveRowsData(_techJournalLog, rowsData);
}
}
public override IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
{
IDictionary<string, TechJournalPosition> positions;
using (var context = new ClickHouseContext(_connectionString))
{
positions = context.GetCurrentLogPositions(_techJournalLog, settings, logBufferItem);
}
return positions;
}
#endregion
}
}
@@ -0,0 +1,29 @@
using System.Collections.Generic;
using YY.TechJournalExportAssistant.Core;
using YY.TechJournalExportAssistant.Core.SharedBuffer;
using YY.TechJournalReaderAssistant;
namespace YY.TechJournalExportAssistant.ClickHouse
{
public class TechJournalOnClickHouseTargetBuilder : ITechJournalOnTargetBuilder
{
public ITechJournalOnTarget CreateTarget(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
{
ITechJournalOnTarget target = new TechJournalOnClickHouse(settings.ConnectionString,
logBufferItem.Key.Portion);
target.SetInformationSystem(new TechJournalLogBase()
{
Name = logBufferItem.Key.Name,
Description = logBufferItem.Key.Description
});
return target;
}
public IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
{
ITechJournalOnTarget target = CreateTarget(settings, logBufferItem);
return target.GetCurrentLogPositions(settings, logBufferItem);
}
}
}
@@ -3,7 +3,7 @@
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<Version>1.0.0.9</Version>
<Version>1.0.1.1</Version>
<Authors>Permitin Yuriy</Authors>
<Product>Technological journal's export assistant to ClickHouse</Product>
<Description>Library for exporting 1C:Enterprise 8.x platform's technological journal files to ClickHouse</Description>
@@ -1,5 +1,5 @@
using System.Collections.Generic;
using System.IO;
using YY.TechJournalExportAssistant.Core.SharedBuffer;
using YY.TechJournalReaderAssistant;
using YY.TechJournalReaderAssistant.Models;
@@ -7,11 +7,14 @@ namespace YY.TechJournalExportAssistant.Core
{
public interface ITechJournalOnTarget
{
TechJournalPosition GetLastPosition();
void SaveLogPosition(FileInfo logFileInfo, TechJournalPosition position);
TechJournalPosition GetLastPosition(string directoryName);
void SaveLogPosition(TechJournalPosition position);
int GetPortionSize();
void SetInformationSystem(TechJournalLogBase techJournalLog);
void Save(EventData eventData, string fileName);
void Save(IList<EventData> eventData, string fileName);
void Save(IDictionary<string, List<EventData>> rowsData);
IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings,
KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem);
}
}
@@ -0,0 +1,14 @@
using System;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs
{
public sealed class OnErrorExportSharedBufferEventArgs : System.EventArgs
{
public OnErrorExportSharedBufferEventArgs(Exception exception)
{
Exception = exception;
}
public Exception Exception { get; }
}
}
@@ -0,0 +1,23 @@
using System.Collections.Generic;
using YY.TechJournalReaderAssistant;
using YY.TechJournalReaderAssistant.Models;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs
{
public sealed class OnSendLogFromSharedBufferEventArgs : System.EventArgs
{
public TechJournalSettings.LogSourceSettings _settings { get; }
public IDictionary<string, List<EventData>> _rows { get; }
public IReadOnlyDictionary<string, TechJournalPosition> _positions { get; }
public OnSendLogFromSharedBufferEventArgs(
TechJournalSettings.LogSourceSettings settings,
IDictionary<string, List<EventData>> rows,
IReadOnlyDictionary<string, TechJournalPosition> positions)
{
_settings = settings;
_rows = rows;
_positions = positions;
}
}
}
@@ -0,0 +1,11 @@
using System;
using System.IO;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
{
public class EventKey
{
public Guid Id { get; set; }
public FileInfo File { get; set; }
}
}
@@ -0,0 +1,11 @@
using System.Collections.Generic;
using YY.TechJournalReaderAssistant;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
{
public interface ITechJournalOnTargetBuilder
{
ITechJournalOnTarget CreateTarget(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem);
IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem);
}
}
@@ -0,0 +1,46 @@
using System;
using System.Collections.Concurrent;
using YY.TechJournalReaderAssistant;
using YY.TechJournalReaderAssistant.Models;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
{
/// <summary>
/// Элемент буфера логов
/// </summary>
public class LogBufferItem
{
/// <summary>
/// Дата создания
/// </summary>
public DateTime Created { get; set; }
/// <summary>
/// Дата последнего обновления
/// </summary>
public DateTime LastUpdate { get; set; }
/// <summary>
/// Количество записей лога в буфере
/// </summary>
public long ItemsCount => LogRows.Count;
/// <summary>
/// Актуальная позиция чтения файлов лога
/// </summary>
public ConcurrentDictionary<string, TechJournalPosition> LogPositions { get; set; }
/// <summary>
/// Записи логов
/// </summary>
public ConcurrentDictionary<EventKey, EventData> LogRows { get; set; }
public LogBufferItem()
{
Created = DateTime.Now;
LastUpdate = DateTime.MinValue;
LogPositions = new ConcurrentDictionary<string, TechJournalPosition>();
LogRows = new ConcurrentDictionary<EventKey, EventData>();
}
}
}
@@ -0,0 +1,166 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using YY.TechJournalReaderAssistant;
using YY.TechJournalReaderAssistant.Models;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
{
public class LogBuffers
{
public readonly ConcurrentDictionary<TechJournalSettings.LogSourceSettings, LogBufferItem> LogBuffer;
public readonly object LockObject;
public LogBuffers()
{
LogBuffer = new ConcurrentDictionary<TechJournalSettings.LogSourceSettings, LogBufferItem>();
LockObject = new object();
}
/// <summary>
/// Общее количество записей логов в буфере
/// </summary>
public long TotalItemsCount
{
get
{
long totalItemsCount = 0;
foreach (var logBufferItem in LogBuffer)
{
//lock (logBufferItem.Key.LockObject)
//{
totalItemsCount += logBufferItem.Value.ItemsCount;
//}
}
return totalItemsCount;
}
}
public DateTime Created
{
get
{
DateTime bufferCreated = DateTime.MinValue;
foreach (var logBufferItem in LogBuffer)
{
//lock (logBufferItem.Key.LockObject)
//{
if (bufferCreated == DateTime.MinValue)
{
bufferCreated = logBufferItem.Value.Created;
} else if (bufferCreated < logBufferItem.Value.Created)
{
bufferCreated = logBufferItem.Value.Created;
}
//}
}
return bufferCreated;
}
}
public void SaveLogsAndPosition(
TechJournalSettings.LogSourceSettings logSettings,
TechJournalPosition position,
IList<EventData> rowsData
)
{
lock (logSettings.LockObject)
{
var logFileInfo = new FileInfo(position.CurrentFileData);
SaveLogPosition(logSettings, logFileInfo, position);
SaveLogs(logSettings, rowsData, logFileInfo);
}
}
private void SaveLogPosition(
TechJournalSettings.LogSourceSettings logSettings,
FileInfo logFileInfo,
TechJournalPosition position)
{
LogBuffer.AddOrUpdate(
logSettings,
(settings) =>
{
var newLogBufferItem = new LogBufferItem();
if (logFileInfo.Directory != null)
newLogBufferItem.LogPositions.TryAdd(
logFileInfo.Directory.Name,
position);
return newLogBufferItem;
},
(settings, logBufferItem) =>
{
if (logFileInfo.Directory != null)
logBufferItem.LogPositions.AddOrUpdate(
logFileInfo.Directory.Name,
(fullDirectoryName) => position,
(fullDirectoryName, oldPosition) => position);
return logBufferItem;
});
}
private void SaveLogs(
TechJournalSettings.LogSourceSettings logSettings,
IList<EventData> rowsData,
FileInfo logFileInfo)
{
LogBuffer.AddOrUpdate(
logSettings,
(settings) =>
{
var newBufferItem = new LogBufferItem();
newBufferItem.LastUpdate = DateTime.Now;
foreach (var rowData in rowsData)
{
newBufferItem.LogRows.TryAdd(new EventKey()
{
Id = Guid.NewGuid(),
File = logFileInfo
}, rowData);
}
return newBufferItem;
},
(settings, logBufferItem) =>
{
DateTime operationDate = DateTime.Now;
if (logBufferItem.Created == DateTime.MinValue)
logBufferItem.Created = operationDate;
logBufferItem.LastUpdate = operationDate;
foreach (var rowData in rowsData)
{
logBufferItem.LogRows.TryAdd(new EventKey()
{
Id = Guid.NewGuid(),
File = logFileInfo
}, rowData);
}
return logBufferItem;
});
}
public TechJournalPosition GetLastPosition(
TechJournalSettings.LogSourceSettings logSettings,
TechJournalLogBase techJournalLog,
string directoryName)
{
TechJournalPosition position = null;
if (LogBuffer.TryGetValue(logSettings, out LogBufferItem bufferItem))
{
bufferItem.LogPositions.TryGetValue(directoryName, out position);
}
return position;
}
}
}
@@ -0,0 +1,268 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs;
using YY.TechJournalReaderAssistant;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
{
public class TechJournalExport
{
#region Private Members
private readonly TechJournalSettings _settings;
private readonly LogBuffers _logBuffers;
private readonly ITechJournalOnTargetBuilder _techJournalTargetBuilder;
#endregion
#region Constructors
public TechJournalExport(TechJournalSettings settings, ITechJournalOnTargetBuilder techJournalTargetBuilder)
{
_settings = settings;
_logBuffers = new LogBuffers();
_techJournalTargetBuilder = techJournalTargetBuilder;
foreach (var logSourceSettings in _settings.LogSources)
{
_logBuffers.LogBuffer.TryAdd(logSourceSettings, new LogBufferItem());
}
foreach (var logBufferItem in _logBuffers.LogBuffer)
{
var logPositions = techJournalTargetBuilder.GetCurrentLogPositions(settings, logBufferItem);
logBufferItem.Value.LogPositions = new ConcurrentDictionary<string, TechJournalPosition>(logPositions);
}
}
#endregion
#region Public Methods
public async Task StartExport()
{
await StartExport(CancellationToken.None);
}
public async Task StartExport(CancellationToken cancellationToken)
{
List<Task> exportJobs = new List<Task>();
foreach (var logSource in _settings.LogSources)
{
Task logExportTask = Task.Run(() => LogExportJob(logSource, cancellationToken), cancellationToken);
exportJobs.Add(logExportTask);
await Task.Delay(1000, cancellationToken);
}
await Task.Delay(10000, cancellationToken);
exportJobs.Add(Task.Run(() => SendLogFromBuffer(cancellationToken), cancellationToken));
Task.WaitAll(exportJobs.ToArray(), cancellationToken);
}
#endregion
#region Private Methods
private async Task LogExportJob(
TechJournalSettings.LogSourceSettings settings,
CancellationToken cancellationToken)
{
string techJournalPath = settings.SourcePath;
TimeZoneInfo timeZone = settings.TimeZone;
while (true)
{
if (cancellationToken.IsCancellationRequested)
break;
bool bufferBlocked = (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxBufferSizeItemsCount);
try
{
if (!bufferBlocked)
{
TechJournalManager tjManager = new TechJournalManager(techJournalPath);
foreach (var tjDirectory in tjManager.Directories)
{
if (cancellationToken.IsCancellationRequested)
break;
bufferBlocked = (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxBufferSizeItemsCount);
if (bufferBlocked)
break;
if (!tjDirectory.DirectoryData.Exists)
continue;
using (TechJournalExportMaster exporter = new TechJournalExportMaster())
{
exporter.SetTechJournalPath(tjDirectory.DirectoryData.FullName, timeZone);
TechJournalOnBuffer target =
new TechJournalOnBuffer(_logBuffers, settings.Portion);
target.SetLogSettings(settings);
target.SetInformationSystem(new TechJournalLogBase()
{
Name = settings.Name,
Description = settings.Description
});
exporter.SetTarget(target);
exporter.OnErrorExportData += OnErrorExportDataToBuffer;
while (exporter.NewDataAvailable())
{
if (cancellationToken.IsCancellationRequested)
break;
exporter.SendData();
if (cancellationToken.IsCancellationRequested)
break;
bufferBlocked = (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxBufferSizeItemsCount);
if (bufferBlocked)
break;
}
}
}
}
if (bufferBlocked)
{
await Task.Delay(100, cancellationToken);
}
else if (_settings.WatchMode.Use)
{
await Task.Delay(_settings.WatchMode.Periodicity, cancellationToken);
}
else
{
break;
}
}
catch (Exception e)
{
RaiseOnError(settings, new OnErrorExportSharedBufferEventArgs(e));
await Task.Delay(60000, cancellationToken);
}
}
}
private async Task SendLogFromBuffer(CancellationToken cancellationToken)
{
while (true)
{
if (cancellationToken.IsCancellationRequested)
break;
bool needExport = false;
if (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxItemCountSize)
{
needExport = true;
}
else
{
DateTime bufferCreated = _logBuffers.Created;
var createTimeLeftMs = (DateTime.Now - bufferCreated).TotalMilliseconds;
if (bufferCreated != DateTime.MinValue && createTimeLeftMs >= _settings.Export.Buffer.MaxSaveDurationMs)
{
needExport = true;
}
}
if (needExport)
{
if (cancellationToken.IsCancellationRequested)
break;
TechJournalSettings.LogSourceSettings lastSettings = null;
try
{
foreach (var logBufferItem in _logBuffers.LogBuffer)
{
lastSettings = logBufferItem.Key;
if (cancellationToken.IsCancellationRequested)
break;
if (logBufferItem.Value.LogRows.Count == 0)
continue;
lock (logBufferItem.Key.LockObject)
{
var eventsByFile = logBufferItem.Value.LogRows
.Select(b => new
{
FileName = b.Key.File.FullName,
EventData = b.Value
})
.GroupBy(g => g.FileName)
.ToDictionary(
g => g.Key,
g => g.Select(e => e.EventData).ToList());
OnSend(logBufferItem.Key, new OnSendLogFromSharedBufferEventArgs(
logBufferItem.Key, eventsByFile, logBufferItem.Value.LogPositions));
ITechJournalOnTarget target = _techJournalTargetBuilder.CreateTarget(_settings, logBufferItem);
target.Save(eventsByFile);
eventsByFile.Clear();
logBufferItem.Value.LogRows.Clear();
foreach (var logPosition in logBufferItem.Value.LogPositions)
{
target.SaveLogPosition(logPosition.Value);
}
logBufferItem.Value.Created = DateTime.MinValue;
logBufferItem.Value.LastUpdate = DateTime.MinValue;
}
}
}
catch (Exception e)
{
RaiseOnError(lastSettings, new OnErrorExportSharedBufferEventArgs(e));
await Task.Delay(1000, cancellationToken);
}
}
await Task.Delay(100, cancellationToken);
if (!_settings.WatchMode.Use
&& _logBuffers.TotalItemsCount == 0)
{
break;
}
}
}
#endregion
#region Events
public delegate void OnSendLogFromSharedBufferEventArgsHandler(TechJournalSettings.LogSourceSettings settings, OnSendLogFromSharedBufferEventArgs args);
public delegate void OnErrorExportSharedBufferEventArgsHandler(TechJournalSettings.LogSourceSettings settings, OnErrorExportSharedBufferEventArgs args);
public event OnSendLogFromSharedBufferEventArgsHandler OnSendLogEvent;
public event OnErrorExportSharedBufferEventArgsHandler OnErrorEvent;
protected void OnSend(
TechJournalSettings.LogSourceSettings settings,
OnSendLogFromSharedBufferEventArgs args)
{
OnSendLogEvent?.Invoke(settings, args);
}
protected void RaiseOnError(
TechJournalSettings.LogSourceSettings settings,
OnErrorExportSharedBufferEventArgs args)
{
OnErrorEvent?.Invoke(settings, args);
}
private void OnErrorExportDataToBuffer(OnErrorExportDataEventArgs e)
{
RaiseOnError(null, new OnErrorExportSharedBufferEventArgs(e.Exception));
}
#endregion
}
}
@@ -0,0 +1,92 @@
using System.Collections.Generic;
using System.Linq;
using YY.TechJournalReaderAssistant;
using EventData = YY.TechJournalReaderAssistant.Models.EventData;
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
{
public class TechJournalOnBuffer : TechJournalOnTarget
{
#region Private Member Variables
private const int _defaultPortion = 1000;
private readonly int _portion;
private TechJournalLogBase _techJournalLog;
private TechJournalPosition _lastTechJournalFilePosition;
private TechJournalSettings.LogSourceSettings _logSettings;
private readonly LogBuffers _logBuffers;
private IList<EventData> _rowsDataForPosition;
#endregion
#region Constructor
public TechJournalOnBuffer() : this(null, _defaultPortion)
{
}
public TechJournalOnBuffer(int portion) : this(null, portion)
{
_portion = portion;
}
public TechJournalOnBuffer(LogBuffers buffers, int portion)
{
_portion = portion;
_rowsDataForPosition = new List<EventData>();
_logBuffers = buffers;
}
#endregion
#region Public Methods
public void SetLogSettings(TechJournalSettings.LogSourceSettings logSettings)
{
_logSettings = logSettings;
}
public void SetLastPosition(TechJournalPosition position)
{
_lastTechJournalFilePosition = position;
}
public override TechJournalPosition GetLastPosition(string directoryName)
{
if (_lastTechJournalFilePosition != null)
return _lastTechJournalFilePosition;
TechJournalPosition position = _logBuffers.GetLastPosition(_logSettings, _techJournalLog, directoryName);
_lastTechJournalFilePosition = position;
return position;
}
public override void SaveLogPosition(TechJournalPosition position)
{
_lastTechJournalFilePosition = position;
_logBuffers.SaveLogsAndPosition(_logSettings, position, _rowsDataForPosition);
_rowsDataForPosition.Clear();
}
public override int GetPortionSize()
{
return _portion;
}
public override void Save(EventData rowData, string fileName)
{
Save(new List<EventData>()
{
rowData
}, fileName);
}
public override void Save(IList<EventData> rowsData, string fileName)
{
_rowsDataForPosition.Clear();
_rowsDataForPosition = rowsData.ToList();
}
public override void SetInformationSystem(TechJournalLogBase techJournalLog)
{
_techJournalLog = techJournalLog;
}
#endregion
}
}
@@ -126,7 +126,9 @@ namespace YY.TechJournalExportAssistant.Core
private TechJournalPosition GetFixedCurrentPosition()
{
TechJournalPosition lastPosition = _target.GetLastPosition();
DirectoryInfo logDirectoryInfo = new DirectoryInfo(_techJournalLogPath);
string directoryName = logDirectoryInfo.Name;
TechJournalPosition lastPosition = _target.GetLastPosition(directoryName);
if (lastPosition != null)
{
@@ -167,9 +169,7 @@ namespace YY.TechJournalExportAssistant.Core
if (reader.CurrentFile != null)
{
_target.SaveLogPosition(
currentFile,
reader.GetCurrentPosition());
_target.SaveLogPosition(reader.GetCurrentPosition());
}
_dataToSend.Clear();
}
@@ -217,13 +217,11 @@ namespace YY.TechJournalExportAssistant.Core
}
private void TechJournalReader_AfterReadFile(TechJournalReader sender, AfterReadFileEventArgs args)
{
FileInfo _lastEventLogDataFileInfo = new FileInfo(args.FileName);
if (_dataToSend.Count >= 0)
SendDataCurrentPortion(sender);
TechJournalPosition position = sender.GetCurrentPosition();
_target.SaveLogPosition(_lastEventLogDataFileInfo, position);
_target.SaveLogPosition(position);
}
private void TechJournalReader_OnErrorEvent(TechJournalReader sender, OnErrorEventArgs args)
{
@@ -3,7 +3,6 @@
public class TechJournalLogBase
{
public virtual string Name { get; set; }
public virtual string DirectoryName { get; set; }
public virtual string Description { get; set; }
public override string ToString()
@@ -1,6 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using YY.TechJournalExportAssistant.Core.SharedBuffer;
using YY.TechJournalReaderAssistant;
using YY.TechJournalReaderAssistant.Models;
@@ -10,7 +10,7 @@ namespace YY.TechJournalExportAssistant.Core
{
#region Public Methods
public virtual TechJournalPosition GetLastPosition()
public virtual TechJournalPosition GetLastPosition(string directoryName)
{
throw new NotImplementedException();
}
@@ -30,7 +30,7 @@ namespace YY.TechJournalExportAssistant.Core
throw new NotImplementedException();
}
public virtual void SaveLogPosition(FileInfo logFileInfo, TechJournalPosition position)
public virtual void SaveLogPosition(TechJournalPosition position)
{
throw new NotImplementedException();
}
@@ -39,7 +39,17 @@ namespace YY.TechJournalExportAssistant.Core
{
throw new NotImplementedException();
}
public virtual void Save(IDictionary<string, List<EventData>> rowsData)
{
throw new NotImplementedException();
}
public virtual IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
{
throw new NotImplementedException();
}
#endregion
}
}
@@ -0,0 +1,126 @@
using System;
using System.Collections.Generic;
using Microsoft.Extensions.Configuration;
namespace YY.TechJournalExportAssistant.Core
{
public class TechJournalSettings
{
public static TechJournalSettings Create(string configFile)
{
IConfiguration Configuration = new ConfigurationBuilder()
.AddJsonFile(configFile, optional: true, reloadOnChange: true)
.Build();
return Create(Configuration);
}
public static TechJournalSettings Create(IConfiguration configuration)
{
return new TechJournalSettings(configuration);
}
public string ConnectionString { get; }
public WatchModeSettings WatchMode { get; }
public ExportSettings Export { get; }
public IReadOnlyList<LogSourceSettings> LogSources { get; }
private TechJournalSettings(IConfiguration configuration)
{
ConnectionString = configuration.GetConnectionString("TechJournalDatabase");
var exportParametersSection = configuration.GetSection("Export");
var bufferSection = exportParametersSection.GetSection("Buffer");
var maxItemCountSize = bufferSection.GetValue("MaxItemCountSize", 10000);
var maxSaveDurationMs = bufferSection.GetValue("MaxSaveDurationMs", 60000);
var maxBufferSizeItemsCount = bufferSection.GetValue("MaxBufferSizeItemsCount", 100000);
Export = new ExportSettings(
new ExportSettings.BufferSettings(maxItemCountSize, maxSaveDurationMs, maxBufferSizeItemsCount));
var watchModeSection = configuration.GetSection("WatchMode");
var useWatchMode = watchModeSection.GetValue("Use", false);
var periodicityWatchMode = watchModeSection.GetValue("Periodicity", 60000);
WatchMode = new WatchModeSettings(useWatchMode, periodicityWatchMode);
var logSourcesSection = configuration.GetSection("LogSources");
var logSources = logSourcesSection.GetChildren();
List<LogSourceSettings> logSourceSettings = new List<LogSourceSettings>();
foreach (var logSource in logSources)
{
var techJournalLogName = logSource.GetValue("Name", string.Empty);
var techJournalLogDescription = logSource.GetValue("Description", string.Empty);
string sourcePath = logSource.GetValue("SourcePath", string.Empty);
int portion = logSource.GetValue("Portion", 10000);
string timeZoneName = logSource.GetValue("TimeZone", string.Empty);
TimeZoneInfo timeZone;
if (string.IsNullOrEmpty(timeZoneName))
timeZone = TimeZoneInfo.Local;
else
timeZone = TimeZoneInfo.FindSystemTimeZoneById(timeZoneName);
logSourceSettings.Add(new LogSourceSettings(
techJournalLogName,
techJournalLogDescription,
sourcePath,
portion,
timeZone));
}
LogSources = logSourceSettings;
}
public class WatchModeSettings
{
public bool Use { get; }
public int Periodicity { get; }
public WatchModeSettings(bool use, int periodicity)
{
Use = use;
Periodicity = periodicity;
}
}
public class LogSourceSettings
{
public string Name { get; }
public string Description { get; }
public string SourcePath { get; }
public int Portion { get; }
public TimeZoneInfo TimeZone { get; }
public object LockObject { get; }
public LogSourceSettings(string name, string description, string sourcePath, int portion, TimeZoneInfo timeZone)
{
Name = name;
Description = description;
Portion = portion;
SourcePath = sourcePath;
TimeZone = timeZone;
LockObject = new object();
}
}
public class ExportSettings
{
public BufferSettings Buffer { get; }
public ExportSettings(BufferSettings bufferSettings)
{
Buffer = bufferSettings;
}
public class BufferSettings
{
public long MaxItemCountSize { get; }
public long MaxSaveDurationMs { get; }
public long MaxBufferSizeItemsCount { get; }
public BufferSettings(long maxItemCountSize, long maxSaveDurationMs, long maxBufferSizeItemsCount)
{
MaxItemCountSize = maxItemCountSize;
MaxSaveDurationMs = maxSaveDurationMs;
MaxBufferSizeItemsCount = maxBufferSizeItemsCount;
}
}
}
}
}
@@ -3,7 +3,7 @@
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<Version>1.0.0.8</Version>
<Version>1.0.1.1</Version>
<Authors>Permitin Yuriy</Authors>
<Product>Technological journal's export assistant. Core library</Product>
<Description>Core library for exporting 1C:Enterprise 8.x platform's technological journal files to different storage</Description>
@@ -15,6 +15,9 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
<PackageReference Include="YY.TechJournalReaderAssistant" Version="1.0.0.19" />
</ItemGroup>
@@ -1,25 +1,32 @@
using Microsoft.Extensions.Configuration;
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Text;
using System.Threading;
using ClickHouse.Client.ADO;
using Newtonsoft.Json;
using Xunit;
using YY.TechJournalExportAssistant.ClickHouse;
using YY.TechJournalExportAssistant.ClickHouse.Helpers;
using YY.TechJournalExportAssistant.Core;
using YY.TechJournalExportAssistant.Core.SharedBuffer;
using YY.TechJournalReaderAssistant;
namespace YY.TechJournalExportAssistant.Tests
{
public class TechJournalExportAssistantToClickHouseTests
{
private readonly string _configFilePath;
private readonly string _logDataPath;
private readonly IConfiguration Configuration;
public TechJournalExportAssistantToClickHouseTests()
{
_configFilePath = GetConfigFile();
var configFilePath = GetConfigFile();
Configuration = new ConfigurationBuilder()
.AddJsonFile(configFilePath, optional: true, reloadOnChange: true)
.Build();
string unitTestDirectory = Directory.GetCurrentDirectory();
@@ -32,10 +39,6 @@ namespace YY.TechJournalExportAssistant.Tests
[Fact]
public void ExportDataTest()
{
IConfiguration Configuration = new ConfigurationBuilder()
.AddJsonFile(_configFilePath, optional: true, reloadOnChange: true)
.Build();
string connectionString = Configuration.GetConnectionString("TechJournalDatabase");
IConfigurationSection techJournalSection = Configuration.GetSection("TechJournal");
@@ -83,7 +86,6 @@ namespace YY.TechJournalExportAssistant.Tests
target.SetInformationSystem(new TechJournalLogBase()
{
Name = techJournalName,
DirectoryName = tjDirectory.DirectoryData.Name,
Description = techJournalDescription
});
exporter.SetTarget(target);
@@ -124,6 +126,85 @@ namespace YY.TechJournalExportAssistant.Tests
Assert.Equal(eventNumberReader, eventNumberFromClickHouse);
}
[Fact]
public async void ExportDataWithSharedBufferTest()
{
string connectionString = Configuration.GetConnectionString("TechJournalDatabase");
IConfigurationSection techJournalSection = Configuration.GetSection("TechJournal");
int watchPeriodSeconds = techJournalSection.GetValue("WatchPeriod", 60);
int watchPeriodSecondsMs = watchPeriodSeconds * 1000;
bool useWatchMode = techJournalSection.GetValue("UseWatchMode", false);
int portion = techJournalSection.GetValue("Portion", 1000);
string timeZoneName = techJournalSection.GetValue("TimeZone", string.Empty);
IConfigurationSection techJournalLogSection = Configuration.GetSection("TechJournalLog");
string techJournalName = techJournalLogSection.GetValue("Name", string.Empty);
string techJournalDescription = techJournalLogSection.GetValue("Description", string.Empty);
var LogSources = new List<dynamic>();
for (int i = 1; i <= 3; i++)
{
LogSources.Add(new
{
Name = $"{techJournalName} {i}",
Description = $"{techJournalDescription} {i}",
SourcePath = _logDataPath,
Portion = portion,
TimeZone = timeZoneName
});
}
var configurationObject = new
{
ConnectionStrings = new
{
TechJournalDatabase = connectionString.Replace("Database=", "Database=SharedBuffer")
},
WatchMode = new
{
Use = useWatchMode,
Periodicity = watchPeriodSecondsMs
},
Export = new
{
Buffer = new
{
MaxItemCountSize = 10,
MaxSaveDurationMs = 5000,
MaxBufferSizeItemsCount = 100
}
},
LogSources
};
var configurationAsJson = JsonConvert.SerializeObject(configurationObject);
IConfiguration configurationForTest = new ConfigurationBuilder().AddJsonStream(new MemoryStream(Encoding.UTF8.GetBytes(configurationAsJson))).Build();
TechJournalSettings settings = TechJournalSettings.Create(configurationForTest);
ClickHouseHelpers.DropDatabaseIfExist(settings.ConnectionString);
TechJournalExport exportMaster = new TechJournalExport(settings, new TechJournalOnClickHouseTargetBuilder());
await exportMaster.StartExport();
int eventNumberFromClickHouse = 0;
using (ClickHouseConnection cn = new ClickHouseConnection(settings.ConnectionString))
{
using (ClickHouseCommand cmd = new ClickHouseCommand(cn))
{
cmd.CommandText =
@"SELECT
COUNT(*)
FROM EventData ed";
var reader = await cmd.ExecuteReaderAsync();
if (await reader.ReadAsync())
eventNumberFromClickHouse = Convert.ToInt32(reader.GetValue(0));
}
}
Assert.Equal(9339, eventNumberFromClickHouse);
}
private string GetConfigFile()
{
// TODO
+7
View File
@@ -38,6 +38,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{7E5B
Scripts\install-clickhouse.sh = Scripts\install-clickhouse.sh
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YY.TechJournalExportAssistantWithSharedBufferConsoleApp", "Apps\YY.TechJournalExportAssistantWithSharedBufferConsoleApp\YY.TechJournalExportAssistantWithSharedBufferConsoleApp.csproj", "{75DDC845-838D-4F33-914B-2D833179A463}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -60,6 +62,10 @@ Global
{62CDAE77-D687-423E-B88C-25967ECC2353}.Debug|Any CPU.Build.0 = Debug|Any CPU
{62CDAE77-D687-423E-B88C-25967ECC2353}.Release|Any CPU.ActiveCfg = Release|Any CPU
{62CDAE77-D687-423E-B88C-25967ECC2353}.Release|Any CPU.Build.0 = Release|Any CPU
{75DDC845-838D-4F33-914B-2D833179A463}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{75DDC845-838D-4F33-914B-2D833179A463}.Debug|Any CPU.Build.0 = Debug|Any CPU
{75DDC845-838D-4F33-914B-2D833179A463}.Release|Any CPU.ActiveCfg = Release|Any CPU
{75DDC845-838D-4F33-914B-2D833179A463}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@@ -71,6 +77,7 @@ Global
{62CDAE77-D687-423E-B88C-25967ECC2353} = {274A8CD4-5F80-4164-B0F6-1F2ACBB8E00B}
{56E62BAA-09E0-4119-AA48-0F3B6E80C1F7} = {300AD5BD-47CB-4AA8-B823-9B061C7654D1}
{7E5B0AC0-4A78-4BC2-9C54-E461317125AB} = {300AD5BD-47CB-4AA8-B823-9B061C7654D1}
{75DDC845-838D-4F33-914B-2D833179A463} = {81F37230-D7FE-4190-A939-B5AE8A71494B}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {B3DABD43-6E3B-4871-B426-2C4BDD6018CC}