mirror of
https://github.com/krugersu/YY.TechJournalExportAssistant.git
synced 2026-06-19 21:46:53 +02:00
Реализован механизм общего буфера и другие улучшения
1. Отказ от таблиц в ClickHouse с движком Buffer из-за проблем со стабильностью работы. 2. Отказ от хранения имени каталога в объекте технологического журнала 3. Улучшены операции массовой вставки данных записей логов 4. Оптимизация количества обращений к ClickHouse при экспорте логов 5. Реализован вспомогательный класс экспорта логов в буфер 6. Реализован механизм общего буфера для передачи записей логов в ClickHouse из различных источников 7. Расширен набор unit-тестов 8. Добавлен пример приложения для работы с общим буфером 9. Обновлены версии библиотек 10. Исправление ошибок и другие небольшие улучшения
This commit is contained in:
@@ -73,7 +73,6 @@ namespace YY.TechJournalExportAssistantConsoleApp
|
||||
target.SetInformationSystem(new TechJournalLogBase()
|
||||
{
|
||||
Name = techJournalName,
|
||||
DirectoryName = tjDirectory.DirectoryData.Name,
|
||||
Description = techJournalDescription
|
||||
});
|
||||
exporter.SetTarget(target);
|
||||
|
||||
+2
@@ -15,4 +15,6 @@
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
<ProjectExtensions><VisualStudio><UserProperties appsettings_1json__JsonSchema="" /></VisualStudio></ProjectExtensions>
|
||||
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using YY.TechJournalExportAssistant.ClickHouse;
|
||||
using YY.TechJournalExportAssistant.Core;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs;
|
||||
|
||||
namespace YY.TechJournalExportAssistantWithSharedBufferConsoleApp
|
||||
{
|
||||
class Program
|
||||
{
|
||||
public static IConfiguration Configuration = new ConfigurationBuilder()
|
||||
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: false)
|
||||
.Build();
|
||||
|
||||
static async Task Main(string[] args)
|
||||
{
|
||||
TechJournalSettings settings = TechJournalSettings.Create(Configuration);
|
||||
|
||||
TechJournalExport exportMaster = new TechJournalExport(settings, new TechJournalOnClickHouseTargetBuilder());
|
||||
exportMaster.OnErrorEvent += OnError;
|
||||
exportMaster.OnSendLogEvent += OnSend;
|
||||
await exportMaster.StartExport();
|
||||
|
||||
Console.WriteLine("Good luck & bye!");
|
||||
}
|
||||
|
||||
private static void OnError(
|
||||
TechJournalSettings.LogSourceSettings settings,
|
||||
OnErrorExportSharedBufferEventArgs args)
|
||||
{
|
||||
Console.WriteLine($"Ошибка при экспорте логов ({settings?.Name ?? "<>"}): {args.Exception}");
|
||||
}
|
||||
|
||||
private static void OnSend(
|
||||
TechJournalSettings.LogSourceSettings settings,
|
||||
OnSendLogFromSharedBufferEventArgs args)
|
||||
{
|
||||
Console.WriteLine($"Отправка данных в хранилище ({settings?.Name ?? "<>"}):\n" +
|
||||
$"Записей: {args._rows.Select(e => e.Value.Count).Sum()}\n" +
|
||||
$"Актуальных позиций чтения: {args._positions.Count}");
|
||||
}
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net5.0</TargetFramework>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\Libs\YY.TechJournalExportAssistant.ClickHouse\YY.TechJournalExportAssistant.ClickHouse.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<None Update="appsettings.json">
|
||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
|
||||
<ProjectExtensions><VisualStudio><UserProperties appsettings_1json__JsonSchema="" /></VisualStudio></ProjectExtensions>
|
||||
|
||||
</Project>
|
||||
@@ -8,6 +8,7 @@ using ClickHouse.Client.Copy;
|
||||
using YY.TechJournalExportAssistant.ClickHouse.Helpers;
|
||||
using YY.TechJournalExportAssistant.Core;
|
||||
using YY.TechJournalExportAssistant.Core.Helpers;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using YY.TechJournalReaderAssistant.Helpers;
|
||||
using YY.TechJournalReaderAssistant.Models;
|
||||
@@ -19,7 +20,6 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
#region Private Members
|
||||
|
||||
private ClickHouseConnection _connection;
|
||||
private long logFileLastId = -1;
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -31,16 +31,12 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
|
||||
_connection = new ClickHouseConnection(connectionSettings);
|
||||
_connection.Open();
|
||||
|
||||
|
||||
var cmdDDL = _connection.CreateCommand();
|
||||
cmdDDL.CommandText = Resource.Query_CreateTable_EventDataStorage;
|
||||
cmdDDL.ExecuteNonQuery();
|
||||
cmdDDL.CommandText = Resource.Query_CreateTable_LogFilesStorage;
|
||||
cmdDDL.ExecuteNonQuery();
|
||||
cmdDDL.CommandText = Resource.Query_CreateTable_EventData;
|
||||
cmdDDL.ExecuteNonQuery();
|
||||
cmdDDL.CommandText = Resource.Query_CreateTable_LogFiles;
|
||||
cmdDDL.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
#endregion
|
||||
@@ -49,18 +45,19 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
|
||||
#region RowsData
|
||||
|
||||
public void SaveRowsData(TechJournalLogBase techJournalLog, List<EventData> eventData, string fileName)
|
||||
public void SaveRowsData(TechJournalLogBase techJournalLog, List<EventData> eventData, string directoryName, string fileName)
|
||||
{
|
||||
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
|
||||
{
|
||||
DestinationTableName = "EventData",
|
||||
BatchSize = 100000
|
||||
BatchSize = 100000,
|
||||
MaxDegreeOfParallelism = 4
|
||||
})
|
||||
{
|
||||
var values = eventData.Select(i => new object[]
|
||||
{
|
||||
techJournalLog.Name,
|
||||
techJournalLog.DirectoryName,
|
||||
directoryName,
|
||||
fileName,
|
||||
i.Id,
|
||||
i.Period,
|
||||
@@ -73,7 +70,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
i.SessionId ?? 0,
|
||||
i.ApplicationName ?? string.Empty,
|
||||
i.ClientId ?? 0,
|
||||
i.ComputerName ?? string.Empty,
|
||||
i.ComputerName ?? string.Empty,
|
||||
i.ConnectionId ?? 0,
|
||||
i.UserName ?? string.Empty,
|
||||
i.ApplicationId ?? 0,
|
||||
@@ -100,7 +97,93 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
bulkResult.Wait();
|
||||
}
|
||||
}
|
||||
public DateTime GetRowsDataMaxPeriod(TechJournalLogBase techJournalLog, string fileName)
|
||||
|
||||
public void SaveRowsData(TechJournalLogBase techJournalLog, IDictionary<string, List<EventData>> eventData)
|
||||
{
|
||||
Dictionary<string, DateTime> maxPeriodByFiles = new Dictionary<string, DateTime>();
|
||||
List<object[]> rowsForInsert = new List<object[]>();
|
||||
foreach (var eventInfo in eventData)
|
||||
{
|
||||
FileInfo logFileInfo = new FileInfo(eventInfo.Key);
|
||||
foreach (var eventItem in eventInfo.Value)
|
||||
{
|
||||
if (!maxPeriodByFiles.TryGetValue(logFileInfo.Name, out DateTime maxPeriodValue))
|
||||
{
|
||||
if (logFileInfo.Directory != null)
|
||||
maxPeriodValue = GetRowsDataMaxPeriod(
|
||||
techJournalLog,
|
||||
logFileInfo.Directory.Name,
|
||||
logFileInfo.Name
|
||||
);
|
||||
maxPeriodByFiles.Add(logFileInfo.Name, maxPeriodValue);
|
||||
}
|
||||
else
|
||||
{
|
||||
maxPeriodValue = DateTime.MinValue;
|
||||
}
|
||||
if (maxPeriodValue != DateTime.MinValue && eventItem.Period <= maxPeriodValue)
|
||||
if (logFileInfo.Directory != null && RowDataExistOnDatabase(techJournalLog, eventItem, logFileInfo.Directory.Name, logFileInfo.Name))
|
||||
continue;
|
||||
|
||||
if (logFileInfo.Directory != null)
|
||||
rowsForInsert.Add(new object[]
|
||||
{
|
||||
techJournalLog.Name,
|
||||
logFileInfo.Directory.Name,
|
||||
eventInfo.Key,
|
||||
eventItem.Id,
|
||||
eventItem.Period,
|
||||
eventItem.Level,
|
||||
eventItem.Duration,
|
||||
eventItem.DurationSec,
|
||||
eventItem.EventName ?? string.Empty,
|
||||
eventItem.ServerContextName ?? string.Empty,
|
||||
eventItem.ProcessName ?? string.Empty,
|
||||
eventItem.SessionId ?? 0,
|
||||
eventItem.ApplicationName ?? string.Empty,
|
||||
eventItem.ClientId ?? 0,
|
||||
eventItem.ComputerName ?? string.Empty,
|
||||
eventItem.ConnectionId ?? 0,
|
||||
eventItem.UserName ?? string.Empty,
|
||||
eventItem.ApplicationId ?? 0,
|
||||
eventItem.Context ?? string.Empty,
|
||||
eventItem.ActionType.GetDescription() ?? string.Empty,
|
||||
eventItem.Database ?? string.Empty,
|
||||
eventItem.DatabaseCopy ?? string.Empty,
|
||||
eventItem.DBMS.GetPresentation() ?? string.Empty,
|
||||
eventItem.DatabasePID ?? string.Empty,
|
||||
eventItem.PlanSQLText ?? string.Empty,
|
||||
eventItem.Rows ?? 0,
|
||||
eventItem.RowsAffected ?? 0,
|
||||
eventItem.SQLText ?? string.Empty,
|
||||
eventItem.SQLQueryOnly ?? string.Empty,
|
||||
eventItem.SQLQueryParametersOnly ?? string.Empty,
|
||||
eventItem.SQLQueryHash ?? string.Empty,
|
||||
eventItem.SDBL ?? string.Empty,
|
||||
eventItem.Description ?? string.Empty,
|
||||
eventItem.Message ?? string.Empty,
|
||||
eventItem.GetCustomFieldsAsJSON() ?? string.Empty
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (rowsForInsert.Count == 0)
|
||||
return;
|
||||
|
||||
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
|
||||
{
|
||||
DestinationTableName = "EventData",
|
||||
BatchSize = 100000,
|
||||
MaxDegreeOfParallelism = 4
|
||||
})
|
||||
{
|
||||
var bulkResult = bulkCopyInterface.WriteToServerAsync(rowsForInsert);
|
||||
bulkResult.Wait();
|
||||
rowsForInsert.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
public DateTime GetRowsDataMaxPeriod(TechJournalLogBase techJournalLog, string directoryName, string fileName)
|
||||
{
|
||||
DateTime output = DateTime.MinValue;
|
||||
|
||||
@@ -114,7 +197,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
AND DirectoryName = {directoryName:String}
|
||||
AND FileName = {fileName:String}";
|
||||
command.AddParameterToCommand("techJournalLog", techJournalLog.Name);
|
||||
command.AddParameterToCommand("directoryName", techJournalLog.DirectoryName);
|
||||
command.AddParameterToCommand("directoryName", directoryName);
|
||||
command.AddParameterToCommand("fileName", fileName);
|
||||
using (var cmdReader = command.ExecuteReader())
|
||||
if (cmdReader.Read()) output = cmdReader.GetDateTime(0);
|
||||
@@ -122,7 +205,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
|
||||
return output;
|
||||
}
|
||||
public bool RowDataExistOnDatabase(TechJournalLogBase techJournalLog, EventData eventData, string fileName)
|
||||
public bool RowDataExistOnDatabase(TechJournalLogBase techJournalLog, EventData eventData, string directoryName, string fileName)
|
||||
{
|
||||
bool output = false;
|
||||
|
||||
@@ -136,11 +219,13 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
FROM EventData AS RD
|
||||
WHERE TechJournalLog = {techJournalLog:String}
|
||||
AND Id = {existId:Int64}
|
||||
AND DirectoryName = {directoryName:String}
|
||||
AND FileName = {fileName:String}
|
||||
AND Period = {existPeriod:DateTime}";
|
||||
command.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
|
||||
command.AddParameterToCommand("existId", DbType.Int64, eventData.Id);
|
||||
command.AddParameterToCommand("existPeriod", DbType.DateTime, eventData.Period);
|
||||
command.AddParameterToCommand("directoryName", DbType.String, directoryName);
|
||||
command.AddParameterToCommand("fileName", DbType.String, fileName);
|
||||
using (var cmdReader = command.ExecuteReader())
|
||||
if (cmdReader.Read()) output = true;
|
||||
@@ -153,7 +238,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
|
||||
#region LogFiles
|
||||
|
||||
public TechJournalPosition GetLogFilePosition(TechJournalLogBase techJournalLog)
|
||||
public TechJournalPosition GetLogFilePosition(TechJournalLogBase techJournalLog, string directoryName)
|
||||
{
|
||||
var cmdGetLastLogFileInfo = _connection.CreateCommand();
|
||||
cmdGetLastLogFileInfo.CommandText =
|
||||
@@ -172,7 +257,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
AND LF_LAST.DirectoryName = {directoryName:String}
|
||||
)";
|
||||
cmdGetLastLogFileInfo.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
|
||||
cmdGetLastLogFileInfo.AddParameterToCommand("directoryName", DbType.AnsiString, techJournalLog.DirectoryName);
|
||||
cmdGetLastLogFileInfo.AddParameterToCommand("directoryName", DbType.AnsiString, directoryName);
|
||||
|
||||
TechJournalPosition output = null;
|
||||
using (var cmdReader = cmdGetLastLogFileInfo.ExecuteReader())
|
||||
@@ -191,76 +276,63 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
|
||||
return output;
|
||||
}
|
||||
public void SaveLogPosition(TechJournalLogBase techJournalLog, FileInfo logFileInfo, TechJournalPosition position)
|
||||
public void SaveLogPosition(TechJournalLogBase techJournalLog, TechJournalPosition position)
|
||||
{
|
||||
var dataFileInfo = new FileInfo(position.CurrentFileData);
|
||||
using (ClickHouseBulkCopy bulkCopyInterface = new ClickHouseBulkCopy(_connection)
|
||||
{
|
||||
DestinationTableName = "LogFiles", BatchSize = 100000
|
||||
DestinationTableName = "LogFiles",
|
||||
BatchSize = 1000000
|
||||
})
|
||||
{
|
||||
IEnumerable<object[]> values = new List<object[]>()
|
||||
if (dataFileInfo.Directory != null)
|
||||
{
|
||||
new object[]
|
||||
IEnumerable<object[]> values = new List<object[]>()
|
||||
{
|
||||
techJournalLog.Name,
|
||||
techJournalLog.DirectoryName,
|
||||
GetLogFileInfoNewId(techJournalLog),
|
||||
logFileInfo.Name,
|
||||
logFileInfo.CreationTimeUtc,
|
||||
logFileInfo.LastWriteTimeUtc,
|
||||
position.EventNumber,
|
||||
position.CurrentFileData.Replace("\\", "\\\\"),
|
||||
position.StreamPosition ?? 0
|
||||
}
|
||||
}.AsEnumerable();
|
||||
var bulkResult = bulkCopyInterface.WriteToServerAsync(values);
|
||||
bulkResult.Wait();
|
||||
}
|
||||
}
|
||||
public long GetLogFileInfoNewId(TechJournalLogBase techJournalLog)
|
||||
{
|
||||
long output = 0;
|
||||
|
||||
if (logFileLastId < 0)
|
||||
{
|
||||
using (var command = _connection.CreateCommand())
|
||||
{
|
||||
command.CommandText =
|
||||
@"SELECT
|
||||
MAX(Id)
|
||||
FROM LogFiles
|
||||
WHERE TechJournalLog = {techJournalLog:String}
|
||||
AND DirectoryName = {directoryName:String}";
|
||||
command.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
|
||||
command.AddParameterToCommand("directoryName", DbType.AnsiString, techJournalLog.DirectoryName);
|
||||
using (var cmdReader = command.ExecuteReader())
|
||||
if (cmdReader.Read()) output = cmdReader.GetInt64(0);
|
||||
new object[]
|
||||
{
|
||||
techJournalLog.Name,
|
||||
dataFileInfo.Directory.Name,
|
||||
DateTime.Now.Ticks,
|
||||
dataFileInfo.Name,
|
||||
dataFileInfo.CreationTimeUtc,
|
||||
dataFileInfo.LastWriteTimeUtc,
|
||||
position.EventNumber,
|
||||
position.CurrentFileData.Replace("\\", "\\\\"),
|
||||
position.StreamPosition ?? 0
|
||||
}
|
||||
}.AsEnumerable();
|
||||
var bulkResult = bulkCopyInterface.WriteToServerAsync(values);
|
||||
bulkResult.Wait();
|
||||
}
|
||||
}
|
||||
else output = logFileLastId;
|
||||
}
|
||||
public IDictionary<string, TechJournalPosition> GetCurrentLogPositions(
|
||||
TechJournalLogBase techJournalLog,
|
||||
TechJournalSettings settings,
|
||||
KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
|
||||
{
|
||||
var cmdGetLastLogFileInfo = _connection.CreateCommand();
|
||||
cmdGetLastLogFileInfo.CommandText = Resource.Query_GetActualPositions;
|
||||
cmdGetLastLogFileInfo.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
|
||||
|
||||
output += 1;
|
||||
logFileLastId = output;
|
||||
IDictionary<string, TechJournalPosition> output = new Dictionary<string, TechJournalPosition>();
|
||||
using (var cmdReader = cmdGetLastLogFileInfo.ExecuteReader())
|
||||
{
|
||||
while (cmdReader.Read())
|
||||
{
|
||||
string fileName = cmdReader.GetString(7).Replace("\\\\", "\\");
|
||||
string directoryName = cmdReader.GetString(1);
|
||||
output.Add(directoryName, new TechJournalPosition(
|
||||
cmdReader.GetInt64(6),
|
||||
fileName,
|
||||
cmdReader.GetInt64(8)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
public void RemoveArchiveLogFileRecords(TechJournalLogBase techJournalLog)
|
||||
{
|
||||
var commandRemoveArchiveLogInfo = _connection.CreateCommand();
|
||||
commandRemoveArchiveLogInfo.CommandText =
|
||||
@"ALTER TABLE LogFilesStorage DELETE
|
||||
WHERE TechJournalLog = {techJournalLog:String}
|
||||
AND DirectoryName = {directoryName:String}
|
||||
AND Id < (
|
||||
SELECT MAX(Id) AS LastId
|
||||
FROM LogFilesStorage lf
|
||||
WHERE TechJournalLog = {techJournalLog:String}
|
||||
AND DirectoryName = {directoryName:String}
|
||||
)";
|
||||
commandRemoveArchiveLogInfo.AddParameterToCommand("techJournalLog", DbType.AnsiString, techJournalLog.Name);
|
||||
commandRemoveArchiveLogInfo.AddParameterToCommand("directoryName", DbType.AnsiString, techJournalLog.DirectoryName);
|
||||
commandRemoveArchiveLogInfo.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
{
|
||||
public class EventDataBulkInsertObject : IEnumerable
|
||||
{
|
||||
public string TechJournalLog { set; get; }
|
||||
public string DirectoryName { set; get; }
|
||||
public string FileName { set; get; }
|
||||
public long Id { set; get; }
|
||||
public DateTime Period { set; get; }
|
||||
public long Level { set; get; }
|
||||
public long Duration { set; get; }
|
||||
public long DurationSec { set; get; }
|
||||
public string EventName { set; get; }
|
||||
public string ServerContextName { set; get; }
|
||||
public string ProcessName { set; get; }
|
||||
public long? SessionId { set; get; }
|
||||
public string ApplicationName { set; get; }
|
||||
public long? ClientId { set; get; }
|
||||
public string ComputerName { set; get; }
|
||||
public long? ConnectionId { set; get; }
|
||||
public string UserName { set; get; }
|
||||
public long? ApplicationId { set; get; }
|
||||
public string Context { set; get; }
|
||||
public string ActionType { set; get; }
|
||||
public string Database { set; get; }
|
||||
public string DatabaseCopy { set; get; }
|
||||
public string DBMS { set; get; }
|
||||
public string DatabasePID { set; get; }
|
||||
public string PlanSQLText { set; get; }
|
||||
public long? Rows { set; get; }
|
||||
public long? RowsAffected { set; get; }
|
||||
public string SQLText { set; get; }
|
||||
public string SQLQueryOnly { set; get; }
|
||||
public string SQLQueryParametersOnly { set; get; }
|
||||
public string SQLQueryHash { set; get; }
|
||||
public string SDBL { set; get; }
|
||||
public string Description { set; get; }
|
||||
public string Message { set; get; }
|
||||
public string CustomEventData { set; get; }
|
||||
|
||||
public IEnumerator GetEnumerator()
|
||||
{
|
||||
yield return TechJournalLog;
|
||||
yield return DirectoryName;
|
||||
yield return FileName;
|
||||
yield return Id;
|
||||
yield return Period;
|
||||
yield return Level;
|
||||
yield return Duration;
|
||||
yield return DurationSec;
|
||||
yield return EventName;
|
||||
yield return ServerContextName;
|
||||
yield return ProcessName;
|
||||
yield return SessionId;
|
||||
yield return ApplicationName;
|
||||
yield return ClientId;
|
||||
yield return ComputerName;
|
||||
yield return ConnectionId;
|
||||
yield return UserName;
|
||||
yield return ApplicationId;
|
||||
yield return Context;
|
||||
yield return ActionType;
|
||||
yield return Database;
|
||||
yield return DatabaseCopy;
|
||||
yield return DBMS;
|
||||
yield return DatabasePID;
|
||||
yield return PlanSQLText;
|
||||
yield return Rows;
|
||||
yield return RowsAffected;
|
||||
yield return SQLText;
|
||||
yield return SQLQueryOnly;
|
||||
yield return SQLQueryParametersOnly;
|
||||
yield return SQLQueryHash;
|
||||
yield return SDBL;
|
||||
yield return Description;
|
||||
yield return Message;
|
||||
yield return CustomEventData;
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
-25
@@ -61,16 +61,7 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS EventData AS EventDataStorage ENGINE = Buffer(currentDatabase(), EventDataStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000).
|
||||
/// </summary>
|
||||
internal static string Query_CreateTable_EventData {
|
||||
get {
|
||||
return ResourceManager.GetString("Query_CreateTable_EventData", resourceCulture);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS EventDataStorage
|
||||
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS EventData
|
||||
///(
|
||||
/// TechJournalLog LowCardinality(String),
|
||||
/// DirectoryName LowCardinality(String),
|
||||
@@ -82,7 +73,8 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
|
||||
/// DurationSec Int64 Codec(DoubleDelta, LZ4),
|
||||
/// EventName LowCardinality(String),
|
||||
/// ServerContextName LowCardinality(String),
|
||||
/// ProcessName LowCardinality(String), [остаток строки не уместился]";.
|
||||
/// ProcessName LowCardinality(String),
|
||||
/// S [остаток строки не уместился]";.
|
||||
/// </summary>
|
||||
internal static string Query_CreateTable_EventDataStorage {
|
||||
get {
|
||||
@@ -91,20 +83,11 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS LogFiles AS LogFilesStorage ENGINE = Buffer(currentDatabase(), LogFilesStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000).
|
||||
/// </summary>
|
||||
internal static string Query_CreateTable_LogFiles {
|
||||
get {
|
||||
return ResourceManager.GetString("Query_CreateTable_LogFiles", resourceCulture);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS LogFilesStorage
|
||||
/// Ищет локализованную строку, похожую на CREATE TABLE IF NOT EXISTS LogFiles
|
||||
///(
|
||||
/// TechJournalLog LowCardinality(String),
|
||||
/// DirectoryName LowCardinality(String),
|
||||
/// Id Int64 Codec(DoubleDelta, LZ4),
|
||||
/// Id Int64 DEFAULT (toUnixTimestamp64Milli(now64())*1000000 + rowNumberInAllBlocks()) Codec(DoubleDelta, LZ4),
|
||||
/// FileName LowCardinality(String),
|
||||
/// CreateDate DateTime Codec(Delta, LZ4),
|
||||
/// ModificationDate DateTime Codec(Delta, LZ4),
|
||||
@@ -112,14 +95,47 @@ namespace YY.TechJournalExportAssistant.ClickHouse {
|
||||
/// LastCurrentFileData LowCardinality(String),
|
||||
/// LastStreamPosition Int64 Codec(DoubleDelta, LZ4)
|
||||
///)
|
||||
///engine = MergeTree()
|
||||
///PARTITION BY toYYYYMM(CreateDate)
|
||||
///PRIMARY KEY CreateD [остаток строки не уместился]";.
|
||||
///engine = [остаток строки не уместился]";.
|
||||
/// </summary>
|
||||
internal static string Query_CreateTable_LogFilesStorage {
|
||||
get {
|
||||
return ResourceManager.GetString("Query_CreateTable_LogFilesStorage", resourceCulture);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ищет локализованную строку, похожую на SELECT
|
||||
/// l.TechJournalLog,
|
||||
/// l.DirectoryName,
|
||||
/// l.Id,
|
||||
/// l.FileName,
|
||||
/// l.CreateDate,
|
||||
/// l.ModificationDate,
|
||||
/// l.LastEventNumber,
|
||||
/// l.LastCurrentFileData,
|
||||
/// l.LastStreamPosition
|
||||
///FROM LogFiles l
|
||||
///
|
||||
///INNER JOIN
|
||||
///
|
||||
///(
|
||||
///SELECT
|
||||
/// TechJournalLog,
|
||||
/// DirectoryName,
|
||||
/// MAX(Id) LastId
|
||||
///FROM LogFiles AS LF_LAST
|
||||
///WHERE LF_LAST.TechJournalLog = {techJournalLog:String}
|
||||
///GROUP BY TechJournalLog,
|
||||
/// DirectoryName
|
||||
///) lglast
|
||||
///
|
||||
///ON l.TechJournalLog = lglast.TechJournalLog
|
||||
/// AND l.DirectoryNam [остаток строки не уместился]";.
|
||||
/// </summary>
|
||||
internal static string Query_GetActualPositions {
|
||||
get {
|
||||
return ResourceManager.GetString("Query_GetActualPositions", resourceCulture);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -117,12 +117,8 @@
|
||||
<resheader name="writer">
|
||||
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
|
||||
</resheader>
|
||||
<data name="Query_CreateTable_EventData" xml:space="preserve">
|
||||
<value>CREATE TABLE IF NOT EXISTS EventData AS EventDataStorage ENGINE = Buffer(currentDatabase(), EventDataStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000)</value>
|
||||
<comment>Создание буфера для новых данных технологического журнала</comment>
|
||||
</data>
|
||||
<data name="Query_CreateTable_EventDataStorage" xml:space="preserve">
|
||||
<value>CREATE TABLE IF NOT EXISTS EventDataStorage
|
||||
<value>CREATE TABLE IF NOT EXISTS EventData
|
||||
(
|
||||
TechJournalLog LowCardinality(String),
|
||||
DirectoryName LowCardinality(String),
|
||||
@@ -167,15 +163,12 @@ ORDER BY Period
|
||||
SETTINGS index_granularity = 8192;</value>
|
||||
<comment>Создание таблицы для хранения данных технологического журнала</comment>
|
||||
</data>
|
||||
<data name="Query_CreateTable_LogFiles" xml:space="preserve">
|
||||
<value>CREATE TABLE IF NOT EXISTS LogFiles AS LogFilesStorage ENGINE = Buffer(currentDatabase(), LogFilesStorage, 16, 10, 60, 100000, 1000000, 10000000, 100000000)</value>
|
||||
</data>
|
||||
<data name="Query_CreateTable_LogFilesStorage" xml:space="preserve">
|
||||
<value>CREATE TABLE IF NOT EXISTS LogFilesStorage
|
||||
<value>CREATE TABLE IF NOT EXISTS LogFiles
|
||||
(
|
||||
TechJournalLog LowCardinality(String),
|
||||
DirectoryName LowCardinality(String),
|
||||
Id Int64 Codec(DoubleDelta, LZ4),
|
||||
Id Int64 DEFAULT (toUnixTimestamp64Milli(now64())*1000000 + rowNumberInAllBlocks()) Codec(DoubleDelta, LZ4),
|
||||
FileName LowCardinality(String),
|
||||
CreateDate DateTime Codec(Delta, LZ4),
|
||||
ModificationDate DateTime Codec(Delta, LZ4),
|
||||
@@ -190,4 +183,36 @@ ORDER BY CreateDate
|
||||
SETTINGS index_granularity = 8192;</value>
|
||||
<comment>Создание таблицы для хранения данных лога обработки файлов</comment>
|
||||
</data>
|
||||
<data name="Query_GetActualPositions" xml:space="preserve">
|
||||
<value>SELECT
|
||||
l.TechJournalLog,
|
||||
l.DirectoryName,
|
||||
l.Id,
|
||||
l.FileName,
|
||||
l.CreateDate,
|
||||
l.ModificationDate,
|
||||
l.LastEventNumber,
|
||||
l.LastCurrentFileData,
|
||||
l.LastStreamPosition
|
||||
FROM LogFiles l
|
||||
|
||||
INNER JOIN
|
||||
|
||||
(
|
||||
SELECT
|
||||
TechJournalLog,
|
||||
DirectoryName,
|
||||
MAX(Id) LastId
|
||||
FROM LogFiles AS LF_LAST
|
||||
WHERE LF_LAST.TechJournalLog = {techJournalLog:String}
|
||||
GROUP BY TechJournalLog,
|
||||
DirectoryName
|
||||
) lglast
|
||||
|
||||
ON l.TechJournalLog = lglast.TechJournalLog
|
||||
AND l.DirectoryName = lglast.DirectoryName
|
||||
AND l.Id = lglast.LastId
|
||||
ORDER BY l.DirectoryName</value>
|
||||
<comment>Получение последних событий</comment>
|
||||
</data>
|
||||
</root>
|
||||
@@ -3,6 +3,7 @@ using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using YY.TechJournalExportAssistant.Core;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using EventData = YY.TechJournalReaderAssistant.Models.EventData;
|
||||
|
||||
@@ -18,8 +19,6 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
private TechJournalLogBase _techJournalLog;
|
||||
private readonly string _connectionString;
|
||||
private TechJournalPosition _lastTechJournalFilePosition;
|
||||
private int _stepsToClearLogFiles = 10000;
|
||||
private int _currentStepToClearLogFiles;
|
||||
|
||||
#endregion
|
||||
|
||||
@@ -53,29 +52,23 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
|
||||
#region Public Methods
|
||||
|
||||
public override TechJournalPosition GetLastPosition()
|
||||
public override TechJournalPosition GetLastPosition(string directoryName)
|
||||
{
|
||||
if (_lastTechJournalFilePosition != null)
|
||||
return _lastTechJournalFilePosition;
|
||||
|
||||
TechJournalPosition position;
|
||||
using(var context = new ClickHouseContext(_connectionString))
|
||||
position = context.GetLogFilePosition(_techJournalLog);
|
||||
position = context.GetLogFilePosition(_techJournalLog, directoryName);
|
||||
|
||||
_lastTechJournalFilePosition = position;
|
||||
return position;
|
||||
}
|
||||
public override void SaveLogPosition(FileInfo logFileInfo, TechJournalPosition position)
|
||||
public override void SaveLogPosition(TechJournalPosition position)
|
||||
{
|
||||
using (var context = new ClickHouseContext(_connectionString))
|
||||
{
|
||||
context.SaveLogPosition(_techJournalLog, logFileInfo, position);
|
||||
if (_currentStepToClearLogFiles == 0 || _currentStepToClearLogFiles >= _stepsToClearLogFiles)
|
||||
{
|
||||
context.RemoveArchiveLogFileRecords(_techJournalLog);
|
||||
_currentStepToClearLogFiles = 0;
|
||||
}
|
||||
_currentStepToClearLogFiles += 1;
|
||||
context.SaveLogPosition(_techJournalLog, position);
|
||||
}
|
||||
|
||||
_lastTechJournalFilePosition = position;
|
||||
@@ -98,14 +91,17 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
if(rowsData.Count == 0)
|
||||
return;
|
||||
|
||||
FileInfo fileInfo = new FileInfo(fileName);
|
||||
using (var context = new ClickHouseContext(_connectionString))
|
||||
{
|
||||
if (_maxPeriodRowData == DateTime.MinValue)
|
||||
{
|
||||
_maxPeriodRowData = context.GetRowsDataMaxPeriod(
|
||||
_techJournalLog,
|
||||
fileName
|
||||
);
|
||||
if (fileInfo.Directory != null)
|
||||
_maxPeriodRowData = context.GetRowsDataMaxPeriod(
|
||||
_techJournalLog,
|
||||
fileInfo.Directory.Name,
|
||||
fileName
|
||||
);
|
||||
}
|
||||
|
||||
List<EventData> newEntities = new List<EventData>();
|
||||
@@ -114,12 +110,15 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
if (itemRow == null)
|
||||
continue;
|
||||
if (_maxPeriodRowData != DateTime.MinValue && itemRow.Period <= _maxPeriodRowData)
|
||||
if (context.RowDataExistOnDatabase(_techJournalLog, itemRow, fileName))
|
||||
if (fileInfo.Directory != null
|
||||
&& context.RowDataExistOnDatabase(_techJournalLog, itemRow, fileInfo.Directory.Name, fileInfo.Name))
|
||||
continue;
|
||||
|
||||
newEntities.Add(itemRow);
|
||||
}
|
||||
context.SaveRowsData(_techJournalLog, newEntities, fileName);
|
||||
|
||||
if (fileInfo.Directory != null)
|
||||
context.SaveRowsData(_techJournalLog, newEntities, fileInfo.Directory.Name, fileName);
|
||||
}
|
||||
}
|
||||
public override void SetInformationSystem(TechJournalLogBase techJournalLog)
|
||||
@@ -127,6 +126,29 @@ namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
_techJournalLog = techJournalLog;
|
||||
}
|
||||
|
||||
public override void Save(IDictionary<string, List<EventData>> rowsData)
|
||||
{
|
||||
if (rowsData.Count == 0)
|
||||
return;
|
||||
|
||||
using (var context = new ClickHouseContext(_connectionString))
|
||||
{
|
||||
context.SaveRowsData(_techJournalLog, rowsData);
|
||||
}
|
||||
}
|
||||
|
||||
public override IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
|
||||
{
|
||||
IDictionary<string, TechJournalPosition> positions;
|
||||
|
||||
using (var context = new ClickHouseContext(_connectionString))
|
||||
{
|
||||
positions = context.GetCurrentLogPositions(_techJournalLog, settings, logBufferItem);
|
||||
}
|
||||
|
||||
return positions;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
using System.Collections.Generic;
|
||||
using YY.TechJournalExportAssistant.Core;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.ClickHouse
|
||||
{
|
||||
public class TechJournalOnClickHouseTargetBuilder : ITechJournalOnTargetBuilder
|
||||
{
|
||||
public ITechJournalOnTarget CreateTarget(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
|
||||
{
|
||||
ITechJournalOnTarget target = new TechJournalOnClickHouse(settings.ConnectionString,
|
||||
logBufferItem.Key.Portion);
|
||||
target.SetInformationSystem(new TechJournalLogBase()
|
||||
{
|
||||
Name = logBufferItem.Key.Name,
|
||||
Description = logBufferItem.Key.Description
|
||||
});
|
||||
|
||||
return target;
|
||||
}
|
||||
|
||||
public IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
|
||||
{
|
||||
ITechJournalOnTarget target = CreateTarget(settings, logBufferItem);
|
||||
return target.GetCurrentLogPositions(settings, logBufferItem);
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.1</TargetFramework>
|
||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||
<Version>1.0.0.9</Version>
|
||||
<Version>1.0.1.1</Version>
|
||||
<Authors>Permitin Yuriy</Authors>
|
||||
<Product>Technological journal's export assistant to ClickHouse</Product>
|
||||
<Description>Library for exporting 1C:Enterprise 8.x platform's technological journal files to ClickHouse</Description>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using YY.TechJournalReaderAssistant.Models;
|
||||
|
||||
@@ -7,11 +7,14 @@ namespace YY.TechJournalExportAssistant.Core
|
||||
{
|
||||
public interface ITechJournalOnTarget
|
||||
{
|
||||
TechJournalPosition GetLastPosition();
|
||||
void SaveLogPosition(FileInfo logFileInfo, TechJournalPosition position);
|
||||
TechJournalPosition GetLastPosition(string directoryName);
|
||||
void SaveLogPosition(TechJournalPosition position);
|
||||
int GetPortionSize();
|
||||
void SetInformationSystem(TechJournalLogBase techJournalLog);
|
||||
void Save(EventData eventData, string fileName);
|
||||
void Save(IList<EventData> eventData, string fileName);
|
||||
void Save(IDictionary<string, List<EventData>> rowsData);
|
||||
IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings,
|
||||
KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem);
|
||||
}
|
||||
}
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
using System;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs
|
||||
{
|
||||
public sealed class OnErrorExportSharedBufferEventArgs : System.EventArgs
|
||||
{
|
||||
public OnErrorExportSharedBufferEventArgs(Exception exception)
|
||||
{
|
||||
Exception = exception;
|
||||
}
|
||||
|
||||
public Exception Exception { get; }
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
using System.Collections.Generic;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using YY.TechJournalReaderAssistant.Models;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs
|
||||
{
|
||||
public sealed class OnSendLogFromSharedBufferEventArgs : System.EventArgs
|
||||
{
|
||||
public TechJournalSettings.LogSourceSettings _settings { get; }
|
||||
public IDictionary<string, List<EventData>> _rows { get; }
|
||||
public IReadOnlyDictionary<string, TechJournalPosition> _positions { get; }
|
||||
|
||||
public OnSendLogFromSharedBufferEventArgs(
|
||||
TechJournalSettings.LogSourceSettings settings,
|
||||
IDictionary<string, List<EventData>> rows,
|
||||
IReadOnlyDictionary<string, TechJournalPosition> positions)
|
||||
{
|
||||
_settings = settings;
|
||||
_rows = rows;
|
||||
_positions = positions;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
|
||||
{
|
||||
public class EventKey
|
||||
{
|
||||
public Guid Id { get; set; }
|
||||
public FileInfo File { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
using System.Collections.Generic;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
|
||||
{
|
||||
public interface ITechJournalOnTargetBuilder
|
||||
{
|
||||
ITechJournalOnTarget CreateTarget(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem);
|
||||
IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using YY.TechJournalReaderAssistant.Models;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
|
||||
{
|
||||
/// <summary>
|
||||
/// Элемент буфера логов
|
||||
/// </summary>
|
||||
public class LogBufferItem
|
||||
{
|
||||
/// <summary>
|
||||
/// Дата создания
|
||||
/// </summary>
|
||||
public DateTime Created { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Дата последнего обновления
|
||||
/// </summary>
|
||||
public DateTime LastUpdate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Количество записей лога в буфере
|
||||
/// </summary>
|
||||
public long ItemsCount => LogRows.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Актуальная позиция чтения файлов лога
|
||||
/// </summary>
|
||||
public ConcurrentDictionary<string, TechJournalPosition> LogPositions { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Записи логов
|
||||
/// </summary>
|
||||
public ConcurrentDictionary<EventKey, EventData> LogRows { get; set; }
|
||||
|
||||
public LogBufferItem()
|
||||
{
|
||||
Created = DateTime.Now;
|
||||
LastUpdate = DateTime.MinValue;
|
||||
LogPositions = new ConcurrentDictionary<string, TechJournalPosition>();
|
||||
LogRows = new ConcurrentDictionary<EventKey, EventData>();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using YY.TechJournalReaderAssistant.Models;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
|
||||
{
|
||||
public class LogBuffers
|
||||
{
|
||||
public readonly ConcurrentDictionary<TechJournalSettings.LogSourceSettings, LogBufferItem> LogBuffer;
|
||||
public readonly object LockObject;
|
||||
|
||||
public LogBuffers()
|
||||
{
|
||||
LogBuffer = new ConcurrentDictionary<TechJournalSettings.LogSourceSettings, LogBufferItem>();
|
||||
LockObject = new object();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Общее количество записей логов в буфере
|
||||
/// </summary>
|
||||
public long TotalItemsCount
|
||||
{
|
||||
get
|
||||
{
|
||||
long totalItemsCount = 0;
|
||||
|
||||
foreach (var logBufferItem in LogBuffer)
|
||||
{
|
||||
//lock (logBufferItem.Key.LockObject)
|
||||
//{
|
||||
totalItemsCount += logBufferItem.Value.ItemsCount;
|
||||
//}
|
||||
}
|
||||
|
||||
return totalItemsCount;
|
||||
}
|
||||
}
|
||||
|
||||
public DateTime Created
|
||||
{
|
||||
get
|
||||
{
|
||||
DateTime bufferCreated = DateTime.MinValue;
|
||||
|
||||
foreach (var logBufferItem in LogBuffer)
|
||||
{
|
||||
//lock (logBufferItem.Key.LockObject)
|
||||
//{
|
||||
if (bufferCreated == DateTime.MinValue)
|
||||
{
|
||||
bufferCreated = logBufferItem.Value.Created;
|
||||
} else if (bufferCreated < logBufferItem.Value.Created)
|
||||
{
|
||||
bufferCreated = logBufferItem.Value.Created;
|
||||
}
|
||||
//}
|
||||
}
|
||||
|
||||
return bufferCreated;
|
||||
}
|
||||
}
|
||||
|
||||
public void SaveLogsAndPosition(
|
||||
TechJournalSettings.LogSourceSettings logSettings,
|
||||
TechJournalPosition position,
|
||||
IList<EventData> rowsData
|
||||
)
|
||||
{
|
||||
lock (logSettings.LockObject)
|
||||
{
|
||||
var logFileInfo = new FileInfo(position.CurrentFileData);
|
||||
SaveLogPosition(logSettings, logFileInfo, position);
|
||||
SaveLogs(logSettings, rowsData, logFileInfo);
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveLogPosition(
|
||||
TechJournalSettings.LogSourceSettings logSettings,
|
||||
FileInfo logFileInfo,
|
||||
TechJournalPosition position)
|
||||
{
|
||||
LogBuffer.AddOrUpdate(
|
||||
logSettings,
|
||||
(settings) =>
|
||||
{
|
||||
var newLogBufferItem = new LogBufferItem();
|
||||
|
||||
if (logFileInfo.Directory != null)
|
||||
newLogBufferItem.LogPositions.TryAdd(
|
||||
logFileInfo.Directory.Name,
|
||||
position);
|
||||
|
||||
return newLogBufferItem;
|
||||
},
|
||||
(settings, logBufferItem) =>
|
||||
{
|
||||
if (logFileInfo.Directory != null)
|
||||
logBufferItem.LogPositions.AddOrUpdate(
|
||||
logFileInfo.Directory.Name,
|
||||
(fullDirectoryName) => position,
|
||||
(fullDirectoryName, oldPosition) => position);
|
||||
|
||||
return logBufferItem;
|
||||
});
|
||||
}
|
||||
|
||||
private void SaveLogs(
|
||||
TechJournalSettings.LogSourceSettings logSettings,
|
||||
IList<EventData> rowsData,
|
||||
FileInfo logFileInfo)
|
||||
{
|
||||
LogBuffer.AddOrUpdate(
|
||||
logSettings,
|
||||
(settings) =>
|
||||
{
|
||||
var newBufferItem = new LogBufferItem();
|
||||
|
||||
newBufferItem.LastUpdate = DateTime.Now;
|
||||
foreach (var rowData in rowsData)
|
||||
{
|
||||
newBufferItem.LogRows.TryAdd(new EventKey()
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
File = logFileInfo
|
||||
}, rowData);
|
||||
}
|
||||
|
||||
return newBufferItem;
|
||||
},
|
||||
(settings, logBufferItem) =>
|
||||
{
|
||||
DateTime operationDate = DateTime.Now;
|
||||
if (logBufferItem.Created == DateTime.MinValue)
|
||||
logBufferItem.Created = operationDate;
|
||||
logBufferItem.LastUpdate = operationDate;
|
||||
foreach (var rowData in rowsData)
|
||||
{
|
||||
logBufferItem.LogRows.TryAdd(new EventKey()
|
||||
{
|
||||
Id = Guid.NewGuid(),
|
||||
File = logFileInfo
|
||||
}, rowData);
|
||||
}
|
||||
|
||||
return logBufferItem;
|
||||
});
|
||||
}
|
||||
|
||||
public TechJournalPosition GetLastPosition(
|
||||
TechJournalSettings.LogSourceSettings logSettings,
|
||||
TechJournalLogBase techJournalLog,
|
||||
string directoryName)
|
||||
{
|
||||
TechJournalPosition position = null;
|
||||
if (LogBuffer.TryGetValue(logSettings, out LogBufferItem bufferItem))
|
||||
{
|
||||
bufferItem.LogPositions.TryGetValue(directoryName, out position);
|
||||
}
|
||||
|
||||
return position;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer.EventArgs;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
|
||||
{
|
||||
public class TechJournalExport
|
||||
{
|
||||
#region Private Members
|
||||
|
||||
private readonly TechJournalSettings _settings;
|
||||
private readonly LogBuffers _logBuffers;
|
||||
private readonly ITechJournalOnTargetBuilder _techJournalTargetBuilder;
|
||||
|
||||
#endregion
|
||||
|
||||
#region Constructors
|
||||
|
||||
public TechJournalExport(TechJournalSettings settings, ITechJournalOnTargetBuilder techJournalTargetBuilder)
|
||||
{
|
||||
_settings = settings;
|
||||
_logBuffers = new LogBuffers();
|
||||
_techJournalTargetBuilder = techJournalTargetBuilder;
|
||||
|
||||
foreach (var logSourceSettings in _settings.LogSources)
|
||||
{
|
||||
_logBuffers.LogBuffer.TryAdd(logSourceSettings, new LogBufferItem());
|
||||
}
|
||||
foreach (var logBufferItem in _logBuffers.LogBuffer)
|
||||
{
|
||||
var logPositions = techJournalTargetBuilder.GetCurrentLogPositions(settings, logBufferItem);
|
||||
logBufferItem.Value.LogPositions = new ConcurrentDictionary<string, TechJournalPosition>(logPositions);
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Public Methods
|
||||
|
||||
public async Task StartExport()
|
||||
{
|
||||
await StartExport(CancellationToken.None);
|
||||
}
|
||||
|
||||
public async Task StartExport(CancellationToken cancellationToken)
|
||||
{
|
||||
List<Task> exportJobs = new List<Task>();
|
||||
foreach (var logSource in _settings.LogSources)
|
||||
{
|
||||
Task logExportTask = Task.Run(() => LogExportJob(logSource, cancellationToken), cancellationToken);
|
||||
exportJobs.Add(logExportTask);
|
||||
await Task.Delay(1000, cancellationToken);
|
||||
}
|
||||
|
||||
await Task.Delay(10000, cancellationToken);
|
||||
|
||||
exportJobs.Add(Task.Run(() => SendLogFromBuffer(cancellationToken), cancellationToken));
|
||||
|
||||
Task.WaitAll(exportJobs.ToArray(), cancellationToken);
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Private Methods
|
||||
|
||||
private async Task LogExportJob(
|
||||
TechJournalSettings.LogSourceSettings settings,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string techJournalPath = settings.SourcePath;
|
||||
TimeZoneInfo timeZone = settings.TimeZone;
|
||||
|
||||
while (true)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
break;
|
||||
bool bufferBlocked = (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxBufferSizeItemsCount);
|
||||
|
||||
try
|
||||
{
|
||||
if (!bufferBlocked)
|
||||
{
|
||||
TechJournalManager tjManager = new TechJournalManager(techJournalPath);
|
||||
foreach (var tjDirectory in tjManager.Directories)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
break;
|
||||
bufferBlocked = (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxBufferSizeItemsCount);
|
||||
if (bufferBlocked)
|
||||
break;
|
||||
if (!tjDirectory.DirectoryData.Exists)
|
||||
continue;
|
||||
|
||||
using (TechJournalExportMaster exporter = new TechJournalExportMaster())
|
||||
{
|
||||
exporter.SetTechJournalPath(tjDirectory.DirectoryData.FullName, timeZone);
|
||||
|
||||
TechJournalOnBuffer target =
|
||||
new TechJournalOnBuffer(_logBuffers, settings.Portion);
|
||||
target.SetLogSettings(settings);
|
||||
target.SetInformationSystem(new TechJournalLogBase()
|
||||
{
|
||||
Name = settings.Name,
|
||||
Description = settings.Description
|
||||
});
|
||||
exporter.SetTarget(target);
|
||||
exporter.OnErrorExportData += OnErrorExportDataToBuffer;
|
||||
|
||||
while (exporter.NewDataAvailable())
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
break;
|
||||
|
||||
exporter.SendData();
|
||||
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
break;
|
||||
bufferBlocked = (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxBufferSizeItemsCount);
|
||||
if (bufferBlocked)
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (bufferBlocked)
|
||||
{
|
||||
await Task.Delay(100, cancellationToken);
|
||||
}
|
||||
else if (_settings.WatchMode.Use)
|
||||
{
|
||||
await Task.Delay(_settings.WatchMode.Periodicity, cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
RaiseOnError(settings, new OnErrorExportSharedBufferEventArgs(e));
|
||||
await Task.Delay(60000, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SendLogFromBuffer(CancellationToken cancellationToken)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
break;
|
||||
|
||||
bool needExport = false;
|
||||
|
||||
if (_logBuffers.TotalItemsCount >= _settings.Export.Buffer.MaxItemCountSize)
|
||||
{
|
||||
needExport = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
DateTime bufferCreated = _logBuffers.Created;
|
||||
var createTimeLeftMs = (DateTime.Now - bufferCreated).TotalMilliseconds;
|
||||
if (bufferCreated != DateTime.MinValue && createTimeLeftMs >= _settings.Export.Buffer.MaxSaveDurationMs)
|
||||
{
|
||||
needExport = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (needExport)
|
||||
{
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
break;
|
||||
TechJournalSettings.LogSourceSettings lastSettings = null;
|
||||
|
||||
try
|
||||
{
|
||||
foreach (var logBufferItem in _logBuffers.LogBuffer)
|
||||
{
|
||||
lastSettings = logBufferItem.Key;
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
break;
|
||||
if (logBufferItem.Value.LogRows.Count == 0)
|
||||
continue;
|
||||
|
||||
lock (logBufferItem.Key.LockObject)
|
||||
{
|
||||
var eventsByFile = logBufferItem.Value.LogRows
|
||||
.Select(b => new
|
||||
{
|
||||
FileName = b.Key.File.FullName,
|
||||
EventData = b.Value
|
||||
})
|
||||
.GroupBy(g => g.FileName)
|
||||
.ToDictionary(
|
||||
g => g.Key,
|
||||
g => g.Select(e => e.EventData).ToList());
|
||||
|
||||
OnSend(logBufferItem.Key, new OnSendLogFromSharedBufferEventArgs(
|
||||
logBufferItem.Key, eventsByFile, logBufferItem.Value.LogPositions));
|
||||
|
||||
ITechJournalOnTarget target = _techJournalTargetBuilder.CreateTarget(_settings, logBufferItem);
|
||||
|
||||
target.Save(eventsByFile);
|
||||
eventsByFile.Clear();
|
||||
logBufferItem.Value.LogRows.Clear();
|
||||
|
||||
foreach (var logPosition in logBufferItem.Value.LogPositions)
|
||||
{
|
||||
target.SaveLogPosition(logPosition.Value);
|
||||
}
|
||||
|
||||
logBufferItem.Value.Created = DateTime.MinValue;
|
||||
logBufferItem.Value.LastUpdate = DateTime.MinValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
RaiseOnError(lastSettings, new OnErrorExportSharedBufferEventArgs(e));
|
||||
await Task.Delay(1000, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
await Task.Delay(100, cancellationToken);
|
||||
|
||||
if (!_settings.WatchMode.Use
|
||||
&& _logBuffers.TotalItemsCount == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Events
|
||||
|
||||
public delegate void OnSendLogFromSharedBufferEventArgsHandler(TechJournalSettings.LogSourceSettings settings, OnSendLogFromSharedBufferEventArgs args);
|
||||
public delegate void OnErrorExportSharedBufferEventArgsHandler(TechJournalSettings.LogSourceSettings settings, OnErrorExportSharedBufferEventArgs args);
|
||||
public event OnSendLogFromSharedBufferEventArgsHandler OnSendLogEvent;
|
||||
public event OnErrorExportSharedBufferEventArgsHandler OnErrorEvent;
|
||||
|
||||
protected void OnSend(
|
||||
TechJournalSettings.LogSourceSettings settings,
|
||||
OnSendLogFromSharedBufferEventArgs args)
|
||||
{
|
||||
OnSendLogEvent?.Invoke(settings, args);
|
||||
}
|
||||
protected void RaiseOnError(
|
||||
TechJournalSettings.LogSourceSettings settings,
|
||||
OnErrorExportSharedBufferEventArgs args)
|
||||
{
|
||||
OnErrorEvent?.Invoke(settings, args);
|
||||
}
|
||||
private void OnErrorExportDataToBuffer(OnErrorExportDataEventArgs e)
|
||||
{
|
||||
RaiseOnError(null, new OnErrorExportSharedBufferEventArgs(e.Exception));
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using EventData = YY.TechJournalReaderAssistant.Models.EventData;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core.SharedBuffer
|
||||
{
|
||||
public class TechJournalOnBuffer : TechJournalOnTarget
|
||||
{
|
||||
#region Private Member Variables
|
||||
|
||||
private const int _defaultPortion = 1000;
|
||||
private readonly int _portion;
|
||||
private TechJournalLogBase _techJournalLog;
|
||||
private TechJournalPosition _lastTechJournalFilePosition;
|
||||
private TechJournalSettings.LogSourceSettings _logSettings;
|
||||
private readonly LogBuffers _logBuffers;
|
||||
private IList<EventData> _rowsDataForPosition;
|
||||
|
||||
|
||||
#endregion
|
||||
|
||||
#region Constructor
|
||||
|
||||
public TechJournalOnBuffer() : this(null, _defaultPortion)
|
||||
{
|
||||
|
||||
}
|
||||
public TechJournalOnBuffer(int portion) : this(null, portion)
|
||||
{
|
||||
_portion = portion;
|
||||
}
|
||||
public TechJournalOnBuffer(LogBuffers buffers, int portion)
|
||||
{
|
||||
_portion = portion;
|
||||
_rowsDataForPosition = new List<EventData>();
|
||||
_logBuffers = buffers;
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
#region Public Methods
|
||||
|
||||
public void SetLogSettings(TechJournalSettings.LogSourceSettings logSettings)
|
||||
{
|
||||
_logSettings = logSettings;
|
||||
}
|
||||
public void SetLastPosition(TechJournalPosition position)
|
||||
{
|
||||
_lastTechJournalFilePosition = position;
|
||||
}
|
||||
public override TechJournalPosition GetLastPosition(string directoryName)
|
||||
{
|
||||
if (_lastTechJournalFilePosition != null)
|
||||
return _lastTechJournalFilePosition;
|
||||
|
||||
TechJournalPosition position = _logBuffers.GetLastPosition(_logSettings, _techJournalLog, directoryName);
|
||||
|
||||
_lastTechJournalFilePosition = position;
|
||||
return position;
|
||||
}
|
||||
public override void SaveLogPosition(TechJournalPosition position)
|
||||
{
|
||||
_lastTechJournalFilePosition = position;
|
||||
_logBuffers.SaveLogsAndPosition(_logSettings, position, _rowsDataForPosition);
|
||||
_rowsDataForPosition.Clear();
|
||||
}
|
||||
public override int GetPortionSize()
|
||||
{
|
||||
return _portion;
|
||||
}
|
||||
public override void Save(EventData rowData, string fileName)
|
||||
{
|
||||
Save(new List<EventData>()
|
||||
{
|
||||
rowData
|
||||
}, fileName);
|
||||
}
|
||||
|
||||
public override void Save(IList<EventData> rowsData, string fileName)
|
||||
{
|
||||
_rowsDataForPosition.Clear();
|
||||
_rowsDataForPosition = rowsData.ToList();
|
||||
}
|
||||
public override void SetInformationSystem(TechJournalLogBase techJournalLog)
|
||||
{
|
||||
_techJournalLog = techJournalLog;
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -126,7 +126,9 @@ namespace YY.TechJournalExportAssistant.Core
|
||||
|
||||
private TechJournalPosition GetFixedCurrentPosition()
|
||||
{
|
||||
TechJournalPosition lastPosition = _target.GetLastPosition();
|
||||
DirectoryInfo logDirectoryInfo = new DirectoryInfo(_techJournalLogPath);
|
||||
string directoryName = logDirectoryInfo.Name;
|
||||
TechJournalPosition lastPosition = _target.GetLastPosition(directoryName);
|
||||
|
||||
if (lastPosition != null)
|
||||
{
|
||||
@@ -167,9 +169,7 @@ namespace YY.TechJournalExportAssistant.Core
|
||||
|
||||
if (reader.CurrentFile != null)
|
||||
{
|
||||
_target.SaveLogPosition(
|
||||
currentFile,
|
||||
reader.GetCurrentPosition());
|
||||
_target.SaveLogPosition(reader.GetCurrentPosition());
|
||||
}
|
||||
_dataToSend.Clear();
|
||||
}
|
||||
@@ -217,13 +217,11 @@ namespace YY.TechJournalExportAssistant.Core
|
||||
}
|
||||
private void TechJournalReader_AfterReadFile(TechJournalReader sender, AfterReadFileEventArgs args)
|
||||
{
|
||||
FileInfo _lastEventLogDataFileInfo = new FileInfo(args.FileName);
|
||||
|
||||
if (_dataToSend.Count >= 0)
|
||||
SendDataCurrentPortion(sender);
|
||||
|
||||
TechJournalPosition position = sender.GetCurrentPosition();
|
||||
_target.SaveLogPosition(_lastEventLogDataFileInfo, position);
|
||||
_target.SaveLogPosition(position);
|
||||
}
|
||||
private void TechJournalReader_OnErrorEvent(TechJournalReader sender, OnErrorEventArgs args)
|
||||
{
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
public class TechJournalLogBase
|
||||
{
|
||||
public virtual string Name { get; set; }
|
||||
public virtual string DirectoryName { get; set; }
|
||||
public virtual string Description { get; set; }
|
||||
|
||||
public override string ToString()
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
using YY.TechJournalReaderAssistant.Models;
|
||||
|
||||
@@ -10,7 +10,7 @@ namespace YY.TechJournalExportAssistant.Core
|
||||
{
|
||||
#region Public Methods
|
||||
|
||||
public virtual TechJournalPosition GetLastPosition()
|
||||
public virtual TechJournalPosition GetLastPosition(string directoryName)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
@@ -30,7 +30,7 @@ namespace YY.TechJournalExportAssistant.Core
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public virtual void SaveLogPosition(FileInfo logFileInfo, TechJournalPosition position)
|
||||
public virtual void SaveLogPosition(TechJournalPosition position)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
@@ -39,7 +39,17 @@ namespace YY.TechJournalExportAssistant.Core
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
|
||||
public virtual void Save(IDictionary<string, List<EventData>> rowsData)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
public virtual IDictionary<string, TechJournalPosition> GetCurrentLogPositions(TechJournalSettings settings, KeyValuePair<TechJournalSettings.LogSourceSettings, LogBufferItem> logBufferItem)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Core
|
||||
{
|
||||
public class TechJournalSettings
|
||||
{
|
||||
public static TechJournalSettings Create(string configFile)
|
||||
{
|
||||
IConfiguration Configuration = new ConfigurationBuilder()
|
||||
.AddJsonFile(configFile, optional: true, reloadOnChange: true)
|
||||
.Build();
|
||||
|
||||
return Create(Configuration);
|
||||
}
|
||||
public static TechJournalSettings Create(IConfiguration configuration)
|
||||
{
|
||||
return new TechJournalSettings(configuration);
|
||||
}
|
||||
|
||||
public string ConnectionString { get; }
|
||||
public WatchModeSettings WatchMode { get; }
|
||||
public ExportSettings Export { get; }
|
||||
public IReadOnlyList<LogSourceSettings> LogSources { get; }
|
||||
|
||||
private TechJournalSettings(IConfiguration configuration)
|
||||
{
|
||||
ConnectionString = configuration.GetConnectionString("TechJournalDatabase");
|
||||
|
||||
var exportParametersSection = configuration.GetSection("Export");
|
||||
var bufferSection = exportParametersSection.GetSection("Buffer");
|
||||
var maxItemCountSize = bufferSection.GetValue("MaxItemCountSize", 10000);
|
||||
var maxSaveDurationMs = bufferSection.GetValue("MaxSaveDurationMs", 60000);
|
||||
var maxBufferSizeItemsCount = bufferSection.GetValue("MaxBufferSizeItemsCount", 100000);
|
||||
Export = new ExportSettings(
|
||||
new ExportSettings.BufferSettings(maxItemCountSize, maxSaveDurationMs, maxBufferSizeItemsCount));
|
||||
|
||||
var watchModeSection = configuration.GetSection("WatchMode");
|
||||
var useWatchMode = watchModeSection.GetValue("Use", false);
|
||||
var periodicityWatchMode = watchModeSection.GetValue("Periodicity", 60000);
|
||||
WatchMode = new WatchModeSettings(useWatchMode, periodicityWatchMode);
|
||||
|
||||
var logSourcesSection = configuration.GetSection("LogSources");
|
||||
var logSources = logSourcesSection.GetChildren();
|
||||
List<LogSourceSettings> logSourceSettings = new List<LogSourceSettings>();
|
||||
foreach (var logSource in logSources)
|
||||
{
|
||||
var techJournalLogName = logSource.GetValue("Name", string.Empty);
|
||||
var techJournalLogDescription = logSource.GetValue("Description", string.Empty);
|
||||
string sourcePath = logSource.GetValue("SourcePath", string.Empty);
|
||||
int portion = logSource.GetValue("Portion", 10000);
|
||||
string timeZoneName = logSource.GetValue("TimeZone", string.Empty);
|
||||
TimeZoneInfo timeZone;
|
||||
if (string.IsNullOrEmpty(timeZoneName))
|
||||
timeZone = TimeZoneInfo.Local;
|
||||
else
|
||||
timeZone = TimeZoneInfo.FindSystemTimeZoneById(timeZoneName);
|
||||
|
||||
logSourceSettings.Add(new LogSourceSettings(
|
||||
techJournalLogName,
|
||||
techJournalLogDescription,
|
||||
sourcePath,
|
||||
portion,
|
||||
timeZone));
|
||||
}
|
||||
LogSources = logSourceSettings;
|
||||
}
|
||||
|
||||
public class WatchModeSettings
|
||||
{
|
||||
public bool Use { get; }
|
||||
public int Periodicity { get; }
|
||||
|
||||
public WatchModeSettings(bool use, int periodicity)
|
||||
{
|
||||
Use = use;
|
||||
Periodicity = periodicity;
|
||||
}
|
||||
}
|
||||
|
||||
public class LogSourceSettings
|
||||
{
|
||||
public string Name { get; }
|
||||
public string Description { get; }
|
||||
public string SourcePath { get; }
|
||||
public int Portion { get; }
|
||||
public TimeZoneInfo TimeZone { get; }
|
||||
public object LockObject { get; }
|
||||
|
||||
public LogSourceSettings(string name, string description, string sourcePath, int portion, TimeZoneInfo timeZone)
|
||||
{
|
||||
Name = name;
|
||||
Description = description;
|
||||
Portion = portion;
|
||||
SourcePath = sourcePath;
|
||||
TimeZone = timeZone;
|
||||
LockObject = new object();
|
||||
}
|
||||
}
|
||||
|
||||
public class ExportSettings
|
||||
{
|
||||
public BufferSettings Buffer { get; }
|
||||
|
||||
public ExportSettings(BufferSettings bufferSettings)
|
||||
{
|
||||
Buffer = bufferSettings;
|
||||
}
|
||||
|
||||
public class BufferSettings
|
||||
{
|
||||
public long MaxItemCountSize { get; }
|
||||
public long MaxSaveDurationMs { get; }
|
||||
public long MaxBufferSizeItemsCount { get; }
|
||||
|
||||
public BufferSettings(long maxItemCountSize, long maxSaveDurationMs, long maxBufferSizeItemsCount)
|
||||
{
|
||||
MaxItemCountSize = maxItemCountSize;
|
||||
MaxSaveDurationMs = maxSaveDurationMs;
|
||||
MaxBufferSizeItemsCount = maxBufferSizeItemsCount;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
<PropertyGroup>
|
||||
<TargetFramework>netstandard2.1</TargetFramework>
|
||||
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
|
||||
<Version>1.0.0.8</Version>
|
||||
<Version>1.0.1.1</Version>
|
||||
<Authors>Permitin Yuriy</Authors>
|
||||
<Product>Technological journal's export assistant. Core library</Product>
|
||||
<Description>Core library for exporting 1C:Enterprise 8.x platform's technological journal files to different storage</Description>
|
||||
@@ -15,6 +15,9 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
|
||||
<PackageReference Include="YY.TechJournalReaderAssistant" Version="1.0.0.19" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
+88
-7
@@ -1,25 +1,32 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using ClickHouse.Client.ADO;
|
||||
using Newtonsoft.Json;
|
||||
using Xunit;
|
||||
using YY.TechJournalExportAssistant.ClickHouse;
|
||||
using YY.TechJournalExportAssistant.ClickHouse.Helpers;
|
||||
using YY.TechJournalExportAssistant.Core;
|
||||
using YY.TechJournalExportAssistant.Core.SharedBuffer;
|
||||
using YY.TechJournalReaderAssistant;
|
||||
|
||||
namespace YY.TechJournalExportAssistant.Tests
|
||||
{
|
||||
public class TechJournalExportAssistantToClickHouseTests
|
||||
{
|
||||
private readonly string _configFilePath;
|
||||
private readonly string _logDataPath;
|
||||
private readonly IConfiguration Configuration;
|
||||
|
||||
public TechJournalExportAssistantToClickHouseTests()
|
||||
{
|
||||
_configFilePath = GetConfigFile();
|
||||
var configFilePath = GetConfigFile();
|
||||
Configuration = new ConfigurationBuilder()
|
||||
.AddJsonFile(configFilePath, optional: true, reloadOnChange: true)
|
||||
.Build();
|
||||
|
||||
string unitTestDirectory = Directory.GetCurrentDirectory();
|
||||
|
||||
@@ -32,10 +39,6 @@ namespace YY.TechJournalExportAssistant.Tests
|
||||
[Fact]
|
||||
public void ExportDataTest()
|
||||
{
|
||||
IConfiguration Configuration = new ConfigurationBuilder()
|
||||
.AddJsonFile(_configFilePath, optional: true, reloadOnChange: true)
|
||||
.Build();
|
||||
|
||||
string connectionString = Configuration.GetConnectionString("TechJournalDatabase");
|
||||
|
||||
IConfigurationSection techJournalSection = Configuration.GetSection("TechJournal");
|
||||
@@ -83,7 +86,6 @@ namespace YY.TechJournalExportAssistant.Tests
|
||||
target.SetInformationSystem(new TechJournalLogBase()
|
||||
{
|
||||
Name = techJournalName,
|
||||
DirectoryName = tjDirectory.DirectoryData.Name,
|
||||
Description = techJournalDescription
|
||||
});
|
||||
exporter.SetTarget(target);
|
||||
@@ -124,6 +126,85 @@ namespace YY.TechJournalExportAssistant.Tests
|
||||
Assert.Equal(eventNumberReader, eventNumberFromClickHouse);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async void ExportDataWithSharedBufferTest()
|
||||
{
|
||||
string connectionString = Configuration.GetConnectionString("TechJournalDatabase");
|
||||
|
||||
IConfigurationSection techJournalSection = Configuration.GetSection("TechJournal");
|
||||
int watchPeriodSeconds = techJournalSection.GetValue("WatchPeriod", 60);
|
||||
int watchPeriodSecondsMs = watchPeriodSeconds * 1000;
|
||||
bool useWatchMode = techJournalSection.GetValue("UseWatchMode", false);
|
||||
int portion = techJournalSection.GetValue("Portion", 1000);
|
||||
string timeZoneName = techJournalSection.GetValue("TimeZone", string.Empty);
|
||||
|
||||
IConfigurationSection techJournalLogSection = Configuration.GetSection("TechJournalLog");
|
||||
string techJournalName = techJournalLogSection.GetValue("Name", string.Empty);
|
||||
string techJournalDescription = techJournalLogSection.GetValue("Description", string.Empty);
|
||||
|
||||
var LogSources = new List<dynamic>();
|
||||
for (int i = 1; i <= 3; i++)
|
||||
{
|
||||
LogSources.Add(new
|
||||
{
|
||||
Name = $"{techJournalName} {i}",
|
||||
Description = $"{techJournalDescription} {i}",
|
||||
SourcePath = _logDataPath,
|
||||
Portion = portion,
|
||||
TimeZone = timeZoneName
|
||||
});
|
||||
}
|
||||
|
||||
var configurationObject = new
|
||||
{
|
||||
ConnectionStrings = new
|
||||
{
|
||||
TechJournalDatabase = connectionString.Replace("Database=", "Database=SharedBuffer")
|
||||
},
|
||||
WatchMode = new
|
||||
{
|
||||
Use = useWatchMode,
|
||||
Periodicity = watchPeriodSecondsMs
|
||||
},
|
||||
Export = new
|
||||
{
|
||||
Buffer = new
|
||||
{
|
||||
MaxItemCountSize = 10,
|
||||
MaxSaveDurationMs = 5000,
|
||||
MaxBufferSizeItemsCount = 100
|
||||
}
|
||||
},
|
||||
LogSources
|
||||
};
|
||||
|
||||
var configurationAsJson = JsonConvert.SerializeObject(configurationObject);
|
||||
IConfiguration configurationForTest = new ConfigurationBuilder().AddJsonStream(new MemoryStream(Encoding.UTF8.GetBytes(configurationAsJson))).Build();
|
||||
TechJournalSettings settings = TechJournalSettings.Create(configurationForTest);
|
||||
|
||||
ClickHouseHelpers.DropDatabaseIfExist(settings.ConnectionString);
|
||||
|
||||
TechJournalExport exportMaster = new TechJournalExport(settings, new TechJournalOnClickHouseTargetBuilder());
|
||||
await exportMaster.StartExport();
|
||||
|
||||
int eventNumberFromClickHouse = 0;
|
||||
using (ClickHouseConnection cn = new ClickHouseConnection(settings.ConnectionString))
|
||||
{
|
||||
using (ClickHouseCommand cmd = new ClickHouseCommand(cn))
|
||||
{
|
||||
cmd.CommandText =
|
||||
@"SELECT
|
||||
COUNT(*)
|
||||
FROM EventData ed";
|
||||
var reader = await cmd.ExecuteReaderAsync();
|
||||
if (await reader.ReadAsync())
|
||||
eventNumberFromClickHouse = Convert.ToInt32(reader.GetValue(0));
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(9339, eventNumberFromClickHouse);
|
||||
}
|
||||
|
||||
private string GetConfigFile()
|
||||
{
|
||||
// TODO
|
||||
|
||||
@@ -38,6 +38,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Scripts", "Scripts", "{7E5B
|
||||
Scripts\install-clickhouse.sh = Scripts\install-clickhouse.sh
|
||||
EndProjectSection
|
||||
EndProject
|
||||
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "YY.TechJournalExportAssistantWithSharedBufferConsoleApp", "Apps\YY.TechJournalExportAssistantWithSharedBufferConsoleApp\YY.TechJournalExportAssistantWithSharedBufferConsoleApp.csproj", "{75DDC845-838D-4F33-914B-2D833179A463}"
|
||||
EndProject
|
||||
Global
|
||||
GlobalSection(SolutionConfigurationPlatforms) = preSolution
|
||||
Debug|Any CPU = Debug|Any CPU
|
||||
@@ -60,6 +62,10 @@ Global
|
||||
{62CDAE77-D687-423E-B88C-25967ECC2353}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{62CDAE77-D687-423E-B88C-25967ECC2353}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{62CDAE77-D687-423E-B88C-25967ECC2353}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
{75DDC845-838D-4F33-914B-2D833179A463}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
|
||||
{75DDC845-838D-4F33-914B-2D833179A463}.Debug|Any CPU.Build.0 = Debug|Any CPU
|
||||
{75DDC845-838D-4F33-914B-2D833179A463}.Release|Any CPU.ActiveCfg = Release|Any CPU
|
||||
{75DDC845-838D-4F33-914B-2D833179A463}.Release|Any CPU.Build.0 = Release|Any CPU
|
||||
EndGlobalSection
|
||||
GlobalSection(SolutionProperties) = preSolution
|
||||
HideSolutionNode = FALSE
|
||||
@@ -71,6 +77,7 @@ Global
|
||||
{62CDAE77-D687-423E-B88C-25967ECC2353} = {274A8CD4-5F80-4164-B0F6-1F2ACBB8E00B}
|
||||
{56E62BAA-09E0-4119-AA48-0F3B6E80C1F7} = {300AD5BD-47CB-4AA8-B823-9B061C7654D1}
|
||||
{7E5B0AC0-4A78-4BC2-9C54-E461317125AB} = {300AD5BD-47CB-4AA8-B823-9B061C7654D1}
|
||||
{75DDC845-838D-4F33-914B-2D833179A463} = {81F37230-D7FE-4190-A939-B5AE8A71494B}
|
||||
EndGlobalSection
|
||||
GlobalSection(ExtensibilityGlobals) = postSolution
|
||||
SolutionGuid = {B3DABD43-6E3B-4871-B426-2C4BDD6018CC}
|
||||
|
||||
Reference in New Issue
Block a user