From d8cc243537a093d31b4da07043865b3ae81474ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BA=D0=BF=D0=B0=D0=B5=D0=B2=20=D0=95=D0=B2=D0=B3?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B0?= =?UTF-8?q?=D0=BD=D0=B4=D1=80=D0=BE=D0=B2=D0=B8=D1=87?= Date: Tue, 10 Nov 2020 14:46:04 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9F=D0=BE=D0=BB=D0=BD=D0=B0=D1=8F=20=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D1=80=D0=B0=D0=B1=D0=BE=D0=BA=D1=82=D0=B0=20?= =?UTF-8?q?=D0=B1=D0=B8=D0=B1=D0=BB=D0=B8=D0=BE=D1=82=D0=B5=D0=BA=D0=B8,?= =?UTF-8?q?=20=D1=83=D0=BB=D1=83=D1=87=D1=88=D0=B5=D0=BD=D0=B8=D0=B5=20?= =?UTF-8?q?=D0=BF=D1=80=D0=BE=D0=B8=D0=B7=D0=B2=D0=BE=D0=B4=D0=B8=D1=82?= =?UTF-8?q?=D0=B5=D0=BB=D1=8C=D0=BD=D0=BE=D1=81=D1=82=D0=B8,=20=D0=B4?= =?UTF-8?q?=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BF?= =?UTF-8?q?=D1=80=D0=BE=D0=B5=D0=BA=D1=82=D0=BE=D0=B2=20=D0=B4=D0=BB=D1=8F?= =?UTF-8?q?=20=D1=8D=D0=BA=D1=81=D0=BF=D0=BE=D1=80=D1=82=D0=B0=20=D0=A2?= =?UTF-8?q?=D0=96=20=D0=B2=20=D1=81=D1=82=D0=BE=D1=80=D0=BE=D0=BD=D0=BD?= =?UTF-8?q?=D0=B8=D0=B5=20=D0=91=D0=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ITechLogFolderReader.cs | 11 ++ .../ITechLogStorage.cs | 9 + .../OneSTools.TechLog.Exporter.Core.csproj | 18 ++ .../TechLogExporterService.cs | 41 ++++ .../TechLogFolderReader.cs | 121 ++++++++++++ ...ools.TechLog.Exporter.ElasticSearch.csproj | 15 ++ .../Program.cs | 41 ++++ .../TechLogStorage.cs | 77 ++++++++ .../appsettings.Development.json | 9 + .../appsettings.json | 18 ++ OneSTools.TechLog.sln | 16 +- OneSTools.TechLog/OneSTools.TechLog.csproj | 20 +- OneSTools.TechLog/TechLogItem.cs | 15 ++ OneSTools.TechLog/TechLogParser.cs | 164 ---------------- OneSTools.TechLog/TechLogReader.cs | 183 ++++++++++++++++++ .../OneSTools.TechLogConsoleApp.csproj | 12 -- OneSTools.TechLogConsoleApp/Program.cs | 40 ---- 17 files changed, 582 insertions(+), 228 deletions(-) create mode 100644 OneSTools.TechLog.Exporter.Core/ITechLogFolderReader.cs create mode 100644 OneSTools.TechLog.Exporter.Core/ITechLogStorage.cs create mode 100644 OneSTools.TechLog.Exporter.Core/OneSTools.TechLog.Exporter.Core.csproj create mode 100644 OneSTools.TechLog.Exporter.Core/TechLogExporterService.cs create mode 100644 OneSTools.TechLog.Exporter.Core/TechLogFolderReader.cs create mode 100644 OneSTools.TechLog.Exporter.ElasticSearch/OneSTools.TechLog.Exporter.ElasticSearch.csproj create mode 100644 OneSTools.TechLog.Exporter.ElasticSearch/Program.cs create mode 100644 OneSTools.TechLog.Exporter.ElasticSearch/TechLogStorage.cs create mode 100644 OneSTools.TechLog.Exporter.ElasticSearch/appsettings.Development.json create mode 100644 OneSTools.TechLog.Exporter.ElasticSearch/appsettings.json create mode 100644 OneSTools.TechLog/TechLogItem.cs delete mode 100644 OneSTools.TechLog/TechLogParser.cs create mode 100644 OneSTools.TechLog/TechLogReader.cs delete mode 100644 OneSTools.TechLogConsoleApp/OneSTools.TechLogConsoleApp.csproj delete mode 100644 OneSTools.TechLogConsoleApp/Program.cs diff --git a/OneSTools.TechLog.Exporter.Core/ITechLogFolderReader.cs b/OneSTools.TechLog.Exporter.Core/ITechLogFolderReader.cs new file mode 100644 index 0000000..9473b7d --- /dev/null +++ b/OneSTools.TechLog.Exporter.Core/ITechLogFolderReader.cs @@ -0,0 +1,11 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace OneSTools.TechLog.Exporter.Core +{ + public interface ITechLogFolderReader : IDisposable + { + Task StartAsync(string logFolder, int portion, bool liveMode = false, CancellationToken cancellationToken = default); + } +} \ No newline at end of file diff --git a/OneSTools.TechLog.Exporter.Core/ITechLogStorage.cs b/OneSTools.TechLog.Exporter.Core/ITechLogStorage.cs new file mode 100644 index 0000000..f0ad3f2 --- /dev/null +++ b/OneSTools.TechLog.Exporter.Core/ITechLogStorage.cs @@ -0,0 +1,9 @@ +using System.Threading.Tasks; + +namespace OneSTools.TechLog.Exporter.Core +{ + public interface ITechLogStorage + { + Task WriteItemsAsync(TechLogItem[] items); + } +} diff --git a/OneSTools.TechLog.Exporter.Core/OneSTools.TechLog.Exporter.Core.csproj b/OneSTools.TechLog.Exporter.Core/OneSTools.TechLog.Exporter.Core.csproj new file mode 100644 index 0000000..3afa6a2 --- /dev/null +++ b/OneSTools.TechLog.Exporter.Core/OneSTools.TechLog.Exporter.Core.csproj @@ -0,0 +1,18 @@ + + + + netstandard2.1 + + + + + + + + + + + + + + diff --git a/OneSTools.TechLog.Exporter.Core/TechLogExporterService.cs b/OneSTools.TechLog.Exporter.Core/TechLogExporterService.cs new file mode 100644 index 0000000..63a6119 --- /dev/null +++ b/OneSTools.TechLog.Exporter.Core/TechLogExporterService.cs @@ -0,0 +1,41 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace OneSTools.TechLog.Exporter.Core +{ + public class TechLogExporterService : BackgroundService + { + private IConfiguration _configuration; + private readonly ILogger _logger; + private readonly ITechLogFolderReader _techLogFolderReader; + private string _logFolder; + private int _portion; + + public TechLogExporterService(IConfiguration configuration, ILogger logger, ITechLogFolderReader techLogFolderReader) + { + _configuration = configuration; + _logger = logger; + _techLogFolderReader = techLogFolderReader; + + _logFolder = configuration.GetValue("Exporter:LogFolder", ""); + + if (_logFolder == "") + throw new Exception("Log folder's path is not set"); + + _portion = configuration.GetValue("Exporter", 10000); + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await _techLogFolderReader.StartAsync(_logFolder, _portion, true, stoppingToken); + } + } +} diff --git a/OneSTools.TechLog.Exporter.Core/TechLogFolderReader.cs b/OneSTools.TechLog.Exporter.Core/TechLogFolderReader.cs new file mode 100644 index 0000000..f865f76 --- /dev/null +++ b/OneSTools.TechLog.Exporter.Core/TechLogFolderReader.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using Microsoft.Extensions.Logging; +using OneSTools.TechLog; + +namespace OneSTools.TechLog.Exporter.Core +{ + public class TechLogFolderReader : IDisposable, ITechLogFolderReader + { + private readonly ILogger _logger; + private ITechLogStorage _techLogStorage; + private string _folder; + private bool _liveMode; + private ActionBlock _writeBlock; + private BatchBlock _batchBlock; + private TransformBlock _parseBlock; + private ActionBlock _readBlock; + + public TechLogFolderReader(ILogger logger, ITechLogStorage techLogStorage) + { + _logger = logger; + _techLogStorage = techLogStorage; + } + + public async Task StartAsync(string folder, int portion, bool liveMode = false, CancellationToken cancellationToken = default) + { + _folder = folder; + _liveMode = liveMode; + + var maxDegree = Environment.ProcessorCount * 10; + + _writeBlock = new ActionBlock(_techLogStorage.WriteItemsAsync, new ExecutionDataflowBlockOptions() + { + BoundedCapacity = 3, + CancellationToken = cancellationToken, + MaxDegreeOfParallelism = maxDegree, + }); + _batchBlock = new BatchBlock(portion, new GroupingDataflowBlockOptions() + { + CancellationToken = cancellationToken, + BoundedCapacity = portion + }); + _parseBlock = new TransformBlock(ParseItemData, new ExecutionDataflowBlockOptions() + { + MaxDegreeOfParallelism = maxDegree, + BoundedCapacity = portion / 2, + CancellationToken = cancellationToken + }); + _readBlock = new ActionBlock(str => ReadItemsData(str, _parseBlock), new ExecutionDataflowBlockOptions() + { + MaxDegreeOfParallelism = maxDegree, + CancellationToken = cancellationToken + }); ; + + _parseBlock.LinkTo(_batchBlock); + _batchBlock.LinkTo(_writeBlock); + + var logFiles = GetLogFiles(); + + foreach (var logFile in logFiles) + { + await _readBlock.SendAsync(logFile); + } + + await _writeBlock.Completion; + } + + private string[] GetLogFiles() + { + return Directory.GetFiles(_folder, "*.log", SearchOption.AllDirectories); + } + + private void ReadItemsData(string logPath, ITargetBlock nextblock) + { + var fileName = Path.GetFileNameWithoutExtension(logPath); + + var fileDateTime = "20" + + fileName.Substring(0, 2) + + "-" + + fileName.Substring(2, 2) + + "-" + + fileName.Substring(4, 2) + + " " + + fileName.Substring(6, 2); + + using var reader = new TechLogReader(logPath); + + while (true) + { + var itemData = reader.ReadItemData(); + + if (itemData != null) + PostData(nextblock, fileDateTime + ":" + itemData); + } + } + + private void PostData(ITargetBlock nextblock, T data) + { + while (true) + { + if (nextblock.Post(data)) + break; + } + } + + private TechLogItem ParseItemData(string itemData) + { + return TechLogReader.ParseItemData(itemData); + } + + public void Dispose() + { + + } + } +} diff --git a/OneSTools.TechLog.Exporter.ElasticSearch/OneSTools.TechLog.Exporter.ElasticSearch.csproj b/OneSTools.TechLog.Exporter.ElasticSearch/OneSTools.TechLog.Exporter.ElasticSearch.csproj new file mode 100644 index 0000000..870a59c --- /dev/null +++ b/OneSTools.TechLog.Exporter.ElasticSearch/OneSTools.TechLog.Exporter.ElasticSearch.csproj @@ -0,0 +1,15 @@ + + + + netcoreapp3.1 + dotnet-OneSTools.TechLog.Exporter.ElasticSearch-146D8E25-B976-4FAB-A231-A91F8DD2EEF5 + + + + + + + + + + diff --git a/OneSTools.TechLog.Exporter.ElasticSearch/Program.cs b/OneSTools.TechLog.Exporter.ElasticSearch/Program.cs new file mode 100644 index 0000000..92c1b01 --- /dev/null +++ b/OneSTools.TechLog.Exporter.ElasticSearch/Program.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OneSTools.TechLog.Exporter.Core; + +namespace OneSTools.TechLog.Exporter.ElasticSearch +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureServices((hostContext, services) => + { + var configuration = hostContext.Configuration; + var host = configuration.GetValue("ElasticSearch:Host", ""); + var port = configuration.GetValue("ElasticSearch:Port", 9200); + var index = configuration.GetValue("ElasticSearch:Index", ""); + var separation = configuration.GetValue("ElasticSearch:Separation", ""); + + services.AddSingleton(sp => + { + var logger = sp.GetService>(); + + return new TechLogStorage(logger, host, port, index, separation); + }); + services.AddSingleton(); + services.AddHostedService(); + }); + } +} diff --git a/OneSTools.TechLog.Exporter.ElasticSearch/TechLogStorage.cs b/OneSTools.TechLog.Exporter.ElasticSearch/TechLogStorage.cs new file mode 100644 index 0000000..d5bb379 --- /dev/null +++ b/OneSTools.TechLog.Exporter.ElasticSearch/TechLogStorage.cs @@ -0,0 +1,77 @@ +using Microsoft.Extensions.Logging; +using Nest; +using OneSTools.TechLog.Exporter.Core; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace OneSTools.TechLog.Exporter.ElasticSearch +{ + public class TechLogStorage : ITechLogStorage + { + private readonly ILogger _logger; + private string _index; + private string _separation; + ElasticClient _client; + + public TechLogStorage(ILogger logger, string host, int port = 9200, string index = "", string separation = "") + { + _logger = logger; + + var uri = new Uri($"{host}:{port}"); + _index = $"{index}-tl"; + + var settings = new ConnectionSettings(uri); + settings.DefaultIndex(_index); + + _separation = separation; + + _client = new ElasticClient(settings); + var response = _client.Ping(); + + if (!response.IsValid) + throw response.OriginalException; + } + + public async Task WriteItemsAsync(TechLogItem[] items) + { + var data = new List<(string IndexName, TechLogItem[] Items)>(); + + switch (_separation) + { + case "H": + var groups = items.GroupBy(c => c.DateTime.ToString("yyyyMMddhh")).OrderBy(c => c.Key); + foreach (IGrouping item in groups) + data.Add(($"{_index}-{item.Key}", item.ToArray())); + break; + case "D": + groups = items.GroupBy(c => c.DateTime.ToString("yyyyMMdd")).OrderBy(c => c.Key); + foreach (IGrouping item in groups) + data.Add(($"{_index}-{item.Key}", item.ToArray())); + break; + case "M": + groups = items.GroupBy(c => c.DateTime.ToString("yyyyMM")).OrderBy(c => c.Key); + foreach (IGrouping item in groups) + data.Add(($"{_index}-{item.Key}", item.ToArray())); + break; + default: + data.Add(($"{_index}-all", items)); + break; + } + + foreach ((string IndexName, TechLogItem[] Entities) item in data) + { + var responseItems = await _client.IndexManyAsync(item.Entities, item.IndexName); + + if (!responseItems.IsValid) + { + throw responseItems.OriginalException; + } + + _logger.LogInformation($"{DateTime.Now:hh:mm:ss:fffff} has written {item.Entities.Length}"); + } + } + } +} diff --git a/OneSTools.TechLog.Exporter.ElasticSearch/appsettings.Development.json b/OneSTools.TechLog.Exporter.ElasticSearch/appsettings.Development.json new file mode 100644 index 0000000..8983e0f --- /dev/null +++ b/OneSTools.TechLog.Exporter.ElasticSearch/appsettings.Development.json @@ -0,0 +1,9 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + } +} diff --git a/OneSTools.TechLog.Exporter.ElasticSearch/appsettings.json b/OneSTools.TechLog.Exporter.ElasticSearch/appsettings.json new file mode 100644 index 0000000..a4debb6 --- /dev/null +++ b/OneSTools.TechLog.Exporter.ElasticSearch/appsettings.json @@ -0,0 +1,18 @@ +{ + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft": "Warning", + "Microsoft.Hosting.Lifetime": "Information" + } + }, + "Exporter": { + "LogFolder": "C:\\Users\\akpaev.e.ENTERPRISE\\Desktop\\TechLog" + }, + "ElasticSearch": { + "Host": "http://192.168.0.95", + "Port": 9200, + "Index": "upp-main", + "Separation": "H" // H - hour, D - day, M - Month + } +} diff --git a/OneSTools.TechLog.sln b/OneSTools.TechLog.sln index 897edb4..0a3ae22 100644 --- a/OneSTools.TechLog.sln +++ b/OneSTools.TechLog.sln @@ -5,7 +5,9 @@ VisualStudioVersion = 16.0.29318.209 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneSTools.TechLog", "OneSTools.TechLog\OneSTools.TechLog.csproj", "{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneSTools.TechLogConsoleApp", "OneSTools.TechLogConsoleApp\OneSTools.TechLogConsoleApp.csproj", "{8C9CA150-5813-4B2B-99B4-B471708DFE8C}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneSTools.TechLog.Exporter.ElasticSearch", "OneSTools.TechLog.Exporter.ElasticSearch\OneSTools.TechLog.Exporter.ElasticSearch.csproj", "{6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneSTools.TechLog.Exporter.Core", "OneSTools.TechLog.Exporter.Core\OneSTools.TechLog.Exporter.Core.csproj", "{02C37C84-911F-4725-A1EC-B81FF8F16227}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -17,10 +19,14 @@ Global {BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Debug|Any CPU.Build.0 = Debug|Any CPU {BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Release|Any CPU.ActiveCfg = Release|Any CPU {BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Release|Any CPU.Build.0 = Release|Any CPU - {8C9CA150-5813-4B2B-99B4-B471708DFE8C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {8C9CA150-5813-4B2B-99B4-B471708DFE8C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {8C9CA150-5813-4B2B-99B4-B471708DFE8C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {8C9CA150-5813-4B2B-99B4-B471708DFE8C}.Release|Any CPU.Build.0 = Release|Any CPU + {6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Release|Any CPU.Build.0 = Release|Any CPU + {02C37C84-911F-4725-A1EC-B81FF8F16227}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {02C37C84-911F-4725-A1EC-B81FF8F16227}.Debug|Any CPU.Build.0 = Debug|Any CPU + {02C37C84-911F-4725-A1EC-B81FF8F16227}.Release|Any CPU.ActiveCfg = Release|Any CPU + {02C37C84-911F-4725-A1EC-B81FF8F16227}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/OneSTools.TechLog/OneSTools.TechLog.csproj b/OneSTools.TechLog/OneSTools.TechLog.csproj index ea9d07c..05aab79 100644 --- a/OneSTools.TechLog/OneSTools.TechLog.csproj +++ b/OneSTools.TechLog/OneSTools.TechLog.csproj @@ -1,15 +1,18 @@ - netstandard2.0 + netstandard2.1 true - Akpaev Evgeniy - Akpaev Evgeniy + Akpaev Evgeny + Akpaev Evgeny https://github.com/akpaevj/OneSTools.TechLog - - Akpaev Evgeniy + https://github.com/akpaevj/OneSTools.TechLog + Akpaev Evgeny Библиотека для парсинга технологического журнала 1С - 1.0.4 + 2.0.0 + 8.0 + onestools_icon_nuget.png + @@ -19,7 +22,10 @@ - + + True + + diff --git a/OneSTools.TechLog/TechLogItem.cs b/OneSTools.TechLog/TechLogItem.cs new file mode 100644 index 0000000..2a7ddb6 --- /dev/null +++ b/OneSTools.TechLog/TechLogItem.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; + +namespace OneSTools.TechLog +{ + public class TechLogItem + { + public DateTime DateTime { get; set; } + public long Duration { get; set; } + public string Event { get; set; } + public int Level { get; set; } + + public Dictionary Properties { get; set; } = new Dictionary(); + } +} diff --git a/OneSTools.TechLog/TechLogParser.cs b/OneSTools.TechLog/TechLogParser.cs deleted file mode 100644 index 55a2a7b..0000000 --- a/OneSTools.TechLog/TechLogParser.cs +++ /dev/null @@ -1,164 +0,0 @@ -using System; -using System.IO; -using System.Text; -using System.Collections; -using System.Collections.Generic; -using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; -using System.Text.RegularExpressions; - -namespace OneSTools.TechLog -{ - /// - /// Represents methods for the parsing of the 1C technological log - /// - public class TechLogParser - { - private string folder; - private Action> eventHandler; - private ExecutionDataflowBlockOptions readBlockOptions; - private ExecutionDataflowBlockOptions parseBlockOptions; - private ExecutionDataflowBlockOptions eventHandlerBlockOptions; - - /// - /// Folder of the technological log data - /// - public string Folder { get => folder; private set => folder = value; } - /// - /// Action for the event handling - /// - public Action> EventHandler { get => eventHandler; set => eventHandler = value; } - - /// - /// Creates a new instance of TechLogParser class - /// - /// - /// - public TechLogParser(string folder, Action> eventHandler) - { - Folder = folder; - EventHandler = eventHandler; - - readBlockOptions = new ExecutionDataflowBlockOptions - { - MaxDegreeOfParallelism = Environment.ProcessorCount - }; - - parseBlockOptions = new ExecutionDataflowBlockOptions - { - MaxDegreeOfParallelism = Environment.ProcessorCount, - BoundedCapacity = 10000 - }; - - eventHandlerBlockOptions = new ExecutionDataflowBlockOptions - { - MaxDegreeOfParallelism = Environment.ProcessorCount, - BoundedCapacity = 10000 - }; - } - - /// - /// Starts parsing of the technological log data - /// - /// - public async Task Parse() - { - var eventHandlerBlock = new ActionBlock>(EventHandler, eventHandlerBlockOptions); - var parseEventBlock = new TransformBlock>(ParseEventData, parseBlockOptions); - var readFileBlock = new ActionBlock((filePath) => ReadFile(filePath, parseEventBlock), readBlockOptions); - - parseEventBlock.LinkTo(eventHandlerBlock); - - var files = GetTechLogFiles(); - - foreach (var filePath in files) - { - SendDataToNextBlock(filePath, readFileBlock); - } - - var readBlockTask = readFileBlock.Completion.ContinueWith(c => parseEventBlock.Complete()); - var parseEventBlockTask = parseEventBlock.Completion.ContinueWith(c => eventHandlerBlock.Complete()); - - readFileBlock.Complete(); - - await Task.WhenAll(readBlockTask, parseEventBlockTask, eventHandlerBlock.Completion); - } - - private void ReadFile(string filePath, ITargetBlock nextBlock) - { - using (var stream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) - using (var reader = new StreamReader(stream)) - { - string fileDateTime = GetFileDateTime(filePath); - - StringBuilder currentEvent = new StringBuilder(); - bool firstEvent = true; - - do - { - var currentLine = reader.ReadLine(); - - if (Regex.IsMatch(currentLine, @"^\d\d:\d\d\.", RegexOptions.Compiled)) - { - if (firstEvent) - { - firstEvent = false; - } - else - { - SendDataToNextBlock(fileDateTime + ":" + currentEvent.ToString(), nextBlock); - - currentEvent.Clear(); - } - - currentEvent.AppendLine(currentLine); - } - else - { - currentEvent.AppendLine(currentLine); - } - } - while (!reader.EndOfStream); - - SendDataToNextBlock(fileDateTime + ":" + currentEvent.ToString(), nextBlock); - } - } - private Dictionary ParseEventData(string eventData) - { - var properties = new Dictionary - { - ["EventName"] = Regex.Match(eventData, @",.*?,", RegexOptions.Compiled).ToString().Trim(','), - ["DateTime"] = Regex.Match(eventData, @"^.*?\.\d+", RegexOptions.Compiled).ToString(), - ["Duration"] = Regex.Match(eventData, @"-\d+?,", RegexOptions.Compiled).ToString().Trim('-', ',') - }; - - var props = Regex.Matches(eventData, @",[\w:]+=.*?(?=(,[\w:]+=|$))", RegexOptions.ExplicitCapture | RegexOptions.Singleline | RegexOptions.Compiled); - - for (int x = 0; x < props.Count; x++) - { - var propText = props[x].ToString(); - var splInd = propText.IndexOf('='); - var propName = propText.Substring(0, splInd).Trim(','); - var propVal = propText.Substring(splInd + 1).Trim('\'', '"'); - - properties[propName] = propVal; - } - - return properties; - } - private void SendDataToNextBlock(T data, ITargetBlock nextBlock) - { - while (!nextBlock.Post(data)) ; - } - private string[] GetTechLogFiles() - { - return Directory.GetFiles(Folder, "*.log"); - } - private string GetFileDateTime(string filePath) - { - var info = Path.GetFileNameWithoutExtension(filePath); - - return "20" + info.Substring(0, 2) + "-" + info.Substring(2, 2) + "-" + info.Substring(4, 2) + " " + info.Substring(6, 2); - } - } -} diff --git a/OneSTools.TechLog/TechLogReader.cs b/OneSTools.TechLog/TechLogReader.cs new file mode 100644 index 0000000..456883f --- /dev/null +++ b/OneSTools.TechLog/TechLogReader.cs @@ -0,0 +1,183 @@ +using System; +using System.Data; +using System.IO; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +namespace OneSTools.TechLog +{ + public class TechLogReader : IDisposable + { + private string _logPath; + private string _fileName; + private FileStream _fileStream; + private StreamReader _streamReader; + private StringBuilder _currentData = new StringBuilder(); + + public TechLogReader(string logPath) + { + _logPath = logPath; + _fileName = Path.GetFileNameWithoutExtension(_logPath); + } + + public TechLogItem ReadNextItem(CancellationToken cancellationToken = default) + { + InitializeStream(); + + var itemData = ReadItemData(cancellationToken); + + if (itemData == null) + return null; + + return ParseItemData(itemData, cancellationToken); + } + + public string ReadItemData(CancellationToken cancellationToken = default) + { + InitializeStream(); + + var currentLine = ""; + + while (!cancellationToken.IsCancellationRequested) + { + currentLine = _streamReader.ReadLine(); + + if (currentLine == null) + { + if (_currentData.Length > 0) + break; + else + return null; + } + + if (currentLine == null || _currentData.Length > 0 && Regex.IsMatch(currentLine, @"^\d\d:\d\d\.", RegexOptions.Compiled)) + break; + else + _currentData.AppendLine(currentLine); + } + + var _strData = _currentData.ToString().Trim(); + + _currentData.Clear(); + + if (currentLine != null) + _currentData.AppendLine(currentLine); + + return _strData; + } + + public static TechLogItem ParseItemData(string itemData, CancellationToken cancellationToken = default) + { + var data = new TechLogItem(); + + int startPosition = 0; + + var dtd = ReadNextPropertyWithoutName(itemData, ref startPosition, ','); + var dtdLength = dtd.Length; + var dtEndIndex = dtd.LastIndexOf('-'); + data.DateTime = DateTime.Parse(dtd.Substring(0, dtEndIndex)); + startPosition -= dtdLength - dtEndIndex; + + data.Duration = long.Parse(ReadNextPropertyWithoutName(itemData, ref startPosition, ',')); + data.Event = ReadNextPropertyWithoutName(itemData, ref startPosition, ','); + data.Level = int.Parse(ReadNextPropertyWithoutName(itemData, ref startPosition, ',')); + + while (!cancellationToken.IsCancellationRequested) + { + var (Name, Value) = ReadNextProperty(itemData, ref startPosition); + + if (data.Properties.ContainsKey(Name)) + { + data.Properties.Add(GetPropertyName(data, Name, 0), Value); + } + else + data.Properties.Add(Name, Value); + + if (startPosition >= itemData.Length) + break; + } + + return data; + } + + private static string GetPropertyName(TechLogItem item, string name, int number = 0) + { + var currentName = $"{name}{number}"; + + if (!item.Properties.ContainsKey(currentName)) + return currentName; + else + { + return GetPropertyName(item, name, number + 1); + } + } + + private static string ReadNextPropertyWithoutName(string strData, ref int startPosition, char delimiter = ',') + { + var endPosition = strData.IndexOf(delimiter, startPosition); + var value = strData.Substring(startPosition, endPosition - startPosition); + startPosition = endPosition + 1; + + return value; + } + + private static (string Name, string Value) ReadNextProperty(string strData, ref int startPosition) + { + var equalPosition = strData.IndexOf('=', startPosition); + var name = strData.Substring(startPosition, equalPosition - startPosition); + startPosition = equalPosition + 1; + + if (startPosition == strData.Length) + return (name, ""); + + var nextChar = strData[startPosition]; + + int endPosition; + switch (nextChar) + { + case '\'': + endPosition = strData.IndexOf("\',", startPosition); + break; + case ',': + startPosition++; + return (name, ""); + case '"': + endPosition = strData.IndexOf("\",", startPosition); + break; + default: + endPosition = strData.IndexOf(',', startPosition); + break; + } + + if (endPosition < 0) + endPosition = strData.Length; + + var value = strData.Substring(startPosition, endPosition - startPosition); + startPosition = endPosition + 1; + + return (name, value); + } + + private void InitializeStream() + { + if (_fileStream == null) + { + _fileStream = new FileStream(_logPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + _streamReader = new StreamReader(_fileStream); + } + } + + public void Dispose() + { + if (_fileStream == null) + { + _streamReader.Dispose(); + + _fileStream = null; + _streamReader = null; + } + } + } +} diff --git a/OneSTools.TechLogConsoleApp/OneSTools.TechLogConsoleApp.csproj b/OneSTools.TechLogConsoleApp/OneSTools.TechLogConsoleApp.csproj deleted file mode 100644 index d2a0136..0000000 --- a/OneSTools.TechLogConsoleApp/OneSTools.TechLogConsoleApp.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - - Exe - netcoreapp3.1 - - - - - - - diff --git a/OneSTools.TechLogConsoleApp/Program.cs b/OneSTools.TechLogConsoleApp/Program.cs deleted file mode 100644 index 258fe50..0000000 --- a/OneSTools.TechLogConsoleApp/Program.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using OneSTools.TechLog; -using System.Diagnostics; - -namespace OneSTools.TechLogConsoleApp -{ - class Program - { - static object locker = new object(); - static int i = 0; - - static async Task Main(string[] args) - { - var parser = new TechLogParser(@"C:\Users\akpaev.e.ENTERPRISE\Desktop\ExpertTools\tl", EventHandler); - - var watch = new Stopwatch(); - watch.Start(); - - await parser.Parse(); - - watch.Stop(); - - Console.WriteLine($"Считано событий: {i}"); - Console.WriteLine($"Время выполнения: {watch.Elapsed}"); - Console.ReadKey(); - } - - private static void EventHandler(Dictionary eventData) - { - lock (locker) - { - i++; - } - } - } -}