Проработка библиотеки

OneSTools.TechLog.Exporter.ClickHouse/OneSTools.TechLog.Exporter.ClickHouse.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk.Worker">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <UserSecretsId>dotnet-OneSTools.TechLog.Exporter.ClickHouse-CB3207E2-F57F-4D50-94AC-73469ECF9AAA</UserSecretsId>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="ClickHouse.Client" Version="2.2.1.285" />
+    <PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\OneSTools.TechLog.Exporter.Core\OneSTools.TechLog.Exporter.Core.csproj" />
+    <ProjectReference Include="..\OneSTools.TechLog\OneSTools.TechLog.csproj" />
+  </ItemGroup>
+</Project>

OneSTools.TechLog.Exporter.ClickHouse/Program.cs

@@ -0,0 +1,27 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using OneSTools.TechLog.Exporter.Core;
+
+namespace OneSTools.TechLog.Exporter.ClickHouse
+{
+    public class Program
+    {
+        public static void Main(string[] args)
+        {
+            CreateHostBuilder(args).Build().Run();
+        }
+
+        public static IHostBuilder CreateHostBuilder(string[] args) =>
+            Host.CreateDefaultBuilder(args)
+                .ConfigureServices((hostContext, services) =>
+                {
+                    services.AddSingleton<ITechLogStorage, TechLogStorage>();
+                    services.AddSingleton<ITechLogExporter, TechLogExporter>();
+                    services.AddHostedService<TechLogExporterService>();
+                });
+    }
+}

OneSTools.TechLog.Exporter.ClickHouse/TechLogStorage.cs

@@ -0,0 +1,86 @@
+using ClickHouse.Client.ADO;
+using ClickHouse.Client.Copy;
+using Microsoft.Extensions.Configuration;
+using Microsoft.Extensions.Logging;
+using OneSTools.TechLog.Exporter.Core;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace OneSTools.TechLog.Exporter.ClickHouse
+{
+    public class TechLogStorage : ITechLogStorage
+    {
+        private readonly ILogger<TechLogStorage> _logger;
+        private readonly ClickHouseConnection _connection;
+
+        public TechLogStorage(ILogger<TechLogStorage> logger, IConfiguration configuration)
+        {
+            _logger = logger;
+
+            var connectionString = configuration.GetConnectionString("Default");
+
+            _connection = new ClickHouseConnection(connectionString);
+            _connection.Open();
+
+            CreateTechLogItemsTable();
+        }
+
+        private void CreateTechLogItemsTable()
+        {
+            var commandText =
+                @"CREATE TABLE IF NOT EXISTS TechLogItems
+                (
+                    DateTime DateTime Codec(Delta, LZ4),
+                    Duration Int64 Codec(DoubleDelta, LZ4),
+                    Event LowCardinality(String),
+                    Level Int64 Codec(DoubleDelta, LZ4)
+                )
+                engine = MergeTree()
+                PARTITION BY (toYYYYMM(DateTime))
+                PRIMARY KEY (DateTime)
+                ORDER BY (DateTime)
+                SETTINGS index_granularity = 8192;";
+
+            using var cmd = _connection.CreateCommand();
+            cmd.CommandText = commandText;
+            cmd.ExecuteNonQuery();
+        }
+
+        public async Task WriteItemsAsync(TechLogItem[] items)
+        {
+            using var copy = new ClickHouseBulkCopy(_connection)
+            {
+                DestinationTableName = "TechLogItems",
+                BatchSize = items.Length
+            };
+
+            var data = items.Select(item => new object[] {
+                item.DateTime,
+                item.Duration,
+                item.Event,
+                item.Level,
+                item.Properties.Select(c => c.Key).ToArray(),
+                item.Properties.Select(c => c.Value).ToArray()
+            }).AsEnumerable();
+
+            try
+            {
+                await copy.WriteToServerAsync(data);
+
+                _logger.LogInformation($"{DateTime.Now:hh:mm:ss:fffff} TechLogExporter has written {items.Length}");
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, $"{DateTime.Now:hh:mm:ss:fffff} Failed to write data into database");
+            }
+        }
+
+        public void Dispose()
+        {
+            _connection?.Dispose();
+        }
+    }
+}

OneSTools.TechLog.Exporter.ClickHouse/appsettings.json

@@ -0,0 +1,31 @@
+{
+  "Logging": {
+    "LogLevel": {
+      "Default": "Information",
+      "Microsoft": "Warning",
+      "Microsoft.Hosting.Lifetime": "Information"
+    }
+  },
+  "ConnectionStrings": {
+    "Default": "Host=172.30.40.163;Port=8123;Username=default;password=;Database=upp_main_tl;"
+  },
+  "Exporter": {
+    "LogFolder": "C:\\Users\\akpaev.e.ENTERPRISE\\Desktop\\TechLog",
+    "RecordingStreams": 3,
+    "Events": [
+      {
+        "Name": "DBMSSQL",
+        "Properties": [
+          {
+            "Name": "Sql",
+            "ColumnInfo": "String Codec(ZSTD)"
+          },
+          {
+            "Name": "Context",
+            "ColumnInfo": "String Codec(ZSTD)"
+          }
+        ]
+      }
+    ]
+  }
+}

OneSTools.TechLog.Exporter.Core/ITechLogEventSubscriber.cs

@@ -0,0 +1,12 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace OneSTools.TechLog.Exporter.Core
+{
+    public interface ITechLogEventSubscriber
+    {
+        public string Event { get; }
+        public Task HandleItemAsync(Dictionary<string, string> item, CancellationToken cancellationToken = default);
+    }
+}

OneSTools.TechLog.Exporter.Core/ITechLogExporter.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace OneSTools.TechLog.Exporter.Core
+{
+    public interface ITechLogExporter : IDisposable
+    {
+        Task StartAsync(CancellationToken cancellationToken = default);
+    }
+
+    public class ExcpEventSubscriber : ITechLogEventSubscriber
+    {
+        public string Event => "EXCP";
+
+        public async Task HandleItemsAsync(Dictionary<string, string> items, CancellationToken cancellationToken = default)
+        {
+        }
+    }
+}

OneSTools.TechLog.Exporter.Core/ITechLogFolderReader.cs

@@ -1,11 +0,0 @@
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace OneSTools.TechLog.Exporter.Core
-{
-    public interface ITechLogFolderReader : IDisposable
-    {
-        Task StartAsync(string logFolder, int portion, bool liveMode = false, CancellationToken cancellationToken = default);
-    }
-}

OneSTools.TechLog.Exporter.Core/ITechLogStorage.cs

@@ -1,8 +1,9 @@
-using System.Threading.Tasks;
+using System;
+using System.Threading.Tasks;
 
 namespace OneSTools.TechLog.Exporter.Core
 {
-    public interface ITechLogStorage
+    public interface ITechLogStorage : IDisposable
     {
         Task WriteItemsAsync(TechLogItem[] items);
     }

OneSTools.TechLog.Exporter.Core/TechLogEventSubscriber.cs

@@ -0,0 +1,16 @@
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace OneSTools.TechLog.Exporter.Core
+{
+    public class TechLogEventSubscriber : ITechLogEventSubscriber
+    {
+        public string Event => "EXCP";
+
+        public async Task HandleItemAsync(Dictionary<string, string> item, CancellationToken cancellationToken = default)
+        {
+            
+        }
+    }
+}

OneSTools.TechLog.Exporter.Core/TechLogExporter.cs

@@ -0,0 +1,166 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using Microsoft.Extensions.Logging;
+using OneSTools.TechLog;
+using Microsoft.Extensions.Configuration;
+
+namespace OneSTools.TechLog.Exporter.Core
+{
+    public class TechLogExporter : IDisposable, ITechLogExporter
+    {
+        private readonly ILogger<TechLogExporter> _logger;
+        private ITechLogStorage _techLogStorage;
+        private string _logFolder;
+        private int _portion;
+        private int _recordingStreams;
+        private bool _liveMode;
+        private ActionBlock<TechLogItem[]> _writeBlock;
+        private BatchBlock<TechLogItem> _batchBlock;
+        private ActionBlock<string> _parseBlock;
+        private ActionBlock<string> _readBlock;
+        private FileSystemWatcher _logFilesWatcher;
+        private HashSet<string> _logFiles = new HashSet<string>();
+
+        public TechLogExporter(ILogger<TechLogExporter> logger, IConfiguration configuration, ITechLogStorage techLogStorage)
+        {
+            _logger = logger;
+            _techLogStorage = techLogStorage;
+
+            _logFolder = configuration.GetValue("Exporter:LogFolder", "");
+            if (_logFolder == string.Empty)
+                throw new Exception("Log folder is not specified");
+
+            _portion = configuration.GetValue("Exporter:Portion", 10000);
+            _recordingStreams = configuration.GetValue("Exporter:RecordingStreams", 1);
+            _liveMode = configuration.GetValue("Exporter:LiveMode", true);
+        }
+
+        public async Task StartAsync(CancellationToken cancellationToken = default)
+        {
+            var maxDegree = Environment.ProcessorCount;
+
+            _writeBlock = new ActionBlock<TechLogItem[]>(_techLogStorage.WriteItemsAsync, new ExecutionDataflowBlockOptions() 
+            { 
+                BoundedCapacity = _recordingStreams,
+                CancellationToken = cancellationToken,
+            });
+            _batchBlock = new BatchBlock<TechLogItem>(_portion, new GroupingDataflowBlockOptions() 
+            {
+                BoundedCapacity = _portion,
+                CancellationToken = cancellationToken
+            });
+            _parseBlock = new ActionBlock<string>(str => ParseItemData(str, _batchBlock), new ExecutionDataflowBlockOptions()
+            {
+                MaxDegreeOfParallelism = maxDegree,
+                BoundedCapacity = _portion / 2,
+                CancellationToken = cancellationToken
+            });
+            _readBlock = new ActionBlock<string>(str => ReadItemsData(str, _parseBlock), new ExecutionDataflowBlockOptions()
+            {
+                MaxDegreeOfParallelism = maxDegree,
+                CancellationToken = cancellationToken
+            }); ;
+
+            _batchBlock.LinkTo(_writeBlock);
+
+            var logFiles = GetLogFiles();
+
+            foreach (var logFile in logFiles)
+                await StartReaderAsync(logFile, cancellationToken);
+
+            if (_liveMode)
+                StartLogFilesWatcher();
+
+            await _writeBlock.Completion;
+        }
+
+        private async Task StartReaderAsync(string logPath, CancellationToken cancellationToken = default)
+        {
+            if (!_logFiles.Contains(logPath))
+            {
+                await SendDataAsync(_readBlock, logPath);
+
+                _logger.LogInformation($"Log reader for \"{logPath}\" is started");
+            }
+        }
+
+        private string[] GetLogFiles()
+        {
+            return Directory.GetFiles(_logFolder, "*.log", SearchOption.AllDirectories);
+        }
+
+        private void ReadItemsData(string logPath, ITargetBlock<string> nextblock)
+        {
+            using var reader = new TechLogReader(logPath, _liveMode);
+
+            do
+            {
+                var itemData = reader.ReadItemData();
+
+                if (reader.Closed)
+                    _logFiles.Remove(logPath);
+
+                if (itemData != null)
+                    PostData(nextblock, itemData);
+            }
+            while (!reader.Closed);
+
+            _logger.LogInformation($"Log reader for \"{logPath}\" is stopped");
+        }
+
+        private void PostData<T>(ITargetBlock<T> block, T data)
+        {
+            while (true)
+            {
+                if (block.Post(data))
+                    break;
+            }
+        }
+
+        private async Task SendDataAsync<T>(ITargetBlock<T> block, T data)
+        {
+            while (true)
+            {
+                if (await block.SendAsync(data))
+                    break;
+            }
+        }
+
+        private void ParseItemData(string itemData, ITargetBlock<TechLogItem> nextblock)
+        {
+            var item = TechLogReader.ParseItemData(itemData);
+
+            PostData(nextblock, item);
+        }
+
+        private void StartLogFilesWatcher()
+        {
+            _logFilesWatcher = new FileSystemWatcher(_logFolder, "*.log")
+            {
+                NotifyFilter = NotifyFilters.CreationTime
+            };
+            _logFilesWatcher.Created += _logFilesWatcher_Created;
+            _logFilesWatcher.IncludeSubdirectories = true;
+            _logFilesWatcher.EnableRaisingEvents = true;
+        }
+
+        private async void _logFilesWatcher_Created(object sender, FileSystemEventArgs e)
+        {
+            if (e.ChangeType == WatcherChangeTypes.Created)
+            {
+                await StartReaderAsync(e.FullPath);
+            }
+        }
+
+        public void Dispose()
+        {
+            _logFilesWatcher?.Dispose();
+        }
+    }
+}

OneSTools.TechLog.Exporter.Core/TechLogExporterService.cs

@@ -13,29 +13,32 @@ namespace OneSTools.TechLog.Exporter.Core
 {
     public class TechLogExporterService : BackgroundService
     {
-        private IConfiguration _configuration;
         private readonly ILogger<TechLogExporterService> _logger;
-        private readonly ITechLogFolderReader _techLogFolderReader;
-        private string _logFolder;
-        private int _portion;
+        private readonly ITechLogExporter _techLogExporter;
 
-        public TechLogExporterService(IConfiguration configuration, ILogger<TechLogExporterService> logger, ITechLogFolderReader techLogFolderReader)
+        public TechLogExporterService(ILogger<TechLogExporterService> logger, ITechLogExporter techLogExporter)
         {
-            _configuration = configuration;
             _logger = logger;
-            _techLogFolderReader = techLogFolderReader;
-
-            _logFolder = configuration.GetValue("Exporter:LogFolder", "");
-
-            if (_logFolder == "")
-                throw new Exception("Log folder's path is not set");
-
-            _portion = configuration.GetValue("Exporter", 10000);
+            _techLogExporter = techLogExporter;
         }
 
         protected override async Task ExecuteAsync(CancellationToken stoppingToken)
         {
-            await _techLogFolderReader.StartAsync(_logFolder, _portion, true, stoppingToken);
+            try
+            {
+                await _techLogExporter.StartAsync(stoppingToken);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogCritical(ex, "Failed to execute TechLogExporter");
+            }
+        }
+
+        public override void Dispose()
+        {
+            _techLogExporter?.Dispose();
+
+            base.Dispose();
         }
     }
 }

OneSTools.TechLog.Exporter.Core/TechLogFolderReader.cs

@@ -1,121 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Text;
-using System.Threading;
-using System.Threading.Tasks;
-using System.Threading.Tasks.Dataflow;
-using Microsoft.Extensions.Logging;
-using OneSTools.TechLog;
-
-namespace OneSTools.TechLog.Exporter.Core
-{
-    public class TechLogFolderReader : IDisposable, ITechLogFolderReader
-    {
-        private readonly ILogger<TechLogFolderReader> _logger;
-        private ITechLogStorage _techLogStorage;
-        private string _folder;
-        private bool _liveMode;
-        private ActionBlock<TechLogItem[]> _writeBlock;
-        private BatchBlock<TechLogItem> _batchBlock;
-        private TransformBlock<string, TechLogItem> _parseBlock;
-        private ActionBlock<string> _readBlock;
-
-        public TechLogFolderReader(ILogger<TechLogFolderReader> logger, ITechLogStorage techLogStorage)
-        {
-            _logger = logger;
-            _techLogStorage = techLogStorage;
-        }
-
-        public async Task StartAsync(string folder, int portion, bool liveMode = false, CancellationToken cancellationToken = default)
-        {
-            _folder = folder;
-            _liveMode = liveMode;
-
-            var maxDegree = Environment.ProcessorCount * 10;
-
-            _writeBlock = new ActionBlock<TechLogItem[]>(_techLogStorage.WriteItemsAsync, new ExecutionDataflowBlockOptions() 
-            { 
-                BoundedCapacity = 3,
-                CancellationToken = cancellationToken,
-                MaxDegreeOfParallelism = maxDegree,
-            });
-            _batchBlock = new BatchBlock<TechLogItem>(portion, new GroupingDataflowBlockOptions() 
-            { 
-                CancellationToken = cancellationToken,
-                BoundedCapacity = portion
-            });
-            _parseBlock = new TransformBlock<string, TechLogItem>(ParseItemData, new ExecutionDataflowBlockOptions()
-            {
-                MaxDegreeOfParallelism = maxDegree,
-                BoundedCapacity = portion / 2,
-                CancellationToken = cancellationToken
-            });
-            _readBlock = new ActionBlock<string>(str => ReadItemsData(str, _parseBlock), new ExecutionDataflowBlockOptions()
-            {
-                MaxDegreeOfParallelism = maxDegree,
-                CancellationToken = cancellationToken
-            }); ;
-
-            _parseBlock.LinkTo(_batchBlock);
-            _batchBlock.LinkTo(_writeBlock);
-
-            var logFiles = GetLogFiles();
-
-            foreach (var logFile in logFiles)
-            {
-                await _readBlock.SendAsync(logFile);
-            }
-
-            await _writeBlock.Completion;
-        }
-
-        private string[] GetLogFiles()
-        {
-            return Directory.GetFiles(_folder, "*.log", SearchOption.AllDirectories);
-        }
-
-        private void ReadItemsData(string logPath, ITargetBlock<string> nextblock)
-        {
-            var fileName = Path.GetFileNameWithoutExtension(logPath);
-
-            var fileDateTime = "20" +
-                fileName.Substring(0, 2) +
-                "-" +
-                fileName.Substring(2, 2) +
-                "-" +
-                fileName.Substring(4, 2) +
-                " " +
-                fileName.Substring(6, 2);
-
-            using var reader = new TechLogReader(logPath);
-
-            while (true)
-            {
-                var itemData = reader.ReadItemData();
-
-                if (itemData != null)
-                    PostData(nextblock, fileDateTime + ":" + itemData);
-            }
-        }
-
-        private void PostData<T>(ITargetBlock<T> nextblock, T data)
-        {
-            while (true)
-            {
-                if (nextblock.Post(data))
-                    break;
-            }
-        }
-
-        private TechLogItem ParseItemData(string itemData)
-        {
-            return TechLogReader.ParseItemData(itemData);
-        }
-
-        public void Dispose()
-        {
-
-        }
-    }
-}

OneSTools.TechLog.Exporter.ElasticSearch/OneSTools.TechLog.Exporter.ElasticSearch.csproj

@@ -1,15 +0,0 @@
-<Project Sdk="Microsoft.NET.Sdk.Worker">
-
-  <PropertyGroup>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
-    <UserSecretsId>dotnet-OneSTools.TechLog.Exporter.ElasticSearch-146D8E25-B976-4FAB-A231-A91F8DD2EEF5</UserSecretsId>
-  </PropertyGroup>
-
-  <ItemGroup>
-    <PackageReference Include="NEST" Version="7.9.0" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <ProjectReference Include="..\OneSTools.TechLog.Exporter.Core\OneSTools.TechLog.Exporter.Core.csproj" />
-  </ItemGroup>
-</Project>

OneSTools.TechLog.Exporter.ElasticSearch/Program.cs

@@ -1,41 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Hosting;
-using Microsoft.Extensions.Logging;
-using OneSTools.TechLog.Exporter.Core;
-
-namespace OneSTools.TechLog.Exporter.ElasticSearch
-{
-    public class Program
-    {
-        public static void Main(string[] args)
-        {
-            CreateHostBuilder(args).Build().Run();
-        }
-
-        public static IHostBuilder CreateHostBuilder(string[] args) =>
-            Host.CreateDefaultBuilder(args)
-                .ConfigureServices((hostContext, services) =>
-                {
-                    var configuration = hostContext.Configuration;
-                    var host = configuration.GetValue("ElasticSearch:Host", "");
-                    var port = configuration.GetValue("ElasticSearch:Port", 9200);
-                    var index = configuration.GetValue("ElasticSearch:Index", "");
-                    var separation = configuration.GetValue("ElasticSearch:Separation", "");
-
-                    services.AddSingleton<ITechLogStorage>(sp =>
-                    {
-                        var logger = sp.GetService<ILogger<TechLogStorage>>();
-
-                        return new TechLogStorage(logger, host, port, index, separation);
-                    });
-                    services.AddSingleton<ITechLogFolderReader, TechLogFolderReader>();
-                    services.AddHostedService<TechLogExporterService>();
-                });
-    }
-}

OneSTools.TechLog.Exporter.ElasticSearch/TechLogStorage.cs

@@ -1,77 +0,0 @@
-using Microsoft.Extensions.Logging;
-using Nest;
-using OneSTools.TechLog.Exporter.Core;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace OneSTools.TechLog.Exporter.ElasticSearch
-{
-    public class TechLogStorage : ITechLogStorage
-    {
-        private readonly ILogger<TechLogStorage> _logger;
-        private string _index;
-        private string _separation;
-        ElasticClient _client;
-
-        public TechLogStorage(ILogger<TechLogStorage> logger, string host, int port = 9200, string index = "", string separation = "")
-        {
-            _logger = logger;
-
-            var uri = new Uri($"{host}:{port}");
-            _index = $"{index}-tl";
-
-            var settings = new ConnectionSettings(uri);
-            settings.DefaultIndex(_index);
-
-            _separation = separation;
-
-            _client = new ElasticClient(settings);
-            var response = _client.Ping();
-
-            if (!response.IsValid)
-                throw response.OriginalException;
-        }
-
-        public async Task WriteItemsAsync(TechLogItem[] items)
-        {
-            var data = new List<(string IndexName, TechLogItem[] Items)>();
-
-            switch (_separation)
-            {
-                case "H":
-                    var groups = items.GroupBy(c => c.DateTime.ToString("yyyyMMddhh")).OrderBy(c => c.Key);
-                    foreach (IGrouping<string, TechLogItem> item in groups)
-                        data.Add(($"{_index}-{item.Key}", item.ToArray()));
-                    break;
-                case "D":
-                    groups = items.GroupBy(c => c.DateTime.ToString("yyyyMMdd")).OrderBy(c => c.Key);
-                    foreach (IGrouping<string, TechLogItem> item in groups)
-                        data.Add(($"{_index}-{item.Key}", item.ToArray()));
-                    break;
-                case "M":
-                    groups = items.GroupBy(c => c.DateTime.ToString("yyyyMM")).OrderBy(c => c.Key);
-                    foreach (IGrouping<string, TechLogItem> item in groups)
-                        data.Add(($"{_index}-{item.Key}", item.ToArray()));
-                    break;
-                default:
-                    data.Add(($"{_index}-all", items));
-                    break;
-            }
-
-            foreach ((string IndexName, TechLogItem[] Entities) item in data)
-            {
-                var responseItems = await _client.IndexManyAsync(item.Entities, item.IndexName);
-
-                if (!responseItems.IsValid)
-                {
-                    throw responseItems.OriginalException;
-                }
-
-                _logger.LogInformation($"{DateTime.Now:hh:mm:ss:fffff} has written {item.Entities.Length}");
-            }
-        }
-    }
-}

OneSTools.TechLog.Exporter.ElasticSearch/appsettings.json

@@ -1,18 +0,0 @@
-{
-  "Logging": {
-    "LogLevel": {
-      "Default": "Information",
-      "Microsoft": "Warning",
-      "Microsoft.Hosting.Lifetime": "Information"
-    }
-  },
-  "Exporter": {
-    "LogFolder": "C:\\Users\\akpaev.e.ENTERPRISE\\Desktop\\TechLog"
-  },
-  "ElasticSearch": {
-    "Host": "http://192.168.0.95",
-    "Port": 9200,
-    "Index": "upp-main",
-    "Separation": "H" // H - hour, D - day, M - Month
-  }
-}

OneSTools.TechLog.sln

@@ -5,9 +5,9 @@ VisualStudioVersion = 16.0.29318.209
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneSTools.TechLog", "OneSTools.TechLog\OneSTools.TechLog.csproj", "{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneSTools.TechLog.Exporter.ElasticSearch", "OneSTools.TechLog.Exporter.ElasticSearch\OneSTools.TechLog.Exporter.ElasticSearch.csproj", "{6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneSTools.TechLog.Exporter.Core", "OneSTools.TechLog.Exporter.Core\OneSTools.TechLog.Exporter.Core.csproj", "{02C37C84-911F-4725-A1EC-B81FF8F16227}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneSTools.TechLog.Exporter.Core", "OneSTools.TechLog.Exporter.Core\OneSTools.TechLog.Exporter.Core.csproj", "{02C37C84-911F-4725-A1EC-B81FF8F16227}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneSTools.TechLog.Exporter.ClickHouse", "OneSTools.TechLog.Exporter.ClickHouse\OneSTools.TechLog.Exporter.ClickHouse.csproj", "{C610C51D-2604-4B3D-AFCA-4204A5821F88}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -19,14 +19,14 @@ Global
 		{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Release|Any CPU.Build.0 = Release|Any CPU
-		{6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{6C61BC30-BA4E-428D-A805-1F3B7D9B4D00}.Release|Any CPU.Build.0 = Release|Any CPU
 		{02C37C84-911F-4725-A1EC-B81FF8F16227}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{02C37C84-911F-4725-A1EC-B81FF8F16227}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{02C37C84-911F-4725-A1EC-B81FF8F16227}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{02C37C84-911F-4725-A1EC-B81FF8F16227}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

OneSTools.TechLog/OneSTools.TechLog.csproj

@@ -13,6 +13,7 @@
     <LangVersion>8.0</LangVersion>
     <PackageIcon>onestools_icon_nuget.png</PackageIcon>
     <PackageLicenseExpression></PackageLicenseExpression>
+    <PackageLicenseFile>LICENSE</PackageLicenseFile>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
@@ -22,6 +23,10 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <None Include="C:\Users\akpaev.e.ENTERPRISE\source\repos\OneSTools.TechLog\LICENSE">
+      <Pack>True</Pack>
+      <PackagePath></PackagePath>
+    </None>
     <None Include="C:\Users\akpaev.e.ENTERPRISE\source\repos\onestools_icon_nuget.png">
       <Pack>True</Pack>
       <PackagePath></PackagePath>

OneSTools.TechLog/TechLogItem.cs

@@ -1,15 +0,0 @@
-using System;
-using System.Collections.Generic;
-
-namespace OneSTools.TechLog
-{
-    public class TechLogItem
-    {
-        public DateTime DateTime { get; set; }
-        public long Duration { get; set; }
-        public string Event { get; set; }
-        public int Level { get; set; }
-
-        public Dictionary<string, string> Properties { get; set; } = new Dictionary<string, string>();
-    }
-}

OneSTools.TechLog/TechLogReader.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Data;
 using System.IO;
 using System.Text;
@@ -11,20 +12,36 @@ namespace OneSTools.TechLog
     public class TechLogReader : IDisposable
     {
         private string _logPath;
-        private string _fileName;
+        private string _fileDateTime;
+        private readonly bool _liveMode;
         private FileStream _fileStream;
         private StreamReader _streamReader;
         private StringBuilder _currentData = new StringBuilder();
+        private ManualResetEvent _logFileChanged;
+        private ManualResetEvent _logFileDeleted;
+        private FileSystemWatcher _logFileWatcher;
 
-        public TechLogReader(string logPath)
+        public bool Closed { get; private set; } = true;
+
+        public TechLogReader(string logPath, bool liveMode = false)
         {
             _logPath = logPath;
-            _fileName = Path.GetFileNameWithoutExtension(_logPath);
+            _liveMode = liveMode;
+
+            var fileName = Path.GetFileNameWithoutExtension(_logPath);
+            _fileDateTime = "20" +
+                fileName.Substring(0, 2) +
+                "-" +
+                fileName.Substring(2, 2) +
+                "-" +
+                fileName.Substring(4, 2) +
+                " " +
+                fileName.Substring(6, 2);
         }
 
-        public TechLogItem ReadNextItem(CancellationToken cancellationToken = default)
+        public Dictionary<string, string> ReadNextItem(CancellationToken cancellationToken = default)
         {
-            InitializeStream();
+            Initialize();
 
             var itemData = ReadItemData(cancellationToken);
 
@@ -36,7 +53,7 @@ namespace OneSTools.TechLog
 
         public string ReadItemData(CancellationToken cancellationToken = default)
         {
-            InitializeStream();
+            Initialize();
 
             var currentLine = "";
 
@@ -49,7 +66,31 @@ namespace OneSTools.TechLog
                     if (_currentData.Length > 0)
                         break;
                     else
-                        return null;
+                    {
+                        if (_liveMode)
+                        {
+                            var handles = new WaitHandle[]
+                            {
+                                _logFileChanged,
+                                _logFileDeleted,
+                                cancellationToken.WaitHandle
+                            };
+
+                            var index = WaitHandle.WaitAny(handles);
+
+                            if (index == 1 || index == 2 || index == WaitHandle.WaitTimeout) // File is deleted / reader stopped / timeout
+                            {
+                                Dispose();
+                                Closed = true;
+                                return null;
+                            }
+                            else if (index == 0) // File is changed, continue reading
+                            {
+                                _logFileChanged.Reset();
+                                continue;
+                            }
+                        }
+                    }
                 }
 
                 if (currentLine == null || _currentData.Length > 0 && Regex.IsMatch(currentLine, @"^\d\d:\d\d\.", RegexOptions.Compiled))
@@ -65,53 +106,51 @@ namespace OneSTools.TechLog
             if (currentLine != null)
                 _currentData.AppendLine(currentLine);
 
-            return _strData;
+            return _fileDateTime + ":" + _strData;
         }
 
-        public static TechLogItem ParseItemData(string itemData, CancellationToken cancellationToken = default)
+        public static Dictionary<string, string> ParseItemData(string itemData, CancellationToken cancellationToken = default)
         {
-            var data = new TechLogItem();
+            var item = new Dictionary<string, string>();
 
             int startPosition = 0;
 
             var dtd = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
             var dtdLength = dtd.Length;
             var dtEndIndex = dtd.LastIndexOf('-');
-            data.DateTime = DateTime.Parse(dtd.Substring(0, dtEndIndex));
+            item["DateTime"] = dtd.Substring(0, dtEndIndex);
             startPosition -= dtdLength - dtEndIndex;
 
-            data.Duration = long.Parse(ReadNextPropertyWithoutName(itemData, ref startPosition, ','));
-            data.Event = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
-            data.Level = int.Parse(ReadNextPropertyWithoutName(itemData, ref startPosition, ','));
+            item["Duration"] = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
+            item["Event"] = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
+            item["Level"] = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
 
             while (!cancellationToken.IsCancellationRequested)
             {
                 var (Name, Value) = ReadNextProperty(itemData, ref startPosition);
 
-                if (data.Properties.ContainsKey(Name))
+                if (item.ContainsKey(Name))
                 {
-                    data.Properties.Add(GetPropertyName(data, Name, 0), Value);
+                    item.Add(GetPropertyName(item, Name, 0), Value);
                 }
                 else
-                    data.Properties.Add(Name, Value);
+                    item.Add(Name, Value);
 
                 if (startPosition >= itemData.Length)
                     break;
             }
 
-            return data;
+            return item;
         }
 
-        private static string GetPropertyName(TechLogItem item, string name, int number = 0)
+        private static string GetPropertyName(Dictionary<string, string> item, string name, int number = 0)
         {
             var currentName = $"{name}{number}";
 
-            if (!item.Properties.ContainsKey(currentName))
+            if (!item.ContainsKey(currentName))
                 return currentName;
             else
-            {
                 return GetPropertyName(item, name, number + 1);
-            }
         }
 
         private static string ReadNextPropertyWithoutName(string strData, ref int startPosition, char delimiter = ',')
@@ -157,7 +196,68 @@ namespace OneSTools.TechLog
             var value = strData.Substring(startPosition, endPosition - startPosition);
             startPosition = endPosition + 1;
 
-            return (name, value);
+            return (name, value.Trim(new char[] { '\'', '"' }).Trim());
+        }
+
+        private static string GetCleanedSql(string data)
+        {
+            throw new NotImplementedException();
+
+            //// Remove paramaters
+            //int startIndex = data.IndexOf("sp_executesql", StringComparison.OrdinalIgnoreCase);
+
+            //if (startIndex < 0)
+            //    startIndex = 0;
+            //else
+            //    startIndex += 16;
+
+            //int e1 = data.IndexOf("', N'@P", StringComparison.OrdinalIgnoreCase);
+            //if (e1 < 0)
+            //    e1 = data.Length;
+
+            //var e2 = data.IndexOf("p_0:", StringComparison.OrdinalIgnoreCase);
+            //if (e2 < 0)
+            //    e2 = data.Length;
+
+            //var endIndex = Math.Min(e1, e2);
+
+            //StringBuilder result = new StringBuilder(data[startIndex..endIndex]);
+
+            //// Remove temp table names
+            //while (true)
+            //{
+            //    var ttsi = result.IndexOf("#tt");
+
+            //    if (ttsi >= 0)
+            //    {
+            //        var ttsei = ttsi + 2;
+
+            //        // Read temp table number
+            //        while (true)
+            //        {
+            //            if (char.IsDigit(result[ttsei]))
+            //                ttsei++;
+            //            else
+            //                break;
+            //        }
+            //    }
+            //    else
+            //        break;
+            //}
+
+            //return result.ToString();
+        }
+
+        private static string GetSqlHash(string sql)
+        {
+            throw new NotImplementedException();
+        }
+
+        private void Initialize()
+        {
+            InitializeStream();
+
+            InitializeWatcher();
         }
 
         private void InitializeStream()
@@ -166,18 +266,47 @@ namespace OneSTools.TechLog
             {
                 _fileStream = new FileStream(_logPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
                 _streamReader = new StreamReader(_fileStream);
+
+                Closed = false;
             }
         }
 
+        private void InitializeWatcher()
+        {
+            if (_liveMode && _logFileWatcher == null)
+            {
+                _logFileChanged = new ManualResetEvent(false);
+                _logFileDeleted = new ManualResetEvent(false);
+
+                _logFileWatcher = new FileSystemWatcher(Path.GetDirectoryName(_logPath), Path.GetFileName(_logPath))
+                {
+                    NotifyFilter = NotifyFilters.CreationTime | NotifyFilters.LastWrite | NotifyFilters.FileName
+                };
+                _logFileWatcher.Changed += _logFileWatcher_Changed;
+                _logFileWatcher.Deleted += _logFileWatcher_Deleted;
+                _logFileWatcher.EnableRaisingEvents = true;
+            }
+        }
+
+        private void _logFileWatcher_Changed(object sender, FileSystemEventArgs e)
+        {
+            if (e.ChangeType == WatcherChangeTypes.Changed)
+                _logFileChanged.Set();
+        }
+
+        private void _logFileWatcher_Deleted(object sender, FileSystemEventArgs e)
+        {
+            if (e.ChangeType == WatcherChangeTypes.Deleted)
+                _logFileDeleted.Set();
+        }
+
         public void Dispose()
         {
-            if (_fileStream == null)
-            {
-                _streamReader.Dispose();
-
-                _fileStream = null;
-                _streamReader = null;
-            }
+            _streamReader?.Dispose();
+            _fileStream = null;
+            _logFileWatcher?.Dispose();
+            _logFileChanged?.Dispose();
+            _logFileDeleted?.Dispose();
         }
     }
 }