1c/OneSTools.TechLog
1
0
Fork 0
mirror of https://github.com/akpaevj/OneSTools.TechLog.git synced 2024-12-06 08:16:09 +02:00
Code Issues Releases Activity

Добавлены объекты для чтения в Live режиме

Browse Source
This commit is contained in:
Акпаев Евгений Александрович 2020-12-11 00:36:01 +03:00
parent 3b2584e184
commit 2dc7824dd9
28 changed files with 913 additions and 726 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
OneSTools.TechLog.Exporter.ClickHouse/OneSTools.TechLog.Exporter.ClickHouse.csproj
View File

@ -1,17 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk.Worker">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<UserSecretsId>dotnet-OneSTools.TechLog.Exporter.ClickHouse-CB3207E2-F57F-4D50-94AC-73469ECF9AAA</UserSecretsId>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="ClickHouse.Client" Version="2.2.1.285" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OneSTools.TechLog.Exporter.Core\OneSTools.TechLog.Exporter.Core.csproj" />
<ProjectReference Include="..\OneSTools.TechLog\OneSTools.TechLog.csproj" />
</ItemGroup>
</Project>

27
OneSTools.TechLog.Exporter.ClickHouse/Program.cs
View File

@ -1,27 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using OneSTools.TechLog.Exporter.Core;
namespace OneSTools.TechLog.Exporter.ClickHouse
{
public class Program
{
public static void Main(string[] args)
{
CreateHostBuilder(args).Build().Run();
}
public static IHostBuilder CreateHostBuilder(string[] args) =>
Host.CreateDefaultBuilder(args)
.ConfigureServices((hostContext, services) =>
{
services.AddSingleton<ITechLogStorage, TechLogStorage>();
services.AddSingleton<ITechLogExporter, TechLogExporter>();
services.AddHostedService<TechLogExporterService>();
});
}
}

86
OneSTools.TechLog.Exporter.ClickHouse/TechLogStorage.cs
View File

@ -1,86 +0,0 @@
using ClickHouse.Client.ADO;
using ClickHouse.Client.Copy;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using OneSTools.TechLog.Exporter.Core;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace OneSTools.TechLog.Exporter.ClickHouse
{
public class TechLogStorage : ITechLogStorage
{
private readonly ILogger<TechLogStorage> _logger;
private readonly ClickHouseConnection _connection;
public TechLogStorage(ILogger<TechLogStorage> logger, IConfiguration configuration)
{
_logger = logger;
var connectionString = configuration.GetConnectionString("Default");
_connection = new ClickHouseConnection(connectionString);
_connection.Open();
CreateTechLogItemsTable();
}
private void CreateTechLogItemsTable()
{
var commandText =
@"CREATE TABLE IF NOT EXISTS TechLogItems
(
DateTime DateTime Codec(Delta, LZ4),
Duration Int64 Codec(DoubleDelta, LZ4),
Event LowCardinality(String),
Level Int64 Codec(DoubleDelta, LZ4)
)
engine = MergeTree()
PARTITION BY (toYYYYMM(DateTime))
PRIMARY KEY (DateTime)
ORDER BY (DateTime)
SETTINGS index_granularity = 8192;";
using var cmd = _connection.CreateCommand();
cmd.CommandText = commandText;
cmd.ExecuteNonQuery();
}
public async Task WriteItemsAsync(TechLogItem[] items)
{
using var copy = new ClickHouseBulkCopy(_connection)
{
DestinationTableName = "TechLogItems",
BatchSize = items.Length
};
var data = items.Select(item => new object[] {
item.DateTime,
item.Duration,
item.Event,
item.Level,
item.Properties.Select(c => c.Key).ToArray(),
item.Properties.Select(c => c.Value).ToArray()
}).AsEnumerable();
try
{
await copy.WriteToServerAsync(data);
_logger.LogInformation($"{DateTime.Now:hh:mm:ss:fffff} TechLogExporter has written {items.Length}");
}
catch (Exception ex)
{
_logger.LogError(ex, $"{DateTime.Now:hh:mm:ss:fffff} Failed to write data into database");
}
}
public void Dispose()
{
_connection?.Dispose();
}
}
}

9
OneSTools.TechLog.Exporter.ClickHouse/appsettings.Development.json
View File

@ -1,9 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
}
}

31
OneSTools.TechLog.Exporter.ClickHouse/appsettings.json
View File

@ -1,31 +0,0 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"ConnectionStrings": {
"Default": "Host=172.30.40.163;Port=8123;Username=default;password=;Database=upp_main_tl;"
},
"Exporter": {
"LogFolder": "C:\\Users\\akpaev.e.ENTERPRISE\\Desktop\\TechLog",
"RecordingStreams": 3,
"Events": [
{
"Name": "DBMSSQL",
"Properties": [
{
"Name": "Sql",
"ColumnInfo": "String Codec(ZSTD)"
},
{
"Name": "Context",
"ColumnInfo": "String Codec(ZSTD)"
}
]
}
]
}
}

12
OneSTools.TechLog.Exporter.Core/ITechLogEventSubscriber.cs
View File

@ -1,12 +0,0 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace OneSTools.TechLog.Exporter.Core
{
public interface ITechLogEventSubscriber
{
public string Event { get; }
public Task HandleItemAsync(Dictionary<string, string> item, CancellationToken cancellationToken = default);
}
}

21
OneSTools.TechLog.Exporter.Core/ITechLogExporter.cs
View File

@ -1,21 +0,0 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace OneSTools.TechLog.Exporter.Core
{
public interface ITechLogExporter : IDisposable
{
Task StartAsync(CancellationToken cancellationToken = default);
}
public class ExcpEventSubscriber : ITechLogEventSubscriber
{
public string Event => "EXCP";
public async Task HandleItemsAsync(Dictionary<string, string> items, CancellationToken cancellationToken = default)
{
}
}
}

10
OneSTools.TechLog.Exporter.Core/ITechLogStorage.cs
View File

@ -1,10 +0,0 @@
using System;
using System.Threading.Tasks;
namespace OneSTools.TechLog.Exporter.Core
{
public interface ITechLogStorage : IDisposable
{
Task WriteItemsAsync(TechLogItem[] items);
}
}

18
OneSTools.TechLog.Exporter.Core/OneSTools.TechLog.Exporter.Core.csproj
View File

@ -1,18 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard2.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Hosting" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="5.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="5.0.0" />
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="5.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OneSTools.TechLog\OneSTools.TechLog.csproj" />
</ItemGroup>
</Project>

16
OneSTools.TechLog.Exporter.Core/TechLogEventSubscriber.cs
View File

@ -1,16 +0,0 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace OneSTools.TechLog.Exporter.Core
{
public class TechLogEventSubscriber : ITechLogEventSubscriber
{
public string Event => "EXCP";
public async Task HandleItemAsync(Dictionary<string, string> item, CancellationToken cancellationToken = default)
{
}
}
}

166
OneSTools.TechLog.Exporter.Core/TechLogExporter.cs
View File

@ -1,166 +0,0 @@
using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.IO;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Microsoft.Extensions.Logging;
using OneSTools.TechLog;
using Microsoft.Extensions.Configuration;
namespace OneSTools.TechLog.Exporter.Core
{
public class TechLogExporter : IDisposable, ITechLogExporter
{
private readonly ILogger<TechLogExporter> _logger;
private ITechLogStorage _techLogStorage;
private string _logFolder;
private int _portion;
private int _recordingStreams;
private bool _liveMode;
private ActionBlock<TechLogItem[]> _writeBlock;
private BatchBlock<TechLogItem> _batchBlock;
private ActionBlock<string> _parseBlock;
private ActionBlock<string> _readBlock;
private FileSystemWatcher _logFilesWatcher;
private HashSet<string> _logFiles = new HashSet<string>();
public TechLogExporter(ILogger<TechLogExporter> logger, IConfiguration configuration, ITechLogStorage techLogStorage)
{
_logger = logger;
_techLogStorage = techLogStorage;
_logFolder = configuration.GetValue("Exporter:LogFolder", "");
if (_logFolder == string.Empty)
throw new Exception("Log folder is not specified");
_portion = configuration.GetValue("Exporter:Portion", 10000);
_recordingStreams = configuration.GetValue("Exporter:RecordingStreams", 1);
_liveMode = configuration.GetValue("Exporter:LiveMode", true);
}
public async Task StartAsync(CancellationToken cancellationToken = default)
{
var maxDegree = Environment.ProcessorCount;
_writeBlock = new ActionBlock<TechLogItem[]>(_techLogStorage.WriteItemsAsync, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = _recordingStreams,
CancellationToken = cancellationToken,
});
_batchBlock = new BatchBlock<TechLogItem>(_portion, new GroupingDataflowBlockOptions()
{
BoundedCapacity = _portion,
CancellationToken = cancellationToken
});
_parseBlock = new ActionBlock<string>(str => ParseItemData(str, _batchBlock), new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegree,
BoundedCapacity = _portion / 2,
CancellationToken = cancellationToken
});
_readBlock = new ActionBlock<string>(str => ReadItemsData(str, _parseBlock), new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = maxDegree,
CancellationToken = cancellationToken
}); ;
_batchBlock.LinkTo(_writeBlock);
var logFiles = GetLogFiles();
foreach (var logFile in logFiles)
await StartReaderAsync(logFile, cancellationToken);
if (_liveMode)
StartLogFilesWatcher();
await _writeBlock.Completion;
}
private async Task StartReaderAsync(string logPath, CancellationToken cancellationToken = default)
{
if (!_logFiles.Contains(logPath))
{
await SendDataAsync(_readBlock, logPath);
_logger.LogInformation($"Log reader for \"{logPath}\" is started");
}
}
private string[] GetLogFiles()
{
return Directory.GetFiles(_logFolder, "*.log", SearchOption.AllDirectories);
}
private void ReadItemsData(string logPath, ITargetBlock<string> nextblock)
{
using var reader = new TechLogReader(logPath, _liveMode);
do
{
var itemData = reader.ReadItemData();
if (reader.Closed)
_logFiles.Remove(logPath);
if (itemData != null)
PostData(nextblock, itemData);
}
while (!reader.Closed);
_logger.LogInformation($"Log reader for \"{logPath}\" is stopped");
}
private void PostData<T>(ITargetBlock<T> block, T data)
{
while (true)
{
if (block.Post(data))
break;
}
}
private async Task SendDataAsync<T>(ITargetBlock<T> block, T data)
{
while (true)
{
if (await block.SendAsync(data))
break;
}
}
private void ParseItemData(string itemData, ITargetBlock<TechLogItem> nextblock)
{
var item = TechLogReader.ParseItemData(itemData);
PostData(nextblock, item);
}
private void StartLogFilesWatcher()
{
_logFilesWatcher = new FileSystemWatcher(_logFolder, "*.log")
{
NotifyFilter = NotifyFilters.CreationTime
};
_logFilesWatcher.Created += _logFilesWatcher_Created;
_logFilesWatcher.IncludeSubdirectories = true;
_logFilesWatcher.EnableRaisingEvents = true;
}
private async void _logFilesWatcher_Created(object sender, FileSystemEventArgs e)
{
if (e.ChangeType == WatcherChangeTypes.Created)
{
await StartReaderAsync(e.FullPath);
}
}
public void Dispose()
{
_logFilesWatcher?.Dispose();
}
}
}

44
OneSTools.TechLog.Exporter.Core/TechLogExporterService.cs
View File

@ -1,44 +0,0 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
namespace OneSTools.TechLog.Exporter.Core
{
public class TechLogExporterService : BackgroundService
{
private readonly ILogger<TechLogExporterService> _logger;
private readonly ITechLogExporter _techLogExporter;
public TechLogExporterService(ILogger<TechLogExporterService> logger, ITechLogExporter techLogExporter)
{
_logger = logger;
_techLogExporter = techLogExporter;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
try
{
await _techLogExporter.StartAsync(stoppingToken);
}
catch (Exception ex)
{
_logger.LogCritical(ex, "Failed to execute TechLogExporter");
}
}
public override void Dispose()
{
_techLogExporter?.Dispose();
base.Dispose();
}
}
}

16
OneSTools.TechLog.sln
View File

@ -5,9 +5,7 @@ VisualStudioVersion = 16.0.29318.209
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneSTools.TechLog", "OneSTools.TechLog\OneSTools.TechLog.csproj", "{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneSTools.TechLog.Exporter.Core", "OneSTools.TechLog.Exporter.Core\OneSTools.TechLog.Exporter.Core.csproj", "{02C37C84-911F-4725-A1EC-B81FF8F16227}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OneSTools.TechLog.Exporter.ClickHouse", "OneSTools.TechLog.Exporter.ClickHouse\OneSTools.TechLog.Exporter.ClickHouse.csproj", "{C610C51D-2604-4B3D-AFCA-4204A5821F88}"
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "OneSTools.TechLogTestApp", "OneSTools.TechLogTestApp\OneSTools.TechLogTestApp.csproj", "{E0AF3F33-EB90-4AD9-939E-63DFBA55202F}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
@ -19,14 +17,10 @@ Global
{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{BC6E4DCE-2722-4E6F-BCBC-8945DE1127DD}.Release|Any CPU.Build.0 = Release|Any CPU
{02C37C84-911F-4725-A1EC-B81FF8F16227}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{02C37C84-911F-4725-A1EC-B81FF8F16227}.Debug|Any CPU.Build.0 = Debug|Any CPU
{02C37C84-911F-4725-A1EC-B81FF8F16227}.Release|Any CPU.ActiveCfg = Release|Any CPU
{02C37C84-911F-4725-A1EC-B81FF8F16227}.Release|Any CPU.Build.0 = Release|Any CPU
{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C610C51D-2604-4B3D-AFCA-4204A5821F88}.Release|Any CPU.Build.0 = Release|Any CPU
{E0AF3F33-EB90-4AD9-939E-63DFBA55202F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E0AF3F33-EB90-4AD9-939E-63DFBA55202F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E0AF3F33-EB90-4AD9-939E-63DFBA55202F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E0AF3F33-EB90-4AD9-939E-63DFBA55202F}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

16
OneSTools.TechLog/AdditionalProperty.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace OneSTools.TechLog
{
public enum AdditionalProperty
{
None,
SqlHash,
CleanedSql,
FirstContextLine,
LastContextLine,
EndPosition
}
}

9
OneSTools.TechLog/LogReaderTimeoutException.cs Normal file
View File

@ -0,0 +1,9 @@
using System;
namespace OneSTools.TechLog
{
public class LogReaderTimeoutException : Exception
{
}
}

6
OneSTools.TechLog/OneSTools.TechLog.csproj
View File

@ -9,7 +9,7 @@
<PackageProjectUrl>https://github.com/akpaevj/OneSTools.TechLog</PackageProjectUrl>
<Copyright>Akpaev Evgeny</Copyright>
<Description>Библиотека для парсинга технологического журнала 1С</Description>
<Version>2.0.0</Version>
<Version>2.1.0</Version>
<LangVersion>8.0</LangVersion>
<PackageIcon>onestools_icon_nuget.png</PackageIcon>
<PackageLicenseExpression></PackageLicenseExpression>
@ -33,4 +33,8 @@
</None>
</ItemGroup>
<ItemGroup>
<PackageReference Include="System.Threading.Tasks.Dataflow" Version="5.0.0" />
</ItemGroup>
</Project>

45
OneSTools.TechLog/StreamReaderExtensions.cs Normal file
View File

@ -0,0 +1,45 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Text;
namespace OneSTools.TechLog
{
internal static class StreamReaderExtensions
{
readonly static FieldInfo charPosField = typeof(StreamReader).GetField("_charPos", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
readonly static FieldInfo byteLenField = typeof(StreamReader).GetField("_byteLen", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
readonly static FieldInfo charBufferField = typeof(StreamReader).GetField("_charBuffer", BindingFlags.NonPublic | BindingFlags.Instance | BindingFlags.DeclaredOnly);
public static long GetPosition(this StreamReader reader)
{
// shift position back from BaseStream.Position by the number of bytes read
// into internal buffer.
int byteLen = (int)byteLenField.GetValue(reader);
var position = reader.BaseStream.Position - byteLen;
// if we have consumed chars from the buffer we need to calculate how many
// bytes they represent in the current encoding and add that to the position.
int charPos = (int)charPosField.GetValue(reader);
if (charPos > 0)
{
var charBuffer = (char[])charBufferField.GetValue(reader);
var encoding = reader.CurrentEncoding;
var bytesConsumed = encoding.GetBytes(charBuffer, 0, charPos).Length;
position += bytesConsumed;
}
return position;
}
public static void SetPosition(this StreamReader reader, long position)
{
reader.DiscardBufferedData();
reader.BaseStream.Seek(position, SeekOrigin.Begin);
if (reader.BaseStream.Position != position)
throw new Exception("Couldn't set the stream position");
}
}
}

38
OneSTools.TechLog/StringBuilderExtensions.cs Normal file
View File

@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Linq;
namespace OneSTools.TechLog
{
public static class StringBuilderExtensions
{
public static int IndexOf(this StringBuilder stringBuilder, char value)
{
return IndexOf(stringBuilder, value, 0);
}
public static int IndexOf(this StringBuilder stringBuilder, char value, int startIndex)
{
for (int i = startIndex; i < stringBuilder.Length; i++)
if (stringBuilder[i] == value)
return i;
return -1;
}
public static int IndexOfAny(this StringBuilder stringBuilder, char[] values)
{
return IndexOfAny(stringBuilder, values, 0);
}
public static int IndexOfAny(this StringBuilder stringBuilder, char[] values, int startIndex)
{
for (int i = startIndex; i < stringBuilder.Length; i++)
if (values.Contains(stringBuilder[i]))
return i;
return -1;
}
}
}

346
OneSTools.TechLog/TechLogFileReader.cs Normal file
View File

@ -0,0 +1,346 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Security.Cryptography;
namespace OneSTools.TechLog
{
public class TechLogFileReader : IDisposable
{
private readonly string _fileDateTime;
private readonly AdditionalProperty _additionalProperty;
private FileStream _fileStream;
private StreamReader _streamReader;
private readonly StringBuilder _data = new StringBuilder();
private long _lastEventEndPosition = -1;
private bool disposedValue;
public string LogPath { get; private set; }
public long Position
{
get => _streamReader.GetPosition();
set => _streamReader.SetPosition(value);
}
public TechLogFileReader(string logPath, AdditionalProperty additionalProperty)
{
LogPath = logPath;
_additionalProperty = additionalProperty;
var fileName = Path.GetFileNameWithoutExtension(LogPath);
_fileDateTime = "20" +
fileName.Substring(0, 2) +
"-" +
fileName.Substring(2, 2) +
"-" +
fileName.Substring(4, 2) +
" " +
fileName.Substring(6, 2);
}
public TechLogItem ReadNextItem(CancellationToken cancellationToken = default)
{
InitializeStream();
var rawItem = ReadRawItem(cancellationToken);
if (rawItem == null)
return null;
return ParseRawItem(rawItem, cancellationToken);
}
private string ReadRawItem(CancellationToken cancellationToken = default)
{
InitializeStream();
string line = null;
while (!cancellationToken.IsCancellationRequested)
{
line = _streamReader.ReadLine();
// This is the end of the event or the end of the stream
if (line is null || _data.Length > 0 && Regex.IsMatch(line, @"^\d\d:\d\d\.", RegexOptions.Compiled))
break;
else
{
_data.AppendLine(line);
_lastEventEndPosition = Position;
}
}
if (_data.Length == 0)
return null;
else
{
var result = $"{_fileDateTime}:{_data}";
_data.Clear();
if (line != null)
_data.AppendLine(line);
return result;
}
}
private TechLogItem ParseRawItem(string rawItem, CancellationToken cancellationToken = default)
{
var item = new TechLogItem();
int startPosition = 0;
var dtd = ReadNextPropertyWithoutName(rawItem, ref startPosition, ',');
var dtdLength = dtd.Length;
var dtEndIndex = dtd.LastIndexOf('-');
item.TrySetPropertyValue("DateTime", dtd.Substring(0, dtEndIndex));
startPosition -= dtdLength - dtEndIndex;
item.TrySetPropertyValue("Duration", ReadNextPropertyWithoutName(rawItem, ref startPosition, ','));
item.TrySetPropertyValue("Event", ReadNextPropertyWithoutName(rawItem, ref startPosition, ','));
item.TrySetPropertyValue("Level", ReadNextPropertyWithoutName(rawItem, ref startPosition, ','));
while (!cancellationToken.IsCancellationRequested)
{
var (Name, Value) = ReadNextProperty(rawItem, ref startPosition);
if (string.IsNullOrEmpty(Name))
break;
// Property with the same name already can exist, so we have to get a new name for the value
if (!item.TrySetPropertyValue(Name, Value))
item.TrySetPropertyValue(GetPropertyName(item, Name, 0), Value);
if (startPosition >= rawItem.Length)
break;
}
SetAdditionalProperties(item);
return item;
}
private bool NeedAdditionalProperty(AdditionalProperty additionalProperty)
=> (_additionalProperty & additionalProperty) == additionalProperty;
private string GetPropertyName(TechLogItem item, string name, int number = 0)
{
var currentName = $"{name}{number}";
if (!item.HasProperty(currentName))
return currentName;
else
return GetPropertyName(item, name, number + 1);
}
private string ReadNextPropertyWithoutName(string strData, ref int startPosition, char delimiter = ',')
{
var endPosition = strData.IndexOf(delimiter, startPosition);
var value = strData[startPosition..endPosition];
startPosition = endPosition + 1;
return value;
}
private (string Name, string Value) ReadNextProperty(string strData, ref int startPosition)
{
var equalPosition = strData.IndexOf('=', startPosition);
if (equalPosition < 0)
return ("", "");
var name = strData[startPosition..equalPosition];
startPosition = equalPosition + 1;
if (startPosition == strData.Length)
return (name, "");
var nextChar = strData[startPosition];
int endPosition;
switch (nextChar)
{
case '\'':
endPosition = strData.IndexOf('\'', startPosition + 1);
break;
case ',':
startPosition++;
return (name, "");
case '"':
endPosition = strData.IndexOf('"', startPosition + 1);
break;
default:
endPosition = strData.IndexOf(',', startPosition);
break;
}
if (endPosition < 0)
endPosition = strData.Length;
var value = strData[startPosition..endPosition];
startPosition = endPosition + 1;
return (name, value.Trim(new char[] { '\'', '"' }).Trim());
}
private void SetAdditionalProperties(TechLogItem item)
{
TrySetCleanSqlProperty(item);
TrySetSqlHashProperty(item);
TrySetFirstContextLineProperty(item);
TrySetLastContextLineProperty(item);
if (NeedAdditionalProperty(AdditionalProperty.EndPosition))
{
item.EndPosition = _lastEventEndPosition;
_lastEventEndPosition = Position;
}
}
private bool TrySetCleanSqlProperty(TechLogItem item)
{
if (NeedAdditionalProperty(AdditionalProperty.CleanedSql) && item.TryGetPropertyValue("Sql", out var sql))
{
item["CleanSql"] = ClearSql(sql);
return true;
}
return false;
}
private string ClearSql(string data)
{
var dataSpan = data.AsSpan();
// Remove parameters
int startIndex = dataSpan.IndexOf("sp_executesql", StringComparison.OrdinalIgnoreCase);
if (startIndex < 0)
startIndex = 0;
else
startIndex += 16;
int e1 = dataSpan.IndexOf("', N'@P", StringComparison.OrdinalIgnoreCase);
if (e1 < 0)
e1 = dataSpan.Length;
var e2 = dataSpan.IndexOf("p_0:", StringComparison.OrdinalIgnoreCase);
if (e2 < 0)
e2 = dataSpan.Length;
var endIndex = Math.Min(e1, e2);
// Remove temp table names, parameters and guids
var result = Regex.Replace(dataSpan[startIndex..endIndex].ToString(), @"(#tt\d+|@P\d+|\d{8}-\d{4}-\d{4}-\d{4}-\d{12})", "{RD}", RegexOptions.ExplicitCapture);
return result;
}
private bool TrySetSqlHashProperty(TechLogItem item)
{
bool needCalculateHash = NeedAdditionalProperty(AdditionalProperty.SqlHash);
if (!item.HasProperty("CleanSql"))
needCalculateHash = TrySetCleanSqlProperty(item);
if (needCalculateHash && item.TryGetPropertyValue("CleanSql", out var cleanedSql))
item["SqlHash"] = GetSqlHash(cleanedSql);
return needCalculateHash;
}
private string GetSqlHash(string cleanedSql)
{
using var cp = MD5.Create();
var src = Encoding.UTF8.GetBytes(cleanedSql);
var res = cp.ComputeHash(src);
return BitConverter.ToString(res).Replace("-", "");
}
private bool TrySetFirstContextLineProperty(TechLogItem item)
{
if (NeedAdditionalProperty(AdditionalProperty.FirstContextLine) && item.TryGetPropertyValue("Context", out var context))
{
item["FirstContextLine"] = GetFirstContextLine(context);
return true;
}
return false;
}
private string GetFirstContextLine(string context)
{
var index = context.IndexOf("\n");
if (index > 0)
return context[0..index];
else
return context;
}
private bool TrySetLastContextLineProperty(TechLogItem item)
{
if (NeedAdditionalProperty(AdditionalProperty.LastContextLine) && item.TryGetPropertyValue("Context", out var context))
{
item["LastContextLine"] = GetLastContextLine(context);
return true;
}
return false;
}
private string GetLastContextLine(string context)
{
var index = context.LastIndexOf("\t");
if (index > 0)
return context[(index + 1)..];
else
return context;
}
private void InitializeStream()
{
if (_fileStream == null)
{
_fileStream = new FileStream(LogPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
_streamReader = new StreamReader(_fileStream);
}
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
}
_streamReader?.Dispose();
disposedValue = true;
}
}
~TechLogFileReader()
{
Dispose(disposing: false);
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

166
OneSTools.TechLog/TechLogFolderReader.cs Normal file
View File

@ -0,0 +1,166 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
namespace OneSTools.TechLog
{
public class TechLogFolderReader : IDisposable
{
private readonly TechLogFolderReaderSettings _settings;
private TechLogFileReader _logReader;
private ManualResetEvent _logChangedCreated;
private FileSystemWatcher _logfilesWatcher;
private bool disposedValue;
public TechLogFolderReader(TechLogFolderReaderSettings settings)
{
_settings = settings;
}
public TechLogItem ReadNextItem(CancellationToken cancellationToken = default)
{
if (_logReader is null)
SetNextLogReader();
if (_settings.LiveMode && _logfilesWatcher is null)
StartLogFilesWatcher();
TechLogItem item = null;
while (!cancellationToken.IsCancellationRequested)
{
try
{
item = _logReader.ReadNextItem(cancellationToken);
}
catch (ObjectDisposedException)
{
item = null;
_logReader = null;
break;
}
catch (Exception ex)
{
throw ex;
}
if (item == null)
{
var newReader = SetNextLogReader();
if (_settings.LiveMode)
{
if (!newReader)
{
_logChangedCreated.Reset();
var waitHandle = WaitHandle.WaitAny(new WaitHandle[] { _logChangedCreated, cancellationToken.WaitHandle }, _settings.ReadingTimeout * 1000);
if (_settings.ReadingTimeout != Timeout.Infinite && waitHandle == WaitHandle.WaitTimeout)
throw new LogReaderTimeoutException();
_logChangedCreated.Reset();
}
}
else
{
if (!newReader)
break;
}
}
else
break;
}
return item;
}
private bool SetNextLogReader()
{
var currentReaderLastWriteDateTime = DateTime.MinValue;
if (_logReader != null)
currentReaderLastWriteDateTime = new FileInfo(_logReader.LogPath).LastWriteTime;
var filesDateTime = new List<(string FilePath, DateTime LastWriteTime)>();
var files = Directory.GetFiles(_settings.Folder, "*.log");
foreach (var file in files)
{
if (_logReader != null)
{
if (_logReader.LogPath != file)
filesDateTime.Add((file, new FileInfo(file).LastWriteTime));
}
else
filesDateTime.Add((file, new FileInfo(file).LastWriteTime));
}
var orderedFiles = filesDateTime.OrderBy(c => c.LastWriteTime).ToList();
var (FilePath, LastWriteTime) = orderedFiles.FirstOrDefault(c => c.LastWriteTime > currentReaderLastWriteDateTime);
if (string.IsNullOrEmpty(FilePath))
return false;
else
{
_logReader?.Dispose();
_logReader = new TechLogFileReader(FilePath, _settings.AdditionalProperty);
return true;
}
}
private void StartLogFilesWatcher()
{
_logChangedCreated = new ManualResetEvent(false);
_logfilesWatcher = new FileSystemWatcher(_settings.Folder, "*.log")
{
NotifyFilter = NotifyFilters.CreationTime | NotifyFilters.LastWrite
};
_logfilesWatcher.Changed += LgpFilesWatcher_Event;
_logfilesWatcher.Created += LgpFilesWatcher_Event;
_logfilesWatcher.EnableRaisingEvents = true;
}
private void LgpFilesWatcher_Event(object sender, FileSystemEventArgs e)
{
if (e.ChangeType == WatcherChangeTypes.Created || e.ChangeType == WatcherChangeTypes.Changed)
_logChangedCreated.Set();
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
}
_logReader?.Dispose();
_logChangedCreated?.Dispose();
_logfilesWatcher?.Dispose();
disposedValue = true;
}
}
~TechLogFolderReader()
{
Dispose(disposing: false);
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

10
OneSTools.TechLog/TechLogFolderReaderSettings.cs Normal file
View File

@ -0,0 +1,10 @@
namespace OneSTools.TechLog
{
public class TechLogFolderReaderSettings
{
public string Folder { get; set; } = "";
public AdditionalProperty AdditionalProperty { get; set; } = AdditionalProperty.None;
public bool LiveMode { get; set; } = false;
public int ReadingTimeout { get; set; } = 1;
}
}

30
OneSTools.TechLog/TechLogItem.cs Normal file
View File

@ -0,0 +1,30 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace OneSTools.TechLog
{
public class TechLogItem
{
private readonly Dictionary<string, string> _properties = new Dictionary<string, string>();
public string this[string property]
{
get => _properties[property];
set => _properties[property] = value;
}
public long EndPosition { get; set; }
public IEnumerable<string> Properties => _properties.Keys;
public bool HasProperty(string property)
=> _properties.ContainsKey(property);
public bool TryGetPropertyValue(string property, out string value)
=> _properties.TryGetValue(property, out value);
public bool TrySetPropertyValue(string property, string value)
=> _properties.TryAdd(property, value);
}
}

362
OneSTools.TechLog/TechLogReader.cs
View File

@ -1,312 +1,160 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using System.Collections.Concurrent;
using System.Diagnostics;
namespace OneSTools.TechLog
{
public class TechLogReader : IDisposable
{
private string _logPath;
private string _fileDateTime;
private readonly bool _liveMode;
private FileStream _fileStream;
private StreamReader _streamReader;
private StringBuilder _currentData = new StringBuilder();
private ManualResetEvent _logFileChanged;
private ManualResetEvent _logFileDeleted;
private FileSystemWatcher _logFileWatcher;
private readonly TechLogReaderSettings _settings;
private ActionBlock<string> _readBlock;
private TransformBlock<TechLogItem, TechLogItem> _tunnelBlock;
private CancellationToken _cancellationToken;
private Timer _flushTimer;
private FileSystemWatcher _logFoldersWatcher;
private bool disposedValue;
public bool Closed { get; private set; } = true;
public TechLogReader(string logPath, bool liveMode = false)
public TechLogReader(TechLogReaderSettings settings)
{
_logPath = logPath;
_liveMode = liveMode;
var fileName = Path.GetFileNameWithoutExtension(_logPath);
_fileDateTime = "20" +
fileName.Substring(0, 2) +
"-" +
fileName.Substring(2, 2) +
"-" +
fileName.Substring(4, 2) +
" " +
fileName.Substring(6, 2);
_settings = settings;
}
public Dictionary<string, string> ReadNextItem(CancellationToken cancellationToken = default)
public async Task ReadAsync(Action<TechLogItem[]> processor, CancellationToken cancellationToken = default)
{
Initialize();
_cancellationToken = cancellationToken;
var itemData = ReadItemData(cancellationToken);
var processorBlock = new ActionBlock<TechLogItem[]>(processor, new ExecutionDataflowBlockOptions() { BoundedCapacity = _settings.BatchFactor });
if (itemData == null)
return null;
var batchBlock = new BatchBlock<TechLogItem>(_settings.BatchSize);
batchBlock.LinkTo(processorBlock, new DataflowLinkOptions() { PropagateCompletion = true });
return ParseItemData(itemData, cancellationToken);
}
_readBlock = new ActionBlock<string>(logFolder => ReadLogFolder(logFolder, batchBlock, cancellationToken), new ExecutionDataflowBlockOptions() { CancellationToken = cancellationToken });
public string ReadItemData(CancellationToken cancellationToken = default)
{
Initialize();
var currentLine = "";
while (!cancellationToken.IsCancellationRequested)
if (_settings.LiveMode)
{
currentLine = _streamReader.ReadLine();
var timeoutMs = _settings.ReadingTimeout * 1000;
if (currentLine == null)
_flushTimer = new Timer(_ => batchBlock.TriggerBatch(), null, timeoutMs, Timeout.Infinite);
_tunnelBlock = new TransformBlock<TechLogItem, TechLogItem>(item =>
{
if (_currentData.Length > 0)
break;
else
{
if (_liveMode)
{
var handles = new WaitHandle[]
{
_logFileChanged,
_logFileDeleted,
cancellationToken.WaitHandle
};
_flushTimer.Change(timeoutMs, Timeout.Infinite);
var index = WaitHandle.WaitAny(handles);
return item;
});
_tunnelBlock.LinkTo(batchBlock, new DataflowLinkOptions() { PropagateCompletion = true });
if (index == 1 || index == 2 || index == WaitHandle.WaitTimeout) // File is deleted / reader stopped / timeout
{
Dispose();
Closed = true;
return null;
}
else if (index == 0) // File is changed, continue reading
{
_logFileChanged.Reset();
continue;
}
}
}
}
if (currentLine == null || _currentData.Length > 0 && Regex.IsMatch(currentLine, @"^\d\d:\d\d\.", RegexOptions.Compiled))
break;
else
_currentData.AppendLine(currentLine);
_ = _readBlock.Completion.ContinueWith(c => _tunnelBlock.Complete());
}
var _strData = _currentData.ToString().Trim();
_currentData.Clear();
if (currentLine != null)
_currentData.AppendLine(currentLine);
return _fileDateTime + ":" + _strData;
}
public static Dictionary<string, string> ParseItemData(string itemData, CancellationToken cancellationToken = default)
{
var item = new Dictionary<string, string>();
int startPosition = 0;
var dtd = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
var dtdLength = dtd.Length;
var dtEndIndex = dtd.LastIndexOf('-');
item["DateTime"] = dtd.Substring(0, dtEndIndex);
startPosition -= dtdLength - dtEndIndex;
item["Duration"] = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
item["Event"] = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
item["Level"] = ReadNextPropertyWithoutName(itemData, ref startPosition, ',');
while (!cancellationToken.IsCancellationRequested)
{
var (Name, Value) = ReadNextProperty(itemData, ref startPosition);
if (item.ContainsKey(Name))
{
item.Add(GetPropertyName(item, Name, 0), Value);
}
else
item.Add(Name, Value);
if (startPosition >= itemData.Length)
break;
}
return item;
}
private static string GetPropertyName(Dictionary<string, string> item, string name, int number = 0)
{
var currentName = $"{name}{number}";
if (!item.ContainsKey(currentName))
return currentName;
else
return GetPropertyName(item, name, number + 1);
_ = _readBlock.Completion.ContinueWith(c => batchBlock.Complete());
foreach (var logFolder in GetExistingLogFolders())
Post(logFolder, _readBlock, cancellationToken);
if (_settings.LiveMode)
InitializeWatcher();
else
_readBlock.Complete();
await processorBlock.Completion;
}
private static string ReadNextPropertyWithoutName(string strData, ref int startPosition, char delimiter = ',')
private void ReadLogFolder(string logFolder, ITargetBlock<TechLogItem> nextBlock, CancellationToken cancellationToken = default)
{
var endPosition = strData.IndexOf(delimiter, startPosition);
var value = strData.Substring(startPosition, endPosition - startPosition);
startPosition = endPosition + 1;
return value;
}
private static (string Name, string Value) ReadNextProperty(string strData, ref int startPosition)
{
var equalPosition = strData.IndexOf('=', startPosition);
var name = strData.Substring(startPosition, equalPosition - startPosition);
startPosition = equalPosition + 1;
if (startPosition == strData.Length)
return (name, "");
var nextChar = strData[startPosition];
int endPosition;
switch (nextChar)
var settings = new TechLogFolderReaderSettings
{
case '\'':
endPosition = strData.IndexOf("\',", startPosition);
break;
case ',':
startPosition++;
return (name, "");
case '"':
endPosition = strData.IndexOf("\",", startPosition);
break;
default:
endPosition = strData.IndexOf(',', startPosition);
break;
}
Folder = logFolder,
AdditionalProperty = _settings.AdditionalProperty,
LiveMode = _settings.LiveMode,
ReadingTimeout = _settings.ReadingTimeout
};
if (endPosition < 0)
endPosition = strData.Length;
using var reader = new TechLogFolderReader(settings);
var value = strData.Substring(startPosition, endPosition - startPosition);
startPosition = endPosition + 1;
TechLogItem item = null;
return (name, value.Trim(new char[] { '\'', '"' }).Trim());
}
private static string GetCleanedSql(string data)
{
throw new NotImplementedException();
//// Remove paramaters
//int startIndex = data.IndexOf("sp_executesql", StringComparison.OrdinalIgnoreCase);
//if (startIndex < 0)
// startIndex = 0;
//else
// startIndex += 16;
//int e1 = data.IndexOf("', N'@P", StringComparison.OrdinalIgnoreCase);
//if (e1 < 0)
// e1 = data.Length;
//var e2 = data.IndexOf("p_0:", StringComparison.OrdinalIgnoreCase);
//if (e2 < 0)
// e2 = data.Length;
//var endIndex = Math.Min(e1, e2);
//StringBuilder result = new StringBuilder(data[startIndex..endIndex]);
//// Remove temp table names
//while (true)
//{
// var ttsi = result.IndexOf("#tt");
// if (ttsi >= 0)
// {
// var ttsei = ttsi + 2;
// // Read temp table number
// while (true)
// {
// if (char.IsDigit(result[ttsei]))
// ttsei++;
// else
// break;
// }
// }
// else
// break;
//}
//return result.ToString();
}
private static string GetSqlHash(string sql)
{
throw new NotImplementedException();
}
private void Initialize()
{
InitializeStream();
InitializeWatcher();
}
private void InitializeStream()
{
if (_fileStream == null)
while (!cancellationToken.IsCancellationRequested)
{
_fileStream = new FileStream(_logPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
_streamReader = new StreamReader(_fileStream);
try
{
item = reader.ReadNextItem(cancellationToken);
}
catch (LogReaderTimeoutException)
{
continue;
}
catch
{
throw;
}
Closed = false;
if (item is null)
break;
else
Post(item, nextBlock, cancellationToken);
}
}
private static void Post<T>(T data, ITargetBlock<T> block, CancellationToken cancellationToken = default)
{
while (!cancellationToken.IsCancellationRequested)
{
if (block.Post(data))
break;
}
}
private string[] GetExistingLogFolders()
=> Directory.GetDirectories(_settings.LogFolder);
private void InitializeWatcher()
{
if (_liveMode && _logFileWatcher == null)
_logFoldersWatcher = new FileSystemWatcher(_settings.LogFolder)
{
_logFileChanged = new ManualResetEvent(false);
_logFileDeleted = new ManualResetEvent(false);
NotifyFilter = NotifyFilters.CreationTime | NotifyFilters.LastWrite | NotifyFilters.FileName
};
_logFoldersWatcher.Created += LogFileWatcherEvent;
_logFoldersWatcher.EnableRaisingEvents = true;
}
_logFileWatcher = new FileSystemWatcher(Path.GetDirectoryName(_logPath), Path.GetFileName(_logPath))
private void LogFileWatcherEvent(object sender, FileSystemEventArgs e)
{
// new log folder has been created
if (e.ChangeType == WatcherChangeTypes.Created && File.GetAttributes(e.FullPath).HasFlag(FileAttributes.Directory))
Post(e.FullPath, _readBlock, _cancellationToken);
}
protected virtual void Dispose(bool disposing)
{
if (!disposedValue)
{
if (disposing)
{
NotifyFilter = NotifyFilters.CreationTime | NotifyFilters.LastWrite | NotifyFilters.FileName
};
_logFileWatcher.Changed += _logFileWatcher_Changed;
_logFileWatcher.Deleted += _logFileWatcher_Deleted;
_logFileWatcher.EnableRaisingEvents = true;
}
_logFoldersWatcher?.Dispose();
disposedValue = true;
}
}
private void _logFileWatcher_Changed(object sender, FileSystemEventArgs e)
~TechLogReader()
{
if (e.ChangeType == WatcherChangeTypes.Changed)
_logFileChanged.Set();
}
private void _logFileWatcher_Deleted(object sender, FileSystemEventArgs e)
{
if (e.ChangeType == WatcherChangeTypes.Deleted)
_logFileDeleted.Set();
Dispose(disposing: false);
}
public void Dispose()
{
_streamReader?.Dispose();
_fileStream = null;
_logFileWatcher?.Dispose();
_logFileChanged?.Dispose();
_logFileDeleted?.Dispose();
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}

12
OneSTools.TechLog/TechLogReaderSettings.cs Normal file
View File

@ -0,0 +1,12 @@
namespace OneSTools.TechLog
{
public class TechLogReaderSettings
{
public string LogFolder { get; set; } = "";
public int BatchSize { get; set; } = 1000;
public int BatchFactor { get; set; } = 2;
public AdditionalProperty AdditionalProperty { get; set; } = AdditionalProperty.None;
public bool LiveMode { get; set; } = false;
public int ReadingTimeout { get; set; } = 1;
}
}

26
OneSTools.TechLogTest/OneSTools.TechLogTest.csproj Normal file
View File

@ -0,0 +1,26 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="coverlet.collector" Version="1.3.0">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\OneSTools.TechLog\OneSTools.TechLog.csproj" />
</ItemGroup>
</Project>

42
OneSTools.TechLogTest/TechLogFolderReaderTest.cs Normal file
View File

@ -0,0 +1,42 @@
using OneSTools.TechLog;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using Xunit;
namespace OneSTools.TechLogTest
{
public class TechLogFolderReaderTest
{
[Fact]
public async Task ReadAsyncTest()
{
// Arrange
var folderReaderSettings = new TechLogReaderSettings()
{
LogFolder = @"C:\Users\akpaev.e.ENTERPRISE\Desktop\TechLog",
//AdditionalProperty = AdditionalProperty.CleanedSql | AdditionalProperty.FirstContextLine | AdditionalProperty.FirstContextLine | AdditionalProperty.LastContextLine,
BatchSize = 10,
BatchFactor = 3,
LiveMode = false
};
using var reader = new TechLogReader(folderReaderSettings);
var cts = new CancellationTokenSource();
int count = 0;
// Act
await reader.ReadAsync(batch =>
{
count += batch.Length;
}, cts.Token);
// Assign
}
}
}

12
OneSTools.TechLogTestApp/OneSTools.TechLogTestApp.csproj Normal file
View File

@ -0,0 +1,12 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\OneSTools.TechLog\OneSTools.TechLog.csproj" />
</ItemGroup>
</Project>

46
OneSTools.TechLogTestApp/Program.cs Normal file
View File

@ -0,0 +1,46 @@
using OneSTools.TechLog;
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
namespace OneSTools.TechLogTestApp
{
class Program
{
static async Task Main(string[] args)
{
// Arrange
var folderReaderSettings = new TechLogReaderSettings()
{
LogFolder = @"C:\Users\akpaev.e.ENTERPRISE\Desktop\TechLog",
AdditionalProperty = AdditionalProperty.SqlHash | AdditionalProperty.FirstContextLine | AdditionalProperty.LastContextLine | AdditionalProperty.EndPosition,
BatchSize = 100,
BatchFactor = 2,
LiveMode = false
};
using var reader = new TechLogReader(folderReaderSettings);
var cts = new CancellationTokenSource();
var stopwatch = Stopwatch.StartNew();
int count = 0;
// Act
await reader.ReadAsync(batch =>
{
count += batch.Length;
}, cts.Token);
stopwatch.Stop();
Console.WriteLine($"Read {count} items for {stopwatch.ElapsedMilliseconds / 1000} s.");
Console.ReadKey();
// Assign
}
}
}