git_bsmd/bsmd.Tool/Echolot.cs

// Copyright (c) 2020- schick Informatik
// Description: The purpose of this tool is to evaluate files sent both through HIS-Nord and dbh
// to evaluate how many classes were sent at what time and by whom to the purpose of improved employee
// time planning 


using ClosedXML.Excel;
using log4net;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;

namespace bsmd.Tool
{
    internal static class Echolot
    {

        private static readonly ILog _log = LogManager.GetLogger(typeof(Echolot));

        static readonly HashSet<string> ValidGroupingKeys = new HashSet<string>
        {
            "VISIT",
            "TRANSIT",
            "NOA_NOD",
            "NOANOD",
            "ATA",
            "ATD",
            "SEC",
            "POBA",
            "POBD",
            "NAME",
            "TIEFA",
            "TIEFD",
            "BKRA",
            "BKRD",
            "STAT",
            "LADG",
            "INFO",
            "SERV",
            "PRE72H",
            "MDH",
            "WAS",
            "CREWA",
            "PASA",
            "BPOL",
            "TOWA",
            "TOWD",
            "HAZA",
            "HAZD",
            "AGNT",
            "STO",
            "CREWD",
            "PASD",
            "WAS_RCPT"
        };

        static readonly HashSet<string> IgnoreGroupingKeys = new HashSet<string>
        {
            "VISIT",
            "TRANSIT",
            "ATA",
            "ATD"
        };

        internal static void Evaluate(string outputFolder, int maxThreads)
        {
            DateTime executionTime = DateTime.Now;

            #region first scan: dbh files

            string inputFolder = Properties.Settings.Default.DBH_Folder;
            var files = Directory.GetFiles(inputFolder, "*.xml");
            var results = new ConcurrentBag<ResultRow>();

            Parallel.ForEach(files, new ParallelOptions { MaxDegreeOfParallelism = maxThreads }, file =>
            {
                try
                {
                    var doc = XDocument.Load(file);

                    // Look for a valid grouping key at the root level
                    var groupingElem = doc.Root.Elements()
                        .FirstOrDefault(x => ValidGroupingKeys.Contains(x.Name.LocalName) && !IgnoreGroupingKeys.Contains(x.Name.LocalName));

                    if (groupingElem == null)
                    {
                        _log.InfoFormat("skipping {0}", file);
                        return; // Skip file
                    }
                        
                    var lastName = doc.Descendants("RPLastName").FirstOrDefault()?.Value?.Trim();
                    var firstName = doc.Descendants("RPFirstName").FirstOrDefault()?.Value?.Trim();
                    var timestampStr = doc.Descendants("Timestamp").FirstOrDefault()?.Value?.Trim();                                        

                    DateTime timestamp = DateTime.Parse(timestampStr);

                    results.Add(new ResultRow
                    {
                        FirstName = firstName,
                        LastName = lastName,
                        Timestamp = timestamp,
                        WeekStart = GetWeekStart(timestamp, executionTime.DayOfWeek),
                        Provider = "DBH"
                    });
                }
                catch (Exception ex)
                {
                    _log.Error(ex.ToString());
                }
            });

            #endregion

            #region second scan: his-nord files

            var inputFolder2 = Properties.Settings.Default.HISNORD_Folder;
            var files2 = Directory.GetFiles(inputFolder2, "*.xml");            

            Parallel.ForEach(files2, new ParallelOptions { MaxDegreeOfParallelism = maxThreads }, file =>
            {
                try
                {
                    var doc = XDocument.Load(file);

                    var match = Regex.Match(file, @"-([A-Z0-9_]+)\.xml$", RegexOptions.None);

                    string key = "";
                    if (match.Success)
                    {
                        key = match.Groups[1].Value;                        
                    }
                    
                    if((key.Length == 0) || IgnoreGroupingKeys.Contains(key))
                    {
                        _log.InfoFormat("skipping {0}", file);
                        return; // Skip file
                    }                    

                    var username = doc.Descendants("firstname").FirstOrDefault()?.Value?.Trim();
                    if(username == null)
                    {
                        _log.WarnFormat("Username not found in file {0}", file);
                        return;
                    }
                    var splitname = username.Split(' ');
                    var lastName = splitname[1].Trim();
                    var firstName = splitname[0].Trim();

                    DateTime timestamp = File.GetCreationTime(file);

                    results.Add(new ResultRow
                    {
                        FirstName = firstName,
                        LastName = lastName,
                        Timestamp = timestamp,
                        WeekStart = GetWeekStart(timestamp, executionTime.DayOfWeek),
                        Provider = "HIS-NORD"
                    });
                }
                catch (Exception ex)
                {
                    _log.Error(ex.ToString());
                }
            });


            #endregion

            var grouped = results
                .GroupBy(r => r.WeekStart)
                .OrderBy(g => g.Key);
            

            // Write Excel
            string excelFile = Path.Combine(outputFolder, $"echolot_{executionTime:yyyyMMdd_HHmmss}.xlsx");
            using (var workbook = new XLWorkbook())
            {
                foreach (var weekGroup in grouped)
                {
                    var ws = workbook.Worksheets.Add(weekGroup.Key.ToString("yyyy-MM-dd"));
                    ws.Cell(1, 1).Value = "Firstname";
                    ws.Cell(1, 2).Value = "Lastname";
                    ws.Cell(1, 3).Value = "Count";
                    int row = 2;

                    var orderedGroups = weekGroup
                        .GroupBy(x => new { x.FirstName, x.LastName })
                        .OrderByDescending(g => g.Count()); // Use OrderBy for ascending


                    foreach (var nameGroup in orderedGroups)
                    {
                        ws.Cell(row, 1).Value = nameGroup.Key.FirstName;
                        ws.Cell(row, 2).Value = nameGroup.Key.LastName;
                        ws.Cell(row, 3).Value = nameGroup.Count();
                        row++;
                    }
                }
                workbook.SaveAs(excelFile);
            }
        }

        static DateTime GetWeekStart(DateTime date, DayOfWeek weekStart)
        {
            int diff = (7 + (date.DayOfWeek - weekStart)) % 7;
            return date.Date.AddDays(-1 * diff);
        }

        class ResultRow
        {
            public string FirstName { get; set; }
            public string LastName { get; set; }
            public DateTime Timestamp { get; set; }
            public DateTime WeekStart { get; set; }
            public string Provider {  get; set; }
        }

    }
}