From 1bbccadd17ecb33cba4ae533e1082c9b5e80378d Mon Sep 17 00:00:00 2001 From: Daniel Schick Date: Fri, 31 Mar 2023 10:05:15 +0200 Subject: [PATCH] updated tool project LOCODE import part 1 --- bsmd.Tool/LocodeSQliteImport.cs | 140 ++++++++++++++++++++++++++++++++ bsmd.Tool/Options.cs | 12 ++- bsmd.Tool/Program.cs | 8 ++ bsmd.Tool/bsmd.Tool.csproj | 14 ++++ bsmd.Tool/bsmd.Tool.csproj.user | 2 +- bsmd.Tool/packages.config | 2 + 6 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 bsmd.Tool/LocodeSQliteImport.cs diff --git a/bsmd.Tool/LocodeSQliteImport.cs b/bsmd.Tool/LocodeSQliteImport.cs new file mode 100644 index 00000000..a0de0fd4 --- /dev/null +++ b/bsmd.Tool/LocodeSQliteImport.cs @@ -0,0 +1,140 @@ +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Data; +using System.Data.SQLite; +using log4net; + +namespace bsmd.Tool +{ + public static class LocodeSQliteImport + { + private static readonly ILog _log = LogManager.GetLogger(typeof(LocodeSQliteImport)); + + /// + /// Importer / Updater für CSV files, die im SVN unter \bsmd\nsw\Archiv abgelegt wurde + /// (offizielle UNLOCODEs).. + /// + /// + /// + public static void Import(string sqliteDBPath, string csvFilePath) + { + if (!File.Exists(sqliteDBPath)) throw new ArgumentException($"file {sqliteDBPath} does not exits"); + if (!File.Exists(csvFilePath)) throw new ArgumentException($"file {csvFilePath} does not exist"); + List currentLocodes = new List(); + + using (var connection = new SQLiteConnection($"Data Source={sqliteDBPath}")) + { + connection.Open(); + + SQLiteCommand lookupCmd = new SQLiteCommand(connection); + lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE"; + SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String); + lookupCmd.Parameters.Add(ccode); + SQLiteParameter lcode = new SQLiteParameter("@LCODE", DbType.String); + lookupCmd.Parameters.Add(lcode); + + SQLiteCommand insertCmd = new SQLiteCommand(connection); + insertCmd.CommandText = "INSERT INTO locodes (country_id, city_code, name, name_wo_diacritics, sub_div, port, rail_terminal, road_terminal, airport, postal_exchange_office, inland_clearance_depot, fixed_transport_functions, border_crossing_function, status, date, iata, coordinates, remarks) VALUES " + + "(@P1, @P2, @P3, @P4, @P5, @P6, @P7, @P8, @P9, @P10, @P11, @P12, @P13, @P14, @P15, @P16, @P17, @P18)"; + + + SQLiteCommand updateCmd = new SQLiteCommand(connection); + updateCmd.CommandText = "UPDATE locodes SET name = @P3, name_wo_diacritics = @P4, sub_div = @P5, port = @P6, rail_terminal = @P7, road_terminal = @P8, airport = @P9, postal_exchange_office = @P10, " + + "inland_clearance_depot = @P11, fixed_transport_functions = @P12, border_crossing_function = @P13, status = @P14, date = @P15, iata = @P16, coordinates = @P17, remarks = @P18 WHERE id = @ID"; + + SQLiteParameter p1 = new SQLiteParameter("@P1", DbType.Int32); + SQLiteParameter p2 = new SQLiteParameter("@P2", DbType.String); + SQLiteParameter p3 = new SQLiteParameter("@P3", DbType.String); + SQLiteParameter p4 = new SQLiteParameter("@P4", DbType.String); + SQLiteParameter p5 = new SQLiteParameter("@P5", DbType.String); + SQLiteParameter p6 = new SQLiteParameter("@P6", DbType.Boolean); + SQLiteParameter p7 = new SQLiteParameter("@P7", DbType.Boolean); + SQLiteParameter p8 = new SQLiteParameter("@P8", DbType.Boolean); + SQLiteParameter p9 = new SQLiteParameter("@P9", DbType.Boolean); + SQLiteParameter p10 = new SQLiteParameter("@P10", DbType.Boolean); + SQLiteParameter p11 = new SQLiteParameter("@P11", DbType.Boolean); + SQLiteParameter p12 = new SQLiteParameter("@P12", DbType.Boolean); + SQLiteParameter p13 = new SQLiteParameter("@P13", DbType.Boolean); + SQLiteParameter p14 = new SQLiteParameter("@P14", DbType.String); + SQLiteParameter p15 = new SQLiteParameter("@P15", DbType.String); + SQLiteParameter p16 = new SQLiteParameter("@P16", DbType.String); + SQLiteParameter p17 = new SQLiteParameter("@P17", DbType.String); + SQLiteParameter p18 = new SQLiteParameter("@P18", DbType.String); + + SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32); + + insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18}); + updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam }); + + string[] csvLines = File.ReadAllLines(csvFilePath); + int updateCnt = 0, insertCnt = 0; + + for(int i = 0; i < csvLines.Length; i++) + { + string line = csvLines[i]; + string[] elems = line.Split(','); + + string country = elems[1].Trim().Replace("\"", ""); + if (country.Length < 2) continue; + string code = elems[2].Trim().Replace("\"", ""); + if (code.Length < 3) continue; + + currentLocodes.Add(country + code); + + ccode.Value = country; + lcode.Value = code; + + // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm + + object lookupResult = lookupCmd.ExecuteScalar(); + if(lookupResult != DBNull.Value) + { + int lid = Convert.ToInt32(lookupResult); + // UPDATE entry + updateCnt++; + } + else + { + // CREATE new entry + insertCnt++; + + } + + + Console.Write($"\r({i}/{csvLines.Length})"); + } + + Console.Write("\n"); + Console.WriteLine($"{insertCnt} new entries, {updateCnt} updated"); + + // jetzt durch alle Ids in der DB laufen und mit dem Import vergleichen + List deleteIds = new List(); + SQLiteCommand cmd = new SQLiteCommand(connection); + cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id"; + SQLiteDataReader reader = cmd.ExecuteReader(); + while(reader.Read()) + { + string locode = reader.GetString(0) + reader.GetString(2); + if (!currentLocodes.Contains(locode)) + deleteIds.Add(reader.GetInt32(1)); + } + + Console.WriteLine($"deleting {deleteIds.Count} obsolete entries"); + + SQLiteCommand delCmd = new SQLiteCommand(connection); + delCmd.CommandText = "DELETE FROM locodes where id = @DELID"; + // diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen + foreach(int deleteId in deleteIds) + { + delCmd.Parameters.AddWithValue("@DELID", deleteId); + if (delCmd.ExecuteNonQuery() != 1) + _log.WarnFormat("{0} affected no rows", deleteId); + } + } + } + } +} diff --git a/bsmd.Tool/Options.cs b/bsmd.Tool/Options.cs index de08e6aa..026e703c 100644 --- a/bsmd.Tool/Options.cs +++ b/bsmd.Tool/Options.cs @@ -7,10 +7,7 @@ // // Copyright (c) 2015 Informatikbüro Daniel Schick. All rights reserved. -using System; -using System.Collections.Generic; using CommandLine; -using CommandLine.Text; namespace bsmd.Tool { @@ -25,6 +22,9 @@ namespace bsmd.Tool [Option('r', "checkrules", HelpText = "Use rule engine on message core")] public bool CheckRules { get; set; } + [Option("locodes", HelpText = "use this flag if you want to import locodes")] + public bool ImportLocodes { get; set; } + [Option('s', "staledays", Default = 30, HelpText ="Delete files older than X days")] public int StaleDays { get; set; } @@ -33,5 +33,11 @@ namespace bsmd.Tool [Option('x', "recursive", Default = false, HelpText ="Cleanup subdirectories recursively")] public bool CleanupRecursive { get; set; } + + [Option("csv", HelpText = "PATH to import LOCODE csv")] + public string LocodeCSV { get; set; } + + [Option("db", HelpText = "PATH to locode db (SQLITE)")] + public string LocodeDB { get; set; } } } diff --git a/bsmd.Tool/Program.cs b/bsmd.Tool/Program.cs index bcc81fff..4d8f4aa2 100644 --- a/bsmd.Tool/Program.cs +++ b/bsmd.Tool/Program.cs @@ -33,6 +33,13 @@ namespace bsmd.Tool { CleanupFiles.Cleanup(options.CleanupFolderRoot, options.StaleDays, options.CleanupRecursive); } + if(o.ImportLocodes) + { + if(!string.IsNullOrEmpty(o.LocodeCSV) && !string.IsNullOrEmpty(o.LocodeDB)) + { + LocodeSQliteImport.Import(o.LocodeDB, o.LocodeCSV); + } + } }); } catch (Exception ex) @@ -40,6 +47,7 @@ namespace bsmd.Tool log.Fatal(ex.Message); result = 1; } + Console.Read(); return result; } } diff --git a/bsmd.Tool/bsmd.Tool.csproj b/bsmd.Tool/bsmd.Tool.csproj index 7e9737b3..92d23e99 100644 --- a/bsmd.Tool/bsmd.Tool.csproj +++ b/bsmd.Tool/bsmd.Tool.csproj @@ -12,6 +12,8 @@ v4.8 512 + + AnyCPU @@ -51,6 +53,10 @@ + + ..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.117.0\lib\net46\System.Data.SQLite.dll + + @@ -70,6 +76,7 @@ + @@ -89,6 +96,13 @@ + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + +