updated tool project LOCODE import part 1

bsmd.Tool/LocodeSQliteImport.cs

@@ -0,0 +1,140 @@
+using System;
+using System.IO;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Data;
+using System.Data.SQLite;
+using log4net;
+
+namespace bsmd.Tool
+{
+    public static class LocodeSQliteImport
+    {
+        private static readonly ILog _log = LogManager.GetLogger(typeof(LocodeSQliteImport));
+
+        /// <summary>
+        /// Importer / Updater für CSV files, die im SVN unter \bsmd\nsw\Archiv abgelegt wurde
+        /// (offizielle UNLOCODEs)..
+        /// </summary>
+        /// <param name="sqliteDBPath"></param>
+        /// <param name="csvFilePath"></param>
+        public static void Import(string sqliteDBPath, string csvFilePath)
+        {
+            if (!File.Exists(sqliteDBPath)) throw new ArgumentException($"file {sqliteDBPath} does not exits");
+            if (!File.Exists(csvFilePath)) throw new ArgumentException($"file {csvFilePath} does not exist");
+            List<string> currentLocodes = new List<string>();
+
+            using (var connection = new SQLiteConnection($"Data Source={sqliteDBPath}"))
+            {
+                connection.Open();
+
+                SQLiteCommand lookupCmd = new SQLiteCommand(connection);
+                lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE";
+                SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String);
+                lookupCmd.Parameters.Add(ccode);
+                SQLiteParameter lcode = new SQLiteParameter("@LCODE", DbType.String);
+                lookupCmd.Parameters.Add(lcode);
+
+                SQLiteCommand insertCmd = new SQLiteCommand(connection);
+                insertCmd.CommandText = "INSERT INTO locodes (country_id, city_code, name, name_wo_diacritics, sub_div, port, rail_terminal, road_terminal, airport, postal_exchange_office, inland_clearance_depot, fixed_transport_functions, border_crossing_function, status, date, iata, coordinates, remarks) VALUES " +
+                    "(@P1, @P2, @P3, @P4, @P5, @P6, @P7, @P8, @P9, @P10, @P11, @P12, @P13, @P14, @P15, @P16, @P17, @P18)";
+
+
+                SQLiteCommand updateCmd = new SQLiteCommand(connection);
+                updateCmd.CommandText = "UPDATE locodes SET name = @P3, name_wo_diacritics = @P4, sub_div = @P5, port = @P6, rail_terminal = @P7, road_terminal = @P8, airport = @P9, postal_exchange_office = @P10, " +
+                    "inland_clearance_depot = @P11, fixed_transport_functions = @P12, border_crossing_function = @P13, status = @P14, date = @P15, iata = @P16, coordinates = @P17, remarks = @P18 WHERE id = @ID";
+
+                SQLiteParameter p1 = new SQLiteParameter("@P1", DbType.Int32);
+                SQLiteParameter p2 = new SQLiteParameter("@P2", DbType.String);
+                SQLiteParameter p3 = new SQLiteParameter("@P3", DbType.String);
+                SQLiteParameter p4 = new SQLiteParameter("@P4", DbType.String);
+                SQLiteParameter p5 = new SQLiteParameter("@P5", DbType.String);
+                SQLiteParameter p6 = new SQLiteParameter("@P6", DbType.Boolean);
+                SQLiteParameter p7 = new SQLiteParameter("@P7", DbType.Boolean);
+                SQLiteParameter p8 = new SQLiteParameter("@P8", DbType.Boolean);
+                SQLiteParameter p9 = new SQLiteParameter("@P9", DbType.Boolean);
+                SQLiteParameter p10 = new SQLiteParameter("@P10", DbType.Boolean);
+                SQLiteParameter p11 = new SQLiteParameter("@P11", DbType.Boolean);
+                SQLiteParameter p12 = new SQLiteParameter("@P12", DbType.Boolean);
+                SQLiteParameter p13 = new SQLiteParameter("@P13", DbType.Boolean);
+                SQLiteParameter p14 = new SQLiteParameter("@P14", DbType.String);
+                SQLiteParameter p15 = new SQLiteParameter("@P15", DbType.String);
+                SQLiteParameter p16 = new SQLiteParameter("@P16", DbType.String);
+                SQLiteParameter p17 = new SQLiteParameter("@P17", DbType.String);
+                SQLiteParameter p18 = new SQLiteParameter("@P18", DbType.String);
+
+                SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32);
+
+                insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18});
+                updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam });
+
+                string[] csvLines = File.ReadAllLines(csvFilePath);
+                int updateCnt = 0, insertCnt = 0;
+
+                for(int i = 0; i < csvLines.Length; i++)
+                {
+                    string line = csvLines[i];
+                    string[] elems = line.Split(',');
+
+                    string country = elems[1].Trim().Replace("\"", "");
+                    if (country.Length < 2) continue;
+                    string code = elems[2].Trim().Replace("\"", "");
+                    if (code.Length < 3) continue;
+
+                    currentLocodes.Add(country + code);
+
+                    ccode.Value = country;
+                    lcode.Value = code;
+
+                    // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm
+
+                    object lookupResult = lookupCmd.ExecuteScalar();
+                    if(lookupResult != DBNull.Value)
+                    {
+                        int lid = Convert.ToInt32(lookupResult);
+                        // UPDATE entry
+                        updateCnt++;
+                    }
+                    else
+                    {
+                        // CREATE new entry
+                        insertCnt++;
+
+                    }
+                    
+
+                    Console.Write($"\r({i}/{csvLines.Length})");
+                }
+
+                Console.Write("\n");
+                Console.WriteLine($"{insertCnt} new entries, {updateCnt} updated");
+
+                // jetzt durch alle Ids in der DB laufen und mit dem Import vergleichen
+                List<int> deleteIds = new List<int>();
+                SQLiteCommand cmd = new SQLiteCommand(connection);
+                cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id";
+                SQLiteDataReader reader = cmd.ExecuteReader();
+                while(reader.Read())
+                {
+                    string locode = reader.GetString(0) + reader.GetString(2);
+                    if (!currentLocodes.Contains(locode))
+                        deleteIds.Add(reader.GetInt32(1));
+                }
+
+                Console.WriteLine($"deleting {deleteIds.Count} obsolete entries");
+
+                SQLiteCommand delCmd = new SQLiteCommand(connection);
+                delCmd.CommandText = "DELETE FROM locodes where id = @DELID";
+                //  diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen
+                foreach(int deleteId in deleteIds)
+                {
+                    delCmd.Parameters.AddWithValue("@DELID", deleteId);
+                    if (delCmd.ExecuteNonQuery() != 1)
+                        _log.WarnFormat("{0} affected no rows", deleteId);
+                }
+            }
+        }
+    }
+}

bsmd.Tool/Options.cs

@@ -7,10 +7,7 @@
 //
 //	Copyright (c) 2015 Informatikbüro Daniel Schick. All rights reserved.
 
-using System;
-using System.Collections.Generic;
 using CommandLine;
-using CommandLine.Text;
 
 namespace bsmd.Tool
 {
@@ -25,6 +22,9 @@ namespace bsmd.Tool
         [Option('r', "checkrules", HelpText = "Use rule engine on message core")]
         public bool CheckRules { get; set; }
 
+        [Option("locodes", HelpText = "use this flag if you want to import locodes")]
+        public bool ImportLocodes { get; set; }
+
         [Option('s', "staledays", Default = 30, HelpText ="Delete files older than X days")]
         public int StaleDays { get; set; }
 
@@ -33,5 +33,11 @@ namespace bsmd.Tool
 
         [Option('x', "recursive", Default = false, HelpText ="Cleanup subdirectories recursively")]
         public bool CleanupRecursive { get; set; }
+
+        [Option("csv", HelpText = "PATH to import LOCODE csv")]
+        public string LocodeCSV { get; set; }
+
+        [Option("db", HelpText = "PATH to locode db (SQLITE)")]
+        public string LocodeDB { get; set; }
     }
 }

bsmd.Tool/Program.cs

@@ -33,6 +33,13 @@ namespace bsmd.Tool
                     {
                         CleanupFiles.Cleanup(options.CleanupFolderRoot, options.StaleDays, options.CleanupRecursive);
                     }
+                    if(o.ImportLocodes)
+                    {
+                        if(!string.IsNullOrEmpty(o.LocodeCSV) &&  !string.IsNullOrEmpty(o.LocodeDB))
+                        {
+                            LocodeSQliteImport.Import(o.LocodeDB, o.LocodeCSV);
+                        }
+                    }
                 });
             }
             catch (Exception ex)
@@ -40,6 +47,7 @@ namespace bsmd.Tool
                 log.Fatal(ex.Message);
                 result = 1;
             }
+            Console.Read();
             return result;
         }
     }

bsmd.Tool/bsmd.Tool.csproj

@@ -12,6 +12,8 @@
     <TargetFrameworkVersion>v4.8</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <TargetFrameworkProfile />
+    <NuGetPackageImportStamp>
+    </NuGetPackageImportStamp>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>AnyCPU</PlatformTarget>
@@ -51,6 +53,10 @@
     <Reference Include="System" />
     <Reference Include="System.Configuration" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Data.SQLite, Version=1.0.117.0, Culture=neutral, PublicKeyToken=db937bc2d44ff139, processorArchitecture=MSIL">
+      <HintPath>..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.117.0\lib\net46\System.Data.SQLite.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Numerics" />
     <Reference Include="System.Web" />
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
@@ -70,6 +76,7 @@
     </Compile>
     <Compile Include="CheckRules.cs" />
     <Compile Include="CleanupFiles.cs" />
+    <Compile Include="LocodeSQliteImport.cs" />
     <Compile Include="Options.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -89,6 +96,13 @@
     </None>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.117.0\build\net46\Stub.System.Data.SQLite.Core.NetFramework.targets" Condition="Exists('..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.117.0\build\net46\Stub.System.Data.SQLite.Core.NetFramework.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.117.0\build\net46\Stub.System.Data.SQLite.Core.NetFramework.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\packages\Stub.System.Data.SQLite.Core.NetFramework.1.0.117.0\build\net46\Stub.System.Data.SQLite.Core.NetFramework.targets'))" />
+  </Target>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">

bsmd.Tool/bsmd.Tool.csproj.user

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Debug|AnyCPU'">
-    <StartArguments>-s 100 -p E:\svnlager\BSMD\nsw\HIS-NORD\Transmitter-Tool -x</StartArguments>
+    <StartArguments>--locodes --csv="C:\svn_lager\bsmd\nsw\Archiv\2022-2.UNLOCODE.all.csv" --db "C:\git_lager\git_bsmd\misc\db.sqlite"</StartArguments>
   </PropertyGroup>
 </Project>

bsmd.Tool/packages.config

@@ -2,4 +2,6 @@
 <packages>
   <package id="CommandLineParser" version="2.9.1" targetFramework="net48" />
   <package id="log4net" version="2.0.15" targetFramework="net48" />
+  <package id="Stub.System.Data.SQLite.Core.NetFramework" version="1.0.117.0" targetFramework="net48" />
+  <package id="System.Data.SQLite.Core" version="1.0.117.0" targetFramework="net48" />
 </packages>