using System; using System.IO; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Data; using System.Data.SQLite; using log4net; namespace bsmd.Tool { public static class LocodeSQliteImport { private static readonly ILog _log = LogManager.GetLogger(typeof(LocodeSQliteImport)); /// /// Importer / Updater für CSV files, die im SVN unter \bsmd\nsw\Archiv abgelegt wurde /// (offizielle UNLOCODEs).. /// /// /// public static void Import(string sqliteDBPath, string csvFilePath) { if (!File.Exists(sqliteDBPath)) throw new ArgumentException($"file {sqliteDBPath} does not exits"); if (!File.Exists(csvFilePath)) throw new ArgumentException($"file {csvFilePath} does not exist"); List currentLocodes = new List(); using (var connection = new SQLiteConnection($"Data Source={sqliteDBPath}")) { connection.Open(); SQLiteCommand lookupCmd = new SQLiteCommand(connection); lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE"; SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String); lookupCmd.Parameters.Add(ccode); SQLiteParameter lcode = new SQLiteParameter("@LCODE", DbType.String); lookupCmd.Parameters.Add(lcode); SQLiteCommand insertCmd = new SQLiteCommand(connection); insertCmd.CommandText = "INSERT INTO locodes (country_id, city_code, name, name_wo_diacritics, sub_div, port, rail_terminal, road_terminal, airport, postal_exchange_office, inland_clearance_depot, fixed_transport_functions, border_crossing_function, status, date, iata, coordinates, remarks) VALUES " + "(@P1, @P2, @P3, @P4, @P5, @P6, @P7, @P8, @P9, @P10, @P11, @P12, @P13, @P14, @P15, @P16, @P17, @P18)"; SQLiteCommand updateCmd = new SQLiteCommand(connection); updateCmd.CommandText = "UPDATE locodes SET name = @P3, name_wo_diacritics = @P4, sub_div = @P5, port = @P6, rail_terminal = @P7, road_terminal = @P8, airport = @P9, postal_exchange_office = @P10, " + "inland_clearance_depot = @P11, fixed_transport_functions = @P12, border_crossing_function = @P13, status = @P14, date = @P15, iata = @P16, coordinates = @P17, remarks = @P18 WHERE id = @ID"; SQLiteParameter p1 = new SQLiteParameter("@P1", DbType.Int32); SQLiteParameter p2 = new SQLiteParameter("@P2", DbType.String); SQLiteParameter p3 = new SQLiteParameter("@P3", DbType.String); SQLiteParameter p4 = new SQLiteParameter("@P4", DbType.String); SQLiteParameter p5 = new SQLiteParameter("@P5", DbType.String); SQLiteParameter p6 = new SQLiteParameter("@P6", DbType.Boolean); SQLiteParameter p7 = new SQLiteParameter("@P7", DbType.Boolean); SQLiteParameter p8 = new SQLiteParameter("@P8", DbType.Boolean); SQLiteParameter p9 = new SQLiteParameter("@P9", DbType.Boolean); SQLiteParameter p10 = new SQLiteParameter("@P10", DbType.Boolean); SQLiteParameter p11 = new SQLiteParameter("@P11", DbType.Boolean); SQLiteParameter p12 = new SQLiteParameter("@P12", DbType.Boolean); SQLiteParameter p13 = new SQLiteParameter("@P13", DbType.Boolean); SQLiteParameter p14 = new SQLiteParameter("@P14", DbType.String); SQLiteParameter p15 = new SQLiteParameter("@P15", DbType.String); SQLiteParameter p16 = new SQLiteParameter("@P16", DbType.String); SQLiteParameter p17 = new SQLiteParameter("@P17", DbType.String); SQLiteParameter p18 = new SQLiteParameter("@P18", DbType.String); SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32); insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18}); updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam }); string[] csvLines = File.ReadAllLines(csvFilePath); int updateCnt = 0, insertCnt = 0; for(int i = 0; i < csvLines.Length; i++) { string line = csvLines[i]; string[] elems = line.Split(','); string country = elems[1].Trim().Replace("\"", ""); if (country.Length < 2) continue; string code = elems[2].Trim().Replace("\"", ""); if (code.Length < 3) continue; currentLocodes.Add(country + code); ccode.Value = country; lcode.Value = code; // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm object lookupResult = lookupCmd.ExecuteScalar(); if(lookupResult != DBNull.Value) { int lid = Convert.ToInt32(lookupResult); // UPDATE entry updateCnt++; } else { // CREATE new entry insertCnt++; } Console.Write($"\r({i}/{csvLines.Length})"); } Console.Write("\n"); Console.WriteLine($"{insertCnt} new entries, {updateCnt} updated"); // jetzt durch alle Ids in der DB laufen und mit dem Import vergleichen List deleteIds = new List(); SQLiteCommand cmd = new SQLiteCommand(connection); cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id"; SQLiteDataReader reader = cmd.ExecuteReader(); while(reader.Read()) { string locode = reader.GetString(0) + reader.GetString(2); if (!currentLocodes.Contains(locode)) deleteIds.Add(reader.GetInt32(1)); } Console.WriteLine($"deleting {deleteIds.Count} obsolete entries"); SQLiteCommand delCmd = new SQLiteCommand(connection); delCmd.CommandText = "DELETE FROM locodes where id = @DELID"; // diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen foreach(int deleteId in deleteIds) { delCmd.Parameters.AddWithValue("@DELID", deleteId); if (delCmd.ExecuteNonQuery() != 1) _log.WarnFormat("{0} affected no rows", deleteId); } } } } }