using System; using System.IO; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Data; using System.Data.SQLite; using log4net; namespace bsmd.Tool { public static class LocodeSQliteImport { private static readonly ILog _log = LogManager.GetLogger(typeof(LocodeSQliteImport)); /// /// Importer / Updater für CSV files, die im SVN unter \bsmd\nsw\Archiv abgelegt wurde /// (offizielle UNLOCODEs).. /// /// /// public static void Import(string sqliteDBPath, string csvFilePath) { if (!File.Exists(sqliteDBPath)) throw new ArgumentException($"file {sqliteDBPath} does not exits"); if (!File.Exists(csvFilePath)) throw new ArgumentException($"file {csvFilePath} does not exist"); List currentLocodes = new List(); using (var connection = new SQLiteConnection($"Data Source={sqliteDBPath}")) { connection.Open(); // preload countries Dictionary countryDict = new Dictionary(); SQLiteCommand countryCmd = new SQLiteCommand(connection); countryCmd.CommandText = "SELECT id, code FROM countries"; SQLiteDataReader reader = countryCmd.ExecuteReader(); while (reader.Read()) { countryDict[reader.GetString(1)] = reader.GetInt32(0); } SQLiteCommand lookupCmd = new SQLiteCommand(connection); lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE"; SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String); lookupCmd.Parameters.Add(ccode); SQLiteParameter lcode = new SQLiteParameter("@LCODE", DbType.String); lookupCmd.Parameters.Add(lcode); SQLiteCommand insertCmd = new SQLiteCommand(connection); insertCmd.CommandText = "INSERT INTO locodes (country_id, city_code, name, name_wo_diacritics, sub_div, port, rail_terminal, road_terminal, airport, postal_exchange_office, inland_clearance_depot, fixed_transport_functions, border_crossing_function, status, date, iata, coordinates, remarks) VALUES " + "(@P1, @P2, @P3, @P4, @P5, @P6, @P7, @P8, @P9, @P10, @P11, @P12, @P13, @P14, @P15, @P16, @P17, @P18)"; SQLiteCommand updateCmd = new SQLiteCommand(connection); updateCmd.CommandText = "UPDATE locodes SET name = @P3, name_wo_diacritics = @P4, sub_div = @P5, port = @P6, rail_terminal = @P7, road_terminal = @P8, airport = @P9, postal_exchange_office = @P10, " + "inland_clearance_depot = @P11, fixed_transport_functions = @P12, border_crossing_function = @P13, status = @P14, date = @P15, iata = @P16, coordinates = @P17, remarks = @P18 WHERE id = @ID"; SQLiteParameter p1 = new SQLiteParameter("@P1", DbType.Int32); SQLiteParameter p2 = new SQLiteParameter("@P2", DbType.String); SQLiteParameter p3 = new SQLiteParameter("@P3", DbType.String); SQLiteParameter p4 = new SQLiteParameter("@P4", DbType.String); SQLiteParameter p5 = new SQLiteParameter("@P5", DbType.String); SQLiteParameter p6 = new SQLiteParameter("@P6", DbType.Boolean); SQLiteParameter p7 = new SQLiteParameter("@P7", DbType.Boolean); SQLiteParameter p8 = new SQLiteParameter("@P8", DbType.Boolean); SQLiteParameter p9 = new SQLiteParameter("@P9", DbType.Boolean); SQLiteParameter p10 = new SQLiteParameter("@P10", DbType.Boolean); SQLiteParameter p11 = new SQLiteParameter("@P11", DbType.Boolean); SQLiteParameter p12 = new SQLiteParameter("@P12", DbType.Boolean); SQLiteParameter p13 = new SQLiteParameter("@P13", DbType.Boolean); SQLiteParameter p14 = new SQLiteParameter("@P14", DbType.String); SQLiteParameter p15 = new SQLiteParameter("@P15", DbType.String); SQLiteParameter p16 = new SQLiteParameter("@P16", DbType.String); SQLiteParameter p17 = new SQLiteParameter("@P17", DbType.String); SQLiteParameter p18 = new SQLiteParameter("@P18", DbType.String); SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32); insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18 }); updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam }); string[] csvLines = File.ReadAllLines(csvFilePath); int updateCnt = 0, insertCnt = 0; for (int i = 0; i < csvLines.Length; i++) { string line = csvLines[i]; string[] elems = line.Split(','); if (elems.Length < 12) continue; string country = elems[1].Trim().Replace("\"", ""); if (country.Length < 2) continue; string code = elems[2].Trim().Replace("\"", ""); if (code.Length < 3) continue; currentLocodes.Add(country + code); ccode.Value = country; lcode.Value = code; // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm object lookupResult = lookupCmd.ExecuteScalar(); if ((lookupResult != null) && (lookupResult != DBNull.Value)) { int lid = Convert.ToInt32(lookupResult); p3.Value = elems[3].Trim().Replace("\"", ""); p4.Value = elems[4].Trim().Replace("\"", ""); p5.Value = elems[5].Trim().Replace("\"", ""); SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13); p14.Value = elems[7].Trim().Replace("\"", ""); p15.Value = elems[8].Trim().Replace("\"", ""); p16.Value = elems[9].Trim().Replace("\"", ""); p17.Value = elems[10].Trim().Replace("\"", ""); p18.Value = elems[11].Trim().Replace("\"", ""); idParam.Value = lid; if (updateCmd.ExecuteNonQuery() == 0) { _log.WarnFormat("Update of {0} affected no rows.", lid); } // UPDATE entry updateCnt++; } else { if(!countryDict.ContainsKey(country)) { _log.WarnFormat("Country {0} not in dictionary!", country); continue; } p1.Value = countryDict[country]; p2.Value = code; p3.Value = elems[3].Trim().Replace("\"", ""); p4.Value = elems[4].Trim().Replace("\"", ""); p5.Value = elems[5].Trim().Replace("\"", ""); SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13); p14.Value = elems[7].Trim().Replace("\"", ""); p15.Value = elems[8].Trim().Replace("\"", ""); p16.Value = elems[9].Trim().Replace("\"", ""); p17.Value = elems[10].Trim().Replace("\"", ""); p18.Value = elems[11].Trim().Replace("\"", ""); if (insertCmd.ExecuteNonQuery() == 0) { _log.Warn("Insert of {0} affected no rows."); } // CREATE new entry insertCnt++; } Console.Write($"\r({i}/{csvLines.Length})"); } Console.Write("\n"); Console.WriteLine($"{insertCnt} new entries, {updateCnt} updated"); // jetzt durch alle Ids in der DB laufen und mit dem Import vergleichen List deleteIds = new List(); SQLiteCommand cmd = new SQLiteCommand(connection); cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id"; SQLiteDataReader reader2 = cmd.ExecuteReader(); while (reader2.Read()) { if (reader2.IsDBNull(2)) { deleteIds.Add(reader2.GetInt32(1)); } else { string locode = reader2.GetString(0) + reader2.GetString(2); if (!currentLocodes.Contains(locode)) deleteIds.Add(reader2.GetInt32(1)); } } Console.WriteLine($"deleting {deleteIds.Count} obsolete entries"); SQLiteCommand delCmd = new SQLiteCommand(connection); delCmd.CommandText = "DELETE FROM locodes where id = @DELID"; // diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen foreach (int deleteId in deleteIds) { delCmd.Parameters.AddWithValue("@DELID", deleteId); if (delCmd.ExecuteNonQuery() != 1) _log.WarnFormat("{0} affected no rows", deleteId); } } } private static void SetBoolParamsFromFunctionString(string func, SQLiteParameter p6, SQLiteParameter p7, SQLiteParameter p8, SQLiteParameter p9, SQLiteParameter p10, SQLiteParameter p11, SQLiteParameter p12, SQLiteParameter p13) { if (func.Length < 8) return; p6.Value = (func[0] == '1'); p7.Value = (func[1] == '2'); p8.Value = (func[2] == '3'); p9.Value = (func[3] == '4'); p10.Value = (func[4] == '5'); p11.Value = (func[5] == '6'); p12.Value = (func[6] == '7'); p13.Value = (func[7] == 'B'); } } }