diff --git a/bsmd.Tool/LocodeSQliteImport.cs b/bsmd.Tool/LocodeSQliteImport.cs index a0de0fd4..bf3833a4 100644 --- a/bsmd.Tool/LocodeSQliteImport.cs +++ b/bsmd.Tool/LocodeSQliteImport.cs @@ -30,6 +30,17 @@ namespace bsmd.Tool { connection.Open(); + // preload countries + Dictionary countryDict = new Dictionary(); + + SQLiteCommand countryCmd = new SQLiteCommand(connection); + countryCmd.CommandText = "SELECT id, code FROM countries"; + SQLiteDataReader reader = countryCmd.ExecuteReader(); + while (reader.Read()) + { + countryDict[reader.GetString(1)] = reader.GetInt32(0); + } + SQLiteCommand lookupCmd = new SQLiteCommand(connection); lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE"; SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String); @@ -67,43 +78,79 @@ namespace bsmd.Tool SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32); - insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18}); + insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18 }); updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam }); string[] csvLines = File.ReadAllLines(csvFilePath); int updateCnt = 0, insertCnt = 0; - for(int i = 0; i < csvLines.Length; i++) + for (int i = 0; i < csvLines.Length; i++) { string line = csvLines[i]; string[] elems = line.Split(','); - + if (elems.Length < 12) continue; string country = elems[1].Trim().Replace("\"", ""); if (country.Length < 2) continue; string code = elems[2].Trim().Replace("\"", ""); if (code.Length < 3) continue; currentLocodes.Add(country + code); - + ccode.Value = country; lcode.Value = code; // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm object lookupResult = lookupCmd.ExecuteScalar(); - if(lookupResult != DBNull.Value) + if ((lookupResult != null) && (lookupResult != DBNull.Value)) { int lid = Convert.ToInt32(lookupResult); + + p3.Value = elems[3].Trim().Replace("\"", ""); + p4.Value = elems[4].Trim().Replace("\"", ""); + p5.Value = elems[5].Trim().Replace("\"", ""); + SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13); + p14.Value = elems[7].Trim().Replace("\"", ""); + p15.Value = elems[8].Trim().Replace("\"", ""); + p16.Value = elems[9].Trim().Replace("\"", ""); + p17.Value = elems[10].Trim().Replace("\"", ""); + p18.Value = elems[11].Trim().Replace("\"", ""); + + idParam.Value = lid; + if (updateCmd.ExecuteNonQuery() == 0) + { + _log.WarnFormat("Update of {0} affected no rows.", lid); + } // UPDATE entry updateCnt++; } else { + if(!countryDict.ContainsKey(country)) + { + _log.WarnFormat("Country {0} not in dictionary!", country); + continue; + } + p1.Value = countryDict[country]; + p2.Value = code; + p3.Value = elems[3].Trim().Replace("\"", ""); + p4.Value = elems[4].Trim().Replace("\"", ""); + p5.Value = elems[5].Trim().Replace("\"", ""); + SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13); + p14.Value = elems[7].Trim().Replace("\"", ""); + p15.Value = elems[8].Trim().Replace("\"", ""); + p16.Value = elems[9].Trim().Replace("\"", ""); + p17.Value = elems[10].Trim().Replace("\"", ""); + p18.Value = elems[11].Trim().Replace("\"", ""); + + if (insertCmd.ExecuteNonQuery() == 0) + { + _log.Warn("Insert of {0} affected no rows."); + } // CREATE new entry insertCnt++; - } - + Console.Write($"\r({i}/{csvLines.Length})"); } @@ -115,12 +162,19 @@ namespace bsmd.Tool List deleteIds = new List(); SQLiteCommand cmd = new SQLiteCommand(connection); cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id"; - SQLiteDataReader reader = cmd.ExecuteReader(); - while(reader.Read()) + SQLiteDataReader reader2 = cmd.ExecuteReader(); + while (reader2.Read()) { - string locode = reader.GetString(0) + reader.GetString(2); - if (!currentLocodes.Contains(locode)) - deleteIds.Add(reader.GetInt32(1)); + if (reader2.IsDBNull(2)) + { + deleteIds.Add(reader2.GetInt32(1)); + } + else + { + string locode = reader2.GetString(0) + reader2.GetString(2); + if (!currentLocodes.Contains(locode)) + deleteIds.Add(reader2.GetInt32(1)); + } } Console.WriteLine($"deleting {deleteIds.Count} obsolete entries"); @@ -128,7 +182,7 @@ namespace bsmd.Tool SQLiteCommand delCmd = new SQLiteCommand(connection); delCmd.CommandText = "DELETE FROM locodes where id = @DELID"; // diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen - foreach(int deleteId in deleteIds) + foreach (int deleteId in deleteIds) { delCmd.Parameters.AddWithValue("@DELID", deleteId); if (delCmd.ExecuteNonQuery() != 1) @@ -136,5 +190,19 @@ namespace bsmd.Tool } } } + + private static void SetBoolParamsFromFunctionString(string func, SQLiteParameter p6, SQLiteParameter p7, SQLiteParameter p8, + SQLiteParameter p9, SQLiteParameter p10, SQLiteParameter p11, SQLiteParameter p12, SQLiteParameter p13) + { + if (func.Length < 8) return; + p6.Value = (func[0] == '1'); + p7.Value = (func[1] == '2'); + p8.Value = (func[2] == '3'); + p9.Value = (func[3] == '4'); + p10.Value = (func[4] == '5'); + p11.Value = (func[5] == '6'); + p12.Value = (func[6] == '7'); + p13.Value = (func[7] == 'B'); + } } } diff --git a/bsmd.Tool/Program.cs b/bsmd.Tool/Program.cs index 4d8f4aa2..54909cb8 100644 --- a/bsmd.Tool/Program.cs +++ b/bsmd.Tool/Program.cs @@ -40,11 +40,12 @@ namespace bsmd.Tool LocodeSQliteImport.Import(o.LocodeDB, o.LocodeCSV); } } + }); } catch (Exception ex) { - log.Fatal(ex.Message); + log.Fatal(ex.ToString()); result = 1; } Console.Read(); diff --git a/misc/db.sqlite b/misc/db.sqlite index 372a0ba0..4eefef87 100644 Binary files a/misc/db.sqlite and b/misc/db.sqlite differ