updated tool project LOCODE import part 2

bsmd.Tool/LocodeSQliteImport.cs

@@ -30,6 +30,17 @@ namespace bsmd.Tool
             {
                 connection.Open();
 
+                // preload countries
+                Dictionary<string, int> countryDict = new Dictionary<string, int>();
+                
+                SQLiteCommand countryCmd = new SQLiteCommand(connection);
+                countryCmd.CommandText = "SELECT id, code FROM countries";
+                SQLiteDataReader reader = countryCmd.ExecuteReader();
+                while (reader.Read())
+                {
+                    countryDict[reader.GetString(1)] = reader.GetInt32(0);
+                }
+
                 SQLiteCommand lookupCmd = new SQLiteCommand(connection);
                 lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE";
                 SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String);
@@ -67,43 +78,79 @@ namespace bsmd.Tool
 
                 SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32);
 
-                insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18});
+                insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18 });
                 updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam });
 
                 string[] csvLines = File.ReadAllLines(csvFilePath);
                 int updateCnt = 0, insertCnt = 0;
 
-                for(int i = 0; i < csvLines.Length; i++)
+                for (int i = 0; i < csvLines.Length; i++)
                 {
                     string line = csvLines[i];
                     string[] elems = line.Split(',');
-
+                    if (elems.Length < 12) continue;
                     string country = elems[1].Trim().Replace("\"", "");
                     if (country.Length < 2) continue;
                     string code = elems[2].Trim().Replace("\"", "");
                     if (code.Length < 3) continue;
 
                     currentLocodes.Add(country + code);
-
+                    
                     ccode.Value = country;
                     lcode.Value = code;
 
                     // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm
 
                     object lookupResult = lookupCmd.ExecuteScalar();
-                    if(lookupResult != DBNull.Value)
+                    if ((lookupResult != null) && (lookupResult != DBNull.Value))
                     {
                         int lid = Convert.ToInt32(lookupResult);
+
+                        p3.Value = elems[3].Trim().Replace("\"", "");
+                        p4.Value = elems[4].Trim().Replace("\"", "");
+                        p5.Value = elems[5].Trim().Replace("\"", "");
+                        SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
+                        p14.Value = elems[7].Trim().Replace("\"", "");
+                        p15.Value = elems[8].Trim().Replace("\"", "");
+                        p16.Value = elems[9].Trim().Replace("\"", "");
+                        p17.Value = elems[10].Trim().Replace("\"", "");
+                        p18.Value = elems[11].Trim().Replace("\"", "");
+
+                        idParam.Value = lid;
+                        if (updateCmd.ExecuteNonQuery() == 0)
+                        {
+                            _log.WarnFormat("Update of {0} affected no rows.", lid);
+                        }
                         // UPDATE entry
                         updateCnt++;
                     }
                     else
                     {
+                        if(!countryDict.ContainsKey(country))
+                        {
+                            _log.WarnFormat("Country {0} not in dictionary!", country);
+                            continue;
+                        }
+                        p1.Value = countryDict[country];
+                        p2.Value = code;
+                        p3.Value = elems[3].Trim().Replace("\"", "");
+                        p4.Value = elems[4].Trim().Replace("\"", "");
+                        p5.Value = elems[5].Trim().Replace("\"", "");
+                        SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
+                        p14.Value = elems[7].Trim().Replace("\"", "");
+                        p15.Value = elems[8].Trim().Replace("\"", "");
+                        p16.Value = elems[9].Trim().Replace("\"", "");
+                        p17.Value = elems[10].Trim().Replace("\"", "");
+                        p18.Value = elems[11].Trim().Replace("\"", "");
+
+                        if (insertCmd.ExecuteNonQuery() == 0)
+                        {
+                            _log.Warn("Insert of {0} affected no rows.");
+                        }
                         // CREATE new entry
                         insertCnt++;
-
                     }
-                    
+
 
                     Console.Write($"\r({i}/{csvLines.Length})");
                 }
@@ -115,12 +162,19 @@ namespace bsmd.Tool
                 List<int> deleteIds = new List<int>();
                 SQLiteCommand cmd = new SQLiteCommand(connection);
                 cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id";
-                SQLiteDataReader reader = cmd.ExecuteReader();
+                SQLiteDataReader reader2 = cmd.ExecuteReader();
-                while(reader.Read())
+                while (reader2.Read())
                 {
-                    string locode = reader.GetString(0) + reader.GetString(2);
+                    if (reader2.IsDBNull(2))
-                    if (!currentLocodes.Contains(locode))
+                    {
-                        deleteIds.Add(reader.GetInt32(1));
+                        deleteIds.Add(reader2.GetInt32(1));
+                    }
+                    else
+                    {
+                        string locode = reader2.GetString(0) + reader2.GetString(2);
+                        if (!currentLocodes.Contains(locode))
+                            deleteIds.Add(reader2.GetInt32(1));
+                    }
                 }
 
                 Console.WriteLine($"deleting {deleteIds.Count} obsolete entries");
@@ -128,7 +182,7 @@ namespace bsmd.Tool
                 SQLiteCommand delCmd = new SQLiteCommand(connection);
                 delCmd.CommandText = "DELETE FROM locodes where id = @DELID";
                 //  diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen
-                foreach(int deleteId in deleteIds)
+                foreach (int deleteId in deleteIds)
                 {
                     delCmd.Parameters.AddWithValue("@DELID", deleteId);
                     if (delCmd.ExecuteNonQuery() != 1)
@@ -136,5 +190,19 @@ namespace bsmd.Tool
                 }
             }
         }
+
+        private static void SetBoolParamsFromFunctionString(string func, SQLiteParameter p6, SQLiteParameter p7, SQLiteParameter p8,
+            SQLiteParameter p9, SQLiteParameter p10, SQLiteParameter p11, SQLiteParameter p12, SQLiteParameter p13)
+        {
+            if (func.Length < 8) return;
+            p6.Value = (func[0] == '1');
+            p7.Value = (func[1] == '2');
+            p8.Value = (func[2] == '3');
+            p9.Value = (func[3] == '4');
+            p10.Value = (func[4] == '5');
+            p11.Value = (func[5] == '6');
+            p12.Value = (func[6] == '7');
+            p13.Value = (func[7] == 'B');
+        }
     }
 }

bsmd.Tool/Program.cs

@@ -40,11 +40,12 @@ namespace bsmd.Tool
                             LocodeSQliteImport.Import(o.LocodeDB, o.LocodeCSV);
                         }
                     }
+      
                 });
             }
             catch (Exception ex)
             {
-                log.Fatal(ex.Message);
+                log.Fatal(ex.ToString());
                 result = 1;
             }
             Console.Read();