dschick/git_bsmd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated tool project LOCODE import part 2

Browse Source
This commit is contained in:
Daniel Schick 2023-04-02 09:14:55 +02:00
parent 1bbccadd17
commit 882861a328
3 changed files with 83 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
bsmd.Tool/LocodeSQliteImport.cs
View File

@ -30,6 +30,17 @@ namespace bsmd.Tool
{ {
connection.Open(); connection.Open();
// preload countries
Dictionary<string, int> countryDict = new Dictionary<string, int>();
SQLiteCommand countryCmd = new SQLiteCommand(connection);
countryCmd.CommandText = "SELECT id, code FROM countries";
SQLiteDataReader reader = countryCmd.ExecuteReader();
while (reader.Read())
{
countryDict[reader.GetString(1)] = reader.GetInt32(0);
}
SQLiteCommand lookupCmd = new SQLiteCommand(connection); SQLiteCommand lookupCmd = new SQLiteCommand(connection);
lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE"; lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE";
SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String); SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String);
@ -67,17 +78,17 @@ namespace bsmd.Tool
SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32); SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32);
insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18}); insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18 });
updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam }); updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam });
string[] csvLines = File.ReadAllLines(csvFilePath); string[] csvLines = File.ReadAllLines(csvFilePath);
int updateCnt = 0, insertCnt = 0; int updateCnt = 0, insertCnt = 0;
for(int i = 0; i < csvLines.Length; i++) for (int i = 0; i < csvLines.Length; i++)
{ {
string line = csvLines[i]; string line = csvLines[i];
string[] elems = line.Split(','); string[] elems = line.Split(',');
if (elems.Length < 12) continue;
string country = elems[1].Trim().Replace("\"", ""); string country = elems[1].Trim().Replace("\"", "");
if (country.Length < 2) continue; if (country.Length < 2) continue;
string code = elems[2].Trim().Replace("\"", ""); string code = elems[2].Trim().Replace("\"", "");
@ -91,17 +102,53 @@ namespace bsmd.Tool
// Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm
object lookupResult = lookupCmd.ExecuteScalar(); object lookupResult = lookupCmd.ExecuteScalar();
if(lookupResult != DBNull.Value) if ((lookupResult != null) && (lookupResult != DBNull.Value))
{ {
int lid = Convert.ToInt32(lookupResult); int lid = Convert.ToInt32(lookupResult);
p3.Value = elems[3].Trim().Replace("\"", "");
p4.Value = elems[4].Trim().Replace("\"", "");
p5.Value = elems[5].Trim().Replace("\"", "");
SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
p14.Value = elems[7].Trim().Replace("\"", "");
p15.Value = elems[8].Trim().Replace("\"", "");
p16.Value = elems[9].Trim().Replace("\"", "");
p17.Value = elems[10].Trim().Replace("\"", "");
p18.Value = elems[11].Trim().Replace("\"", "");
idParam.Value = lid;
if (updateCmd.ExecuteNonQuery() == 0)
{
_log.WarnFormat("Update of {0} affected no rows.", lid);
}
// UPDATE entry // UPDATE entry
updateCnt++; updateCnt++;
} }
else else
{ {
if(!countryDict.ContainsKey(country))
{
_log.WarnFormat("Country {0} not in dictionary!", country);
continue;
}
p1.Value = countryDict[country];
p2.Value = code;
p3.Value = elems[3].Trim().Replace("\"", "");
p4.Value = elems[4].Trim().Replace("\"", "");
p5.Value = elems[5].Trim().Replace("\"", "");
SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
p14.Value = elems[7].Trim().Replace("\"", "");
p15.Value = elems[8].Trim().Replace("\"", "");
p16.Value = elems[9].Trim().Replace("\"", "");
p17.Value = elems[10].Trim().Replace("\"", "");
p18.Value = elems[11].Trim().Replace("\"", "");
if (insertCmd.ExecuteNonQuery() == 0)
{
_log.Warn("Insert of {0} affected no rows.");
}
// CREATE new entry // CREATE new entry
insertCnt++; insertCnt++;
} }
@ -115,12 +162,19 @@ namespace bsmd.Tool
List<int> deleteIds = new List<int>(); List<int> deleteIds = new List<int>();
SQLiteCommand cmd = new SQLiteCommand(connection); SQLiteCommand cmd = new SQLiteCommand(connection);
cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id"; cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id";
SQLiteDataReader reader = cmd.ExecuteReader(); SQLiteDataReader reader2 = cmd.ExecuteReader();
while(reader.Read()) while (reader2.Read())
{ {
string locode = reader.GetString(0) + reader.GetString(2); if (reader2.IsDBNull(2))
{
deleteIds.Add(reader2.GetInt32(1));
}
else
{
string locode = reader2.GetString(0) + reader2.GetString(2);
if (!currentLocodes.Contains(locode)) if (!currentLocodes.Contains(locode))
deleteIds.Add(reader.GetInt32(1)); deleteIds.Add(reader2.GetInt32(1));
}
} }
Console.WriteLine($"deleting {deleteIds.Count} obsolete entries"); Console.WriteLine($"deleting {deleteIds.Count} obsolete entries");
@ -128,7 +182,7 @@ namespace bsmd.Tool
SQLiteCommand delCmd = new SQLiteCommand(connection); SQLiteCommand delCmd = new SQLiteCommand(connection);
delCmd.CommandText = "DELETE FROM locodes where id = @DELID"; delCmd.CommandText = "DELETE FROM locodes where id = @DELID";
// diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen // diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen
foreach(int deleteId in deleteIds) foreach (int deleteId in deleteIds)
{ {
delCmd.Parameters.AddWithValue("@DELID", deleteId); delCmd.Parameters.AddWithValue("@DELID", deleteId);
if (delCmd.ExecuteNonQuery() != 1) if (delCmd.ExecuteNonQuery() != 1)
@ -136,5 +190,19 @@ namespace bsmd.Tool
} }
} }
} }
private static void SetBoolParamsFromFunctionString(string func, SQLiteParameter p6, SQLiteParameter p7, SQLiteParameter p8,
SQLiteParameter p9, SQLiteParameter p10, SQLiteParameter p11, SQLiteParameter p12, SQLiteParameter p13)
{
if (func.Length < 8) return;
p6.Value = (func[0] == '1');
p7.Value = (func[1] == '2');
p8.Value = (func[2] == '3');
p9.Value = (func[3] == '4');
p10.Value = (func[4] == '5');
p11.Value = (func[5] == '6');
p12.Value = (func[6] == '7');
p13.Value = (func[7] == 'B');
}
} }
} }

3
bsmd.Tool/Program.cs
View File

@ -40,11 +40,12 @@ namespace bsmd.Tool
LocodeSQliteImport.Import(o.LocodeDB, o.LocodeCSV); LocodeSQliteImport.Import(o.LocodeDB, o.LocodeCSV);
} }
} }
}); });
} }
catch (Exception ex) catch (Exception ex)
{ {
log.Fatal(ex.Message); log.Fatal(ex.ToString());
result = 1; result = 1;
} }
Console.Read(); Console.Read();

BIN
misc/db.sqlite
View File

Binary file not shown.