dschick/git_bsmd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated tool project LOCODE import part 2

Browse Source
This commit is contained in:
Daniel Schick 2023-04-02 09:14:55 +02:00
parent 1bbccadd17
commit 882861a328
3 changed files with 83 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
bsmd.Tool/LocodeSQliteImport.cs
View File

@ -30,6 +30,17 @@ namespace bsmd.Tool
{
connection.Open();
// preload countries
Dictionary<string, int> countryDict = new Dictionary<string, int>();
SQLiteCommand countryCmd = new SQLiteCommand(connection);
countryCmd.CommandText = "SELECT id, code FROM countries";
SQLiteDataReader reader = countryCmd.ExecuteReader();
while (reader.Read())
{
countryDict[reader.GetString(1)] = reader.GetInt32(0);
}
SQLiteCommand lookupCmd = new SQLiteCommand(connection);
lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE";
SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String);
@ -67,17 +78,17 @@ namespace bsmd.Tool
SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32);
insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18});
insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18 });
updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam });
string[] csvLines = File.ReadAllLines(csvFilePath);
int updateCnt = 0, insertCnt = 0;
for(int i = 0; i < csvLines.Length; i++)
for (int i = 0; i < csvLines.Length; i++)
{
string line = csvLines[i];
string[] elems = line.Split(',');
if (elems.Length < 12) continue;
string country = elems[1].Trim().Replace("\"", "");
if (country.Length < 2) continue;
string code = elems[2].Trim().Replace("\"", "");
@ -91,17 +102,53 @@ namespace bsmd.Tool
// Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm
object lookupResult = lookupCmd.ExecuteScalar();
if(lookupResult != DBNull.Value)
if ((lookupResult != null) && (lookupResult != DBNull.Value))
{
int lid = Convert.ToInt32(lookupResult);
p3.Value = elems[3].Trim().Replace("\"", "");
p4.Value = elems[4].Trim().Replace("\"", "");
p5.Value = elems[5].Trim().Replace("\"", "");
SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
p14.Value = elems[7].Trim().Replace("\"", "");
p15.Value = elems[8].Trim().Replace("\"", "");
p16.Value = elems[9].Trim().Replace("\"", "");
p17.Value = elems[10].Trim().Replace("\"", "");
p18.Value = elems[11].Trim().Replace("\"", "");
idParam.Value = lid;
if (updateCmd.ExecuteNonQuery() == 0)
{
_log.WarnFormat("Update of {0} affected no rows.", lid);
}
// UPDATE entry
updateCnt++;
}
else
{
if(!countryDict.ContainsKey(country))
{
_log.WarnFormat("Country {0} not in dictionary!", country);
continue;
}
p1.Value = countryDict[country];
p2.Value = code;
p3.Value = elems[3].Trim().Replace("\"", "");
p4.Value = elems[4].Trim().Replace("\"", "");
p5.Value = elems[5].Trim().Replace("\"", "");
SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
p14.Value = elems[7].Trim().Replace("\"", "");
p15.Value = elems[8].Trim().Replace("\"", "");
p16.Value = elems[9].Trim().Replace("\"", "");
p17.Value = elems[10].Trim().Replace("\"", "");
p18.Value = elems[11].Trim().Replace("\"", "");
if (insertCmd.ExecuteNonQuery() == 0)
{
_log.Warn("Insert of {0} affected no rows.");
}
// CREATE new entry
insertCnt++;
}
@ -115,12 +162,19 @@ namespace bsmd.Tool
List<int> deleteIds = new List<int>();
SQLiteCommand cmd = new SQLiteCommand(connection);
cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id";
SQLiteDataReader reader = cmd.ExecuteReader();
while(reader.Read())
SQLiteDataReader reader2 = cmd.ExecuteReader();
while (reader2.Read())
{
string locode = reader.GetString(0) + reader.GetString(2);
if (reader2.IsDBNull(2))
{
deleteIds.Add(reader2.GetInt32(1));
}
else
{
string locode = reader2.GetString(0) + reader2.GetString(2);
if (!currentLocodes.Contains(locode))
deleteIds.Add(reader.GetInt32(1));
deleteIds.Add(reader2.GetInt32(1));
}
}
Console.WriteLine($"deleting {deleteIds.Count} obsolete entries");
@ -128,7 +182,7 @@ namespace bsmd.Tool
SQLiteCommand delCmd = new SQLiteCommand(connection);
delCmd.CommandText = "DELETE FROM locodes where id = @DELID";
// diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen
foreach(int deleteId in deleteIds)
foreach (int deleteId in deleteIds)
{
delCmd.Parameters.AddWithValue("@DELID", deleteId);
if (delCmd.ExecuteNonQuery() != 1)
@ -136,5 +190,19 @@ namespace bsmd.Tool
}
}
}
private static void SetBoolParamsFromFunctionString(string func, SQLiteParameter p6, SQLiteParameter p7, SQLiteParameter p8,
SQLiteParameter p9, SQLiteParameter p10, SQLiteParameter p11, SQLiteParameter p12, SQLiteParameter p13)
{
if (func.Length < 8) return;
p6.Value = (func[0] == '1');
p7.Value = (func[1] == '2');
p8.Value = (func[2] == '3');
p9.Value = (func[3] == '4');
p10.Value = (func[4] == '5');
p11.Value = (func[5] == '6');
p12.Value = (func[6] == '7');
p13.Value = (func[7] == 'B');
}
}
}

3
bsmd.Tool/Program.cs
View File

@ -40,11 +40,12 @@ namespace bsmd.Tool
LocodeSQliteImport.Import(o.LocodeDB, o.LocodeCSV);
}
}
});
}
catch (Exception ex)
{
log.Fatal(ex.Message);
log.Fatal(ex.ToString());
result = 1;
}
Console.Read();

BIN
misc/db.sqlite
View File

Binary file not shown.