using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Data;
using System.Data.SQLite;
using log4net;
namespace bsmd.Tool
{
public static class LocodeSQliteImport
{
private static readonly ILog _log = LogManager.GetLogger(typeof(LocodeSQliteImport));
///
/// Importer / Updater für CSV files, die im SVN unter \bsmd\nsw\Archiv abgelegt wurde
/// (offizielle UNLOCODEs)..
///
///
///
public static void Import(string sqliteDBPath, string csvFilePath)
{
if (!File.Exists(sqliteDBPath)) throw new ArgumentException($"file {sqliteDBPath} does not exits");
if (!File.Exists(csvFilePath)) throw new ArgumentException($"file {csvFilePath} does not exist");
List currentLocodes = new List();
using (var connection = new SQLiteConnection($"Data Source={sqliteDBPath}"))
{
connection.Open();
// preload countries
Dictionary countryDict = new Dictionary();
SQLiteCommand countryCmd = new SQLiteCommand(connection);
countryCmd.CommandText = "SELECT id, code FROM countries";
SQLiteDataReader reader = countryCmd.ExecuteReader();
while (reader.Read())
{
countryDict[reader.GetString(1)] = reader.GetInt32(0);
}
SQLiteCommand lookupCmd = new SQLiteCommand(connection);
lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE";
SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String);
lookupCmd.Parameters.Add(ccode);
SQLiteParameter lcode = new SQLiteParameter("@LCODE", DbType.String);
lookupCmd.Parameters.Add(lcode);
SQLiteCommand insertCmd = new SQLiteCommand(connection);
insertCmd.CommandText = "INSERT INTO locodes (country_id, city_code, name, name_wo_diacritics, sub_div, port, rail_terminal, road_terminal, airport, postal_exchange_office, inland_clearance_depot, fixed_transport_functions, border_crossing_function, status, date, iata, coordinates, remarks) VALUES " +
"(@P1, @P2, @P3, @P4, @P5, @P6, @P7, @P8, @P9, @P10, @P11, @P12, @P13, @P14, @P15, @P16, @P17, @P18)";
SQLiteCommand updateCmd = new SQLiteCommand(connection);
updateCmd.CommandText = "UPDATE locodes SET name = @P3, name_wo_diacritics = @P4, sub_div = @P5, port = @P6, rail_terminal = @P7, road_terminal = @P8, airport = @P9, postal_exchange_office = @P10, " +
"inland_clearance_depot = @P11, fixed_transport_functions = @P12, border_crossing_function = @P13, status = @P14, date = @P15, iata = @P16, coordinates = @P17, remarks = @P18 WHERE id = @ID";
SQLiteParameter p1 = new SQLiteParameter("@P1", DbType.Int32);
SQLiteParameter p2 = new SQLiteParameter("@P2", DbType.String);
SQLiteParameter p3 = new SQLiteParameter("@P3", DbType.String);
SQLiteParameter p4 = new SQLiteParameter("@P4", DbType.String);
SQLiteParameter p5 = new SQLiteParameter("@P5", DbType.String);
SQLiteParameter p6 = new SQLiteParameter("@P6", DbType.Boolean);
SQLiteParameter p7 = new SQLiteParameter("@P7", DbType.Boolean);
SQLiteParameter p8 = new SQLiteParameter("@P8", DbType.Boolean);
SQLiteParameter p9 = new SQLiteParameter("@P9", DbType.Boolean);
SQLiteParameter p10 = new SQLiteParameter("@P10", DbType.Boolean);
SQLiteParameter p11 = new SQLiteParameter("@P11", DbType.Boolean);
SQLiteParameter p12 = new SQLiteParameter("@P12", DbType.Boolean);
SQLiteParameter p13 = new SQLiteParameter("@P13", DbType.Boolean);
SQLiteParameter p14 = new SQLiteParameter("@P14", DbType.String);
SQLiteParameter p15 = new SQLiteParameter("@P15", DbType.String);
SQLiteParameter p16 = new SQLiteParameter("@P16", DbType.String);
SQLiteParameter p17 = new SQLiteParameter("@P17", DbType.String);
SQLiteParameter p18 = new SQLiteParameter("@P18", DbType.String);
SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32);
insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18 });
updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam });
string[] csvLines = File.ReadAllLines(csvFilePath);
int updateCnt = 0, insertCnt = 0;
for (int i = 0; i < csvLines.Length; i++)
{
string line = csvLines[i];
string[] elems = line.Split(',');
if (elems.Length < 12) continue;
string country = elems[1].Trim().Replace("\"", "");
if (country.Length < 2) continue;
string code = elems[2].Trim().Replace("\"", "");
if (code.Length < 3) continue;
currentLocodes.Add(country + code);
ccode.Value = country;
lcode.Value = code;
// Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm
object lookupResult = lookupCmd.ExecuteScalar();
if ((lookupResult != null) && (lookupResult != DBNull.Value))
{
int lid = Convert.ToInt32(lookupResult);
p3.Value = elems[3].Trim().Replace("\"", "");
p4.Value = elems[4].Trim().Replace("\"", "");
p5.Value = elems[5].Trim().Replace("\"", "");
SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
p14.Value = elems[7].Trim().Replace("\"", "");
p15.Value = elems[8].Trim().Replace("\"", "");
p16.Value = elems[9].Trim().Replace("\"", "");
p17.Value = elems[10].Trim().Replace("\"", "");
p18.Value = elems[11].Trim().Replace("\"", "");
idParam.Value = lid;
if (updateCmd.ExecuteNonQuery() == 0)
{
_log.WarnFormat("Update of {0} affected no rows.", lid);
}
// UPDATE entry
updateCnt++;
}
else
{
if(!countryDict.ContainsKey(country))
{
_log.WarnFormat("Country {0} not in dictionary!", country);
continue;
}
p1.Value = countryDict[country];
p2.Value = code;
p3.Value = elems[3].Trim().Replace("\"", "");
p4.Value = elems[4].Trim().Replace("\"", "");
p5.Value = elems[5].Trim().Replace("\"", "");
SetBoolParamsFromFunctionString(elems[6].Trim().Replace("\"", ""), p6, p7, p8, p9, p10, p11, p12, p13);
p14.Value = elems[7].Trim().Replace("\"", "");
p15.Value = elems[8].Trim().Replace("\"", "");
p16.Value = elems[9].Trim().Replace("\"", "");
p17.Value = elems[10].Trim().Replace("\"", "");
p18.Value = elems[11].Trim().Replace("\"", "");
if (insertCmd.ExecuteNonQuery() == 0)
{
_log.Warn("Insert of {0} affected no rows.");
}
// CREATE new entry
insertCnt++;
}
Console.Write($"\r({i}/{csvLines.Length})");
}
Console.Write("\n");
Console.WriteLine($"{insertCnt} new entries, {updateCnt} updated");
// jetzt durch alle Ids in der DB laufen und mit dem Import vergleichen
List deleteIds = new List();
SQLiteCommand cmd = new SQLiteCommand(connection);
cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id";
SQLiteDataReader reader2 = cmd.ExecuteReader();
while (reader2.Read())
{
if (reader2.IsDBNull(2))
{
deleteIds.Add(reader2.GetInt32(1));
}
else
{
string locode = reader2.GetString(0) + reader2.GetString(2);
if (locode.Equals("ZZUKN")) continue;
if (locode.Equals("ZZNOK")) continue;
if (!currentLocodes.Contains(locode))
deleteIds.Add(reader2.GetInt32(1));
}
}
Console.WriteLine($"deleting {deleteIds.Count} obsolete entries");
SQLiteCommand delCmd = new SQLiteCommand(connection);
delCmd.CommandText = "DELETE FROM locodes where id = @DELID";
// diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen
foreach (int deleteId in deleteIds)
{
delCmd.Parameters.AddWithValue("@DELID", deleteId);
if (delCmd.ExecuteNonQuery() != 1)
_log.WarnFormat("{0} affected no rows", deleteId);
}
}
}
private static void SetBoolParamsFromFunctionString(string func, SQLiteParameter p6, SQLiteParameter p7, SQLiteParameter p8,
SQLiteParameter p9, SQLiteParameter p10, SQLiteParameter p11, SQLiteParameter p12, SQLiteParameter p13)
{
if (func.Length < 8) return;
p6.Value = (func[0] == '1');
p7.Value = (func[1] == '2');
p8.Value = (func[2] == '3');
p9.Value = (func[3] == '4');
p10.Value = (func[4] == '5');
p11.Value = (func[5] == '6');
p12.Value = (func[6] == '7');
p13.Value = (func[7] == 'B');
}
}
}