git_bsmd/bsmd.Tool/LocodeSQliteImport.cs

using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Data;
using System.Data.SQLite;
using log4net;

namespace bsmd.Tool
{
    public static class LocodeSQliteImport
    {
        private static readonly ILog _log = LogManager.GetLogger(typeof(LocodeSQliteImport));

        /// <summary>
        /// Importer / Updater für CSV files, die im SVN unter \bsmd\nsw\Archiv abgelegt wurde
        /// (offizielle UNLOCODEs)..
        /// </summary>
        /// <param name="sqliteDBPath"></param>
        /// <param name="csvFilePath"></param>
        public static void Import(string sqliteDBPath, string csvFilePath)
        {
            if (!File.Exists(sqliteDBPath)) throw new ArgumentException($"file {sqliteDBPath} does not exits");
            if (!File.Exists(csvFilePath)) throw new ArgumentException($"file {csvFilePath} does not exist");
            List<string> currentLocodes = new List<string>();

            using (var connection = new SQLiteConnection($"Data Source={sqliteDBPath}"))
            {
                connection.Open();

                SQLiteCommand lookupCmd = new SQLiteCommand(connection);
                lookupCmd.CommandText = "SELECT locodes.id FROM locodes INNER JOIN countries ON locodes.country_id = countries.id WHERE countries.code = @CCODE AND locodes.city_code = @LCODE";
                SQLiteParameter ccode = new SQLiteParameter("@CCODE", DbType.String);
                lookupCmd.Parameters.Add(ccode);
                SQLiteParameter lcode = new SQLiteParameter("@LCODE", DbType.String);
                lookupCmd.Parameters.Add(lcode);

                SQLiteCommand insertCmd = new SQLiteCommand(connection);
                insertCmd.CommandText = "INSERT INTO locodes (country_id, city_code, name, name_wo_diacritics, sub_div, port, rail_terminal, road_terminal, airport, postal_exchange_office, inland_clearance_depot, fixed_transport_functions, border_crossing_function, status, date, iata, coordinates, remarks) VALUES " +
                    "(@P1, @P2, @P3, @P4, @P5, @P6, @P7, @P8, @P9, @P10, @P11, @P12, @P13, @P14, @P15, @P16, @P17, @P18)";


                SQLiteCommand updateCmd = new SQLiteCommand(connection);
                updateCmd.CommandText = "UPDATE locodes SET name = @P3, name_wo_diacritics = @P4, sub_div = @P5, port = @P6, rail_terminal = @P7, road_terminal = @P8, airport = @P9, postal_exchange_office = @P10, " +
                    "inland_clearance_depot = @P11, fixed_transport_functions = @P12, border_crossing_function = @P13, status = @P14, date = @P15, iata = @P16, coordinates = @P17, remarks = @P18 WHERE id = @ID";

                SQLiteParameter p1 = new SQLiteParameter("@P1", DbType.Int32);
                SQLiteParameter p2 = new SQLiteParameter("@P2", DbType.String);
                SQLiteParameter p3 = new SQLiteParameter("@P3", DbType.String);
                SQLiteParameter p4 = new SQLiteParameter("@P4", DbType.String);
                SQLiteParameter p5 = new SQLiteParameter("@P5", DbType.String);
                SQLiteParameter p6 = new SQLiteParameter("@P6", DbType.Boolean);
                SQLiteParameter p7 = new SQLiteParameter("@P7", DbType.Boolean);
                SQLiteParameter p8 = new SQLiteParameter("@P8", DbType.Boolean);
                SQLiteParameter p9 = new SQLiteParameter("@P9", DbType.Boolean);
                SQLiteParameter p10 = new SQLiteParameter("@P10", DbType.Boolean);
                SQLiteParameter p11 = new SQLiteParameter("@P11", DbType.Boolean);
                SQLiteParameter p12 = new SQLiteParameter("@P12", DbType.Boolean);
                SQLiteParameter p13 = new SQLiteParameter("@P13", DbType.Boolean);
                SQLiteParameter p14 = new SQLiteParameter("@P14", DbType.String);
                SQLiteParameter p15 = new SQLiteParameter("@P15", DbType.String);
                SQLiteParameter p16 = new SQLiteParameter("@P16", DbType.String);
                SQLiteParameter p17 = new SQLiteParameter("@P17", DbType.String);
                SQLiteParameter p18 = new SQLiteParameter("@P18", DbType.String);

                SQLiteParameter idParam = new SQLiteParameter("@ID", DbType.Int32);

                insertCmd.Parameters.AddRange(new[] { p1, p2, p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18});
                updateCmd.Parameters.AddRange(new[] { p3, p4, p5, p6, p7, p8, p9, p10, p11, p12, p13, p14, p15, p16, p17, p18, idParam });

                string[] csvLines = File.ReadAllLines(csvFilePath);
                int updateCnt = 0, insertCnt = 0;

                for(int i = 0; i < csvLines.Length; i++)
                {
                    string line = csvLines[i];
                    string[] elems = line.Split(',');

                    string country = elems[1].Trim().Replace("\"", "");
                    if (country.Length < 2) continue;
                    string code = elems[2].Trim().Replace("\"", "");
                    if (code.Length < 3) continue;

                    currentLocodes.Add(country + code);

                    ccode.Value = country;
                    lcode.Value = code;

                    // Eingabeformat: https://service.unece.org/trade/locode/Service/LocodeColumn.htm

                    object lookupResult = lookupCmd.ExecuteScalar();
                    if(lookupResult != DBNull.Value)
                    {
                        int lid = Convert.ToInt32(lookupResult);
                        // UPDATE entry
                        updateCnt++;
                    }
                    else
                    {
                        // CREATE new entry
                        insertCnt++;

                    }
                    

                    Console.Write($"\r({i}/{csvLines.Length})");
                }

                Console.Write("\n");
                Console.WriteLine($"{insertCnt} new entries, {updateCnt} updated");

                // jetzt durch alle Ids in der DB laufen und mit dem Import vergleichen
                List<int> deleteIds = new List<int>();
                SQLiteCommand cmd = new SQLiteCommand(connection);
                cmd.CommandText = "SELECT countries.code, locodes.id, locodes.city_code FROM countries INNER JOIN locodes on locodes.country_id = countries.id";
                SQLiteDataReader reader = cmd.ExecuteReader();
                while(reader.Read())
                {
                    string locode = reader.GetString(0) + reader.GetString(2);
                    if (!currentLocodes.Contains(locode))
                        deleteIds.Add(reader.GetInt32(1));
                }

                Console.WriteLine($"deleting {deleteIds.Count} obsolete entries");

                SQLiteCommand delCmd = new SQLiteCommand(connection);
                delCmd.CommandText = "DELETE FROM locodes where id = @DELID";
                //  diejenigen löschen, die nicht mehr in der Quell CSV Datei auftauchen
                foreach(int deleteId in deleteIds)
                {
                    delCmd.Parameters.AddWithValue("@DELID", deleteId);
                    if (delCmd.ExecuteNonQuery() != 1)
                        _log.WarnFormat("{0} affected no rows", deleteId);
                }
            }
        }
    }
}