YP_AutoCSVDownloader/CSVDownloader/Program.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using System.Linq;

using OpenQA.Selenium;
using OpenQA.Selenium.Chrome;
using OpenQA.Selenium.Support.UI;
using SeleniumExtras.WaitHelpers;

using CSVDownloader.File;
using CSVDownloader.Store;
using CSVDownloader.Store.CreditCSVData;
using CSVDownloader.Store.QRCSVData;
using CSVDownloader.Store.ElectronicMoneyDataStore;

using CSVDownloader.Web;
using CSVDownloader.Exceptions;

using MySql.Data.MySqlClient;

namespace CSVDownloader {


    class Program {
        static void Main(string[] args) {



            // 作業ディレクトリを実行ファイルに合わせる。
            var exe_path = Assembly.GetEntryAssembly().Location;
            var exe_directory_path = Path.GetDirectoryName(exe_path);
            Directory.SetCurrentDirectory(exe_directory_path);

            var controller = new Controller();

            String ret = controller.Start();


            if (ret.Length != 0) {
                // エラー検知
                Console.WriteLine(ret);
                Console.WriteLine("エラー発生　詳細はログを参照");
                Environment.Exit(1);
            }



        }
    }

    class Controller {

        private log4net.ILog logger_ = log4net.LogManager.GetLogger("");

        private List<WebController> web_controller_list_ = new List<WebController>();

        private HistoryDAO history_dao_;

        private ParkingCreditCardAgenciesSpotIDDAO parking_credit_card_agencies_spotID_dao_;

        private DaitoCreitDataStore daito_credit_store_;

        private ZeusCreditDataStore zeus_credit_store_;

        private ItecCreditDataStore itec_credit_store_;

        private HelloTechnoCreditDataStore hello_techno_credit_store_;

        private DaitoQRDataStore daito_qr_store_;

        private CreditCSVDataGMOStore gmo_credit_store_;

        private GMOElectronicMoneyDataStore gmo_electronic_money_store_;

        private ParkingCreditCardAgenciesSpotIDErrorDAO parking_credit_card_agencies_spotID_error_dao_;

        private IConfigReader config_;

        private ChromeDriver driver_;

        private MySqlConnection conn_;

        private MySqlTransaction transaction_;

        private Dictionary<Code.CreditAgent, CreditCSVDataStore> credit_datastore_map_ = new Dictionary<Code.CreditAgent, CreditCSVDataStore>();

        private Dictionary<Code.CreditAgent, QRCSVDataStore> qr_datastore_map_ = new Dictionary<Code.CreditAgent, QRCSVDataStore>();

        private Dictionary<Code.CreditAgent, ElectronicMoneyCSVDataStore> electronic_money_datastore_map_ = new Dictionary<Code.CreditAgent, ElectronicMoneyCSVDataStore>();

        private int save_chunk_num_ = 100;


        public String Start() {
            Console.WriteLine("★★★★★★自動CSVダウンロード 起動★★★★★");
            logger_.Info("★★★★★★自動CSVダウンロード　起動★★★★★");
            String returnable = "";

            // 各種ストアを用意
            history_dao_ = new HistoryDAO();
            config_ = new ConfigReader("config/config.ini");
            var mysql_param = new MySqlConnectionParameter() {
                host = "192.168.0.28",
                port = 3306,
                database = "ypark",
                user = "ypuser2",
                password = "ypp@ssw0rd2"
            };
            //var mysql_param = new MySqlConnectionParameter() {
            //    host = "localhost",
            //    port = 3306,
            //    database = "yp",
            //    user = "docker",
            //    password = "docker",
            //};

            try {
                conn_ = MySQL.GetConnection(mysql_param);


                parking_credit_card_agencies_spotID_dao_ = new ParkingCreditCardAgenciesSpotIDDAO(conn_);
                parking_credit_card_agencies_spotID_error_dao_ = new ParkingCreditCardAgenciesSpotIDErrorDAO(conn_);
                daito_credit_store_ = new DaitoCreitDataStore(conn_);
                zeus_credit_store_ = new ZeusCreditDataStore(conn_);
                itec_credit_store_ = new ItecCreditDataStore(conn_);
                hello_techno_credit_store_ = new HelloTechnoCreditDataStore(conn_);
                daito_qr_store_ = new DaitoQRDataStore(conn_);
                gmo_credit_store_ = new CreditCSVDataGMOStore(conn_);
                gmo_electronic_money_store_ = new GMOElectronicMoneyDataStore(conn_);

            } catch (Exception e) {
                logger_.Error("DB接続失敗");
                logger_.Error(e.Message);
                return "DBへのコネクト失敗";
            }

            // エラーリストを初期化する（全削除）
            parking_credit_card_agencies_spotID_error_dao_.DeleteAll();

            // サイトごとにコントローラーを用意する。
            MakeConttollers();

            // 各コントローラーを起動する。
            foreach (var web_controller in web_controller_list_) {
                var ret = HandleWebController(web_controller);
                if (ret == Code.ResultCode.NG) {
                    logger_.Error($"処理失敗 {web_controller.GetCreditAgent().ToString()}");
                    returnable = "処理失敗";
                }
            }

            // ブラウザの終了
            if (driver_ != null) {
                driver_.Quit();
            }

            // 終了
            logger_.Info("処理成功");
            return returnable;
        }

        private void MakeConttollers() {

            driver_ = DriverFactory.GetDriver();


            // CREVAS
            {
                var controller = new DaitoController(driver_);
                var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent());
                controller.SetParkingDic(dic);
                web_controller_list_.Add(controller);
                credit_datastore_map_.Add(controller.GetCreditAgent(), daito_credit_store_);
                qr_datastore_map_.Add(controller.GetCreditAgent(), daito_qr_store_);

            }
            // Zeus
            {
                var controller = new ZeusController(driver_);
                var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent());
                controller.SetParkingDic(dic);
                web_controller_list_.Add(controller);
                credit_datastore_map_.Add(controller.GetCreditAgent(), zeus_credit_store_);
            }
            // Itec
            {
                var controller = new ItecController(driver_);
                var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent());
                controller.SetParkingDic(dic);
                web_controller_list_.Add(controller);
                credit_datastore_map_.Add(controller.GetCreditAgent(), itec_credit_store_);
            }

            // HelloTechno
            {
                //var controller = new HelloTechnoController(driver_);
                //var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent());
                //controller.SetParkingDic(dic);
                //web_controller_list_.Add(controller);
                //credit_datastore_map_.Add(controller.GetCreditAgent(), hello_techno_credit_store_);
            }
            // HelloTechno(GMO)
            {
                var controller = new GMOController(driver_);
                var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent());
                controller.SetParkingDic(dic);
                web_controller_list_.Add(controller);
                credit_datastore_map_.Add(controller.GetCreditAgent(), gmo_credit_store_);
                electronic_money_datastore_map_.Add(controller.GetCreditAgent(), gmo_electronic_money_store_);
            }



        }

        private Code.ResultCode HandleWebController(WebController web_controller) {


            Code.ResultCode result_code = Code.ResultCode.OK;
            try {
                // 各ダウンロードディレクトリをクリーンする。
                CreanDirectries();

                // カード会社を特定する。
                Code.CreditAgent agent = web_controller.GetCreditAgent();
                logger_.Info($"ダウンロード開始 対象：{agent}");

                // 履歴を参照する。
                var history = history_dao_.GetHistory(agent);

                // 履歴から取得範囲を特定する。
                (var from, var to) = GetFromTo(agent, history);
                logger_.Info($"取得範囲 {from.ToString("yyyy/MM/dd")} ～ {to.ToString("yyyy/MM/dd")}");

                // ログイン情報を取得
                var login_info = GetLoginInfo(agent);

                // ログインを開始
                result_code = web_controller.Login(login_info);
                if (result_code != Code.ResultCode.OK) {
                    throw new Exception("ログイン失敗");
                }
                logger_.Info("ログイン成功");

                // ダウンロード開始
                result_code = web_controller.Download(from, to);
                if (result_code != Code.ResultCode.OK) {
                    throw new Exception("ダウンロード失敗");
                }
                logger_.Info("ダウンロード成功");

                // ログアウト
                result_code = web_controller.Logout();
                if (result_code != Code.ResultCode.OK) {
                    throw new Exception("ログアウト失敗");
                }
                logger_.Info("ログアウト成功");


                // ロード画面の表示
                var load_html_path = Path.GetFullPath(@"static\html\load.html");
                driver_.Navigate().GoToUrl($"file://{load_html_path}");


                // データリストの取得
                // 取得に失敗したものはエラーリストとして登録する。
                bool list_get_success = true;
                List<CreditCSVData> credit_info_list = null;
                List<QRCSVData> qr_info_list = null;
                List<ElectronicMoneyCSVData> electronic_money_info_list = null;

                try {
                    credit_info_list = web_controller.GetCreditCSVDataList();
                } catch (SpotNameNotMatchException e) {

                    list_get_success = false;
                    parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList());
                }

                try {
                    qr_info_list = web_controller.GetQRCSVDataList();
                } catch (SpotNameNotMatchException e) {
                    list_get_success = false;
                    parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList());
                }

                try {
                    electronic_money_info_list = web_controller.GetElectronicMoneyCSVDataList();
                } catch (SpotNameNotMatchException e) {
                    list_get_success = false;
                    parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList());
                }


                if (!list_get_success) {
                    throw new Exception("データリスト取得失敗");
                }

                // データ保存
                int delete_count_credit = 0;
                int delete_count_qr = 0;
                int delete_count_electronic_money = 0;
                transaction_ = conn_.BeginTransaction();

                // クレジット
                if (0 < credit_info_list.Count) {
                    ShowTotalCount(credit_info_list.Count);
                    var store = credit_datastore_map_[agent];
                    delete_count_credit = store.Delete(from, to);
                    var current_count = 0;
                    ShowCurrentCount(current_count);
                    var chunk_data_list = credit_info_list.Select((CreditCSVData data, int index) => new { data, index })
                        .GroupBy(x => x.index / save_chunk_num_)
                        .Select(g => g.Select(r => r.data));
                    foreach (var data_list in chunk_data_list) {
                        var list = data_list.ToList();
                        store.Save(list);
                        current_count += list.Count;
                        ShowCurrentCount(current_count);
                    }
                }
                // QR
                if (0 < qr_info_list.Count) {
                    ShowTotalCount(qr_info_list.Count);
                    var store = qr_datastore_map_[agent];
                    delete_count_qr = store.Delete(from, to);
                    var current_count = 0;
                    ShowCurrentCount(current_count);
                    var chunk_data_list = qr_info_list.Select((QRCSVData data, int index) => new { data, index })
                       .GroupBy(x => x.index / save_chunk_num_)
                       .Select(g => g.Select(r => r.data));
                    foreach (var data_list in chunk_data_list) {
                        var list = data_list.ToList();
                        store.Save(list);
                        current_count += list.Count;
                        ShowCurrentCount(current_count);
                    }
                }
                // 電子マネー
                if (0 < electronic_money_info_list.Count) {
                    ShowTotalCount(electronic_money_info_list.Count);
                    var store = electronic_money_datastore_map_[agent];
                    delete_count_electronic_money = store.Delete(from, to);
                    var current_count = 0;
                    ShowCurrentCount(current_count);
                    var chunk_data_list = electronic_money_info_list.Select((ElectronicMoneyCSVData data, int index) => new { data, index })
                       .GroupBy(x => x.index / save_chunk_num_)
                       .Select(g => g.Select(r => r.data));
                    foreach (var data_list in chunk_data_list) {
                        var list = data_list.ToList();
                        store.Save(list);
                        current_count += list.Count;
                        ShowCurrentCount(current_count);
                    }
                }


                // コミット処理
                transaction_.Commit();
                logger_.Info($"削除件数 CREDIT     :{delete_count_credit}");
                logger_.Info($"削除件数 QR         :{delete_count_qr}");
                logger_.Info($"削除件数 電子マネー :{delete_count_electronic_money}");
                logger_.Info($"登録件数 CREDIT     :{credit_info_list.Count}");
                logger_.Info($"登録件数 QR         :{qr_info_list.Count}");
                logger_.Info($"登録件数 電子マネー :{electronic_money_info_list.Count}");
                web_controller.Archive(GetArchiveFilename(agent, DateTime.Now));

                // 履歴の登録
                history_dao_.Save(agent, DateTime.Now);

                logger_.Info($"ダウンロード終了 対象：{agent}");
            } catch (Exception e) {
                // ロールバック処理
                logger_.Error(e.Message);
                logger_.Error(e.StackTrace);
                if (transaction_ != null) {
                    transaction_.Rollback();
                }
                transaction_ = null;
                return Code.ResultCode.NG;
            }

            transaction_ = null;

            return Code.ResultCode.OK;
        }

        /// <summary>
        /// データ取得範囲を特定する。<br/>
        /// </summary>
        /// <param name="history"></param>
        /// <returns></returns>
        private (DateTime, DateTime) GetFromTo(Code.CreditAgent agent, List<History> history) {
            DateTime from, to;

            if (agent == Code.CreditAgent.Zeus) {
                // Zeusの場合は、データが修正されるケースがあるため
                // 期間を制御する
                // 前回取得日から1か月遡る。
                // var prev_date = DateTime.Now.AddMonths(-1);
                // from = new DateTime(prev_date.Year, prev_date.Month, 1);
                // to = from.AddMonths(1).AddDays(-1);

                var now_date = DateTime.Now.Date;
                from = now_date.AddMonths(-1);
                to = now_date;


            } else {
                var now_date = DateTime.Now.Date;
                from = now_date.AddDays(-20);
                to = now_date;

                if (history.Count != 0) {
                    var prev_date = history[0].date;
                    if (from < prev_date) {
                        from = prev_date.Date;
                    }
                }
            }

            return (from, to);

        }

        /// <summary>
        /// コンフィグファイルよりログイン情報を取得する。
        /// </summary>
        /// <param name="agent"></param>
        /// <returns></returns>
        private LoginInfo GetLoginInfo(Code.CreditAgent agent) {
            var login_info = new LoginInfo();
            String sesction = $"LOGIN_INFO_{agent.ToString()}";
            login_info.user_name = config_.Read(sesction, "user");
            if (login_info.user_name.Length == 0) {
                throw new Exception($"ログイン情報不足 host {agent.ToString()}");
            }
            login_info.password = config_.Read(sesction, "password");
            if (login_info.password.Length == 0) {
                throw new Exception($"ログイン情報不足 password {agent.ToString()}");
            }

            login_info.company_name = config_.Read(sesction, "company");


            return login_info;
        }

        private String GetArchiveFilename(Code.CreditAgent agent, DateTime now) {
            return $"{agent}_{now.ToString("yyyyMMddHHmmss")}.zip";
        }

        private void CreanDirectries() {
            var dir_download = new DirectoryInfo(DriverFactory.GetDownloadDir());
            foreach (var file in dir_download.GetFiles()) {
                file.Delete();
            }
            var tmp_download = new DirectoryInfo(DriverFactory.GetTmpDownloadDir());
            foreach (var file in tmp_download.GetFiles()) {
                file.Delete();
            }
        }

        private void ShowTotalCount(int num) {
            driver_.ExecuteScript($"setTotal({num})");
        }

        private void ShowCurrentCount(int num) {
            driver_.ExecuteScript($"setCount({num})");
        }
    }
}