using System; using System.Collections.Generic; using System.IO; using System.Reflection; using System.Linq; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; using OpenQA.Selenium.Support.UI; using SeleniumExtras.WaitHelpers; using CSVDownloader.File; using CSVDownloader.Store; using CSVDownloader.Store.CreditCSVData; using CSVDownloader.Store.QRCSVData; using CSVDownloader.Store.ElectronicMoneyDataStore; using CSVDownloader.Web; using CSVDownloader.Exceptions; using MySql.Data.MySqlClient; namespace CSVDownloader { class Program { static void Main(string[] args) { // 作業ディレクトリを実行ファイルに合わせる。 var exe_path = Assembly.GetEntryAssembly().Location; var exe_directory_path = Path.GetDirectoryName(exe_path); Directory.SetCurrentDirectory(exe_directory_path); var controller = new Controller(); String ret = controller.Start(); if (ret.Length != 0) { // エラー検知 Console.WriteLine(ret); Console.WriteLine("エラー発生 詳細はログを参照"); Environment.Exit(1); } } } class Controller { private log4net.ILog logger_ = log4net.LogManager.GetLogger(""); private List web_controller_list_ = new List(); private HistoryDAO history_dao_; private ParkingCreditCardAgenciesSpotIDDAO parking_credit_card_agencies_spotID_dao_; private DaitoCreitDataStore daito_credit_store_; private ZeusCreditDataStore zeus_credit_store_; private ItecCreditDataStore itec_credit_store_; private HelloTechnoCreditDataStore hello_techno_credit_store_; private DaitoQRDataStore daito_qr_store_; private CreditCSVDataGMOStore gmo_credit_store_; private GMOElectronicMoneyDataStore gmo_electronic_money_store_; private ParkingCreditCardAgenciesSpotIDErrorDAO parking_credit_card_agencies_spotID_error_dao_; private IConfigReader config_; private ChromeDriver driver_; private MySqlConnection conn_; private MySqlTransaction transaction_; private Dictionary credit_datastore_map_ = new Dictionary(); private Dictionary qr_datastore_map_ = new Dictionary(); private Dictionary electronic_money_datastore_map_ = new Dictionary(); private int save_chunk_num_ = 100; public String Start() { Console.WriteLine("★★★★★★自動CSVダウンロード 起動★★★★★"); logger_.Info("★★★★★★自動CSVダウンロード 起動★★★★★"); String returnable = ""; // 各種ストアを用意 history_dao_ = new HistoryDAO(); config_ = new ConfigReader("config/config.ini"); var mysql_param = new MySqlConnectionParameter() { host = "192.168.0.28", port = 3306, database = "ypark", user = "ypuser2", password = "ypp@ssw0rd2" }; //var mysql_param = new MySqlConnectionParameter() { // host = "localhost", // port = 3306, // database = "yp", // user = "docker", // password = "docker", //}; try { conn_ = MySQL.GetConnection(mysql_param); parking_credit_card_agencies_spotID_dao_ = new ParkingCreditCardAgenciesSpotIDDAO(conn_); parking_credit_card_agencies_spotID_error_dao_ = new ParkingCreditCardAgenciesSpotIDErrorDAO(conn_); daito_credit_store_ = new DaitoCreitDataStore(conn_); zeus_credit_store_ = new ZeusCreditDataStore(conn_); itec_credit_store_ = new ItecCreditDataStore(conn_); hello_techno_credit_store_ = new HelloTechnoCreditDataStore(conn_); daito_qr_store_ = new DaitoQRDataStore(conn_); gmo_credit_store_ = new CreditCSVDataGMOStore(conn_); gmo_electronic_money_store_ = new GMOElectronicMoneyDataStore(conn_); } catch (Exception e) { logger_.Error("DB接続失敗"); logger_.Error(e.Message); return "DBへのコネクト失敗"; } // エラーリストを初期化する(全削除) parking_credit_card_agencies_spotID_error_dao_.DeleteAll(); // サイトごとにコントローラーを用意する。 MakeConttollers(); // 各コントローラーを起動する。 foreach (var web_controller in web_controller_list_) { var ret = HandleWebController(web_controller); if (ret == Code.ResultCode.NG) { logger_.Error($"処理失敗 {web_controller.GetCreditAgent().ToString()}"); returnable = "処理失敗"; } } // ブラウザの終了 if (driver_ != null) { driver_.Quit(); } // 終了 logger_.Info("処理成功"); return returnable; } private void MakeConttollers() { driver_ = DriverFactory.GetDriver(); // CREVAS { var controller = new DaitoController(driver_); var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); controller.SetParkingDic(dic); web_controller_list_.Add(controller); credit_datastore_map_.Add(controller.GetCreditAgent(), daito_credit_store_); qr_datastore_map_.Add(controller.GetCreditAgent(), daito_qr_store_); } // Zeus { var controller = new ZeusController(driver_); var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); controller.SetParkingDic(dic); web_controller_list_.Add(controller); credit_datastore_map_.Add(controller.GetCreditAgent(), zeus_credit_store_); } // Itec { var controller = new ItecController(driver_); var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); controller.SetParkingDic(dic); web_controller_list_.Add(controller); credit_datastore_map_.Add(controller.GetCreditAgent(), itec_credit_store_); } // HelloTechno { //var controller = new HelloTechnoController(driver_); //var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); //controller.SetParkingDic(dic); //web_controller_list_.Add(controller); //credit_datastore_map_.Add(controller.GetCreditAgent(), hello_techno_credit_store_); } // HelloTechno(GMO) { var controller = new GMOController(driver_); var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); controller.SetParkingDic(dic); web_controller_list_.Add(controller); credit_datastore_map_.Add(controller.GetCreditAgent(), gmo_credit_store_); electronic_money_datastore_map_.Add(controller.GetCreditAgent(), gmo_electronic_money_store_); } } private Code.ResultCode HandleWebController(WebController web_controller) { Code.ResultCode result_code = Code.ResultCode.OK; try { // 各ダウンロードディレクトリをクリーンする。 CreanDirectries(); // カード会社を特定する。 Code.CreditAgent agent = web_controller.GetCreditAgent(); logger_.Info($"ダウンロード開始 対象:{agent}"); // 履歴を参照する。 var history = history_dao_.GetHistory(agent); // 履歴から取得範囲を特定する。 (var from, var to) = GetFromTo(agent, history); logger_.Info($"取得範囲 {from.ToString("yyyy/MM/dd")} ~ {to.ToString("yyyy/MM/dd")}"); // ログイン情報を取得 var login_info = GetLoginInfo(agent); // ログインを開始 result_code = web_controller.Login(login_info); if (result_code != Code.ResultCode.OK) { throw new Exception("ログイン失敗"); } logger_.Info("ログイン成功"); // ダウンロード開始 result_code = web_controller.Download(from, to); if (result_code != Code.ResultCode.OK) { throw new Exception("ダウンロード失敗"); } logger_.Info("ダウンロード成功"); // ログアウト result_code = web_controller.Logout(); if (result_code != Code.ResultCode.OK) { throw new Exception("ログアウト失敗"); } logger_.Info("ログアウト成功"); // ロード画面の表示 var load_html_path = Path.GetFullPath(@"static\html\load.html"); driver_.Navigate().GoToUrl($"file://{load_html_path}"); // データリストの取得 // 取得に失敗したものはエラーリストとして登録する。 bool list_get_success = true; List credit_info_list = null; List qr_info_list = null; List electronic_money_info_list = null; try { credit_info_list = web_controller.GetCreditCSVDataList(); } catch (SpotNameNotMatchException e) { list_get_success = false; parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList()); } try { qr_info_list = web_controller.GetQRCSVDataList(); } catch (SpotNameNotMatchException e) { list_get_success = false; parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList()); } try { electronic_money_info_list = web_controller.GetElectronicMoneyCSVDataList(); } catch (SpotNameNotMatchException e) { list_get_success = false; parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList()); } if (!list_get_success) { throw new Exception("データリスト取得失敗"); } // データ保存 int delete_count_credit = 0; int delete_count_qr = 0; int delete_count_electronic_money = 0; transaction_ = conn_.BeginTransaction(); // クレジット if (0 < credit_info_list.Count) { ShowTotalCount(credit_info_list.Count); var store = credit_datastore_map_[agent]; delete_count_credit = store.Delete(from, to); var current_count = 0; ShowCurrentCount(current_count); var chunk_data_list = credit_info_list.Select((CreditCSVData data, int index) => new { data, index }) .GroupBy(x => x.index / save_chunk_num_) .Select(g => g.Select(r => r.data)); foreach (var data_list in chunk_data_list) { var list = data_list.ToList(); store.Save(list); current_count += list.Count; ShowCurrentCount(current_count); } } // QR if (0 < qr_info_list.Count) { ShowTotalCount(qr_info_list.Count); var store = qr_datastore_map_[agent]; delete_count_qr = store.Delete(from, to); var current_count = 0; ShowCurrentCount(current_count); var chunk_data_list = qr_info_list.Select((QRCSVData data, int index) => new { data, index }) .GroupBy(x => x.index / save_chunk_num_) .Select(g => g.Select(r => r.data)); foreach (var data_list in chunk_data_list) { var list = data_list.ToList(); store.Save(list); current_count += list.Count; ShowCurrentCount(current_count); } } // 電子マネー if (0 < electronic_money_info_list.Count) { ShowTotalCount(electronic_money_info_list.Count); var store = electronic_money_datastore_map_[agent]; delete_count_electronic_money = store.Delete(from, to); var current_count = 0; ShowCurrentCount(current_count); var chunk_data_list = electronic_money_info_list.Select((ElectronicMoneyCSVData data, int index) => new { data, index }) .GroupBy(x => x.index / save_chunk_num_) .Select(g => g.Select(r => r.data)); foreach (var data_list in chunk_data_list) { var list = data_list.ToList(); store.Save(list); current_count += list.Count; ShowCurrentCount(current_count); } } // コミット処理 transaction_.Commit(); logger_.Info($"削除件数 CREDIT :{delete_count_credit}"); logger_.Info($"削除件数 QR :{delete_count_qr}"); logger_.Info($"削除件数 電子マネー :{delete_count_electronic_money}"); logger_.Info($"登録件数 CREDIT :{credit_info_list.Count}"); logger_.Info($"登録件数 QR :{qr_info_list.Count}"); logger_.Info($"登録件数 電子マネー :{electronic_money_info_list.Count}"); web_controller.Archive(GetArchiveFilename(agent, DateTime.Now)); // 履歴の登録 history_dao_.Save(agent, DateTime.Now); logger_.Info($"ダウンロード終了 対象:{agent}"); } catch (Exception e) { // ロールバック処理 logger_.Error(e.Message); logger_.Error(e.StackTrace); if (transaction_ != null) { transaction_.Rollback(); } transaction_ = null; return Code.ResultCode.NG; } transaction_ = null; return Code.ResultCode.OK; } /// /// データ取得範囲を特定する。
///
/// /// private (DateTime, DateTime) GetFromTo(Code.CreditAgent agent, List history) { DateTime from, to; if (agent == Code.CreditAgent.Zeus) { // Zeusの場合は、データが修正されるケースがあるため // 期間を制御する // 前回取得日から1か月遡る。 // var prev_date = DateTime.Now.AddMonths(-1); // from = new DateTime(prev_date.Year, prev_date.Month, 1); // to = from.AddMonths(1).AddDays(-1); var now_date = DateTime.Now.Date; from = now_date.AddMonths(-1); to = now_date; } else { var now_date = DateTime.Now.Date; from = now_date.AddDays(-20); to = now_date; if (history.Count != 0) { var prev_date = history[0].date; if (from < prev_date) { from = prev_date.Date; } } } return (from, to); } /// /// コンフィグファイルよりログイン情報を取得する。 /// /// /// private LoginInfo GetLoginInfo(Code.CreditAgent agent) { var login_info = new LoginInfo(); String sesction = $"LOGIN_INFO_{agent.ToString()}"; login_info.user_name = config_.Read(sesction, "user"); if (login_info.user_name.Length == 0) { throw new Exception($"ログイン情報不足 host {agent.ToString()}"); } login_info.password = config_.Read(sesction, "password"); if (login_info.password.Length == 0) { throw new Exception($"ログイン情報不足 password {agent.ToString()}"); } login_info.company_name = config_.Read(sesction, "company"); return login_info; } private String GetArchiveFilename(Code.CreditAgent agent, DateTime now) { return $"{agent}_{now.ToString("yyyyMMddHHmmss")}.zip"; } private void CreanDirectries() { var dir_download = new DirectoryInfo(DriverFactory.GetDownloadDir()); foreach (var file in dir_download.GetFiles()) { file.Delete(); } var tmp_download = new DirectoryInfo(DriverFactory.GetTmpDownloadDir()); foreach (var file in tmp_download.GetFiles()) { file.Delete(); } } private void ShowTotalCount(int num) { driver_.ExecuteScript($"setTotal({num})"); } private void ShowCurrentCount(int num) { driver_.ExecuteScript($"setCount({num})"); } } }