using System; using System.Collections.Generic; using System.IO; using System.Reflection; using System.Linq; using System.Text.RegularExpressions; using OpenQA.Selenium; using OpenQA.Selenium.Chrome; using OpenQA.Selenium.Support.UI; using SeleniumExtras.WaitHelpers; using CSVDownloader.Code; using CSVDownloader.File; using CSVDownloader.Store; using CSVDownloader.Store.CreditCSVData; using CSVDownloader.Store.QRCSVData; using CSVDownloader.Store.ElectronicMoneyDataStore; using CSVDownloader.Web; using CSVDownloader.Exceptions; using MySql.Data.MySqlClient; namespace CSVDownloader { class Program { static void Main(string[] args) { // 作業ディレクトリを実行ファイルに合わせる。 var exe_path = Assembly.GetEntryAssembly().Location; var exe_directory_path = Path.GetDirectoryName(exe_path); Directory.SetCurrentDirectory(exe_directory_path); var controller = new Controller(args); String ret = controller.Start(); if (ret.Length != 0) { // エラー検知 Console.WriteLine(ret); Console.WriteLine("エラー発生 詳細はログを参照"); Environment.Exit(1); } } } class Controller { private log4net.ILog logger_ = log4net.LogManager.GetLogger(""); private List web_controller_list_ = new List(); private HistoryDAO history_dao_; private ParkingCreditCardAgenciesSpotIDDAO parking_credit_card_agencies_spotID_dao_; private DaitoCreitDataStore daito_credit_store_; private ZeusCreditDataStore zeus_credit_store_; private ItecCreditDataStore itec_credit_store_; private HelloTechnoCreditDataStore hello_techno_credit_store_; private DaitoQRDataStore daito_qr_store_; private DaitoElectronicMoneyDataStore daito_electronic_money_store_; private CreditCSVDataGMOStore gmo_credit_store_; private GMOElectronicMoneyDataStore gmo_electronic_money_store_; private ParkingCreditCardAgenciesSpotIDErrorDAO parking_credit_card_agencies_spotID_error_dao_; private IConfigReader config_; private ChromeDriver driver_; private MySqlConnection conn_; private MySqlTransaction transaction_; private Dictionary credit_datastore_map_ = new Dictionary(); private Dictionary qr_datastore_map_ = new Dictionary(); private Dictionary electronic_money_datastore_map_ = new Dictionary(); private int save_chunk_num_ = 100; // パラメータあり起動時の動作条件------ START private bool hasParam_ = false; private DateTime paramDateFrom_ = DateTime.Today; private DateTime paramDateTo_ = DateTime.Today; private string paramAgentCode_ = ""; // パラメータあり起動時の動作条件------ END public Controller(string[] args) { // 引数なしは通常動作 if (args.Count() == 0) { return; } // 開始終了日付指定 決済会社コード指定 if (args.Count() == 3) { DateTime from = DateTime.Today; DateTime to = DateTime.Today; var fromStr = args[0]; var toStr = args[1]; var dateReg = new Regex(@"^\d{4}/\d{2}/\d{2}$"); if (dateReg.IsMatch(fromStr) && !DateTime.TryParse(fromStr, out from)) { throw new Exception("起動引数不正 FROM 日付フォーマット"); } if (dateReg.IsMatch(toStr) && !DateTime.TryParse(toStr, out to)) { throw new Exception("起動引数不正 TO 日付フォーマット"); } if (to < from) { throw new Exception("起動引数不正 FROMが大きい"); } hasParam_ = true; paramDateFrom_ = from; paramDateTo_ = to; paramAgentCode_ = args[2]; var message = $"★★★★★★起動引数あり FROM:{fromStr} TO:{toStr} AGENT:{paramAgentCode_}★★★★★★"; Console.WriteLine(message); logger_.Info(message); return; } throw new Exception("起動引数不正"); } public String Start() { Console.WriteLine("★★★★★★自動CSVダウンロード 起動★★★★★"); logger_.Info("★★★★★★自動CSVダウンロード 起動★★★★★"); String returnable = ""; // 各種ストアを用意 history_dao_ = new HistoryDAO(); config_ = new ConfigReader("config/config.ini"); var mysql_param = new MySqlConnectionParameter() { host = "192.168.0.28", port = 3306, database = "ypark", user = "ypuser2", password = "ypp@ssw0rd2" }; //var mysql_param = new MySqlConnectionParameter() { // host = "localhost", // port = 3306, // database = "yp", // user = "docker", // password = "docker", //}; try { conn_ = MySQL.GetConnection(mysql_param); parking_credit_card_agencies_spotID_dao_ = new ParkingCreditCardAgenciesSpotIDDAO(conn_); parking_credit_card_agencies_spotID_error_dao_ = new ParkingCreditCardAgenciesSpotIDErrorDAO(conn_); daito_credit_store_ = new DaitoCreitDataStore(conn_); zeus_credit_store_ = new ZeusCreditDataStore(conn_); itec_credit_store_ = new ItecCreditDataStore(conn_); hello_techno_credit_store_ = new HelloTechnoCreditDataStore(conn_); daito_qr_store_ = new DaitoQRDataStore(conn_); gmo_credit_store_ = new CreditCSVDataGMOStore(conn_); gmo_electronic_money_store_ = new GMOElectronicMoneyDataStore(conn_); daito_electronic_money_store_ = new DaitoElectronicMoneyDataStore(conn_); } catch (Exception e) { logger_.Error("DB接続失敗"); logger_.Error(e.Message); return "DBへのコネクト失敗"; } // エラーリストを初期化する(全削除) parking_credit_card_agencies_spotID_error_dao_.DeleteAll(); // サイトごとにコントローラーを用意する。 MakeConttollers(); // 各コントローラーを起動する。 foreach (var web_controller in web_controller_list_) { var ret = HandleWebController(web_controller); if (ret == Code.ResultCode.NG) { logger_.Error($"処理失敗 {web_controller.GetCreditAgent().ToString()}"); returnable = "処理失敗"; } } // ブラウザの終了 if (driver_ != null) { driver_.Quit(); } // 終了 logger_.Info("処理成功"); return returnable; } private void MakeConttollers() { driver_ = DriverFactory.GetDriver(); // CREVAS { var controller = new DaitoController(driver_); if (!hasParam_ || paramAgentCode_ == controller.GetCreditAgentCode()) { var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); controller.SetParkingDic(dic); web_controller_list_.Add(controller); credit_datastore_map_.Add(controller.GetCreditAgent(), daito_credit_store_); qr_datastore_map_.Add(controller.GetCreditAgent(), daito_qr_store_); electronic_money_datastore_map_.Add(controller.GetCreditAgent(), daito_electronic_money_store_); } } // Zeus { var controller = new ZeusController(driver_); if (!hasParam_ || paramAgentCode_ == controller.GetCreditAgentCode()) { var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); controller.SetParkingDic(dic); web_controller_list_.Add(controller); credit_datastore_map_.Add(controller.GetCreditAgent(), zeus_credit_store_); } } // Itec { //var controller = new ItecController(driver_); //var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); //controller.SetParkingDic(dic); //web_controller_list_.Add(controller); //credit_datastore_map_.Add(controller.GetCreditAgent(), itec_credit_store_); } // HelloTechno { //var controller = new HelloTechnoController(driver_); //var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); //controller.SetParkingDic(dic); //web_controller_list_.Add(controller); //credit_datastore_map_.Add(controller.GetCreditAgent(), hello_techno_credit_store_); } // HelloTechno(GMO) { //var controller = new GMOController(driver_); //if (!hasParam_ || paramAgentCode_ == controller.GetCreditAgentCode()) { // var dic = parking_credit_card_agencies_spotID_dao_.GetDictionary(controller.GetCreditAgent()); // controller.SetParkingDic(dic); // web_controller_list_.Add(controller); // credit_datastore_map_.Add(controller.GetCreditAgent(), gmo_credit_store_); // electronic_money_datastore_map_.Add(controller.GetCreditAgent(), gmo_electronic_money_store_); //} } } private Code.ResultCode HandleWebController(WebController web_controller) { Code.ResultCode result_code = Code.ResultCode.OK; // 実行対象日判定 if (hasParam_) { logger_.Info($"起動引数ありのため実行対象日判定をスキップ"); } else if (!web_controller.IsWorkDate()) { logger_.Info($"処理対象日外:{ web_controller.GetCreditAgent()}"); return result_code; } try { // 各ダウンロードディレクトリをクリーンする。 CreanDirectries(); // カード会社を特定する。 Code.CreditAgent agent = web_controller.GetCreditAgent(); logger_.Info($"ダウンロード開始 対象:{agent}"); // 履歴を参照する。 var history = history_dao_.GetHistory(agent); // 履歴から取得範囲を特定する。 (var from, var to) = GetFromTo(agent, history); logger_.Info($"取得範囲 {from.ToString("yyyy/MM/dd")} ~ {to.ToString("yyyy/MM/dd")}"); // ログイン情報を取得 var login_info = GetLoginInfo(agent); // ログインを開始 result_code = web_controller.Login(login_info); if (result_code != Code.ResultCode.OK) { throw new Exception("ログイン失敗"); } logger_.Info("ログイン成功"); // ダウンロード開始 result_code = web_controller.Download(from, to); if (result_code != Code.ResultCode.OK) { throw new Exception("ダウンロード失敗"); } logger_.Info("ダウンロード成功"); // ログアウト result_code = web_controller.Logout(); if (result_code != Code.ResultCode.OK) { throw new Exception("ログアウト失敗"); } logger_.Info("ログアウト成功"); // ロード画面の表示 var load_html_path = Path.GetFullPath(@"static\html\load.html"); driver_.Navigate().GoToUrl($"file://{load_html_path}"); // データリストの取得 // 取得に失敗したものはエラーリストとして登録する。 bool list_get_success = true; List credit_info_list = null; List qr_info_list = null; List electronic_money_info_list = null; try { credit_info_list = web_controller.GetCreditCSVDataList(); } catch (SpotNameNotMatchException e) { list_get_success = false; parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList()); } try { qr_info_list = web_controller.GetQRCSVDataList(); } catch (SpotNameNotMatchException e) { list_get_success = false; parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList()); } try { electronic_money_info_list = web_controller.GetElectronicMoneyCSVDataList(); } catch (SpotNameNotMatchException e) { list_get_success = false; parking_credit_card_agencies_spotID_error_dao_.Save(e.GetList()); } if (!list_get_success) { throw new Exception("データリスト取得失敗"); } // データ保存 int delete_count_credit = 0; int delete_count_qr = 0; int delete_count_electronic_money = 0; transaction_ = conn_.BeginTransaction(); // クレジット if (0 < credit_info_list.Count) { ShowTotalCount(credit_info_list.Count); var store = credit_datastore_map_[agent]; delete_count_credit = store.Delete(from, to); var current_count = 0; ShowCurrentCount(current_count); var chunk_data_list = credit_info_list.Select((CreditCSVData data, int index) => new { data, index }) .GroupBy(x => x.index / save_chunk_num_) .Select(g => g.Select(r => r.data)); foreach (var data_list in chunk_data_list) { var list = data_list.ToList(); store.Save(list); current_count += list.Count; ShowCurrentCount(current_count); } } // QR if (0 < qr_info_list.Count) { ShowTotalCount(qr_info_list.Count); var store = qr_datastore_map_[agent]; delete_count_qr = store.Delete(from, to); var current_count = 0; ShowCurrentCount(current_count); var chunk_data_list = qr_info_list.Select((QRCSVData data, int index) => new { data, index }) .GroupBy(x => x.index / save_chunk_num_) .Select(g => g.Select(r => r.data)); foreach (var data_list in chunk_data_list) { var list = data_list.ToList(); store.Save(list); current_count += list.Count; ShowCurrentCount(current_count); } } // 電子マネー if (0 < electronic_money_info_list.Count) { ShowTotalCount(electronic_money_info_list.Count); var store = electronic_money_datastore_map_[agent]; delete_count_electronic_money = store.Delete(from, to); var current_count = 0; ShowCurrentCount(current_count); var chunk_data_list = electronic_money_info_list.Select((ElectronicMoneyCSVData data, int index) => new { data, index }) .GroupBy(x => x.index / save_chunk_num_) .Select(g => g.Select(r => r.data)); foreach (var data_list in chunk_data_list) { var list = data_list.ToList(); store.Save(list); current_count += list.Count; ShowCurrentCount(current_count); } } // コミット処理 transaction_.Commit(); logger_.Info($"削除件数 CREDIT :{delete_count_credit}"); logger_.Info($"削除件数 QR :{delete_count_qr}"); logger_.Info($"削除件数 電子マネー :{delete_count_electronic_money}"); logger_.Info($"登録件数 CREDIT :{credit_info_list.Count}"); logger_.Info($"登録件数 QR :{qr_info_list.Count}"); logger_.Info($"登録件数 電子マネー :{electronic_money_info_list.Count}"); web_controller.Archive(GetArchiveFilename(agent, DateTime.Now)); // 履歴の登録 if (!hasParam_) { history_dao_.Save(agent, DateTime.Now); } else { logger_.Info("パラメータ指定起動のため、取得履歴は保存しない"); } logger_.Info($"ダウンロード終了 対象:{agent}"); } catch (Exception e) { // ロールバック処理 logger_.Error(e.Message); logger_.Error(e.StackTrace); if (transaction_ != null) { transaction_.Rollback(); } transaction_ = null; return Code.ResultCode.NG; } transaction_ = null; return Code.ResultCode.OK; } /// /// データ取得範囲を特定する。
///
/// /// private (DateTime, DateTime) GetFromTo(Code.CreditAgent agent, List history) { // 起動パラメータがある場合はそれを設定する if (hasParam_) { return (paramDateFrom_, paramDateTo_); } DateTime from, to; if (agent == Code.CreditAgent.Zeus) { // Zeusの場合は、データが修正されるケースがあるため // 期間を制御する // 前回取得日から1か月遡る。 // var prev_date = DateTime.Now.AddMonths(-1); // from = new DateTime(prev_date.Year, prev_date.Month, 1); // to = from.AddMonths(1).AddDays(-1); var now_date = DateTime.Now.Date; from = now_date.AddMonths(-1); to = now_date; } else { var now_date = DateTime.Now.Date; from = now_date.AddDays(-20); to = now_date; if (history.Count != 0) { var prev_date = history[0].date; if (from < prev_date) { from = prev_date.Date; } } } return (from, to); } /// /// コンフィグファイルよりログイン情報を取得する。 /// /// /// private LoginInfo GetLoginInfo(Code.CreditAgent agent) { var login_info = new LoginInfo(); String sesction = $"LOGIN_INFO_{agent.ToString()}"; login_info.user_name = config_.Read(sesction, "user"); if (login_info.user_name.Length == 0) { throw new Exception($"ログイン情報不足 host {agent.ToString()}"); } login_info.password = config_.Read(sesction, "password"); if (login_info.password.Length == 0) { throw new Exception($"ログイン情報不足 password {agent.ToString()}"); } login_info.company_name = config_.Read(sesction, "company"); return login_info; } private String GetArchiveFilename(Code.CreditAgent agent, DateTime now) { return $"{agent}_{now.ToString("yyyyMMddHHmmss")}.zip"; } private void CreanDirectries() { var dir_download = new DirectoryInfo(DriverFactory.GetDownloadDir()); foreach (var file in dir_download.GetFiles()) { file.Delete(); } var tmp_download = new DirectoryInfo(DriverFactory.GetTmpDownloadDir()); foreach (var file in tmp_download.GetFiles()) { file.Delete(); } } private void ShowTotalCount(int num) { driver_.ExecuteScript($"setTotal({num})"); } private void ShowCurrentCount(int num) { driver_.ExecuteScript($"setCount({num})"); } } }