From 41cb298cdd600435b8fb7b064cd989e3e5af93cc Mon Sep 17 00:00:00 2001 From: prahul11 Date: Tue, 7 Nov 2023 14:29:09 +0530 Subject: [PATCH] Update common_task.py --- common_task.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/common_task.py b/common_task.py index 51da29a..1dac0b5 100644 --- a/common_task.py +++ b/common_task.py @@ -132,8 +132,8 @@ def run_india_scraper(today_date): stat.write(f"Jobdata program took {duration_hours:.2f} hours to run.\n") current_date = datetime.now() today_date = current_date.strftime('%d-%m-%Y') - upload_file_to_bucket(expired_output, f"Compete_1_India_Archive_{today_date}.csv" , today_date) - upload_file_to_bucket(india_detail_file, f"Compete_1_India_Active_{today_date}.csv" , today_date) + upload_file_to_bucket(expired_output, f"Compete_1_India_Archive_{today_date}.csv" , today_date, None) + upload_file_to_bucket(india_detail_file, f"Compete_1_India_Active_{today_date}.csv" , today_date, None) def run_gulf_scraper(today_date): gulfSearch() @@ -146,7 +146,7 @@ def run_gulf_scraper(today_date): expired_output = f"gulf_data/daily_upload_folder/Compete_1_Gulf_Archive_{today_date}.csv" common_output = f"gulf_data/daily_common_folder/common_data_on_{today_date}.csv" do_the_difference(gulf_search_file, last_file, "jdURL", fresh_output, expired_output, common_output, "g") - upload_file_to_bucket(expired_output, f"Compete_1_Gulf_Archive_{today_date}.csv" ,today_date) + upload_file_to_bucket(expired_output, f"Compete_1_Gulf_Archive_{today_date}.csv" ,today_date, None) start_time = time.time() gulf_detail_file = f"gulf_data/daily_upload_folder/Compete_1_Gulf_Active_{today_date}.csv" gulf_detail_error_file = f"india_data/daily_error_folder/error_on_India_detail_{today_date}.txt" @@ -200,6 +200,32 @@ if __name__ == "__main__": upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes") jstIdJobDetailScraper(fresh_output, f"Compete_2_Indonesia_Active_{today_date}.csv") upload_file_to_bucket(detail_file, f"Compete_2_Indonesia_Active_{today_date}.csv" ,today_date, "yes") + elif option =="sg": + search_file = f"singapore_data/daily_search_results/sg_search_{today_date}.csv" + search_jst_id(search_file) + folder_path = "singapore_data/daily_search_results/" + search_pattern = "sg_search_*.csv" + last_file = find_second_latest_file(folder_path, search_pattern) + fresh_output = f"singapore_data/daily_process_folder/new_jobs_on_{today_date}.csv" + expired_output = f"singapore_data/daily_upload_folder/Compete_2_Singapore_Archive_{today_date}.csv" + detail_file = f"singapore_data/daily_upload_folder/Compete_2_Singapore_Active_{today_date}.csv" + do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id") + upload_file_to_bucket(expired_output, f"Compete_2_Singapore_Archive_{today_date}.csv" ,today_date,"yes") + jstIdJobDetailScraper(fresh_output, f"Compete_2_Singapore_Active_{today_date}.csv") + upload_file_to_bucket(detail_file, f"Compete_2_Singapore_Active_{today_date}.csv" ,today_date, "yes") + elif option =="my": + search_file = f"malaysia_data/daily_search_results/id_search_{today_date}.csv" + search_jst_id(search_file) + folder_path = "malaysia_data/daily_search_results/" + search_pattern = "my_search_*.csv" + last_file = find_second_latest_file(folder_path, search_pattern) + fresh_output = f"malaysia_data/daily_process_folder/new_jobs_on_{today_date}.csv" + expired_output = f"malaysia_data/daily_upload_folder/Compete_2_Malaysia_Archive_{today_date}.csv" + detail_file = f"malaysia_data/daily_upload_folder/Compete_2_Malaysia_Active_{today_date}.csv" + do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id") + upload_file_to_bucket(expired_output, f"Compete_2_Malaysia_Archive_{today_date}.csv" ,today_date,"yes") + jstIdJobDetailScraper(fresh_output, f"Compete_2_Malaysia_Active_{today_date}.csv") + upload_file_to_bucket(detail_file, f"Compete_2_Malaysia_Active_{today_date}.csv" ,today_date, "yes") else: print("Invalid argument. Please use 'gulf' or 'india' as the argument.")