Update common_task.py
parent
0cd1352bb9
commit
e5ddc1d4a0
|
@ -202,9 +202,9 @@ if __name__ == "__main__":
|
|||
expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv"
|
||||
detail_file = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Active_{today_date}.csv"
|
||||
do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id")
|
||||
upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes")
|
||||
# upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes")
|
||||
jstIdJobDetailScraper(fresh_output, detail_file)
|
||||
upload_file_to_bucket(detail_file, f"Compete_2_Indonesia_Active_{today_date}.csv" ,today_date, "yes")
|
||||
# upload_file_to_bucket(detail_file, f"Compete_2_Indonesia_Active_{today_date}.csv" ,today_date, "yes")
|
||||
elif option =="sg":
|
||||
search_file = f"singapore_data/daily_search_results/sg_search_{today_date}.csv"
|
||||
search_jst_sg(search_file)
|
||||
|
@ -215,9 +215,9 @@ if __name__ == "__main__":
|
|||
expired_output = f"singapore_data/daily_upload_folder/Compete_2_Singapore_Archive_{today_date}.csv"
|
||||
detail_file = f"singapore_data/daily_upload_folder/Compete_2_Singapore_Active_{today_date}.csv"
|
||||
do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id")
|
||||
upload_file_to_bucket(expired_output, f"Compete_2_Singapore_Archive_{today_date}.csv" ,today_date,"yes")
|
||||
# upload_file_to_bucket(expired_output, f"Compete_2_Singapore_Archive_{today_date}.csv" ,today_date,"yes")
|
||||
jstSGJobDetailScraper(fresh_output, detail_file)
|
||||
upload_file_to_bucket(detail_file, f"Compete_2_Singapore_Active_{today_date}.csv" ,today_date, "yes")
|
||||
# upload_file_to_bucket(detail_file, f"Compete_2_Singapore_Active_{today_date}.csv" ,today_date, "yes")
|
||||
elif option =="my":
|
||||
search_file = f"malaysia_data/daily_search_results/my_search_{today_date}.csv"
|
||||
search_jst_malay(search_file)
|
||||
|
@ -228,9 +228,9 @@ if __name__ == "__main__":
|
|||
expired_output = f"malaysia_data/daily_upload_folder/Compete_2_Malaysia_Archive_{today_date}.csv"
|
||||
detail_file = f"malaysia_data/daily_upload_folder/Compete_2_Malaysia_Active_{today_date}.csv"
|
||||
do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id")
|
||||
upload_file_to_bucket(expired_output, f"Compete_2_Malaysia_Archive_{today_date}.csv" ,today_date,"yes")
|
||||
# upload_file_to_bucket(expired_output, f"Compete_2_Malaysia_Archive_{today_date}.csv" ,today_date,"yes")
|
||||
jstMalayJobDetailScraper(fresh_output, detail_file)
|
||||
upload_file_to_bucket(detail_file, f"Compete_2_Malaysia_Active_{today_date}.csv" ,today_date, "yes")
|
||||
# upload_file_to_bucket(detail_file, f"Compete_2_Malaysia_Active_{today_date}.csv" ,today_date, "yes")
|
||||
|
||||
else:
|
||||
print("Invalid argument. Please use 'gulf' or 'india' as the argument.")
|
||||
|
|
Loading…
Reference in New Issue