foundit
/
compete_jobs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ddd

prahul11 2023-10-18 10:34:07 +05:30
parent 9c2ccafa58
commit 3949b0ef0c
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
common_task.py
View File

@ -124,7 +124,7 @@ def run_gulf_scraper(today_date):
expired_output = f"gulf_data/daily_upload_folder/Compete_1_Gulf_Archieve_{today_date}.csv" expired_output = f"gulf_data/daily_upload_folder/Compete_1_Gulf_Archieve_{today_date}.csv"
common_output = f"gulf_data/daily_common_folder/common_data_on_{today_date}.csv" common_output = f"gulf_data/daily_common_folder/common_data_on_{today_date}.csv"
do_the_difference(gulf_search_file, last_file, "jdURL", fresh_output, expired_output, common_output) do_the_difference(gulf_search_file, last_file, "jdURL", fresh_output, expired_output, common_output)
upload_file_to_bucket(expired_output, f"Compete_1_Gulf_Archieve_{today_date}.csv" ) upload_file_to_bucket(expired_output, f"Compete_1_Gulf_Archieve_{today_date}.csv" ,today_date)
start_time = time.time() start_time = time.time()
gulf_detail_file = f"gulf_data/daily_upload_folder/Compete_1_Gulf_Active_{today_date}.csv" gulf_detail_file = f"gulf_data/daily_upload_folder/Compete_1_Gulf_Active_{today_date}.csv"
gulf_detail_error_file = f"india_data/daily_error_folder/error_on_India_detail_{today_date}.txt" gulf_detail_error_file = f"india_data/daily_error_folder/error_on_India_detail_{today_date}.txt"
@ -135,7 +135,7 @@ def run_gulf_scraper(today_date):
print(f"Jobdata program took {duration_hours:.2f} hours to run.") print(f"Jobdata program took {duration_hours:.2f} hours to run.")
with open(f'gulf_data/daily_stats_folder/stats_file_of_{today_date}.txt', "a") as stat: with open(f'gulf_data/daily_stats_folder/stats_file_of_{today_date}.txt', "a") as stat:
stat.write(f"Jobdata program took {duration_hours:.2f} hours to run.\n") stat.write(f"Jobdata program took {duration_hours:.2f} hours to run.\n")
upload_file_to_bucket(gulf_detail_file, f"Compete_1_Gulf_Active_{today_date}.csv" ) upload_file_to_bucket(gulf_detail_file, f"Compete_1_Gulf_Active_{today_date}.csv" , today_date)
if __name__ == "__main__": if __name__ == "__main__":