diff --git a/common_task.py b/common_task.py index 383ea8c..08fa458 100644 --- a/common_task.py +++ b/common_task.py @@ -201,7 +201,6 @@ if __name__ == "__main__": fresh_detail_output = f"indonesia_data/daily_upload_folder/new_jobs_on_{today_date}.csv" expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv" detail_file = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Active_{today_date}.csv" - do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id") upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes") jstIdJobDetailScraper(fresh_output, detail_file) @@ -220,7 +219,7 @@ if __name__ == "__main__": jstSGJobDetailScraper(fresh_output, detail_file) upload_file_to_bucket(detail_file, f"Compete_2_Singapore_Active_{today_date}.csv" ,today_date, "yes") elif option =="my": - search_file = f"malaysia_data/daily_search_results/id_search_{today_date}.csv" + search_file = f"malaysia_data/daily_search_results/my_search_{today_date}.csv" search_jst_malay(search_file) folder_path = "malaysia_data/daily_search_results/" search_pattern = "my_search_*.csv" diff --git a/jobstreet/jst_malay_detail.py b/jobstreet/jst_malay_detail.py index f4505c7..d35e94b 100644 --- a/jobstreet/jst_malay_detail.py +++ b/jobstreet/jst_malay_detail.py @@ -20,7 +20,9 @@ column = [ "Key Skills", "Minimum Experience", "Maximum Experience", - "Salary Detail" + "Salary Detail", + "Country" + ] def jstMalayJobDetailScraper(search_file, jd_file): @@ -102,6 +104,8 @@ def jstMalayJobDetailScraper(search_file, jd_file): "Key Skills":"", "Minimum Experience" : result['data']['jobDetail'].get('jobDetail', {'jobRequirement':""}).get('jobRequirement', {"yearsOfExperience":""}).get('yearsOfExperience'), # "Salary Detail" : result['data']['jobDetail']['header']['salary'] + "Country" : "Malaysia" + } j_writer.writerow(job) del jobIds[0] diff --git a/jobstreet/jst_sg_detail.py b/jobstreet/jst_sg_detail.py index 28fe387..2e83a43 100644 --- a/jobstreet/jst_sg_detail.py +++ b/jobstreet/jst_sg_detail.py @@ -20,7 +20,9 @@ column = [ "Key Skills", "Minimum Experience", "Maximum Experience", - "Salary Detail" + "Salary Detail", + "Country" + ] def jstSGJobDetailScraper(search_file, jd_file): @@ -102,6 +104,8 @@ def jstSGJobDetailScraper(search_file, jd_file): "Key Skills":"", "Minimum Experience" : result['data']['jobDetail'].get('jobDetail', {'jobRequirement':""}).get('jobRequirement', {"yearsOfExperience":""}).get('yearsOfExperience'), # "Salary Detail" : result['data']['jobDetail']['header']['salary'] + "Country" : "Singapore" + } j_writer.writerow(job) del jobIds[0]