jhg
parent
300650d08f
commit
0cd1352bb9
|
@ -201,7 +201,6 @@ if __name__ == "__main__":
|
||||||
fresh_detail_output = f"indonesia_data/daily_upload_folder/new_jobs_on_{today_date}.csv"
|
fresh_detail_output = f"indonesia_data/daily_upload_folder/new_jobs_on_{today_date}.csv"
|
||||||
expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv"
|
expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv"
|
||||||
detail_file = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Active_{today_date}.csv"
|
detail_file = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Active_{today_date}.csv"
|
||||||
|
|
||||||
do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id")
|
do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id")
|
||||||
upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes")
|
upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes")
|
||||||
jstIdJobDetailScraper(fresh_output, detail_file)
|
jstIdJobDetailScraper(fresh_output, detail_file)
|
||||||
|
@ -220,7 +219,7 @@ if __name__ == "__main__":
|
||||||
jstSGJobDetailScraper(fresh_output, detail_file)
|
jstSGJobDetailScraper(fresh_output, detail_file)
|
||||||
upload_file_to_bucket(detail_file, f"Compete_2_Singapore_Active_{today_date}.csv" ,today_date, "yes")
|
upload_file_to_bucket(detail_file, f"Compete_2_Singapore_Active_{today_date}.csv" ,today_date, "yes")
|
||||||
elif option =="my":
|
elif option =="my":
|
||||||
search_file = f"malaysia_data/daily_search_results/id_search_{today_date}.csv"
|
search_file = f"malaysia_data/daily_search_results/my_search_{today_date}.csv"
|
||||||
search_jst_malay(search_file)
|
search_jst_malay(search_file)
|
||||||
folder_path = "malaysia_data/daily_search_results/"
|
folder_path = "malaysia_data/daily_search_results/"
|
||||||
search_pattern = "my_search_*.csv"
|
search_pattern = "my_search_*.csv"
|
||||||
|
|
|
@ -20,7 +20,9 @@ column = [
|
||||||
"Key Skills",
|
"Key Skills",
|
||||||
"Minimum Experience",
|
"Minimum Experience",
|
||||||
"Maximum Experience",
|
"Maximum Experience",
|
||||||
"Salary Detail"
|
"Salary Detail",
|
||||||
|
"Country"
|
||||||
|
|
||||||
]
|
]
|
||||||
|
|
||||||
def jstMalayJobDetailScraper(search_file, jd_file):
|
def jstMalayJobDetailScraper(search_file, jd_file):
|
||||||
|
@ -102,6 +104,8 @@ def jstMalayJobDetailScraper(search_file, jd_file):
|
||||||
"Key Skills":"",
|
"Key Skills":"",
|
||||||
"Minimum Experience" : result['data']['jobDetail'].get('jobDetail', {'jobRequirement':""}).get('jobRequirement', {"yearsOfExperience":""}).get('yearsOfExperience'),
|
"Minimum Experience" : result['data']['jobDetail'].get('jobDetail', {'jobRequirement':""}).get('jobRequirement', {"yearsOfExperience":""}).get('yearsOfExperience'),
|
||||||
# "Salary Detail" : result['data']['jobDetail']['header']['salary']
|
# "Salary Detail" : result['data']['jobDetail']['header']['salary']
|
||||||
|
"Country" : "Malaysia"
|
||||||
|
|
||||||
}
|
}
|
||||||
j_writer.writerow(job)
|
j_writer.writerow(job)
|
||||||
del jobIds[0]
|
del jobIds[0]
|
||||||
|
|
|
@ -20,7 +20,9 @@ column = [
|
||||||
"Key Skills",
|
"Key Skills",
|
||||||
"Minimum Experience",
|
"Minimum Experience",
|
||||||
"Maximum Experience",
|
"Maximum Experience",
|
||||||
"Salary Detail"
|
"Salary Detail",
|
||||||
|
"Country"
|
||||||
|
|
||||||
]
|
]
|
||||||
|
|
||||||
def jstSGJobDetailScraper(search_file, jd_file):
|
def jstSGJobDetailScraper(search_file, jd_file):
|
||||||
|
@ -102,6 +104,8 @@ def jstSGJobDetailScraper(search_file, jd_file):
|
||||||
"Key Skills":"",
|
"Key Skills":"",
|
||||||
"Minimum Experience" : result['data']['jobDetail'].get('jobDetail', {'jobRequirement':""}).get('jobRequirement', {"yearsOfExperience":""}).get('yearsOfExperience'),
|
"Minimum Experience" : result['data']['jobDetail'].get('jobDetail', {'jobRequirement':""}).get('jobRequirement', {"yearsOfExperience":""}).get('yearsOfExperience'),
|
||||||
# "Salary Detail" : result['data']['jobDetail']['header']['salary']
|
# "Salary Detail" : result['data']['jobDetail']['header']['salary']
|
||||||
|
"Country" : "Singapore"
|
||||||
|
|
||||||
}
|
}
|
||||||
j_writer.writerow(job)
|
j_writer.writerow(job)
|
||||||
del jobIds[0]
|
del jobIds[0]
|
||||||
|
|
Loading…
Reference in New Issue