mjb
parent
362ef3f366
commit
4b7333e17f
|
@ -67,13 +67,13 @@ def jstMalayJobDetailScraper(search_file, jd_file):
|
||||||
# print(response.text)
|
# print(response.text)
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
print("yes 200")
|
# print("yes 200")
|
||||||
result = response.json()
|
result = response.json()
|
||||||
# print("result", result)
|
# print("result", result)
|
||||||
if jfile.tell() == 0:
|
if jfile.tell() == 0:
|
||||||
j_writer.writeheader()
|
j_writer.writeheader()
|
||||||
if result.get('data', {'jobDetail' :''}).get('jobDetail'):
|
if result.get('data', {'jobDetail' :''}).get('jobDetail'):
|
||||||
print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
# print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
||||||
# print(result['data']['jobDetail'])
|
# print(result['data']['jobDetail'])
|
||||||
try :
|
try :
|
||||||
job_industry = result['data']['jobDetail']\
|
job_industry = result['data']['jobDetail']\
|
||||||
|
|
|
@ -58,12 +58,12 @@ def search_jst_malay(csv_file):
|
||||||
total_pages = ceil(result['data']['jobs']['totalJobs']/30)
|
total_pages = ceil(result['data']['jobs']['totalJobs']/30)
|
||||||
if len(result['data']['jobs']['jobs']) > 0 :
|
if len(result['data']['jobs']['jobs']) > 0 :
|
||||||
column = ['Company_Name'] + list(result['data']['jobs']['jobs'][0].keys())[:6]
|
column = ['Company_Name'] + list(result['data']['jobs']['jobs'][0].keys())[:6]
|
||||||
print("writing to file")
|
# print("writing to file")
|
||||||
with open(csv_file, 'a+', newline='', encoding='utf-8') as csvfile:
|
with open(csv_file, 'a+', newline='', encoding='utf-8') as csvfile:
|
||||||
writer = DictWriter(csvfile, fieldnames=column, extrasaction='ignore')
|
writer = DictWriter(csvfile, fieldnames=column, extrasaction='ignore')
|
||||||
if csvfile.tell() == 0:
|
if csvfile.tell() == 0:
|
||||||
writer.writeheader()
|
writer.writeheader()
|
||||||
print("writing each array")
|
# print("writing each array")
|
||||||
for a_job in result['data']['jobs']['jobs']:
|
for a_job in result['data']['jobs']['jobs']:
|
||||||
a_job['Company_Name'] = a_job['companyMeta']['name']
|
a_job['Company_Name'] = a_job['companyMeta']['name']
|
||||||
writer.writerow(a_job)
|
writer.writerow(a_job)
|
||||||
|
|
|
@ -66,7 +66,7 @@ def ph_search(searchfilename):
|
||||||
|
|
||||||
response = requests.get(base_url, params=query_params, headers=headers, timeout=20)
|
response = requests.get(base_url, params=query_params, headers=headers, timeout=20)
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
print("Request was successful")
|
# print("Request was successful")
|
||||||
total_pages = ceil(response.json()['totalCount']/100)
|
total_pages = ceil(response.json()['totalCount']/100)
|
||||||
for each_job in response.json()['data']:
|
for each_job in response.json()['data']:
|
||||||
write_to_file( searchfilename, each_job)
|
write_to_file( searchfilename, each_job)
|
||||||
|
|
|
@ -67,13 +67,13 @@ def jstMalayJobDetailScraper(search_file, jd_file):
|
||||||
# print(response.text)
|
# print(response.text)
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
print("yes 200")
|
# print("yes 200")
|
||||||
result = response.json()
|
result = response.json()
|
||||||
# print("result", result)
|
# print("result", result)
|
||||||
if jfile.tell() == 0:
|
if jfile.tell() == 0:
|
||||||
j_writer.writeheader()
|
j_writer.writeheader()
|
||||||
if result.get('data', {'jobDetail' :''}).get('jobDetail'):
|
if result.get('data', {'jobDetail' :''}).get('jobDetail'):
|
||||||
print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
# print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
||||||
# print(result['data']['jobDetail'])
|
# print(result['data']['jobDetail'])
|
||||||
try :
|
try :
|
||||||
job_industry = result['data']['jobDetail']\
|
job_industry = result['data']['jobDetail']\
|
||||||
|
|
|
@ -58,12 +58,12 @@ def search_jst_malay(csv_file):
|
||||||
total_pages = ceil(result['data']['jobs']['totalJobs']/30)
|
total_pages = ceil(result['data']['jobs']['totalJobs']/30)
|
||||||
if len(result['data']['jobs']['jobs']) > 0 :
|
if len(result['data']['jobs']['jobs']) > 0 :
|
||||||
column = ['Company_Name'] + list(result['data']['jobs']['jobs'][0].keys())[:6]
|
column = ['Company_Name'] + list(result['data']['jobs']['jobs'][0].keys())[:6]
|
||||||
print("writing to file")
|
# print("writing to file")
|
||||||
with open(csv_file, 'a+', newline='', encoding='utf-8') as csvfile:
|
with open(csv_file, 'a+', newline='', encoding='utf-8') as csvfile:
|
||||||
writer = DictWriter(csvfile, fieldnames=column, extrasaction='ignore')
|
writer = DictWriter(csvfile, fieldnames=column, extrasaction='ignore')
|
||||||
if csvfile.tell() == 0:
|
if csvfile.tell() == 0:
|
||||||
writer.writeheader()
|
writer.writeheader()
|
||||||
print("writing each array")
|
# print("writing each array")
|
||||||
for a_job in result['data']['jobs']['jobs']:
|
for a_job in result['data']['jobs']['jobs']:
|
||||||
a_job['Company_Name'] = a_job['companyMeta']['name']
|
a_job['Company_Name'] = a_job['companyMeta']['name']
|
||||||
writer.writerow(a_job)
|
writer.writerow(a_job)
|
||||||
|
|
Loading…
Reference in New Issue