mjb
parent
362ef3f366
commit
4b7333e17f
|
@ -67,13 +67,13 @@ def jstMalayJobDetailScraper(search_file, jd_file):
|
|||
# print(response.text)
|
||||
|
||||
if response.status_code == 200:
|
||||
print("yes 200")
|
||||
# print("yes 200")
|
||||
result = response.json()
|
||||
# print("result", result)
|
||||
if jfile.tell() == 0:
|
||||
j_writer.writeheader()
|
||||
if result.get('data', {'jobDetail' :''}).get('jobDetail'):
|
||||
print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
||||
# print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
||||
# print(result['data']['jobDetail'])
|
||||
try :
|
||||
job_industry = result['data']['jobDetail']\
|
||||
|
|
|
@ -58,12 +58,12 @@ def search_jst_malay(csv_file):
|
|||
total_pages = ceil(result['data']['jobs']['totalJobs']/30)
|
||||
if len(result['data']['jobs']['jobs']) > 0 :
|
||||
column = ['Company_Name'] + list(result['data']['jobs']['jobs'][0].keys())[:6]
|
||||
print("writing to file")
|
||||
# print("writing to file")
|
||||
with open(csv_file, 'a+', newline='', encoding='utf-8') as csvfile:
|
||||
writer = DictWriter(csvfile, fieldnames=column, extrasaction='ignore')
|
||||
if csvfile.tell() == 0:
|
||||
writer.writeheader()
|
||||
print("writing each array")
|
||||
# print("writing each array")
|
||||
for a_job in result['data']['jobs']['jobs']:
|
||||
a_job['Company_Name'] = a_job['companyMeta']['name']
|
||||
writer.writerow(a_job)
|
||||
|
|
|
@ -66,7 +66,7 @@ def ph_search(searchfilename):
|
|||
|
||||
response = requests.get(base_url, params=query_params, headers=headers, timeout=20)
|
||||
if response.status_code == 200:
|
||||
print("Request was successful")
|
||||
# print("Request was successful")
|
||||
total_pages = ceil(response.json()['totalCount']/100)
|
||||
for each_job in response.json()['data']:
|
||||
write_to_file( searchfilename, each_job)
|
||||
|
|
|
@ -67,13 +67,13 @@ def jstMalayJobDetailScraper(search_file, jd_file):
|
|||
# print(response.text)
|
||||
|
||||
if response.status_code == 200:
|
||||
print("yes 200")
|
||||
# print("yes 200")
|
||||
result = response.json()
|
||||
# print("result", result)
|
||||
if jfile.tell() == 0:
|
||||
j_writer.writeheader()
|
||||
if result.get('data', {'jobDetail' :''}).get('jobDetail'):
|
||||
print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
||||
# print(result.get('data', {'jobDetail' :''}).get('jobDetail'))
|
||||
# print(result['data']['jobDetail'])
|
||||
try :
|
||||
job_industry = result['data']['jobDetail']\
|
||||
|
|
|
@ -58,12 +58,12 @@ def search_jst_malay(csv_file):
|
|||
total_pages = ceil(result['data']['jobs']['totalJobs']/30)
|
||||
if len(result['data']['jobs']['jobs']) > 0 :
|
||||
column = ['Company_Name'] + list(result['data']['jobs']['jobs'][0].keys())[:6]
|
||||
print("writing to file")
|
||||
# print("writing to file")
|
||||
with open(csv_file, 'a+', newline='', encoding='utf-8') as csvfile:
|
||||
writer = DictWriter(csvfile, fieldnames=column, extrasaction='ignore')
|
||||
if csvfile.tell() == 0:
|
||||
writer.writeheader()
|
||||
print("writing each array")
|
||||
# print("writing each array")
|
||||
for a_job in result['data']['jobs']['jobs']:
|
||||
a_job['Company_Name'] = a_job['companyMeta']['name']
|
||||
writer.writerow(a_job)
|
||||
|
|
Loading…
Reference in New Issue