From c55aeda72abb351ecfef0e37c09ff4e102e5a704 Mon Sep 17 00:00:00 2001 From: prahul11 Date: Wed, 11 Oct 2023 23:33:51 +0530 Subject: [PATCH] u4 --- naukri/jobdata_gulf_r.py | 4 +++- naukri/search_gulf_r.py | 14 +++++++------- naukri/search_india.py | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/naukri/jobdata_gulf_r.py b/naukri/jobdata_gulf_r.py index ff33cb4..4d80976 100644 --- a/naukri/jobdata_gulf_r.py +++ b/naukri/jobdata_gulf_r.py @@ -123,7 +123,9 @@ class NaukriGulfJobDetailScraper: "Key Skills" : ', '.join([y['title'] for y in json_response['keywordInterlinking']]), "Minimum Experience" : json_response.get('desiredCandidate').get('experience').get('min'), "Maximum Experience" : json_response.get('desiredCandidate').get('experience').get('max'), - "Salary Detail" : json_response.get('compensation') + "Salary Detail" : json_response.get('compensation'), + "Country" : json_response.get('compensation',{'country':''}).get('country') + } return json_data diff --git a/naukri/search_gulf_r.py b/naukri/search_gulf_r.py index b1bc850..6c3faa9 100644 --- a/naukri/search_gulf_r.py +++ b/naukri/search_gulf_r.py @@ -42,10 +42,10 @@ def parse_and_save(json_data, csv_filename, city): for job in json_data["jobs"]: parsed_item = {field: job.get(field, None) for field in keys_to_extract} parsed_item['city'] = city - print("parsed_item ---", parsed_item) - print(parsed_item.get('company', {'name':''}).get('name')) - print(parsed_item.get('company', {'id':''}).get('id')) - print(parsed_item.get('company', {'url':''}).get('url')) + # print("parsed_item ---", parsed_item) + # print(parsed_item.get('company', {'name':''}).get('name')) + # print(parsed_item.get('company', {'id':''}).get('id')) + # print(parsed_item.get('company', {'url':''}).get('url')) for key, value in parsed_item.get('company', {'name':'', 'id':'', 'url':''}).items(): parsed_item["Company" + key] = value try: @@ -57,7 +57,7 @@ def parse_and_save(json_data, csv_filename, city): #parsed_data.extend(city) with open(csv_filename, "a", newline="", encoding="utf-8") as csvfile: - print("csv_filename---", csv_filename) + # print("csv_filename---", csv_filename) csv_writer = csv.DictWriter(csvfile, fieldnames= fields_to_write) if csvfile.tell() == 0: csv_writer.writeheader() @@ -83,7 +83,7 @@ def main(): while total_pages>0: url = base_url.format(city[0],(jobs_per_pages*(start_page-1)),start_page) - print("url", url) + # print("url", url) # input() response = requests.get(url, headers=headers) @@ -106,7 +106,7 @@ def main(): total_pages = total_pages-1 start_page = start_page+1 - print("Data saved to output_new.json") + # print("Data saved to output_new.json") print(error_pages) if __name__ == "__main__": diff --git a/naukri/search_india.py b/naukri/search_india.py index 3ae5323..5bf722d 100644 --- a/naukri/search_india.py +++ b/naukri/search_india.py @@ -79,7 +79,8 @@ class NaukriJobScraper: url = self.base_url.format(industry_name, start_page, industry_q) try: # print(url) - response = requests.get(url, headers=self.headers, timeout=self.timeout, proxies=self.proxies) + response = requests.get(url, headers=self.headers, timeout=self.timeout, + proxies=self.proxies) # print(f"{response.status_code} for {url}")