foundit
/
compete_jobs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

added logger

prahul11 2023-10-11 01:36:16 +05:30
parent a401cf37fc
commit 7e18ef91ff
2 changed files with 36 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
naukri/jobdata_india.py
View File

@ -3,6 +3,11 @@ import csv
import time import time
import json import json
import os import os
import logging
# Configure the logging settings
logging.basicConfig(filename='search_india_error.log', level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger()
# Global variables # Global variables
input_file = "data_naukri/search_result_india.csv" input_file = "data_naukri/search_result_india.csv"
@ -116,6 +121,7 @@ class NaukriJobDetailScraper:
url = self.base_url.format(job_id) url = self.base_url.format(job_id)
time.sleep(0.5) time.sleep(0.5)
try:
response = requests.get(url, headers=self.headers, timeout=self.timeout, proxies=self.proxies) response = requests.get(url, headers=self.headers, timeout=self.timeout, proxies=self.proxies)
print(f"{response.status_code} for {url}") print(f"{response.status_code} for {url}")
@ -151,6 +157,8 @@ class NaukriJobDetailScraper:
else: else:
print(f"Error for job ID {job_id}") print(f"Error for job ID {job_id}")
time.sleep(10) time.sleep(10)
except Exception as e1:
logging.error(url + '\n'+ str(e1) + '\n')
def main(): def main():

2
naukri/search_india.py
View File

@ -105,7 +105,7 @@ class NaukriJobScraper:
print(f"Industry: {industry_name}, pages remaining: {total_pages}, start page: {start_page}") print(f"Industry: {industry_name}, pages remaining: {total_pages}, start page: {start_page}")
time.sleep(1) time.sleep(1)
except Exception as e1: except Exception as e1:
logging.error(url + '\n'+ str(e1)) logging.error(url + '\n'+ str(e1) + '\n')
def main(): def main():