added logger
parent
a401cf37fc
commit
7e18ef91ff
|
@ -3,6 +3,11 @@ import csv
|
|||
import time
|
||||
import json
|
||||
import os
|
||||
import logging
|
||||
|
||||
# Configure the logging settings
|
||||
logging.basicConfig(filename='search_india_error.log', level=logging.ERROR, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger()
|
||||
|
||||
# Global variables
|
||||
input_file = "data_naukri/search_result_india.csv"
|
||||
|
@ -116,6 +121,7 @@ class NaukriJobDetailScraper:
|
|||
url = self.base_url.format(job_id)
|
||||
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
response = requests.get(url, headers=self.headers, timeout=self.timeout, proxies=self.proxies)
|
||||
|
||||
print(f"{response.status_code} for {url}")
|
||||
|
@ -151,6 +157,8 @@ class NaukriJobDetailScraper:
|
|||
else:
|
||||
print(f"Error for job ID {job_id}")
|
||||
time.sleep(10)
|
||||
except Exception as e1:
|
||||
logging.error(url + '\n'+ str(e1) + '\n')
|
||||
|
||||
def main():
|
||||
|
||||
|
|
|
@ -105,7 +105,7 @@ class NaukriJobScraper:
|
|||
print(f"Industry: {industry_name}, pages remaining: {total_pages}, start page: {start_page}")
|
||||
time.sleep(1)
|
||||
except Exception as e1:
|
||||
logging.error(url + '\n'+ str(e1))
|
||||
logging.error(url + '\n'+ str(e1) + '\n')
|
||||
|
||||
|
||||
def main():
|
||||
|
|
Loading…
Reference in New Issue