print command removed

prahul11 2023-10-09 14:35:09 +05:30
parent 45015d9b4f
commit d1e98fac17
2 changed files with 3 additions and 2 deletions

View File

@ -69,6 +69,7 @@ class NaukriGulfSearchSpiderSpider(scrapy.Spider):
def parse(self, response):
# status_code = response.status
print(self.crawler.stats.inc_value('urls_crawled'))
total_pages = response.meta.get('total_pages')
start_page = response.meta.get('start_page')
base_url = response.meta.get('base_url')
@ -96,7 +97,7 @@ class NaukriGulfSearchSpiderSpider(scrapy.Spider):
except:
pass
yield parsed_item
print(f"Processed{url} : {start_page}/{total_page_num}/{total_pages}")
# print(f"Processed {url} : {start_page}/{total_page_num}/{total_pages}")
total_pages = total_pages-1
start_page = start_page+1
else:

View File

@ -4,7 +4,7 @@ import logging
import json
import math
input_file_path = "static_data/_industry_urls.csv"
input_file_path = "_industry_urls.csv"
headers = {
"authority": "www.naukri.com",
"accept": "application/json",