From 9945b2991f3796ff75f12e4a6061d38282d68967 Mon Sep 17 00:00:00 2001 From: prahul11 Date: Fri, 13 Oct 2023 21:25:22 +0530 Subject: [PATCH] j --- common_task.py | 14 ++++++++------ naukri/search_india.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/common_task.py b/common_task.py index f990ddb..9ef6b70 100644 --- a/common_task.py +++ b/common_task.py @@ -5,6 +5,7 @@ import glob from naukri.search_india import NaukriJobScraper from naukri.jobdata_india import NaukriJobDetailScraper +# from naukri.search_gulf_r import import time import os @@ -79,12 +80,12 @@ def run_india_scraper(): india_search_output_file = f"india_data/daily_search_results/search_result_india_{today_date}.csv" india_search_error_file = f"india_data/daily_error_folder/search_error_india_{today_date}.csv" india_search_stats_file = f"india_data/stats_india_{today_date}.txt" - start_time = time.time() - scraper = NaukriJobScraper(india_search_input_file, india_search_output_file, india_search_error_file) - scraper.scrape() - end_time = time.time() - duration_hours = (end_time - start_time) / 3600 - print(f"Search program took {duration_hours:.2f} hours to run.") + # start_time = time.time() + # scraper = NaukriJobScraper(india_search_input_file, india_search_output_file, india_search_error_file) + # scraper.scrape() + # end_time = time.time() + # duration_hours = (end_time - start_time) / 3600 + # print(f"Search program took {duration_hours:.2f} hours to run.") with open(india_search_stats_file, "a") as stat: stat.write(f"Search program took {duration_hours:.2f} hours to run. \n") folder_path = "india_data/daily_search_results/" @@ -109,6 +110,7 @@ def run_india_scraper(): upload_file_to_bucket(india_detail_file, f"Compete_1_India_{today_date}.csv" ) def run_gulf_scraper(): + pass if __name__ == "__main__": diff --git a/naukri/search_india.py b/naukri/search_india.py index f76e925..0ab1657 100644 --- a/naukri/search_india.py +++ b/naukri/search_india.py @@ -91,7 +91,7 @@ class NaukriJobScraper: print(f"Error with page {start_page} for industry {industry_name}") with open(self.error_file_path, "a") as file: file.write(f"Error with page {start_page} for industry {industry_name}\n") - time.sleep(10) + # time.sleep(10) continue # if 200 response