diff --git a/common_task.py b/common_task.py index bdc668c..0268b8a 100644 --- a/common_task.py +++ b/common_task.py @@ -8,15 +8,19 @@ from naukri.jobdata_india import NaukriJobDetailScraper from naukri.search_gulf_r import main as gulfSearch, output_filename_csv as gulf_search_file from naukri.jobdata_gulf_r import NaukriGulfJobDetailScraper from jobstreet.jst_id_search import search_jst_id +from jobstreet.jst_id_detail import jstIdJobDetailScraper import time import os import sys -def upload_file_to_bucket(localFilePath, localFileName, today_date): +def upload_file_to_bucket(localFilePath, localFileName, today_date, c2): s3 = boto3.client('s3') bucket_name = 'compete-syndication' file_path = localFilePath - s3_key = f'naukri/{today_date}/{localFileName}' + if not c2: + s3_key = f'naukri/{today_date}/{localFileName}' + else: + s3_key = f'jobstreet/{today_date}/{localFileName}' s3.upload_file(file_path, bucket_name, s3_key) print(f'File "{file_path}" uploaded to S3 bucket "{bucket_name}" as "{s3_key}"') @@ -191,7 +195,11 @@ if __name__ == "__main__": last_file = find_second_latest_file(folder_path, search_pattern) fresh_output = f"indonesia_data/daily_process_folder/new_jobs_on_{today_date}.csv" expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv" + detail_file = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Active_{today_date}.csv" do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id") + upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes") + jstIdJobDetailScraper(search_file, f"Compete_2_Indonesia_Active_{today_date}.csv") + upload_file_to_bucket(detail_file, f"Compete_2_Indonesia_Active_{today_date}.csv" ,today_date, "yes") else: print("Invalid argument. Please use 'gulf' or 'india' as the argument.")