foundit
/
compete_jobs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update common_task.py

prahul11 2023-11-06 16:24:04 +05:30
parent d999c3454e
commit d4af8ed0f3
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
common_task.py
View File

@ -8,15 +8,19 @@ from naukri.jobdata_india import NaukriJobDetailScraper
from naukri.search_gulf_r import main as gulfSearch, output_filename_csv as gulf_search_file from naukri.search_gulf_r import main as gulfSearch, output_filename_csv as gulf_search_file
from naukri.jobdata_gulf_r import NaukriGulfJobDetailScraper from naukri.jobdata_gulf_r import NaukriGulfJobDetailScraper
from jobstreet.jst_id_search import search_jst_id from jobstreet.jst_id_search import search_jst_id
from jobstreet.jst_id_detail import jstIdJobDetailScraper
import time import time
import os import os
import sys import sys
def upload_file_to_bucket(localFilePath, localFileName, today_date): def upload_file_to_bucket(localFilePath, localFileName, today_date, c2):
s3 = boto3.client('s3') s3 = boto3.client('s3')
bucket_name = 'compete-syndication' bucket_name = 'compete-syndication'
file_path = localFilePath file_path = localFilePath
if not c2:
s3_key = f'naukri/{today_date}/{localFileName}' s3_key = f'naukri/{today_date}/{localFileName}'
else:
s3_key = f'jobstreet/{today_date}/{localFileName}'
s3.upload_file(file_path, bucket_name, s3_key) s3.upload_file(file_path, bucket_name, s3_key)
print(f'File "{file_path}" uploaded to S3 bucket "{bucket_name}" as "{s3_key}"') print(f'File "{file_path}" uploaded to S3 bucket "{bucket_name}" as "{s3_key}"')
@ -191,7 +195,11 @@ if __name__ == "__main__":
last_file = find_second_latest_file(folder_path, search_pattern) last_file = find_second_latest_file(folder_path, search_pattern)
fresh_output = f"indonesia_data/daily_process_folder/new_jobs_on_{today_date}.csv" fresh_output = f"indonesia_data/daily_process_folder/new_jobs_on_{today_date}.csv"
expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv" expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv"
detail_file = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Active_{today_date}.csv"
do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id") do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id")
upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes")
jstIdJobDetailScraper(search_file, f"Compete_2_Indonesia_Active_{today_date}.csv")
upload_file_to_bucket(detail_file, f"Compete_2_Indonesia_Active_{today_date}.csv" ,today_date, "yes")
else: else:
print("Invalid argument. Please use 'gulf' or 'india' as the argument.") print("Invalid argument. Please use 'gulf' or 'india' as the argument.")