Update common_task.py
parent
d999c3454e
commit
d4af8ed0f3
|
@ -8,15 +8,19 @@ from naukri.jobdata_india import NaukriJobDetailScraper
|
|||
from naukri.search_gulf_r import main as gulfSearch, output_filename_csv as gulf_search_file
|
||||
from naukri.jobdata_gulf_r import NaukriGulfJobDetailScraper
|
||||
from jobstreet.jst_id_search import search_jst_id
|
||||
from jobstreet.jst_id_detail import jstIdJobDetailScraper
|
||||
import time
|
||||
import os
|
||||
import sys
|
||||
|
||||
def upload_file_to_bucket(localFilePath, localFileName, today_date):
|
||||
def upload_file_to_bucket(localFilePath, localFileName, today_date, c2):
|
||||
s3 = boto3.client('s3')
|
||||
bucket_name = 'compete-syndication'
|
||||
file_path = localFilePath
|
||||
s3_key = f'naukri/{today_date}/{localFileName}'
|
||||
if not c2:
|
||||
s3_key = f'naukri/{today_date}/{localFileName}'
|
||||
else:
|
||||
s3_key = f'jobstreet/{today_date}/{localFileName}'
|
||||
s3.upload_file(file_path, bucket_name, s3_key)
|
||||
print(f'File "{file_path}" uploaded to S3 bucket "{bucket_name}" as "{s3_key}"')
|
||||
|
||||
|
@ -191,7 +195,11 @@ if __name__ == "__main__":
|
|||
last_file = find_second_latest_file(folder_path, search_pattern)
|
||||
fresh_output = f"indonesia_data/daily_process_folder/new_jobs_on_{today_date}.csv"
|
||||
expired_output = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Archive_{today_date}.csv"
|
||||
detail_file = f"indonesia_data/daily_upload_folder/Compete_2_Indonesia_Active_{today_date}.csv"
|
||||
do_the_difference(search_file, last_file, 'id', fresh_output, expired_output, None, "id")
|
||||
upload_file_to_bucket(expired_output, f"Compete_2_Indonesia_Archive_{today_date}.csv" ,today_date,"yes")
|
||||
jstIdJobDetailScraper(search_file, f"Compete_2_Indonesia_Active_{today_date}.csv")
|
||||
upload_file_to_bucket(detail_file, f"Compete_2_Indonesia_Active_{today_date}.csv" ,today_date, "yes")
|
||||
|
||||
else:
|
||||
print("Invalid argument. Please use 'gulf' or 'india' as the argument.")
|
||||
|
|
Loading…
Reference in New Issue