import base64 import requests import json import os from os import path import shutil import glob import pymysql.cursors import pymysql import uuid import email import getpass, imaplib import os import sys import time from email.header import decode_header import html2text db = pymysql.connect (host="cldy-hub-db-prod-do-user-1524670-0.a.db.ondigitalocean.com", user="doadmin", passwd="jmbly6e4obtma5z0", db="defaultdb", port=25060, charset='utf8mb4', cursorclass=pymysql.cursors.DictCursor) def standardizeJob(body): userkey = '3SYCRFQHOOT' subUserId = 'Raj_Beri' version = '3.0' # service url- provided by RChilli url="http://jdrest.rchilli.com/JDParser/RChilli/ParseJDText" data=body data_bytes = data.encode("utf-8") encoded_string = base64.b64encode(data_bytes) data64 = encoded_string.decode('UTF-8') headers = {'content-type': 'application/json'} body = """{"base64text":\""""+data64+"""\","userkey":\""""+ userkey+"""\",\"version\":\""""+version+"""\",\"subuserid\":\""""+subUserId+"""\"}""" #print (body) response = requests.post(url,data=body,headers=headers) resp =json.loads(response.text) #print (resp) #please handle error too job_json =resp["JDParsedData"] #read values from response return job_json def getData(query): with db.cursor() as cursor: cursor.execute(query) return cursor def getOrgId(name, website, domain, industry, city, state, country, linkedin): if (domain != ''): sql = "select `id` from organization where lower(domain) = lower('" + domain + "') limit 1" else: sql = "select `id` from organization where lower(name) = lower('" + name + "') limit 1" rowcount = getData(sql).rowcount if rowcount > 0: for record in getData(sql): return record["id"] else: with db.cursor() as cursor: sql = "insert into `organization` (`uuid`, `type`, `source`, `name`, `industry`, `legal_name`, `dba`, `city`, `state_province`, `country`, `linkedin_url`, `website`, `domain`) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" cursor.execute(sql, (str(uuid.uuid1()), 'Recruiter', 'Job Posting Email', name, industry, name, name, city, state, country, linkedin, website, domain)) o_id = cursor.lastrowid db.commit() return o_id def getPersonId(org_id, org_name, first_name, last_name, title, email, city, state, country, linkedin_url): sql = "select `id` from person where lower(email) = lower('" + email + "') limit 1" rowcount = getData(sql).rowcount if rowcount > 0: for record in getData(sql): return record["id"] else: with db.cursor() as cursor: sql = "insert into `person` (`organization`, `org_id`, `uuid`, `type`, `source`, `name`, `first_name`, `last_name`, `job_profile`, `email`, `city`, `state_province`, `country`, `linkedin_url`) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" cursor.execute(sql, (org_name, org_id, str(uuid.uuid1()), 'Hiring Manager', 'LinkedIn', first_name + ' ' + last_name, first_name, last_name, title, email, city, state, country, linkedin_url)) o_id = cursor.lastrowid db.commit() return o_id def createJob(reference_id, title, company, org_id, city, state, country, summary, posting_date, details, source, status, details_html, job_json): print(job_json) with db.cursor() as cursor: sql = "insert into `jobs` (`company`, `reference_id`, `org_id`, `title`, `summary`, `posting_date`, `details`, `source`, `status`, `details_html`, `city`, `state_province`, `country`, `job_json`) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" cursor.execute(sql, (company, reference_id, org_id, title, summary, '', details, source, status, details_html, city, state, country, json.dumps(job_json))) o_id = cursor.lastrowid db.commit() return o_id userName = 'job@gigminds.com' passwd = 'Ch@ngeTheW0rld' print ("Starting to Process Job Inbox") imapSession = imaplib.IMAP4_SSL('imap.gmail.com',993) typ, accountDetails = imapSession.login(userName, passwd) if typ != 'OK': print ('Not able to sign in!') raise imapSession.select('Inbox') # typ, data = imapSession.search(None, 'ALL') typ, data = imapSession.search(None, '(UNSEEN)') if typ != 'OK': print ('Error searching Inbox.') raise # Iterating over all emails for msgId in data[0].split(): res, msg = imapSession.fetch(msgId, '(RFC822)') from_ = "" subject_ = "" body_ = "" for response in msg: if isinstance(response, tuple): msg = email.message_from_bytes(response[1]) subject_ = decode_header(msg["Subject"])[0][0] message_id = decode_header(msg["Message-ID"])[0][0] if isinstance(subject_, bytes): subject_ = subject_.decode() from_ = msg.get("From") if msg.is_multipart(): # iterate over email parts for part in msg.walk(): # extract content type of email content_type = part.get_content_type() content_disposition = str(part.get("Content-Disposition")) try: # get the email body body = part.get_payload(decode=True).decode() except: pass if content_type == "text/plain" and "attachment" not in content_disposition: # print text/plain emails and skip attachments #print(body) body_ = body elif "attachment" in content_disposition: # download attachment filename = part.get_filename() ##if filename: ## if not os.path.isdir(subject): ## # make a folder for this email (named after the subject) ## os.mkdir(subject) ## filepath = os.path.join(subject, filename) ## # download attachment and save it ## open(filepath, "wb").write(part.get_payload(decode=True)) else: # extract content type of email content_type = msg.get_content_type() # get the email body body = msg.get_payload(decode=True).decode() if content_type == "text/plain": # print only text email parts #print(body) body_ = body if content_type == "text/html": #print(body) body_ = body # if it's HTML, create a new HTML file and open it in browser ##if not os.path.isdir(subject): ## # make a folder for this email (named after the subject) ## os.mkdir(subject) ##filename = f"{subject[:50]}.html" ##filepath = os.path.join(subject, filename) # write the file ##open(filepath, "w").write(body) # open in the default browser ##webbrowser.open(filepath) #body_ = html2text.html2text(body_) #print ("From: ", from_) #print ("Subject: ", subject_) #print ("Body: ", body_) email_str = from_.replace('"', '').replace(' <','|').replace('>','') from_name, from_email = email_str.split('|') try: from_first_name, from_last_name = from_name.split(' ') except ValueError: from_first_name = from_name from_last_name = '' #print(from_name, ":", from_first_name, from_last_name, from_email) job_json = standardizeJob(body_) title = job_json["JobProfile"]["Title"] description_text = job_json["JobDescription"] print(title) if title != '' and description_text != '': organization = job_json["Organization"] description_text = job_json["JobDescription"] description_html = job_json["JobDescription"].replace('\n', '
') contact_email = job_json["ContactEmail"] contact_phone = job_json["ContactPhone"] contact_person_name = job_json["ContactPersonName"] website = job_json["WebSite"] job_location = job_json["JobLocation"]["Location"] job_city = job_json["JobLocation"]["City"] job_state = job_json["JobLocation"]["State"] job_country = job_json["JobLocation"]["Country"] if contact_email != '' and contact_person_name != '': from_email = contact_email #print(">>", contact_person_name, contact_email) try: from_first_name, from_last_name = contact_person_name.split(' ') except ValueError: from_first_name = contact_person_name from_last_name = '' domain = from_email.split('@')[1] website = "www." + domain if organization == '': organization = domain #print(from_first_name, ":", from_last_name, ":", from_email, domain) #print(description_html) #print("="*100) summary = description_text[0:255] org_id = getOrgId(organization, website, domain, '', job_city, job_state, job_country, '') print(org_id) person_id = getPersonId(org_id, organization, from_first_name, from_last_name, '', from_email, job_city, job_state, job_country, '') print(person_id) job_id = createJob(message_id, title, organization, org_id, job_city, job_state, job_country, summary, '', description_text, 'Gigminds', 'Active', description_html, job_json) print(job_id) imapSession.close() imapSession.logout()