import pandas as pd
import requests
import json
from bs4 import BeautifulSoup


def get_page_cnt(SITE_URL):
    """get total pages' cnt"""
    try:
        first_response = requests.get(url=SITE_URL)
    except ConnectionError as e:
        first_response = requests.get(url=SITE_URL)
    first_page = first_response.content.decode('utf-8')
    soup = BeautifulSoup(first_page, "lxml")
    PAGE_CNT = soup.find_all('span', class_=None)[1].get_text().split()[-2]
    return int(PAGE_CNT)


def parse_page(soup):
    """parse page to DataFrame."""
    page_return = soup.find_all('tr', class_="tr-bd")
    page_result = list()
    for item in page_return:
        result = dict()
        result['pol'] = item.find('div', class_='pol').get_text()
        result['pod'] = item.find('div', class_='fd').get_text()
        result['carrier'] = item.find(
            'div', class_='t-muted carrier').get_text()
        result['schedule'] = item.find('div', class_='sailingsche').get_text()
        result['pot'] = item.find_all('td')[2].find_all('div')[0].get_text()
        result['date'] = item.find('div', class_='tt').get_text(strip=True)
        for i in item.find_all('td', class_='td-price'):
            result[i.find('h5').get_text()] = i.find('div').get_text()
        result['start_date'] = item.find_all('td', class_=None)[-2].get_text()
        result['end_date'] = item.find_all('td', class_=None)[-1].get_text()
        page_result.append(result)
    return pd.DataFrame(page_result)


def update_headers_pageCnt(POD, POD_TYPE):
    """get headers and page counts for this port.

     Params:
         POD: standlized code for pod;
         POD_TYPE: ports' type like 'FCL';

     Return:
         headers: request headers;
         page_cnt: prices' amount for this port;
    """

    SITE_URL = "https://portal.kun-logistics.com/portal/fclratequery/form/index/?pol=CNNGB&fd=" + \
        POD + "&loadingtype=" + POD_TYPE

    headers = {
        "Cookie": "ASP.NET_SessionId=qlxcrlplrvfgkc24p2mn500z; uid=6ad87ee9-13b0-4d18-906a-d0a7dcae1322",
        "Content-Type": "application/json; charset=UTF-8",
        "Host": "portal.kun-logistics.com",
        "Referer": SITE_URL,
        "Accept": "*/*",
        "X-Requested-With": "XMLHttpRequest",
        "Accept-Encoding": "gzip, deflate, br",
        "Accept-Language": "zh-CN,zh;q=0.8"
    }

    PAGE_CNT = get_page_cnt(SITE_URL)
    return headers, PAGE_CNT


def update_payload(Payload_page, POD, POD_TYPE):
    """update the page number u want to crawl.

    Params:
        Paload_page: int, page number;

    Return:
        Payload: payload dict.

    """
    Payload_data = "{\"carrierid\":\"\",\"sailingdate\":\"\",\"portofloadingcode\":\"CNNGB\",\"finaldestinationcode\":\"" + \
        POD + "\",\"loadingtype\":\"" + POD_TYPE + \
        "\",\"page\":\"" + str(Payload_page) + "\"}"
    Payload = {
        "key": "getlist",
        "data": Payload_data,
        "xc_token": Payload_xc_token
    }
    return Payload


def get_pod_result(POD, POD_TYPE):
    headers, PAGE_CNT = update_headers_pageCnt(POD=POD, POD_TYPE=POD_TYPE)
    FNL_DF = pd.DataFrame()

    for Payload_page in range(1, PAGE_CNT + 1):
        Payload = update_payload(Payload_page, POD=POD, POD_TYPE=POD_TYPE)
        reponse = requests.post(
            url=PAGE_URL, data=json.dumps(Payload), headers=headers)
        one_page = reponse.content.decode('utf-8')
        soup = BeautifulSoup(one_page, "lxml")
        page_df = parse_page(soup)
        FNL_DF = FNL_DF.append(page_df)
    return FNL_DF.reset_index(drop=True)


if __name__ == '__main__':
    PAGE_URL = "https://portal.kun-logistics.com/portal/fclratequery/form/ajax"
    PORT_PATH = 'POD.xlsx'
    OUTPUT_PATH = 'OUTPUT.xlsx'

    Payload_xc_token = "ztMp8r"

    pod_data = pd.read_excel(PORT_PATH).dropna()
    print("【INFO】searching for {0} ports' prices".format(pod_data.shape[0]))

    ALL_PRICE = pd.DataFrame()
    for pod, pod_type in zip(pod_data['CODE'], pod_data['TYPE']):
        pod_result = get_pod_result(POD=pod, POD_TYPE=pod_type)
        print("【RESULT】{0} have {1} records!".format(pod, pod_result.shape[0]))
        ALL_PRICE = ALL_PRICE.append(pod_result)

    if ALL_PRICE.shape[0]:
        print("【FINALLY】total records: {0}".format(ALL_PRICE.shape[0]))
        ALL_PRICE.to_excel(OUTPUT_PATH)
    else:
        print("【WARN】TRY ANOTHER TOKEN PLEASE!")