# flightlinespider.py
# Group: Group bald
# Date: 20200712
import requests
import bs4

i = 0
# 显示所有航线的网页
web_url = 'https://flights.ctrip.com/schedule/.szx.html'
r = requests.get(web_url)
soup = bs4.BeautifulSoup(r.text, "html5lib")
route_list = []
list1 = soup.find_all('a')
list2 = soup.find_all('a',{'target':'_blank'})
list3 = list(set(list1).difference(set(list2)))
for item in list3:
    if '-深圳' in item.text:
        route_list.append(item.text)
route_list.sort()
# print(route_list)

# Group: Group bald
# Date: 20200712
from selenium import webdriver
import time
import pandas as pd
from flightlinespider import *
import random


class CTSpider(object):
    def __init__(self, i):
        self.i = 0
        self.url = 'https://flights.ctrip.com/schedule/.szx.html'
        self.browser = driver = webdriver.Chrome(
            r'D:/anaconda/new/Lib/site-packages/selenium/webdriver/chrome/chromedriver.exe')
        self.index = i # 循环遍历的标记

    # 点击特定路线
    def get_html(self):
        self.browser.get(self.url) # 打开url
        # 查找页面中route_list[index]的路线（文字），点击
        self.browser.find_elements_by_link_text(route_list[self.index])[0].click()

        time.sleep(random.uniform(2, 4))  # 给商品页面加载时间

    # 解析页面
    def parse_html(self):
		# 查找到当前访问的网页中class="flight_logo"的部分，返回一个列表
        flights = self.browser.find_elements_by_class_name("flight_logo")
        flightno_list = []
        flight_list = []
        for flight in flights: # 循环访问列表中的内容
            text = flight.text.split('/n机型:') # 用.text提取出文字内容
            flightno_list.append(text[0]) # 将所需内容添加到列中
            flight_list.append(text[1])

        arv = self.browser.find_elements_by_class_name("arrive")
        arrive_time = []  # 到达时间
        arrive_airport = [] # 到达机场
        for i in arv:
            a = i.text.split('/n') # 爬取到的内容每条间会有换行
            arrive_time.append(a[0])
            arrive_airport.append(a[1])
		
        # 找到所有的class为week的部分
        flights = self.browser.find_elements_by_class_name("weeks")
        flights = flights[1:] # flihts[0]为表头，去除
        week_list_mon = []  # 班期
        week_list_tue = []	# 这段就是非常愚蠢的代码，如果有改良的方法欢迎给我发邮件或者私信我
        week_list_wed = []	# 我不知道怎么生成低维不定长的二维数组
        week_list_thu = []
        week_list_fri = []
        week_list_sat = []
        week_list_sun = []
        # 将数字和星期几对应，方便后续处理
        dic = {0: '一', 1: '二', 2: '三', 3: '四', 4: '五', 5: '六', 6: '日'}
        dic_ = {'一': 0, '二': 1, '三': 2, '四': 3, '五': 4, '六': 5, '日': 6}
		
        # 若周一有班次，则"一"的class为"blue",否则为空
        # 在每个块（flight）里继续查找class="blue"即当日有班次的航班
        for flight in flights:
            weeks = flight.find_elements_by_class_name("blue")
            weekli = []
            for i in weeks: # 将单个航班中显示为蓝色的星期几的文字调整为列表
                weekli.append(i.text)
                # 例如，如果一三五有班期，则weekli=['一','三','五']
            m = 0
            if dic[m] in weekli: # 如果weekli中包含周一
                week_list_mon.append(1) # 则周一的列表中添加元素1
            else: # 否则周一的列表中添加元素0
                week_list_mon.append(0)

            m = 1 # 周二
            if dic[m] in weekli:
                week_list_tue.append(1)
            else:
                week_list_tue.append(0)

            m = 2
            if dic[m] in weekli:
                week_list_wed.append(1)
            else:
                week_list_wed.append(0)

            m = 3
            if dic[m] in weekli:
                week_list_thu.append(1)
            else:
                week_list_thu.append(0)

            m = 4
            if dic[m] in weekli:
                week_list_fri.append(1)
            else:
                week_list_fri.append(0)

            m = 5
            if dic[m] in weekli:
                week_list_sat.append(1)
            else:
                week_list_sat.append(0)

            m = 6 # 周日
            if dic[m] in weekli:
                week_list_sun.append(1)
            else:
                week_list_sun.append(0)
		
        # 准点率
        punc = self.browser.find_elements_by_class_name("punctuality")
        punc_list = []
        for i in punc:
            if i.text != '准点率':
                punc_list.append(i.text)

        # 将数据放到一个database里面
        data = {'航班号':flightno_list,'机型': flight_list, '到达时间': arrive_time,
                '到达机场': arrive_airport,'周一': week_list_mon, '周二': week_list_tue,
                '周三': week_list_wed, '周四': week_list_thu, '周五': week_list_fri,
                '周六': week_list_sat, '周日': week_list_sun, '准点率': punc_list}
        df = pd.DataFrame(data)
		df=df.drop_duplicates() # 去除由于“经停”带来的重复数据
        df.to_csv('Data3.csv', mode='a', header=False) # 以追加方式写入文件

    # 主函数
    def main(self):
        self.get_html()
        self.parse_html()


if __name__ == '__main__':
    global route_list
    for i in range(0, len(route_list)):
        print(i, route_list[i]) # 打印进度，如果中间出现问题，调整后可以从出错处继续运行
        spider = CTSpider(i)
        spider.main()