# -*- coding: utf-8 -*-
# !/usr/bin/env python
"""
-------------------------------------------------
   File Name：     GetFreeProxy.py
   Description :  抓取免费代理
   Author :       JHao
   date：          2016/11/25
-------------------------------------------------
   Change Activity:
                   2016/11/25: 
-------------------------------------------------
"""
import re
import sys
import json
import requests
from time import sleep

sys.path.append('..')

from Util.WebRequest import WebRequest
from Util.utilFunction import getHtmlTree
import requests
from DB.DbClient import DbClient
from Config.setting import USEFUL_PROXY_COUNT
from Config.setting import ZHIMA_PROXY_API

# for debug to disable insecureWarning
requests.packages.urllib3.disable_warnings()
proxy_dict = {
    "proxy": "",
    "region": "",
    "anonimity": "",
    "proxy_type": "",
    "source": ""
}
db = DbClient()


def init_proxy_dict():
    for _ in proxy_dict.keys():
        proxy_dict[_] = ""


class GetFreeProxy(object):
    """
    proxy getter
    """

    # @staticmethod
    # def freeProxy01():
    #     """
    #     无忧代理 http://www.data5u.com/
    #     几乎没有能用的
    #     :return:
    #     """
    #     url_list = [
    #         'http://www.data5u.com/',
    #         'http://www.data5u.com/free/gngn/index.shtml',
    #         'http://www.data5u.com/free/gnpt/index.shtml'
    #     ]
    #     key = 'ABCDEFGHIZ'
    #     for url in url_list:
    #         html_tree = getHtmlTree(url)
    #         ul_list = html_tree.xpath('//ul[@class="l2"]')
    #         for ul in ul_list:
    #             try:
    #                 ip = ul.xpath('./span[1]/li/text()')[0]
    #                 classnames = ul.xpath('./span[2]/li/attribute::class')[0]
    #                 classname = classnames.split(' ')[1]
    #                 port_sum = 0
    #                 for c in classname:
    #                     port_sum *= 10
    #                     port_sum += key.index(c)
    #                 port = port_sum >> 3
    #                 yield '{}:{}'.format(ip, port)
    #             except Exception as e:
    #                 print(e)

    @staticmethod
    def freeProxy02(count=20):
        """
        代理66 http://www.66ip.cn/
        :param count: 提取数量
        :return:
        """
        urls = [
            "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",
            "http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s"
            "tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip"
        ]

        try:
            import execjs
            import requests

            headers = {
                'User-Agent':
                    'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
                'Accept': '*/*',
                'Connection': 'keep-alive',
                'Accept-Language': 'zh-CN,zh;q=0.8'
            }
            session = requests.session()
            src = session.get("http://www.66ip.cn/", headers=headers).text
            src = src.split("</script>")[0] + '}'
            src = src.replace("<script>", "function test() {")
            src = src.replace("while(z++)try{eval(",
                              ';var num=10;while(z++)try{var tmp=')
            src = src.replace(
                ");break}",
                ";num--;if(tmp.search('cookie') != -1 | num<0){return tmp}}")
            ctx = execjs.compile(src)
            src = ctx.call("test")
            src = src[src.find("document.cookie="):src.find("};if((")]
            src = src.replace("document.cookie=", "")
            src = "function test() {var window={}; return %s }" % src
            cookie = execjs.compile(src).call('test')
            js_cookie = cookie.split(";")[0].split("=")[-1]
        except Exception as e:
            print(e)
            return

        for url in urls:
            try:
                html = session.get(url.format(count),
                                   cookies={
                                       "__jsl_clearance": js_cookie
                                   },
                                   headers=headers).text
                ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}",
                                 html)
                for ip in ips:
                    init_proxy_dict()
                    proxy_dict["proxy"] = ip.strip()
                    yield json.dumps(proxy_dict)
            except Exception as e:
                print(e)
                pass

    @staticmethod
    def freeProxy03(page_count=1):
        """
        西刺代理 http://www.xicidaili.com
        :return:
        """
        url_list = [
            'http://www.xicidaili.com/nn/',  # 高匿
            # 'http://www.xicidaili.com/nt/',  # 透明
        ]
        for each_url in url_list:
            for i in range(1, page_count + 1):
                page_url = each_url + str(i)
                tree = getHtmlTree(page_url)
                proxy_list = tree.xpath(
                    './/table[@id="ip_list"]//tr[position()>1]')
                for proxy in proxy_list:
                    try:
                        init_proxy_dict()
                        proxy_dict["proxy"] = ':'.join(
                            proxy.xpath('./td/text()')[0:2])
                        proxy_dict["proxy_type"] = proxy.xpath('./td/text()')[5]
                        yield json.dumps(proxy_dict)
                    except Exception as e:
                        pass

    @staticmethod
    def freeProxy04():
        """
        guobanjia http://www.goubanjia.com/
        :return:
        """
        url = "http://www.goubanjia.com/"
        tree = getHtmlTree(url)
        proxy_list = tree.xpath('//td[@class="ip"]')
        proxy_attr = tree.xpath('//tr[@class="success" or @class="warning"]')
        # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
        # 需要过滤掉<p style="display:none;">的内容
        xpath_str = """.//*[not(contains(@style, 'display: none'))
                                        and not(contains(@style, 'display:none'))
                                        and not(contains(@class, 'port'))
                                        ]/text()
                                """
        for each_proxy, each_attr in proxy_list, proxy_attr:
            try:
                # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
                ip_addr = ''.join(each_proxy.xpath(xpath_str))

                # HTML中的port是随机数，真正的端口编码在class后面的字母中。
                # 比如这个：
                # <span class="port CFACE">9054</span>
                # CFACE解码后对应的是3128。
                port = 0
                for _ in each_proxy.xpath(".//span[contains(@class, 'port')]"
                                          "/attribute::class")[0]. \
                        replace("port ", ""):
                    port *= 10
                    port += (ord(_) - ord('A'))
                port /= 8
                init_proxy_dict()
                proxy_dict["proxy"] = '{}:{}'.format(ip_addr, int(port))
                proxy_dict["anonimity"] = each_attr.xpath(".//td/text()")[2]
                proxy_dict["proxy_type"] = each_attr.xpath(".//td/text()")[3]
                yield json.dumps(proxy_dict)
            except Exception as e:
                pass

    @staticmethod
    def freeProxy05():
        """
        快代理 https://www.kuaidaili.com
        """
        url_list = [
            'https://www.kuaidaili.com/free/inha/'
            # 'https://www.kuaidaili.com/free/intr/'
        ]
        for url in url_list:
            tree = getHtmlTree(url)
            proxy_list = tree.xpath('.//table//tr')
            sleep(1)  # 必须sleep 不然第二条请求不到数据
            for tr in proxy_list[1:]:
                init_proxy_dict()
                proxy_dict['proxy'] = ':'.join(tr.xpath('./td/text()')[0:2])
                proxy_dict['proxy_type'] = tr.xpath('./td/text()')[3]
                proxy_dict['anonimity'] = tr.xpath('./td/text()')[2]
                yield json.dumps(proxy_dict)

    # @staticmethod
    # def freeProxy06():
    #     """
    #     码农代理 https://proxy.coderbusy.com/
    #     :return:
    #     """
    #     urls = ['https://proxy.coderbusy.com/']
    #     for url in urls:
    #         tree = getHtmlTree(url)
    #         proxy_list = tree.xpath('.//table//tr')
    #         for tr in proxy_list[1:]:
    #             init_proxy_dict()
    #             proxy_dict['proxy'] = ':'.join(tr.xpath('./td/text()')[0:2])
    #             yield json.dumps(proxy_dict)

    @staticmethod
    def freeProxy07():
        """
        云代理 http://www.ip3366.net/free/
        :return:
        """
        urls = [
            'http://www.ip3366.net/free/?stype=1',
            "http://www.ip3366.net/free/?stype=2"
        ]
        request = WebRequest()
        for url in urls:
            tree = getHtmlTree(url)
            proxy_list = tree.xpath('.//table//tr')
            sleep(1)
            for tr in proxy_list[1:]:
                init_proxy_dict()
                proxy_dict['proxy'] = ':'.join(tr.xpath('./td/text()')[0:2])
                proxy_dict['proxy_type'] = tr.xpath('./td/text()')[3]
                proxy_dict['anonimity'] = tr.xpath('./td/text()')[2]
                yield json.dumps(proxy_dict)

            # r = request.get(url, timeout=10)
            # proxies = re.findall(
            #     r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>',
            #     r.text)
            # for proxy in proxies:
            #     init_proxy_dict()
            #     proxy_dict['proxy'] = ":".join(proxy)
            #     yield json.dumps(proxy_dict)

    # @staticmethod
    # def freeProxy08():
    #     """
    #     IP海 http://www.iphai.com/free/ng
    #     :return:
    #     """
    #     urls = [
    #         'http://www.iphai.com/free/ng', 'http://www.iphai.com/free/np',
    #         'http://www.iphai.com/free/wg', 'http://www.iphai.com/free/wp'
    #     ]
    #     request = WebRequest()
    #     for url in urls:
    #         r = request.get(url, timeout=10)
    #         proxies = re.findall(
    #             r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',
    #             r.text)
    #         for proxy in proxies:
    #             init_proxy_dict()
    #             proxy_dict['proxy'] = ":".join(proxy)
    #             yield json.dumps(proxy_dict)

    @staticmethod
    def freeProxy09(page_count=1):
        """
        http://ip.jiangxianli.com/?page=
        免费代理库
        :return:
        """
        for i in range(1, page_count + 1):
            url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)
            html_tree = getHtmlTree(url)
            for index, tr in enumerate(html_tree.xpath("//table//tr")):
                if index == 0:
                    continue
                init_proxy_dict()
                proxy_dict['proxy'] = ":".join(tr.xpath("./td/text()")[0:2]).strip()
                proxy_dict['proxy_type'] = tr.xpath("./td/text()")[3]
                proxy_dict['anonimity'] = tr.xpath("./td/text()")[2]
                yield json.dumps(proxy_dict)

    # @staticmethod
    # def freeProxy10():
    #     """
    #     墙外网站 cn-proxy
    #     :return:
    #     """
    #     urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
    #     request = WebRequest()
    #     for url in urls:
    #         r = request.get(url, timeout=10)
    #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
    #         for proxy in proxies:
    #             yield ':'.join(proxy)

    # @staticmethod
    # def freeProxy11():
    #     """
    #     https://proxy-list.org/english/index.php
    #     :return:
    #     """
    #     urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]
    #     request = WebRequest()
    #     import base64
    #     for url in urls:
    #         r = request.get(url, timeout=10)
    #         proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
    #         for proxy in proxies:
    #             yield base64.b64decode(proxy).decode()

    # @staticmethod
    # def freeProxy12():
    #     urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
    #     request = WebRequest()
    #     for url in urls:
    #         r = request.get(url, timeout=10)
    #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
    #         for proxy in proxies:
    #             yield ':'.join(proxy)

    @staticmethod
    def freeProxy13(max_page=2):
        """
        http://www.qydaili.com/free/?action=china&page=1
        齐云代理
        :param max_page:
        :return:
        """
        base_url = 'http://www.qydaili.com/free/?action=china&page='
        for page in range(1, max_page + 1):
            url = base_url + str(page)
            tree = getHtmlTree(url)
            proxy_list = tree.xpath('.//table//tr')
            sleep(1)
            for tr in proxy_list[1:]:
                init_proxy_dict()
                proxy_dict['proxy'] = ':'.join(tr.xpath('./td/text()')[0:2])
                proxy_dict['proxy_type'] = tr.xpath('./td/text()')[3]
                proxy_dict['anonimity'] = tr.xpath('./td/text()')[2]
                yield json.dumps(proxy_dict)

            # r = request.get(url, timeout=10)
            # proxies = re.findall(
            #     r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
            #     r.text)
            # for proxy in proxies:
            #     init_proxy_dict()
            #     proxy_dict['proxy'] = ':'.join(proxy)
            #     yield json.dumps(proxy_dict)

    # @staticmethod
    # def freeProxy14(max_page=2):
    #     """
    #     http://www.89ip.cn/index.html
    #     89免费代理
    #     :param max_page:
    #     :return:
    #     """
    #     base_url = 'http://www.89ip.cn/index_{}.html'
    #     request = WebRequest()
    #     for page in range(1, max_page + 1):
    #         url = base_url.format(page)
    #         r = request.get(url, timeout=10)
    #         proxies = re.findall(
    #             r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
    #             r.text)
    #         for proxy in proxies:
    #             init_proxy_dict()
    #             proxy_dict['proxy'] = ':'.join(proxy)
    #             yield json.dumps(proxy_dict)

    @staticmethod
    def freeProxy15():
        urls = [
            # 'http://www.xiladaili.com/putong/',
            "http://www.xiladaili.com/gaoni/"
            # "http://www.xiladaili.com/http/",
            # "http://www.xiladaili.com/https/"
        ]
        for url in urls:
            tree = getHtmlTree(url)
            proxy_list = tree.xpath('.//table//tr')
            sleep(1)
            for tr in proxy_list[1:]:
                init_proxy_dict()
                proxy_dict['proxy'] = tr.xpath('./td/text()')[0]
                proxy_dict['proxy_type'] = re.sub(re.compile(r'[\u4e00-\u9fa5]'), '', tr.xpath('./td/text()')[1])
                proxy_dict['anonimity'] = tr.xpath('./td/text()')[2]
                yield json.dumps(proxy_dict)

    @staticmethod
    def zhimaProxy():
        # db.changeTable('ProxyPool:useful_proxy_3')
        db.changeTable('ProxyPool:useful_proxy_63')
        if db.getNumber() < USEFUL_PROXY_COUNT / 2:
            rsp = json.loads(requests.get(ZHIMA_PROXY_API).text)
            if rsp['success']:
                for proxy in rsp['data']:
                    proxy_dict['proxy'] = proxy['ip'] + ':' + str(proxy['port'])
                    yield json.dumps(proxy_dict)


if __name__ == '__main__':
    from CheckProxy import CheckProxy

    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy02)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy03)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy04)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy05)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy06)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy07)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy08)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy09)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy13)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy14)
    # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy15)

    CheckProxy.checkAllGetProxyFunc()