136 lines
3.8 KiB
Python
136 lines
3.8 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
# !/usr/bin/env python
|
|||
|
|
"""
|
|||
|
|
-------------------------------------------------
|
|||
|
|
File Name： SsdbClient.py
|
|||
|
|
Description : 封装SSDB操作
|
|||
|
|
Author : JHao
|
|||
|
|
date： 2016/12/2
|
|||
|
|
-------------------------------------------------
|
|||
|
|
Change Activity:
|
|||
|
|
2016/12/2:
|
|||
|
|
2017/09/22: PY3中 redis-py返回的数据是bytes型
|
|||
|
|
2017/09/27: 修改pop()方法 返回{proxy:value}字典
|
|||
|
|
-------------------------------------------------
|
|||
|
|
"""
|
|||
|
|
__author__ = 'JHao'
|
|||
|
|
|
|||
|
|
from Config.setting import PY3
|
|||
|
|
|
|||
|
|
from redis.connection import BlockingConnectionPool
|
|||
|
|
from redis import Redis
|
|||
|
|
|
|||
|
|
|
|||
|
|
class SsdbClient(object):
|
|||
|
|
"""
|
|||
|
|
SSDB client
|
|||
|
|
|
|||
|
|
SSDB中代理存放的结构为hash：
|
|||
|
|
原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
|
|||
|
|
验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
|
|||
|
|
|
|||
|
|
"""
|
|||
|
|
def __init__(self, name, **kwargs):
|
|||
|
|
"""
|
|||
|
|
init
|
|||
|
|
:param name: hash name
|
|||
|
|
:param host: host
|
|||
|
|
:param port: port
|
|||
|
|
:param password: password
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
self.name = name
|
|||
|
|
self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
|
|||
|
|
|
|||
|
|
def get(self, proxy_str):
|
|||
|
|
"""
|
|||
|
|
从hash中获取对应的proxy, 使用前需要调用changeTable()
|
|||
|
|
:param proxy_str: proxy str
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
data = self.__conn.hget(name=self.name, key=proxy_str)
|
|||
|
|
if data:
|
|||
|
|
return data.decode('utf-8') if PY3 else data
|
|||
|
|
else:
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def put(self, proxy_obj):
|
|||
|
|
"""
|
|||
|
|
将代理放入hash, 使用changeTable指定hash name
|
|||
|
|
:param proxy_obj: Proxy obj
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
|
|||
|
|
return data
|
|||
|
|
|
|||
|
|
def delete(self, proxy_str):
|
|||
|
|
"""
|
|||
|
|
移除指定代理, 使用changeTable指定hash name
|
|||
|
|
:param proxy_str: proxy str
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
self.__conn.hdel(self.name, proxy_str)
|
|||
|
|
|
|||
|
|
def exists(self, proxy_str):
|
|||
|
|
"""
|
|||
|
|
判断指定代理是否存在, 使用changeTable指定hash name
|
|||
|
|
:param proxy_str: proxy str
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
return self.__conn.hexists(self.name, proxy_str)
|
|||
|
|
|
|||
|
|
def update(self, proxy_obj):
|
|||
|
|
"""
|
|||
|
|
更新 proxy 属性
|
|||
|
|
:param proxy_obj:
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
|
|||
|
|
|
|||
|
|
def pop(self):
|
|||
|
|
"""
|
|||
|
|
弹出一个代理
|
|||
|
|
:return: dict {proxy: value}
|
|||
|
|
"""
|
|||
|
|
# proxies = self.__conn.hkeys(self.name)
|
|||
|
|
# if proxies:
|
|||
|
|
# proxy = random.choice(proxies)
|
|||
|
|
# value = self.__conn.hget(self.name, proxy)
|
|||
|
|
# self.delete(proxy)
|
|||
|
|
# return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
|
|||
|
|
# 'value': value.decode('utf-8') if PY3 and value else value}
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
def getAll(self):
|
|||
|
|
"""
|
|||
|
|
列表形式返回所有代理, 使用changeTable指定hash name
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
item_dict = self.__conn.hgetall(self.name)
|
|||
|
|
if PY3:
|
|||
|
|
return [value.decode('utf8') for key, value in item_dict.items()]
|
|||
|
|
else:
|
|||
|
|
return item_dict.values()
|
|||
|
|
|
|||
|
|
def clear(self):
|
|||
|
|
"""
|
|||
|
|
清空所有代理, 使用changeTable指定hash name
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
return self.__conn.execute_command("hclear", self.name)
|
|||
|
|
|
|||
|
|
def getNumber(self):
|
|||
|
|
"""
|
|||
|
|
返回代理数量
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
return self.__conn.hlen(self.name)
|
|||
|
|
|
|||
|
|
def changeTable(self, name):
|
|||
|
|
"""
|
|||
|
|
切换操作对象
|
|||
|
|
:param name: raw_proxy/useful_proxy
|
|||
|
|
:return:
|
|||
|
|
"""
|
|||
|
|
self.name = name
|