134 lines
3.7 KiB
Python
134 lines
3.7 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
-------------------------------------------------
|
||
File Name： RedisClient
|
||
Description : 封装Redis相关操作
|
||
Author : JHao
|
||
date： 2019/8/9
|
||
-------------------------------------------------
|
||
Change Activity:
|
||
2019/8/9: 封装Redis相关操作
|
||
-------------------------------------------------
|
||
"""
|
||
__author__ = 'JHao'
|
||
|
||
from Config.setting import PY3
|
||
|
||
from redis.connection import BlockingConnectionPool
|
||
from redis import Redis
|
||
|
||
|
||
class RedisClient(object):
|
||
"""
|
||
Redis client 和SSDB协议一致 数据结构一致, 但部分方法不通用
|
||
|
||
Redis中代理存放的结构为hash：
|
||
原始代理存放在name为raw_proxy的hash中, key为代理的ip:por, value为代理属性的字典;
|
||
验证后的代理存放在name为useful_proxy的hash中, key为代理的ip:port, value为代理属性的字典;
|
||
|
||
"""
|
||
|
||
def __init__(self, name, **kwargs):
|
||
"""
|
||
init
|
||
:param name: hash name
|
||
:param host: host
|
||
:param port: port
|
||
:param password: password
|
||
:return:
|
||
"""
|
||
self.name = name
|
||
self.__conn = Redis(connection_pool=BlockingConnectionPool(**kwargs))
|
||
|
||
def get(self, proxy_str):
|
||
"""
|
||
从hash中获取对应的proxy, 使用前需要调用changeTable()
|
||
:param proxy_str: proxy str
|
||
:return:
|
||
"""
|
||
data = self.__conn.hget(name=self.name, key=proxy_str)
|
||
if data:
|
||
return data.decode('utf-8') if PY3 else data
|
||
else:
|
||
return None
|
||
|
||
def put(self, proxy_obj):
|
||
"""
|
||
将代理放入hash, 使用changeTable指定hash name
|
||
:param proxy_obj: Proxy obj
|
||
:return:
|
||
"""
|
||
data = self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
|
||
return data
|
||
|
||
def delete(self, proxy_str):
|
||
"""
|
||
移除指定代理, 使用changeTable指定hash name
|
||
:param proxy_str: proxy str
|
||
:return:
|
||
"""
|
||
self.__conn.hdel(self.name, proxy_str)
|
||
|
||
def exists(self, proxy_str):
|
||
"""
|
||
判断指定代理是否存在, 使用changeTable指定hash name
|
||
:param proxy_str: proxy str
|
||
:return:
|
||
"""
|
||
return self.__conn.hexists(self.name, proxy_str)
|
||
|
||
def update(self, proxy_obj):
|
||
"""
|
||
更新 proxy 属性
|
||
:param proxy_obj:
|
||
:return:
|
||
"""
|
||
self.__conn.hset(self.name, proxy_obj.proxy, proxy_obj.info_json)
|
||
|
||
def pop(self):
|
||
"""
|
||
弹出一个代理
|
||
:return: dict {proxy: value}
|
||
"""
|
||
# proxies = self.__conn.hkeys(self.name)
|
||
# if proxies:
|
||
# proxy = random.choice(proxies)
|
||
# value = self.__conn.hget(self.name, proxy)
|
||
# self.delete(proxy)
|
||
# return {'proxy': proxy.decode('utf-8') if PY3 else proxy,
|
||
# 'value': value.decode('utf-8') if PY3 and value else value}
|
||
return None
|
||
|
||
def getAll(self):
|
||
"""
|
||
列表形式返回所有代理, 使用changeTable指定hash name
|
||
:return:
|
||
"""
|
||
item_dict = self.__conn.hgetall(self.name)
|
||
if PY3:
|
||
return [value.decode('utf8') for key, value in item_dict.items()]
|
||
else:
|
||
return item_dict.values()
|
||
|
||
def clear(self):
|
||
"""
|
||
清空所有代理, 使用changeTable指定hash name
|
||
:return:
|
||
"""
|
||
return self.__conn.delete(self.name)
|
||
|
||
def getNumber(self):
|
||
"""
|
||
返回代理数量
|
||
:return:
|
||
"""
|
||
return self.__conn.hlen(self.name)
|
||
|
||
def changeTable(self, name):
|
||
"""
|
||
切换操作对象
|
||
:param name: raw_proxy/useful_proxy
|
||
:return:
|
||
"""
|
||
self.name = name
|