import os, sys
os_sep = os.sep
base_dir = os.path.dirname(os_sep.join(os.path.abspath(file).split(os_sep)[0:-2]))
sys.path.append(base_dir)
from core.utils import MysqlHelper
import time
import logging
import requests
import threading
from selenium import webdriver
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s[thread:%(thread)d][process:%(process)d]',
datefmt='%a, %d %b %Y %H:%M:%S',
filename='%s/log/%s%s.log' % (
base_dir, time.strftime('%Y%m%d', time.localtime(time.time())), file.split(os_sep)[-1]),
filemode='a')
class MyThread(threading.Thread):
def init(self, func, args, name):
threading.Thread.init(self)
self.func, self.args, self.name = func, args, name
def run(self):
self.func(self.args)
url_counter = 0
def main():
# 最近3600秒内url在test_tab0具有唯一性
mysql_obj = MysqlHelper()
#
"""
联合检测
q = 'SELECT url FROM ( SELECT url FROM test_tab0 WHERE unix_timestamp(now()) - create_time<=3600 UNION ALL SELECT url FROM test_tab0_error where status!=2 ) AS t GROUP BY url;'
未失效url检测
q = 'SELECT url FROM test_tab0 WHERE unix_timestamp(now()) <expire_time;'
当前半小时
q = 'SELECT url FROM test_tab0 WHERE unix_timestamp(now()) - create_time<= 1800;'
后期任务：
test_tab0_error积累一定数据后对url重新检测
#3个功能点：当前半个小时、当前未失效的url test_tab0内url的异常情况（当前的2个功能点）、（后期任务：test_tab0_error积累一定数据后对url重新检测）
"""
q = 'SELECT url FROM test_tab0 WHERE unix_timestamp(now()) <expire_time;'
tuple_l = mysql_obj.select(q)
del mysql_obj
if len(tuple_l) == 0:
s = '无待检测url，程序退出'
print(s)
logging.info(s)
# 考虑到每1小时执行下该脚本，对url异常的处理为：第一次请求为预期则终止请求，反之，间隔30后，再至多请求2次，每次间隔10s
sleep_counter, sleep_step, sleep_seconds, mycode_l, repeat_times, repeat_sleep_times = 0, 20, 1, [
'g3user.com', '51g3.com.cn'], 4, 10
# 重构到基类 where list
# d当前为为了f_l字段的需求改动
def get_onerow(url, f_l=['title', 'uid', 'money_total'], tab='test_tab0'):
mysql_obj = MysqlHelper()
f_s = ','.join(f_l)
q = 'SELECT %s FROM %s WHERE url="%s" ORDER BY id DESC LIMIT 1' % (f_s, tab, url)
s = '%s%s' % (' DB ', q)
logging.info(s)
t = mysql_obj.select(q)
if t != -1:
t = t[0]
del mysql_obj
return t
def chk_exception_url(url, sleep_seconds=0, http_tag='http://'):
time.sleep(sleep_seconds)
global url_counter
ret = {}
# db url状态值 状态 0：打不开 1：打开无广告 2:已处理
ret['ok'], ret['status_code'], s = -1, -1, time.strftime('%Y%m%d %H:%M:%S', time.localtime(time.time()))
try:
if url.find('http') == -1:
url = '%s%s' % (http_tag, url)
r = requests.get(url)
ret['status_code'], txt_pos = int(r.status_code), -1
s = '%s,%s,%s,%s' % (s, ret['status_code'], url, r)
except Exception as e:
ret['ok'] = 0
s = '%s %s %s' % (s, ' SPIDER ', e)
logging.error(s)
print(e, url)
# 当前，仅考虑目标站返回200
if ret['status_code'] == 200:
driver = webdriver.PhantomJS(
executable_path='/usr/local/phantomjs/bin/phantomjs')
driver.get(url)
time.sleep(1)
page_source = driver.page_source
for ii in mycode_l:
if page_source.find(ii) > -1:
ret['ok'], txt_pos = 1, 1
break
if txt_pos == -1:
s = '%s%s' % (s, '返回200,但是在html中未检测到我公司代码。')
ret['ok'], ret['info'] = 0, s
# elif ret['status_code'] == 403:
# www.hsdcw.com/fenlei/41668214.html
elif ret['status_code'] == 403:
pass
else:
ret['ok'], ret['info'] = 0, s
url_counter += 1
s = '%s/%s%s%s' % (url_counter, len(tuple_l), 'chk-ret', s)
print(s)
if ret['ok'] == 0:
logging.warning(s)
else:
logging.info(s)
return ret
tn, tl, tstep = len(tuple_l), [], 5000
def tf(ts):
te = ts + tstep
te = min(te, tn)
for i in tuple_l[ts:te]:
url = i[0]
"""
针对新浪爱问的规则: 不检测
"""
if url.find('iask.sina.com') > -1:
continue
write_db_flag = 1
for t in range(0, repeat_times, 1):
print('threadID', threading.get_ident(), url)
ret = chk_exception_url(url, repeat_sleep_times)
if ret['ok'] == 1:
write_db_flag = 0
break
if write_db_flag == 1:
try:
title, uid, money_total = get_onerow(url)
except Exception as e:
s = '%s %s %s' % (' DB Exception-去test_tab0查', url, e)
logging.info(s)
print(s)
break
# 多线程 考虑到原包的 数据库限制，每次均实例化数据库类，用后删除
mysql_obj = MysqlHelper()
q = 'SELECT id FROM test_tab0_error WHERE url="%s" LIMIT 1' % (url)
print(q)
try:
r = mysql_obj.select(q)
s = '%s%s' % ('-SQL-', q)
logging.info(s)
print(q)
except Exception as e:
s = '%s%s %s' % (' DB Exception-', q, e)
logging.info(s)
print(s)
break
ctime = int(time.time())
db_status = 1 if ret['status_code'] == 200 else 0
if len(r) == 0:
q = 'INSERT INTO test_tab0_error (title,url,status,remarks,update_time,create_time,uid,money) VALUES ("%s","%s","%s","%s","%s","%s","%s","%s")' % (
title, url, db_status, ret['info'], ctime, ctime, uid, money_total)
elif len(r) == 1:
continue
"""
q = 'UPDATE test_tab0_error SET title="%s",status="%s",remarks="%s",update_time="%s" ,uid="%s", money="%s"' % (
title, db_status, ret['info'], ctime, uid, money_total)
后期处理test_tab0_error二次检测的更新
"""
try:
mysql_obj.execute(q)
mysql_obj.commit()
del mysql_obj
s = '%s%s' % (' DB SQL ok ', q)
logging.info(s)
print(s)
except Exception as e:
s = '%s%s %s' % (' DB Exception-', q, e)
logging.error(s)
print(s)
for i in range(0, tn, tstep):
if i >= tn:
break
thread_instance = MyThread(tf, (i), tf.__name__)
tl.append(thread_instance)
for t in tl:
t.setDaemon = False
t.start()
for t in tl:
t.join()
if name == 'main':
main()
