能大量爬淘宝的出来工资就能10k+了吧
■网友
https://github.com/Ckend/taobao_scrapy
可以爬任意关键词的商品信息，包括销量等等

■网友
可以看一下我在的文章

■网友
淘宝反爬虫还是比较屌，刚入门写写别的再来爬淘宝吧
■网友
http://www.lovelucy.info/python-crawl-pages.htmlfyi
■网友
Python爬虫，爬取符合条件的淘宝联盟网站的商品。实现如下：
#coding=utf-8__author__ = \u0026#39;yansong\u0026#39;# 2018.07.12# 抓取淘宝联盟 比率\u0026gt;10 ，描述、服务、物流3项参数高于或持平于同行业的商品图片。import jsonimport demjsonimport urllib2import osimport timeimport requestsimport refrom selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.common.keys import Keysfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.support.wait import WebDriverWaitimport sysreload(sys)sys.setdefaultencoding(\u0026#39;utf8\u0026#39;)path_name = u\u0026#39;鞋子\u0026#39;Myname = u\u0026#39;鞋子\u0026#39;# 创建文件夹path = os.getcwd() \t\t\t\t # 获取此脚本所在目录new_path = os.path.join(path,path_name)if not os.path.isdir(new_path):\tos.mkdir(new_path)#url请求，获取响应的json数据，并将json数据转换成dict 采用selenium实现def get_browser_text(url): #browser = webdriver.Chrome(executable_path="C:\\\\Program Files (x86)\\\\Google\\\\Chrome\\\\Application\\\\chromedriver.exe") browser = webdriver.Firefox(executable_path="C:\\\\Program Files (x86)\\\\Mozilla Firefox\\\\geckodriver.exe") try: browser.get(url) print(browser.page_source) browserdata = browser.page_source browser.close() # res = r\u0026#39;\u0026lt;a .*?\u0026gt;(.*?)\u0026lt;/a\u0026gt;\u0026#39; res = r\u0026#39;\u0026lt;pre .*?\u0026gt;(.*?)\u0026lt;/pre\u0026gt;\u0026#39; json_data = re.findall(res, browserdata, re.S | re.M) print json_data for value in json_data: print value dict_data = demjson.decode(json_data) print \u0026#39;dict_data:\u0026#39; print dict_data # print type(dict_data) return dict_data except: return ""#url请求，获取响应的json数据，并将json数据转换成dict 采用 urllib2 实现def getJSONText(url): try: page = urllib2.urlopen(url) data = https://www.zhihu.com/api/v4/questions/35566856/page.read() #print (data) #print (type(data)) #dict_data = json.loads(data) dict_data = demjson.decode(data) #print dict_data #print type(dict_data) return dict_data except: return""# 获取单个商品的HTML代码并用正则匹配出描述、服务、物流3项参数 采用urllib2def getHTMLText(url): try: data = https://www.zhihu.com/api/v4/questions/35566856/urllib2.urlopen(url).read() res = r/u0026#39;/u0026lt;dd class="tb-rate-(.*?)"\u0026#39; data_list = re.findall(res, data, re.S | re.M) print type(data_list) print data_list #for value in mm: # print value return data_list except: return ""#url请求，获取响应的json数据，并将json数据转换成dict 采用 添加请求头、设置代理的方式 实现def getJSONText2(url): try: proxies = { "http": "http://221.10.159.234:1337", "https": "https://60.255.186.169:8888", } headers = {\u0026#39;User-Agent\u0026#39;: \u0026#39;Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36\u0026#39;} data = https://www.zhihu.com/api/v4/questions/35566856/requests.get(url, headers=headers,proxies=proxies).text print (data) print (type(data)) # dict_data = json.loads(data) dict_data = demjson.decode(data) print dict_data print type(dict_data) return dict_data except: return""def getJSONText3(url): try: driver = webdriver.Chrome(executable_path = "C:\\\\Program Files (x86)\\\\Google\\\\Chrome\\\\Application\\\\chromedriver.exe") driver.get(url) # 访问淘宝宝贝页面，获取cookie # driver.get(taobao_comment_url) # 直接访问宝贝评论会被反爬虫检测到。上一步获得cookie后可得到评论数据 print(driver.find_element_by_xpath(\u0026#39;/html/body\u0026#39;).text) data = driver.find_element_by_xpath(\u0026#39;/html/body\u0026#39;).text #data = https://www.zhihu.com/api/v4/questions/35566856/requests.get(url, headers=headers,proxies=proxies).text print (data) print (type(data)) # dict_data = json.loads(data) dict_data = demjson.decode(data) print dict_data print type(dict_data) return dict_data except: return""def mytaobao_spider(): #每一页的url 循环抓取50页 for page in range(3, 60): url = (\u0026#39;http://pub.alimama.com/items/channel/qqhd.json?q=%s\u0026amp;channel=qqhd\u0026amp;_t=1531121449018\u0026amp;toPage=%d\u0026amp;perPageSize=50\u0026amp;shopTag=\u0026amp;t=1531125125414\u0026amp;_tb_token_=eeee6ee3be688\u0026amp;pvid=19_118.112.188.32_688_1531125125232\u0026#39;%(Myname,page)) #url = (\u0026#39;http://pub.alimama.com/items/search.json?q=%E9%9E%8B%E5%AD%90\u0026amp;_t=1531368912715\u0026amp;auctionTag=\u0026amp;perPageSize=50\u0026amp;shopTag=yxjh\u0026amp;t=1531368913289\u0026amp;_tb_token_=e370663ebef17\u0026amp;pvid=10_118.112.188.32_9532_1531368912863\u0026#39;) print url time.sleep(2) #延时2秒，添加延时操作是因为淘宝有反爬虫机制，过于频繁的访问IP可能会被限制 url_data=https://www.zhihu.com/api/v4/questions/35566856/getJSONText(url) #一页中每件商品的标签信息 for i in range(0, 50): time.sleep(1) try: #print type(url_data) pictUrl = url_data #图片url sellerId = url_data # 商品id auctionUrl = url_data # 淘宝链接 auctionId = url_data # 淘宝链接 = /u0026#39;http://item.taobao.com/item.htm?id=%d\u0026#39;%(auctionId) tkRate = url_data # 比率 zkPrice = url_data # 价格 #需要抓取比率大于10.00的商品信息 if tkRate \u0026gt; 10.00: #time.sleep(1) # print \u0026#39;详细信息:\u0026#39; # print type(tkRate) # print type(zkPrice) # print \u0026#39;比率:%f\u0026#39; % (tkRate) # print \u0026#39;价格:%f\u0026#39; % (zkPrice) # print sellerId # print auctionId # print pictUrl # print auctionUrl # 淘宝链接 # print type(sellerId) print auctionUrl # 每件商品的子url (描述相符、发货速度、服务态度 等信息) #sub_url = (\u0026#39;http://pub.alimama.com/pubauc/searchPromotionInfo.json?oriMemberId=%d\u0026amp;blockId=\u0026amp;t=1531369204612\u0026amp;_tb_token_=e370663ebef17\u0026amp;pvid=10_118.112.188.32_760_1531368931581\u0026#39; % (sellerId)) sub_url = auctionUrl #每件商品的淘宝url sub_url_data = getHTMLText(sub_url) #获取店铺的 描述、服务、物流 信息 print type(sub_url_data) print len(sub_url_data) #如果返回的是空字符串, 则说明没有取到我们想要的字段，是因为淘宝有不同的页面，对于这种页面我们需要进一步分析下面的url if (len(sub_url_data) == 0): info_url = (\u0026#39;https://world.taobao.com/item/%d.htm\u0026#39; % (auctionId)) info_data = urllib2.urlopen(info_url).read() res_info = r\u0026#39;\u0026lt;li class="(.*?)\u0026lt;img\u0026#39; tmp_url_data = https://www.zhihu.com/api/v4/questions/35566856/re.findall(res_info, info_data, re.S | re.M) print"tmp_url_data:" for value1 in tmp_url_data: print value1 sub_url_data= score_list = for x in tmp_url_data] #截取前面5位 print \u0026#39;new_list:\u0026#39; for score in score_list: print score if score == \u0026#39;down\u0026#39;: score = \u0026#39;lower\u0026#39; #d第一种页面与第二种页面返回的店铺评定信息不同，需转换成统一的方便后面处理，将 down 转换为 lower sub_url_data.append(score) print \u0026#39;替换后的list元素:\u0026#39; for level_data in sub_url_data: print level_data #如果3项评定参数都不是‘lower’ 就将图片和相关信息抓取出来 任意一项参数为‘lower’都不符合要求 if((not(sub_url_data == \u0026#39;lower\u0026#39; ) ) and (not(sub_url_data == \u0026#39;lower\u0026#39; ) ) and (not(sub_url_data == \u0026#39;lower\u0026#39; ) ) ): #for value in sub_url_data: # print value mypictUrl = \u0026#39;http:\u0026#39; + pictUrl #图片url picture_content = urllib2.urlopen(mypictUrl).read() picture_name = auctionUrl + \u0026#39;.jpg\u0026#39; #拼接图片名称 print picture_name time.sleep(1) #需要写入文件的信息 spider_info = \u0026#39;url:\u0026#39; + url + \u0026#39;\\u0026#39; + \u0026#39; sub_url:\u0026#39; + sub_url + \u0026#39;\\u0026#39; + \u0026#39; 淘宝链接:\u0026#39; + auctionUrl + \u0026#39;\\u0026#39; + \u0026#39; mypictUrl:\u0026#39; + mypictUrl + \u0026#39;\\\u0026#39; try: #写图片 index_num = picture_name.index(\u0026#39;id=\u0026#39;) with open(path_name + \u0026#39;/\u0026#39; + picture_name, \u0026#39;wb\u0026#39;) as code: code.write(picture_content) #写URL信息 with open(path_name + \u0026#39;/\u0026#39; + \u0026#39;spider.txt\u0026#39;, \u0026#39;a\u0026#39;) as spider_code: spider_code.write(spider_info) except (IOError, ZeroDivisionError), e: print e print "Error: 没有找到图片文件或读取文件失败" else: print "图片写入成功" time.sleep(1) except (IndexError, KeyError, TypeError), e: print e print "每件商品信息读取失败" else: pass #print "每件商品的标签信息读取成功"if __name__ == \u0026#39;__main__\u0026#39;: mytaobao_spider()