■网友
蟹腰先上代码：# -*- coding: utf-8 -*-import requestsimport reimport datetimeimport osimport csvclass Spider(): def __init__(self): self.url=u\u0026#39;http://www.nea.gov.sg/anti-pollution-radiation-protection/air-pollution-control/psi/historical-psi-readings/year/yearNum/month/monthNum/day/dayNum\u0026#39; self.headers={ \u0026#39;Host\u0026#39;: \u0026#39;www.nea.gov.sg\u0026#39;, \u0026#39;User-Agent\u0026#39;: \u0026#39;Mozilla/5.0 (Windows NT 6.1; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0\u0026#39;, \u0026#39;Connection\u0026#39;: \u0026#39;keep-alive\u0026#39; } zuhe1=u\u0026#39;\u0026lt;tr\u0026gt;.*?\u0026#39; zuhe2=u\u0026#39;\u0026lt;td\u0026gt;(.*?)\u0026lt;/td\u0026gt;.*?\u0026#39; zuhe3=u\u0026#39;\u0026lt;/tr\u0026gt;\u0026#39; self.zuhe=zuhe1+zuhe2*7+zuhe3 def tool(self,x): x=re.sub(re.compile(\u0026#39;\u0026lt;.*?\u0026gt;\u0026#39;),\u0026#39;\u0026#39;,x) x=re.sub(re.compile(\u0026#39;\\u0026#39;),\u0026#39;\u0026#39;,x) x=re.sub(re.compile(\u0026#39;\\r\u0026#39;),\u0026#39;\u0026#39;,x) x=x.encode(\u0026#39;utf-8\u0026#39;) return x.strip() def handleDate(self,year,month,day): #返回日期数据 date=datetime.date(year,month,day)# print date.datetime.datetime.strftime(\u0026#39;%Y-%m-%d\u0026#39;) return date #日期对象 def timeDelta(self,year,month): #计算一个月有多少天 date=datetime.date(year,month,1) try: date2=datetime.date(date.year,date.month+1,date.day) except: date2=datetime.date(date.year+1,1,date.day) dateDelta=(date2-date).days return dateDelta def getPageContent(self,date): url=self.url url=url.replace(u\u0026#39;yearNum\u0026#39;,str(date.year)) url=url.replace(u\u0026#39;monthNum\u0026#39;,str(date.month)) url=url.replace(u\u0026#39;dayNum\u0026#39;,str(date.day))# print url r=requests.get(url) r.encoding=\u0026#39;utf-8\u0026#39; pageContent=r.text# f=open(\u0026#39;content.html\u0026#39;,\u0026#39;w\u0026#39;)# f.write(pageContent.encode(\u0026#39;utf-8\u0026#39;))# f.close() return pageContent def getPageInfos(self,pageContent): pattern1=re.compile(u\u0026#39;\u0026lt;tbody\u0026gt;(.*?)\u0026lt;/tbody\u0026gt;\u0026#39;,re.S) result1=re.search(pattern1,pageContent) content1=result1.group(1) pattern2=re.compile(self.zuhe,re.S) infos=re.findall(pattern2,content1) return infos def saveInfo(self,info,date): fileName= \u0026#39;psi/\u0026#39;+datetime.datetime.strftime(date,\u0026#39;%Y\u0026#39;)+\u0026#39;/\u0026#39;+datetime.datetime.strftime(date,\u0026#39;%m\u0026#39;)+\u0026#39;/\u0026#39;+datetime.datetime.strftime(date,\u0026#39;%d\u0026#39;)+\u0026#39;.csv\u0026#39; if os.path.exists(fileName): mode=\u0026#39;ab\u0026#39; else: mode=\u0026#39;wb\u0026#39; csvfile=file(fileName,mode) writer=csv.writer(csvfile)# if mode==\u0026#39;wb\u0026#39;:# writer.writerow(self.rowName) writer.writerow() csvfile.close() def mkdir(self,date): #创建目录 path = \u0026#39;psi/\u0026#39;+datetime.datetime.strftime(date,\u0026#39;%Y\u0026#39;)+\u0026#39;/\u0026#39;+datetime.datetime.strftime(date,\u0026#39;%m\u0026#39;) isExists=os.path.exists(path) if not isExists: os.makedirs(path) def saveAllInfo(self,infos,date): for (i,info) in enumerate(infos): self.mkdir(date) self.saveInfo(info,date)# print \u0026#39;save info from link\u0026#39;,i+1,\u0026#39;/\u0026#39;,len(infos)