python爬虫最全教程,爬虫python

python爬虫爬古诗网站导入库import requests #获取网页from lxml import etree #分析网页库函数1 :分析网页defget_html(k 获取type_v1 ) 3360 URL=' 3359 so.gush hhi p={ } c={ } t='.format (k1，type_v1 ) #网址print(URL ) ua={ ' Windows NT 6.1； Trident/5.0；' }resp=requests.get(URL， headers=ua (打印(resp.status _ code )打印状态代码#print ) resp.text ) # 打印网页返回resp函数2 :转换格式defchange_html ) resp ) 30 2.将分析元素html=etree.html(resp.text )网页转换为xpath可分析的格式result=2.1分析标题title_V1=html.XPath (/title/text ) ) ) print ) title _ v1 ) 2.2分析名句和作者verse_V1=html.XPath ) div ) ) class='left ' )/div ) @class='sons ' )/div/a verse _ href (author _ v1=html.XPath ()/div ) claref div [ @ class=' sons ' ]/div/a [2]/@ href ' ] # print (len (author _ v1 ) ) print author_href )函数4 :保存的数据def type_v1 ) 3360 'a ' ) as wf: wf.write (页面(n(.format ) k1 ) ) forIinrange(len(verse_V1 ) ) : wf.wrrat author_href[i] )函数5 :主题deftheme_def(html ) :theme_V1=html.XPath((/div ) @class='left ' )/text () (print ) theme_V1 ) return theme_v1函数6 :获取页数type_v1 ) #网页获取函数html=change_html(resp ) )转换函数PPP div [ @ class=' son1' ]/h1/span/text (' ) return page_v1主函数def main ) ) : #1.网页theme_list=[ '无限制] '获取的“植物”、“食物”、“经典”、“女性”、“父亲”、“母亲”、“老师”、“孩子”(] print )主题为(n )、(| . Join ) ) theme_list ) ) type page_v1=page_def(type_v1 ) print ) page_num=int 调用网页获取函数html=change_html(resp ) #转换函数print (html ) resu=elem _ def (html ) #进行元素分析函数save_def ) resu[0]