爬取 小米商城 首页中商品的 “商品图片连接”,“商品链接”,“商品名”,“价格”
#-*- coding = utf-8 -*-#@Time:2020/7/19 9:35#@Author:huxuehao#@File:#@Software:PyCharm#@Emial:#@Theme:#下面是爬虫所使用的库import re #正则表达式import urllib.request,urllib.error #指定url,获取网页数据import xlwt #进行Excel操作import csv #进行csv操作from bs4 import BeautifulSoup #网页解析#注意下面路径中的必须是“start=”，因为我们通过手动添加start的值进行锁定爬取页面src="https://www.mi.com/"#获取网页源码def gethtml(url): head={ "User-Agent": "Mozilla / 5.0(Windows NT 10.0;Win64;x64) AppleWebKit / 537.36(KHTML, likeGecko) Chrome / 83.0.4103.116Safari / 537.36" } #User-Agent的信息为Chrome的信息，请将默认浏览器设置为Chrome浏览器 req=urllib.request.Request(url,headers=head) response=urllib.request.urlopen(req) html=response.read().decode("utf-8","ignore") # print(html) soup=BeautifulSoup(html,"html.parser") # print(soup) return soupsoups=gethtml(src)#爬取信息的正则表达式findtupianlink=re.compile(r'data-src="(.*?)" height="110" src="//i1.mifile.cn/f/i/2014/cn/placeholder-220!110x110.png"',re.S)findshangpinlink=re.compile(r'<a data-log_code="(.*?)" rel="external nofollow" rel="external nofollow" href="(.*?)" target="_blank">',re.S)findname=re.compile(r'<div class="title">(.*?)</div>',re.S)findmoney=re.compile(r'<p class="price">(.*?)(元|元起)</p>',re.S)datalist=[]# for item in soups.find_all('li',class_=""):for item in soups.find_all('a'): # print(item) item=str(item) data=[] tupian = re.findall(findtupianlink, item) if len(tupian)< 1: continue else: data.append(tupian[0]) shangpin = re.findall(findshangpinlink, item) if len(shangpin) < 1: continue else: #因为a data-log_code="(.*?)" rel="external nofollow" rel="external nofollow" href="(.*?)" target="_blank">，所以会返回一个元组("","") #且保存在shangpin[0]中 #我们想获得元组的的第二个元素，所以要shangpin[0][1] data.append(shangpin[0][1]) name = re.findall(findname, item) if len(name) < 1: continue else: data.append(name[0]) money = re.findall(findmoney, item) if len(money) < 1: continue else: data.append(money[0][0]) datalist.append(data)# for i in datalist:# for j in i:# print(j,end="t")# print()# print(len(datalist))#将list中的信息写入txt文档f=open("xiami.txt","w",encoding="utf-8")for i in datalist: # print("已写入第%d条" % (i + 1)) for j in i: string=str(j) #转换成字符串 f.write(string) #写入 f.write(",") #用','进行隔开 f.write("n")f.close#将list中的信息存到Excel中book = xlwt.Workbook(encoding="utf-8",style_compression=0) # 创建book对象sheet = book.add_sheet('mi',cell_overwrite_ok=True) # 创建工作表col=("商品图片连接","商品链接","商品名","价格")for i in range(len(col)): sheet.write(0,i,col[i]) #将col中的信息写入第一行的1-4列for i in range(0,len(datalist)): # print("已写入第%d条"%(i+1)) data=datalist[i] for j in range(0,4): sheet.write(i+1,j,data[j])book.save("xiaomi.xls")#将list中的数据存储到csv文件中with open('xiaomi.csv', 'w', encoding='utf-8',newline='') as csvfile: writer = csv.writer(csvfile, delimiter=',') # 这里在初始化写入对象时传入delimiter为','，此时输出结果的每一列就是以','分隔了 writer.writerow(["商品图片连接","商品链接","商品名","价格"]) for i in datalist: #将datalist中的列表循环写入 writer.writerow(i)print("爬取信息保存完毕!")