python-requests多进程下载文件
如下代码实现的是登录某网站后,调用get接口下载某个文件,启动进程池Pool
import requests
import json
import time
from multiprocessing import Pool
import multiprocessing
baseurl = "http://8.25.3.33:28090"
userName = "xxxxx"
password = "xxxxxx"
# 进程数量
pnum = 3
# 总任务队列数量
tnum = 10
def login():
s = requests.Session()
url1 = baseurl + "/appstore/user/login"
formdata1 = {
"userName": userName,
"password": password,
}
header = {"Content-type": "application/json;charset=UTF-8"}
s.headers.update(header)
r1 = s.post(url1, data=json.dumps(formdata1))
return s
def downloadapp(i):
name = multiprocessing.current_process().name
filename = "appstore" + str(i)
s = login()
speed = 0.0
url4 = baseurl + "/appstore/userDownload/1"
r4 = s.get(url4, stream=True)
print(r4.headers.items())
start_time = time.time()
offset = 1
with open(filename, "wb") as code:
for chunk in r4.iter_content(chunk_size=1024*1024):
if chunk:
# dural_time = time.time() - start_time
code.write(chunk)
offset = offset + len(chunk)
# print("offset:",offset)
# try:
# speed = offset / 1024 / 1024 / dural_time
# except ZeroDivisionError as err:
# pass
# print("time zero",err)
# print("threadname: {0}, filename : {1}, speed: {2:.2f}M/s".format(name, filename, speed))
endtime = time.time()
totaltime = endtime - start_time
totalsize = offset / 1024 / 1024
print("{name} : total size {totalsize}MB".format(name=name, totalsize=totalsize))
print("{name} : total cost time {totaltime}".format(name=name, totaltime=totaltime))
totalspeed = offset / 1024 / 1024 / totaltime
print("threadname: {0}, filename : {1}, speed: {2:.2f}M/s".format(name, filename, totalspeed))
s.close()
if __name__ == '__main__':
# set the processes max number 3
pool = Pool(processes=pnum)
# i 总的队列数量
for i in range(1, tnum+1):
result = pool.apply_async(downloadapp, (i,))
pool.close()
pool.join()