3 ms·
from __future__ import print_function import re import requests from bs4 import BeautifulSoup def download_file(download_url, name):
by diminoten 8y ago
from __future__ import print_function
import re
import requests
from bs4 import BeautifulSoup
def download_file(download_url, name):
r = requests.get(download_url, stream = True)
with open("repo" + name, 'wb') as f:
for chunk in r.iter_content(chunk_size = 1024*1024):
if chunk:
f.write(chunk)
html = requests.get("https://getpolarized.io/2019/01/08/top-pdfs-of-2018-hackernews.html")
soup = BeautifulSoup(html.content, features='html.parser')
sAll = soup.findAll("a")
for href in sAll:
if href.has_attr('href'):
link = href['href']
if link.find(".pdf") > 0:
print(link)
last_index = link.rindex("/")
name = link[last_index + 1:]
print(name)
try:
download_file(link, name)
except:
print("error downloading " + link )