4 ms·
Just in case if you thinking to download these links import re import requests from bs4 import BeautifulSoup def download_file(download_url, name): #create r
by createdjustnow 8y ago
Just in case if you thinking to download these links
import re
import requests
from bs4 import BeautifulSoup
def download_file(download_url, name):
#create response object
r = requests.get(download_url, stream = True)
#download started
with open("repo" + name, 'wb') as f:
for chunk in r.iter_content(chunk_size = 1024*1024):
if chunk:
f.write(chunk)
html = requests.get("https://getpolarized.io/2019/01/08/top-pdfs-of-2018-hackernews.html" https://getpolarized.io/2019/01/08/top-pdfs-of-2018-hackerne...)
soup = BeautifulSoup(html.content)
sAll = soup.findAll("a")
for href in sAll:
if(href.has_attr('href')):
link = href['href']
if(link.find(".pdf") > 0):
print(link)
last_index = link.rindex("/")
name = link[last_index + 1:]
print(name)
try:
download_file(link, name)
except:
print("error downloading " + link )
- diminoten 8y agofrom __future__ import print_function import re import requests from bs4 import BeautifulSoup def download_file(download_url, name): r = requests.get(download_url, stream = True) with open("repo" + name, 'wb') as f: for chunk in r.iter_content(chunk_size = 1024*1024): if chunk: f.write(chunk) html = requests.get("https://getpolarized.io/2019/01/08/top-pdfs-of-2018-hackernews.html") soup = BeautifulSoup(html.content, features='html.parser') sAll = soup.findAll("a") for href in sAll: if href.has_attr('href'): link = href['href'] if link.find(".pdf") > 0: print(link) last_index = link.rindex("/") name = link[last_index + 1:] print(name) try: download_file(link, name) except: print("error downloading " + link )