rp = robotparser.RobotFileParser() rp.set_url("https://news.ycombinator.com/news/robots.txt" https://news.ycombinator.com/news/robots.txt") rp.read() # Reads

rp = robotparser.RobotFileParser() rp.set_url("https://news.ycombinator.com/news/robots.txt" https://news.ycombinator.com/news/robots.txt") rp.read() # Reads the robots.txt rp.can_fetch("*", 'https://news.ycombinator.com/news' https://news.ycombinator.com/news') >>>> True