https://www.archiveteam.org/index.php?title=Wget_with_WARC_output https://www.archiveteam.org/index.php?title=Wget_with_WARC_o... https://github.com/oduwsdl/ip

https://www.archiveteam.org/index.php?title=Wget_with_WARC_output https://www.archiveteam.org/index.php?title=Wget_with_WARC_o... https://github.com/oduwsdl/ipwb https://github.com/oduwsdl/ipwb #!/bin/sh cd ${HOME}/data/archive user_agent="Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27" domain="${1}" wget \ --mirror \ --page-requisites \ --restrict-file-names=nocontrol \ --timeout 60 \ --tries 5 \ --wait 1 \ --waitretry 5 \ --warc-cdx \ --warc-file="${domain}" \ --warc-header "operator: Wayback Mesh" \ -U "${user_agent}" \ -e robots=off \ "http://${domain}" ipwb index "${domain}.warc" > "${domain}.ipfs.cdxj"