2 ms·
Tests client () { Range=bytes=0-$1 yy025|nc -vv h1 80|yy054|gzip -dc 2>&- ;} # h1 is an entry in a HOSTS file # could be address of TLS wrapper, e.g.
by 1vuio0pswjnm7 3y ago
Tests
client () { Range=bytes=0-$1 yy025|nc -vv h1 80|yy054|gzip -dc 2>&- ;}
# h1 is an entry in a HOSTS file
# could be address of TLS wrapper, e.g., stunnel, or TLS fwd proxy
# could also be address of data.commoncrawl.org as HTTPS is optional
# test MAIN
echo p|yy074 0|sed 1q|yy025|nc -vv h1 80 > 1
# test MAIN robotstxt warc
yy074 0 < 1|sed 1q|client 1024|yy074 0|sed 1q >> 1
tail -1 1|client > 1.warc # 5.8 MiB
sqlite3 1.db < 1.sql
# test MAIN warc
yy074 0 < 1|sed -n 2p|client 1024|yy074 0|sed 1q >> 1
tail -1 1|client 10000000 > 1.warc # 35 MiB
yy074 5 < 1.warc|sqlite3 1.db
# test MAIN metadata warc.wat
yy074 0 < 1|sed -n 3p|client 1024|yy074 0|sed 1q >> 1
tail -1 1|client 10000000 > 1.warc # 39.1 MiB
yy074 5 < 1.warc|sqlite3 1.db
# test NEWS 2022
echo p|yy074 0|sed -n /NEWS.2022/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed 1q|client|sed 1q >>1
tail -1 1|yy074 0|client 10000000 > 1.warc # 51.4 MiB
yy074 5 < 1.warc|sqlite3 1.db
# test NEWS 2021
echo p|yy074 0|sed -n /NEWS.2021/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed 1q|client|sed 1q >>1
tail -1 1|yy074 0|client 10000000 > 1.warc # 46.8 MiB
yy074 5 < 1.warc|sqlite3 1.db
# test NEWS 2019
echo p|yy074 0|sed -n /NEWS.2019/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed -n 3p|client|sed 1q >>1
tail -1 1|yy074 0|client 10000000 > 1.warc # 46.9 MiB
grep -m1 ^WARC-IP-Address 1.warc # no DNS data
# test NEWS 2018
echo p|yy074 0|sed -n /NEWS.2018/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed -n 7p|client|sed 1q >>1
tail -1 1|yy074 0|client 10000000 > 1.warc # 42.6 MiB
grep -m1 ^WARC-IP-Address 1.warc # no DNS data
# test cc-index
echo p|yy074 0|sed -n /cc-index/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed 1q|client|sed 1q >>1
tail -1 1|yy074 0|client 1024|yy074 0|grep -m1 robotstxt >>1
tail -1 1|client >1.warc # 3.9 MiB
yy074 5 < 1|sqlite3 1.db
# test cc-index 2012 (IA ARC format)
echo p|yy074 0|sed -n /cc-index/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed -n /parse/p|client >>1
yy074 0 < 1|tail -1|client 1024|yy074 0 >>1
sed -n /arc.gz/p 1|tail -1|(read x;
# modified tnftp that accepts byte range
tnftp -4o'|gzip -dc 2>&- >1.warc' -c 0-10000000 $x) # 42.2 MiB
yy074 5 < 1.warc|sqlite3 1.db
# test cc-index 2009-2010 (IA_ARC format)
echo p|yy074 0|sed -n /cc-index/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed -n /crawl-002/p|client >>1
yy074 0 < 1|tail -1|client 1024|yy074 0 >>1
grep arc.gz 1|tail -1|client 30000000 >1.warc # 35.7 MiB
yy074 5 < 1.warc|sqlite3 1.db
# test cc-index 2008-2009 (IA_ARC format)
echo p|yy074 0|sed -n /cc-index/p|yy025|nc -vv h1 80 > 1
yy074 0 < 1|sed -n /crawl-001/p|client >>1
yy074 0 < 1|tail -1|client 1024|yy074 0 >>1
grep arc.gz 1|tail -1|client 1000000 >1.warc # 4 MiB
yy074 5 < 1.warc|sqlite3 1.db
#rm 1 1.warc