2 ms·
PART 0: Make URLs ^p\n if(http){ _X_;fwrite("/crawl-data/index.html\n",1,23,yyout); t=time(0);tm=*localtime(&t);strftime(date,5,"%Y",&tm); f
by 1vuio0pswjnm7 3y ago
PART 0: Make URLs
^p\n if(http){
_X_;fwrite("/crawl-data/index.html\n",1,23,yyout);
t=time(0);tm=*localtime(&t);strftime(date,5,"%Y",&tm);
for(x=atoi(date)-1;x>2015;x--)
{_X_;printf("/crawl-data/CC-NEWS/%d/index.html\n",x);};
_X_;fwrite("/cc-index/collections/index.html\n",1,33,yyout);
}
"href=\"./2"[^/]+ if(http){
_X_;fwrite("/crawl-data/CC-NEWS/",1,20,yyout);
for(x=8;x<yyleng;x++)putc(yytext[x],yyout);
fwrite("/index.html\n",1,12,yyout);
}
"<title>Common Crawl News Dataset "[^<]{4} if(http){
y=0;for(x=33;x<yyleng;x++){date[y]=yytext[x];y++;}
}
"href=\"./"[0-9]{2}"/warc.paths.gz" if(http){
_X_;fwrite("/crawl-data/CC-NEWS/",1,20,yyout);
fwrite(date,1,4,yyout);
for(x=7;x<yyleng;x++)putc(yytext[x],yyout);
putc(10,yyout);
}
"href=\"./"[^/]+ if(http){
_X_;fwrite("/crawl-data/",1,12,yyout);
for(x=8;x<yyleng;x++)putc(yytext[x],yyout);
fwrite("/robotstxt.paths.gz\n",1,20,yyout);
_X_;fwrite("/crawl-data/",1,12,yyout);
for(x=8;x<yyleng;x++)putc(yytext[x],yyout);
fwrite("/warc.paths.gz\n",1,15,yyout);
_X_;fwrite("/crawl-data/",1,12,yyout);
for(x=8;x<yyleng;x++)putc(yytext[x],yyout);
fwrite("/wat.paths.gz\n",1,14,yyout);
}
^cc-index[^\n]+.gz if(http){
_X_;putc(47,yyout);
fwrite(yytext,1,yyleng,yyout);
putc(10,yyout);
}
"href=\"/crawl-00"[12][^\"]+ |\
"href=\"/parse-output"[^\"]+ |\
"href=\"/crawl-data"[^\"]+.gz if(http){
_X_;for(x=6;x<yyleng;x++)putc(yytext[x],yyout);
putc(10,yyout);
}
^crawl-data\/CC-[^\r\n]+.(warc|wat).gz if(http){
_X_;putc(47,yyout);fwrite(yytext,1,yyleng,yyout);
putc(10,yyout);
}
\"filename\":\"[^\"]+ if(http){
_X_;putc(47,yyout);
for(x=12;x<yyleng;x++)putc(yytext[x],yyout);
putc(10,yyout);
}
\"filename\":[ ]\"crawl-data\/[^\"]+ if(http){
_X_;putc(47,yyout);
for(x=13;x<yyleng;x++)putc(yytext[x],yyout);
putc(10,yyout);
}