2 ms·
Test Processing Practice Expt (ETL) DNS Data from WARC yy074
/*
Extract DNS Data from ARC_IA/WARC
Detect sitemap availability
0 - dontknow
1 - nositemapfound
2 - sitemapfound
*/
int fileno (FILE *);
#define jmp (yy_start) = 1 + 2 *
int v,w,x,y,z,request,response;
int dnsd,hosts,map,nsd,sqlite3,tinydns,http,append;
char addr[255];
char name[255];
char date[21];
int addrlen,namelen,datelen;
#include <time.h>
void bzero(void *,size_t);
char *strptime(const char *s, const char *f, struct tm *tm);
struct tm tm;
time_t t;
#define _X_ fwrite("https://data.commoncrawl.org",1,28,yyout);
#define _Y_(x) "x) ON CONFLICT(addrname) DO UPDATE SET date=excluded.date, sitemap=excluded.sitemap WHERE sitemap = x;\n"
%s ARC_IA WARC WAT
%option nounput noinput noyywrap
%%
^"1 0 CommonCrawl\nURL IP-address Archive-date Content-type Archive-length\n" {
if(http)yyterminate();jmp ARC_IA;}
^WARC\/1 if(http)yyterminate();jmp WARC;
<WARC>^"WARC-Type: metadata" jmp WAT;
<WAT>^WARC\/1. jmp WARC;
^q\n if(http)yyterminate();
/* insert PART 0: make URLs */
/* insert PART 1: ARC_IA */
/* insert PART 2: WAT */
/* insert PART 3: WARC */
.|\n
%%
int main(int argc,char* argv[])
{
if(argc)if(!argv[1]){
puts("usage: yy074 [0-6][a]");
puts("options:");
puts(" 0 - urls");
puts(" 1 - dnsd");
puts(" 2 - hosts");
puts(" 3 - map");
puts(" 4 - nsd");
puts(" 5 - sqlite3");
puts(" 6 - tinydns");
puts(" a - append");
goto x;
}
if(argc)if(argv[1])switch(argv[1][0]){
case '0': http++;break;
case '1': dnsd++;break;
case '2': hosts++;break;
case '3': map++;break;
case '4': nsd++;break;
case '5': sqlite3++;break;
case '6': tinydns++;break;
default: goto x;
}
if(argc)if(argv[1])if(argv[1][1]=='a')append++;
if(nsd)if(!append)puts(". 1 IN SOA from. arc_ia/warc. 1 0 1 0 1");
if(tinydns)if(!append)puts(".");
if(sqlite3)puts("PRAGMA foreign_keys=OFF;");
if(sqlite3)puts("BEGIN TRANSACTION;");
if(sqlite3)puts("CREATE TABLE IF NOT EXISTS t1(date INTEGER,addrname TEXT PRIMARY KEY,sitemap INTEGER);");
yylex();
if(v==4)if(response)if(sqlite3)if(z)
{
fwrite(_Y_(2),1,103,yyout);
fwrite("COMMIT;\n",1,8,yyout);
goto x;
}
if(v==4)if(response)if(sqlite3)if(!z)
{
fwrite(_Y_(0),1,103,yyout);
fwrite("COMMIT;\n",1,8,yyout);
goto x;
}
if(v==2)if(sqlite3)if(response)fwrite("0);\n",1,4,yyout);
if(v==2)if(sqlite3)fwrite("COMMIT;\n",1,8,yyout);
if((v==1)||(v==3))if(sqlite3)fwrite("COMMIT;\n",1,8,yyout);
x:
exit(0);
}
- 1vuio0pswjnm7 3y agoPART 3: WARC <WARC>^WARC-Type:[ ]request { request++; if(v==4)v=0; z=0; if(sqlite3)fwrite("INSERT or IGNORE INTO t1 VALUES(",1,32,stdout); if(sqlite3)x=0; } <WARC>^WARC-Type:[ ]response response++; <WARC>^WARC-IP-Address:[ ][0-9.]+ if(request){ v=2; if(!map)if(!dnsd)if(!nsd)if(!tinydns)if(!sqlite3) for(x=17;x<yyleng;x++)putc(yytext[x],yyout); if(hosts)putc(32,yyout); if(!hosts) { y=0;for(x=17;x<yyleng;x++) if(y<255){addr[y]=yytext[x];y++;};addrlen=y; } } <WARC>^"WARC-Date: "[-ZT:0-9]{20} if(request){ if((map)|(sqlite3)){y=0;for(x=11;x<yyleng;x++) if(y<21){date[y]=yytext[x];y++;};datelen=y;} if(sqlite3)strptime(date,"%Y-%m-%dT%TZ",&tm); if(sqlite3)printf("%ld,'",mktime(&tm)); } <WARC>^WARC-Target-URI:[ ](https|http):\/\/[^/\n\r]+ if(request){ request--; if(tinydns)putc(61,yyout); if(yytext[21]==115)w=25; if(yytext[21]==58)w=24; if(!sqlite3) for(x=w;x<yyleng;x++)putc(yytext[x],yyout); y=0;for(x=w;x<yyleng;x++)if(y<255){name[y]=yytext[x];y++;}; namelen=y; if(nsd)fwrite(". 1 IN A ",1,9,yyout); if((nsd)||(sqlite3))fwrite(addr,1,addrlen,yyout); if(sqlite3)putc(32,yyout); if(sqlite3)fwrite(name,1,namelen,yyout); if(sqlite3)fwrite("',",1,2,yyout); if(tinydns)putc(58,yyout); if(tinydns)fwrite(addr,1,addrlen,yyout); if(tinydns)fwrite(":1\n",1,3,yyout); if((map)||(dnsd))putc(32,yyout); if((map)||(dnsd))fwrite(addr,1,addrlen,yyout); if(map)fwrite(" # ",1,3,yyout); if(map)strptime(date,"%Y-%m-%dT%TZ",&tm); if(map)printf("%ld",mktime(&tm)); if(!sqlite3)if(!tinydns)putc(10,yyout); } <WARC>^GET[ ]\/robots.txt[ ] v=4; <WARC>[sS][iI][tT][eE][mM][aA][pP].*\.[xX][mM][lL][\r\n] if(response)if(v==4){ response--; z++; if(sqlite3)fwrite(_Y_(2),1,103,yyout); } <WARC>^WARC\/1. if(response){ response--; if(v==4)if(sqlite3)fwrite(_Y_(1),1,103,yyout); if(v!=4)if(v)if(sqlite3)fwrite("0);\n",1,4,yyout); }
- 1vuio0pswjnm7 3y agoPART 2: WAT <WAT>\"WARC-Date\":\"[^\"]+ { if((map)|(sqlite3)){y=0;for(x=13;x<yyleng;x++) if(y<21){date[y]=yytext[x];y++;};datelen=y;} } <WAT>\"WARC-IP-Address\":\"[^\"]+ { v=3; if(!map)if(!dnsd)if(!nsd)if(!tinydns)if(!sqlite3) for(x=19;x<yyleng;x++)putc(yytext[x],yyout); if(hosts)putc(32,yyout); if(!hosts){y=0;for(x=19;x<yyleng;x++) if(y<255){addr[y]=yytext[x];y++;};addrlen=y;}; } <WAT>\"WARC-Target-URI\":\"(https|http):\/\/[^/\"]+ { if(sqlite3)fwrite("INSERT or IGNORE INTO t1 VALUES(",1,32,stdout); if(sqlite3)strptime(date,"%Y-%m-%dT%TZ",&tm); if(sqlite3)printf("%ld,'",mktime(&tm)); if(sqlite3)fwrite(addr,1,addrlen,yyout); if(sqlite3)putc(32,yyout); if(tinydns)putc(61,yyout); if(yytext[23]==115)w=27; if(yytext[23]==58)w=26; for(x=w;x<yyleng;x++)putc(yytext[x],yyout); y=0;for(x=w;x<yyleng;x++)if(y<255){name[y]=yytext[x];y++;}; namelen=y; if(nsd)fwrite(". 1 IN A ",1,9,yyout); if(sqlite3)fwrite("',",1,2,yyout); if(nsd)fwrite(addr,1,addrlen,yyout); if(tinydns)putc(58,yyout); if(tinydns)fwrite(addr,1,addrlen,yyout); if(tinydns)fwrite(":1\n",1,3,yyout); if((map)||(dnsd))putc(32,yyout); if((map)||(dnsd))fwrite(addr,1,addrlen,yyout); if(map)fwrite(" # ",1,3,yyout); if(map)strptime(date,"%Y-%m-%dT%TZ",&tm); if(map)printf("%ld",mktime(&tm)); if(!sqlite3)if(!tinydns)putc(10,yyout); if(sqlite3)fwrite("0);\n",1,4,yyout); }