4 ms·
Text Processing Practice Expt: 27 SERP Types to SQLite (yy084)
/*
make sqlite from s.e.r.p.
*/
int fileno (FILE *);
FILE *f;
#define jmp (yy_start) = 1 + 2 *
int mode;
int v,w,x,y,z;
int utf8,tag;
int datelen,querylen,urllen,desclen;
char desc[255];
char url[255];
char *a;
int setenv (const char *, const char *, int);
int printf(const char *__restrict, ...);
int dprintf(int, const char *__restrict, ...);
unsigned int time(unsigned int *tloc);
int CDp8iULPuCinsert(int xyz)
{
fwrite("INSERT or IGNORE INTO t1 VALUES(",1,32,stdout);
printf("%03d,'",xyz);
fwrite(getenv("query"),1,querylen,stdout);
fwrite("','",1,3,yyout);
fwrite(getenv("date"),1,datelen,yyout);
fwrite("','",1,3,yyout);
return 0;
}
%s M1 M2 M3 M4 M5 M6 M7 M9
%s M10 M11 M12 M13 M14 M15 M16
%s M17 M18 M19 M20 M21 M22 M23
%s M24 M25 M26 M27
%s M2_DESC M3_DESC M5_DESC M11_URL
%s M11_DESC1 M11_DESC2 M13_DESC M24_DESC
%s M25_DESC M27_DESC
%option nounput noinput noyywrap
%%
TWVGHkppk6Q6cODqP7Kqdate:\"[^\"]+ {
y=0;for(x=26;x<yyleng;x++){if(yytext[x])yytext[y]=yytext[x];
if(yytext[yyleng])yytext[yyleng]=0;y++;};
setenv("date",yytext,1);
datelen=y;
}
TWVGHkppk6Q6cODqP7Kqquery:\"[^\"]+ {
y=0;for(x=27;x<yyleng;x++){if(yytext[x])yytext[y]=yytext[x];
if(yytext[yyleng])yytext[yyleng]=0;y++;};
setenv("query",yytext,1);
querylen=y;
}
TWVGHkppk6Q6cODqP7Kqmode:\"[^\"]+ {
y=0;for(x=26;x<yyleng;x++){if(yytext[x])yytext[y]=yytext[x];
if(yytext[yyleng])yytext[yyleng]=0;y++;};
mode=atoi(yytext);
switch(mode){
case 1: jmp M1;break; /* duckduckgo */
case 2: jmp M2;break; /* google */
case 3: jmp M3;break; /* qwant */
case 4: jmp M4;break; /* startpage */
case 5: jmp M5;break; /* github */
case 6: jmp M6;break; /* google news */
case 7: jmp M7;break; /* mojeek */
case 10: jmp M10;break; /* f-droid */
case 11: jmp M11;break; /* bing */
case 13: jmp M13;break; /* yandex */
case 14: jmp M14;break; /* tailsx */
case 15: jmp M15;break; /* google scholar */
case 16: jmp M16;break; /* ecosia */
case 17: jmp M17;break; /* wiby */
case 18: jmp M18;break; /* marginalia */
case 19: jmp M19;break; /* yahoo */
case 20: jmp M20;break; /* youtube */
case 21: jmp M21;break; /* wikipedia */
case 22: jmp M22;break; /* pubmed */
case 23: jmp M23;break; /* brave */
case 24: jmp M24;break; /* mwmbl */
case 25: jmp M25;break; /* crossref */
case 26: jmp M26;break; /* aol */
case 27: jmp M27;break; /* sec */
default: break;
}
}
^HTTP\/1.[01][ ]200[ ]O[Kk]\r {
utf8=0;tag=0;urllen=0;desclen=0;v=0;w=0;x=0;y=0;z=0;}
/* INSERT SERPS HERE */
.|\n
%%
int main(int argc,char *argv[]){
if(argc>2)
{
printf("TWVGHkppk6Q6cODqP7Kqmode:\"%s\"\n",argv[1]);
fwrite("TWVGHkppk6Q6cODqP7Kqquery:\"",1,27,stdout);
if(*argv){++argv;++argv;};
while((a=*argv++)){x++;printf("%s",a);if((argc-x)>2)putc(32,stdout);};
fwrite("\"\n",1,2,stdout);
printf("TWVGHkppk6Q6cODqP7Kqdate:\"%u\"\n",time((unsigned int *)0));
goto x;
}
fwrite("PRAGMA foreign_keys=OFF;\n",1,25,stdout);
fwrite("BEGIN TRANSACTION;\n",1,19,stdout);
fwrite("CREATE TABLE IF NOT EXISTS t1(site INTEGER,query TEXT,gmt TEXT,url TEXT PRIMARY KEY,desc TEXT);\n",1,96,stdout);
yylex();
fwrite("COMMIT;\n",1,8,stdout);
x:
exit(0);
}
- 1vuio0pswjnm7 3y ago/*````````aol`````````*/ <M26>"lh-24\" href=\""[^\"]+ { CDp8iULPuCinsert(26); for(x=13;x<yyleng;x++)putc(yytext[x],yyout);putc(44,yyout); fwrite("','",1,3,yyout); } <M26>\"[a-f0-9]{13}\"[ ]aria-label=\"[^\"]+ { for(x=28;x<yyleng;x++)putc(yytext[x],yyout); fwrite("');\n",1,4,yyout); } /*`````````bing`````````*/ <M11>\<h2\>\<a[ ]href=\" { CDp8iULPuCinsert(11); jmp M11_URL; } <M11_URL>\"[ ]h=\" fwrite("','",1,3,yyout);jmp M11_DESC1; <M11_URL>[^\"]+ for(x=0;x<yyleng;x++){if(yytext[x]==39)putc(39,yyout);putc(yytext[x],yyout);}; <M11_DESC1>\> jmp M11_DESC2; <M11_DESC1>\< jmp M11_URL; <M11_DESC2>"</h2>" fwrite("');\n",1,4,yyout);jmp M11; <M11_DESC2>"</a>" <M11_DESC2>"<strong>"|"</strong>"|"</a>" /* example: less(1) shows hex <93> bvi \e28093 E2 80 93 \342 \200 \??? printf "%o" 0x93 223 \342 \200 \223 */ <M11_DESC2>\342\200 utf8++; <M11_DESC2>\223|\224 putc(45,yyout);utf8=0; <M11_DESC2>\231 putc(39,yyout);putc(39,yyout);utf8=0; <M11_DESC2>\234 putc(34,yyout);putc(34,yyout);utf8=0; <M11_DESC2>\235 putc(34,yyout);putc(34,yyout);utf8=0; <M11_DESC2>\246 putc(46,yyout);putc(46,yyout);putc(46,yyout);utf8=0; <M11_DESC2>. if(yytext[0]==39)putc(39,yyout);fwrite(yytext,1,yyleng,yyout); /*```````````brave``````````*/ <M23>\{title:\"[^\"]+\",url:\"[^\"]+ { CDp8iULPuCinsert(23); y=0;z=0;for(x=8;x<yyleng;x++) { if(yytext[x]==34)y=x; if(!y)if(z<255){desc[z]=yytext[x];z++;}; } for(x=y+1;x<yyleng;x++) { putc(yytext[x],yyout); if(yytext[x]==39)putc(39,yyout); } fwrite("','",1,3,yyout); for(x=0;x<z;x++) { if(desc[x]==39)putc(39,yyout); putc(desc[x],yyout); } if(z==255)fwrite(" [TRUNCATED]",1,12,yyout); fwrite("');\n",1,4,yyout); } To test: #!/bin/sh test $1||exec sed -n '/^ *;;/p' ${0##*/}; read x; case $1 in :) ;;11) # bing shift yy084 11 "$x first=${1-1}" x=$(echo "$x"|sed 's/ /+/g;s/++/+/g'); curl -si04A "" "https://www.bing.com/search?q=$x&cc=US&first=${1-1}" ;;23) # brave shift yy084 23 "$x offset=${1-0}"; x=$(echo "$x"|sed 's/ /+/g;s/++/+/g'); curl -si04A "" "https://search.brave.com/search?q=$x&source=web&offset=${1-0}" ;;26) # aol shift yy084 26 "$x b=${1-1}"; x=$(echo "$x"|sed 's/ /+/g;s/++/+/g') curl -si40A "" "https://search.aol.com/search?q=$x&b=${1-1}" esac echo research|1.sh 11 > 11.serp echo research|1.sh 23 > 23.serp echo research|1.sh 26 > 26.serp cat [0-9]*.serp|yy084|sqlite3 test.db
- 1vuio0pswjnm7 3y ago