4 ms·
Text Processing Practice Experiment: 20 SERP Types to SQLite yy084
/*
make sqlite from s.e.r.p.
ddg 001,
google 002,
qwant 003, SNI
startpage 004,
github 005,
google news 006, pub date as date
mojeek 007,
f-droid 010,
bing 011, SNI
yandex 013,
tailsx 014,
google scholar 015,
ecosia 016,
wiby.me 017,
marginalia 018,
yahoo 019, cache url as desc
youtube 020,
wikipedia 021,
pubmed 022, SNI
brave 023 SNI?
*/
int fileno (FILE *);
#define jmp (yy_start) = 1 + 2 *
int w,x,y,z;
int utf8,tag;
int datelen,querylen,urllen,desclen;
int bing,bing_url,bing_desc;
int ddg_url,ddg_desc,ddg_info;
int google_url,google_desc,google_desc2;
int mojeek_url,mojeek_desc,mojeek_info;
int yandex,yandex_url,yandex_desc;
int tailsx,tailsx_url,tailsx_desc,tailsx_info;
int google_news;
int startpage;
int yahoo,yahoo_url,yahoo_bingj;
int google_scholar_url;
int ecosia,ecosia_desc;
int wikipedia_url;
int pubmed_url;
int youtube1,youtube2,youtube_itag,youtube_url,youtube_desc;
int brave;
int qwant,qwant_json;
int f_droid;
int github;
int setenv (const char *, const char *, int);
char desc[255];
char url[255];
int puts(const char *);
int printf(const char *__restrict, ...);
int dprintf(int, const char *__restrict, ...);
int CDp8iULPuCinsert(int xyz)
{
fwrite("INSERT or IGNORE INTO t1 VALUES(",1,32,stdout);
printf("%03d,'",xyz);
fwrite(getenv("query"),1,querylen,stdout);
fwrite("','",1,3,yyout);
fwrite(getenv("date"),1,datelen,yyout);
fwrite("','",1,3,yyout);
return 0;
}
%s YANDEX_DESC BING_URL BING_DESC1 BING_DESC2
%s QWANT_DESC GITHUB_DESC GOOGLE_DESC
%option nounput noinput noyywrap
%%
"class=\"startpage-html startpage-html--" startpage++;
"<title>Yandex</title>" yandex++;
"policyref=\"https://policies.yahoo.com/w3c/p3p.xml\"" yahoo++;
^[Ss]erver:[ ]Qwant\r qwant++;
TWVGHkppk6Q6cODqP7Kqdate:\"[^\"]+ {
y=0;for(x=26;x<yyleng;x++){if(yytext[x])yytext[y]=yytext[x];
if(yytext[yyleng])yytext[yyleng]=0;y++;};
setenv("date",yytext,1);
datelen=y;
}
TWVGHkppk6Q6cODqP7Kqquery:\"[^\"]+ {
y=0;for(x=27;x<yyleng;x++){if(yytext[x])yytext[y]=yytext[x];
if(yytext[yyleng])yytext[yyleng]=0;y++;};
setenv("query",yytext,1);
querylen=y;
}
^HTTP\/1.[01][ ]200[ ]O[Kk]\r {
bing_desc=0;
ddg_info=0;
tag=0;
tailsx_info=0;
yahoo=0;
yahoo_bingj=0;
bing=0;
bing_url=0;
brave=0;
ddg_url=0;
desclen=0;
ecosia=0;
ecosia_desc=0;
google_scholar_url=0;
mojeek_desc=0;
mojeek_info=0;
mojeek_url=0;
pubmed_url=0;
qwant=0;
startpage=0;
tailsx=0;
tailsx_desc=0;
tailsx_url=0;
w=0;
wikipedia_url=0;
x=0;
yahoo=0;
yahoo_url=0;
youtube_itag=0;
z=0;
}
/* ````````````insert search engines below```````````````*/
.|\n
%%
int main(){
fwrite("PRAGMA foreign_keys=OFF;\n",1,25,stdout);
fwrite("BEGIN TRANSACTION;\n",1,19,stdout);
fwrite("CREATE TABLE IF NOT EXISTS t1(site INTEGER,query TEXT,gmt TEXT,url TEXT PRIMARY KEY,desc TEXT);\n",1,96,stdout);
yylex();
fwrite("COMMIT;\n",1,8,stdout);
exit(0);
}
- 1vuio0pswjnm7 3y ago/*```````````duckduckgo```````````*/ "Zero-click info: " ddg_info++; "<!-- Web results are present -->" ddg_info=0; [ ]{8}"<a rel=\"nofollow\" href=\""[^\n\r"]+ if(!ddg_info){ ddg_url++; CDp8iULPuCinsert(1); for(x=32;x<yyleng;x++){putc(yytext[x],yyout);if(yytext[x]==39)putc(39,stdout);}; fwrite("','",1,3,yyout); } "class='result-link'><span class=\"result__type\">"[^<]+"</span>"[^<]+ if(ddg_url){ for(x=57;x<yyleng;x++)putc(yytext[x],yyout); fwrite("');\n",1,4,yyout); } "class='result-link'>"[^<]+ if(ddg_url){ for(x=20;x<yyleng;x++)putc(yytext[x],yyout); fwrite("');\n",1,4,yyout); } To test: $!/bin/sh read x; echo TWVGHkppk6Q6cODqP7Kqquery:\""$x"\" echo TWVGHkppk6Q6cODqP7Kqdate:\"$(epoch)\" curl -si0A "" -d "q=$x&p=&s=${1-0}&dc=$(($1+1))&api=%2Fd.js&o=json&kl=wt-wt" https://lite.duckduckgo.com/lite/ echo this is a test|001.sh > 1.htm yy084 < 1.htm|sqlite3 test.db # epoch prints UNIX timestamp epoch.c: int puts(const char *); unsigned int time(unsigned int *tloc); char *itoa(int x){ static char buf[21]; char *s=buf+20; if(x>=0){do{*--s='0'+(x%10);x/=10;}while(x!=0);return s;} else{do{*--s='0'-(x%10);x/=10;}while(x!=0);*--s='-';}; return s;} int main(){ puts(itoa(time(0))); }
- 1vuio0pswjnm7 3y ago/*```````````google``````````*/ "<div class=\"egMi0 kCrYT\"><a href=\"/url?q="[^"\n\r ]+"&sa=U" { google_desc=0;google_desc2=0; google_url++; CDp8iULPuCinsert(11); for(x=41;x<yyleng-9;x++){ if(yytext[x]==39)putc(39,stdout); putc(yytext[x],yyout); } fwrite("','",1,3,yyout); } "<div class=\"egMi0 kCrYT\"><a href=\""[hf][^"\n\r ]+"&sa="[UX] { google_url++; CDp8iULPuCinsert(11); for(x=34;x<yyleng-9;x++) { if(yytext[x]==39)putc(39,stdout); putc(yytext[x],yyout); } fwrite("','",1,3,yyout); } "<div class=\"BNeawe vvjwJb AP7Wnd\">" google_desc++;jmp GOOGLE_DESC; <GOOGLE_DESC>\< fwrite("');\n",1,4,yyout);jmp 0; <GOOGLE_DESC>\xE9 putc(101,yyout); <GOOGLE_DESC>[^\xE9<\n\r] if(yytext[0]==39)putc(39,yyout);putc(yytext[0],yyout); "<span dir=\"rtl\">"[^\xE9<\n\r]+ google_desc2++;jmp GOOGLE_DESC; To test: #!/bin/sh read x; echo TWVGHkppk6Q6cODqP7Kqquery:\""$x"\" echo TWVGHkppk6Q6cODqP7Kqdate:\"$(epoch)\" curl -si0A "" "https://www.google.com/search?q=$x&num=100&filter=0&start=${1-1}&scoring=d" echo this is a test|002.sh > 1.htm yy084 < 1.htm|sqlite3 test.db