3 ms·
Convert YouTube JSON to TSV
Example usage:
(1.sh search utility and yy059 are described in earlier submissions)
1.sh search_query > 1.yts
yy059 < 1.yts|1.sh >> 1.yts
yy059 < 1.yts|1.sh >> 1.yts
1.sh /c/channelname > 1.ytc
yy059 < 1.ytc|1.sh >> 1.ytc
cat 1.yt[sc]|yy062 > 1.tsv
less -SN 1.tsv
1.sh $(sed -n 1000p 1.tsv|cut -f3) >> 1.tsv
This is made for me and thus only includes the columns I want. I am often selecting videos based on running length.
- 1vuio0pswjnm7 5y agoAdd: pattern_user "/user/"
- 1vuio0pswjnm7 5y agosed 's/.\{4\}//' << eof|flex -8iCrf /* youtube json to tsv */ int fileno (FILE *); int setenv (const char *, const char *, int); #define echo do{if(fwrite(yytext,(size_t)yyleng,1,yyout)){}}while(0) #define jmp (yy_start) = 1 + 2 * #define pc(x) putchar(x) int request_type; int count_text; int count_simpletext; int count_baseurl; int count_videoid; void save_request(){setenv("request",yytext,1);} void save_channel(){setenv("channel",yytext,1);} void print_saved_channel(){fprintf(stdout,"\t%s",getenv("channel"));} void print_saved_request(){fprintf(stdout,"\t%s",getenv("request"));} void print_saved_request_type(){pc(9);if(request_type==1)pc(115);if(request_type==2)pc(99);} void try_to_omit_premieres(){if(yytext[0]=='*')jmp 0;} pattern_initial_baseurl \"canonicalBaseUrl\":\" pattern_continuation_baseurl \"canonicalBaseUrl\":\40\" pattern_originalurl \"originalUrl\":\"https:\/\/www.youtube.com pattern_initial_video \"videoRenderer\": pattern_initial_grid \"gridVideoRenderer\": pattern_initial_movie \"movieRenderer\": pattern_continuation_compact\"compactVideoRenderer\":\40\173 pattern_initial_videoid \"videoId\":\" pattern_continuation_videoid \"videoId\":\40\" pattern_initial_text \"text\":\" pattern_continuation_text \"text\":\40\" pattern_initial_simpletext \"simpleText\":\" pattern_channel_name "/c/" pattern_channel_id "/channel/" pattern_search_query "/results?search_query".{6} %option nounput noinput noyywrap %s state_print_column1 %s state_print_column2 %s state_print_column4 %s state_save_request_type1 %s state_save_request_type2 %s state_save_request %s state_save_channel %% /* originalUrl */ {pattern_originalurl} jmp state_save_request; /* will use as cols 5,6 */ <state_save_request>\" jmp 0; <state_save_request>{pattern_channel_name}|{pattern_channel_id} jmp state_save_request_type2; <state_save_request>{pattern_search_query} jmp state_save_request_type1; <state_save_request_type2>[^/]* request_type=2;save_request();jmp 0; <state_save_request_type1>[^\"]* request_type=1;save_request();jmp 0; /* videoRenderer gridVideoRenderer movieRenderer compactVideRenderer start of record reset counters */ {pattern_initial_video}|{pattern_initial_grid}|{pattern_initial_movie}|{pattern_continuation_compact} { count_videoid=0; count_text=0; count_simpletext=0; count_baseurl=0;} /* videoId */ {pattern_initial_videoid}|{pattern_continuation_videoid} { count_videoid++;jmp state_print_column1;} /* print 1 */ <state_print_column1>\" jmp 0; <state_print_column1>[^\"]* if(count_videoid==1){echo;pc(9);} /* canonicalBaseUrl */ {pattern_initial_baseurl}|{pattern_continuation_baseurl} { count_baseurl++;jmp state_save_channel;} /* print 3 if search json */ <state_save_channel>\" jmp 0; <state_save_channel>[^\"]* { save_channel(); if(request_type==1) if(count_baseurl==1) print_saved_channel();} /* text */ {pattern_initial_text} { count_text++; if(count_text==1) jmp state_print_column2;} /* print 2 if search json print 2 plus 3 if channel json */ <state_print_column2>\" jmp 0; <state_print_column2>[^\"]* { echo; if(request_type==2) print_saved_channel();} /* also print 4 if continuation json */ {pattern_continuation_text} { count_text++; if(count_text==1) jmp state_print_column2; if(request_type==2) if(count_text==4) jmp state_print_column4; if(request_type==1) if(count_text==5) jmp state_print_column4;} /* simpleText print 4 if initial json */ {pattern_initial_simpletext} { count_simpletext++; if(request_type==1) if(count_simpletext==2) jmp state_print_column4; if(request_type==2) if(count_simpletext==4) jmp state_print_column4;} /* print 4 plus 5,6 */ <state_print_column4>\" jmp 0; <state_print_column4>[^\"]* { try_to_omit_premieres(); if(yytext[0]>57)fprintf(stdout,"\t%s","No_views");else {pc(9);echo;}; print_saved_request(); print_saved_request_type();pc(10);jmp 0;} \n|. %% int main(){ yylex();exit(0);} /* ---------------------------------------------------------- initial json and continuation json differ continuation json has space after the colon columns: 1 video ID 2 description (may contain unicode) 3 channel name or channel ID 4 running time (NB. missing for videos with No views) 5 search query_string or channel name/ID 6 search or channel ("s" or "c") do meta tags from video page html allow unicode search_query retrieval is "request type 1" channel retrieval is "request type 2" ------------------------------------------------------------ */ eof cc -std=c89 -pedantic -Wall -static lex.yy.cc -o yy062
- Maclennan 5y ago[dead]
- 1vuio0pswjnm7 5y agoTODO: Support playlists