5 ms·
Does anyone know of an easy way to download the nucleotide sequences for the entire set of pneumonia / coronavirus viruses? I've looked at the FTP mirror but ca
by Smerity 7y ago
Does anyone know of an easy way to download the nucleotide sequences for the entire set of pneumonia / coronavirus viruses? I've looked at the FTP mirror but can't connect it with the nucleotide locus I find on the site itself.
I work in language modeling and want to see about using those unsupervised / self-supervised methods for genome annotation / phylogenetic tree construction.
Even if it's more of a curiousity than a useful tool I have experience with small datasets, most recently focused on character level language modeling on ~90MB of training data, so if I can get (90MB / (29 kilobases * 2 bits per base) =) approximately 12,000 related samples I should be able to at least make a dataset out of it.
- jerven 7y agohttps://viralzone.expasy.org/8676 https://viralzone.expasy.org/8676 Is a good start.
- BioGeek 7y agoUsing Biopython. Note that the search query that I am using currently returns 70605 results, so you might want to tweak it fit your needs. from Bio import Entrez import time from urllib.error import HTTPError DB = 'nucleotide' QUERY = '("pneumoviridae"[Organism] OR "Coronaviridae"[Organism])' Entrez.email = 'your.email@provider.com' handle = Entrez.esearch(db=DB, term=QUERY, rettype='fasta') record = Entrez.read(handle) handle = Entrez.esearch(db=DB, term=QUERY, retmax=count, rettype='fasta') record = Entrez.read(handle) id_list = record['IdList'] count = len(id_list) post_xml = Entrez.epost(DB, id=",".join(id_list)) search_results = Entrez.read(post_xml) webenv = search_results['WebEnv'] query_key = search_results['QueryKey'] batch_size = 200 with open('viruses.fasta', 'w') as out_handle: for start in range(0, count, batch_size): end = min(count, start+batch_size) print(f"Going to download record {start+1} to {end}") attempt = 0 success = False while attempt < 3 and not success: attempt += 1 try: fetch_handle = Entrez.efetch(db=DB, rettype='fasta', retstart=start, retmax=batch_size, webenv=webenv, query_key=query_key) success = True except HTTPError as err: if 500 <= err.code <= 599: print(f"Received error from server {err}") print("Attempt {attempt} of 3") time.sleep(15) else: raise data = fetch_handle.read() fetch_handle.close() out_handle.write(data)