极客观点
项目管理
HarmonyOS
#!/usr/bin/env python
# -*- coding: utf-8 -*-
infile2 = open('genemark.gff3', 'r')
infile1 = set(line1.strip() for line1 in open('1.txt', 'r'))
for line in infile2:
line = line.strip().split()
if line[2] == 'gene':
chr, start, end = line[0], int(line[3]), int(line[4])
for line1 in infile1:
line1 = line1.split()
chr1, start1, end1 = line1[1], int(line1[2]), int(line1[3])
if chr1 == chr:
if start1 < start < end1:
print line1[0], line[-1]
if start1 < end < end1:
print line1[0], line[-1]
if start1 > start and end > end1:
print line1[0], line[-1]genemark.gff3 格式类似下边:
chr1D GeneMark.hmm gene 2705930 2711118 . + . ID=1903228_g;Name=1903228_g
chr1D GeneMark.hmm mRNA 2705930 2711118 . + . ID=1903228_t;Name=1903228_t;Parent=1903228_g1.txt:
UN011157 chr1D 2705329 2706342 98.4 95.0 972 30 21 0
UN003843 chr1D 2705681 2721144 61.4 97.4 633 12 5 0附上原始文件的百度云链接，希望感兴趣的参考
点击下载 密码 enu8
综合楼下各位朋友的答案，现推荐两种
第一种 根据 @ferstar @用筹兮用严 的答案，即并行版
#!/usr/bin/env python
# encoding: utf-8
from collections import defaultdict
from multiprocessing import Pool, cpu_count
from functools import partial
def find_sth(f2, f1=None):
start, end = int(f2[3]), int(f2[4])
for uno1, start1, end1 in f1[f2[0]]:
if (start1 <= start and start <= end1) or (start1 <= end and end <= end1) or (start1 >= start and end >= end1):
with open("out.txt", "a") as fh:
fh.write(uno1 + "\t" + f2[-1] + "\n")
#print(uno1, f2[-1])
def main():
with open('1.txt', 'r') as f1:
infile1 = defaultdict(set)
for uno1, chr1, start1, end1, *others in map(str.split, f1):
infile1[chr1].add((uno1, int(start1), int(end1)))
with open('genemark.gff3', 'r') as f2:
infile2 = [x for x in map(str.split, f2) if x[2] == 'gene']
pool = Pool(cpu_count())
pool.map(partial(find_sth, f1=infile1), infile2)
pool.close()
pool.join()
if __name__ == "__main__":
main()第二种 @citaret 他的版本（单核版），对单核来说，不逊于上述代码。但是两者结果稍有不同，并行版结果更全（这里少了73条，出在判断条件的边界问题，由于对intervaltree熟悉，怎么改还不知道），现在边界问题已修改，两种代码结果完全一样，perfect！
如下
from collections import defaultdict
from intervaltree import Interval, IntervalTree
with open('1.txt') as f:
d1 = defaultdict(list)
xs = map(lambda x: x.strip().split(), f)
for x in xs:
y = (x[0], int(x[2]), int(x[3]))
d1[x[1]].append(y)
for k, v in d1.items():
d1[k] = IntervalTree(Interval(s, e, u) for u, s, e in v)
with open('genemark.gff3') as f:
for line in f:
line = line.strip().split()
if line[2] == 'gene':
chr, start, end = line[0], int(line[3]), int(line[4])
for start1, end1, un1 in d1[chr][start-1:end+1]:
print(un1, line[-1])
python
update 2016.6.5
关于如何使用多进程加速，并且使用 intervaltree 的方法，我试了很多代码，抛弃掉使用进程池按行分发task的方法，原因是处理一行的计算量很小，而频繁给进程函数频繁传参区间树字典的损耗却很大，大家可以试一试处理楼主的4G文件，比但进程要慢多了。
改进的思路是，文件按块读取，这样IO会很快，而进程函数处理的单位以大块计，才有了分摊计算量的意义。我还是用了多进程+Queue，进程数和和每次读取文件块的大小也许可以进一步调优。在我的xps13笔记本上，跑完4G文件用的时间是：
以下是代码，希望能抛砖引玉
------------- history