scrapy写多爬虫 并行报错 ,求人指导

from scrapy.utils.project import get_project_settings
from Espider.spiders.chinafof import chinafof
from Espider.spiders.ifenxi import ifenxi
from Espider.spiders.vcbeat import vcbeat
from Espider.spiders.zdreview import zdreview
from scrapy.crawler import CrawlerProcess
from multiprocessing import Process

def run_crawl():

process=CrawlerProcess(get_project_settings())
process.crawl(chinafof)
process.crawl(ifenxi)
process.crawl(vcbeat)
process.crawl(zdreview)
process.start()


if __name__=='__main__':

[Process(target=run_crawl).start() for _ in range(4)]


File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 86, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
--- <exception caught here> ---
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/posixbase.py", line 619, in _doReadOrWrite
    why = selectable.doWrite()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1186, in doWrite
    "doWrite called on a %s" % reflect.qual(self.__class__))
builtins.RuntimeError: doWrite called on a twisted.internet.tcp.Port

2018-12-02 17:25:54 [twisted] CRITICAL: Unhandled Error
Traceback (most recent call last):
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 103, in callWithLogger
    return callWithContext({"system": lp}, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/log.py", line 86, in callWithContext
    return context.call({ILogContext: newCtx}, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 122, in callWithContext
    return self.currentContext().callWithContext(ctx, func, *args, **kw)
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/python/context.py", line 85, in callWithContext
    return func(*args,**kw)
--- <exception caught here> ---
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/posixbase.py", line 619, in _doReadOrWrite
    why = selectable.doWrite()
  File "/home/shenjianlin/.local/lib/python3.4/site-packages/twisted/internet/base.py", line 1186, in doWrite
    "doWrite called on a %s" % reflect.qual(self.__class__))
builtins.RuntimeError: doWrite called on a twisted.internet.tcp.Port

阅读 1.6k
撰写回答
你尚未登录,登录后可以
  • 和开发者交流问题的细节
  • 关注并接收问题和回答的更新提醒
  • 参与内容的编辑和改进,让解决方法与时俱进
推荐问题