# 请确保本地Python环境是3.6+
git clone https://github.com/howie6879/ruia.git
# 安装pipenv
pip install pipenv 
# 安装依赖包
pipenv install --dev

Response

import asyncio
import aiohttp
import async_timeout

from typing import Coroutine

class Request:
# Default config
REQUEST_CONFIG = {
‘RETRIES’: 3,
‘DELAY’: 0,
‘TIMEOUT’: 10,
‘RETRY_FUNC’: Coroutine,
‘VALID’: Coroutine
}

METHOD = [<span class="hljs-string">'GET'</span>, <span class="hljs-string">'POST'</span>]

<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self, url, method=<span class="hljs-string">'GET'</span>, request_config=None, request_session=None)</span>:</span>
    self.url = url
    self.method = method.upper()
    self.request_config = request_config <span class="hljs-keyword">or</span> self.REQUEST_CONFIG
    self.request_session = request_session

@property
def current_request_session(self):
if self.request_session is None:
self.request_session = aiohttp.ClientSession()
self.close_request_session = True
return self.request_session

<span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">fetch</span><span class="hljs-params">(self)</span>:</span>
    <span class="hljs-string">"""Fetch all the information by using aiohttp"""</span>
    <span class="hljs-keyword">if</span> self.request_config.get(<span class="hljs-string">'DELAY'</span>, <span class="hljs-number">0</span>) &gt; <span class="hljs-number">0</span>:
        <span class="hljs-keyword">await</span> asyncio.sleep(self.request_config[<span class="hljs-string">'DELAY'</span>])

    timeout = self.request_config.get(<span class="hljs-string">'TIMEOUT'</span>, <span class="hljs-number">10</span>)
    <span class="hljs-keyword">async</span> <span class="hljs-keyword">with</span> async_timeout.timeout(timeout):
        resp = <span class="hljs-keyword">await</span> self._make_request()
    <span class="hljs-keyword">try</span>:
        resp_data = <span class="hljs-keyword">await</span> resp.text()
    <span class="hljs-keyword">except</span> UnicodeDecodeError:
        resp_data = <span class="hljs-keyword">await</span> resp.read()
    resp_dict = dict(
        rl=self.url,
        method=self.method,
        encoding=resp.get_encoding(),
        html=resp_data,
        cookies=resp.cookies,
        headers=resp.headers,
        status=resp.status,
        history=resp.history
    )
    <span class="hljs-keyword">await</span> self.request_session.close()
    <span class="hljs-keyword">return</span> type(<span class="hljs-string">'Response'</span>, (), resp_dict)


<span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">_make_request</span><span class="hljs-params">(self)</span>:</span>
    <span class="hljs-keyword">if</span> self.method == <span class="hljs-string">'GET'</span>:
        request_func = self.current_request_session.get(self.url)
    <span class="hljs-keyword">else</span>:
        request_func = self.current_request_session.post(self.url)
    resp = <span class="hljs-keyword">await</span> request_func
    <span class="hljs-keyword">return</span> resp

if name == ‘main’:
loop = asyncio.get_event_loop()
resp = loop.run_until_complete(Request(‘https://docs.python-ruia.org/’).fetch())
print(resp.status)

实际运行一下，会输出请求状态 200 ，就这样简单封装一下，我们已经有了自己的请求类 Request ，接下来只需要再完善一下重试机制以及将返回的属性封装一下就基本完成了：

# 重试函数
async def _retry(self):
    if self.retry_times > 0:
        retry_times = self.request_config.get('RETRIES', 3) - self.retry_times + 1
        self.retry_times -= 1
        retry_func = self.request_config.get('RETRY_FUNC')
        if retry_func and iscoroutinefunction(retry_func):
            request_ins = await retry_func(weakref.proxy(self))
            if isinstance(request_ins, Request):
                return await request_ins.fetch()
        return await self.fetch()

最终代码见 ruia/request.py 即可，接下来就可以利用 Request 来实际请求一个目标网页，如下：

这段代码请求了目标网页 https://docs.python-ruia.org/ 并返回了 Response 对象，其中 Response 提供属性介绍如下：

Field & Item

实现了对目标网页的请求，接下来就是对目标网页进行字段提取，我觉得 ORM 的思想很适合用在这里，我们只需要定义一个 Item 类，类里面每个属性都可以用 Field 类来定义，然后只需要传入 url 或者 html ，执行过后 Item 类里面 定义的属性会自动被提取出来变成目标字段值

可能说起来比较拗口，下面直接演示一下可能你就明白这样写的好，假设你的需求是获取 HackerNews 网页的 title 和 url ，可以这样实现：

import asyncio

from ruia import AttrField, TextField, Item

class HackerNewsItem(Item):
target_item = TextField(css_select=‘tr.athing’)
title = TextField(css_select=‘a.storylink’)
url = AttrField(css_select=‘a.storylink’, attr=‘href’)

async def main():
async for item in HackerNewsItem.get_items(url=“https://news.ycombinator.com/”):
print(item.title, item.url)

if name == ‘main’:
items = asyncio.run(main())

从输出结果可以看到， title 和 url 属性已经被赋与实际的目标值，这样写起来是不是很简洁清晰也很明了呢？

来看看怎么实现， Field 类的目的是提供多种方式让开发者提取网页字段，比如：

• XPath
• CSS Selector
• RE

所以我们只需要根据需求，定义父类然后再利用不同的提取方式实现子类即可，代码如下：

class BaseField(object):
    """
    BaseField class
    """
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self, default: str = <span class="hljs-string">''</span>, many: bool = False)</span>:</span>
    <span class="hljs-string">"""
    Init BaseField class
    url: http://lxml.de/index.html
    :param default: default value
    :param many: if there are many fields in one page
    """</span>
    self.default = default
    self.many = many

<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">extract</span><span class="hljs-params">(self, *args, **kwargs)</span>:</span>
    <span class="hljs-keyword">raise</span> NotImplementedError(<span class="hljs-string">'extract is not implemented.'</span>)

class _LxmlElementField(BaseField):
pass

class AttrField(_LxmlElementField):
“”"
This field is used to get attribute.
“”"
pass

class HtmlField(_LxmlElementField):
“”"
This field is used to get raw html data.
“”"
pass

class TextField(_LxmlElementField):
“”"
This field is used to get text.
“”"
pass

class RegexField(BaseField):
“”"
This field is used to get raw html code by regular expression.
RegexField uses standard library re inner, that is to say it has a better performance than _LxmlElementField.
“”"
pass

核心类就是上面的代码，具体实现请看 ruia/field.py

接下来继续说 Item 部分，这部分实际上是对 ORM 那块的实现，用到的知识点是 元类 ，因为我们需要控制类的创建行为：

class ItemMeta(type):
    """
    Metaclass for an item
    """
<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__new__</span><span class="hljs-params">(cls, name, bases, attrs)</span>:</span>
    __fields = dict({(field_name, attrs.pop(field_name))
                     <span class="hljs-keyword">for</span> field_name, object <span class="hljs-keyword">in</span> list(attrs.items())
                     <span class="hljs-keyword">if</span> isinstance(object, BaseField)})
    attrs[<span class="hljs-string">'__fields'</span>] = __fields
    new_class = type.__new__(cls, name, bases, attrs)
    <span class="hljs-keyword">return</span> new_class

class Item(metaclass=ItemMeta):
“”"
Item class for each item
“”"

<span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">__init__</span><span class="hljs-params">(self)</span>:</span>
    self.ignore_item = <span class="hljs-keyword">False</span>
    self.results = {}</pre> 

这一层弄明白接下来就很简单了，还记得上一篇文章《谈谈对Python爬虫的理解》里面说的四个类型的目标网页么：

• 单页面单目标
• 单页面多目标
• 多页面单目标
• 多页面多目标

本质来说就是要获取网页的单目标以及多目标（多页面可以放在Spider那块实现）， Item 类只需要定义两个方法就能实现：

target_item

具体实现见： ruia/item.py

Spider

在 Ruia 框架中，为什么要有 Spider ，有以下原因：

Spider

接下来说说代码实现， Ruia 框架的 API 写法我有参考 Scrapy ，各个函数之间的联结也是使用回调，但是你也可以直接使用 await ，可以直接看代码示例：

from ruia import AttrField, TextField, Item, Spider

class HackerNewsItem(Item):
target_item = TextField(css_select=‘tr.athing’)
title = TextField(css_select=‘a.storylink’)
url = AttrField(css_select=‘a.storylink’, attr=‘href’)

class HackerNewsSpider(Spider):
start_urls = [f’https://news.ycombinator.com/news?p={index}’ for index in range(1, 3)]

<span class="hljs-keyword">async</span> <span class="hljs-function"><span class="hljs-keyword">def</span> <span class="hljs-title">parse</span><span class="hljs-params">(self, response)</span>:</span>
    <span class="hljs-keyword">async</span> <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> HackerNewsItem.get_items(html=response.html):
        <span class="hljs-keyword">yield</span> item

if name == ‘main’:
HackerNewsSpider.start()

使用起来还是挺简洁的，输出如下：

[2019:03:14 10:29:04] INFO  Spider  Spider started!
[2019:03:14 10:29:04] INFO  Spider  Worker started: 4380434912
[2019:03:14 10:29:04] INFO  Spider  Worker started: 4380435048
[2019:03:14 10:29:04] INFO  Request <GET: https://news.ycombinator.com/news?p=1>
[2019:03:14 10:29:04] INFO  Request <GET: https://news.ycombinator.com/news?p=2>
[2019:03:14 10:29:08] INFO  Spider  Stopping spider: Ruia
[2019:03:14 10:29:08] INFO  Spider  Total requests: 2
[2019:03:14 10:29:08] INFO  Spider  Time usage: 0:00:03.426335
[2019:03:14 10:29:08] INFO  Spider  Spider finished!

Spider 的核心部分在于对请求 URL 的请求控制，目前采用的是生产消费者模式来处理，具体函数如下：

详细代码，见 ruia/spider.py

更多

至此，爬虫框架的核心部分已经实现完毕，基础功能同样一个不落地实现了，接下来要做的就是：

• 实现更多优雅地功能
• 实现更多的插件，让生态丰富起来
• 修BUG

项目地址点击阅读原文或者在 github 搜索 ruia ，如果你有兴趣，请参与进来吧！

如果觉得写得不错，点个好看来个 star 呗~