首先需要创建接个文件

Main.py (入口文件)

html_parser(解析下载的html 文件)

downloader (下载文件)

outputer(存储文件)

url_manager(爬虫链接管理文件)

mysql_help(mysql 操作文件)

理解爬虫步骤：

首先向 爬取一个网页，在网页里解析《a》标签将已经解析到的超链接放到url_manager(爬虫链接管理文件)里待后面取出，将爬下来的页面用BS4对其取出有用的资源

存到字典中，待爬虫全部结束后再循环写入到数据库中。

源代码如下

Main.py (入口文件)

# -*- coding:utf-8 -*-
import url_manager
import html_parser
import htmldownloader
import htmloutputer
class Main(object):
    def __init__(self):
        self.urls = url_manager.UrlManager()
        self.parser = html_parser.HtmlParser()
        self.downloader = htmldownloader.HtmlDownloader()
        self.outputer = htmloutputer.OutPuter()

    def craw(self,root_url):
        self.urls.add_new_url(root_url)
        count = 1
        while self.urls.has_new_url():
            # try:
                new_url = self.urls.get_new_url()
                print u"执行：%d  %s"%(count,new_url)
                html_cont = self.downloader.download(new_url)#下载页面内容
                new_urls,new_data = self.parser.parser(new_url,html_cont)#解析器
                self.urls.add_new_urls(new_urls)
                self.outputer.collect_data(new_data)

                if count == 1000:#多少停止
                    break
                count = count+1
            # except:
            #     print u"页面访问失败"

        # self.outputer.output_html()
        self.outputer.set_mysql()

if __name__ == "__main__":
    root_url = "https://www.qiushibaike.com/article/118853097"
    obj_main = Main()
    obj_main.craw(root_url)

html_parser(解析下载的html 文件)

# -*- coding:utf-8 -*-
from bs4 import BeautifulSoup
import re
import urlparse
class HtmlParser(object):
    # 获取该页的所有url
    def _get_new_urls(self, page_url, soup):
        new_urls = set()
        links = soup.find_all('a', href=re.compile(r"/article/\.*"))
        # print links
        # quit()
        for link in links:
            new_url = link['href']
            new_urls.add(urlparse.urljoin(page_url, new_url))
        return new_urls

    # 获取数据
    def _get_new_data(self, page_url, soup):
        res_data = {}
        # 抓取的路由
        res_data['url'] = page_url

        #抓取用户头像
        head_img_url = soup.find("div",class_="author").find('img')
        res_data['head_img_url'] = urlparse.urljoin(page_url, head_img_url['src'])

        #获取用户姓名
        title_node = soup.find('div', class_='author').find('h2')
        res_data['name'] = title_node.get_text()

        #获取段子
        summary_node = soup.find('div', id="single-next-link")
        res_data['content'] = summary_node.get_text()
        try:
            content_img = summary_node.find('div',class_='thumb').find('img')
            res_data['content_img'] = urlparse.urljoin(page_url, content_img['src'])
        except:
            res_data['content_img'] =''

        return res_data
    #解析出新的url,和数据
    def parser(self,page_url,html_cont):
        if page_url is None or html_cont is None:
            return
        soup = BeautifulSoup(html_cont,'html.parser',from_encoding='utf-8')
        new_urls = self._get_new_urls(page_url,soup)
        new_data = self._get_new_data(page_url,soup)
        return new_urls, new_data

downloader (下载文件)

# -*- coding:utf -8-*-
import urllib2,cookielib
class HtmlDownloader(object):

    def download(self,url):
        if url is None:
            return
        # cj = cookielib.CookieJar()
        # res = urllib2.urlopen(url)
        # urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'
        }
        req = urllib2.Request(
            url=url,
            headers=headers
        )
        res = urllib2.urlopen(req)
        # print res.getcode()
        # quit()
        if res.getcode() != 200:
            return
        return res.read()

outputer(存储文件)

# -*- coding:utf-8 -*-
import MyswlHelp
import codecs
class OutPuter(object):
    def __init__(self):
        self.datas = []

    def collect_data(self,data):
        if data is None:
            return
        self.datas.append(data)

    #存入文件
    def output_html(self):
        file = codecs.open('text.html','a',encoding='utf-8')
        file.seek(1)
        for v in self.datas:
            try:
                file.write("{url:%s,title:%s,summary:%s}" % (v['url'], v['name'].encode('gb18030'), v['content'].encode('gb18030')))

            except:
                file.write("{url:%s,title:%s,summary:%s}" % (v['url'], v['name'].encode('utf-8'), v['content'].encode('utf-8')))            #     continue

        file.flush()
        file.close()

    #存到数据库
    def set_mysql(self):
        my_help = MyswlHelp.MysqlHelp()

        for v in self.datas:
            try:
                if v['content_img'] == '':
                    sql = "insert into qqbk (name,content,head_img_url) VALUES ('"+v['name'].encode('utf-8')+"','"+v['content'].encode('utf-8')+"','"+v['head_img_url'].encode('utf-8')+"')"
                else:
                    sql = "insert into qqbk (name,content,content_img,head_img_url) VALUES ('"+v['name'].encode('utf-8')+"','"+v['content'].encode('utf-8')+"','"+v['content_img'].encode('utf-8')+"','"+v['head_img_url'].encode('utf-8')+"')"
                # print sql
                my_help.execute(sql)
            except:
                continue

url_manager(爬虫链接管理文件)

# -*- coding:utf-8 -*-
class UrlManager(object):
    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()

    #填加单个url
    def add_new_url(self,url):
        if url is None:
            return
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)

    #批量添加
    def add_new_urls(self,urls):
        if urls is None or len(urls) == 0:
            return
        for url in urls:
            self.add_new_url(url)

    #获取一个url
    def get_new_url(self):
        url = self.new_urls.pop()
        self.old_urls.add(url)
        return url

    #判断
    def has_new_url(self):
        return len(self.new_urls) != 0

mysql_help(mysql 操作文件)

#!/usr/bin/python
# -*- coding: UTF-8 -*-
import MySQLdb
class MysqlHelp:
    "mysql 操作类"
    localhost="127.0.0.1"#地址
    db_name="python"#数据库名称
    access="root"#账号
    password=""#密码
    db=""
    cursor=""

    #初始化
    def __init__(self):
        # 打开数据库连接
        self.db = MySQLdb.connect(self.localhost, self.access, self.password, self.db_name,charset="utf8")
        # 使用cursor()方法获取操作游标
        self.cursor = self.db.cursor()

    #增删改
    def execute(self,sql):
        try:
            # 执行SQL语句
            self.cursor.execute(sql)
            # 提交到数据库执行
            res = self.db.commit()
            return res
        except:
            # 发生错误时回滚
            self.db.rollback()
        # 关闭数据库连接
        self.db.close()


    #查询
    def query(self,sql):
        try:
            self.cursor.execute(sql)
            res = self.cursor.fetchall()
        except:
            res= "Error: unable to fecth data"
        self.db.close()
        return res

执行：93  https://www.qiushibaike.com/article/115767514
执行：94  https://www.qiushibaike.com/article/118964920
执行：95  https://www.qiushibaike.com/article/51456225
执行：96  https://www.qiushibaike.com/article/116486378
执行：97  https://www.qiushibaike.com/article/288171
执行：98  https://www.qiushibaike.com/article/119549247
执行：99  https://www.qiushibaike.com/article/213184
执行：100  https://www.qiushibaike.com/article/119441053

看查看数据库

多出了100条记录