简介

前几天，由于意见征集稿的出现

所以决定先把一些文章给爬下来，存着以待学习，转换成markdown的格式以便观看

转换md用的第三方模块：html2text

---

Code

思路：

• 用BS获取对应位置的内容
• 然后用html2text转换成md格式

Ps：先知的标题奇奇怪怪的，如果要用来做文件名，请用正则进行过滤，否则你会爬一天的。。。。

import requests
import html2text,os,codecs,re
from bs4 import BeautifulSoup

# 获取网站源码
def get_html(url):
    result = requests.get(url)
    if result.status_code == 200:
        html = BeautifulSoup(result.text,"lxml")
        return html
    else:
        exit(0)

# 转换markdown格式
def markdown(url):
    html = get_html(url)
    filename = html.find('span', class_="content-title").text
    day = html.find('div', class_='topic-info').select('span > span')[1].text
    content = html.find('div', id="topic_content")

    symbols = re.compile("[^\s\w*]")
    result = symbols.findall(filename)
    for i in result:
        if i in filename:
            filename = filename.replace(i, "")
    if "*" in filename:
        filename = filename.replace("*", "")

    markdown = html2text.HTML2Text()
    markdown.ignore_links = True
    print("[+] {}.md".format(filename))
    file = codecs.open("./MD/"+filename + ".md", "w+",'utf-8')
    file.write("`文章创作时间`: {}\n".format(day)+markdown.handle(str(content)))
    file.close()


# 当前页面链接
def get_content(page):
    url = "https://xz.aliyun.com/"
    html = get_html(page)
    links = html.find_all("a",class_="topic-title")
    for i in links:
        markdown(url+ i['href'])


# 获取页码
def get_page():
    url = "https://xz.aliyun.com/?page="
    html = get_html(url)
    page_count = html.find('ul',class_="pull-right").select("ul > li")[1].select('li > a')[0].text.split("/")[1]
    for page in range(int(page_count)):
        page += 1
        get_content(url+str(page))

if __name__ == "__main__":
    if not os.path.exists("MD"):
        os.makedirs("MD/images")
    get_page()

该爬虫仅供参考学习，需要图片下载的话，请自行添加