import json
import time
import os
import re

import requests
from bs4 import BeautifulSoup
import chardet


"""
url_list = [
	'http://tech.163.com/special/00097UHL/tech_datalist_02.js?callback=data_callback',
	'http://ent.163.com/special/000380VU/newsdata_index_02.js?callback=data_callback',
	'http://sports.163.com/special/000587PR/newsdata_n_index_02.js?callback=data_callback',
	'http://money.163.com/special/002557S5/newsdata_idx_index.js?callback=data_callback',
	'http://edu.163.com/special/002987KB/newsdata_edu_hot_02.js?callback=data_callback'
]
"""

def crawl(pn):
	headers = {
		'Accept': '*/*',
		'Connection': 'keep-alive',
		'Host': 'ent.163.com',
		'Referer': 'http://ent.163.com/',
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
	}

	request_url = 'http://ent.163.com/special/000380VU/newsdata_index_0' + str(pn) + '.js?callback=data_callback'
	# print('is crawling page ' + request_url + '......')
	response = requests.get(request_url, headers=headers)
	if response.status_code == 200:
		page_encoding = chardet.detect(response.content)['encoding']
		temp_str = response.text.replace('data_callback(', '')
		# temp_str = response.content.decode(page_encoding).replace('data_callback(', '')
		temp_str = temp_str.replace(temp_str[-1], '')
		for each_news in json.loads(temp_str):
			print(each_news['docurl'])
			download_news_content(each_news['title'], each_news['docurl'])

	elif response.status_code == 404:
		raise Exception('No Page Found! ' + request_url)
	else:
		print('ERROR! ' + str(response.status_code))


def download_news_content(title, news_url):

	if news_url.startswith('http://v'):
		print('This page contains video ...')

	else:
		# r_content = re.compile('<img \w')
		r_title = re.compile('[\?\＂\？\：\s\/\·]')

		file_dir = 'd:/网易新闻/娱乐'
		if not os.path.exists(file_dir) or not os.path.isdir(file_dir):
			os.makedirs(file_dir)

		headers = {
			'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
			'Connection': 'keep-alive',
			'Upgrade-Insecure-Requests': '1',
			'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
		}
		response = requests.get(news_url, headers=headers)
		if response.status_code == 200:
			soup = BeautifulSoup(response.text, 'html5lib')

			if response.url.startswith('http://digi.163.com') or response.url.startswith('http://tech.163.com') or response.url.startswith('http://ent.163.com'):
				text_soup = soup.select('#endText')[0]
				content_text = text_soup.get_text()
			elif response.url.startswith('http://dy.163.com'):
				text_soup = soup.select('#content')[0]
				content_text = text_soup.get_text()
			elif response.url.startswith('http://mobile.163.com'):
				text_soup = soup.select('#epContentLeft')[0]
				content_text = text_soup.get_text()

			with open(file_dir + os.path.sep + re.sub(r_title, '', title, count=0) + '.txt', mode='wt', encoding='utf-8') as f:
				f.write(content_text)
				f.flush()
				f.close()
				print(title + '.txt has been written done!')

if __name__ == '__main__':
	for i in range(2, 10, 1):
		crawl(i)
		time.sleep(5)