import requests  
import os  
import re  
import time  
import hashlib  
import random  
from bs4 import BeautifulSoup  
from urllib.parse import urljoin, urlparse  
from datetime import datetime  
  
# ======================  
# 1. 配置模块  
# ======================  
# 全局会话配置  
session = requests.Session()  
session.headers = {  
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',  
    'Referer': 'https://www.gdmu.edu.cn/'  
}  
  
# 文件存储配置  
news_folder = '学校要闻'  
image_folder = os.path.join(news_folder, 'images')  
  
# 初始化文章存储列表  
essay_list = []  
  
  
# ======================  
# 2. 爬取模块  
# ======================  
def create_folders():  
    """创建必要的存储文件夹"""  
    os.makedirs(news_folder, exist_ok=True)  
    os.makedirs(image_folder, exist_ok=True)  
  
def fetch_page(url):  
    """获取网页内容"""  
    try:  
        response = session.get(url, timeout=15)  
        response.encoding = 'utf-8'  
        response.raise_for_status()  
        return response  
    except requests.exceptions.RequestException as e:  
        print(f"请求失败：{str(e)}")  
        return None  
  
def get_click_count(clicktype, owner, clickid):  
    """调用点击量接口"""  
    api_url = f"https://www.gdmu.edu.cn/system/resource/code/news/click/dynclicks.jsp?clickid={clickid}&owner={owner}&clicktype={clicktype}"  
    try:  
        response = session.get(api_url,  timeout=15)  
        return response.text  if response.status_code  == 200 else "获取失败"  
    except Exception as e:  
        print(f"点击量接口异常：{str(e)}")  
        return "未知"  
  
  
def collect_article_links():  
    """动态收集文章链接"""  
    global essay_list  
    seen_urls = set()  
    current_url = 'https://www.gdmu.edu.cn/xxyw1.htm'  
    failure_count = 0  
  
    while True:  
        if (response := fetch_page(current_url)) is None:  
            failure_count += 1  
            if failure_count >= 3:  
                print("连续3次请求失败，终止分页抓取")  
                break  
            continue  
        failure_count = 0  
  
        # 提取文章链接  
        matches = re.findall(r'href=["\']((?:\.\./)?info/\d+/\d+\.htm)', response.text)  
        for match in matches:  
            full_url = urljoin(current_url, match)  
            if full_url not in seen_urls:  
                seen_urls.add(full_url)  
                essay_list.append(full_url)  
  
        # 寻找下一页  
        soup = BeautifulSoup(response.text, 'html.parser')  
        next_link = soup.find('a', string=re.compile(r'下页|下一页'))  
        if not next_link or not (next_href := next_link.get('href')):  
            break  
  
        # 解析下一页地址  
        current_url = urljoin(current_url, next_href)  
        if urlparse(current_url).path == urlparse(response.url).path:  
            break  
  
    print(f"共收集到 {len(essay_list)} 篇有效文章链接")  
  
  
# ======================  
# 3. 转换模块  
# ======================  
def download_image(img_url, article_url):  
    """高效图片下载处理"""  
    try:  
        full_url = urljoin(article_url, img_url)  
        parsed = urlparse(full_url)  
        clean_path = parsed.path.split('?')[0]  
        file_ext = os.path.splitext(clean_path)[1][:10]  
  
        hash_name = hashlib.md5(full_url.encode()).hexdigest()  
        filename = f"{hash_name}{file_ext if file_ext else '.jpg'}"  
        save_path = os.path.join(image_folder, filename)  
  
        if os.path.exists(save_path):  
            return f'images/{filename}'  
  
        # 单次请求处理  
        response = session.get(full_url, stream=True, timeout=20)  
        if response.status_code == 200 and 'image' in response.headers.get('Content-Type', ''):  
            with open(save_path, 'wb') as f:  
                for chunk in response.iter_content(chunk_size=8192):  
                    f.write(chunk)  
            return f'images/{filename}'  
        return None  
    except Exception as e:  
        print(f"图片处理异常：{str(e)}")  
        return None  
  
  
def html_to_md(html_content, article_url):  
    """优化的HTML转换处理"""  
    soup = BeautifulSoup(html_content, 'html.parser')  
  
    # 批量处理图片  
    img_mapping = {}  
    for img in soup.find_all('img', {'src': True}):  
        if (local_path := download_image(img['src'], article_url)):  
            alt = img.get('alt', '').strip() or "图片描述"  
            img_mapping[img['src']] = f"![{alt}]({local_path})"  
  
    # 转换处理  
    content = []  
    for element in soup.descendants:  
        if element.name == 'img' and element['src'] in img_mapping:  
            content.append(img_mapping[element['src']])  
        elif element.name in ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']:  
            content.append(f"\n{'#'*int(element.name[1])} {element.get_text().strip()}\n")  
        elif element.name == 'p' and element.get_text(strip=True):  
            content.append(element.get_text(' ', strip=True))  
  
    return '\n\n'.join([c for c in content if c.strip()])  
  
  
# ======================  
# 4. 处理模块  
# ======================  
def process_single_article(url):  
    """优化的文章处理流程"""  
    if (response := fetch_page(url)) is None:  
        return  
  
    soup = BeautifulSoup(response.text, 'html.parser')  
    crawl_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")  
  
    # 元数据提取  
    title = (soup.find(class_='arc-tit') or soup.title).get_text().strip()[:100]  
    edit_date = re.search(r'\d{4}-\d{2}-\d{2}', soup.get_text()).group() if re.search(r'\d{4}-\d{2}-\d{2}', soup.get_text()) else "未知日期"  
  
    # 点击量获取  
    click_params = re.search(r'_showDynClicks\(\s*"(\w+)"\s*,\s*(\d+)\s*,\s*(\d+)\s*\)', response.text)  
    view_count = get_click_count(*click_params.groups()) if click_params else "未知"  
  
    # 内容处理  
    content_div = soup.find(class_='uarc-con') or soup.find('article')  
    if not content_div:  
        print(f"⛔ 未找到正文内容：{url}")  
        return  
  
    md_content = html_to_md(str(content_div), url)  
    image_links = [urljoin(url, img['src']) for img in content_div.find_all('img', {'src': True})]  
  
    # 文件保存  
    cleaned_title = re.sub(r'[^\w\u4e00-\u9fff-]', '', title)[:50]  
    filename = f"{cleaned_title}.md"  
    try:  
        with open(os.path.join(news_folder, filename), 'w', encoding='utf-8') as f:  
            f.write(f"# {title}\n\n")  
            f.write(f"**🕒 编辑时间**: {edit_date}  \n")  
            f.write(f"**📅 爬取时间**: {crawl_time}\n")  
            f.write(f"**👁️ 点击次数**: {view_count}\n")  
            f.write(f"**🔗 文章链接**: [{url}]({url})\n")  
            f.write(f"**🖼️ 图片链接**:\n" + "\n".join(f"- [{link}]({link})" for link in image_links) + "\n\n")  
            f.write(md_content)  
        print(f"✅ 成功保存：{filename}")  
    except Exception as e:  
        print(f"💾 文件保存失败：{str(e)}")  
  
  
# ======================  
# 5. 主程序  
# ======================  
def main():  
    create_folders()  
    collect_article_links()  
  
    try:  
        num_to_process = int(input(f"请输入处理数量（共{len(essay_list)}篇，默认全部）：") or len(essay_list))  
    except:  
        num_to_process = len(essay_list)  
  
    success = 0  
    for idx, url in enumerate(essay_list[:num_to_process], 1):  
        print(f"\n📝 处理进度：{idx}/{num_to_process} - {url}")  
        try:  
            process_single_article(url)  
            success += 1  
            if idx % 3 == 0:  
                delay = random.uniform(1.5, 3.5)  
                print(f"⏳ 随机等待 {delay:.1f}秒...")  
                time.sleep(delay)  
        except Exception as e:  
            print(f"⚠️ 处理异常：{str(e)}")  
  
    print(f"\n🎉 完成！成功率：{success/num_to_process:.1%}")  
  
  
if __name__ == "__main__":  
    main()

import os  
from datetime import datetime  
import pandas as pd  
import matplotlib.pyplot as plt  
import matplotlib.dates as mdates  
from openpyxl.drawing.image import Image  
from openpyxl.styles import Alignment, Font  
from openpyxl.utils import get_column_letter  
  
# ======================  
# 1. 提取模块  
# ======================  
def parse_md_files(news_folder):  
    """解析Markdown文件提取元数据"""  
    articles = []  
  
    for filename in os.listdir(news_folder):  
        if not filename.endswith('.md'):  
            continue  
  
        file_path = os.path.join(news_folder, filename)  
        article_info = {  
            'title': '',  
            'edit_time': None,  
            'crawl_time': None,  
            'click_count': '未知',  
            'url': '',  
            'filename': filename,  
            'image_links': []  # 新增图片链接字段  
        }  
  
        try:  
            with open(file_path, 'r', encoding='utf-8') as f:  
                lines = f.readlines()  
                in_image_section = False  # 标记是否进入图片链接区域  
  
                # 提取标题（第一行）  
                if lines:  
                    title_line = lines[0].strip()  
                    article_info['title'] = title_line.lstrip('#').strip()  
  
                # 提取元数据  
                for line in lines[1:100]:  # 扩大扫描范围到前100行  
                    line = line.strip()  
  
                    # 图片链接提取逻辑  
                    if line.startswith('**🖼️ 图片链接**:'):  
                        in_image_section = True  
                        continue  
                    if in_image_section:  
                        if line.startswith('- ['):  
                            img_url = line.split('(')[-1].split(')')[0]  
                            article_info['image_links'].append(img_url.strip())  
                        else:  
                            in_image_section = False  # 退出图片链接区域  
  
                    # 提取编辑时间  
                    if line.startswith('**🕒 编辑时间**:'):  
                        edit_time_str = line.split(': ')[1].strip()  
                        try:  
                            article_info['edit_time'] = datetime.strptime(edit_time_str, "%Y-%m-%d")  
                        except ValueError:  
                            pass  
  
                    # 提取爬取时间  
                    elif line.startswith('**📅 爬取时间**:'):  
                        crawl_time_str = line.split(': ')[1].strip()  
                        try:  
                            article_info['crawl_time'] = datetime.strptime(crawl_time_str, "%Y-%m-%d %H:%M:%S")  
                        except ValueError:  
                            pass  
  
                    # 提取点击量  
                    elif line.startswith('**👁️ 点击次数**:'):  
                        article_info['click_count'] = line.split(': ')[1].strip()  
  
                    # 提取文章链接  
                    elif line.startswith('**🔗 文章链接**:'):  
                        url_part = line.split('(')[-1].split(')')[0]  
                        article_info['url'] = url_part.strip()  
  
                # 有效性检查  
                if article_info['edit_time'] and article_info['crawl_time']:  
                    articles.append(article_info)  
  
        except Exception as e:  
            print(f"解析文件 {filename} 出错: {str(e)}")  
  
    return articles  
  
# ======================  
# 2. 保存模块  
# ======================  
def save_to_excel(data, output_file, chart_image_path=None):  
    """将数据保存到Excel文件（增强版含全局居中对齐）"""  
    df = pd.DataFrame(data)  
  
    # 添加分组标识列  
    df['_group_id_'] = df.index  
    df = df.explode('image_links').reset_index(drop=True)  
    df['image_links'] = df['image_links'].fillna('无图片')  
  
    # 重命名字段（保持不变）  
    df = df.rename(columns={  
        'title': '文章标题',  
        'edit_time': '编辑时间',  
        'crawl_time': '爬取时间',  
        'click_count': '点击量',  
        'url': '文章链接',  
        'filename': '文件名',  
        'image_links': '图片链接'  
    })  
  
    # 格式化时间字段（保持不变）  
    df['编辑时间'] = df['编辑时间'].dt.strftime('%Y-%m-%d')  
    df['爬取时间'] = df['爬取时间'].dt.strftime('%Y-%m-%d  %H:%M:%S')  
  
    with pd.ExcelWriter(output_file, engine='openpyxl') as writer:  
        df.to_excel(writer, index=False, sheet_name='文章数据')  
        workbook = writer.book  
        worksheet = writer.sheets['文章数据']  
  
        # 创建居中对齐样式（新增核心代码）  
        center_alignment = Alignment(  
            horizontal='center',  
            vertical='center',  
            wrap_text=True  # 自动换行以适配合并单元格  
        )  
  
        # 应用全局对齐样式（新增）  
        for row in worksheet.iter_rows():  
            for cell in row:  
                cell.alignment = center_alignment  
  
                # 原有列宽设置逻辑（保持不变）  
        for col in df.columns:  
            col_idx = df.columns.get_loc(col) + 1  
            col_letter = get_column_letter(col_idx)  
            if col == '图片链接':  
                worksheet.column_dimensions[col_letter].width = 105  
            else:  
                max_len = max(df[col].astype(str).map(len).max(), len(str(col))) + 2  
                worksheet.column_dimensions[col_letter].width = min(max_len, 105)  
  
        # 合并单元格逻辑（保持原有功能）  
        for group_id, group in df.groupby('_group_id_'):  
            if len(group) > 1:  
                start_row = group.index.min() + 2  
                end_row = group.index.max() + 2  
                for col_num in [1, 2, 3, 4, 5, 6]:  
                    # 设置合并单元格对齐（新增）  
                    merged_cell = worksheet.cell(row=start_row, column=col_num)  
                    merged_cell.alignment = center_alignment  
                    worksheet.merge_cells(  
                        start_row=start_row,  
                        end_row=end_row,  
                        start_column=col_num,  
                        end_column=col_num  
                    )  
  
        # 超链接样式设置（保持颜色特征）  
        hyperlink_font = Font(color="0563C1", underline="single")  
        image_font = Font(color="FF69B4", underline="single")  
        for row in range(2, len(df) + 2):  
            # 文章链接  
            article_cell = worksheet[f'E{row}']  
            if df.at[row - 2, '文章链接']:  
                article_cell.font = hyperlink_font  
                # 图片链接  
            img_cell = worksheet[f'G{row}']  
            if df.at[row - 2, '图片链接'] != '无图片':  
                img_cell.font = image_font  
  
                # 图表插入逻辑（保持不变）  
        if chart_image_path and os.path.exists(chart_image_path):  
            try:  
                img = Image(chart_image_path)  
                img.width = 800  
                img.height = 500  
                worksheet.add_image(img, f'A{len(df) + 3}')  
            except Exception as e:  
                print(f"图表插入失败：{str(e)}")  
  
    print(f"成功保存带全局居中对齐的Excel文件: {output_file}")  
  
  
def truncate_title(title, max_length=7):  
    """截断长标题"""  
    return title[:max_length] + '..' if len(title) > max_length else title  
  
  
# ======================  
# 3. 图片模块  
# ======================  
def visualize_timeline(data):  
    """生成时间线可视化图表"""  
    # 设置中文字体  
    plt.rcParams['font.sans-serif'] = ['SimHei', 'Microsoft YaHei', 'WenQuanYi Micro Hei']  
    plt.rcParams['axes.unicode_minus'] = False  
  
    # 按点击量降序排序（保持纵向排列顺序）  
    sorted_data = sorted(data, key=lambda x: x['click_count'], reverse=True)  
  
    # 创建索引映射表（用于定位文章在图表中的纵向位置）  
    index_map = {a['filename']: idx for idx, a in enumerate(sorted_data)}  
  
    # 动态调整画布高度  
    plt.figure(figsize=(14, len(sorted_data) * 0.6 + 4))  
    ax = plt.gca()  
    ax.invert_yaxis()  # 高点击量在上方  
  
    # 设置日期格式  
    date_format = mdates.DateFormatter('%m-%d %H:%M')  
    ax.xaxis.set_major_formatter(date_format)  
    ax.xaxis.set_major_locator(mdates.AutoDateLocator())  
  
    # 按编辑时间排序（用于生成趋势线）  
    edit_sorted = sorted(data, key=lambda x: x['edit_time'])  
    edit_x = [a['edit_time'] for a in edit_sorted]  
    edit_y = [index_map[a['filename']] for a in edit_sorted]  # 保持点击量排序的纵向位置  
  
    # 按爬取时间排序（用于生成趋势线）  
    crawl_sorted = sorted(data, key=lambda x: x['crawl_time'])  
    crawl_x = [a['crawl_time'] for a in crawl_sorted]  
    crawl_y = [index_map[a['filename']] for a in crawl_sorted]  
  
    # 绘制时间趋势线（连接时间相邻的点）  
    ax.plot(edit_x, edit_y,  
            color='#2ecc71', linewidth=2, alpha=0.7,  
            marker='o', markersize=8, label='编辑时间趋势')  
    ax.plot(crawl_x, crawl_y,  
            color='#e74c3c', linewidth=2, alpha=0.7,  
            marker='s', markersize=8, label='爬取时间趋势')  
  
    # 绘制个体连接线  
    for idx, article in enumerate(sorted_data):  
        edit_time = article['edit_time']  
        crawl_time = article['crawl_time']  
  
        ax.plot(  
            [edit_time, crawl_time],  
            [idx, idx],  
            linestyle='--',  
            color='gray',  
            alpha=0.4,  
            linewidth=1  
        )  
  
        # 智能标注系统  
        truncated_title = truncate_title(article['title'])  
        annotation_text = f"{truncated_title}\n点击: {article['click_count']}"  
  
        # 根据时间间隔决定标注方向  
        time_gap = (crawl_time - edit_time).total_seconds()  
        if time_gap < 3600 * 6:  # 6小时内的短间隔  
            xy = (crawl_time, idx)  
            offset = (-10, 0)  
            ha = 'right'  
        else:  
            xy = (edit_time, idx)  
            offset = (10, 0)  
            ha = 'left'  
  
        ax.annotate(  
            annotation_text,  
            xy,  
            xytext=offset,  
            textcoords='offset points',  
            fontsize=8,  
            alpha=0.9,  
            ha=ha,  
            va='center',  
            arrowprops=dict(  
                arrowstyle='->',  
                color='gray',  
                alpha=0.3,  
                linewidth=0.8  
            )  
        )  
  
    # 图表美化  
    plt.yticks(range(len(sorted_data)), [truncate_title(a['title']) for a in sorted_data], fontsize=8)  
    plt.xlabel('时间轴', fontsize=12)  
    plt.ylabel('文章（点击量降序）', fontsize=12)  
    plt.title('文章时效性分析（时间趋势线+点击量排序）', fontsize=14, pad=20)  
    plt.grid(True, alpha=0.2, linestyle=':')  
  
    # 优化图例  
    plt.legend(  
        loc='upper left',  
        bbox_to_anchor=(1, 0.9),  
        frameon=True,  
        shadow=True,  
        fontsize=10  
    )  
  
    plt.tight_layout()  
    plt.savefig('数据分析图_时间趋势版.png', dpi=300, bbox_inches='tight')  
    print("已生成可视化图表: 数据分析图_时间趋势版.png")  
  
  
# ======================  
# 4.主程序  
# ======================  
def main():  
    # 参数配置  
    NEWS_FOLDER = '学校要闻'  
    EXCEL_OUTPUT = '广东医科大学--学校要闻.xlsx'  
    CHART_IMAGE = '数据分析图_时间趋势版.png'  # 新增图表路径常量  
  
    # 处理数据  
    articles_data = parse_md_files(NEWS_FOLDER)  
  
    if not articles_data:  
        print("未找到有效文章数据！")  
        return  
  
    # 生成可视化图表（必须在保存Excel之前）  
    valid_data = [  
        a for a in articles_data  
        if isinstance(a.get('click_count'), str) and a['click_count'].isdigit()  
    ]  
    for a in valid_data:  
        a['click_count'] = int(a['click_count'])  
    visualize_timeline(valid_data)  # 生成图表文件  
  
    # 保存到Excel（传递图表路径）  
    save_to_excel(articles_data, EXCEL_OUTPUT, CHART_IMAGE)  
  
  
if __name__ == "__main__":  
    main()