""" 字节直播数据采集工具 - GUI 版本 整合 commerce 和 entertainment 两个爬虫 """ import sys import os import threading import queue import json import hashlib import requests from datetime import datetime from pathlib import Path # 处理打包后的路径 if getattr(sys, 'frozen', False): # 打包后运行 BASE_DIR = Path(sys._MEIPASS) GUI_DIR = BASE_DIR CRAWLER_DIR = BASE_DIR # 设置 Playwright 浏览器路径 playwright_path = BASE_DIR / 'ms-playwright' if playwright_path.exists(): os.environ['PLAYWRIGHT_BROWSERS_PATH'] = str(playwright_path) else: # 开发模式运行 GUI_DIR = Path(__file__).parent CRAWLER_DIR = GUI_DIR.parent # 添加项目路径 sys.path.insert(0, str(CRAWLER_DIR)) import customtkinter as ctk from tkinter import messagebox import importlib def import_with_crawler_context(crawler_type: str, module_path: str): """ 在指定爬虫的上下文中导入模块 解决 commerce 和 entertainment 都有同名 config/core/utils 模块的冲突问题 Args: crawler_type: 'commerce' 或 'entertainment' module_path: 要导入的模块路径，如 'core.browser_login' Returns: 导入的模块 """ crawler_path = str(CRAWLER_DIR / crawler_type) # 清除可能冲突的缓存模块 conflicting_prefixes = ('config', 'core', 'utils', 'services') modules_to_remove = [ name for name in sys.modules.keys() if any(name == prefix or name.startswith(prefix + '.') for prefix in conflicting_prefixes) ] for mod_name in modules_to_remove: del sys.modules[mod_name] # 确保爬虫路径在 sys.path 最前面 if crawler_path in sys.path: sys.path.remove(crawler_path) sys.path.insert(0, crawler_path) # 导入模块（不再在 finally 中移除路径，因为模块运行时可能还需要） return importlib.import_module(module_path) # 设置外观 ctk.set_appearance_mode("dark") ctk.set_default_color_theme("blue") # 平台配置（与浏览器插件保持一致） PLATFORMS = { 'commerce': { 'name': '巨量百应-达人账号', 'loginUrl': 'https://buyin.jinritemai.com/mpa/account/institution-role-select' }, 'entertainment': { 'name': '字节联盟-主播排行', 'loginUrl': 'https://union.bytedance.com/open/portal/data/leaderboard?appId=3000' } } class GlobalConfigManager: """全局配置管理器，用于管理上传服务器配置""" DEFAULT_CONFIG = { 'serverUrl': 'https://admin.api.skzhijia.com', 'apiKey': 'sk_7A353DEF2BFD4EADA4BF364CCC7C8FDE2AAD1' } def __init__(self): # 配置文件保存在 gui 目录下 if getattr(sys, 'frozen', False): self.config_file = Path(sys.executable).parent / 'upload_config.json' else: self.config_file = GUI_DIR / 'upload_config.json' def load_config(self) -> dict: """加载配置""" if self.config_file.exists(): try: with open(self.config_file, 'r', encoding='utf-8') as f: config = json.load(f) # 合并默认配置 return {**self.DEFAULT_CONFIG, **config} except Exception: pass return self.DEFAULT_CONFIG.copy() def save_config(self, config: dict) -> bool: """保存配置""" try: with open(self.config_file, 'w', encoding='utf-8') as f: json.dump(config, f, ensure_ascii=False, indent=2) return True except Exception: return False # 全局配置管理器实例 global_config_manager = GlobalConfigManager() class ConfigDialog(ctk.CTkToplevel): """配置对话框""" def __init__(self, parent, crawler_type, on_save_callback=None): super().__init__(parent) self.crawler_type = crawler_type self.on_save_callback = on_save_callback self.title(f"配置 - {crawler_type}") self.geometry("500x480") self.resizable(False, False) # 模态对话框 self.transient(parent) self.grab_set() self.load_config() self.setup_ui() # 居中显示 self.update_idletasks() x = (self.winfo_screenwidth() - self.winfo_width()) // 2 y = (self.winfo_screenheight() - self.winfo_height()) // 2 self.geometry(f"+{x}+{y}") def load_config(self): """加载配置""" try: config_module = import_with_crawler_context(self.crawler_type, 'config.settings') self.config_values = { 'REPORT_URL': getattr(config_module, 'REPORT_URL', ''), 'REQUEST_TIMEOUT': getattr(config_module, 'REQUEST_TIMEOUT', 30), 'BROWSER_TIMEOUT': getattr(config_module, 'BROWSER_TIMEOUT', 300), 'FANS_FETCH_CONCURRENCY': getattr(config_module, 'FANS_FETCH_CONCURRENCY', 5), } if self.crawler_type == 'commerce': self.config_values['DAEMON_INTERVAL_HOURS'] = getattr(config_module, 'DAEMON_INTERVAL_HOURS', 1) else: self.config_values['DAEMON_INTERVAL_NO_FANS_HOURS'] = getattr(config_module, 'DAEMON_INTERVAL_NO_FANS_HOURS', 1) self.config_values['DAEMON_INTERVAL_WITH_FANS_HOURS'] = getattr(config_module, 'DAEMON_INTERVAL_WITH_FANS_HOURS', 6) except Exception as e: self.config_values = {} messagebox.showerror("错误", f"加载配置失败: {e}") # 加载全局上传配置 self.global_config = global_config_manager.load_config() def setup_ui(self): """设置UI""" # 滚动容器 scroll_frame = ctk.CTkScrollableFrame(self) scroll_frame.pack(fill="both", expand=True, padx=10, pady=10) self.entries = {} self.global_entries = {} # Cookie 上传配置（全局配置） ctk.CTkLabel(scroll_frame, text="Cookie 上传配置", font=ctk.CTkFont(weight="bold")).pack(anchor="w", pady=(0, 5)) # 服务器地址 self._add_global_entry(scroll_frame, "服务器地址:", 'serverUrl', width=400) # API Key self._add_global_entry(scroll_frame, "API Key:", 'apiKey', width=400, show="*") # 通用配置 ctk.CTkLabel(scroll_frame, text="爬虫配置", font=ctk.CTkFont(weight="bold")).pack(anchor="w", pady=(15, 5)) # 数据上报URL self._add_entry(scroll_frame, "数据上报URL:", 'REPORT_URL', width=400) # 请求超时 self._add_entry(scroll_frame, "请求超时(秒):", 'REQUEST_TIMEOUT', width=100) # 浏览器登录超时 self._add_entry(scroll_frame, "登录超时(秒):", 'BROWSER_TIMEOUT', width=100) # 粉丝抓取并发数 self._add_entry(scroll_frame, "粉丝抓取并发数:", 'FANS_FETCH_CONCURRENCY', width=100) # 守护进程间隔 ctk.CTkLabel(scroll_frame, text="守护进程配置", font=ctk.CTkFont(weight="bold")).pack(anchor="w", pady=(15, 5)) if self.crawler_type == 'commerce': self._add_entry(scroll_frame, "抓取间隔(小时):", 'DAEMON_INTERVAL_HOURS', width=100) else: self._add_entry(scroll_frame, "无粉丝抓取间隔(小时):", 'DAEMON_INTERVAL_NO_FANS_HOURS', width=100) self._add_entry(scroll_frame, "含粉丝抓取间隔(小时):", 'DAEMON_INTERVAL_WITH_FANS_HOURS', width=100) # 按钮区域 btn_frame = ctk.CTkFrame(self) btn_frame.pack(fill="x", padx=10, pady=10) ctk.CTkButton(btn_frame, text="保存", width=80, command=self.on_save).pack(side="right", padx=5) ctk.CTkButton(btn_frame, text="取消", width=80, command=self.destroy).pack(side="right", padx=5) def _add_entry(self, parent, label, key, width=200): """添加配置项""" frame = ctk.CTkFrame(parent, fg_color="transparent") frame.pack(fill="x", pady=3) ctk.CTkLabel(frame, text=label, width=150, anchor="w").pack(side="left") entry = ctk.CTkEntry(frame, width=width) entry.pack(side="left", padx=5) value = self.config_values.get(key, '') entry.insert(0, str(value) if value else '') self.entries[key] = entry def _add_global_entry(self, parent, label, key, width=200, show=None): """添加全局配置项""" frame = ctk.CTkFrame(parent, fg_color="transparent") frame.pack(fill="x", pady=3) ctk.CTkLabel(frame, text=label, width=150, anchor="w").pack(side="left") entry = ctk.CTkEntry(frame, width=width, show=show) entry.pack(side="left", padx=5) value = self.global_config.get(key, '') entry.insert(0, str(value) if value else '') self.global_entries[key] = entry def on_save(self): """保存配置""" # 保存全局上传配置 new_global_config = {} for key, entry in self.global_entries.items(): new_global_config[key] = entry.get().strip() if global_config_manager.save_config(new_global_config): messagebox.showinfo("提示", "配置已保存。\n注意：部分爬虫配置需要重启程序才能生效。") else: messagebox.showerror("错误", "保存配置失败") return if self.on_save_callback: self.on_save_callback() self.destroy() class LogHandler: """日志处理器，将日志重定向到 GUI""" def __init__(self, text_widget, log_queue): self.text_widget = text_widget self.log_queue = log_queue def write(self, message): if message.strip(): self.log_queue.put(message) def flush(self): pass class CrawlerTab(ctk.CTkFrame): """爬虫 Tab 基类""" def __init__(self, parent, crawler_type, log_queue): super().__init__(parent) self.crawler_type = crawler_type self.log_queue = log_queue self.running_thread = None self.stop_flag = threading.Event() self.is_logged_in = False self.is_running = False # 是否有任务正在运行 self.setup_ui() self.check_cookie_status() def setup_ui(self): # Cookie 状态区域 status_frame = ctk.CTkFrame(self) status_frame.pack(fill="x", padx=10, pady=10) self.cookie_label = ctk.CTkLabel( status_frame, text="Cookie 状态: 检查中...", font=ctk.CTkFont(size=14) ) self.cookie_label.pack(side="left", padx=10, pady=5) # 按钮区域 btn_frame = ctk.CTkFrame(self) btn_frame.pack(fill="x", padx=10, pady=5) self.login_btn = ctk.CTkButton( btn_frame, text="登录", width=80, command=self.on_login ) self.login_btn.pack(side="left", padx=5, pady=5) self.logout_btn = ctk.CTkButton( btn_frame, text="退出登录", width=80, fg_color="orange", hover_color="darkorange", command=self.on_logout, state="disabled" ) self.logout_btn.pack(side="left", padx=5, pady=5) self.upload_btn = ctk.CTkButton( btn_frame, text="上传Cookie", width=90, fg_color="#28a745", hover_color="#218838", command=self.on_upload_cookie, state="disabled" ) self.upload_btn.pack(side="left", padx=5, pady=5) self.status_btn = ctk.CTkButton( btn_frame, text="查看状态", width=80, command=self.on_status, state="disabled" ) self.status_btn.pack(side="left", padx=5, pady=5) self.crawl_btn = ctk.CTkButton( btn_frame, text="单次抓取", width=80, command=self.on_crawl, state="disabled" ) self.crawl_btn.pack(side="left", padx=5, pady=5) self.daemon_btn = ctk.CTkButton( btn_frame, text="启动守护", width=80, command=self.on_daemon, state="disabled" ) self.daemon_btn.pack(side="left", padx=5, pady=5) self.stop_btn = ctk.CTkButton( btn_frame, text="停止", width=80, fg_color="red", hover_color="darkred", command=self.on_stop, state="disabled" ) self.stop_btn.pack(side="left", padx=5, pady=5) self.config_btn = ctk.CTkButton( btn_frame, text="配置", width=80, fg_color="gray", hover_color="darkgray", command=self.on_config ) self.config_btn.pack(side="left", padx=5, pady=5) # 参数设置区域 params_frame = ctk.CTkFrame(self) params_frame.pack(fill="x", padx=10, pady=5) self.setup_params(params_frame) def setup_params(self, parent): """子类重写此方法设置参数""" pass def log(self, message): """写入日志""" timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.log_queue.put(f"[{timestamp}] [{self.crawler_type}] {message}\n") def check_cookie_status(self): """检查 Cookie 状态""" try: cookie_module = import_with_crawler_context(self.crawler_type, 'core.cookie_manager') CookieManager = cookie_module.CookieManager cookie_manager = CookieManager() if cookie_manager.is_cookie_exists(): cookie_data = cookie_manager.load_cookies() if cookie_data: metadata = cookie_data.get("metadata", {}) username = metadata.get("username", "未知") self.cookie_label.configure( text=f"Cookie 状态: 已登录 ({username})", text_color="green" ) self.is_logged_in = True self.update_buttons_state() return self.cookie_label.configure( text="Cookie 状态: 未登录", text_color="red" ) self.is_logged_in = False self.update_buttons_state() except Exception as e: self.cookie_label.configure( text=f"Cookie 状态: 检查失败 ({e})", text_color="orange" ) self.is_logged_in = False self.update_buttons_state() def update_buttons_state(self): """根据登录状态和运行状态更新按钮""" if self.is_running: # 任务运行中：只启用停止按钮 self.login_btn.configure(state="disabled") self.logout_btn.configure(state="disabled") self.upload_btn.configure(state="disabled") self.status_btn.configure(state="disabled") self.crawl_btn.configure(state="disabled") self.daemon_btn.configure(state="disabled") self.stop_btn.configure(state="normal") self.config_btn.configure(state="disabled") elif self.is_logged_in: # 已登录且未运行：禁用登录，启用其他功能 self.login_btn.configure(state="disabled") self.logout_btn.configure(state="normal") self.upload_btn.configure(state="normal") self.status_btn.configure(state="normal") self.crawl_btn.configure(state="normal") self.daemon_btn.configure(state="normal") self.stop_btn.configure(state="disabled") self.config_btn.configure(state="normal") else: # 未登录：只启用登录按钮 self.login_btn.configure(state="normal") self.logout_btn.configure(state="disabled") self.upload_btn.configure(state="disabled") self.status_btn.configure(state="disabled") self.crawl_btn.configure(state="disabled") self.daemon_btn.configure(state="disabled") self.stop_btn.configure(state="disabled") self.config_btn.configure(state="normal") def on_login(self): """登录""" self.log("正在启动浏览器登录...") self.is_running = True self.update_buttons_state() def run(): try: browser_module = import_with_crawler_context(self.crawler_type, 'core.browser_login') BrowserLogin = browser_module.BrowserLogin browser_login = BrowserLogin() cookies = browser_login.trigger_login() if cookies: self.log("登录成功!") else: self.log("登录失败或被取消") except Exception as e: self.log(f"登录出错: {e}") finally: # 确保在主线程更新UI self.after(0, self._on_login_complete) self.running_thread = threading.Thread(target=run, daemon=True) self.running_thread.start() def _on_login_complete(self): """登录完成后的处理""" self.is_running = False self.check_cookie_status() def on_logout(self): """退出登录""" if messagebox.askyesno("确认", "确定要退出登录吗？\n这将清除保存的Cookie。"): try: cookie_module = import_with_crawler_context(self.crawler_type, 'core.cookie_manager') CookieManager = cookie_module.CookieManager cookie_manager = CookieManager() cookie_file = cookie_manager.cookie_file if cookie_file.exists(): cookie_file.unlink() self.log("已退出登录，Cookie 已清除") else: self.log("Cookie 文件不存在") self.check_cookie_status() except Exception as e: self.log(f"退出登录失败: {e}") def on_config(self): """打开配置对话框""" ConfigDialog(self.winfo_toplevel(), self.crawler_type) def on_status(self): """查看状态""" try: cookie_module = import_with_crawler_context(self.crawler_type, 'core.cookie_manager') CookieManager = cookie_module.CookieManager cookie_manager = CookieManager() if cookie_manager.is_cookie_exists(): cookie_data = cookie_manager.load_cookies() if cookie_data: metadata = cookie_data.get("metadata", {}) cookies = cookie_data.get("cookies", []) info = f"""Cookie 状态信息: 文件路径: {cookie_manager.cookie_file} Cookie 数量: {len(cookies)} 用户名: {metadata.get('username', '未知')} 创建时间: {metadata.get('created_at', '未知')} 最后验证: {metadata.get('last_validated', '未知')} 登录URL: {metadata.get('url', '未知')}""" self.log(info) else: self.log("Cookie 文件存在但加载失败") else: self.log("Cookie 文件不存在，请先登录") self.check_cookie_status() except Exception as e: self.log(f"查看状态出错: {e}") def on_upload_cookie(self): """上传 Cookie 到服务器""" # 检查配置 config = global_config_manager.load_config() server_url = config.get('serverUrl', '').strip() api_key = config.get('apiKey', '').strip() if not server_url or not api_key: messagebox.showwarning("配置缺失", "请先在配置中设置服务器地址和 API Key") return self.log("正在上传 Cookie 到服务器...") self.set_running_state(True) def run(): try: # 获取 cookie 数据 cookie_module = import_with_crawler_context(self.crawler_type, 'core.cookie_manager') CookieManager = cookie_module.CookieManager cookie_manager = CookieManager() if not cookie_manager.is_cookie_exists(): self.log("Cookie 文件不存在，请先登录") return cookie_data = cookie_manager.load_cookies() if not cookie_data: self.log("加载 Cookie 失败") return cookies = cookie_data.get("cookies", []) if not cookies: self.log("Cookie 为空") return # 将 cookie 列表转换为字符串格式：name=value; name2=value2 cookie_string = '; '.join([f"{c['name']}={c['value']}" for c in cookies]) # 计算 MD5 md5_hash = hashlib.md5(cookie_string.encode('utf-8')).hexdigest() # 获取平台配置 platform = PLATFORMS.get(self.crawler_type, {}) login_url = platform.get('loginUrl', '') # 上传到服务器 upload_url = f"{server_url}/api/crawler/cookie/upload" payload = { 'code': self.crawler_type, 'cookieString': cookie_string, 'md5': md5_hash, 'url': login_url } headers = { 'X-API-Key': api_key, 'Content-Type': 'application/json' } self.log(f"正在上传到: {upload_url}") response = requests.post(upload_url, json=payload, headers=headers, timeout=30) if response.status_code != 200: self.log(f"上传失败: HTTP {response.status_code} - {response.text}") return result = response.json() if result.get('code') == 200: self.log(f"Cookie 上传成功!") if result.get('data'): self.log(f"服务器返回: {result.get('data')}") else: self.log(f"上传失败: {result.get('msg', '未知错误')}") except requests.exceptions.Timeout: self.log("上传超时，请检查网络连接") except requests.exceptions.ConnectionError: self.log("连接服务器失败，请检查服务器地址") except Exception as e: self.log(f"上传出错: {e}") finally: self.after(0, lambda: self.set_running_state(False)) self.running_thread = threading.Thread(target=run, daemon=True) self.running_thread.start() def on_crawl(self): """单次抓取 - 子类重写""" pass def on_daemon(self): """守护进程 - 子类重写""" pass def on_stop(self): """停止""" self.stop_flag.set() self.log("正在停止...") # 给一个短暂的延迟后检查任务是否已停止 self.after(1000, self._check_task_stopped) def _check_task_stopped(self): """检查任务是否已停止""" if self.running_thread and self.running_thread.is_alive(): # 任务仍在运行，继续等待 self.after(1000, self._check_task_stopped) else: # 任务已停止 self.is_running = False self.update_buttons_state() self.log("任务已停止") def set_running_state(self, running): """设置运行状态""" self.is_running = running if running: self.stop_flag.clear() self.update_buttons_state() class CommerceTab(CrawlerTab): """巨量百应 Tab""" def __init__(self, parent, log_queue): super().__init__(parent, "commerce", log_queue) def setup_params(self, parent): # 每页数量 ctk.CTkLabel(parent, text="每页数量:").pack(side="left", padx=(10, 5), pady=5) self.page_size_var = ctk.StringVar(value="20") self.page_size_entry = ctk.CTkEntry(parent, width=60, textvariable=self.page_size_var) self.page_size_entry.pack(side="left", padx=5, pady=5) # 间隔时间 ctk.CTkLabel(parent, text="抓取间隔(小时):").pack(side="left", padx=(20, 5), pady=5) self.interval_var = ctk.StringVar(value="1") self.interval_entry = ctk.CTkEntry(parent, width=60, textvariable=self.interval_var) self.interval_entry.pack(side="left", padx=5, pady=5) def on_crawl(self): """单次抓取""" self.log("开始抓取达人账号数据...") self.set_running_state(True) def run(): try: service_module = import_with_crawler_context('commerce', 'services.daren_account_service') DarenAccountService = service_module.DarenAccountService page_size = int(self.page_size_var.get()) service = DarenAccountService() filepath = service.crawl(page_size=page_size, status=1) if filepath: self.log(f"抓取完成! 文件: {filepath}") else: self.log("抓取失败") except Exception as e: self.log(f"抓取出错: {e}") finally: self.after(0, lambda: self.set_running_state(False)) self.running_thread = threading.Thread(target=run, daemon=True) self.running_thread.start() def on_daemon(self): """守护进程模式""" self.log("启动守护进程模式...") self.set_running_state(True) def run(): try: import time service_module = import_with_crawler_context('commerce', 'services.daren_account_service') DarenAccountService = service_module.DarenAccountService page_size = int(self.page_size_var.get()) interval_hours = float(self.interval_var.get()) interval_seconds = interval_hours * 3600 self.log(f"守护进程已启动 - 间隔: {interval_hours} 小时") cycle = 0 while not self.stop_flag.is_set(): cycle += 1 self.log(f"第 {cycle} 次抓取开始...") try: service = DarenAccountService() filepath = service.crawl(page_size=page_size, status=1) if filepath: self.log(f"第 {cycle} 次抓取完成: {filepath.name}") else: self.log(f"第 {cycle} 次抓取失败") except Exception as e: self.log(f"第 {cycle} 次抓取出错: {e}") # 等待下次抓取 self.log(f"等待 {interval_hours} 小时后进行下次抓取...") # 分段等待，以便响应停止信号 waited = 0 while waited < interval_seconds and not self.stop_flag.is_set(): time.sleep(min(30, interval_seconds - waited)) waited += 30 self.log("守护进程已停止") except Exception as e: self.log(f"守护进程出错: {e}") finally: self.after(0, lambda: self.set_running_state(False)) self.running_thread = threading.Thread(target=run, daemon=True) self.running_thread.start() class EntertainmentTab(CrawlerTab): """娱乐版 Tab""" def __init__(self, parent, log_queue): super().__init__(parent, "entertainment", log_queue) def setup_params(self, parent): # 每页数量 ctk.CTkLabel(parent, text="每页数量:").pack(side="left", padx=(10, 5), pady=5) self.size_var = ctk.StringVar(value="10") self.size_entry = ctk.CTkEntry(parent, width=60, textvariable=self.size_var) self.size_entry.pack(side="left", padx=5, pady=5) # 抓取粉丝 self.fetch_fans_var = ctk.BooleanVar(value=True) self.fetch_fans_cb = ctk.CTkCheckBox( parent, text="抓取粉丝数据", variable=self.fetch_fans_var ) self.fetch_fans_cb.pack(side="left", padx=20, pady=5) # 间隔时间 ctk.CTkLabel(parent, text="抓取间隔(小时):").pack(side="left", padx=(20, 5), pady=5) self.interval_var = ctk.StringVar(value="1") self.interval_entry = ctk.CTkEntry(parent, width=60, textvariable=self.interval_var) self.interval_entry.pack(side="left", padx=5, pady=5) def on_crawl(self): """单次抓取""" self.log("开始抓取排行榜数据...") self.set_running_state(True) def run(): try: service_module = import_with_crawler_context('entertainment', 'services.crawler_service') CrawlerService = service_module.CrawlerService size = int(self.size_var.get()) fetch_fans = self.fetch_fans_var.get() service = CrawlerService() filepath = service.crawl( rank_type=0, size=size, filter_type='anchor', fetch_fans=fetch_fans ) if filepath: self.log(f"抓取完成! 文件: {filepath}") else: self.log("抓取失败") except Exception as e: self.log(f"抓取出错: {e}") finally: self.after(0, lambda: self.set_running_state(False)) self.running_thread = threading.Thread(target=run, daemon=True) self.running_thread.start() def on_daemon(self): """守护进程模式""" self.log("启动守护进程模式...") self.set_running_state(True) def run(): try: import time service_module = import_with_crawler_context('entertainment', 'services.crawler_service') CrawlerService = service_module.CrawlerService size = int(self.size_var.get()) fetch_fans = self.fetch_fans_var.get() interval_hours = float(self.interval_var.get()) interval_seconds = interval_hours * 3600 self.log(f"守护进程已启动 - 间隔: {interval_hours} 小时") cycle = 0 while not self.stop_flag.is_set(): cycle += 1 self.log(f"第 {cycle} 次抓取开始...") try: service = CrawlerService() filepath = service.crawl( rank_type=0, size=size, filter_type='anchor', fetch_fans=fetch_fans ) if filepath: self.log(f"第 {cycle} 次抓取完成: {filepath.name}") else: self.log(f"第 {cycle} 次抓取失败") except Exception as e: self.log(f"第 {cycle} 次抓取出错: {e}") # 等待下次抓取 self.log(f"等待 {interval_hours} 小时后进行下次抓取...") # 分段等待 waited = 0 while waited < interval_seconds and not self.stop_flag.is_set(): time.sleep(min(30, interval_seconds - waited)) waited += 30 self.log("守护进程已停止") except Exception as e: self.log(f"守护进程出错: {e}") finally: self.after(0, lambda: self.set_running_state(False)) self.running_thread = threading.Thread(target=run, daemon=True) self.running_thread.start() class App(ctk.CTk): """主应用程序""" # 日志最大行数，超过后自动截断 MAX_LOG_LINES = 1000 def __init__(self): super().__init__() self.title("字节直播数据采集工具") self.geometry("900x650") self.minsize(800, 550) # 日志队列 self.log_queue = queue.Queue() # 重定向标准输出到日志 self.setup_stdout_redirect() self.setup_ui() self.process_log_queue() def setup_stdout_redirect(self): """重定向标准输出和错误输出到日志队列""" class StdoutRedirector: def __init__(self, log_queue, original_stdout): self.log_queue = log_queue self.original_stdout = original_stdout self.buffer = "" def write(self, message): # 同时输出到原始stdout（控制台） if self.original_stdout: try: self.original_stdout.write(message) except: pass # 处理消息，按行分割 if message: self.buffer += message while '\n' in self.buffer: line, self.buffer = self.buffer.split('\n', 1) if line.strip(): self.log_queue.put(f"{line}\n") def flush(self): if self.original_stdout: try: self.original_stdout.flush() except: pass # 刷新缓冲区中剩余的内容 if self.buffer.strip(): self.log_queue.put(f"{self.buffer}\n") self.buffer = "" # 保存原始的stdout和stderr self.original_stdout = sys.stdout self.original_stderr = sys.stderr # 重定向 sys.stdout = StdoutRedirector(self.log_queue, self.original_stdout) sys.stderr = StdoutRedirector(self.log_queue, self.original_stderr) def setup_ui(self): # 标题 title_label = ctk.CTkLabel( self, text="直播数据采集工具", font=ctk.CTkFont(size=20, weight="bold") ) title_label.pack(pady=10) # Tab 视图 self.tabview = ctk.CTkTabview(self) self.tabview.pack(fill="both", expand=True, padx=10, pady=5) # 添加 Tab self.tabview.add("巨量百应") self.tabview.add("字节联盟-主播排行") # 创建 Tab 内容 self.commerce_tab = CommerceTab( self.tabview.tab("巨量百应"), self.log_queue ) self.commerce_tab.pack(fill="both", expand=True) self.entertainment_tab = EntertainmentTab( self.tabview.tab("字节联盟-主播排行"), self.log_queue ) self.entertainment_tab.pack(fill="both", expand=True) # 设置默认选中的 Tab self.tabview.set("字节联盟-主播排行") # 日志区域 log_frame = ctk.CTkFrame(self) log_frame.pack(fill="both", expand=True, padx=10, pady=10) # 日志标题行 log_header = ctk.CTkFrame(log_frame, fg_color="transparent") log_header.pack(fill="x", padx=5, pady=(5, 0)) log_label = ctk.CTkLabel(log_header, text="运行日志:", anchor="w") log_label.pack(side="left") # 日志行数显示 self.log_count_label = ctk.CTkLabel(log_header, text="(0 行)", anchor="w", text_color="gray") self.log_count_label.pack(side="left", padx=5) self.log_text = ctk.CTkTextbox(log_frame, height=150) self.log_text.pack(fill="both", expand=True, padx=5, pady=5) # 按钮区域 btn_frame = ctk.CTkFrame(log_frame, fg_color="transparent") btn_frame.pack(fill="x", padx=5, pady=5) # 左侧按钮 clear_btn = ctk.CTkButton( btn_frame, text="清空日志", width=80, command=self.clear_log ) clear_btn.pack(side="left", padx=5) # 右侧按钮 exit_btn = ctk.CTkButton( btn_frame, text="退出程序", width=80, fg_color="red", hover_color="darkred", command=self.on_exit ) exit_btn.pack(side="right", padx=5) # 初始日志 self.log("程序启动完成，请选择爬虫类型并操作") def log(self, message): """写入日志""" timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.log_queue.put(f"[{timestamp}] {message}\n") def clear_log(self): """清空日志""" self.log_text.delete("1.0", "end") self.update_log_count() def update_log_count(self): """更新日志行数显示""" try: # 获取行数 content = self.log_text.get("1.0", "end-1c") line_count = content.count('\n') + 1 if content else 0 self.log_count_label.configure(text=f"({line_count} 行)") except: pass def truncate_log_if_needed(self): """如果日志超过最大行数，截断旧日志""" try: content = self.log_text.get("1.0", "end-1c") lines = content.split('\n') if len(lines) > self.MAX_LOG_LINES: # 保留最新的日志 keep_lines = lines[-self.MAX_LOG_LINES:] self.log_text.delete("1.0", "end") self.log_text.insert("1.0", '\n'.join(keep_lines)) # 添加截断提示 self.log_text.insert("1.0", f"[日志已截断，保留最新 {self.MAX_LOG_LINES} 行]\n") except: pass def process_log_queue(self): """处理日志队列""" message_count = 0 try: while True: message = self.log_queue.get_nowait() self.log_text.insert("end", message) message_count += 1 # 每处理50条消息检查一次是否需要截断 if message_count % 50 == 0: self.truncate_log_if_needed() except queue.Empty: pass if message_count > 0: self.log_text.see("end") self.update_log_count() # 处理完一批消息后检查截断 self.truncate_log_if_needed() # 每 100ms 检查一次 self.after(100, self.process_log_queue) def on_exit(self): """退出程序""" if messagebox.askyesno("确认", "确定要退出程序吗？"): # 恢复标准输出 sys.stdout = self.original_stdout sys.stderr = self.original_stderr self.destroy() def main(): app = App() app.mainloop() if __name__ == "__main__": main()