From 5d33a1d3d4922d0e295feec7d8c4a4637633e4fa Mon Sep 17 00:00:00 2001 From: xwfe Date: Mon, 2 Jun 2025 14:13:09 +0800 Subject: [PATCH] feat: dynamic scraper listing --- src/cli/scraper_cmd.rs | 31 ++++++++++++------- src/docs/angular/clean.rs | 34 +++++++++++++++++++++ src/docs/angular/entries.rs | 59 +++++++++++++++++++++++++++++++++++++ src/docs/angular/mod.rs | 16 ++++++++++ src/docs/angular/scraper.rs | 49 ++++++++++++++++++++++++++++++ src/docs/javascript/mod.rs | 7 +++++ src/docs/mod.rs | 7 +++-- src/docs/registry.rs | 11 +++++++ src/docs/typescript/mod.rs | 7 +++++ 9 files changed, 208 insertions(+), 13 deletions(-) create mode 100644 src/docs/angular/clean.rs create mode 100644 src/docs/angular/entries.rs create mode 100644 src/docs/angular/mod.rs create mode 100644 src/docs/angular/scraper.rs diff --git a/src/cli/scraper_cmd.rs b/src/cli/scraper_cmd.rs index f316347..7cd07ce 100644 --- a/src/cli/scraper_cmd.rs +++ b/src/cli/scraper_cmd.rs @@ -1,26 +1,35 @@ //! 文档抓取命令处理 use std::error::Error; +use crate::docs::{init, get_scraper, get_scraper_names}; /// 列出所有可用的抓取器 pub fn list_scrapers() -> Result<(), Box> { + init()?; + println!("可用的文档抓取器:"); - println!(" babel - Babel 文档抓取器"); - println!(" html - HTML 文档抓取器"); - println!(" css - CSS 文档抓取器"); - println!(" javascript - JavaScript 文档抓取器"); - println!(" typescript - TypeScript 文档抓取器"); - println!(" rust - Rust 文档抓取器"); - println!(" url - 通用URL抓取器 (需要指定URL)"); - + let names = get_scraper_names(); + if names.is_empty() { + println!(" 没有可用的文档抓取器"); + } else { + for name in names { + if let Some(scraper) = get_scraper(&name) { + let version = scraper.version().unwrap_or("latest"); + println!(" {} - {} 文档抓取器 (版本: {})", name, scraper.name(), version); + } + } + } + Ok(()) } /// 运行指定的抓取器 pub async fn run_scraper(name: &str, version: &str, output: Option<&str>) -> Result<(), Box> { + init()?; + println!("运行抓取器: {} (版本: {})", name, version); - - let output_str = output.unwrap_or(""); - + + let output_str = output.unwrap_or("docs"); + crate::scrape_async(name, version, output_str).await } diff --git a/src/docs/angular/clean.rs b/src/docs/angular/clean.rs new file mode 100644 index 0000000..1481e42 --- /dev/null +++ b/src/docs/angular/clean.rs @@ -0,0 +1,34 @@ +use crate::core::error::Result; +use crate::core::scraper::filter::{Filter, FilterContext}; +use nipper::Document; +use std::any::Any; + +#[derive(Clone, Default)] +pub struct AngularCleanHtmlFilter; + +impl AngularCleanHtmlFilter { + pub fn new() -> Self { + Self::default() + } +} + +impl Filter for AngularCleanHtmlFilter { + fn apply(&self, html: &str, _context: &mut FilterContext) -> Result { + let mut document = Document::from(html); + document.select("header").remove(); + document.select("footer").remove(); + Ok(document.html().to_string_lossy().into_owned()) + } + + fn box_clone(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn Any { + self as &dyn Any + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self as &mut dyn Any + } +} diff --git a/src/docs/angular/entries.rs b/src/docs/angular/entries.rs new file mode 100644 index 0000000..4c2a667 --- /dev/null +++ b/src/docs/angular/entries.rs @@ -0,0 +1,59 @@ +use crate::core::scraper::filter::{EntryDefinitionProvider, FilterContext}; +use crate::core::error::Result; +use crate::core::scraper::filter::Filter; +use nipper::Document; +use std::any::Any; + +#[derive(Clone, Default)] +pub struct AngularEntriesFilter; + +impl AngularEntriesFilter { + pub fn new() -> Self { + Self::default() + } +} + +impl EntryDefinitionProvider for AngularEntriesFilter { + fn get_entry_name(&self, document: &Document, _context: &FilterContext) -> Option { + let title = document.select("h1").text(); + if title.is_empty() { None } else { Some(title) } + } + + fn get_entry_type(&self, name: &str, context: &FilterContext) -> Option { + if context.current_path.contains("/api/") { + Some("API".to_string()) + } else if context.current_path.contains("/guide/") || context.current_path.contains("/tutorial") { + Some("Guide".to_string()) + } else { + Some("Page".to_string()) + } + } + + fn box_clone(&self) -> Box { + Box::new(self.clone()) + } +} + +impl Filter for AngularEntriesFilter { + fn apply(&self, html: &str, context: &mut FilterContext) -> Result { + let document = Document::from(html); + if let Some(name) = self.get_entry_name(&document, context) { + let type_str = self.get_entry_type(&name, context).unwrap_or_default(); + let entry = crate::core::models::Entry::new( + Some(name), + Some(context.current_path.clone()), + Some(type_str), + )?; + context.entries.push(entry); + } + Ok(html.to_string()) + } + + fn box_clone(&self) -> Box { + Box::new(self.clone()) + } + + fn as_any(&self) -> &dyn Any { self } + + fn as_any_mut(&mut self) -> &mut dyn Any { self } +} diff --git a/src/docs/angular/mod.rs b/src/docs/angular/mod.rs new file mode 100644 index 0000000..0f22c57 --- /dev/null +++ b/src/docs/angular/mod.rs @@ -0,0 +1,16 @@ +//! Angular 文档模块 + +mod scraper; +mod clean; +mod entries; + +pub use scraper::AngularScraper; +pub use clean::AngularCleanHtmlFilter; +pub use entries::AngularEntriesFilter; + +/// 注册 Angular 文档抓取器 +pub fn register() { + let scraper = AngularScraper::new("latest", "output/angular"); + crate::docs::registry::register_scraper("angular", scraper); + println!("注册了 Angular 文档抓取器"); +} diff --git a/src/docs/angular/scraper.rs b/src/docs/angular/scraper.rs new file mode 100644 index 0000000..03fbca0 --- /dev/null +++ b/src/docs/angular/scraper.rs @@ -0,0 +1,49 @@ +//! Angular 文档抓取器 + +use crate::core::error::Result; +use crate::core::scraper::base::{Scraper, UrlScraper}; +use async_trait::async_trait; +use crate::docs::angular::{AngularCleanHtmlFilter, AngularEntriesFilter}; + +/// Angular 文档爬虫 +pub struct AngularScraper { + scraper: UrlScraper, +} + +impl AngularScraper { + /// 创建新的 Angular 文档爬虫 + pub fn new(version: &str, output_path: &str) -> Self { + let base_url = "https://angular.io/docs"; + let mut scraper = UrlScraper::new("Angular", version, base_url, output_path); + + let initial_paths = vec![ + "/".to_string(), + "/guide/quickstart".to_string(), + "/tutorial".to_string(), + "/api".to_string(), + ]; + + scraper = scraper + .with_initial_paths(initial_paths) + .with_filter(Box::new(AngularCleanHtmlFilter::new())) + .with_filter(Box::new(AngularEntriesFilter::new())); + + Self { scraper } + } +} + +#[async_trait] +impl Scraper for AngularScraper { + fn name(&self) -> &str { + self.scraper.name() + } + + fn version(&self) -> &str { + self.scraper.version() + } + + async fn run(&mut self) -> Result<()> { + println!("开始抓取 Angular 文档..."); + self.scraper.run().await + } +} diff --git a/src/docs/javascript/mod.rs b/src/docs/javascript/mod.rs index 6b4f497..1316f9c 100644 --- a/src/docs/javascript/mod.rs +++ b/src/docs/javascript/mod.rs @@ -9,3 +9,10 @@ mod scraper; pub use clean::JavaScriptCleanHtmlFilter; pub use entries::JavaScriptEntriesFilter; pub use scraper::JavaScriptScraper; + +/// 注册 JavaScript 文档抓取器 +pub fn register() { + let scraper = JavaScriptScraper::new("latest", "output/javascript"); + crate::docs::registry::register_scraper("javascript", scraper); + println!("注册了 JavaScript 文档抓取器"); +} diff --git a/src/docs/mod.rs b/src/docs/mod.rs index 2c95f29..0fe0c60 100644 --- a/src/docs/mod.rs +++ b/src/docs/mod.rs @@ -16,6 +16,7 @@ pub mod html; pub mod javascript; pub mod rust; pub mod typescript; +pub mod angular; // 重新导出 pub use documentation::Documentation; @@ -35,8 +36,10 @@ pub use crate::{create_entries_filter, create_filter, register_doc}; pub fn init() -> crate::core::error::Result<()> { // 注册具体文档 babel::register(); - - // 可以在这里添加更多文档的注册 + javascript::register(); + typescript::register(); + angular::register(); + // 注册所有抓取器 register_all_scrapers()?; diff --git a/src/docs/registry.rs b/src/docs/registry.rs index 308c77f..a04562c 100644 --- a/src/docs/registry.rs +++ b/src/docs/registry.rs @@ -122,6 +122,11 @@ impl DocRegistry { self.scrapers.get(name) } + /// 获取所有已注册的抓取器名称 + pub fn get_scraper_names(&self) -> Vec { + self.scrapers.keys().cloned().collect() + } + /// 加载所有文档从磁盘 pub fn load_from_disk(&mut self, path: &str) -> Result<()> { use std::fs; @@ -328,6 +333,12 @@ pub fn get_scraper(name: &str) -> Option> { registry.get_scraper(name).map(|s| s.box_clone()) } +/// 获取所有已注册的抓取器名称 +pub fn get_scraper_names() -> Vec { + let registry = DOC_REGISTRY.lock().unwrap(); + registry.get_scraper_names() +} + /// 加载所有文档从磁盘 pub fn load_docs_from_disk(path: &str) -> Result<()> { let mut registry = DOC_REGISTRY.lock().unwrap(); diff --git a/src/docs/typescript/mod.rs b/src/docs/typescript/mod.rs index 00cd2a9..ef0234e 100644 --- a/src/docs/typescript/mod.rs +++ b/src/docs/typescript/mod.rs @@ -5,3 +5,10 @@ mod scraper; pub use scraper::TypeScriptScraper; + +/// 注册 TypeScript 文档抓取器 +pub fn register() { + let scraper = TypeScriptScraper::new("latest", "output/typescript"); + crate::docs::registry::register_scraper("typescript", scraper); + println!("注册了 TypeScript 文档抓取器"); +}