Implement dynamic scraper listing

src/cli/scraper_cmd.rs

@@ -1,26 +1,35 @@
 //! 文档抓取命令处理
 
 use std::error::Error;
+use crate::docs::{init, get_scraper, get_scraper_names};
 
 /// 列出所有可用的抓取器
 pub fn list_scrapers() -> Result<(), Box<dyn Error>> {
+    init()?;
+
     println!("可用的文档抓取器:");
-    println!("  babel - Babel 文档抓取器");
-    println!("  html - HTML 文档抓取器");
-    println!("  css - CSS 文档抓取器");
-    println!("  javascript - JavaScript 文档抓取器");
-    println!("  typescript - TypeScript 文档抓取器");
-    println!("  rust - Rust 文档抓取器");
-    println!("  url - 通用URL抓取器 (需要指定URL)");
-
+    let names = get_scraper_names();
+    if names.is_empty() {
+        println!("  没有可用的文档抓取器");
+    } else {
+        for name in names {
+            if let Some(scraper) = get_scraper(&name) {
+                let version = scraper.version().unwrap_or("latest");
+                println!("  {} - {} 文档抓取器 (版本: {})", name, scraper.name(), version);
+            }
+        }
+    }
+
     Ok(())
 }
 
 /// 运行指定的抓取器
 pub async fn run_scraper(name: &str, version: &str, output: Option<&str>) -> Result<(), Box<dyn Error>> {
+    init()?;
+
     println!("运行抓取器: {} (版本: {})", name, version);
-    
-    let output_str = output.unwrap_or("");
-    
+
+    let output_str = output.unwrap_or("docs");
+
     crate::scrape_async(name, version, output_str).await
 }

src/docs/angular/clean.rs

@@ -0,0 +1,34 @@
+use crate::core::error::Result;
+use crate::core::scraper::filter::{Filter, FilterContext};
+use nipper::Document;
+use std::any::Any;
+
+#[derive(Clone, Default)]
+pub struct AngularCleanHtmlFilter;
+
+impl AngularCleanHtmlFilter {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+impl Filter for AngularCleanHtmlFilter {
+    fn apply(&self, html: &str, _context: &mut FilterContext) -> Result<String> {
+        let mut document = Document::from(html);
+        document.select("header").remove();
+        document.select("footer").remove();
+        Ok(document.html().to_string_lossy().into_owned())
+    }
+
+    fn box_clone(&self) -> Box<dyn Filter> {
+        Box::new(self.clone())
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self as &dyn Any
+    }
+
+    fn as_any_mut(&mut self) -> &mut dyn Any {
+        self as &mut dyn Any
+    }
+}

src/docs/angular/entries.rs

@@ -0,0 +1,59 @@
+use crate::core::scraper::filter::{EntryDefinitionProvider, FilterContext};
+use crate::core::error::Result;
+use crate::core::scraper::filter::Filter;
+use nipper::Document;
+use std::any::Any;
+
+#[derive(Clone, Default)]
+pub struct AngularEntriesFilter;
+
+impl AngularEntriesFilter {
+    pub fn new() -> Self {
+        Self::default()
+    }
+}
+
+impl EntryDefinitionProvider for AngularEntriesFilter {
+    fn get_entry_name(&self, document: &Document, _context: &FilterContext) -> Option<String> {
+        let title = document.select("h1").text();
+        if title.is_empty() { None } else { Some(title) }
+    }
+
+    fn get_entry_type(&self, name: &str, context: &FilterContext) -> Option<String> {
+        if context.current_path.contains("/api/") {
+            Some("API".to_string())
+        } else if context.current_path.contains("/guide/") || context.current_path.contains("/tutorial") {
+            Some("Guide".to_string())
+        } else {
+            Some("Page".to_string())
+        }
+    }
+
+    fn box_clone(&self) -> Box<dyn EntryDefinitionProvider> {
+        Box::new(self.clone())
+    }
+}
+
+impl Filter for AngularEntriesFilter {
+    fn apply(&self, html: &str, context: &mut FilterContext) -> Result<String> {
+        let document = Document::from(html);
+        if let Some(name) = self.get_entry_name(&document, context) {
+            let type_str = self.get_entry_type(&name, context).unwrap_or_default();
+            let entry = crate::core::models::Entry::new(
+                Some(name),
+                Some(context.current_path.clone()),
+                Some(type_str),
+            )?;
+            context.entries.push(entry);
+        }
+        Ok(html.to_string())
+    }
+
+    fn box_clone(&self) -> Box<dyn Filter> {
+        Box::new(self.clone())
+    }
+
+    fn as_any(&self) -> &dyn Any { self }
+
+    fn as_any_mut(&mut self) -> &mut dyn Any { self }
+}

src/docs/angular/mod.rs

@@ -0,0 +1,16 @@
+//! Angular 文档模块
+
+mod scraper;
+mod clean;
+mod entries;
+
+pub use scraper::AngularScraper;
+pub use clean::AngularCleanHtmlFilter;
+pub use entries::AngularEntriesFilter;
+
+/// 注册 Angular 文档抓取器
+pub fn register() {
+    let scraper = AngularScraper::new("latest", "output/angular");
+    crate::docs::registry::register_scraper("angular", scraper);
+    println!("注册了 Angular 文档抓取器");
+}

src/docs/angular/scraper.rs

@@ -0,0 +1,49 @@
+//! Angular 文档抓取器
+
+use crate::core::error::Result;
+use crate::core::scraper::base::{Scraper, UrlScraper};
+use async_trait::async_trait;
+use crate::docs::angular::{AngularCleanHtmlFilter, AngularEntriesFilter};
+
+/// Angular 文档爬虫
+pub struct AngularScraper {
+    scraper: UrlScraper,
+}
+
+impl AngularScraper {
+    /// 创建新的 Angular 文档爬虫
+    pub fn new(version: &str, output_path: &str) -> Self {
+        let base_url = "https://angular.io/docs";
+        let mut scraper = UrlScraper::new("Angular", version, base_url, output_path);
+
+        let initial_paths = vec![
+            "/".to_string(),
+            "/guide/quickstart".to_string(),
+            "/tutorial".to_string(),
+            "/api".to_string(),
+        ];
+
+        scraper = scraper
+            .with_initial_paths(initial_paths)
+            .with_filter(Box::new(AngularCleanHtmlFilter::new()))
+            .with_filter(Box::new(AngularEntriesFilter::new()));
+
+        Self { scraper }
+    }
+}
+
+#[async_trait]
+impl Scraper for AngularScraper {
+    fn name(&self) -> &str {
+        self.scraper.name()
+    }
+
+    fn version(&self) -> &str {
+        self.scraper.version()
+    }
+
+    async fn run(&mut self) -> Result<()> {
+        println!("开始抓取 Angular 文档...");
+        self.scraper.run().await
+    }
+}

src/docs/javascript/mod.rs

@@ -9,3 +9,10 @@ mod scraper;
 pub use clean::JavaScriptCleanHtmlFilter;
 pub use entries::JavaScriptEntriesFilter;
 pub use scraper::JavaScriptScraper;
+
+/// 注册 JavaScript 文档抓取器
+pub fn register() {
+    let scraper = JavaScriptScraper::new("latest", "output/javascript");
+    crate::docs::registry::register_scraper("javascript", scraper);
+    println!("注册了 JavaScript 文档抓取器");
+}

src/docs/mod.rs

@@ -16,6 +16,7 @@ pub mod html;
 pub mod javascript;
 pub mod rust;
 pub mod typescript;
+pub mod angular;
 
 // 重新导出
 pub use documentation::Documentation;
@@ -35,8 +36,10 @@ pub use crate::{create_entries_filter, create_filter, register_doc};
 pub fn init() -> crate::core::error::Result<()> {
     // 注册具体文档
     babel::register();
-
-    // 可以在这里添加更多文档的注册
+    javascript::register();
+    typescript::register();
+    angular::register();
+
 
     // 注册所有抓取器
     register_all_scrapers()?;

src/docs/registry.rs

@@ -122,6 +122,11 @@ impl DocRegistry {
         self.scrapers.get(name)
     }
 
+    /// 获取所有已注册的抓取器名称
+    pub fn get_scraper_names(&self) -> Vec<String> {
+        self.scrapers.keys().cloned().collect()
+    }
+
     /// 加载所有文档从磁盘
     pub fn load_from_disk(&mut self, path: &str) -> Result<()> {
         use std::fs;
@@ -328,6 +333,12 @@ pub fn get_scraper(name: &str) -> Option<Box<dyn ScraperTrait>> {
     registry.get_scraper(name).map(|s| s.box_clone())
 }
 
+/// 获取所有已注册的抓取器名称
+pub fn get_scraper_names() -> Vec<String> {
+    let registry = DOC_REGISTRY.lock().unwrap();
+    registry.get_scraper_names()
+}
+
 /// 加载所有文档从磁盘
 pub fn load_docs_from_disk(path: &str) -> Result<()> {
     let mut registry = DOC_REGISTRY.lock().unwrap();

src/docs/typescript/mod.rs

@@ -5,3 +5,10 @@
 mod scraper;
 
 pub use scraper::TypeScriptScraper;
+
+/// 注册 TypeScript 文档抓取器
+pub fn register() {
+    let scraper = TypeScriptScraper::new("latest", "output/typescript");
+    crate::docs::registry::register_scraper("typescript", scraper);
+    println!("注册了 TypeScript 文档抓取器");
+}