diff --git a/openkb/cli.py b/openkb/cli.py index c9e543181..761c70bad 100644 --- a/openkb/cli.py +++ b/openkb/cli.py @@ -452,6 +452,59 @@ def add_single_file( return _add_single_file_locked(file_path, kb_dir, stage=stage, bundle=bundle) +def _delete_if_auto_cleanup_enabled( + file_path: Path, status: Literal["added", "skipped", "failed"], config: dict +) -> bool: + """Delete file if auto_delete_added_files is enabled and ingestion succeeded/skipped. + + Deletes on both "added" (successful ingestion) and "skipped" (duplicate already + in KB) to keep raw/ directory clean. Preserves files on "failed" to allow retries. + + Args: + file_path: Path to the file to potentially delete. + status: Result status from add_single_file ("added", "skipped", or "failed"). + config: Configuration dict (typically from resolve_effective_config). + + Returns: + True if file was deleted, False otherwise. + """ + if status in ("added", "skipped") and config.get("auto_delete_added_files", False): + try: + file_path.unlink(missing_ok=True) + return True + except Exception as exc: + logger.warning(f"Failed to delete {file_path.name}: {exc}") + return False + return False + + +def _cleanup_empty_directories(start_dir: Path) -> int: + """Recursively delete empty directories under start_dir. + + Walks from deepest subdirectories up, deleting directories that become + empty after file cleanup. + + Args: + start_dir: Root directory to clean up (e.g., kb_dir / "raw"). + + Returns: + Number of directories deleted. + """ + deleted_count = 0 + try: + for directory in sorted(start_dir.rglob("*"), key=lambda p: len(p.parts), reverse=True): + if directory.is_dir() and directory != start_dir: + try: + if not list(directory.iterdir()): + directory.rmdir() + deleted_count += 1 + except OSError: + pass + except Exception as exc: + logger.warning(f"Error during directory cleanup: {exc}") + return deleted_count + + def _add_single_file_locked( file_path: Path, kb_dir: Path, *, stage: bool = True, bundle=None ) -> Literal["added", "skipped", "failed"]: @@ -1086,6 +1139,10 @@ def add(ctx, path, from_pageindex_cloud): Alternatively, pass --from-pageindex-cloud to import a document that is already indexed in PageIndex Cloud, with no local file. Requires the PAGEINDEX_API_KEY environment variable. + + If ``auto_delete_added_files`` is enabled in config.yaml, files are + automatically deleted after ingestion (both on successful addition and + on skip/duplicate). """ kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override")) if kb_dir is None: @@ -1106,6 +1163,8 @@ def add(ctx, path, from_pageindex_cloud): click.echo("Provide a PATH or use --from-pageindex-cloud .") return + config = resolve_effective_config(kb_dir)[0] + # URL ingest: download into raw/ first, then call add_single_file explicitly. # Keep staged conversion enabled so converted source artifacts do not touch # the live KB before the mutation snapshot exists. The tri-state outcome @@ -1123,6 +1182,8 @@ def add(ctx, path, from_pageindex_cloud): # indexing has already succeeded but compilation didn't. if outcome == "skipped": fetched.unlink(missing_ok=True) + else: + _delete_if_auto_cleanup_enabled(fetched, outcome, config) return target = Path(path) @@ -1143,7 +1204,8 @@ def add(ctx, path, from_pageindex_cloud): click.echo(f"Found {total} supported file(s) in {path}.") for i, f in enumerate(files, 1): click.echo(f"\n[{i}/{total}] ", nl=False) - add_single_file(f, kb_dir) + outcome = add_single_file(f, kb_dir) + _delete_if_auto_cleanup_enabled(f, outcome, config) else: if target.suffix.lower() not in SUPPORTED_EXTENSIONS: click.echo( @@ -1151,7 +1213,68 @@ def add(ctx, path, from_pageindex_cloud): f"Supported: {', '.join(sorted(SUPPORTED_EXTENSIONS))}" ) return - add_single_file(target, kb_dir) + outcome = add_single_file(target, kb_dir) + _delete_if_auto_cleanup_enabled(target, outcome, config) + + +@cli.command() +@click.pass_context +@_with_kb_lock(exclusive=True) +def add_all(ctx): + """Process all files in the ``raw/`` directory and add them to the knowledge base. + + This command walks the ``raw/`` directory recursively for all supported + document types and ingests them into the KB. If ``auto_delete_added_files`` + is enabled in config.yaml, files are automatically deleted after ingestion + (both on successful addition and on skip/duplicate), and empty subdirectories + are cleaned up. + + Returns a summary of the operation (added, skipped, failed, deleted counts). + """ + kb_dir = _find_kb_dir(ctx.obj.get("kb_dir_override")) + if kb_dir is None: + click.echo("No knowledge base found. Run `openkb init` first.") + return + + raw_dir = kb_dir / "raw" + if not raw_dir.is_dir(): + click.echo(f"No raw/ directory found at {raw_dir}") + return + + files = [ + f + for f in sorted(raw_dir.rglob("*")) + if f.is_file() and f.suffix.lower() in SUPPORTED_EXTENSIONS + ] + if not files: + click.echo("No supported files found in raw/ directory.") + return + + config = resolve_effective_config(kb_dir)[0] + total = len(files) + added = skipped = failed = deleted = dirs_deleted = 0 + + click.echo(f"Processing {total} file(s) from raw/ directory...") + for i, f in enumerate(files, 1): + click.echo(f"\n[{i}/{total}] ", nl=False) + outcome = add_single_file(f, kb_dir) + if outcome == "added": + added += 1 + elif outcome == "skipped": + skipped += 1 + else: + failed += 1 + if _delete_if_auto_cleanup_enabled(f, outcome, config): + deleted += 1 + + # Clean up empty subdirectories if auto-cleanup is enabled + if config.get("auto_delete_added_files", False): + dirs_deleted = _cleanup_empty_directories(raw_dir) + + summary = f"Added: {added}, Skipped: {skipped}, Failed: {failed}, Deleted: {deleted}" + if dirs_deleted > 0: + summary += f", Empty dirs cleaned: {dirs_deleted}" + click.echo(f"\n\nSummary: {summary}") def _stream_to_tty() -> bool: diff --git a/openkb/config.py b/openkb/config.py index 95ca8691f..efd7ac382 100644 --- a/openkb/config.py +++ b/openkb/config.py @@ -36,6 +36,7 @@ # global/KB list overrides it wholesale; resolve_entity_types cleans the # effective value on read. "entity_types": list(DEFAULT_ENTITY_TYPES), + "auto_delete_added_files": False, } GLOBAL_CONFIG_DIR = Path.home() / ".config" / "openkb"