diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index 8d87b1d..0000000 --- a/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -node_modules/* diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index 29f384f..0000000 --- a/.eslintrc.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "root": true, - "env": { - "es6": true, - "node": true, - "mocha": true - }, - "parserOptions": { - "ecmaVersion": 8 - }, - "extends": [ - "eslint:recommended" - ] -} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..2c24a8c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [main, v2] + pull_request: + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + node: [20, 22, 24] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + cache: npm + - run: npm ci + - run: npm run lint + - run: npm run test:types + - run: npm test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d705087..0000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -branches: - only: - - main -dist: xenial -language: node_js -node_js: - - 10 - - 11 - - 12 - - 13 - - 14 - - node -os: - - linux - - osx -install: - npm install --dev -script: - - npm run lint - - npm run test:sitemap - - npm run test:urls diff --git a/CHANGELOG.md b/CHANGELOG.md index 9928fd7..8093fe3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,50 @@ # check-http-status changelog +## 2.0.0 - Oct 2, 2026 + +### New + +- Crawl a whole website (`crawl`) and report every link's status with the pages it was found on and the link text +- Command line tool: `npx check-http-status ` with `crawl` and `check` commands +- Include / exclude rules (substring, wildcard or regex). Excluded URLs are listed and checked, but not crawled +- External links are checked but never crawled +- Sitemap index (``) and gzipped sitemap (`.xml.gz`) support +- Orphan page detection (pages in the sitemap that aren't linked from anywhere) +- Optional asset checks (images, scripts, stylesheets, icons, iframes) +- New report formats: interactive HTML and JSON; reports list every URL found once (URL, Found On, Redirect To, Status, …); Excel report has Summary, All URLs, Issues and External Links sheets +- `maxPages`, `maxDepth`, `concurrency`, `maxRedirects`, `subdomains`, `ignoreQuery`, `onProgress` options +- `--fail` flag to fail CI builds when broken links are found +- Upload reports to Google Drive (`--drive` / `googleDrive`), with Excel/CSV converted to Google Sheets; `drive-login` command; service account support for CI +- Upload reports to OneDrive (`--onedrive` / `oneDrive`), personal or work/school; `onedrive-login` command +- Public "anyone with the link" sharing for uploads (`--public`, `--drive-public`, `--onedrive-public`) +- Retries for 429/502/503/504 and dropped connections, honouring `Retry-After` (`retries`, `retryDelay`) +- `delay` between requests to the same site, and a custom `userAgent` +- For apps built on the library: `signal` (stop a crawl), `onResult` (live results), `recheck()` and `writeReport()` +- Optional robots.txt support (`--respect-robots` / `respectRobots`), including Crawl-delay +- Config file (`check-http-status.config.json` or `--config `) with a JSON Schema; `loadConfig()` / `findConfig()` +- TypeScript type definitions +- Bot-protection challenges (Cloudflare, DataDome, Akamai, PerimeterX, Sucuri) are reported as `Blocked` instead of broken links +- The terminal lists issues even when reports are saved with `-o` +- `checkHttpStatus()` returns `{ summary, results }` + +### Fixed + +- Requests ran in batches of 10 that waited for the slowest request; now a worker pool keeps every slot busy +- Redirect loops caused infinite recursion +- Relative `Location` headers (e.g. `/new-page`) were reported as errors +- Sitemap index files crashed the sitemap reader +- Progress output crashed when stdout was not a terminal (CI, pipes) +- De-duplicating large sitemaps was O(n²) +- HTTP Basic auth credentials are no longer sent to external sites +- Export location no longer requires a trailing slash + +### Breaking + +- Requires Node.js 18.17+ +- Throws errors instead of calling `process.exit()` +- Excel/CSV columns changed (one row per URL with its full redirect chain) +- Replaced `axios`, `columnify` and `xml2js` with built-in `fetch` and `cheerio` + ## 1.3.0 - Oct 15, 2021 ### Enhancement diff --git a/README.md b/README.md index 7d5c544..c4a424a 100644 --- a/README.md +++ b/README.md @@ -2,141 +2,424 @@ [![NPM version][npm-image]][npm-url] [![Downloads][downloads-image]][npm-url] -[![Build Status][travis-image]][travis-url] +[![CI][ci-image]][ci-url] -Easily check status codes, response headers, and redirect chains in `Node.js` -similar as done from the [httpstatus](https://httpstatus.io/) website. +Crawl a website or check a list of URLs, and get HTTP status codes, full +redirect chains, and broken links, along with **the pages each broken link was found on**. +Export to Excel, CSV, JSON or a shareable, interactive HTML report. -When the site is on VPN so this is where it plays an important role. You can -simply connect your system/machine with VPN and run this package locally so it -can check the status of your VPN connected URL(s). +**[Website & live demo report →](https://trunkcode.github.io/check-http-status/)** -It can also check the website that are secured with HTTP Authentication. +```bash +npx check-http-status https://www.example.com --exclude /docs -o report.xlsx +``` + +It runs on your own machine (macOS, Windows, Linux), so it can also check +staging sites, password-protected sites (HTTP Basic auth) and sites that are +only reachable over a VPN. + +## Features + +- **Crawl a whole site** from a start URL, breadth-first, with `maxPages` and `maxDepth` limits. +- **Where was it found?** Every broken link and redirect lists the pages that link to it, with the link text. +- **Redirect chains:** every hop, the final URL and status, and redirect-loop detection. +- **External links** are checked but never crawled. +- **No false alarms from bot protection:** links to sites behind Cloudflare, DataDome and similar services, which block all automated checkers, are reported as **Blocked** (check them in a browser) instead of broken. +- **Include / exclude rules** (substring, `*` wildcard or `/regex/`). Excluded URLs are still listed and checked, just not crawled. +- **Sitemaps:** ``, `` (Yoast, Rank Math, WordPress core) and `.xml.gz`. Combined with a crawl, it flags **orphan pages** (in the sitemap but not linked from anywhere). +- **Assets** (optional): images, scripts, stylesheets, icons, iframes. +- **Reports:** Excel, CSV, JSON and HTML. Reports always list **every URL found, once**. The first columns are **URL**, **Found On** (every page that links to it, in one cell) and **Redirect To**, followed by the status. The Excel workbook has these sheets: + - **Summary:** totals per status. + - **All URLs:** every crawled page, asset and external link. + - **Issues:** redirects, 4xx, 5xx and connection errors. + - **External Links:** external links (checked, not crawled). +- **Google Drive & OneDrive:** upload reports straight to the cloud, optionally with a public link. On Google Drive, spreadsheets become Google Sheets. +- **Polite and resilient:** retries temporary errors (honouring `Retry-After`), with an optional delay between requests, a custom User-Agent and optional robots.txt support. +- **Config file** shared by the CLI, the GitHub Action and your own apps, with a JSON Schema for editor autocomplete. +- **TypeScript** type definitions included. +- **CI friendly:** `--fail` exits with code 1 when broken links are found. ## Install -Via `npm` +Requires Node.js 18.17 or newer. ```bash +# Run without installing +npx check-http-status --help + +# Or install globally / as a dev dependency +npm install -g check-http-status npm install check-http-status --save-dev ``` -Via Yarn +## Command line ```bash -yarn add check-http-status --dev +# Crawl a site (the terminal lists issues only; saved reports list every URL) +check-http-status https://www.example.com + +# Don't crawl /docs, but still list and check those links; save Excel + HTML reports +check-http-status crawl example.com -e /docs -o report.xlsx -o report.html + +# Optional: add a sitemap index → also finds orphan pages +check-http-status crawl example.com -s https://example.com/sitemap_index.xml + +# Only check the given URLs (no crawling) +check-http-status check https://example.com/old-page https://example.com/pricing --all + +# Check every URL in a sitemap, or in a text file (one URL per line) +check-http-status check -s https://example.com/sitemap.xml --issues-only +check-http-status check -f urls.txt -o results.csv + +# Password-protected staging site +CHS_AUTH=user:pass check-http-status https://staging.example.com + +# Fail a CI job if broken links are found +check-http-status https://www.example.com --fail -o report.html ``` -## Examples +Example output: -### Status Code of the Sitemap(s) URL(s) +``` +301 https://example.com/summer-sale → https://example.com/new-arrivals + found on https://example.com/ ("Summer sale") +301 https://example.com/products/sleeping-bags → https://example.com/products/sleeping-bags/ ⇒ https://example.com/collections/sleeping-bags (404, 2 hops) + found on https://example.com/products/ ("Sleeping bags") +410 https://example.com/blog/2019/old-news + found on https://example.com/blog/ ("Old news") +500 https://example.com/products/stoves + found on https://example.com/products/ ("Stoves") + found on https://example.com/blog/packing-list ("Stoves") -```node +36 URLs (17 pages crawled, 2s): 22 OK, 7 redirects, 4 not found, 2 other 4xx/5xx, 1 errors +``` + +Run `check-http-status --help` for every option. + +## Node.js + +```js const checkHttpStatus = require('check-http-status'); -checkHttpStatus({ - 'sitemaps': [ - 'https://www.trunkcode.com/page-sitemap.xml', - 'https://www.trunkcode.com/post-sitemap.xml' +// Just pass the website URL - pages are found by crawling, no sitemap needed. +const report = await checkHttpStatus({ + 'crawl': 'https://www.example.com', + 'exclude': ['/docs'], + 'maxPages': 2000, + 'export': [ + { 'format': 'xlsx', 'location': '/Users/me/Desktop/' }, + { 'format': 'html', 'file': '/Users/me/Desktop/report.html' } ], - 'skip200': true, // Do not report the URLs having HTTP code 200. - 'export': { - 'format': 'xlsx', - 'location': '/Users/trunkcode/Desktop/', - }, 'options': { - 'auth': { - 'password': 'Testing1234', - 'username': 'trunkcode' - }, - 'headers': { - 'Accept': 'text/html', - } + 'auth': { 'username': 'user', 'password': 'pass' }, + 'headers': { 'Accept-Language': 'en' }, + 'timeout': 15000 } }); + +console.log(report.summary.counts); // { OK: 412, Redirect: 9, 'Not Found': 3, ... } + +for (const result of report.results.filter((r) => r.category === 'Not Found')) { + console.log(result.url, 'found on', result.foundOn.map((source) => source.page)); +} ``` -### Status Code of the particular URL(s) +### Optional: add a sitemap -```node -const checkHttpStatus = require('check-http-status'); +Sitemaps are never required. Adding one to a crawl also checks sitemap pages that +nothing links to, and flags them as orphans: +```js checkHttpStatus({ - 'urls': [ - 'http://trunkcode.com/', - 'https://example.com/', - 'https://example1234.com/', - 'https://www.trunkcode.com/', - 'https://www.trunkcode.com/test/' - ], - 'skip200': true, // Do not report the URLs having HTTP code 200. + 'crawl': 'https://www.example.com', + 'sitemaps': ['https://www.example.com/sitemap_index.xml'] +}); +``` + +### Check a list of URLs (v1 style) + +```js +checkHttpStatus({ + 'sitemaps': ['https://www.trunkcode.com/page-sitemap.xml'], + 'urls': ['https://example.com/', 'https://www.trunkcode.com/test/'], + 'skip200': true, 'export': { 'format': 'xlsx', - 'location': '/Users/trunkcode/Desktop/', - }, - 'options': { - 'auth': { - 'password': 'Testing1234', - 'username': 'trunkcode' - }, - 'headers': { - 'Accept': 'text/html', - } + 'location': '/Users/trunkcode/Desktop/' } }); ``` -### Status Code of the Sitemap(s) URL(s) with particular URL(s) +### Parameters -```node -const checkHttpStatus = require('check-http-status'); +| Parameter | Type | Default | Description | +|---|---|---|---| +| `crawl` | String / Array | | Website URL to crawl. Pages are discovered by following internal links. | +| `urls` | Array | | URLs to check (with their redirects), without crawling. | +| `sitemaps` | Array | | *Optional.* Sitemap / sitemap index URLs. With `crawl`, their pages are crawled too and orphans are flagged; without it, their URLs are checked. | -checkHttpStatus({ - 'sitemaps': [ - 'https://www.trunkcode.com/page-sitemap.xml', - 'https://www.trunkcode.com/post-sitemap.xml' - ], - 'urls': [ - 'http://trunkcode.com/', - 'https://example.com/', - 'https://example1234.com/', - 'https://www.trunkcode.com/', - 'https://www.trunkcode.com/test/' +Provide at least one of `crawl`, `urls` or `sitemaps`. For a full site audit, `crawl` on its own is enough. +| `include` | Array / String | | Only crawl URLs matching one of these rules. | +| `exclude` | Array / String | | Don't crawl URLs matching these rules. They are still listed and checked. | +| `maxPages` | Number | `1000` | Maximum pages to crawl. Later pages are listed as "Skipped (max pages)". | +| `maxDepth` | Number | unlimited | Maximum link depth from the start URL. | +| `checkExternal` | Boolean | `true` | Check the status of external links. Excluded pages and assets are always checked. | +| `checkAssets` | Boolean | `false` | Also check images, scripts, stylesheets, icons and iframes. | +| `subdomains` | Boolean | `false` | Treat subdomains of the start URL as internal. | +| `ignoreQuery` | Boolean | `false` | Strip query strings before comparing URLs. | +| `respectRobots` | Boolean | `false` | Don't request URLs disallowed by the site's robots.txt (listed as "Blocked by robots.txt"), and honour `Crawl-delay`. | +| `concurrency` | Number | `10` | Parallel requests. | +| `delay` | Number | `0` | Minimum milliseconds between requests to the same site. Use it for sites that rate-limit. | +| `retries` | Number | `2` | Retries for 429, 502, 503, 504 and dropped connections, honouring `Retry-After`. 404s, DNS failures and refused connections are not retried. | +| `retryDelay` | Number | `1000` | Wait before the first retry (milliseconds), doubled for each further retry. | +| `userAgent` | String | identifies check-http-status | User-Agent header. | +| `maxRedirects` | Number | `10` | Redirects to follow per URL. | +| `skip200` | Boolean | `false` | Leave 200 URLs out of the console output and the URL sheet/CSV (v1 behaviour). By default reports contain every URL. | +| `export` | Object / Array | | `{ format, location }` or `{ format, file }`. Formats: `xlsx`, `csv`, `json`, `html`. | +| `options.auth` | Object | | `{ username, password }` for HTTP Basic auth. Only sent to the crawled/checked hosts. | +| `options.headers` | Object | | Extra request headers. | +| `options.timeout` | Number | `15000` | Timeout per request in milliseconds. | +| `googleDrive` | Boolean / Object | | Upload reports to Google Drive: `{ folderId, folderName, convert, public, credentials, keyFile }`. See [Upload reports to Google Drive](#upload-reports-to-google-drive). | +| `oneDrive` | Boolean / Object | | Upload reports to OneDrive: `{ folderPath, public, credentials, clientId, refreshToken, tenant }`. See [Upload reports to OneDrive](#upload-reports-to-onedrive). | +| `silent` | Boolean | `false` | No console output. | +| `signal` | AbortSignal | | Stop the crawl. Everything checked so far is returned; unfinished URLs are "Not checked". | +| `onResult` | Function | | Called with each result as soon as its URL is checked (live results). | +| `onProgress` | Function | | Called with progress stats after each URL. | + +Rules match as a case-insensitive substring (`/docs`), a wildcard +(`*/products/*`) or a regular expression (`/\/page\/\d+/`). + +### TypeScript + +Type definitions are included: + +```ts +import checkHttpStatus = require('check-http-status'); + +const report: checkHttpStatus.Report = await checkHttpStatus({ crawl: 'https://www.example.com' }); +const broken: checkHttpStatus.Result[] = report.results.filter((r) => r.category === 'Not Found'); +``` + +### Use it in your own app + +`signal`, `onResult` and `recheck()` are made for apps built on top of the +library (like a desktop UI): stop a crawl, show results as they arrive, and +re-check URLs after fixing them without crawling again. + +```js +const { checkHttpStatus, recheck, writeReport } = require('check-http-status'); + +const controller = new AbortController(); +stopButton.onclick = () => controller.abort(); + +let report = await checkHttpStatus({ + 'crawl': 'https://www.example.com', + 'signal': controller.signal, + 'silent': true, + 'onResult': (result) => table.addRow(result), + 'onProgress': (progress) => statusBar.update(progress) +}); + +// Later, after fixing some links. Where each URL was found is kept from the crawl. +report = await recheck(report, ['https://www.example.com/old-page']); +await writeReport(report, 'report.xlsx', 'xlsx'); +``` + +### Result + +`checkHttpStatus()` resolves to `{ summary, results }`. Each result has: + +| Field | Description | +|---|---| +| `url` | The URL. | +| `status`, `statusText`, `category` | e.g. `404`, `Not Found`, and one of `OK`, `Redirect`, `Not Found`, `Client Error`, `Server Error`, `Error`, `Blocked`, `Not checked`. | +| `botProtection` | Set when a bot-protection service (Cloudflare, DataDome, Akamai, PerimeterX, Sucuri) answered instead of the page. These are reported as `Blocked`: check them in a browser. They are not counted as broken and don't trigger `--fail`. | +| `error` | Connection error (timeout, DNS, TLS…) if the request failed. | +| `type`, `linkType`, `scope` | `internal`/`external`, `page`/`asset`, and whether it was crawled, excluded, checked or skipped. | +| `redirectTo`, `redirectChain`, `finalUrl`, `finalStatus`, `redirectCount` | Redirect details. `note` is set for loops or too many redirects. | +| `foundOn` | `[{ page, anchors, count, kind }]`, where `kind` is `link`, `redirect` or `sitemap`. | +| `title`, `contentType`, `responseTime`, `depth`, `inSitemap`, `orphan` | Page details. | + +## Config file + +Save your settings in **`check-http-status.config.json`**. The CLI picks it up +from the current folder automatically, or use `--config `. The same file +works for the GitHub Action and for apps that call `loadConfig()`. + +```json +{ + "$schema": "https://unpkg.com/check-http-status/schema.json", + "crawl": "https://www.example.com", + "exclude": ["/docs", "/wp-admin"], + "maxPages": 2000, + "respectRobots": true, + "delay": 250, + "export": [ + { "format": "xlsx", "file": "reports/link-report.xlsx" }, + { "format": "html", "file": "reports/link-report.html" } ], - 'skip200': true, // Do not report the URLs having HTTP code 200. - 'export': { - 'format': 'xlsx', - 'location': '/Users/trunkcode/Desktop/', - }, - 'options': { - 'auth': { - 'password': 'Testing1234', - 'username': 'trunkcode' - }, - 'headers': { - 'Accept': 'text/html', - } - } + "fail": true +} +``` + +```bash +check-http-status # uses check-http-status.config.json +check-http-status --max-pages 50 # flags override the file +check-http-status -e /blog # -i / -e / -s add to the file's lists +check-http-status --config audits/client-a.json +``` + +- **Options:** it accepts the same options as the Node.js API (see [Parameters](#parameters)), plus `"fail": true`. +- **Units:** durations are in **milliseconds**, as in Node.js. On the command line they are seconds. +- **Paths:** relative paths (report files, `googleDrive.keyFile`) are relative to the config file. +- **Typos:** unknown options are reported as errors. +- **Editor support:** the `$schema` line gives autocomplete and validation in VS Code and other editors. +- **Secrets:** keep passwords and tokens out of the file and use environment variables (`CHS_AUTH`, `CHS_GOOGLE_CREDENTIALS`, `CHS_ONEDRIVE_CREDENTIALS`). + +```js +const { checkHttpStatus, loadConfig } = require('check-http-status'); +const report = await checkHttpStatus({ ...loadConfig('check-http-status.config.json'), 'silent': true }); +``` + +## robots.txt + +By default every link is checked, which is usually what you want when auditing +your own site. Add `--respect-robots` (or `"respectRobots": true`) to skip URLs +that your robots.txt disallows: + +- Blocked URLs are still listed, with the pages that link to them, as **"Blocked by robots.txt"**, but are never requested. +- **`Crawl-delay`** is honoured (up to 30 seconds). +- **Which rules apply:** the rules for your User-Agent's product token (`check-http-status`, or e.g. `AuditBot` for `--user-agent "AuditBot/1.0"`) are used if present, otherwise the `*` rules. +- **Scope:** only the site being crawled is affected. External links are single status checks. +- **Missing or unreachable robots.txt:** a missing robots.txt (404) allows everything. As RFC 9309 requires, one that can't be fetched (5xx or network error) blocks everything. + +## Upload reports to Google Drive + +Add `--drive` to upload the report when the crawl finishes. Excel and CSV +reports are converted to **Google Sheets** (use `--no-convert` to keep the +file). Without `-o`, an Excel report is created and uploaded. + +```bash +check-http-status https://www.example.com --drive +check-http-status https://www.example.com -o report.xlsx -o report.html --drive +``` + +Files go into a **"HTTP Status Reports"** folder that the tool creates in your +Drive (rename it with `--drive-folder-name`). By default the tool can only see +files and folders it created itself, never the rest of your Drive. + +### One-time setup (your own Google account) + +1. In the [Google Cloud console](https://console.cloud.google.com/), create a project and **enable the Google Drive API**. +2. Set up the **OAuth consent screen** (External), add your email as a test user, then **publish** the app. While it's in "Testing", Google expires the login after 7 days. +3. Under **Credentials**, create an **OAuth client ID** of type **Desktop app**. +4. Sign in once. The refresh token is saved to `~/.config/check-http-status/` (on Windows, `%APPDATA%\check-http-status\`): + + ```bash + check-http-status drive-login --client-id --client-secret + ``` + +To upload into **an existing folder of yours**, sign in with +`drive-login --full-access`, then pass the folder ID from its URL +(`drive.google.com/drive/folders/`) with `--drive-folder `. + +### CI / GitHub Actions + +Either reuse your login: store the contents of the saved +`google-credentials.json` as a secret named `CHS_GOOGLE_CREDENTIALS`. Or use a +**service account**: store its JSON key as `CHS_GOOGLE_CREDENTIALS` (or point +`GOOGLE_APPLICATION_CREDENTIALS` / `--drive-key` at the file), share the target +folder with the service account's email, and pass `--drive-folder`. Service +accounts have no storage of their own, so the folder must be in a **Shared +Drive** (Google Workspace). + +```yaml +- run: npx check-http-status https://www.example.com --drive --drive-folder ${{ vars.DRIVE_FOLDER_ID }} + env: + CHS_GOOGLE_CREDENTIALS: ${{ secrets.CHS_GOOGLE_CREDENTIALS }} +``` + +In Node.js, pass `googleDrive`. The uploaded files are returned in `report.uploads`: + +```js +const report = await checkHttpStatus({ + 'crawl': 'https://www.example.com', + 'googleDrive': { 'folderId': '1AbC…', 'convert': true } // or `googleDrive: true` +}); + +console.log(report.uploads); // [{ service, name, url, public, id, file }] +console.log(report.uploadError); // set if the upload failed (the crawl results are still returned) +``` + +### Share with anyone + +Add `--public` (or `'public': true`) to make each uploaded file viewable by +anyone with the link. The link is printed and returned in `report.uploads`. +Google Workspace and Microsoft 365 admins can block public links. If yours does, +the upload still succeeds and you get an error explaining why. + +```bash +check-http-status https://www.example.com --drive --public +``` + +## Upload reports to OneDrive + +Works with personal OneDrive and OneDrive for work or school. Excel reports open +directly in Excel for the web. + +```bash +check-http-status https://www.example.com --onedrive +check-http-status https://www.example.com -o report.xlsx --onedrive --onedrive-public +``` + +Files go into the app's own folder, **Apps/<your app name>**. The tool can't +see anything else in your OneDrive. + +### One-time setup + +1. In the [Microsoft Entra admin center](https://entra.microsoft.com/) (or Azure portal), open **App registrations**, then **New registration**. + Choose who can sign in: personal accounts, work accounts, or both. +2. Under **Authentication**, add the platform **Mobile and desktop applications** with the redirect URI `http://localhost`. +3. Copy the **Application (client) ID**. No client secret is needed. +4. Sign in once: + + ```bash + check-http-status onedrive-login --client-id + # Work/school only: --tenant . Personal only: --tenant consumers + ``` + +To upload into **your own folder** (e.g. `Documents/SEO`), sign in with +`onedrive-login --full-access` and pass `--onedrive-folder "Documents/SEO"`. +Missing folders are created. + +For CI, store the contents of the saved `onedrive-credentials.json` as a +secret named `CHS_ONEDRIVE_CREDENTIALS`. + +In Node.js: + +```js +const report = await checkHttpStatus({ + 'crawl': 'https://www.example.com', + 'oneDrive': { 'folderPath': 'Documents/SEO', 'public': true } // or `oneDrive: true` }); ``` -## Parameters +## Upgrading from v1 -| Attributes | Type | Required | Default | Description | -|:----------:|:-------:|:--------:|:-------:|:------------------------------------------------------------------------------------------------:| -| sitemaps | Array | Yes | | Sitemap(s) URL(s) where the Actual site URL(s) needs to be fetched for checking the HTTP Status. | -| urls | Array | Yes | | URL(s) for which HTTP Status needs to be checked. | -| skip200 | Boolean | No | `false` | Whether to list the HTTP status `200` URL(s) or not. | -| export | Object | No | `{}` | Whether to export the status report or not. By default it logs the report on the screen. | -| options | Object | No | `{}` | Define options like HTTP Auth credentials if the site is locked or headers etc. | +v1 configs keep working. Changes in v2: -**NOTE:** `sitemaps` or `urls` is required. You can define both parameters as -well to fetch URL(s) from sitemap and the URL(s) that are not listed in the -ssitemap, you can provide them separately. +- Node.js 18.17+ is required (it uses the built-in `fetch`, so axios is no longer a dependency). +- `checkHttpStatus()` now **returns** `{ summary, results }`. +- Invalid config **throws** instead of calling `process.exit()`. +- Excel/CSV columns changed: one row per URL with its full redirect chain and the pages it was found on, plus Summary, Issues and External Links sheets. +- `export.location` no longer needs a trailing slash. + +## License + +MIT [npm-image]: https://img.shields.io/npm/v/check-http-status.svg [npm-url]: https://www.npmjs.com/package/check-http-status [downloads-image]: https://img.shields.io/npm/dt/check-http-status.svg - -[travis-image]: https://api.travis-ci.com/trunkcode/check-http-status.svg?branch=main -[travis-url]: https://travis-ci.com/trunkcode/check-http-status +[ci-image]: https://github.com/trunkcode/check-http-status/actions/workflows/ci.yml/badge.svg +[ci-url]: https://github.com/trunkcode/check-http-status/actions/workflows/ci.yml diff --git a/bin/check-http-status.js b/bin/check-http-status.js new file mode 100755 index 0000000..e5b81e1 --- /dev/null +++ b/bin/check-http-status.js @@ -0,0 +1,441 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { parseArgs } = require('util'); +const checkHttpStatus = require('../index'); +const { findConfig, loadConfig } = require('../lib/config'); +const pkg = require('../package.json'); + +const HELP = `check-http-status v${pkg.version} + +Crawl a website or check a list of URLs for HTTP status codes, redirect chains +and broken links, and see which pages link to them. + +Usage + check-http-status crawl [options] Crawl a website starting from + check-http-status check [options] Check only the given URLs / sitemaps + check-http-status [options] Same as "crawl" + check-http-status drive-login [options] Connect your Google Drive account (once) + check-http-status onedrive-login [options] Connect your OneDrive account (once) + +Crawl options + -i, --include Only crawl URLs containing (repeatable) + -e, --exclude Don't crawl URLs containing ; they are still listed + and their status is checked (repeatable) + Rules may use * wildcards or /regex/ + --max-pages Max pages to crawl (default 1000) + --max-depth Max link depth from the start URL (default unlimited) + --no-external Don't check the status of external links (excluded + pages and assets are still checked) + --assets Also check images, scripts, stylesheets and iframes + --subdomains Treat subdomains as internal + --ignore-query Ignore query strings (?a=b) when comparing URLs + --respect-robots Don't request URLs that the site's robots.txt disallows + (they are listed as "Blocked by robots.txt"); also + honours Crawl-delay. Off by default + +Config file + --config Load settings from a JSON config file. Flags override it; + -i/-e/-s add to its lists. Without --config, + check-http-status.config.json in the current folder is + used if present + --no-config Ignore check-http-status.config.json + +Optional input + -s, --sitemap Read URLs from a sitemap or sitemap index (repeatable). + Not needed for crawling: pages are found from links. + With "crawl", sitemap pages are crawled too and pages + not linked from anywhere are reported as orphans + -f, --file Read URLs from a text file, one per line + +Requests + -c, --concurrency Parallel requests (default 10) + -t, --timeout Timeout per request (default 15) + --delay Wait between requests to the same site, e.g. 0.5 + (default 0). Use it if the site rate-limits you + --retries Retries for 429/502/503/504 and dropped connections, + honouring Retry-After (default 2) + --max-redirects Max redirects to follow per URL (default 10) + --user-agent User-Agent header (default identifies check-http-status) + -a, --auth HTTP Basic auth (only sent to the site being checked). + Can also be set with the CHS_AUTH environment variable + -H, --header Extra request header (repeatable) + +Output + -o, --output Save a report; format from the extension: + .xlsx, .csv, .json or .html (repeatable) + --issues-only Leave 200 OK URLs out of the terminal output and reports + (the terminal shows issues only for "crawl" by default; + saved reports always list every URL unless this is set) + --all Also list 200 OK URLs in the terminal + --fail Exit with code 1 if broken links (4xx/5xx/errors) are found + -q, --quiet No progress or console output + +Cloud upload (without -o, an Excel report is created and uploaded) + --public Make uploaded files viewable by anyone with the link + (both services) + + Google Drive + --drive Upload the report(s) to Google Drive. Excel/CSV become + Google Sheets + --drive-public Anyone with the link can view the Google Drive files + --drive-folder Upload into this existing folder (ID from its URL). Needs + "drive-login --full-access" or a service account the + folder is shared with. Default: a "HTTP Status Reports" + folder created by this tool + --drive-folder-name Name of the folder this tool creates and reuses + --drive-key Service account key file (for CI). Default: the account + from "drive-login", or CHS_GOOGLE_CREDENTIALS / + GOOGLE_APPLICATION_CREDENTIALS + --no-convert Upload .xlsx/.csv as files instead of Google Sheets + + OneDrive + --onedrive Upload the report(s) to OneDrive (personal or work) + --onedrive-public Return "anyone with the link" view links + --onedrive-folder Upload into this folder path, e.g. "Reports/SEO". + Needs "onedrive-login --full-access". Default: the + app's own folder (Apps/) + + Login commands + --client-id OAuth client ID (or CHS_GOOGLE_CLIENT_ID / CHS_ONEDRIVE_CLIENT_ID) + --client-secret Google OAuth client secret (or CHS_GOOGLE_CLIENT_SECRET) + --tenant OneDrive: "common" (default), "consumers", + "organizations" or your tenant ID + --full-access Allow uploads into your own folders, not just the + tool's own folder + + -h, --help Show this help + -v, --version Show version + +Examples + check-http-status https://www.example.com -e /docs -o report.xlsx + check-http-status crawl example.com --sitemap https://example.com/sitemap.xml -o report.html + check-http-status check https://example.com/old-page https://example.com/missing + check-http-status check --sitemap https://example.com/sitemap_index.xml --issues-only --fail + check-http-status https://www.example.com --delay 0.5 --user-agent "MyAuditBot/1.0" + check-http-status https://www.example.com --drive --public + check-http-status https://www.example.com -o report.xlsx --onedrive --onedrive-folder Reports +`; + +function fail(message) { + console.error(`Error: ${message}\nRun "check-http-status --help" for usage.`); + process.exit(2); +} + +function parseHeaders(list) { + const headers = {}; + for (const header of list) { + const index = header.indexOf(':'); + if (index < 1) { + fail(`Invalid header "${header}". Use "Name: value".`); + } + + headers[header.slice(0, index).trim()] = header.slice(index + 1).trim(); + } + + return headers; +} + +function parseAuth(value) { + if (!value) { + return undefined; + } + + const index = value.indexOf(':'); + return { + 'password': index === -1 ? '' : value.slice(index + 1), + 'username': index === -1 ? value : value.slice(0, index) + }; +} + +function seconds(value) { + return value === undefined ? undefined : Math.round(parseFloat(value) * 1000); +} + +async function driveLogin(values) { + const { login } = require('../lib/google-drive'); + const clientId = values['client-id'] || process.env.CHS_GOOGLE_CLIENT_ID; + const clientSecret = values['client-secret'] || process.env.CHS_GOOGLE_CLIENT_SECRET; + + if (!clientId) { + fail('drive-login needs an OAuth client ID: --client-id --client-secret .\n' + + 'Create one at https://console.cloud.google.com/apis/credentials (type "Desktop app") and enable the Google Drive API.'); + } + + const file = await login({ clientId, clientSecret, 'fullAccess': values['full-access'] }); + console.log(`Google Drive connected. Credentials saved to ${file}`); + console.log('Upload reports with: check-http-status --drive'); +} + +async function oneDriveLogin(values) { + const { login } = require('../lib/onedrive'); + const clientId = values['client-id'] || process.env.CHS_ONEDRIVE_CLIENT_ID; + + if (!clientId) { + fail('onedrive-login needs an application (client) ID: --client-id .\n' + + 'Register one at https://entra.microsoft.com → App registrations, with platform "Mobile and desktop applications" and redirect URI http://localhost.'); + } + + const file = await login({ clientId, 'fullAccess': values['full-access'], 'tenant': values.tenant }); + console.log(`OneDrive connected. Credentials saved to ${file}`); + console.log('Upload reports with: check-http-status --onedrive'); +} + +async function main() { + let parsed; + try { + parsed = parseArgs({ + 'allowPositionals': true, + 'options': { + 'all': { 'type': 'boolean' }, + 'assets': { 'type': 'boolean' }, + 'auth': { 'short': 'a', 'type': 'string' }, + 'client-id': { 'type': 'string' }, + 'client-secret': { 'type': 'string' }, + 'concurrency': { 'short': 'c', 'type': 'string' }, + 'config': { 'type': 'string' }, + 'delay': { 'type': 'string' }, + 'drive': { 'type': 'boolean' }, + 'drive-folder': { 'type': 'string' }, + 'drive-folder-name': { 'type': 'string' }, + 'drive-key': { 'type': 'string' }, + 'drive-public': { 'type': 'boolean' }, + 'exclude': { 'multiple': true, 'short': 'e', 'type': 'string' }, + 'fail': { 'type': 'boolean' }, + 'file': { 'multiple': true, 'short': 'f', 'type': 'string' }, + 'full-access': { 'type': 'boolean' }, + 'header': { 'multiple': true, 'short': 'H', 'type': 'string' }, + 'help': { 'short': 'h', 'type': 'boolean' }, + 'ignore-query': { 'type': 'boolean' }, + 'include': { 'multiple': true, 'short': 'i', 'type': 'string' }, + 'issues-only': { 'type': 'boolean' }, + 'max-depth': { 'type': 'string' }, + 'max-pages': { 'type': 'string' }, + 'max-redirects': { 'type': 'string' }, + 'no-config': { 'type': 'boolean' }, + 'no-convert': { 'type': 'boolean' }, + 'no-external': { 'type': 'boolean' }, + 'onedrive': { 'type': 'boolean' }, + 'onedrive-folder': { 'type': 'string' }, + 'onedrive-public': { 'type': 'boolean' }, + 'output': { 'multiple': true, 'short': 'o', 'type': 'string' }, + 'public': { 'type': 'boolean' }, + 'quiet': { 'short': 'q', 'type': 'boolean' }, + 'respect-robots': { 'type': 'boolean' }, + 'retries': { 'type': 'string' }, + 'sitemap': { 'multiple': true, 'short': 's', 'type': 'string' }, + 'subdomains': { 'type': 'boolean' }, + 'tenant': { 'type': 'string' }, + 'timeout': { 'short': 't', 'type': 'string' }, + 'user-agent': { 'type': 'string' }, + 'version': { 'short': 'v', 'type': 'boolean' } + } + }); + } catch (error) { + fail(error.message); + } + + const { values } = parsed; + let positionals = parsed.positionals; + + if (values.help) { + process.stdout.write(HELP); + return; + } else if (values.version) { + console.log(pkg.version); + return; + } + + if (positionals[0] === 'drive-login' || positionals[0] === 'onedrive-login') { + try { + await (positionals[0] === 'drive-login' ? driveLogin(values) : oneDriveLogin(values)); + } catch (error) { + console.error(`Error: ${error.message}`); + process.exit(1); + } + return; + } + + let command = 'crawl'; + let explicitCommand = null; + if (['crawl', 'check'].includes(positionals[0])) { + command = explicitCommand = positionals[0]; + positionals = positionals.slice(1); + } + + let urls = [...positionals]; + for (const file of values.file || []) { + try { + urls = urls.concat(fs.readFileSync(file, 'utf8').split(/\r?\n/).map((line) => line.trim()).filter((line) => line && !line.startsWith('#'))); + } catch (error) { + fail(`Cannot read ${file}: ${error.message}`); + } + } + + // Settings from a config file come first; flags given on the command line win. + let fileConfig = {}; + const configFile = values.config || (values['no-config'] ? null : findConfig()); + if (configFile) { + try { + fileConfig = loadConfig(configFile); + } catch (error) { + fail(error.message); + } + + if (!values.quiet) { + console.error(`Using config ${path.relative(process.cwd(), path.resolve(configFile)) || configFile}`); + } + } + + const exportList = (values.output || []).map((file) => { + const format = path.extname(file).slice(1).toLowerCase(); + if (!['xlsx', 'csv', 'json', 'html'].includes(format)) { + fail(`Unknown report format for "${file}". Use .xlsx, .csv, .json or .html.`); + } + + return { 'file': path.resolve(file), format }; + }); + + const cli = {}; + const set = (key, value) => { + if (value !== undefined) { + cli[key] = value; + } + }; + + set('checkAssets', values.assets); + set('checkExternal', values['no-external'] ? false : undefined); + set('concurrency', values.concurrency); + set('delay', seconds(values.delay)); + set('export', exportList.length ? exportList : undefined); + set('fail', values.fail); + set('ignoreQuery', values['ignore-query']); + set('maxDepth', values['max-depth']); + set('maxPages', values['max-pages']); + set('maxRedirects', values['max-redirects']); + set('respectRobots', values['respect-robots']); + set('retries', values.retries); + set('silent', values.quiet); + set('skip200', values['issues-only']); + set('subdomains', values.subdomains); + set('userAgent', values['user-agent']); + + const config = { ...fileConfig, ...cli }; + + // Rules and sitemaps from the command line are added to the config file's. + for (const key of ['include', 'exclude', 'sitemaps']) { + const fromCli = values[key === 'sitemaps' ? 'sitemap' : key]; + if (fromCli) { + config[key] = [...[].concat(fileConfig[key] || []), ...fromCli]; + } + } + + const fileOptions = fileConfig.options || {}; + config.options = { + ...fileOptions, + 'auth': parseAuth(values.auth || process.env.CHS_AUTH) || fileOptions.auth, + 'headers': { ...(fileOptions.headers || {}), ...parseHeaders(values.header || []) }, + 'timeout': seconds(values.timeout) || fileOptions.timeout + }; + + if (values.drive || values['drive-public'] || values['drive-folder'] || values['drive-folder-name'] || values['drive-key'] || values['no-convert'] || (values.public && fileConfig.googleDrive)) { + const base = typeof fileConfig.googleDrive === 'object' ? fileConfig.googleDrive : {}; + config.googleDrive = { ...base }; + const drive = config.googleDrive; + if (values['no-convert']) { + drive.convert = false; + } + if (values['drive-folder']) { + drive.folderId = values['drive-folder']; + } + if (values['drive-folder-name']) { + drive.folderName = values['drive-folder-name']; + } + if (values['drive-key']) { + drive.keyFile = values['drive-key']; + } + if (values.public || values['drive-public']) { + drive.public = true; + } + } + + if (values.onedrive || values['onedrive-public'] || values['onedrive-folder'] || (values.public && fileConfig.oneDrive)) { + const base = typeof fileConfig.oneDrive === 'object' ? fileConfig.oneDrive : {}; + config.oneDrive = { ...base }; + if (values['onedrive-folder']) { + config.oneDrive.folderPath = values['onedrive-folder']; + } + if (values.public || values['onedrive-public']) { + config.oneDrive.public = true; + } + } + + if (values.public && !config.googleDrive && !config.oneDrive) { + fail('--public needs --drive and/or --onedrive.'); + } + + if (urls.length) { + // With several URLs, the first is the start page and the rest are extra seeds. + if (command === 'crawl') { + config.crawl = urls; + delete config.urls; + } else { + config.urls = urls; + delete config.crawl; + } + } else if (explicitCommand === 'check' && config.crawl) { + // "check" with a config that crawls: check the start URL(s) only. + config.urls = [].concat(config.urls || [], config.crawl); + delete config.crawl; + } + + const isCrawl = [].concat(config.crawl || []).length > 0; + if (!isCrawl && ![].concat(config.urls || []).length && ![].concat(config.sitemaps || []).length) { + process.stdout.write(HELP); + process.exit(2); + } + + // Terminal shows issues only; reports keep every URL. + config.consoleIssuesOnly = !values.all && isCrawl; + config.printResults = true; + const failOnBroken = !!config.fail; + delete config.fail; + + // First Ctrl+C stops and keeps results; second one quits. + const controller = new AbortController(); + const onInterrupt = () => { + if (controller.signal.aborted) { + process.exit(130); + } + + console.error('\nStopping… (press Ctrl+C again to quit immediately)'); + controller.abort(); + }; + config.signal = controller.signal; + process.on('SIGINT', onInterrupt); + + let report; + try { + report = await checkHttpStatus(config); + } catch (error) { + fail(error.message); + } finally { + process.removeListener('SIGINT', onInterrupt); + } + + if (report.uploadError) { + process.exitCode = 1; + } + + if (failOnBroken) { + const counts = report.summary.counts; + const broken = (counts['Not Found'] || 0) + (counts['Client Error'] || 0) + (counts['Server Error'] || 0) + (counts['Error'] || 0); + if (broken > 0) { + process.exitCode = 1; + } + } +} + +main(); diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/docs/demo.html b/docs/demo.html new file mode 100644 index 0000000..cd9d9eb --- /dev/null +++ b/docs/demo.html @@ -0,0 +1,323 @@ + + + + + +Demo crawl report · check-http-status + + + +
+

HTTP status report

+

+ +
+ +
+
+ +
+ + +
+
+
+
+ + + + + + + + +
StatusURLTypeFound on
+
+
+
+ + +
+ + + + + diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 0000000..5281a71 --- /dev/null +++ b/docs/index.html @@ -0,0 +1,296 @@ + + + + + +check-http-status · Crawl a site, find broken links and redirects + + + + +
+
+ check-http-status + +
+ +
+
+

Find every broken link and redirect on your website.

+

Crawl a site or check a list of URLs. See status codes, full redirect chains, and the exact pages each broken link was found on. Export to Excel, CSV, JSON or an interactive HTML report.

+
$npx check-http-status https://your-site.com -o report.xlsx
+
+ View demo report → + Star on GitHub +
+
+
+
+
$ npx check-http-status acme-outdoor.example -e /docs
+
+301 /summer-sale → /new-arrivals
+      found on / ("Summer sale")
+301 /products/sleeping-bags ⇒ /collections/
+      sleeping-bags (404, 2 hops)
+      found on /products/ ("Sleeping bags")
+404 partner-site.example/deleted-article
+      found on /blog/best-hiking-trails
+410 /blog/2019/old-news
+      found on /blog/ ("Old news")
+500 /products/stoves
+      found on /products/ ("Stoves")
+      found on /blog/packing-list ("Stoves")
+
+36 URLs (17 pages crawled, 2s): 22 OK,
+7 redirects, 4 not found, 2 5xx, 1 error
+3 orphan page(s): in the sitemap but not linked.
+
+
+ +
+

Everything you need for a link audit

+

It runs on your machine (macOS, Windows or Linux), so it can also check staging sites, password-protected sites and sites behind a VPN.

+
+
⟳

Full-site crawl

Follows every internal link from a start page, breadth-first, with page and depth limits.

+
⌖

Where it was found

Every broken link and redirect lists the pages that link to it, plus the link text.

+
↪

Redirect chains

Each hop is reported, with the final URL and status. Redirect loops are detected.

+
⇥

External links

External links are checked but never crawled, so they're listed without leaving your site.

+
⊘

Include / exclude rules

Skip sections like /docs. Excluded URLs are still listed and checked, just not crawled. Supports wildcards and regex.

+
≡

Sitemaps & orphans

Reads sitemap indexes and .xml.gz files, and flags pages in the sitemap that nothing links to.

+
▦

Excel, CSV, JSON, HTML

Every URL found, listed once with all the pages it was found on in one cell. Excel sheets: Summary, All URLs, Issues and External Links.

+
🔒

Staging & auth

HTTP Basic auth (sent only to your site, never to external links) and custom headers.

+
☁

Google Drive & OneDrive

Add --drive or --onedrive to upload the report. Add --public to get a link anyone can open.

+
{}

Config file & TypeScript

Save settings in check-http-status.config.json (with a JSON Schema for autocomplete). Type definitions included.

+
✓

CI friendly

--fail exits with code 1 when broken links are found, so you can block a deploy.

+
+
+ +
+
+
+

See a real report

+

A crawl of a sample store with 40 URLs, including 404s, a 410, 5xx errors, a two-hop redirect chain, a redirect loop, a DNS failure, a broken image and orphan pages. Filter it, search it, and click any row to see where the link was found.

+
+
+ Open demo report + Sample .xlsx +
+
+
+ +
+

Usage

+

Requires Node.js 18.17 or newer. Run it with npx, install it globally, or use it as a library.

+
+ + + +
+ 
# Crawl a site, skip /docs (still listed), save an Excel report
+npx check-http-status https://www.example.com --exclude /docs -o report.xlsx
+
+# Optional: add a sitemap to also find orphan pages; HTML report
+npx check-http-status crawl example.com -s https://example.com/sitemap_index.xml -o report.html
+
+# Only check specific URLs, or every URL in a sitemap
+npx check-http-status check https://example.com/old-page https://example.com/pricing
+npx check-http-status check -s https://example.com/sitemap.xml --issues-only
+
+# Password-protected staging site
+CHS_AUTH=user:pass npx check-http-status https://staging.example.com
+
+# Install globally
+npm install -g check-http-status
+check-http-status --help
+ + +
+ +
+

Options

+

The CLI flags and Node.js options are equivalent. Run check-http-status --help for the full list.

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CLINode.jsDefaultDescription
crawl <url>crawlWebsite URL. Pages are found by following internal links.
check <url…>urlsCheck only these URLs (and their redirects).
-s, --sitemapsitemapsOptional. Sitemap or sitemap index URL(s). Not needed for crawling. Gzip supported.
-i, --includeincludeOnly crawl URLs matching one of these rules.
-e, --excludeexcludeDon't crawl matching URLs. They're still listed and checked.
--max-pagesmaxPages1000Maximum pages to crawl.
--max-depthmaxDepth∞Maximum link depth from the start URL.
--no-externalcheckExternal: falsecheckedDon't check external links. Excluded pages and assets are still checked.
--assetscheckAssetsoffAlso check images, scripts, CSS and iframes.
--subdomainssubdomainsoffTreat subdomains as internal.
--respect-robotsrespectRobotsoffDon't request URLs your robots.txt disallows (they're listed as blocked); honour Crawl-delay.
--configloadConfig()Load settings from a JSON file. check-http-status.config.json in the current folder is used automatically.
--ignore-queryignoreQueryoffTreat /a?x=1 and /a as the same URL.
-c, --concurrencyconcurrency10Parallel requests.
-t, --timeoutoptions.timeout15 sPer-request timeout (seconds on the CLI, milliseconds in Node.js).
--delaydelay0Wait between requests to the same site (seconds on the CLI, milliseconds in Node.js).
--retriesretries2Retry 429/502/503/504 and dropped connections, honouring Retry-After.
--user-agentuserAgentCustom User-Agent header.
-a, --authoptions.authHTTP Basic auth, sent only to the site being checked.
-H, --headeroptions.headersExtra request headers.
-o, --outputexport.xlsx, .csv, .json or .html report.
--issues-only / --allskip200Hide or show 200 OK URLs in the terminal. --issues-only also drops them from saved reports, which otherwise list every URL.
--drivegoogleDriveUpload the report(s) to Google Drive. Excel/CSV become Google Sheets. Run check-http-status drive-login once first.
--onedriveoneDriveUpload the report(s) to OneDrive. Run check-http-status onedrive-login once first.
--publicpublic: trueAnyone with the link can view the uploaded files.
--failExit with code 1 if any 4xx, 5xx or connection errors are found.
+
+
+ +
+
+
+

Why run it locally?

+

Online checkers can't reach staging servers, intranet sites or VPN-only sites, and many sites block their datacenter IPs. Running on your own machine gets past all of that, and you can crawl as many pages as you need.

+
+
+

Upgrading from v1

+

v1 configs (urls, sitemaps, skip200, export, options) still work. v2 adds crawling and a CLI, and now returns the results. It also throws errors instead of exiting the process, and requires Node.js 18.17+.

+
+
+
+ + +
+ + + diff --git a/docs/sample-report.xlsx b/docs/sample-report.xlsx new file mode 100644 index 0000000..2b37086 Binary files /dev/null and b/docs/sample-report.xlsx differ diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..a829239 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,19 @@ +'use strict'; + +const globals = require('globals'); +const js = require('@eslint/js'); + +module.exports = [ + { + 'ignores': ['docs/**', 'node_modules/**'] + }, + js.configs.recommended, + { + 'files': ['**/*.js'], + 'languageOptions': { + 'ecmaVersion': 2022, + 'globals': globals.node, + 'sourceType': 'commonjs' + } + } +]; diff --git a/examples/check-http-status.config.json b/examples/check-http-status.config.json new file mode 100644 index 0000000..160aa15 --- /dev/null +++ b/examples/check-http-status.config.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://unpkg.com/check-http-status/schema.json", + "crawl": "https://www.example.com", + "exclude": ["/docs", "/wp-admin"], + "maxPages": 2000, + "checkAssets": true, + "respectRobots": true, + "delay": 250, + "userAgent": "ExampleAudit/1.0 (+https://www.example.com/bot)", + "export": [ + { "format": "xlsx", "file": "reports/link-report.xlsx" }, + { "format": "html", "file": "reports/link-report.html" } + ], + "googleDrive": { "public": false }, + "fail": true +} diff --git a/examples/runkit.js b/examples/runkit.js new file mode 100644 index 0000000..18c412d --- /dev/null +++ b/examples/runkit.js @@ -0,0 +1,19 @@ +'use strict'; + +const checkHttpStatus = require('check-http-status'); + +// Crawl a site, skip /docs (still listed), and print the broken links with the pages they were found on. +checkHttpStatus({ + 'crawl': 'https://www.trunkcode.com/', + 'exclude': ['/docs'], + 'maxPages': 50, + 'silent': true +}).then((report) => { + console.log(report.summary.counts); + + for (const result of report.results) { + if (['Not Found', 'Client Error', 'Server Error', 'Error'].includes(result.category)) { + console.log(result.status || result.error, result.url, 'found on', result.foundOn.map((source) => source.page)); + } + } +}); diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..3d6815a --- /dev/null +++ b/index.d.ts @@ -0,0 +1,261 @@ +// Type definitions for check-http-status 2.x + +declare namespace checkHttpStatus { + /** `Blocked`: bot protection answered; not counted as broken. */ + type Category = 'OK' | 'Redirect' | 'Not Found' | 'Client Error' | 'Server Error' | 'Error' | 'Blocked' | 'Not checked' | 'Other'; + + /** What happened to a URL: crawled, checked, excluded, external, asset, skipped (limit/depth) or blocked by robots.txt. */ + type Scope = 'crawl' | 'list' | 'excluded' | 'external' | 'asset' | 'limit' | 'depth' | 'robots'; + + type ExportFormat = 'xlsx' | 'csv' | 'json' | 'html'; + + interface ExportOptions { + /** Report format. Default `xlsx`. */ + format?: ExportFormat; + /** Full path of the report file. */ + file?: string; + /** Folder for an auto-named report. */ + location?: string; + } + + interface RequestOptions { + /** HTTP Basic auth, only sent to the checked site. */ + auth?: { username?: string; password?: string }; + /** Extra request headers. */ + headers?: Record; + /** Timeout per request in milliseconds. Default 15000. */ + timeout?: number; + /** User-Agent header (same as the top-level `userAgent`). */ + userAgent?: string; + } + + interface GoogleServiceAccount { + type: 'service_account'; + client_email: string; + private_key: string; + [key: string]: unknown; + } + + interface GoogleAuthorizedUser { + type?: 'authorized_user'; + client_id: string; + client_secret?: string; + refresh_token: string; + } + + interface GoogleDriveOptions { + /** Existing folder ID. Default: the tool's own folder. */ + folderId?: string; + /** Name of the folder the tool creates and reuses. */ + folderName?: string; + /** Convert .xlsx/.csv to Google Sheets. Default true. */ + convert?: boolean; + /** Anyone with the link can view the uploaded files. */ + public?: boolean; + /** Service account key or authorized-user credentials. */ + credentials?: GoogleServiceAccount | GoogleAuthorizedUser | string; + /** Path to a service account key file. */ + keyFile?: string; + clientId?: string; + clientSecret?: string; + refreshToken?: string; + /** Which exported formats to upload. Default: all. */ + formats?: ExportFormat | ExportFormat[]; + } + + interface OneDriveCredentials { + client_id: string; + refresh_token: string; + tenant?: string; + scope?: string; + client_secret?: string; + } + + interface OneDriveOptions { + /** e.g. "Documents/SEO". Default: the app's own folder. */ + folderPath?: string; + /** Return "anyone with the link" view links. */ + public?: boolean; + credentials?: OneDriveCredentials | string; + clientId?: string; + refreshToken?: string; + /** "common" (default), "consumers", "organizations" or a tenant ID. */ + tenant?: string; + /** Which exported formats to upload. Default: all. */ + formats?: ExportFormat | ExportFormat[]; + } + + interface Config { + /** Website URL(s) to crawl. */ + crawl?: string | string[]; + /** URLs to check without crawling. */ + urls?: string | string[]; + /** Optional sitemap / sitemap index URLs. */ + sitemaps?: string | string[]; + /** Only crawl matching URLs (substring, `*` or `/regex/`). */ + include?: string | string[]; + /** Don't crawl matching URLs; they're still listed and checked. */ + exclude?: string | string[]; + /** Default 1000. */ + maxPages?: number; + /** Maximum link depth from the start URL. Default unlimited. */ + maxDepth?: number | null; + /** Default 10. */ + maxRedirects?: number; + /** Check external links. Default true. */ + checkExternal?: boolean; + /** Also check images, scripts and CSS. Default false. */ + checkAssets?: boolean; + /** Treat subdomains as internal. Default false. */ + subdomains?: boolean; + /** Strip query strings before comparing URLs. Default false. */ + ignoreQuery?: boolean; + /** Honour robots.txt and Crawl-delay. Default false. */ + respectRobots?: boolean; + /** Parallel requests. Default 10. */ + concurrency?: number; + /** Milliseconds between requests to the same site. Default 0. */ + delay?: number; + /** Retries for temporary failures. Default 2. */ + retries?: number; + /** First retry wait in ms, doubled each time. Default 1000. */ + retryDelay?: number; + /** User-Agent header. */ + userAgent?: string; + /** Leave 200 OK URLs out of output and reports. */ + skip200?: boolean; + /** No console output. */ + silent?: boolean; + /** Reports to write. */ + export?: ExportOptions | ExportOptions[]; + options?: RequestOptions; + /** Upload reports to Google Drive. */ + googleDrive?: boolean | GoogleDriveOptions; + /** Upload reports to OneDrive. */ + oneDrive?: boolean | OneDriveOptions; + /** Stop the crawl; results so far are returned. */ + signal?: AbortSignal; + /** Called as each URL is checked. */ + onResult?: (result: Result) => void; + /** Called with progress after each URL. */ + onProgress?: (progress: Progress) => void; + } + + interface FoundOn { + /** Page, sitemap or redirecting URL. */ + page: string; + /** Link text(s) used on that page. */ + anchors: string[]; + /** Times the link appears on that page. */ + count: number; + kind: 'link' | 'redirect' | 'sitemap' | 'seed'; + } + + interface RedirectHop { + url: string; + status: number | null; + } + + interface Result { + url: string; + status: number | null; + statusText: string; + category: Category; + /** Timeout, DNS or connection error. */ + error: string; + type: 'internal' | 'external'; + linkType: 'page' | 'asset'; + scope: Scope; + redirectTo: string; + redirectChain: RedirectHop[]; + redirectCount: number; + finalUrl: string; + finalStatus: number | null; + /** e.g. "Cloudflare" when bot protection answered. */ + botProtection: string; + /** e.g. "Redirect loop" or "Stopped after 10 redirects". */ + note: string; + foundOn: FoundOn[]; + title: string; + contentType: string; + responseTime: number | null; + depth: number; + crawled: boolean; + inSitemap: boolean; + /** In the sitemap but not linked. */ + orphan: boolean; + /** Set by `recheck()`. */ + rechecked?: boolean; + } + + interface Progress { + state: 'idle' | 'running' | 'done' | 'stopped'; + mode: 'crawl' | 'list'; + discovered: number; + pagesCrawled: number; + checked: number; + queued: number; + active: number; + counts: Partial>; + startedAt: number | null; + finishedAt: number | null; + elapsed: number; + pageLimitReached: boolean; + } + + interface Summary extends Progress { + crawl: string[]; + sitemaps: string[]; + include: string[]; + exclude: string[]; + issues: number; + orphans: number; + robotsBlocked: number; + recheckedAt?: number; + } + + interface Upload { + service: 'googleDrive' | 'oneDrive'; + /** Local file that was uploaded. */ + file: string; + id: string; + name: string; + /** Link to the uploaded file. */ + url: string; + public: boolean; + mimeType?: string; + } + + interface Report { + summary: Summary; + results: Result[]; + /** When a cloud upload was requested. */ + uploads?: Upload[]; + /** Set when an upload failed. */ + uploadError?: string; + } + + interface RecheckConfig extends Pick {} + + /** Crawl a website and/or check a list of URLs. */ + function checkHttpStatus(config: Config): Promise; + + /** Re-check URLs of a report; found-on pages are kept. */ + function recheck(report: Report, urls: string | string[], config?: RecheckConfig): Promise; + + /** Write a report to a file. */ + function writeReport(report: Report, file: string, format: ExportFormat, options?: { skip200?: boolean }): Promise; + + /** Render a report as a self-contained interactive HTML page. */ + function renderHtml(report: Report, options?: { title?: string }): string; + + /** Load a config file; paths are relative to it. */ + function loadConfig(file: string): Config & { fail?: boolean }; + + /** Find check-http-status.config.json in a folder. */ + function findConfig(dir?: string): string | null; +} + +declare function checkHttpStatus(config: checkHttpStatus.Config): Promise; + +export = checkHttpStatus; diff --git a/index.js b/index.js index 6aab47d..8fc538b 100644 --- a/index.js +++ b/index.js @@ -1,79 +1,343 @@ 'use strict'; -const consoleColumns = require('./lib/console-columns'); -const fetchAllSitemaps = require('./lib/fetch-from-sitemaps'); const fs = require('fs'); -const generateExcel = require('./lib/generate-excel'); -const httpList = require('./lib/http-list'); +const os = require('os'); +const path = require('path'); +const fetchSitemaps = require('./lib/sitemaps'); +const googleDrive = require('./lib/google-drive'); +const oneDrive = require('./lib/onedrive'); +const writeCsv = require('./lib/report/csv'); +const writeExcel = require('./lib/report/excel'); +const { Checker, ISSUE_CATEGORIES, categorize } = require('./lib/checker'); +const { createProgressPrinter, printReport } = require('./lib/report/console'); +const { createRequester } = require('./lib/request'); +const { createRobots } = require('./lib/robots'); +const { findConfig, loadConfig } = require('./lib/config'); +const { renderHtml, writeHtml } = require('./lib/report/html'); + +const EXPORT_FORMATS = ['xlsx', 'csv', 'json', 'html']; +const CLOUD_SERVICES = { + 'googleDrive': { 'label': 'Google Drive', 'resolveCredentials': googleDrive.resolveCredentials, 'upload': googleDrive.uploadToDrive }, + 'oneDrive': { 'label': 'OneDrive', 'resolveCredentials': oneDrive.resolveCredentials, 'upload': oneDrive.uploadToOneDrive } +}; +// Fields `recheck()` refreshes; where a URL was found stays as crawled. +const STATUS_FIELDS = ['category', 'contentType', 'error', 'finalStatus', 'finalUrl', 'note', 'redirectChain', 'redirectCount', 'redirectTo', 'responseTime', 'status', 'statusText']; + +const toArray = (value) => { + if (value === undefined || value === null || value === '') { + return []; + } + + return Array.isArray(value) ? value : [value]; +}; + +const toInt = (value, fallback, min, max) => { + const number = parseInt(value, 10); + return Number.isNaN(number) ? fallback : Math.min(max, Math.max(min, number)); +}; + +function normalizeUrl(url) { + let value = String(url).trim(); + if (!/^https?:\/\//i.test(value)) { + value = 'https://' + value; + } + + try { + return new URL(value).href; + } catch { + throw new Error(`Invalid URL: ${url}`); + } +} + +function normalizeConfig(config) { + if (!config || typeof config !== 'object') { + throw new Error('Missing required parameters.'); + } + + const options = config.options || {}; + const rules = (value) => toArray(value).flatMap((rule) => String(rule).split(/\r?\n/)).map((rule) => rule.trim()).filter(Boolean); + const normalized = { + 'checkAssets': !!config.checkAssets, + 'checkExternal': config.checkExternal !== false, + 'concurrency': toInt(config.concurrency, 10, 1, 100), + // Only filters the terminal output; saved reports keep every URL. + 'consoleIssuesOnly': !!config.consoleIssuesOnly, + 'crawl': toArray(config.crawl).map(normalizeUrl), + 'exclude': rules(config.exclude), + 'ignoreQuery': !!config.ignoreQuery, + 'include': rules(config.include), + 'maxDepth': config.maxDepth === undefined || config.maxDepth === null || config.maxDepth === '' ? null : toInt(config.maxDepth, null, 0, 10000), + 'maxPages': toInt(config.maxPages, 1000, 1, 1000000), + 'maxRedirects': toInt(config.maxRedirects, 10, 0, 50), + 'onProgress': typeof config.onProgress === 'function' ? config.onProgress : null, + 'onResult': typeof config.onResult === 'function' ? config.onResult : null, + // CLI: print the URL list even when reports are saved. + 'printResults': !!config.printResults, + 'respectRobots': !!config.respectRobots, + 'options': { + 'auth': options.auth, + 'delay': toInt(config.delay, 0, 0, 600000), + 'headers': options.headers, + 'retries': toInt(config.retries, 2, 0, 10), + 'retryDelay': toInt(config.retryDelay, 1000, 0, 60000), + 'signal': config.signal, + 'timeout': toInt(options.timeout, 15000, 1000, 600000), + 'userAgent': config.userAgent || options.userAgent + }, + 'signal': config.signal, + 'silent': !!config.silent, + 'sitemaps': toArray(config.sitemaps).map(normalizeUrl), + 'skip200': !!config.skip200, + 'subdomains': !!config.subdomains, + 'urls': [...new Set(toArray(config.urls).map(normalizeUrl))] + }; + + if (!normalized.crawl.length && !normalized.urls.length && !normalized.sitemaps.length) { + throw new Error('Provide at least one of `crawl`, `urls` or `sitemaps`.'); + } + + normalized.export = toArray(config.export).map((details) => { + const format = (details.format || 'xlsx').toLowerCase(); + if (!EXPORT_FORMATS.includes(format)) { + throw new Error(`Unsupported export format "${format}". Use one of: ${EXPORT_FORMATS.join(', ')}.`); + } + + let file = details.file; + if (!file) { + if (!details.location) { + throw new Error('Missing export location.'); + } else if (!fs.existsSync(details.location)) { + throw new Error(`Export location does not exist: ${details.location}`); + } + + file = path.join(details.location, `httpstatuschecker-results-${Math.floor(Date.now() / 1000)}.${format}`); + } + + return { file, format }; + }); + + normalized.cloud = []; + for (const [key, service] of Object.entries(CLOUD_SERVICES)) { + if (!config[key]) { + continue; + } + + const cloudOptions = config[key] === true ? {} : { ...config[key] }; + // Fail before crawling, not after, if no credentials can be found. + cloudOptions.credentials = service.resolveCredentials(cloudOptions); + normalized.cloud.push({ key, 'options': cloudOptions }); + } + + // Nothing to upload yet: create an Excel report in the temp folder. + if (normalized.cloud.length && !normalized.export.length) { + const host = new URL(normalized.crawl[0] || normalized.urls[0] || normalized.sitemaps[0]).hostname; + const stamp = new Date().toISOString().slice(0, 16).replace('T', ' ').replace(':', '-'); + normalized.export.push({ + 'file': path.join(os.tmpdir(), `HTTP status - ${host} - ${stamp}.xlsx`), + 'format': 'xlsx', + 'temporary': true + }); + } + + return normalized; +} + +function buildSummary(results, extra) { + const counts = {}; + for (const result of results) { + counts[result.category] = (counts[result.category] || 0) + 1; + } + + return { + ...extra, + 'counts': counts, + 'discovered': results.length, + 'issues': results.filter((result) => ISSUE_CATEGORIES.includes(result.category)).length, + 'orphans': results.filter((result) => result.orphan).length, + 'robotsBlocked': results.filter((result) => result.scope === 'robots').length + }; +} + +// Upload failures are collected, not thrown, so the crawl results are never lost. +async function uploadReports(settings, report, log) { + const uploads = []; + const errors = []; + + for (const { key, options } of settings.cloud) { + const service = CLOUD_SERVICES[key]; + const formats = options.formats ? toArray(options.formats) : EXPORT_FORMATS; + const files = settings.export.filter((entry) => formats.includes(entry.format)); + + try { + const uploaded = await service.upload(files, options); + uploaded.forEach((upload) => log(`Uploaded to ${service.label}${upload.public ? ' (public link)' : ''}: ${upload.name} → ${upload.url}`)); + uploads.push(...uploaded); + } catch (error) { + errors.push(error.message); + log(`Error: ${error.message}`); + } + } + + report.uploads = uploads; + if (errors.length) { + report.uploadError = errors.join(' '); + return; + } + + for (const entry of settings.export.filter((item) => item.temporary)) { + await fs.promises.rm(entry.file, { 'force': true }); + } +} -/** - * Main function that handles config, and trigger functions according to the - * config. - */ async function checkHttpStatus(config) { - const allowedExportTypes = [ - 'csv', - 'xlsx', - ]; - var skip200 = false; - var urlsList = []; - - if (config.skip200) { - skip200 = true; - } - - if (!config) { - console.error('\x1b[31m%s\x1b[0m', 'Error: Missing required Parameters.'); - process.exit(); - } else if (!config.options) { - config.options = {}; - } - - if (config.sitemaps) { - urlsList = await fetchAllSitemaps(config.sitemaps); - } - - if (config.urls && Array.isArray(config.urls)) { - urlsList = urlsList.concat(config.urls); - } - - if (urlsList.length === 0) { - console.error('\x1b[31m%s\x1b[0m', 'Error: No URL(s) found.'); - process.exit(); - } else if (config.export && !config.export.location) { - console.error('\x1b[31m%s\x1b[0m', 'Error: Missing export location.'); - process.exit(); - } else if (config.export && !fs.existsSync(config.export.location)) { - console.error('\x1b[31m%s\x1b[0m', 'Error: Export Location is undefined.'); - process.exit(); - } - - const httpStatusList = await httpList(urlsList, config.options, skip200); - - if (config.export && !config.export.format) { - config.export.format = 'xlsx'; - } - - if (skip200 && httpStatusList.length === 0) { - // Add empty line - console.log(); - console.log('\x1b[32m%s\x1b[0m', 'All the URLs are 200 and there is nothing to worry about!'); - } else if (config.export && allowedExportTypes.includes(config.export.format)) { - const urlLength = Math.max(...urlsList.map((el) => el.length)); - const rowLength = { - 'errorMessage': 50, - 'requestedUrl': urlLength, - 'url': urlLength + 20 - }; - - generateExcel(httpStatusList, rowLength, config.export); - } else { - consoleColumns(httpStatusList); - } - - // Add empty line - console.log(); - console.log('\x1b[32m%s\x1b[0m', 'HTTP Status check completed!'); + const settings = normalizeConfig(config); + const authHosts = [...settings.crawl, ...settings.urls, ...settings.sitemaps].map((url) => new URL(url).hostname); + const request = createRequester({ ...settings.options, authHosts }); + const progress = settings.silent ? null : createProgressPrinter(); + const log = (...args) => { + if (!settings.silent) { + console.error(...args); + } + }; + + let sitemapEntries = []; + if (settings.sitemaps.length) { + sitemapEntries = await fetchSitemaps(settings.sitemaps, request, (url, message) => { + log(`Error: could not read sitemap ${url} (${message})`); + }); + + if (!sitemapEntries.length && !settings.crawl.length && !settings.urls.length) { + throw new Error('No URL(s) found in the sitemap(s).'); + } + } + + const userProgress = settings.onProgress; + const checker = new Checker({ + ...settings, + 'onProgress': (state) => { + if (progress) { + progress.update(state); + } + + if (userProgress) { + userProgress(state); + } + } + }, request, settings.respectRobots ? createRobots(request, settings.options.userAgent) : null); + + // Aborting keeps everything checked so far; unfinished URLs are "Not checked". + const stop = () => checker.stop(); + if (settings.signal) { + if (settings.signal.aborted) { + stop(); + } + settings.signal.addEventListener('abort', stop, { 'once': true }); + } + + try { + await checker.run(sitemapEntries); + } finally { + if (settings.signal) { + settings.signal.removeEventListener('abort', stop); + } + if (progress) { + progress.clear(); + } + } + + const results = checker.results(); + const report = { + 'results': results, + 'summary': buildSummary(results, { + ...checker.progress(), + 'crawl': settings.crawl, + 'exclude': settings.exclude, + 'include': settings.include, + 'sitemaps': settings.sitemaps + }) + }; + + for (const { file, format, temporary } of settings.export) { + await writeReport(report, file, format, settings); + if (!temporary) { + log(`Saved ${format.toUpperCase()} report: ${file}`); + } + } + + if (settings.cloud.length) { + await uploadReports(settings, report, log); + } + + if (!settings.silent && (settings.printResults || !settings.export.length)) { + printReport(report, { ...settings, 'skip200': settings.skip200 || settings.consoleIssuesOnly }); + } else if (!settings.silent) { + log(`${report.summary.discovered} URLs checked, ${report.summary.issues} issue(s) found.`); + } + + return report; +} + +async function writeReport(report, file, format, options = {}) { + if (format === 'xlsx') { + await writeExcel(report, file, options); + } else if (format === 'csv') { + await writeCsv(report, file, options); + } else if (format === 'json') { + await fs.promises.writeFile(file, JSON.stringify(report, null, 2)); + } else if (format === 'html') { + await writeHtml(report, file); + } else { + throw new Error(`Unsupported export format "${format}". Use one of: ${EXPORT_FORMATS.join(', ')}.`); + } +} + +async function recheck(report, urls, config = {}) { + const targets = toArray(urls); + if (!targets.length) { + return report; + } + + const fresh = await checkHttpStatus({ + 'delay': config.delay, + 'maxRedirects': config.maxRedirects, + 'onResult': config.onResult, + 'options': config.options, + 'retries': config.retries, + 'signal': config.signal, + 'silent': true, + 'urls': targets, + 'userAgent': config.userAgent + }); + + const byUrl = new Map(report.results.map((result) => [result.url, result])); + for (const result of fresh.results) { + const existing = byUrl.get(result.url); + if (!existing) { + // A new redirect target: keep it so redirect chains stay complete. + byUrl.set(result.url, { ...result, 'scope': 'list' }); + continue; + } + + const updated = { ...existing, 'rechecked': true }; + STATUS_FIELDS.forEach((field) => { + updated[field] = result[field]; + }); + updated.category = categorize(updated); + byUrl.set(result.url, updated); + } + + const results = [...byUrl.values()]; + return { + ...report, + 'results': results, + 'summary': buildSummary(results, { ...report.summary, 'recheckedAt': Date.now() }) + }; } module.exports = checkHttpStatus; +module.exports.checkHttpStatus = checkHttpStatus; +module.exports.findConfig = findConfig; +module.exports.loadConfig = loadConfig; +module.exports.recheck = recheck; +module.exports.renderHtml = renderHtml; +module.exports.writeReport = writeReport; diff --git a/lib/check-status-code.js b/lib/check-status-code.js deleted file mode 100644 index 591b156..0000000 --- a/lib/check-status-code.js +++ /dev/null @@ -1,134 +0,0 @@ -'use strict'; - -const axios = require('axios'); - -/** - * Check HTTP Status as per settings and return status code and other - * information for further process. - * - * @param string urlToCheck URL that needs to be requested. - * @param object axiosOptions Axios Options. - * @param bool redirect Whether the URL is crawling or the redirected URL is crawling. - * - * @returns array Response of checked URL. - */ -function axiosRequest(urlToCheck, axiosOptions, redirect) { - var httpStatus = [ - '', - 0, - '', - urlToCheck, - '' - ]; - - if (!redirect) { - httpStatus[0] = urlToCheck; - } - - return new Promise((resolve) => { - axios.get(urlToCheck, axiosOptions) - .then((response) => { - httpStatus[2] = response.status; - if (redirect) { - httpStatus[1] = ''; - } - - resolve(httpStatus); - }) - .catch(async (error) => { - var statusLists = []; - - if (error.response && error.response.status) { - httpStatus[2] = error.response.status; - if (error.response.status >= 300 && error.response.status < 400) { - const redUrl = error.response.headers.location; - const checkType = await axiosRequest(redUrl, axiosOptions, true); - - httpStatus[3] = error.response.headers.location; - statusLists[0] = httpStatus; - - if (redirect) { - httpStatus[1] = ''; - } else if (typeof checkType[0] === 'object') { - httpStatus[1] = checkType.length; - } else { - httpStatus[1] = 1; - } - - if (!redirect && typeof checkType[0] === 'object') { - let childLinks = 1; - checkType.forEach((element) => { - statusLists[childLinks] = element; - childLinks += 1; - }); - } else { - statusLists[1] = checkType; - } - } else { - statusLists = httpStatus; - } - } else { - httpStatus[1] = ''; - httpStatus[3] = ''; - if (error.message) { - httpStatus[4] = error.message; - } else { - httpStatus[4] = error; - } - - statusLists = httpStatus; - } - - resolve(statusLists); - }); - }); -} - -/** - * Call main function to generate array with all the required information - * and await until that all are not completed. - * - * @param string urlToCheck URL that needs to be checked. - * @param object options Axios options. - * @param bool skip200 Whether to log URLs that returns 200 status code or not. - * - * @returns array List of URLs with their status code and other information. - */ -async function checkStatusCode(urlToCheck, options, skip200) { - const statusList = []; - - var axiosOptions = { - 'maxRedirects': 0 - }; - var statusArray = []; - - // Custom headers to be sent. - if (options.headers) { - axiosOptions.headers = options.headers; - } - - // Specifies the number of milliseconds before the request times out. - if (options.timeout) { - axiosOptions.timeout = options.timeout; - } - - // Indicates that HTTP Basic auth should be used, and supplies credentials. - if (options.auth) { - axiosOptions.auth = options.auth; - } - - statusArray = await axiosRequest(urlToCheck, axiosOptions); - if (typeof statusArray[0] === 'object') { - statusArray.forEach((row) => { - statusList.push(row); - }); - } else { - if (statusArray[2] !== 200 || !skip200) { - statusList.push(statusArray); - } - } - - return Promise.resolve(statusList); -} - -module.exports = checkStatusCode; diff --git a/lib/checker.js b/lib/checker.js new file mode 100644 index 0000000..4ba23e5 --- /dev/null +++ b/lib/checker.js @@ -0,0 +1,559 @@ +'use strict'; + +const cheerio = require('cheerio'); +const compilePatterns = require('./patterns'); +const { AbortedError, bodyToText } = require('./request'); + +const SKIP_SCHEMES = /^(mailto:|tel:|javascript:|data:|sms:|ftp:|file:|blob:|#)/i; +const ISSUE_CATEGORIES = ['Redirect', 'Not Found', 'Client Error', 'Server Error', 'Error']; + +const SCOPE_LABELS = { + 'asset': 'Asset', + 'crawl': 'Crawled', + 'depth': 'Skipped (max depth)', + 'excluded': 'Excluded by rule', + 'external': 'External (not crawled)', + 'limit': 'Skipped (max pages)', + 'list': 'Checked', + 'robots': 'Blocked by robots.txt' +}; + +function categorize(record) { + const status = record.status; + + if (record.error) { + return 'Error'; + } else if (record.botProtection) { + return 'Blocked'; + } else if (status === null || status === undefined) { + return 'Not checked'; + } else if (status >= 200 && status < 300) { + return 'OK'; + } else if (status >= 300 && status < 400) { + return 'Redirect'; + } else if (status === 404 || status === 410) { + return 'Not Found'; + } else if (status >= 400 && status < 500) { + return 'Client Error'; + } else if (status >= 500) { + return 'Server Error'; + } + + return 'Other'; +} + +const stripWww = (host) => host.toLowerCase().replace(/^www\./, ''); + +class Checker { + constructor(config, request, robots = null) { + this.config = config; + this.request = request; + this.robots = robots; + this.include = compilePatterns(config.include); + this.exclude = compilePatterns(config.exclude); + this.crawlMode = config.crawl.length > 0; + // In list mode "internal" means the hosts of the URLs the user supplied. + this.rootHosts = new Set((this.crawlMode ? config.crawl : config.urls).map((url) => stripWww(new URL(url).host))); + + this.records = new Map(); + this.pageQueue = []; + this.checkQueue = []; + this.active = 0; + this.crawlQueued = 0; + this.pagesCrawled = 0; + this.checked = 0; + this.stopped = false; + this.state = 'idle'; + this.startedAt = null; + this.finishedAt = null; + } + + run(sitemapEntries = []) { + this.state = 'running'; + this.startedAt = Date.now(); + + for (const url of this.config.crawl) { + this.discover(url, { + 'kind': 'seed', + 'seed': 'crawl' + }); + } + + for (const url of this.config.urls) { + this.discover(url, { + 'kind': 'seed', + 'seed': 'list' + }); + } + + for (const entry of sitemapEntries) { + if (!this.crawlMode) { + this.rootHosts.add(stripWww(new URL(entry.url).host)); + } + + this.discover(entry.url, { + 'fromUrl': entry.sitemap, + 'kind': 'sitemap', + 'seed': 'sitemap' + }); + } + + return new Promise((resolve) => { + this.resolveDone = resolve; + this.pump(); + this.maybeFinish(); + }); + } + + stop() { + this.stopped = true; + this.pageQueue.length = 0; + this.checkQueue.length = 0; + this.maybeFinish(); + } + + isInternal(url) { + const host = stripWww(url.host); + + if (this.rootHosts.has(host)) { + return true; + } + + if (this.config.subdomains) { + for (const root of this.rootHosts) { + if (host.endsWith('.' + root)) { + return true; + } + } + } + + return false; + } + + isAllowed(href) { + if (this.include.length && !this.include.some((pattern) => pattern.test(href))) { + return false; + } + + return !this.exclude.some((pattern) => pattern.test(href)); + } + + normalize(raw, base) { + const href = String(raw || '').trim(); + if (!href || SKIP_SCHEMES.test(href)) { + return null; + } + + let url; + try { + url = new URL(href, base); + } catch { + return null; + } + + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + return null; + } + + url.hash = ''; + if (this.config.ignoreQuery) { + url.search = ''; + } + + return url; + } + + classify(url, { from, kind, seed, isAsset }) { + const href = url.href; + const depth = from ? from.depth + (kind === 'redirect' ? 0 : 1) : 0; + + if (seed === 'crawl') { + return { depth, 'scope': 'crawl' }; + } else if (seed === 'list' || (seed === 'sitemap' && !this.crawlMode)) { + return { depth, 'scope': 'list' }; + } else if (kind === 'redirect' && from.scope === 'list') { + return { depth, 'scope': 'list' }; + } else if (!this.isInternal(url)) { + return { depth, 'scope': 'external' }; + } else if (isAsset) { + return { depth, 'scope': 'asset' }; + } else if (!this.isAllowed(href)) { + return { depth, 'scope': 'excluded' }; + } else if (this.config.maxDepth !== null && depth > this.config.maxDepth) { + return { depth, 'scope': 'depth' }; + } else if (this.crawlQueued >= this.config.maxPages) { + return { depth, 'scope': 'limit' }; + } + + return { depth, 'scope': 'crawl' }; + } + + discover(raw, { from = null, fromUrl = null, kind = 'link', anchor = '', base, isAsset = false, seed = null } = {}) { + const url = this.normalize(raw, base || (from && from.url) || undefined); + if (!url) { + return; + } + + const key = url.href; + let record = this.records.get(key); + + if (!record) { + const { depth, scope } = this.classify(url, { from, isAsset, kind, seed }); + if (scope === 'crawl') { + this.crawlQueued++; + } + + record = { + 'contentType': '', + 'crawled': false, + 'depth': depth, + 'error': '', + 'internal': this.isInternal(url), + 'isStart': seed === 'crawl', + 'linkType': isAsset ? 'asset' : 'page', + 'note': '', + 'queued': false, + 'redirectHops': kind === 'redirect' ? from.redirectHops + 1 : 0, + 'redirectTo': '', + 'responseTime': null, + 'scope': scope, + 'sources': new Map(), + 'status': null, + 'statusText': '', + 'title': '', + 'url': key + }; + this.records.set(key, record); + this.enqueue(record); + } else if (seed === 'list' && !record.queued) { + // A URL the user asked for explicitly is always checked. + record.scope = 'list'; + this.enqueue(record); + } + + const sourceUrl = from ? from.url : fromUrl; + if (sourceUrl) { + let source = record.sources.get(sourceUrl); + if (!source) { + source = { + 'anchors': new Set(), + 'count': 0, + 'kind': kind + }; + record.sources.set(sourceUrl, source); + } + + source.count++; + if (anchor) { + source.anchors.add(anchor); + } + } + } + + enqueue(record) { + const scope = record.scope; + // `checkExternal` only applies to external links. + const shouldCheck = ['crawl', 'list', 'excluded', 'asset'].includes(scope) || (scope === 'external' && this.config.checkExternal); + + if (this.stopped || record.queued || !shouldCheck) { + return; + } + + record.queued = true; + if (scope === 'crawl') { + this.pageQueue.push(record); + } else { + this.checkQueue.push(record); + } + } + + pump() { + while (!this.stopped && this.active < this.config.concurrency && (this.pageQueue.length || this.checkQueue.length)) { + // Pages first (breadth-first) so the site structure is discovered early. + const record = this.pageQueue.length ? this.pageQueue.shift() : this.checkQueue.shift(); + + this.active++; + this.process(record) + .then(() => true, (error) => { + if (error instanceof AbortedError) { + // Stopped mid-request: leave the URL as "Not checked" instead of an error. + record.status = null; + return false; + } + + record.error = record.error || error.message || String(error); + return true; + }) + .then((finished) => { + this.active--; + if (finished) { + this.checked++; + if (this.config.onResult) { + this.config.onResult(this.toResult(record)); + } + } + if (this.config.onProgress) { + this.config.onProgress(this.progress()); + } + this.pump(); + this.maybeFinish(); + }); + } + } + + maybeFinish() { + if (this.state !== 'running' || this.active > 0) { + return; + } + + if (this.stopped || (this.pageQueue.length === 0 && this.checkQueue.length === 0)) { + this.state = this.stopped ? 'stopped' : 'done'; + this.finishedAt = Date.now(); + this.resolveDone(); + } + } + + async process(record) { + // robots.txt only applies to the crawled site. + if (this.robots && this.crawlMode && record.internal) { + const { allowed, crawlDelay } = await this.robots.check(record.url); + if (crawlDelay && this.request.setHostDelay) { + this.request.setHostDelay(new URL(record.url).host, crawlDelay); + } + + if (!allowed) { + record.scope = 'robots'; + return; + } + } + + const parse = record.scope === 'crawl'; + let response; + + if (parse) { + response = await this.request(record.url, { + 'readBody': true + }); + } else { + // HEAD is cheap but many servers mishandle it, so retry with GET on any error. + try { + response = await this.request(record.url, { + 'method': 'HEAD' + }); + } catch (error) { + if (error instanceof AbortedError) { + throw error; + } + response = null; + } + + if (!response || response.status >= 400) { + response = await this.request(record.url); + } + } + + record.status = response.status; + record.statusText = response.statusText; + record.contentType = response.contentType.split(';')[0].trim(); + record.responseTime = response.responseTime; + if (response.botProtection) { + record.botProtection = response.botProtection; + record.note = `Bot protection (${response.botProtection}): check in a browser`; + return; + } + + if (response.status >= 300 && response.status < 400 && response.location) { + const target = this.normalize(response.location, record.url); + if (!target) { + return; + } + + record.redirectTo = target.href; + if (record.redirectHops >= this.config.maxRedirects) { + record.note = `Stopped after ${this.config.maxRedirects} redirects`; + return; + } + + this.discover(target.href, { + 'anchor': '(redirect)', + 'from': record, + 'isAsset': record.linkType === 'asset', + 'kind': 'redirect' + }); + return; + } + + if (parse && response.body && /html/i.test(response.contentType)) { + record.crawled = true; + this.pagesCrawled++; + this.extractLinks(record, bodyToText(response.body)); + } + } + + extractLinks(record, html) { + const $ = cheerio.load(html); + record.title = $('title').first().text().replace(/\s+/g, ' ').trim().slice(0, 300); + + let base = record.url; + const baseHref = $('base[href]').attr('href'); + if (baseHref) { + try { + base = new URL(baseHref, record.url).href; + } catch { + // Ignore an invalid . + } + } + + $('a[href], area[href]').each((_, el) => { + const $el = $(el); + const text = $el.text().replace(/\s+/g, ' ').trim() || + $el.attr('aria-label') || + $el.attr('title') || + $el.find('img[alt]').attr('alt') || + ''; + + this.discover($el.attr('href'), { + 'anchor': text.slice(0, 200), + 'base': base, + 'from': record + }); + }); + + if (this.config.checkAssets) { + const assets = [ + ['img[src]', 'src', 'image'], + ['script[src]', 'src', 'script'], + ['link[rel~="stylesheet"][href]', 'href', 'stylesheet'], + ['link[rel~="icon"][href]', 'href', 'icon'], + ['source[src], video[src], audio[src]', 'src', 'media'], + ['iframe[src]', 'src', 'iframe'] + ]; + + for (const [selector, attr, label] of assets) { + $(selector).each((_, el) => { + this.discover($(el).attr(attr), { + 'anchor': `(${label})`, + 'base': base, + 'from': record, + 'isAsset': true + }); + }); + } + } + } + + redirectChain(record) { + const chain = []; + const seen = new Set(); + let current = record; + + while (current && current.redirectTo) { + if (seen.has(current.url)) { + return { chain, 'loop': true }; + } + + seen.add(current.url); + const next = this.records.get(current.redirectTo); + chain.push({ + 'status': next ? next.status : null, + 'url': current.redirectTo + }); + current = next; + } + + return { chain, 'loop': false }; + } + + // Linked directly, or via a URL that redirects here. + isLinked(record, seen = new Set()) { + if (seen.has(record.url)) { + return false; + } + seen.add(record.url); + + for (const [page, source] of record.sources) { + if (source.kind === 'link') { + return true; + } + + const from = source.kind === 'redirect' && this.records.get(page); + if (from && this.isLinked(from, seen)) { + return true; + } + } + + return false; + } + + progress() { + const counts = {}; + for (const record of this.records.values()) { + const category = categorize(record); + counts[category] = (counts[category] || 0) + 1; + } + + return { + 'active': this.active, + 'checked': this.checked, + 'counts': counts, + 'discovered': this.records.size, + 'elapsed': this.startedAt ? (this.finishedAt || Date.now()) - this.startedAt : 0, + 'finishedAt': this.finishedAt, + 'mode': this.crawlMode ? 'crawl' : 'list', + 'pageLimitReached': this.crawlMode && this.crawlQueued >= this.config.maxPages, + 'pagesCrawled': this.pagesCrawled, + 'queued': this.pageQueue.length + this.checkQueue.length, + 'startedAt': this.startedAt, + 'state': this.state + }; + } + + results() { + return [...this.records.values()].map((record) => this.toResult(record)); + } + + toResult(record) { + const { chain, loop } = record.redirectTo ? this.redirectChain(record) : { 'chain': [], 'loop': false }; + const final = chain[chain.length - 1]; + const foundOn = [...record.sources].map(([page, source]) => ({ + 'anchors': [...source.anchors], + 'count': source.count, + 'kind': source.kind, + 'page': page + })); + const inSitemap = foundOn.some((source) => source.kind === 'sitemap'); + + return { + 'botProtection': record.botProtection || '', + 'category': categorize(record), + 'contentType': record.contentType, + 'crawled': record.crawled, + 'depth': record.depth, + 'error': record.error, + 'finalStatus': final ? final.status : null, + 'finalUrl': final ? final.url : '', + 'foundOn': foundOn, + 'inSitemap': inSitemap, + 'linkType': record.linkType, + 'note': loop ? 'Redirect loop' : record.note, + 'orphan': this.crawlMode && inSitemap && !record.isStart && !this.isLinked(record), + 'redirectChain': chain, + 'redirectCount': chain.length, + 'redirectTo': record.redirectTo, + 'responseTime': record.responseTime, + 'scope': record.scope, + 'status': record.status, + 'statusText': record.statusText, + 'title': record.title, + 'type': record.internal ? 'internal' : 'external', + 'url': record.url + }; + } +} + +module.exports = { + Checker, + ISSUE_CATEGORIES, + SCOPE_LABELS, + categorize +}; diff --git a/lib/config.js b/lib/config.js new file mode 100644 index 0000000..9db5387 --- /dev/null +++ b/lib/config.js @@ -0,0 +1,81 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const DEFAULT_NAMES = ['check-http-status.config.json', '.check-http-statusrc.json']; + +// `checkHttpStatus()` options plus `fail` (CLI / Action). +const KNOWN_KEYS = [ + '$schema', 'checkAssets', 'checkExternal', 'concurrency', 'crawl', 'delay', 'exclude', 'export', 'fail', + 'googleDrive', 'ignoreQuery', 'include', 'maxDepth', 'maxPages', 'maxRedirects', 'oneDrive', 'options', + 'respectRobots', 'retries', 'retryDelay', 'silent', 'sitemaps', 'skip200', 'subdomains', 'urls', 'userAgent' +]; + +function findConfig(dir = process.cwd()) { + for (const name of DEFAULT_NAMES) { + const file = path.join(dir, name); + if (fs.existsSync(file)) { + return file; + } + } + + return null; +} + +// Relative paths are resolved against the config file's folder. +function loadConfig(file) { + const absolute = path.resolve(file); + let config; + + if (/\.c?js$/.test(absolute)) { + config = require(absolute); + } else { + let text; + try { + text = fs.readFileSync(absolute, 'utf8'); + } catch (error) { + throw new Error(`Cannot read config file ${file}: ${error.message}`, { 'cause': error }); + } + + try { + config = JSON.parse(text.replace(/^\uFEFF/, '')); + } catch (error) { + throw new Error(`Config file ${file} is not valid JSON: ${error.message}`, { 'cause': error }); + } + } + + if (!config || typeof config !== 'object' || Array.isArray(config)) { + throw new Error(`Config file ${file} must contain an object.`); + } + + const unknown = Object.keys(config).filter((key) => !KNOWN_KEYS.includes(key)); + if (unknown.length) { + throw new Error(`Unknown option(s) in ${file}: ${unknown.join(', ')}.`); + } + + const dir = path.dirname(absolute); + const resolve = (value) => (value && !path.isAbsolute(value) ? path.join(dir, value) : value); + const result = { ...config }; + delete result.$schema; + + if (result.export) { + result.export = (Array.isArray(result.export) ? result.export : [result.export]).map((entry) => ({ + ...entry, + 'file': resolve(entry.file), + 'location': resolve(entry.location) + })); + } + + if (result.googleDrive && typeof result.googleDrive === 'object' && result.googleDrive.keyFile) { + result.googleDrive = { ...result.googleDrive, 'keyFile': resolve(result.googleDrive.keyFile) }; + } + + return result; +} + +module.exports = { + KNOWN_KEYS, + findConfig, + loadConfig +}; diff --git a/lib/console-columns.js b/lib/console-columns.js deleted file mode 100644 index d172faa..0000000 --- a/lib/console-columns.js +++ /dev/null @@ -1,39 +0,0 @@ -'use strict'; - -const columnify = require('columnify'); - -/** - * List down all the URLs with the status code in console. - * - * @param array statusList List of URLs with their status code and other - * information to be logged on console. - */ -function terminalColumns(statusList) { - const data = []; - - // Add empty line - console.log(); - - statusList.forEach((singleStatus) => { - data.push({ - '# Redirects': singleStatus[1], - 'Error Message': singleStatus[4], - 'Requested URL': singleStatus[0], - 'Status Code': singleStatus[2], - 'URL': singleStatus[3], - }); - }); - - console.log(columnify(data, { - 'columnSplitter': '|', - 'columns': [ - 'Requested URL', - '# Redirects', - 'Status Code', - 'URL', - 'Error Message' - ] - })); -} - -module.exports = terminalColumns; diff --git a/lib/fetch-from-sitemaps.js b/lib/fetch-from-sitemaps.js deleted file mode 100644 index cec890d..0000000 --- a/lib/fetch-from-sitemaps.js +++ /dev/null @@ -1,79 +0,0 @@ -'use strict'; - -const axios = require('axios'); -const xml2js = require('xml2js'); - -/** - * Fetch Sitemap URL and dumb all the URLs from the XML to a variable. - * - * @param string sitemapUrl Sitemap URL from which list of URLs needs to be fetched. - * - * @returns array List of URLs fetched from the sitemap. - */ -function fetchFromSitemap(sitemapUrl) { - var errorMessage = 'Error: ' + sitemapUrl + ' returns with status code '; - var fetchedUrls = []; - - return new Promise((resolve) => { - axios.get(sitemapUrl) - .then((response) => { - if (response.status === 200) { - if (response.data.indexOf(' { - for (const readUrl of result.urlset.url) { - fetchedUrls.push(readUrl.loc[0]); - } - }); - } - } - - resolve(fetchedUrls); - }) - .catch((error) => { - if (error.response && error.response.status) { - errorMessage += error.response.status; - console.error('\x1b[31m%s\x1b[0m', errorMessage); - } else { - console.error(error); - } - - resolve(fetchedUrls); - }); - }); -} - -/** - * Fetch URLs from the sitemap one-by-one. - * - * @param array sitemapUrls List of Sitemap URL(s). - * - * @returns array List of URLs fetched from all the sitemap(s). - */ -async function fetchAllSitemaps(sitemapUrls) { - const sitemapUrlsLists = []; - const urlsList = []; - - var mergeLists = []; - for (const sitemapUrl of sitemapUrls) { - sitemapUrlsLists.push(fetchFromSitemap(sitemapUrl)); - } - - const allUrls = await Promise.all(sitemapUrlsLists); - for (const singleList of allUrls) { - mergeLists = mergeLists.concat(singleList); - } - - for (const singleUrl of mergeLists) { - if (!urlsList.includes(singleUrl)) { - urlsList.push(singleUrl); - } - } - - return urlsList; -} - -module.exports = fetchAllSitemaps; diff --git a/lib/generate-excel.js b/lib/generate-excel.js deleted file mode 100644 index 4642feb..0000000 --- a/lib/generate-excel.js +++ /dev/null @@ -1,77 +0,0 @@ -'use strict'; - -const excelJS = require('exceljs'); - -/** - * Generate filename with location to save the exported file. - * - * @param object exportDetails Pass export information like format type, location etc. - * - * @returns string File Location with filename. - */ -function exportFileName(exportDetails) { - const currentTimestamp = Math.floor(Date.now() / 1000); - const fileNameSuffix = currentTimestamp + '.' + exportDetails.format; - const fileName = 'httpstatuschecker-results-' + fileNameSuffix; - const location = exportDetails.location + fileName; - - return location; -} - -/** - * List down all the URLs with the status code in .xlsx format. - * - * @param array statusList List of the URLs needs to bee added in excel file. - * @param object rowLength Length that needs to be set for each column. - * @param object exportDetails Pass export information like format type, location etc. - */ -function generateExcel(statusList, rowLength, exportDetails) { - const location = exportFileName(exportDetails); - const workbook = new excelJS.Workbook(); - const worksheet = workbook.addWorksheet('Sheet 1', { - 'views': [ - { - 'state': 'frozen', - 'xSplit': 1, - 'ySplit': 1 - } - ] - }); - - worksheet.columns = [ - { - 'header': 'Requested URL', - 'key': 'url', - 'width': rowLength.requestedUrl - }, - { - 'header': '# Redirects', - 'key': 'redirects', - 'width': 12 - }, - { - 'header': 'Status Code', - 'key': 'status', - 'width': 12 - }, - { - 'header': 'URL', - 'key': 'url', - 'width': rowLength.url - }, - { - 'header': 'Error Message', - 'key': 'error_message', - 'width': rowLength.errorMessage - }, - ]; - - worksheet.getRow(1).font = { - 'bold': true - }; - - worksheet.addRows(statusList); - workbook[exportDetails.format].writeFile(location); -} - -module.exports = generateExcel; diff --git a/lib/google-drive.js b/lib/google-drive.js new file mode 100644 index 0000000..0c353aa --- /dev/null +++ b/lib/google-drive.js @@ -0,0 +1,277 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); +const { base64url, configDir, loopbackLogin, parseJson, postToken, saveCredentials } = require('./oauth'); + +const SERVICE = 'Google Drive'; +// `drive.file` only sees files this tool created; existing folders need `drive`. +const SCOPE_APP_FILES = 'https://www.googleapis.com/auth/drive.file'; +const SCOPE_FULL = 'https://www.googleapis.com/auth/drive'; +const DEFAULT_FOLDER = 'HTTP Status Reports'; +const FOLDER_MIME = 'application/vnd.google-apps.folder'; +const GOOGLE_SHEET = 'application/vnd.google-apps.spreadsheet'; +const DEFAULT_ENDPOINTS = { + 'auth': 'https://accounts.google.com/o/oauth2/v2/auth', + 'files': 'https://www.googleapis.com/drive/v3/files', + 'token': 'https://oauth2.googleapis.com/token', + 'upload': 'https://www.googleapis.com/upload/drive/v3/files' +}; +const MIME_TYPES = { + 'csv': 'text/csv', + 'html': 'text/html', + 'json': 'application/json', + 'xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' +}; + +function savedLoginPath() { + return path.join(configDir(), 'google-credentials.json'); +} + +// Order: options, environment variables, saved login. +function resolveCredentials(options = {}) { + const read = (file) => parseJson(fs.readFileSync(file, 'utf8'), `${SERVICE}: ${file}`); + let credentials = null; + + if (options.credentials) { + credentials = typeof options.credentials === 'string' ? parseJson(options.credentials, `${SERVICE}: credentials`) : options.credentials; + } else if (options.keyFile) { + credentials = read(options.keyFile); + } else if (options.clientId && options.refreshToken) { + credentials = { + 'client_id': options.clientId, + 'client_secret': options.clientSecret || '', + 'refresh_token': options.refreshToken, + 'type': 'authorized_user' + }; + } else if (process.env.CHS_GOOGLE_CREDENTIALS) { + credentials = parseJson(process.env.CHS_GOOGLE_CREDENTIALS, `${SERVICE}: CHS_GOOGLE_CREDENTIALS`); + } else if (process.env.GOOGLE_APPLICATION_CREDENTIALS) { + credentials = read(process.env.GOOGLE_APPLICATION_CREDENTIALS); + } else if (fs.existsSync(savedLoginPath())) { + credentials = read(savedLoginPath()); + } + + if (!credentials) { + throw new Error(`${SERVICE}: no credentials found. Run "check-http-status drive-login", or set CHS_GOOGLE_CREDENTIALS / GOOGLE_APPLICATION_CREDENTIALS to a service account key.`); + } + + if (credentials.type === 'service_account' && credentials.client_email && credentials.private_key) { + return credentials; + } else if (credentials.refresh_token && credentials.client_id) { + return { ...credentials, 'type': 'authorized_user' }; + } + + throw new Error(`${SERVICE}: credentials must be a service account key or contain client_id, client_secret and refresh_token.`); +} + +async function getAccessToken(credentials, endpoints, scope) { + if (credentials.type === 'service_account') { + const now = Math.floor(Date.now() / 1000); + const header = base64url(JSON.stringify({ 'alg': 'RS256', 'typ': 'JWT' })); + const claims = base64url(JSON.stringify({ + 'aud': endpoints.token, + 'exp': now + 3600, + 'iat': now, + 'iss': credentials.client_email, + 'scope': scope + })); + const signature = crypto.createSign('RSA-SHA256').update(`${header}.${claims}`).sign(credentials.private_key); + const data = await postToken(endpoints.token, { + 'assertion': `${header}.${claims}.${base64url(signature)}`, + 'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer' + }, SERVICE); + + return data.access_token; + } + + const data = await postToken(endpoints.token, { + 'client_id': credentials.client_id, + 'client_secret': credentials.client_secret || '', + 'grant_type': 'refresh_token', + 'refresh_token': credentials.refresh_token + }, SERVICE); + + return data.access_token; +} + +async function driveError(response) { + const data = await response.json().catch(() => null); + const message = data && data.error && (data.error.message || data.error); + let hint = ''; + + if (/storage quota/i.test(String(message))) { + hint = ' - service accounts have no storage of their own: upload into a Shared Drive folder, or use "check-http-status drive-login" instead'; + } else if (response.status === 404) { + hint = ' - check the folder ID and that the folder is shared with this account. To upload into a folder you created yourself, sign in with "check-http-status drive-login --full-access"'; + } + + return `${response.status}${message ? ': ' + message : ''}${hint}`; +} + +// With drive.file, the search only sees folders this tool created. +async function findOrCreateFolder(name, accessToken, endpoints) { + const headers = { 'Authorization': `Bearer ${accessToken}` }; + const query = `name = '${name.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}' and mimeType = '${FOLDER_MIME}' and trashed = false`; + const search = await fetch(`${endpoints.files}?${new URLSearchParams({ 'fields': 'files(id)', 'pageSize': '1', 'q': query, 'spaces': 'drive' })}`, { headers }); + + if (!search.ok) { + throw new Error(`${SERVICE}: could not look up the "${name}" folder (${await driveError(search)}).`); + } + + const found = (await search.json()).files || []; + if (found.length) { + return found[0].id; + } + + const create = await fetch(`${endpoints.files}?fields=id`, { + 'body': JSON.stringify({ 'mimeType': FOLDER_MIME, name }), + 'headers': { ...headers, 'Content-Type': 'application/json' }, + 'method': 'POST' + }); + + if (!create.ok) { + throw new Error(`${SERVICE}: could not create the "${name}" folder (${await driveError(create)}).`); + } + + return (await create.json()).id; +} + +// Resumable upload: works for any file size. +async function uploadFile(file, format, accessToken, options, endpoints) { + const convert = options.convert !== false && (format === 'xlsx' || format === 'csv'); + const name = convert ? path.basename(file, path.extname(file)) : path.basename(file); + const metadata = { name }; + + if (convert) { + metadata.mimeType = GOOGLE_SHEET; + } + + if (options.folderId) { + metadata.parents = [options.folderId]; + } + + const query = 'uploadType=resumable&supportsAllDrives=true&fields=id,name,mimeType,webViewLink'; + const start = await fetch(`${endpoints.upload}?${query}`, { + 'body': JSON.stringify(metadata), + 'headers': { + 'Authorization': `Bearer ${accessToken}`, + 'Content-Type': 'application/json; charset=UTF-8', + 'X-Upload-Content-Type': MIME_TYPES[format] + }, + 'method': 'POST' + }); + + if (!start.ok || !start.headers.get('location')) { + throw new Error(`${SERVICE}: could not start upload of ${path.basename(file)} (${await driveError(start)}).`); + } + + const finish = await fetch(start.headers.get('location'), { + 'body': await fs.promises.readFile(file), + 'headers': { 'Content-Type': MIME_TYPES[format] }, + 'method': 'PUT' + }); + + if (!finish.ok) { + throw new Error(`${SERVICE}: upload of ${path.basename(file)} failed (${await driveError(finish)}).`); + } + + const uploaded = await finish.json(); + const result = { + 'file': file, + 'id': uploaded.id, + 'mimeType': uploaded.mimeType, + 'name': uploaded.name, + 'public': false, + 'service': 'googleDrive', + 'url': uploaded.webViewLink || `https://drive.google.com/file/d/${uploaded.id}/view` + }; + + if (options.public) { + await makePublic(uploaded.id, accessToken, endpoints, uploaded.name); + result.public = true; + } + + return result; +} + +async function makePublic(fileId, accessToken, endpoints, name) { + const response = await fetch(`${endpoints.files}/${encodeURIComponent(fileId)}/permissions?supportsAllDrives=true`, { + 'body': JSON.stringify({ 'role': 'reader', 'type': 'anyone' }), + 'headers': { + 'Authorization': `Bearer ${accessToken}`, + 'Content-Type': 'application/json' + }, + 'method': 'POST' + }); + + if (!response.ok) { + const reason = await driveError(response); + const hint = response.status === 403 ? ' - your Google Workspace admin may block sharing files publicly' : ''; + throw new Error(`${SERVICE}: uploaded ${name} but could not make it public (${reason}${hint}).`); + } +} + +async function uploadToDrive(files, options = {}) { + const endpoints = { ...DEFAULT_ENDPOINTS, ...(options.endpoints || {}) }; + const credentials = resolveCredentials(options); + // A service account only sees folders shared with it, so full scope is safe there. + const scope = credentials.type === 'service_account' && options.folderId ? SCOPE_FULL : SCOPE_APP_FILES; + const accessToken = await getAccessToken(credentials, endpoints, scope); + const folderId = options.folderId || await findOrCreateFolder(options.folderName || DEFAULT_FOLDER, accessToken, endpoints); + const uploads = []; + + for (const { file, format } of files) { + uploads.push(await uploadFile(file, format, accessToken, { ...options, folderId }, endpoints)); + } + + return uploads; +} + +async function login(options) { + const endpoints = { ...DEFAULT_ENDPOINTS, ...(options.endpoints || {}) }; + const { code, redirectUri, verifier } = await loopbackLogin({ + 'authEndpoint': endpoints.auth, + 'log': options.log, + 'openBrowser': options.openBrowser, + 'params': { + 'access_type': 'offline', + 'client_id': options.clientId, + 'prompt': 'consent', + 'scope': options.fullAccess ? SCOPE_FULL : SCOPE_APP_FILES + }, + 'redirectHost': '127.0.0.1', + 'service': SERVICE + }); + + const tokens = await postToken(endpoints.token, { + 'client_id': options.clientId, + 'client_secret': options.clientSecret || '', + 'code': code, + 'code_verifier': verifier, + 'grant_type': 'authorization_code', + 'redirect_uri': redirectUri + }, SERVICE); + + if (!tokens.refresh_token) { + throw new Error(`${SERVICE}: Google did not return a refresh token. Remove the app at https://myaccount.google.com/permissions and try again.`); + } + + const file = options.saveTo || savedLoginPath(); + saveCredentials(file, { + 'client_id': options.clientId, + 'client_secret': options.clientSecret || '', + 'refresh_token': tokens.refresh_token, + 'type': 'authorized_user' + }); + + return file; +} + +module.exports = { + login, + resolveCredentials, + savedLoginPath, + uploadToDrive +}; diff --git a/lib/http-list.js b/lib/http-list.js deleted file mode 100644 index 80143c5..0000000 --- a/lib/http-list.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; - -const checkStatusCode = require('./check-status-code'); - -/** - * Show Progress of HTTP Status. - * - * @param int progressStatus Currently Processing URL(s). - * @param int totalList Total URL(s) needs to be processed. - */ -function printProgress(progressList, totalList) { - process.stdout.clearLine(); - process.stdout.cursorTo(0); - process.stdout.write('Processing ' + progressList + ' of ' + totalList + ' ... '); -} - -/** - * List down all the URLs with the status code. - * - * @param array urlsList List of URLs. - * @param object configOptions config options for Axios like HTTP Auth etc. - * @param bool skip200 Whether to log URLs that returns 200 status code or not. - * - * @returns array Result of each requested with its status code etc. - */ -async function httpList(urlsList, configOptions, skip200) { - const httpStatus = []; - const totalUrls = urlsList.length; - - var combineResults = []; - var currentProgress = 0; - var maxLimit = 0; - - - printProgress(currentProgress, totalUrls); - for (const checkUrl of urlsList) { - httpStatus.push(checkStatusCode(checkUrl, configOptions, skip200)); - currentProgress += 1; - maxLimit += 1; - - printProgress(currentProgress, totalUrls); - - // Resolve previous promises before pushing more. - if (maxLimit === 10) { - await Promise.all(httpStatus); - maxLimit = 0; - } - } - - const httpResults = await Promise.all(httpStatus); - for (const status of httpResults) { - combineResults = combineResults.concat(status); - } - - return combineResults; -} - -module.exports = httpList; diff --git a/lib/oauth.js b/lib/oauth.js new file mode 100644 index 0000000..8ff237a --- /dev/null +++ b/lib/oauth.js @@ -0,0 +1,146 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const http = require('http'); +const os = require('os'); +const path = require('path'); + +const base64url = (input) => Buffer.from(input).toString('base64').replace(/=+$/, '').replace(/\+/g, '-').replace(/\//g, '_'); + +function configDir() { + const base = process.platform === 'win32' + ? (process.env.APPDATA || path.join(os.homedir(), 'AppData', 'Roaming')) + : (process.env.XDG_CONFIG_HOME || path.join(os.homedir(), '.config')); + + return path.join(base, 'check-http-status'); +} + +function saveCredentials(file, data) { + fs.mkdirSync(path.dirname(file), { 'recursive': true }); + fs.writeFileSync(file, JSON.stringify(data, null, 2), { 'mode': 0o600 }); +} + +function parseJson(text, label) { + try { + return JSON.parse(text); + } catch { + throw new Error(`${label} is not valid JSON.`); + } +} + +async function postToken(endpoint, params, service) { + const response = await fetch(endpoint, { + 'body': new URLSearchParams(params), + 'headers': { 'Content-Type': 'application/x-www-form-urlencoded' }, + 'method': 'POST' + }); + const data = await response.json().catch(() => ({})); + + if (!response.ok || !data.access_token) { + const reason = data.error_description ? data.error_description.split(/\r?\n/)[0] : (data.error || response.status); + throw new Error(`${service}: authentication failed (${reason}).`); + } + + return data; +} + +function openBrowser(url) { + const { spawn } = require('child_process'); + const [command, args] = process.platform === 'darwin' + ? ['open', [url]] + : process.platform === 'win32' + ? ['cmd', ['/c', 'start', '""', url.replace(/&/g, '^&')]] + : ['xdg-open', [url]]; + + try { + spawn(command, args, { 'detached': true, 'stdio': 'ignore' }).on('error', () => {}).unref(); + } catch { + // The URL is printed as well, so the user can open it manually. + } +} + +// Browser sign-in with a loopback redirect and PKCE (RFC 8252). +function loopbackLogin(options) { + const verifier = base64url(crypto.randomBytes(32)); + const state = base64url(crypto.randomBytes(16)); + const log = options.log || console.error; + const servers = []; + let redirectUri = ''; + + return new Promise((resolve, reject) => { + let settled = false; + const finish = (error, value) => { + if (settled) { + return; + } + + settled = true; + servers.forEach((server) => server.close()); + if (error) { + reject(error); + } else { + resolve(value); + } + }; + + const handler = (req, res) => { + const url = new URL(req.url, 'http://localhost'); + if (url.pathname !== '/') { + res.writeHead(404); + return res.end(); + } + + const error = url.searchParams.get('error'); + const ok = !error && url.searchParams.get('state') === state && url.searchParams.get('code'); + const description = String(url.searchParams.get('error_description') || error || 'Invalid response.').replace(/[<>&]/g, ''); + + res.writeHead(ok ? 200 : 400, { 'Connection': 'close', 'Content-Type': 'text/html; charset=utf-8' }); + res.end(ok + ? `

check-http-status is connected to ${options.service}.

You can close this tab.

` + : `

${options.service} sign-in failed.

${description}

`); + + if (ok) { + finish(null, { 'code': url.searchParams.get('code'), redirectUri, verifier }); + } else { + finish(new Error(`${options.service} login failed${error ? ': ' + description : '.'}`)); + } + }; + + // "localhost" may resolve to IPv4 or IPv6. + const primary = http.createServer(handler); + servers.push(primary); + primary.on('error', (error) => finish(error)); + primary.listen(0, '127.0.0.1', () => { + const port = primary.address().port; + const secondary = http.createServer(handler); + secondary.on('error', () => {}); + secondary.listen(port, '::1'); + servers.push(secondary); + + redirectUri = `http://${options.redirectHost || '127.0.0.1'}:${port}`; + const authUrl = `${options.authEndpoint}?${new URLSearchParams({ + ...options.params, + 'code_challenge': base64url(crypto.createHash('sha256').update(verifier).digest()), + 'code_challenge_method': 'S256', + 'redirect_uri': redirectUri, + 'response_type': 'code', + 'state': state + })}`; + + log(`Opening your browser to sign in to ${options.service}. If it does not open, visit:\n\n${authUrl}\n`); + (options.openBrowser || openBrowser)(authUrl); + }); + + setTimeout(() => finish(new Error(`${options.service} login timed out after 5 minutes.`)), 5 * 60 * 1000).unref(); + }); +} + +module.exports = { + base64url, + configDir, + loopbackLogin, + parseJson, + postToken, + saveCredentials +}; diff --git a/lib/onedrive.js b/lib/onedrive.js new file mode 100644 index 0000000..a4be484 --- /dev/null +++ b/lib/onedrive.js @@ -0,0 +1,238 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { configDir, loopbackLogin, parseJson, postToken, saveCredentials } = require('./oauth'); + +const SERVICE = 'OneDrive'; +// AppFolder scope only sees "Apps/". +const SCOPE_APP_FOLDER = 'offline_access Files.ReadWrite.AppFolder'; +const SCOPE_FULL = 'offline_access Files.ReadWrite'; +const DEFAULT_ENDPOINTS = { + 'graph': 'https://graph.microsoft.com/v1.0', + 'login': 'https://login.microsoftonline.com' +}; +// Upload session chunks must be a multiple of 320 KiB. +const CHUNK_SIZE = 320 * 1024 * 32; + +function savedLoginPath() { + return path.join(configDir(), 'onedrive-credentials.json'); +} + +// Order: options, CHS_ONEDRIVE_CREDENTIALS, saved login. +function resolveCredentials(options = {}) { + let credentials = null; + + if (options.credentials) { + credentials = typeof options.credentials === 'string' ? parseJson(options.credentials, `${SERVICE}: credentials`) : options.credentials; + } else if (options.clientId && options.refreshToken) { + credentials = { + 'client_id': options.clientId, + 'refresh_token': options.refreshToken, + 'tenant': options.tenant + }; + } else if (process.env.CHS_ONEDRIVE_CREDENTIALS) { + credentials = parseJson(process.env.CHS_ONEDRIVE_CREDENTIALS, `${SERVICE}: CHS_ONEDRIVE_CREDENTIALS`); + } else if (fs.existsSync(savedLoginPath())) { + credentials = { ...parseJson(fs.readFileSync(savedLoginPath(), 'utf8'), `${SERVICE}: ${savedLoginPath()}`), 'savedFile': savedLoginPath() }; + } + + if (!credentials) { + throw new Error(`${SERVICE}: no credentials found. Run "check-http-status onedrive-login", or set CHS_ONEDRIVE_CREDENTIALS.`); + } + + if (!credentials.client_id || !credentials.refresh_token) { + throw new Error(`${SERVICE}: credentials must contain client_id and refresh_token.`); + } + + return { ...credentials, 'tenant': credentials.tenant || 'common' }; +} + +async function getAccessToken(credentials, endpoints) { + const params = { + 'client_id': credentials.client_id, + 'grant_type': 'refresh_token', + 'refresh_token': credentials.refresh_token, + 'scope': credentials.scope || SCOPE_APP_FOLDER + }; + + if (credentials.client_secret) { + params.client_secret = credentials.client_secret; + } + + const data = await postToken(`${endpoints.login}/${encodeURIComponent(credentials.tenant)}/oauth2/v2.0/token`, params, SERVICE); + + // Microsoft rotates refresh tokens; keep the saved login fresh. + if (credentials.savedFile && data.refresh_token && data.refresh_token !== credentials.refresh_token) { + const { savedFile, ...saved } = credentials; + try { + saveCredentials(savedFile, { ...saved, 'refresh_token': data.refresh_token }); + } catch { + // Not fatal: the old refresh token stays valid until it expires. + } + } + + return data.access_token; +} + +async function graphError(response) { + const data = await response.json().catch(() => null); + const message = data && data.error && (data.error.message || data.error.code || data.error); + return `${response.status}${message ? ': ' + message : ''}`; +} + +function itemPath(folderPath, name) { + const encode = (segment) => encodeURIComponent(segment).replace(/'/g, '%27'); + if (!folderPath) { + return `/me/drive/special/approot:/${encode(name)}:`; + } + + const segments = folderPath.split(/[\\/]+/).filter(Boolean).map(encode); + return `/me/drive/root:/${[...segments, encode(name)].join('/')}:`; +} + +async function uploadFile(file, accessToken, options, endpoints) { + const name = path.basename(file); + const session = await fetch(`${endpoints.graph}${itemPath(options.folderPath, name)}/createUploadSession`, { + 'body': JSON.stringify({ 'item': { '@microsoft.graph.conflictBehavior': 'rename' } }), + 'headers': { + 'Authorization': `Bearer ${accessToken}`, + 'Content-Type': 'application/json' + }, + 'method': 'POST' + }); + + if (!session.ok) { + const hint = session.status === 403 && options.folderPath ? ' - uploading into your own folders needs "check-http-status onedrive-login --full-access"' : ''; + throw new Error(`${SERVICE}: could not start upload of ${name} (${await graphError(session)}${hint}).`); + } + + const { uploadUrl } = await session.json(); + const body = await fs.promises.readFile(file); + let item = null; + + // The pre-authenticated upload URL must not receive the Authorization header. + for (let offset = 0; offset < body.length; offset += CHUNK_SIZE) { + const chunk = body.subarray(offset, offset + CHUNK_SIZE); + const end = offset + chunk.length - 1; + const response = await fetch(uploadUrl, { + 'body': chunk, + // fetch sets Content-Length; setting it by hand breaks on Node 20/22. + 'headers': { 'Content-Range': `bytes ${offset}-${end}/${body.length}` }, + 'method': 'PUT' + }); + + if (!response.ok) { + throw new Error(`${SERVICE}: upload of ${name} failed (${await graphError(response)}).`); + } + + if (response.status === 200 || response.status === 201) { + item = await response.json(); + break; + } + + await response.body.cancel().catch(() => {}); + } + + if (!item) { + throw new Error(`${SERVICE}: upload of ${name} did not complete.`); + } + + const result = { + 'file': file, + 'id': item.id, + 'name': item.name, + 'public': false, + 'service': 'oneDrive', + 'url': item.webUrl + }; + + if (options.public) { + result.url = await createPublicLink(item.id, accessToken, endpoints, item.name); + result.public = true; + } + + return result; +} + +async function createPublicLink(itemId, accessToken, endpoints, name) { + const response = await fetch(`${endpoints.graph}/me/drive/items/${encodeURIComponent(itemId)}/createLink`, { + 'body': JSON.stringify({ 'scope': 'anonymous', 'type': 'view' }), + 'headers': { + 'Authorization': `Bearer ${accessToken}`, + 'Content-Type': 'application/json' + }, + 'method': 'POST' + }); + + if (!response.ok) { + const hint = response.status === 403 || response.status === 400 ? ' - your organization may not allow "anyone with the link" sharing' : ''; + throw new Error(`${SERVICE}: uploaded ${name} but could not create a public link (${await graphError(response)}${hint}).`); + } + + return (await response.json()).link.webUrl; +} + +async function uploadToOneDrive(files, options = {}) { + const endpoints = { ...DEFAULT_ENDPOINTS, ...(options.endpoints || {}) }; + const credentials = resolveCredentials(options); + const accessToken = await getAccessToken(credentials, endpoints); + const uploads = []; + + for (const { file } of files) { + uploads.push(await uploadFile(file, accessToken, options, endpoints)); + } + + return uploads; +} + +async function login(options) { + const endpoints = { ...DEFAULT_ENDPOINTS, ...(options.endpoints || {}) }; + const tenant = options.tenant || 'common'; + const scope = options.fullAccess ? SCOPE_FULL : SCOPE_APP_FOLDER; + const base = `${endpoints.login}/${encodeURIComponent(tenant)}/oauth2/v2.0`; + + const { code, redirectUri, verifier } = await loopbackLogin({ + 'authEndpoint': `${base}/authorize`, + 'log': options.log, + 'openBrowser': options.openBrowser, + 'params': { + 'client_id': options.clientId, + 'prompt': 'select_account', + 'scope': scope + }, + // Microsoft matches "http://localhost" redirect URIs on any port. + 'redirectHost': 'localhost', + 'service': SERVICE + }); + + const tokens = await postToken(`${base}/token`, { + 'client_id': options.clientId, + 'code': code, + 'code_verifier': verifier, + 'grant_type': 'authorization_code', + 'redirect_uri': redirectUri, + 'scope': scope + }, SERVICE); + + if (!tokens.refresh_token) { + throw new Error(`${SERVICE}: Microsoft did not return a refresh token (the offline_access permission is required).`); + } + + const file = options.saveTo || savedLoginPath(); + saveCredentials(file, { + 'client_id': options.clientId, + 'refresh_token': tokens.refresh_token, + 'scope': scope, + 'tenant': tenant + }); + + return file; +} + +module.exports = { + login, + resolveCredentials, + savedLoginPath, + uploadToOneDrive +}; diff --git a/lib/patterns.js b/lib/patterns.js new file mode 100644 index 0000000..b8f3efc --- /dev/null +++ b/lib/patterns.js @@ -0,0 +1,31 @@ +'use strict'; + +// Rules: "/regex/flags", "*" wildcard, or a case-insensitive substring. +function compilePatterns(input) { + let rules = input; + if (!Array.isArray(rules)) { + rules = String(rules || '').split(/\r?\n|,/); + } + + return rules + .map((rule) => String(rule).trim()) + .filter(Boolean) + .map((rule) => { + const regex = rule.match(/^\/(.+)\/([imsuy]*)$/); + if (regex) { + return new RegExp(regex[1], regex[2]); + } + + if (rule.includes('*')) { + const escaped = rule.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*'); + return new RegExp(escaped, 'i'); + } + + const needle = rule.toLowerCase(); + return { + 'test': (url) => url.toLowerCase().includes(needle) + }; + }); +} + +module.exports = compilePatterns; diff --git a/lib/report/console.js b/lib/report/console.js new file mode 100644 index 0000000..1d8dcd8 --- /dev/null +++ b/lib/report/console.js @@ -0,0 +1,130 @@ +'use strict'; + +const { ISSUE_CATEGORIES } = require('../checker'); + +const useColor = (stream) => stream.isTTY && !process.env.NO_COLOR; +const paint = (code, text, stream = process.stdout) => (useColor(stream) ? `\x1b[${code}m${text}\x1b[0m` : text); + +const CATEGORY_COLORS = { + 'Blocked': 35, + 'Client Error': 31, + 'Error': 31, + 'Not Found': 31, + 'OK': 32, + 'Redirect': 33, + 'Server Error': 31 +}; + +// Progress line on stderr; silent when stderr isn't a terminal (CI, pipes). +function createProgressPrinter(stream = process.stderr) { + let last = 0; + + return { + clear() { + if (stream.isTTY) { + stream.clearLine(0); + stream.cursorTo(0); + } + }, + update(progress, force = false) { + if (!stream.isTTY || (!force && Date.now() - last < 150)) { + return; + } + + last = Date.now(); + const issues = ISSUE_CATEGORIES.reduce((sum, key) => sum + (progress.counts[key] || 0), 0); + const text = progress.mode === 'crawl' + ? `Crawled ${progress.pagesCrawled} pages · checked ${progress.checked}/${progress.checked + progress.queued + progress.active} URLs · ${issues} issues` + : `Checked ${progress.checked}/${progress.checked + progress.queued + progress.active} URLs · ${issues} issues`; + + stream.clearLine(0); + stream.cursorTo(0); + stream.write(text.slice(0, (stream.columns || 120) - 1)); + } + }; +} + +function printReport(report, options = {}) { + const { summary } = report; + const maxSources = options.maxSources === undefined ? 3 : options.maxSources; + const rows = report.results + .filter((result) => !options.skip200 || result.category !== 'OK') + .sort((a, b) => (a.status || 999) - (b.status || 999) || a.url.localeCompare(b.url)); + + console.log(); + for (const result of rows) { + const label = result.error ? 'ERR' : String(result.status === null ? '---' : result.status); + let line = `${paint(CATEGORY_COLORS[result.category] || 90, label.padEnd(4))} ${result.url}`; + + if (result.redirectTo) { + const final = result.redirectCount > 1 ? ` ⇒ ${result.finalUrl} (${result.finalStatus === null ? '?' : result.finalStatus}, ${result.redirectCount} hops)` : ''; + line += paint(90, ` → ${result.redirectTo}${final}`); + } + + if (result.error) { + line += paint(90, ` ${result.error}`); + } + + if (result.note) { + line += paint(90, ` [${result.note}]`); + } + + console.log(line); + + if (maxSources > 0 && ISSUE_CATEGORIES.includes(result.category)) { + const linked = result.foundOn.filter((source) => source.kind !== 'redirect'); + const sources = linked.length ? linked : result.foundOn; + for (const source of sources.slice(0, maxSources)) { + const anchor = source.anchors.find((text) => text && text[0] !== '('); + const label = source.kind === 'redirect' ? 'redirected from' : 'found on'; + console.log(paint(90, ` ${label} ${source.page}${anchor ? ` ("${anchor}")` : ''}`)); + } + + if (sources.length > maxSources) { + console.log(paint(90, ` … and ${sources.length - maxSources} more page(s)`)); + } + } + } + + printSummary(summary, rows.length === 0 && options.skip200); +} + +function printSummary(summary, allOk) { + const counts = summary.counts; + const parts = [ + paint(32, `${counts['OK'] || 0} OK`), + paint(33, `${counts['Redirect'] || 0} redirects`), + paint(31, `${counts['Not Found'] || 0} not found`), + paint(31, `${(counts['Client Error'] || 0) + (counts['Server Error'] || 0)} other 4xx/5xx`), + paint(31, `${counts['Error'] || 0} errors`) + ]; + if (counts['Blocked']) { + parts.push(paint(35, `${counts['Blocked']} blocked by bot protection`)); + } + + console.log(); + if (allOk) { + console.log(paint(32, 'All the URLs are 200 and there is nothing to worry about!')); + } + + const crawled = summary.mode === 'crawl' ? `${summary.pagesCrawled} pages crawled, ` : ''; + console.log(`${summary.discovered} URLs (${crawled}${Math.round(summary.elapsed / 1000)}s): ${parts.join(', ')}`); + + if (summary.orphans) { + console.log(paint(33, `${summary.orphans} orphan page(s): listed in the sitemap but not linked from any crawled page.`)); + } + + if (summary.robotsBlocked) { + console.log(paint(33, `${summary.robotsBlocked} URL(s) skipped because robots.txt disallows them.`)); + } + + if (summary.pageLimitReached) { + console.log(paint(33, 'Max pages reached - increase maxPages (--max-pages) to crawl more.')); + } +} + +module.exports = { + createProgressPrinter, + paint, + printReport +}; diff --git a/lib/report/csv.js b/lib/report/csv.js new file mode 100644 index 0000000..e8f902d --- /dev/null +++ b/lib/report/csv.js @@ -0,0 +1,45 @@ +'use strict'; + +const fs = require('fs'); +const { urlRows } = require('./rows'); + +const COLUMNS = [ + ['url', 'URL'], + ['foundOn', 'Found On'], + ['redirectTo', 'Redirect To'], + ['status', 'Status'], + ['category', 'Category'], + ['type', 'Type'], + ['linkType', 'Link Type'], + ['scope', 'Crawl'], + ['redirectCount', '# Redirects'], + ['finalUrl', 'Final URL'], + ['finalStatus', 'Final Status'], + ['title', 'Title'], + ['contentType', 'Content Type'], + ['depth', 'Depth'], + ['responseTime', 'Response (ms)'], + ['inSitemap', 'In Sitemap'], + ['orphan', 'Orphan'], + ['foundOnCount', '# Found On'] +]; + +function cell(value) { + const text = String(value === null || value === undefined ? '' : value); + return /[",\n\r]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; +} + +// UTF-8 BOM so Excel detects the encoding. +async function writeCsv(report, file, options = {}) { + const results = options.skip200 ? report.results.filter((result) => result.category !== 'OK') : report.results; + const rows = urlRows(results); + const lines = [COLUMNS.map(([, header]) => cell(header)).join(',')]; + + for (const row of rows) { + lines.push(COLUMNS.map(([key]) => cell(row[key])).join(',')); + } + + await fs.promises.writeFile(file, '\ufeff' + lines.join('\r\n') + '\r\n'); +} + +module.exports = writeCsv; diff --git a/lib/report/excel.js b/lib/report/excel.js new file mode 100644 index 0000000..861ac6d --- /dev/null +++ b/lib/report/excel.js @@ -0,0 +1,176 @@ +'use strict'; + +const { externalRows, issueRows, urlRows } = require('./rows'); + +const HEADER_FILL = 'FF1F3A5F'; +const CATEGORY_FILLS = { + 'Blocked': 'FFEDE7F6', + 'Client Error': 'FFFDE2E1', + 'Error': 'FFFBD3D0', + 'Not Found': 'FFFDE2E1', + 'OK': 'FFE6F4EA', + 'Redirect': 'FFFFF4D6', + 'Server Error': 'FFFBD3D0' +}; + +// Every sheet has one row per URL; all pages a URL was found on share one cell. +const URL = { 'header': 'URL', 'key': 'url', 'width': 60 }; +const FOUND_ON = { 'header': 'Found On', 'key': 'foundOn', 'width': 60 }; +const REDIRECT_TO = { 'header': 'Redirect To', 'key': 'redirectTo', 'width': 50 }; +const STATUS = { 'header': 'Status', 'key': 'status', 'width': 22 }; +const CATEGORY = { 'header': 'Category', 'key': 'category', 'width': 14 }; +const FINAL_URL = { 'header': 'Final URL', 'key': 'finalUrl', 'width': 50 }; +const FINAL_STATUS = { 'header': 'Final Status', 'key': 'finalStatus', 'width': 12 }; +const FOUND_ON_COUNT = { 'header': '# Found On', 'key': 'foundOnCount', 'width': 11 }; + +const URL_COLUMNS = [ + URL, + FOUND_ON, + REDIRECT_TO, + STATUS, + CATEGORY, + { 'header': 'Type', 'key': 'type', 'width': 10 }, + { 'header': 'Link Type', 'key': 'linkType', 'width': 10 }, + { 'header': 'Crawl', 'key': 'scope', 'width': 22 }, + { 'header': '# Redirects', 'key': 'redirectCount', 'width': 11 }, + FINAL_URL, + FINAL_STATUS, + { 'header': 'Title', 'key': 'title', 'width': 40 }, + { 'header': 'Content Type', 'key': 'contentType', 'width': 20 }, + { 'header': 'Depth', 'key': 'depth', 'width': 7 }, + { 'header': 'Response (ms)', 'key': 'responseTime', 'width': 13 }, + { 'header': 'In Sitemap', 'key': 'inSitemap', 'width': 10 }, + { 'header': 'Orphan', 'key': 'orphan', 'width': 8 }, + FOUND_ON_COUNT +]; + +const ISSUE_COLUMNS = [ + URL, + FOUND_ON, + REDIRECT_TO, + STATUS, + CATEGORY, + { 'header': 'Type', 'key': 'type', 'width': 10 }, + FINAL_URL, + FINAL_STATUS, + FOUND_ON_COUNT +]; + +const EXTERNAL_COLUMNS = [ + URL, + FOUND_ON, + REDIRECT_TO, + STATUS, + CATEGORY, + { 'header': 'Link Type', 'key': 'linkType', 'width': 10 }, + FINAL_URL, + FINAL_STATUS, + FOUND_ON_COUNT +]; + +function addTable(workbook, name, columns, rows) { + const worksheet = workbook.addWorksheet(name, { + 'views': [ + { + 'state': 'frozen', + 'ySplit': 1 + } + ] + }); + + worksheet.columns = columns; + worksheet.getColumn('foundOn').alignment = { 'vertical': 'top', 'wrapText': true }; + worksheet.getRow(1).font = { + 'bold': true, + 'color': { 'argb': 'FFFFFFFF' } + }; + worksheet.getRow(1).fill = { + 'fgColor': { 'argb': HEADER_FILL }, + 'pattern': 'solid', + 'type': 'pattern' + }; + + for (const row of rows) { + const added = worksheet.addRow(row); + const fill = CATEGORY_FILLS[row.category]; + if (fill) { + added.getCell('category').fill = { + 'fgColor': { 'argb': fill }, + 'pattern': 'solid', + 'type': 'pattern' + }; + } + } + + if (rows.length) { + worksheet.autoFilter = { + 'from': { 'column': 1, 'row': 1 }, + 'to': { 'column': columns.length, 'row': 1 } + }; + } + + return worksheet; +} + +function addSummary(workbook, report) { + const { summary } = report; + const counts = summary.counts; + const worksheet = workbook.addWorksheet('Summary'); + + worksheet.columns = [{ 'width': 34 }, { 'width': 80 }]; + const lines = [ + ['Report', 'check-http-status'], + ['Mode', summary.mode === 'crawl' ? 'Crawl website' : 'Check URL list'], + ['Start URL(s)', summary.crawl.join('\n')], + ['Sitemap(s)', summary.sitemaps.join('\n')], + ['Generated', new Date(summary.finishedAt || Date.now()).toLocaleString()], + ['Duration (s)', Math.round(summary.elapsed / 1000)], + ['Status', summary.state], + ['URLs found', summary.discovered], + ['Pages crawled', summary.pagesCrawled], + ['External links (not crawled)', report.results.filter((result) => result.type === 'external').length], + ['Max pages reached', summary.pageLimitReached ? 'Yes - increase maxPages to crawl more' : 'No'], + [], + ['OK (2xx)', counts['OK'] || 0], + ['Redirect (3xx)', counts['Redirect'] || 0], + ['Not Found (404 / 410)', counts['Not Found'] || 0], + ['Client Error (other 4xx)', counts['Client Error'] || 0], + ['Server Error (5xx)', counts['Server Error'] || 0], + ['Error (timeout / DNS / connection)', counts['Error'] || 0], + ['Blocked by bot protection (check in a browser)', counts['Blocked'] || 0], + ['Not checked', counts['Not checked'] || 0], + ['Orphan pages (in sitemap, not linked)', summary.orphans], + ['Blocked by robots.txt', summary.robotsBlocked || 0] + ]; + + if (summary.include.length || summary.exclude.length) { + lines.push([], ['Include rules', summary.include.join('\n') || '(none)'], ['Exclude rules', summary.exclude.join('\n') || '(none)']); + } + + lines.forEach((line) => worksheet.addRow(line)); + worksheet.getColumn(1).font = { 'bold': true }; + worksheet.getColumn(2).alignment = { 'vertical': 'top', 'wrapText': true }; +} + +async function writeExcel(report, file, options = {}) { + // Loaded on demand: exceljs adds ~0.3s to startup and is only needed here. + const excelJS = require('exceljs'); + const workbook = new excelJS.Workbook(); + workbook.creator = 'check-http-status'; + workbook.created = new Date(); + + const results = options.skip200 ? report.results.filter((result) => result.category !== 'OK') : report.results; + + addSummary(workbook, report); + addTable(workbook, options.skip200 ? 'URLs (non-200)' : 'All URLs', URL_COLUMNS, urlRows(results)); + addTable(workbook, 'Issues', ISSUE_COLUMNS, issueRows(report.results)); + + const external = externalRows(report.results); + if (external.length) { + addTable(workbook, 'External Links', EXTERNAL_COLUMNS, external); + } + + await workbook.xlsx.writeFile(file); +} + +module.exports = writeExcel; diff --git a/lib/report/html.js b/lib/report/html.js new file mode 100644 index 0000000..96a0500 --- /dev/null +++ b/lib/report/html.js @@ -0,0 +1,26 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +const TEMPLATE = path.join(__dirname, 'template.html'); + +function renderHtml(report, options = {}) { + const template = fs.readFileSync(TEMPLATE, 'utf8'); + // Escape "<" so URLs or page titles can never close the + + + diff --git a/lib/request.js b/lib/request.js new file mode 100644 index 0000000..ed5b56b --- /dev/null +++ b/lib/request.js @@ -0,0 +1,242 @@ +'use strict'; + +const zlib = require('zlib'); + +const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; check-http-status/2.0; +https://github.com/trunkcode/check-http-status)'; +const MAX_BODY_BYTES = 15 * 1024 * 1024; +const MAX_RETRY_WAIT = 60 * 1000; +// Temporary failures only; DNS, refused and TLS errors aren't retried. +const RETRY_STATUSES = [429, 502, 503, 504]; +const RETRY_ERRORS = ['ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN', 'EPIPE', 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'TIMEOUT']; + +class AbortedError extends Error { + constructor() { + super('Aborted'); + this.name = 'AbortedError'; + } +} + +const sleep = (ms, signal) => new Promise((resolve, reject) => { + if (signal && signal.aborted) { + return reject(new AbortedError()); + } + + const onAbort = () => { + clearTimeout(timer); + reject(new AbortedError()); + }; + const timer = setTimeout(() => { + if (signal) { + signal.removeEventListener('abort', onAbort); + } + resolve(); + }, ms); + + if (signal) { + signal.addEventListener('abort', onAbort, { 'once': true }); + } +}); + +// Challenge pages a browser would pass; not broken links. +function detectBotProtection(response) { + if (![403, 429, 503].includes(response.status)) { + return ''; + } + + const header = (name) => response.headers.get(name); + if (header('cf-mitigated') === 'challenge') { + return 'Cloudflare'; + } else if (header('x-datadome') || /datadome=/.test(header('set-cookie') || '')) { + return 'DataDome'; + } else if (/_abck=|ak_bmsc=/.test(header('set-cookie') || '')) { + return 'Akamai Bot Manager'; + } else if (header('x-px-blocked') || /_px[23]?=/.test(header('set-cookie') || '')) { + return 'PerimeterX'; + } else if (header('x-sucuri-block')) { + return 'Sucuri'; + } + + return ''; +} + +// Seconds or HTTP date from a Retry-After header, in milliseconds. +function retryAfterMs(header) { + if (!header) { + return null; + } + + const seconds = Number(header); + if (!Number.isNaN(seconds)) { + return seconds * 1000; + } + + const date = Date.parse(header); + return Number.isNaN(date) ? null : Math.max(0, date - Date.now()); +} + +// Redirects are never followed automatically, so every hop can be reported. +function createRequester(options) { + const headers = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en', + 'User-Agent': options.userAgent || DEFAULT_USER_AGENT, + ...(options.headers || {}) + }; + const timeout = options.timeout || 15000; + const retries = options.retries === undefined ? 2 : options.retries; + const retryDelay = options.retryDelay === undefined ? 1000 : options.retryDelay; + const delay = options.delay || 0; + const signal = options.signal; + const authHosts = new Set((options.authHosts || []).map((host) => host.toLowerCase())); + const nextSlot = new Map(); + // Per-host minimum spacing, e.g. a robots.txt Crawl-delay. + const hostDelays = new Map(); + + if (signal) { + // One abort listener per in-flight request. + require('events').setMaxListeners(1000, signal); + } + + let authHeader = ''; + if (options.auth && (options.auth.username || options.auth.password)) { + const token = Buffer.from(`${options.auth.username || ''}:${options.auth.password || ''}`).toString('base64'); + authHeader = `Basic ${token}`; + } + + async function waitForSlot(host) { + const spacing = Math.max(delay, hostDelays.get(host) || 0); + if (!spacing) { + return; + } + + const now = Date.now(); + const slot = Math.max(now, nextSlot.get(host) || 0); + nextSlot.set(host, slot + spacing); + if (slot > now) { + await sleep(slot - now, signal); + } + } + + async function attempt(url, method, readBody) { + if (signal && signal.aborted) { + throw new AbortedError(); + } + + const host = new URL(url).host.toLowerCase(); + await waitForSlot(host); + + const controller = new AbortController(); + const onAbort = () => controller.abort(); + const timer = setTimeout(() => controller.abort(), timeout); + const requestHeaders = { ...headers }; + const startedAt = Date.now(); + + if (signal) { + signal.addEventListener('abort', onAbort, { 'once': true }); + } + + // Never send credentials to external links. + if (authHeader && authHosts.has(new URL(url).hostname.toLowerCase())) { + requestHeaders['Authorization'] = authHeader; + } + + try { + const response = await fetch(url, { + 'headers': requestHeaders, + 'method': method, + 'redirect': 'manual', + 'signal': controller.signal + }); + const contentType = response.headers.get('content-type') || ''; + const contentLength = parseInt(response.headers.get('content-length') || '0', 10); + + let body = null; + if (readBody && response.ok && contentLength <= MAX_BODY_BYTES) { + body = Buffer.from(await response.arrayBuffer()); + if (body.length > MAX_BODY_BYTES) { + body = null; + } + } else if (response.body) { + await response.body.cancel().catch(() => {}); + } + + return { + 'body': body, + 'contentType': contentType, + 'location': response.headers.get('location'), + 'responseTime': Date.now() - startedAt, + 'botProtection': detectBotProtection(response), + 'retryAfter': response.headers.get('retry-after'), + 'status': response.status, + 'statusText': response.statusText + }; + } catch (error) { + if (signal && signal.aborted) { + throw new AbortedError(); + } + + if (error.name === 'AbortError') { + const timeoutError = new Error(`Timeout after ${timeout / 1000}s`); + timeoutError.code = 'TIMEOUT'; + throw timeoutError; + } + + const cause = error.cause || {}; + const wrapped = new Error(cause.code ? `${cause.code}${cause.message ? ': ' + cause.message : ''}` : error.message); + wrapped.code = cause.code; + throw wrapped; + } finally { + clearTimeout(timer); + if (signal) { + signal.removeEventListener('abort', onAbort); + } + } + } + + async function request(url, { method = 'GET', readBody = false } = {}) { + for (let tryNumber = 0; ; tryNumber++) { + let response; + try { + response = await attempt(url, method, readBody); + } catch (error) { + if (error instanceof AbortedError || tryNumber >= retries || !RETRY_ERRORS.includes(error.code)) { + throw error; + } + + await sleep(retryDelay * 2 ** tryNumber, signal); + continue; + } + + if (tryNumber >= retries || !RETRY_STATUSES.includes(response.status) || response.botProtection) { + response.retries = tryNumber; + return response; + } + + const wait = retryAfterMs(response.retryAfter); + await sleep(Math.min(MAX_RETRY_WAIT, wait === null ? retryDelay * 2 ** tryNumber : wait), signal); + } + } + + request.setHostDelay = (host, ms) => hostDelays.set(host.toLowerCase(), ms); + return request; +} + +// Gunzips .xml.gz sitemaps. +function bodyToText(body) { + if (!body) { + return ''; + } + + if (body[0] === 0x1f && body[1] === 0x8b) { + return zlib.gunzipSync(body).toString('utf8'); + } + + return body.toString('utf8'); +} + +module.exports = { + AbortedError, + DEFAULT_USER_AGENT, + bodyToText, + createRequester +}; diff --git a/lib/robots.js b/lib/robots.js new file mode 100644 index 0000000..ce5b87d --- /dev/null +++ b/lib/robots.js @@ -0,0 +1,157 @@ +'use strict'; + +const { DEFAULT_USER_AGENT, bodyToText } = require('./request'); + +const MAX_CRAWL_DELAY = 30 * 1000; + +// e.g. "MyBot/1.0 (+https://…)" → "mybot" +function productToken(userAgent) { + const ua = userAgent || DEFAULT_USER_AGENT; + const compatible = ua.match(/compatible;\s*([^/;\s)]+)/i); + const token = compatible ? compatible[1] : ua.split(/[\s/]/)[0]; + return token.toLowerCase(); +} + +// RFC 9309 +function parseRobots(text, token) { + const groups = []; + let current = null; + let lastWasAgent = false; + + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.replace(/#.*$/, '').trim(); + const match = line.match(/^([a-z-]+)\s*:\s*(.*)$/i); + if (!match) { + continue; + } + + const field = match[1].toLowerCase(); + const value = match[2].trim(); + + if (field === 'user-agent') { + // Consecutive user-agent lines share one group. + if (!current || !lastWasAgent) { + current = { 'agents': [], 'crawlDelay': null, 'rules': [] }; + groups.push(current); + } + current.agents.push(value.toLowerCase()); + lastWasAgent = true; + continue; + } + + lastWasAgent = false; + if (!current) { + continue; + } + + if ((field === 'allow' || field === 'disallow') && value) { + current.rules.push({ 'allow': field === 'allow', 'pattern': value }); + } else if (field === 'crawl-delay' && !Number.isNaN(Number(value))) { + current.crawlDelay = Number(value) * 1000; + } + } + + // Use the groups naming this crawler; fall back to "*". + let matching = groups.filter((group) => group.agents.includes(token)); + if (!matching.length) { + matching = groups.filter((group) => group.agents.includes('*')); + } + + const delays = matching.map((group) => group.crawlDelay).filter((delay) => delay !== null); + return { + 'crawlDelay': delays.length ? Math.min(MAX_CRAWL_DELAY, Math.max(...delays)) : null, + 'rules': matching.flatMap((group) => group.rules) + }; +} + +function patternToRegExp(pattern) { + const anchored = pattern.endsWith('$'); + const body = (anchored ? pattern.slice(0, -1) : pattern) + .replace(/[.+?^{}()|[\]\\]/g, '\\$&') + .replace(/\*/g, '.*'); + return new RegExp('^' + body + (anchored ? '$' : '')); +} + +// The longest matching rule wins; Allow wins a tie. +function isAllowed(rules, url) { + const target = url.pathname + url.search; + let best = null; + + for (const rule of rules) { + let pathToMatch = target; + try { + // Rules may be written percent-encoded or not; compare like with like. + pathToMatch = /%[0-9a-f]{2}/i.test(rule.pattern) ? target : decodeURI(target); + } catch { + // Keep the raw path if it isn't valid percent-encoding. + } + + if (!patternToRegExp(rule.pattern).test(pathToMatch)) { + continue; + } + + const length = rule.pattern.length; + if (!best || length > best.length || (length === best.length && rule.allow)) { + best = { 'allow': rule.allow, length }; + } + } + + return !best || best.allow; +} + +function createRobots(request, userAgent) { + const token = productToken(userAgent); + const cache = new Map(); + + function load(origin) { + if (!cache.has(origin)) { + cache.set(origin, (async () => { + try { + // RFC 9309: follow up to five redirects. + let url = `${origin}/robots.txt`; + let response = await request(url, { 'readBody': true }); + for (let hops = 0; hops < 5 && response.status >= 300 && response.status < 400 && response.location; hops++) { + url = new URL(response.location, url).href; + response = await request(url, { 'readBody': true }); + } + + if (response.status >= 200 && response.status < 300) { + return parseRobots(bodyToText(response.body), token); + } else if (response.status >= 500) { + // RFC 9309: an unreachable robots.txt means "disallow everything". + return { 'crawlDelay': null, 'disallowAll': true, 'rules': [] }; + } + } catch (error) { + if (error.name === 'AbortedError') { + throw error; + } + return { 'crawlDelay': null, 'disallowAll': true, 'rules': [] }; + } + + // 4xx (usually 404): no restrictions. + return { 'crawlDelay': null, 'rules': [] }; + })()); + } + + return cache.get(origin); + } + + return { + async check(href) { + const url = new URL(href); + const robots = await load(url.origin); + return { + 'allowed': !robots.disallowAll && isAllowed(robots.rules, url), + 'crawlDelay': robots.crawlDelay + }; + }, + token + }; +} + +module.exports = { + createRobots, + isAllowed, + parseRobots, + productToken +}; diff --git a/lib/sitemaps.js b/lib/sitemaps.js new file mode 100644 index 0000000..ff71ee8 --- /dev/null +++ b/lib/sitemaps.js @@ -0,0 +1,90 @@ +'use strict'; + +const cheerio = require('cheerio'); +const { bodyToText } = require('./request'); + +const MAX_SITEMAPS = 500; + +// Supports , and gzipped sitemaps. +async function fetchSitemaps(sitemapUrls, request, onError = () => {}) { + const queue = [...sitemapUrls]; + const seenSitemaps = new Set(); + const pages = new Map(); + + while (queue.length && seenSitemaps.size < MAX_SITEMAPS) { + const batch = queue.splice(0, 5).filter((url) => !seenSitemaps.has(url)); + batch.forEach((url) => seenSitemaps.add(url)); + + await Promise.all(batch.map(async (sitemapUrl) => { + try { + const { locs, isIndex } = await fetchSitemap(sitemapUrl, request); + for (const loc of locs) { + if (isIndex) { + queue.push(loc); + } else if (!pages.has(loc)) { + pages.set(loc, sitemapUrl); + } + } + } catch (error) { + onError(sitemapUrl, error.message); + } + })); + } + + return [...pages].map(([url, sitemap]) => ({ + 'sitemap': sitemap, + 'url': url + })); +} + +async function fetchSitemap(sitemapUrl, request) { + let url = sitemapUrl; + let response; + + for (let hops = 0; hops <= 10; hops++) { + response = await request(url, { + 'readBody': true + }); + + if (response.status >= 300 && response.status < 400 && response.location) { + url = new URL(response.location, url).href; + continue; + } + + break; + } + + if (response.status !== 200) { + throw new Error(`HTTP ${response.status}`); + } + + const xml = bodyToText(response.body); + if (!/<(urlset|sitemapindex)[\s>]/i.test(xml)) { + throw new Error('Not a sitemap XML file'); + } + + const $ = cheerio.load(xml, { + 'xml': true + }); + const isIndex = $('sitemapindex').length > 0; + const selector = isIndex ? 'sitemapindex > sitemap > loc' : 'urlset > url > loc'; + const locs = $(selector) + .map((_, el) => $(el).text().trim()) + .get() + .filter(Boolean) + .map((loc) => { + try { + return new URL(loc, url).href; + } catch { + return null; + } + }) + .filter(Boolean); + + return { + 'isIndex': isIndex, + 'locs': locs + }; +} + +module.exports = fetchSitemaps; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..d244fa0 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,2743 @@ +{ + "name": "check-http-status", + "version": "2.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "check-http-status", + "version": "2.0.0", + "license": "MIT", + "dependencies": { + "cheerio": "^1.0.0", + "exceljs": "^4.4.0" + }, + "bin": { + "check-http-status": "bin/check-http-status.js" + }, + "devDependencies": { + "@eslint/js": "^10.0.1", + "@types/node": "^26.6.4", + "eslint": "^10.11.0", + "globals": "^17.13.0", + "typescript": "^7.0.2" + }, + "engines": { + "node": ">=18.17" + } + }, + "node_modules/@cacheable/memory": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@cacheable/memory/-/memory-2.2.0.tgz", + "integrity": "sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.5.0", + "@keyv/bigmap": "^1.3.1", + "hookified": "^1.15.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/utils": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.5.0.tgz", + "integrity": "sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.5.1", + "keyv": "^5.6.0" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/eslint-utils/node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/config-array": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/@eslint/config-array/-/config-array-0.23.5.tgz", + "integrity": "sha512-Y3kKLvC1dvTOT+oGlqNQ1XLqK6D1HU2YXPc52NmAlJZbMMWDzGYXMiPRJ8TYD39muD/OTjlZmNJ4ib7dvSrMBA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/object-schema": "^3.0.5", + "debug": "^4.3.1", + "minimatch": "^10.2.4" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/config-array/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/@eslint/config-array/node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/@eslint/config-array/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@eslint/config-helpers": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@eslint/config-helpers/-/config-helpers-0.7.0.tgz", + "integrity": "sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/core": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@eslint/core/-/core-1.2.1.tgz", + "integrity": "sha512-MwcE1P+AZ4C6DWlpin/OmOA54mmIZ/+xZuJiQd4SyB29oAJjN30UW9wkKNptW2ctp4cEsvhlLY/CsQ1uoHDloQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@types/json-schema": "^7.0.15" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/js": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-10.0.1.tgz", + "integrity": "sha512-zeR9k5pd4gxjZ0abRoIaxdc7I3nDktoXZk2qOv9gCNWx3mVwEn32VRhyLaRsDiJjTs0xq/T8mfPtyuXu7GWBcA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "eslint": "^10.0.0" + }, + "peerDependenciesMeta": { + "eslint": { + "optional": true + } + } + }, + "node_modules/@eslint/object-schema": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@eslint/object-schema/-/object-schema-3.0.5.tgz", + "integrity": "sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@eslint/plugin-kit": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/@eslint/plugin-kit/-/plugin-kit-0.7.3.tgz", + "integrity": "sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@eslint/core": "^1.2.1", + "levn": "^0.4.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + } + }, + "node_modules/@fast-csv/format": { + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/@fast-csv/format/-/format-4.3.5.tgz", + "integrity": "sha512-8iRn6QF3I8Ak78lNAa+Gdl5MJJBM5vRHivFtMRUWINdevNo00K7OXxS2PshawLKTejVwieIlPmK5YlLu6w4u8A==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.isboolean": "^3.0.3", + "lodash.isequal": "^4.5.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0" + } + }, + "node_modules/@fast-csv/format/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, + "node_modules/@fast-csv/parse": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@fast-csv/parse/-/parse-4.3.6.tgz", + "integrity": "sha512-uRsLYksqpbDmWaSmzvJcuApSEe38+6NQZBUsuAyMZKqHxH0g1wcJgsKUvN3WC8tewaqFjBMMGrkHmC+T7k8LvA==", + "license": "MIT", + "dependencies": { + "@types/node": "^14.0.1", + "lodash.escaperegexp": "^4.1.2", + "lodash.groupby": "^4.6.0", + "lodash.isfunction": "^3.0.9", + "lodash.isnil": "^4.0.0", + "lodash.isundefined": "^3.0.1", + "lodash.uniq": "^4.5.0" + } + }, + "node_modules/@fast-csv/parse/node_modules/@types/node": { + "version": "14.18.63", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.18.63.tgz", + "integrity": "sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==", + "license": "MIT" + }, + "node_modules/@humanfs/core": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", + "integrity": "sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/types": "^0.15.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/node": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@humanfs/node/-/node-0.16.8.tgz", + "integrity": "sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanfs/core": "^0.19.2", + "@humanfs/types": "^0.15.0", + "@humanwhocodes/retry": "^0.4.0" + }, + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanfs/types": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/@humanfs/types/-/types-0.15.0.tgz", + "integrity": "sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/retry": { + "version": "0.4.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/retry/-/retry-0.4.3.tgz", + "integrity": "sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.18" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@keyv/bigmap": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@keyv/bigmap/-/bigmap-1.3.1.tgz", + "integrity": "sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hashery": "^1.4.0", + "hookified": "^1.15.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "keyv": "^5.6.0" + } + }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/esrecurse": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz", + "integrity": "sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "26.6.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.6.4.tgz", + "integrity": "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.9.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/acorn": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.18.0.tgz", + "integrity": "sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/archiver": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-5.3.2.tgz", + "integrity": "sha512-+25nxyyznAXF7Nef3y0EbBeqmGZgeN/BxHX29Rs39djAfaFalmQ89SE6CWyDCHzGL0yt/ycBtNOmGTW0FyGWNw==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^2.1.0", + "async": "^3.2.4", + "buffer-crc32": "^0.2.1", + "readable-stream": "^3.6.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^2.2.0", + "zip-stream": "^4.1.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/archiver-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-2.1.0.tgz", + "integrity": "sha512-bEL/yUb/fNNiNTuUz979Z0Yg5L+LzLxGJz8x79lYmR54fmTIb6ob/hNQgkQnIUDWIFjZVQwl9Xs356I6BAMHfw==", + "license": "MIT", + "dependencies": { + "glob": "^7.1.4", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/archiver-utils/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/archiver-utils/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/big-integer": { + "version": "1.6.52", + "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", + "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", + "license": "Unlicense", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/binary": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", + "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", + "license": "MIT", + "dependencies": { + "buffers": "~0.1.1", + "chainsaw": "~0.1.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bluebird": { + "version": "3.4.7", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", + "integrity": "sha512-iD3898SR7sWVRHbiQv+sHUtHnMvC1o3nW5rAcqnq3uOn07DSAppZYUkIGslDz6gXC7HfunPe7YVBgoEJASPcHA==", + "license": "MIT" + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/brace-expansion": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/buffer-indexof-polyfill": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", + "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/buffers": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", + "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", + "engines": { + "node": ">=0.2.0" + } + }, + "node_modules/cacheable": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/cacheable/-/cacheable-2.5.0.tgz", + "integrity": "sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cacheable/memory": "^2.2.0", + "@cacheable/utils": "^2.5.0", + "hookified": "^1.15.0", + "keyv": "^5.6.0", + "qified": "^0.10.1" + } + }, + "node_modules/chainsaw": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", + "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", + "license": "MIT/X11", + "dependencies": { + "traverse": ">=0.3.0 <0.4" + }, + "engines": { + "node": "*" + } + }, + "node_modules/cheerio": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", + "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.1.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.19.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/compress-commons": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-4.1.2.tgz", + "integrity": "sha512-D3uMHtGc/fcO1Gt1/L7i1e33VOvD4A9hfQLP+6ewd+BvG/gQ84Yh4oftEhAdjSMgBgwGL+jsppT7JYNpo6MHHg==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "^0.2.13", + "crc32-stream": "^4.0.2", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-4.0.3.tgz", + "integrity": "sha512-NT7w2JVU7DFroFdYkeq8cywxrgjPHWkdX1wjpRQXPX5Asews3tA+Ght6lddQO5Mkumffp3X7GEqku3epj2toIw==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^3.4.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/dayjs": { + "version": "1.11.23", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.23.tgz", + "integrity": "sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "10.11.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-10.11.0.tgz", + "integrity": "sha512-P7a6UEEqb9G95MYAtqkmsTbVXIYyzIfl6NGOIJk162PaahFxFyeGcrlXYFSiagECg4sEm8IseJdZBKR3rx6MsQ==", + "dev": true, + "license": "MIT", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@eslint-community/eslint-utils": "^4.8.0", + "@eslint-community/regexpp": "^4.12.2", + "@eslint/config-array": "^0.23.5", + "@eslint/config-helpers": "^0.7.0", + "@eslint/core": "^1.2.1", + "@eslint/plugin-kit": "^0.7.3", + "@humanfs/node": "^0.16.6", + "@humanwhocodes/module-importer": "^1.0.1", + "@humanwhocodes/retry": "^0.4.2", + "@types/estree": "^1.0.6", + "ajv": "^6.14.0", + "cross-spawn": "^7.0.6", + "debug": "^4.3.2", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^9.1.2", + "eslint-visitor-keys": "^5.0.1", + "espree": "^11.2.0", + "esquery": "^1.7.0", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "11.1.5 || >11.1.6 <12", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "minimatch": "^10.2.5", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } + } + }, + "node_modules/eslint-scope": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-9.1.2.tgz", + "integrity": "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "@types/esrecurse": "^4.3.1", + "@types/estree": "^1.0.8", + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint/node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/eslint/node_modules/brace-expansion": { + "version": "5.0.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", + "integrity": "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/eslint/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/espree": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-11.2.0.tgz", + "integrity": "sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.16.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^5.0.1" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/exceljs": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/exceljs/-/exceljs-4.4.0.tgz", + "integrity": "sha512-XctvKaEMaj1Ii9oDOqbW/6e1gXknSY4g/aLCDicOXqBE4M0nRWkUu0PTp++UPNzoFY12BNHMfs/VadKIS6llvg==", + "license": "MIT", + "dependencies": { + "archiver": "^5.0.0", + "dayjs": "^1.8.34", + "fast-csv": "^4.3.1", + "jszip": "^3.10.1", + "readable-stream": "^3.6.0", + "saxes": "^5.0.1", + "tmp": "^0.2.0", + "unzipper": "^0.10.11", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=8.3.0" + } + }, + "node_modules/fast-csv": { + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-4.3.6.tgz", + "integrity": "sha512-2RNSpuwwsJGP0frGsOmTb9oUF+VkFSM4SyLTDgwf2ciHWTarN0lQTC+F2f/t5J9QjW+c65VFIAAu85GsvMIusw==", + "license": "MIT", + "dependencies": { + "@fast-csv/format": "4.3.5", + "@fast-csv/parse": "4.3.6" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/file-entry-cache": { + "version": "11.1.5", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-11.1.5.tgz", + "integrity": "sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^6.1.23" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "6.1.23", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-6.1.23.tgz", + "integrity": "sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "cacheable": "^2.5.0", + "flatted": "^3.4.2", + "hookified": "^1.15.0" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/fstream": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", + "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", + "deprecated": "This package is no longer supported.", + "license": "ISC", + "dependencies": { + "graceful-fs": "^4.1.2", + "inherits": "~2.0.0", + "mkdirp": ">=0.5 0", + "rimraf": "2" + }, + "engines": { + "node": ">=0.6" + } + }, + "node_modules/fstream/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "17.13.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-17.13.0.tgz", + "integrity": "sha512-RwMTC61u7hrG3ZJMkZQOK4JLJrKS8QtoTii63EmWvFXHqycY9FObBJQCbsLxSO8kjKhWE5kV1GC+Vb1g3RI0Zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/hashery": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.1.tgz", + "integrity": "sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^1.15.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "license": "MIT" + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/jszip": { + "version": "3.10.2", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.2.tgz", + "integrity": "sha512-3l+rb15IOWtUhU0H5MFqES/T6Kh7abYwjosBey/vD6hDt8zoEffkSC5Ws5SGtgVw3gBx2NEbhTeSW1+kWkpyTQ==", + "license": "(MIT OR GPL-3.0-or-later)", + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, + "node_modules/jszip/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/jszip/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/jszip/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "license": "MIT", + "dependencies": { + "immediate": "~3.0.5" + } + }, + "node_modules/listenercount": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", + "integrity": "sha512-3mk/Zag0+IJxeDrxSgaDPy4zZ3w05PRZeJNnlWhzFz5OkX49J4krc+A8X2d2M69vGMBEX0uyl8M+W+8gH+kBqQ==", + "license": "ISC" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "license": "MIT" + }, + "node_modules/lodash.difference": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.difference/-/lodash.difference-4.5.0.tgz", + "integrity": "sha512-dS2j+W26TQ7taQBGN8Lbbq04ssV3emRw4NY58WErlTO29pIqS0HmoT5aJ9+TUQ1N3G+JOZSji4eugsWwGp9yPA==", + "license": "MIT" + }, + "node_modules/lodash.escaperegexp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz", + "integrity": "sha512-TM9YBvyC84ZxE3rgfefxUWiQKLilstD6k7PTGt6wfbtXF8ixIJLOL3VYyV/z+ZiPLsVxAsKAFVwWlWeb2Y8Yyw==", + "license": "MIT" + }, + "node_modules/lodash.flatten": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flatten/-/lodash.flatten-4.4.0.tgz", + "integrity": "sha512-C5N2Z3DgnnKr0LOpv/hKCgKdb7ZZwafIrsesve6lmzvZIRZRGaZ/l6Q8+2W7NaT+ZwO3fFlSCzCzrDCFdJfZ4g==", + "license": "MIT" + }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isequal": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.isequal/-/lodash.isequal-4.5.0.tgz", + "integrity": "sha512-pDo3lu8Jhfjqls6GkMgpahsF9kCyayhgykjyLMNFTKWrpVdAQtYyB4muAMWozBB4ig/dtWAmsMxLEI8wuz+DYQ==", + "deprecated": "This package is deprecated. Use require('node:util').isDeepStrictEqual instead.", + "license": "MIT" + }, + "node_modules/lodash.isfunction": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/lodash.isfunction/-/lodash.isfunction-3.0.9.tgz", + "integrity": "sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==", + "license": "MIT" + }, + "node_modules/lodash.isnil": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.isnil/-/lodash.isnil-4.0.0.tgz", + "integrity": "sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isundefined": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/lodash.isundefined/-/lodash.isundefined-3.0.1.tgz", + "integrity": "sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==", + "license": "MIT" + }, + "node_modules/lodash.union": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.union/-/lodash.union-4.6.0.tgz", + "integrity": "sha512-c4pB2CdGrGdjMKYLA+XiRDO7Y0PRQbm/Gzg8qMj+QH+pFVAoTp5sBpO0odL3FjoPCGjK96p6qsP+yQoiLoOBcw==", + "license": "MIT" + }, + "node_modules/lodash.uniq": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz", + "integrity": "sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==", + "license": "MIT" + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "license": "(MIT AND Zlib)" + }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/qified": { + "version": "0.10.1", + "resolved": "https://registry.npmjs.org/qified/-/qified-0.10.1.tgz", + "integrity": "sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA==", + "dev": true, + "license": "MIT", + "dependencies": { + "hookified": "^2.1.1" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/qified/node_modules/hookified": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-2.2.0.tgz", + "integrity": "sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA==", + "dev": true, + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.7.tgz", + "integrity": "sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/saxes": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/saxes/-/saxes-5.0.1.tgz", + "integrity": "sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==", + "license": "ISC", + "dependencies": { + "xmlchars": "^2.2.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/traverse": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", + "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", + "license": "MIT/X11", + "engines": { + "node": "*" + } + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici": { + "version": "7.30.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.0.tgz", + "integrity": "sha512-dkrQXeHSaoamnItlYbmzG0wFYrM0ZwDxCIg0A7aKjTyyhh9svRzCNFEzV+Vm05/yehjCzjDZ31KXfGEjYSztDQ==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", + "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", + "dev": true, + "license": "MIT" + }, + "node_modules/unzipper": { + "version": "0.10.14", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.14.tgz", + "integrity": "sha512-ti4wZj+0bQTiX2KmKWuwj7lhV+2n//uXEotUmGuQqrbVZSEGFMbI68+c6JCQ8aAmUWYvtHEz2A8K6wXvueR/6g==", + "license": "MIT", + "dependencies": { + "big-integer": "^1.6.17", + "binary": "~0.3.0", + "bluebird": "~3.4.1", + "buffer-indexof-polyfill": "~1.0.0", + "duplexer2": "~0.1.4", + "fstream": "^1.0.12", + "graceful-fs": "^4.2.2", + "listenercount": "~1.0.1", + "readable-stream": "~2.3.6", + "setimmediate": "~1.0.4" + } + }, + "node_modules/unzipper/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/unzipper/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/unzipper/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/xmlchars": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", + "license": "MIT" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zip-stream": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-4.1.1.tgz", + "integrity": "sha512-9qv4rlDiopXg4E69k+vMHjNN63YFMe9sZMrdlvKnCjlCRWeCBswPPMPUfx+ipsAWq1LXHe70RcbaHdJJpS6hyQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^3.0.4", + "compress-commons": "^4.1.2", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/zip-stream/node_modules/archiver-utils": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-3.0.4.tgz", + "integrity": "sha512-KVgf4XQVrTjhyWmx6cte4RxonPLR9onExufI1jhvw/MQ4BB6IsZD5gT8Lq+u/+pRkWna/6JoHpiQioaqFP5Rzw==", + "license": "MIT", + "dependencies": { + "glob": "^7.2.3", + "graceful-fs": "^4.2.0", + "lazystream": "^1.0.0", + "lodash.defaults": "^4.2.0", + "lodash.difference": "^4.5.0", + "lodash.flatten": "^4.4.0", + "lodash.isplainobject": "^4.0.6", + "lodash.union": "^4.6.0", + "normalize-path": "^3.0.0", + "readable-stream": "^3.6.0" + }, + "engines": { + "node": ">= 10" + } + } + } +} diff --git a/package.json b/package.json index 84f8ba3..14741fa 100644 --- a/package.json +++ b/package.json @@ -1,19 +1,30 @@ { "name": "check-http-status", - "description": "Check HTTP Status and create a list for all the HTTP status code.", - "version": "1.3.0", + "description": "Crawl a website or check a list of URLs: HTTP status codes, redirect chains, broken links and the pages they were found on. Export to Excel, CSV, JSON or an HTML report.", + "version": "2.0.0", "main": "index.js", + "types": "index.d.ts", + "bin": { + "check-http-status": "bin/check-http-status.js" + }, "files": [ "LICENSE", "index.js", - "test/runkitExample.js", - "lib/*" + "index.d.ts", + "schema.json", + "bin/*", + "lib/**/*", + "examples/*" ], + "engines": { + "node": ">=18.17" + }, + "runkitExampleFilename": "examples/runkit.js", "scripts": { + "build:demo": "node scripts/build-demo.js", "lint": "eslint .", - "test:sitemaps": "node test/sitemaps.js", - "test:sitemaps-urls": "node test/sitemaps-urls.js", - "test:urls": "node test/urls.js" + "test": "node --test", + "test:types": "tsc -p test-d" }, "repository": { "type": "git", @@ -23,6 +34,11 @@ "http status", "redirect chain", "link checker", + "broken links", + "crawler", + "site crawler", + "sitemap", + "seo", "status", "redirects", "301", @@ -33,14 +49,16 @@ "bugs": { "url": "https://github.com/trunkcode/check-http-status/issues" }, - "homepage": "https://github.com/trunkcode/check-http-status", + "homepage": "https://trunkcode.github.io/check-http-status/", "dependencies": { - "axios": "^0.23.0", - "columnify": "^1.5.4", - "exceljs": "^4.3.0", - "xml2js": "^0.4.23" + "cheerio": "^1.0.0", + "exceljs": "^4.4.0" }, "devDependencies": { - "eslint": "^8.0.1" + "@eslint/js": "^10.0.1", + "@types/node": "^26.6.4", + "eslint": "^10.11.0", + "globals": "^17.13.0", + "typescript": "^7.0.2" } } diff --git a/schema.json b/schema.json new file mode 100644 index 0000000..28822e4 --- /dev/null +++ b/schema.json @@ -0,0 +1,101 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "$id": "https://unpkg.com/check-http-status/schema.json", + "title": "check-http-status config", + "description": "Settings shared by the check-http-status CLI (--config), the GitHub Action and apps built on the library. Durations are in milliseconds.", + "type": "object", + "additionalProperties": false, + "definitions": { + "stringList": { + "oneOf": [ + { "type": "string" }, + { "type": "array", "items": { "type": "string" } } + ] + }, + "exportEntry": { + "type": "object", + "additionalProperties": false, + "properties": { + "format": { "enum": ["xlsx", "csv", "json", "html"], "default": "xlsx" }, + "file": { "type": "string", "description": "Report path (relative to the config file)." }, + "location": { "type": "string", "description": "Folder for an auto-named report (relative to the config file)." } + } + } + }, + "properties": { + "$schema": { "type": "string" }, + "crawl": { "$ref": "#/definitions/stringList", "description": "Website URL(s) to crawl. Pages are found by following internal links." }, + "urls": { "$ref": "#/definitions/stringList", "description": "URLs to check without crawling." }, + "sitemaps": { "$ref": "#/definitions/stringList", "description": "Optional sitemap / sitemap index URLs." }, + "include": { "$ref": "#/definitions/stringList", "description": "Only crawl URLs matching one of these rules (substring, * wildcard or /regex/)." }, + "exclude": { "$ref": "#/definitions/stringList", "description": "Don't crawl URLs matching these rules. They are still listed and checked." }, + "maxPages": { "type": "integer", "minimum": 1, "default": 1000 }, + "maxDepth": { "type": ["integer", "null"], "minimum": 0, "description": "Maximum link depth from the start URL. Unlimited when omitted." }, + "maxRedirects": { "type": "integer", "minimum": 0, "maximum": 50, "default": 10 }, + "checkExternal": { "type": "boolean", "default": true, "description": "Check the status of external links. Excluded pages and assets are always checked." }, + "checkAssets": { "type": "boolean", "default": false, "description": "Also check images, scripts, stylesheets, icons and iframes." }, + "subdomains": { "type": "boolean", "default": false, "description": "Treat subdomains as internal." }, + "ignoreQuery": { "type": "boolean", "default": false, "description": "Strip query strings before comparing URLs." }, + "respectRobots": { "type": "boolean", "default": false, "description": "Don't request URLs disallowed by robots.txt; honour Crawl-delay." }, + "concurrency": { "type": "integer", "minimum": 1, "maximum": 100, "default": 10 }, + "delay": { "type": "integer", "minimum": 0, "default": 0, "description": "Milliseconds between requests to the same site." }, + "retries": { "type": "integer", "minimum": 0, "maximum": 10, "default": 2, "description": "Retries for 429/502/503/504 and dropped connections." }, + "retryDelay": { "type": "integer", "minimum": 0, "default": 1000, "description": "Milliseconds before the first retry, doubled each time." }, + "userAgent": { "type": "string" }, + "skip200": { "type": "boolean", "default": false, "description": "Leave 200 OK URLs out of the output and reports." }, + "silent": { "type": "boolean", "default": false }, + "fail": { "type": "boolean", "default": false, "description": "CLI / GitHub Action: exit with code 1 if broken links are found." }, + "export": { + "oneOf": [ + { "$ref": "#/definitions/exportEntry" }, + { "type": "array", "items": { "$ref": "#/definitions/exportEntry" } } + ] + }, + "options": { + "type": "object", + "additionalProperties": false, + "properties": { + "auth": { + "type": "object", + "description": "HTTP Basic auth. Prefer the CHS_AUTH environment variable over storing passwords here.", + "properties": { "username": { "type": "string" }, "password": { "type": "string" } } + }, + "headers": { "type": "object", "additionalProperties": { "type": "string" } }, + "timeout": { "type": "integer", "minimum": 1000, "default": 15000, "description": "Milliseconds per request." }, + "userAgent": { "type": "string" } + } + }, + "googleDrive": { + "oneOf": [ + { "type": "boolean" }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "folderId": { "type": "string" }, + "folderName": { "type": "string", "default": "HTTP Status Reports" }, + "convert": { "type": "boolean", "default": true }, + "public": { "type": "boolean", "default": false }, + "keyFile": { "type": "string", "description": "Service account key file (relative to the config file)." }, + "formats": { "$ref": "#/definitions/stringList" } + } + } + ] + }, + "oneDrive": { + "oneOf": [ + { "type": "boolean" }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "folderPath": { "type": "string" }, + "public": { "type": "boolean", "default": false }, + "tenant": { "type": "string", "default": "common" }, + "formats": { "$ref": "#/definitions/stringList" } + } + } + ] + } + } +} diff --git a/scripts/build-demo.js b/scripts/build-demo.js new file mode 100644 index 0000000..b9d308d --- /dev/null +++ b/scripts/build-demo.js @@ -0,0 +1,50 @@ +'use strict'; + +// Builds docs/demo.html and docs/sample-report.xlsx from the fixture site. +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const checkHttpStatus = require('../index'); +const startSite = require('../test/fixtures/site'); +const writeExcel = require('../lib/report/excel'); +const { renderHtml } = require('../lib/report/html'); + +const DOCS = path.join(__dirname, '..', 'docs'); +const SITE = 'https://www.acme-outdoor.example'; +const EXTERNAL = 'https://partner-site.example'; + +async function main() { + const site = await startSite(); + + try { + const report = await checkHttpStatus({ + 'checkAssets': true, + 'crawl': site.origin + '/', + 'exclude': ['/docs'], + 'silent': true, + 'sitemaps': [site.origin + '/sitemap_index.xml'] + }); + + const json = JSON.stringify(report) + .split(site.origin).join(SITE) + .split(site.externalOrigin).join(EXTERNAL); + const demo = JSON.parse(json); + + fs.writeFileSync(path.join(DOCS, 'demo.html'), renderHtml(demo, { 'title': 'Demo crawl report · check-http-status' })); + + // Write to a temp file first: exceljs needs a real path. + const tmpFile = path.join(os.tmpdir(), `chs-demo-${Date.now()}.xlsx`); + await writeExcel(demo, tmpFile); + fs.copyFileSync(tmpFile, path.join(DOCS, 'sample-report.xlsx')); + fs.rmSync(tmpFile); + + console.log(`Demo built: ${demo.results.length} URLs, ${demo.summary.issues} issues.`); + } finally { + await site.close(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/test-d/tsconfig.json b/test-d/tsconfig.json new file mode 100644 index 0000000..0a75110 --- /dev/null +++ b/test-d/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "strict": true, + "noEmit": true, + "module": "commonjs", + "target": "es2022", + "types": ["node"], + "skipLibCheck": false + }, + "files": ["usage.ts"] +} diff --git a/test-d/usage.ts b/test-d/usage.ts new file mode 100644 index 0000000..2a972e0 --- /dev/null +++ b/test-d/usage.ts @@ -0,0 +1,63 @@ +// Compiled with `npm run test:types` to check index.d.ts. Never executed. +import checkHttpStatus = require('..'); +import { findConfig, loadConfig, recheck, renderHtml, writeReport } from '..'; + +async function main(): Promise { + const controller = new AbortController(); + + const report: checkHttpStatus.Report = await checkHttpStatus({ + crawl: 'https://www.example.com', + exclude: ['/docs'], + maxDepth: null, + respectRobots: true, + delay: 500, + userAgent: 'AuditBot/1.0', + export: [{ format: 'xlsx', file: 'report.xlsx' }, { format: 'html', location: './reports' }], + options: { auth: { username: 'u', password: 'p' }, headers: { 'Accept-Language': 'en' }, timeout: 10000 }, + googleDrive: { public: true, convert: false }, + oneDrive: { folderPath: 'Documents/SEO', tenant: 'consumers' }, + signal: controller.signal, + onResult: (result) => { + const status: number | null = result.status; + const pages: string[] = result.foundOn.map((source) => source.page); + void status; + void pages; + }, + onProgress: (progress) => { + const state: 'idle' | 'running' | 'done' | 'stopped' = progress.state; + void state; + } + }); + + const notFound = report.results.filter((result) => result.category === 'Not Found'); + const firstUpload = report.uploads?.[0]?.url; + const issues: number = report.summary.issues; + void notFound; + void firstUpload; + void issues; + + const updated = await recheck(report, ['https://www.example.com/old'], { retries: 1 }); + await writeReport(updated, 'report.csv', 'csv'); + const html: string = renderHtml(updated, { title: 'Report' }); + const file: string | null = findConfig(); + const config = loadConfig('check-http-status.config.json'); + const fails: boolean | undefined = config.fail; + void html; + void file; + void fails; + + // Same function via the named export. + await checkHttpStatus.checkHttpStatus({ urls: 'https://example.com' }); + + // @ts-expect-error unknown export format + await checkHttpStatus({ crawl: 'https://example.com', export: { format: 'pdf' } }); + + // @ts-expect-error misspelled option + await checkHttpStatus({ crawl: 'https://example.com', excludes: ['/docs'] }); + + // @ts-expect-error category is a fixed set of strings + const category: checkHttpStatus.Category = 'Broken'; + void category; +} + +void main; diff --git a/test/checker.test.js b/test/checker.test.js new file mode 100644 index 0000000..d72e5dd --- /dev/null +++ b/test/checker.test.js @@ -0,0 +1,368 @@ +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { after, before, describe, test } = require('node:test'); +const { execFile } = require('child_process'); +const excelJS = require('exceljs'); +const checkHttpStatus = require('../index'); +const startSite = require('./fixtures/site'); + +let site; +let tmp; + +before(async () => { + site = await startSite(); + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'chs-test-')); +}); + +after(async () => { + await site.close(); + fs.rmSync(tmp, { 'force': true, 'recursive': true }); +}); + +const find = (report, pathname, origin = site.origin) => report.results.find((result) => result.url === origin + pathname); + +describe('crawl mode', () => { + let report; + + before(async () => { + report = await checkHttpStatus({ + 'checkAssets': true, + 'crawl': site.origin + '/', + 'exclude': ['/docs'], + 'options': { 'timeout': 5000 }, + 'silent': true, + 'sitemaps': [site.origin + '/sitemap_index.xml'] + }); + }); + + test('crawls internal pages and records titles', () => { + const tent = find(report, '/products/tents/summit-2p'); + assert.strictEqual(tent.status, 200); + assert.strictEqual(tent.scope, 'crawl'); + assert.strictEqual(tent.crawled, true); + assert.strictEqual(tent.title, 'Summit 2P | Acme Outdoor'); + }); + + test('excluded URLs are listed and checked but not crawled', () => { + const docs = find(report, '/docs/'); + assert.strictEqual(docs.scope, 'excluded'); + assert.strictEqual(docs.status, 200); + assert.strictEqual(docs.crawled, false); + assert.strictEqual(find(report, '/docs/returns'), undefined); + }); + + test('external links are checked but never crawled', () => { + const partner = find(report, '/partners', site.externalOrigin); + assert.strictEqual(partner.type, 'external'); + assert.strictEqual(partner.scope, 'external'); + assert.strictEqual(partner.status, 200); + assert.strictEqual(find(report, '/should-not-be-crawled', site.externalOrigin), undefined); + assert.strictEqual(find(report, '/deleted-article', site.externalOrigin).category, 'Not Found'); + }); + + test('falls back to GET when HEAD is not allowed', () => { + assert.strictEqual(find(report, '/head-not-allowed', site.externalOrigin).status, 200); + }); + + test('skips mailto: and #fragment links', () => { + assert.ok(!report.results.some((result) => result.url.startsWith('mailto:') || result.url.includes('#'))); + }); + + test('reports broken links with the pages and link text they were found on', () => { + const gone = find(report, '/blog/2019/old-news'); + assert.strictEqual(gone.status, 410); + assert.strictEqual(gone.category, 'Not Found'); + assert.deepStrictEqual(gone.foundOn.map((source) => source.page), [site.origin + '/blog/']); + assert.deepStrictEqual(gone.foundOn[0].anchors, ['Old news']); + + const stoves = find(report, '/products/stoves'); + assert.strictEqual(stoves.category, 'Server Error'); + assert.strictEqual(stoves.foundOn.length, 2); + }); + + test('a link repeated on every page is counted once per page', () => { + const privacy = find(report, '/privacy'); + assert.ok(privacy.foundOn.length > 10); + assert.ok(privacy.foundOn.every((source) => source.count === 1)); + }); + + test('follows redirect chains, including relative Location headers', () => { + const bags = find(report, '/products/sleeping-bags'); + assert.strictEqual(bags.status, 301); + assert.strictEqual(bags.redirectCount, 2); + assert.strictEqual(bags.finalUrl, site.origin + '/collections/sleeping-bags'); + assert.strictEqual(bags.finalStatus, 404); + + const sale = find(report, '/summer-sale'); + assert.strictEqual(sale.redirectTo, site.origin + '/new-arrivals'); + assert.strictEqual(sale.finalStatus, 200); + assert.strictEqual(find(report, '/new-arrivals').crawled, true); + }); + + test('detects redirect loops without hanging', () => { + assert.strictEqual(find(report, '/store-locator').note, 'Redirect loop'); + }); + + test('reports DNS failures as errors', () => { + const careers = report.results.find((result) => result.url.startsWith('https://jobs.example.invalid')); + assert.strictEqual(careers.category, 'Error'); + assert.ok(careers.error); + }); + + test('checks assets when enabled', () => { + const image = find(report, '/images/missing-trail.jpg'); + assert.strictEqual(image.linkType, 'asset'); + assert.strictEqual(image.status, 404); + assert.strictEqual(find(report, '/assets/style.css').status, 200); + }); + + test('reads sitemap indexes and gzipped sitemaps, and flags orphan pages', () => { + const landing = find(report, '/unlinked-landing-page'); + assert.strictEqual(landing.inSitemap, true); + assert.strictEqual(landing.orphan, true); + assert.strictEqual(landing.crawled, true); + assert.strictEqual(find(report, '/blog/deleted-post').category, 'Not Found'); + assert.strictEqual(find(report, '/about-us').orphan, false); + assert.ok(report.summary.orphans >= 2); + }); +}); + +describe('crawl limits and rules', () => { + test('maxPages stops crawling and lists the rest as not checked', async () => { + const report = await checkHttpStatus({ 'crawl': site.origin, 'maxPages': 3, 'silent': true }); + assert.strictEqual(report.results.filter((result) => result.scope === 'crawl').length, 3); + assert.ok(report.results.some((result) => result.scope === 'limit' && result.status === null)); + assert.strictEqual(report.summary.pageLimitReached, true); + }); + + test('checkExternal: false skips external links but still checks excluded pages and assets', async () => { + const report = await checkHttpStatus({ 'checkAssets': true, 'checkExternal': false, 'crawl': site.origin, 'exclude': ['/docs'], 'silent': true }); + assert.strictEqual(find(report, '/partners', site.externalOrigin).category, 'Not checked'); + assert.strictEqual(find(report, '/docs/').status, 200); + assert.strictEqual(find(report, '/images/missing-trail.jpg').status, 404); + }); + + test('a page only reachable through a redirect is not an orphan', async () => { + const report = await checkHttpStatus({ 'checkExternal': false, 'crawl': site.origin, 'silent': true, 'sitemaps': [site.origin + '/sitemap_index.xml'] }); + const arrivals = find(report, '/new-arrivals'); + assert.strictEqual(arrivals.inSitemap, true); + assert.strictEqual(arrivals.orphan, false, 'linked as /summer-sale, which redirects here'); + assert.strictEqual(find(report, '/unlinked-landing-page').orphan, true); + }); + + test('maxDepth limits how far links are followed', async () => { + const report = await checkHttpStatus({ 'checkExternal': false, 'crawl': site.origin, 'maxDepth': 1, 'silent': true }); + assert.strictEqual(find(report, '/products/tents').crawled, true); + assert.strictEqual(find(report, '/products/tents/summit-2p').scope, 'depth'); + }); + + test('include rules restrict crawling, wildcards and regex work', async () => { + const report = await checkHttpStatus({ 'checkExternal': false, 'crawl': site.origin, 'exclude': '/\\/basecamp-\\d+p$/', 'include': ['*/products/*'], 'silent': true }); + assert.strictEqual(find(report, '/products/tents').crawled, true); + assert.strictEqual(find(report, '/blog/').scope, 'excluded'); + assert.strictEqual(find(report, '/products/tents/basecamp-4p').scope, 'excluded'); + }); + + test('ignoreQuery merges URLs that only differ by query string', async () => { + const report = await checkHttpStatus({ 'checkExternal': false, 'crawl': site.origin, 'ignoreQuery': true, 'silent': true }); + assert.ok(find(report, '/blog/packing-list')); + assert.ok(!report.results.some((result) => result.url.includes('?'))); + }); +}); + +describe('list mode', () => { + test('checks only the given URLs and follows their redirects', async () => { + const report = await checkHttpStatus({ + 'silent': true, + 'urls': [site.origin + '/old-blog', site.origin + '/nope', site.origin + '/products/tents'] + }); + + assert.strictEqual(report.summary.mode, 'list'); + assert.strictEqual(find(report, '/old-blog').finalStatus, 200); + assert.strictEqual(find(report, '/nope').status, 404); + assert.strictEqual(find(report, '/products/tents').crawled, false); + assert.strictEqual(find(report, '/products/tents/summit-2p'), undefined); + }); + + test('reads URLs from sitemaps', async () => { + const report = await checkHttpStatus({ 'silent': true, 'sitemaps': [site.origin + '/sitemap_index.xml'] }); + assert.strictEqual(find(report, '/blog/deleted-post').status, 404); + assert.strictEqual(find(report, '/unlinked-landing-page').status, 200); + }); + + test('sends basic auth only to the checked site, plus custom headers', async () => { + const report = await checkHttpStatus({ + 'options': { 'auth': { 'password': 'secret', 'username': 'admin' }, 'headers': { 'X-Test': 'yes' } }, + 'silent': true, + 'urls': [site.origin + '/staff'] + }); + assert.strictEqual(find(report, '/staff').status, 200); + + const unauthorized = await checkHttpStatus({ 'silent': true, 'urls': [site.origin + '/staff'] }); + assert.strictEqual(find(unauthorized, '/staff').status, 401); + }); + + test('bot-protection challenges are reported as Blocked, not broken, and not retried', async () => { + const before = site.requests.length; + const report = await checkHttpStatus({ 'retryDelay': 10, 'silent': true, 'urls': [site.origin + '/cf-protected', site.origin + '/datadome-protected', site.origin + '/plain-403'] }); + const cf = find(report, '/cf-protected'); + assert.strictEqual(cf.category, 'Blocked'); + assert.strictEqual(cf.botProtection, 'Cloudflare'); + assert.match(cf.note, /check in a browser/); + assert.strictEqual(find(report, '/datadome-protected').botProtection, 'DataDome'); + assert.strictEqual(find(report, '/plain-403').category, 'Client Error'); + assert.strictEqual(report.summary.issues, 1, 'Blocked is not an issue'); + assert.strictEqual(site.requests.slice(before).filter((r) => r.path === '/datadome-protected').length, 2, 'HEAD + GET, no retries'); + }); + + test('maxRedirects stops following long chains', async () => { + const report = await checkHttpStatus({ 'maxRedirects': 1, 'silent': true, 'urls': [site.origin + '/products/sleeping-bags'] }); + assert.match(find(report, '/products/sleeping-bags/').note, /Stopped after 1 redirects/); + }); + + test('throws instead of exiting the process on bad input', async () => { + await assert.rejects(checkHttpStatus({}), /Provide at least one/); + await assert.rejects(checkHttpStatus({ 'export': { 'location': '/does/not/exist' }, 'urls': ['https://example.com'] }), /does not exist/); + }); +}); + +describe('exports', () => { + test('v1 config shape still works and writes all formats', async () => { + const report = await checkHttpStatus({ + 'crawl': site.origin, + 'export': [ + { 'format': 'xlsx', 'location': tmp }, + { 'file': path.join(tmp, 'report.csv'), 'format': 'csv' }, + { 'file': path.join(tmp, 'report.json'), 'format': 'json' }, + { 'file': path.join(tmp, 'report.html'), 'format': 'html' } + ], + 'silent': true, + 'skip200': true + }); + + const xlsxFile = fs.readdirSync(tmp).find((file) => file.startsWith('httpstatuschecker-results-') && file.endsWith('.xlsx')); + assert.ok(xlsxFile, 'xlsx written to the export location'); + + const workbook = new excelJS.Workbook(); + await workbook.xlsx.readFile(path.join(tmp, xlsxFile)); + assert.deepStrictEqual(workbook.worksheets.map((sheet) => sheet.name), ['Summary', 'URLs (non-200)', 'Issues', 'External Links']); + + const issues = workbook.getWorksheet('Issues'); + const header = issues.getRow(1).values; + const urlCol = header.indexOf('URL'); + const foundOnCol = header.indexOf('Found On'); + const goneRow = issues.getRows(2, issues.rowCount - 1).find((row) => row.getCell(urlCol).value === site.origin + '/blog/2019/old-news'); + assert.strictEqual(goneRow.getCell(foundOnCol).value, site.origin + '/blog/'); + + const csv = fs.readFileSync(path.join(tmp, 'report.csv'), 'utf8'); + assert.ok(csv.startsWith('\ufeffURL,Found On,Redirect To,Status')); + assert.ok(!csv.includes(',OK,'), 'skip200 removes OK rows from CSV'); + + const json = JSON.parse(fs.readFileSync(path.join(tmp, 'report.json'), 'utf8')); + assert.strictEqual(json.results.length, report.results.length); + + const html = fs.readFileSync(path.join(tmp, 'report.html'), 'utf8'); + assert.ok(html.includes('/blog/2019/old-news')); + assert.ok(!html.includes('__REPORT_DATA__')); + }); + + test('the Found On cell is capped below the Excel cell limit', () => { + const { urlRows } = require('../lib/report/rows'); + const foundOn = Array.from({ 'length': 2000 }, (_, i) => ({ 'anchors': [], 'count': 1, 'kind': 'link', 'page': `https://example.com/a-long-page-path-number-${i}` })); + const [row] = urlRows([{ 'category': 'OK', 'foundOn': foundOn, 'redirectChain': [], 'type': 'internal', 'url': 'https://example.com/privacy' }]); + assert.ok(row.foundOn.length < 32767); + assert.match(row.foundOn, /… and \d+ more \(see the JSON or HTML report\)$/); + assert.strictEqual(row.foundOnCount, 2000); + }); + + test('HTML report escapes data so it cannot break out of the script tag', () => { + const { renderHtml } = require('../index'); + const html = renderHtml({ 'results': [{ 'title': '' }], 'summary': {} }); + assert.ok(!html.includes('