♻️ refactor: support reject pattern

Author: cy948

packages/web-crawler/src/crawImpl/browserless.ts

@@ -5,6 +5,7 @@ import { CrawlImpl, CrawlSuccessResult } from '../type';
 import { htmlToMarkdown } from '../utils/htmlToMarkdown';
 
 const BASE_URL = process.env.BROWSERLESS_URL ?? 'https://chrome.browserless.io';
+const REJECT_REQUEST_PATTERN = process.env.BROWSERLESS_REJECT_REQUEST_PATTERN ?? '';
 const BROWSERLESS_TOKEN = process.env.BROWSERLESS_TOKEN;
 
 class BrowserlessInitError extends Error {
@@ -21,6 +22,7 @@ export const browserless: CrawlImpl = async (url, { filterOptions }) => {
 
   const input = {
     gotoOptions: { waitUntil: 'networkidle2' },
+    rejectRequestPattern: [REJECT_REQUEST_PATTERN],
     url,
   };