From 9799aa873a7d20bcb0be366826364890a383cbed Mon Sep 17 00:00:00 2001 From: suyao Date: Fri, 28 Mar 2025 00:25:10 +0800 Subject: [PATCH] feat(OCR): improve PDF handling and update package versions --- package.json | 31 ++--- src/main/ocr/BaseOcrProvider.ts | 119 ++++++++++++++----- src/main/ocr/Doc2xOcrProvider.ts | 34 +++--- src/main/ocr/MacSysOcrProvider.ts | 98 ++++++++++----- yarn.lock | 190 ++++++------------------------ 5 files changed, 229 insertions(+), 243 deletions(-) diff --git a/package.json b/package.json index 2830bec15..3cbd017aa 100644 --- a/package.json +++ b/package.json @@ -50,18 +50,18 @@ "prepare": "husky" }, "dependencies": { - "@cherrystudio/embedjs": "^0.1.28", - "@cherrystudio/embedjs-libsql": "^0.1.28", - "@cherrystudio/embedjs-loader-csv": "^0.1.28", - "@cherrystudio/embedjs-loader-image": "^0.1.28", - "@cherrystudio/embedjs-loader-markdown": "^0.1.28", - "@cherrystudio/embedjs-loader-msoffice": "^0.1.28", - "@cherrystudio/embedjs-loader-pdf": "^0.1.28", - "@cherrystudio/embedjs-loader-sitemap": "^0.1.28", - "@cherrystudio/embedjs-loader-web": "^0.1.28", - "@cherrystudio/embedjs-loader-xml": "^0.1.28", - "@cherrystudio/embedjs-openai": "^0.1.28", - "@cherrystudio/mac-system-ocr": "^0.1.0", + "@cherrystudio/embedjs": "^0.1.29", + "@cherrystudio/embedjs-libsql": "^0.1.29", + "@cherrystudio/embedjs-loader-csv": "^0.1.29", + "@cherrystudio/embedjs-loader-image": "^0.1.29", + "@cherrystudio/embedjs-loader-markdown": "^0.1.29", + "@cherrystudio/embedjs-loader-msoffice": "^0.1.29", + "@cherrystudio/embedjs-loader-pdf": "^0.1.29", + "@cherrystudio/embedjs-loader-sitemap": "^0.1.29", + "@cherrystudio/embedjs-loader-web": "^0.1.29", + "@cherrystudio/embedjs-loader-xml": "^0.1.29", + "@cherrystudio/embedjs-openai": "^0.1.29", + "@cherrystudio/mac-system-ocr": "^0.2.0", "@electron-toolkit/utils": "^3.0.0", "@electron/notarize": "^2.5.0", "@google/generative-ai": "^0.21.0", @@ -73,6 +73,7 @@ "@types/react-infinite-scroll-component": "^5.0.0", "@xyflow/react": "^12.4.4", "adm-zip": "^0.5.16", + "canvas": "3.1.0", "docx": "^9.0.2", "electron-log": "^5.1.5", "electron-store": "^8.2.0", @@ -84,8 +85,8 @@ "fs-extra": "^11.2.0", "markdown-it": "^14.1.0", "officeparser": "^4.1.1", - "pdf-parse": "^1.1.1", "pdf-to-img": "^4.4.0", + "pdfjs-dist": "4.2.67", "proxy-agent": "^6.5.0", "tar": "^7.4.3", "undici": "^7.4.0", @@ -119,7 +120,6 @@ "@types/md5": "^2.3.5", "@types/node": "^18.19.9", "@types/pako": "^1.0.2", - "@types/pdf-parse": "^1.1.4", "@types/react": "^19.0.12", "@types/react-dom": "^19.0.4", "@types/react-infinite-scroll-component": "^5.0.0", @@ -188,7 +188,8 @@ "pdf-parse@npm:1.1.1": "patch:pdf-parse@npm%3A1.1.1#~/.yarn/patches/pdf-parse-npm-1.1.1-04a6109b2a.patch", "@langchain/openai@npm:^0.3.16": "patch:@langchain/openai@npm%3A0.3.16#~/.yarn/patches/@langchain-openai-npm-0.3.16-e525b59526.patch", "@langchain/openai@npm:>=0.1.0 <0.4.0": "patch:@langchain/openai@npm%3A0.3.16#~/.yarn/patches/@langchain-openai-npm-0.3.16-e525b59526.patch", - "openai@npm:^4.77.0": "patch:openai@npm%3A4.77.3#~/.yarn/patches/openai-npm-4.77.3-59c6d42e7a.patch" + "openai@npm:^4.77.0": "patch:openai@npm%3A4.77.3#~/.yarn/patches/openai-npm-4.77.3-59c6d42e7a.patch", + "canvas": "3.1.0" }, "packageManager": "yarn@4.6.0", "lint-staged": { diff --git a/src/main/ocr/BaseOcrProvider.ts b/src/main/ocr/BaseOcrProvider.ts index 1aba42ebe..13944d1ac 100644 --- a/src/main/ocr/BaseOcrProvider.ts +++ b/src/main/ocr/BaseOcrProvider.ts @@ -3,9 +3,10 @@ import path from 'node:path' import { windowService } from '@main/services/WindowService' import { FileSource, LocalFileSource, OcrProvider } from '@types' +import { createCanvas, loadImage } from 'canvas' import { app } from 'electron' -import Logger from 'electron-log' -import pdfParse from 'pdf-parse' +import { TypedArray } from 'pdfjs-dist/types/src/display/api' + export default abstract class BaseOcrProvider { protected provider: OcrProvider private storageDir = path.join(app.getPath('userData'), 'Data', 'Files') @@ -24,33 +25,18 @@ export default abstract class BaseOcrProvider { return new Promise((resolve) => setTimeout(resolve, ms)) } - /** - * 获取PDF文件的页数和文件大小 - * @param filePath PDF文件路径 - * @returns 包含页数和文件大小(字节)的对象 - */ - public async getPdfInfo(filePath: string): Promise<{ pageCount: number; fileSize: number }> { - try { - Logger.info(`Getting PDF info for: ${filePath}`) - - if (!filePath.toLowerCase().endsWith('.pdf')) { - throw new Error('File is not a PDF') - } - - const stats = fs.statSync(filePath) - const fileSize = stats.size - const dataBuffer = fs.readFileSync(filePath) - const pdfData = await pdfParse(dataBuffer) - - Logger.info(`File ${filePath} has ${pdfData.numpages} pages and size: ${fileSize} bytes`) - return { - pageCount: pdfData.numpages, - fileSize: fileSize - } - } catch (error) { - Logger.error(`Failed to get PDF info for ${filePath}: ${error instanceof Error ? error.message : String(error)}`) - throw new Error('Failed to get PDF information') + public async readPdf( + source: string | URL | TypedArray, + passwordCallback?: (fn: (password: string) => void, reason: string) => string + ) { + const { getDocument } = await import('pdfjs-dist/legacy/build/pdf.mjs') + const documentLoadingTask = getDocument(source) + if (passwordCallback) { + documentLoadingTask.onPassword = passwordCallback } + + const document = await documentLoadingTask.promise + return document } public async sendOcrProgress(sourceId: string, progress: number): Promise { @@ -86,4 +72,81 @@ export default abstract class BaseOcrProvider { } return movedPaths } + + public async cropImage(image: Buffer | string) { + const img = await loadImage(image) + const width = img.width + const height = img.height + + const canvas = createCanvas(width, height) + const context = canvas.getContext('2d') + + context.drawImage(img, 0, 0) + + const data = context.getImageData(0, 0, width, height).data + + const top = scanY(true) + const bottom = scanY(false) + const left = scanX(true) + const right = scanX(false) + + if (top === null || bottom === null || left === null || right === null) { + console.error('image is empty') + return canvas.toBuffer() + } + + const new_width = right - left + const new_height = bottom - top + + canvas.width = new_width + canvas.height = new_height + + context.drawImage(img, left, top, new_width, new_height, 0, 0, new_width, new_height) + + return canvas.toBuffer() + + // get pixel RGB data: + function getRGB(x: number, y: number) { + return { + red: data[(width * y + x) * 4], + green: data[(width * y + x) * 4 + 1], + blue: data[(width * y + x) * 4 + 2] + } + } + + // check if pixel is a color other than white: + function isColor(rgb: { red: number; green: number; blue: number }) { + return rgb.red == 255 && rgb.green == 255 && rgb.blue == 255 + } + + // scan top and bottom edges of image: + function scanY(top: boolean) { + const offset = top ? 1 : -1 + + for (let y = top ? 0 : height - 1; top ? y < height : y > -1; y += offset) { + for (let x = 0; x < width; x++) { + if (!isColor(getRGB(x, y))) { + return y + } + } + } + + return null + } + + // scan left and right edges of image: + function scanX(left: boolean) { + const offset = left ? 1 : -1 + + for (let x = left ? 0 : width - 1; left ? x < width : x > -1; x += offset) { + for (let y = 0; y < height; y++) { + if (!isColor(getRGB(x, y))) { + return x + } + } + } + + return null + } + } } diff --git a/src/main/ocr/Doc2xOcrProvider.ts b/src/main/ocr/Doc2xOcrProvider.ts index d4df2c11d..df6e3a9fa 100644 --- a/src/main/ocr/Doc2xOcrProvider.ts +++ b/src/main/ocr/Doc2xOcrProvider.ts @@ -34,26 +34,32 @@ export default class Doc2xOcrProvider extends BaseOcrProvider { super(provider) } + private async validateFile(filePath: string): Promise { + const pdfBuffer = await fs.promises.readFile(filePath) + + const doc = await this.readPdf(new Uint8Array(pdfBuffer)) + + // 文件页数小于1000页 + if (doc.numPages >= 1000) { + throw new Error(`PDF page count (${doc.numPages}) exceeds the limit of 1000 pages`) + } + // 文件大小小于300MB + if (pdfBuffer.length >= 300 * 1024 * 1024) { + const fileSizeMB = Math.round(pdfBuffer.length / (1024 * 1024)) + throw new Error(`PDF file size (${fileSizeMB}MB) exceeds the limit of 300MB`) + } + } + public async parseFile(sourceId: string, file: FileType): Promise<{ processedFile: FileType }> { try { Logger.info(`OCR processing started: ${file.path}`) - const pdfInfo = await this.getPdfInfo(file.path) - - // 文件页数小于1000页 - if (pdfInfo.pageCount >= 1000) { - throw new Error(`PDF page count (${pdfInfo.pageCount}) exceeds the limit of 1000 pages`) - } - // 文件大小小于300MB - if (pdfInfo.fileSize >= 300 * 1024 * 1024) { - const fileSizeMB = Math.round(pdfInfo.fileSize / (1024 * 1024)) - throw new Error(`PDF file size (${fileSizeMB}MB) exceeds the limit of 300MB`) - } - // 步骤1: 准备上传 const { uid, url } = await this.preupload() Logger.info(`OCR preupload completed: uid=${uid}`) + await this.validateFile(file.path) + // 步骤2: 上传文件 await this.putFile(file.path, url) @@ -148,8 +154,8 @@ export default class Doc2xOcrProvider extends BaseOcrProvider { private async putFile(filePath: string, url: string): Promise { try { - const fileContent = fs.readFileSync(filePath) - const response = await axios.put(url, fileContent) + const fileStream = fs.createReadStream(filePath) + const response = await axios.put(url, fileStream) if (response.status !== 200) { throw new Error(`HTTP status ${response.status}: ${response.statusText}`) diff --git a/src/main/ocr/MacSysOcrProvider.ts b/src/main/ocr/MacSysOcrProvider.ts index cbb15f6ad..aa684c82f 100644 --- a/src/main/ocr/MacSysOcrProvider.ts +++ b/src/main/ocr/MacSysOcrProvider.ts @@ -1,32 +1,80 @@ -import MacOCR, { OCRResult } from '@cherrystudio/mac-system-ocr' +import MacOCR from '@cherrystudio/mac-system-ocr' import { FileSource, isLocalFile, LocalFileSource, OcrProvider } from '@types' import Logger from 'electron-log' import * as fs from 'fs' import PQueue from 'p-queue' import * as path from 'path' +import { TextItem } from 'pdfjs-dist/types/src/display/api' import BaseOcrProvider from './BaseOcrProvider' export default class MacSysOcrProvider extends BaseOcrProvider { + private readonly BATCH_SIZE = 5 + private readonly CONCURRENCY = 5 + private readonly MIN_TEXT_LENGTH = 1000 constructor(provider: OcrProvider) { super(provider) } - private async processPage( - pageBuffer: Buffer, - pageNum: number, + private async processPages( + results: any, totalPages: number, sourceId: string, writeStream: fs.WriteStream ): Promise { - try { - const text: OCRResult = await MacOCR.recognizeFromBuffer(pageBuffer) - writeStream.write(text.text + '\n') - await this.sendOcrProgress(sourceId, (pageNum / totalPages) * 100) - } catch (error) { - Logger.error(`[OCR] Error processing page ${pageNum}:`, error) - throw error + const queue = new PQueue({ concurrency: this.CONCURRENCY }) + const batches: Promise[] = [] + + // Create ordered batches + for (let startPage = 0; startPage < totalPages; startPage += this.BATCH_SIZE) { + const endPage = Math.min(startPage + this.BATCH_SIZE, totalPages) + const batchPromise = queue.add(async () => { + // Convert pages to buffers + const pageBuffers: Buffer[] = [] + for (let i = startPage; i < endPage; i++) { + const pageNum = i + 1 + const pageBuffer = await results.getPage(pageNum) + const croppedPageBuffer = await this.cropImage(pageBuffer) + pageBuffers.push(croppedPageBuffer) + } + + // Process batch + const ocrResults = await MacOCR.recognizeBatchFromBuffer(pageBuffers, { + ocrOptions: { + recognitionLevel: MacOCR.RECOGNITION_LEVEL_ACCURATE + } + }) + + // Write results in order + for (const result of ocrResults) { + writeStream.write(result.text + '\n') + } + + // Update progress + await this.sendOcrProgress(sourceId, (endPage / totalPages) * 100) + }) + batches.push(batchPromise) } + + // Wait for all batches to complete in order + await Promise.all(batches) + } + + public async isScanPdf(buffer: Buffer): Promise { + const doc = await this.readPdf(new Uint8Array(buffer)) + const pageLength = doc.numPages + let counts = 0 + const pagesToCheck = Math.min(pageLength, 10) + for (let i = 0; i < pagesToCheck; i++) { + const page = await doc.getPage(i + 1) + const pageData = await page.getTextContent() + const pageText = pageData.items.map((item) => (item as TextItem).str).join('') + counts += pageText.length + if (counts >= this.MIN_TEXT_LENGTH) { + return false + } + } + return true } public async parseFile(sourceId: string, file: FileSource): Promise<{ processedFile: LocalFileSource }> { @@ -35,9 +83,14 @@ export default class MacSysOcrProvider extends BaseOcrProvider { if (file.ext === '.pdf') { try { const { pdf } = await import('pdf-to-img') - const readStream = fs.createReadStream(file.path) - const results = await pdf(readStream) - const pages = results.length + const pdfBuffer = await fs.promises.readFile(file.path) + const isScanPdf = await this.isScanPdf(pdfBuffer) + if (!isScanPdf) { + Logger.info('[OCR] PDF is not a scan version, skipping OCR') + return { processedFile: file } + } + const results = await pdf(pdfBuffer) + const totalPages = results.length Logger.info('[OCR] PDF successfully converted to image') const baseDir = path.dirname(file.path) @@ -45,22 +98,8 @@ export default class MacSysOcrProvider extends BaseOcrProvider { const txtFileName = `${baseName}.txt` const txtFilePath = path.join(baseDir, txtFileName) - // Create a write stream const writeStream = fs.createWriteStream(txtFilePath) - - const queue = new PQueue({ concurrency: 5 }) - const tasks: Promise[] = [] - - for (let i = 0; i < pages; i++) { - const pageNum = i + 1 - const task = queue.add(async () => { - const pageBuffer = await results.getPage(pageNum) - await this.processPage(pageBuffer, pageNum, pages, sourceId, writeStream) - }) - tasks.push(task) - } - - await Promise.all(tasks) + await this.processPages(results, totalPages, sourceId, writeStream) await new Promise((resolve, reject) => { writeStream.end(() => { @@ -70,7 +109,6 @@ export default class MacSysOcrProvider extends BaseOcrProvider { writeStream.on('error', reject) }) const movedPaths = this.moveToAttachmentsDir(file.id, [txtFilePath]) - console.log('movedPaths', movedPaths) return { processedFile: { ...file, diff --git a/yarn.lock b/yarn.lock index f3ec2f4e1..e2fe33c28 100644 --- a/yarn.lock +++ b/yarn.lock @@ -474,7 +474,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-libsql@npm:^0.1.28": +"@cherrystudio/embedjs-libsql@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-libsql@npm:0.1.29" dependencies: @@ -486,7 +486,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-csv@npm:^0.1.28": +"@cherrystudio/embedjs-loader-csv@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-csv@npm:0.1.29" dependencies: @@ -499,7 +499,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-image@npm:^0.1.28": +"@cherrystudio/embedjs-loader-image@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-image@npm:0.1.29" dependencies: @@ -515,7 +515,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-markdown@npm:^0.1.28": +"@cherrystudio/embedjs-loader-markdown@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-markdown@npm:0.1.29" dependencies: @@ -530,7 +530,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-msoffice@npm:^0.1.28": +"@cherrystudio/embedjs-loader-msoffice@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-msoffice@npm:0.1.29" dependencies: @@ -543,7 +543,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-pdf@npm:^0.1.28": +"@cherrystudio/embedjs-loader-pdf@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-pdf@npm:0.1.29" dependencies: @@ -556,7 +556,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-sitemap@npm:^0.1.28": +"@cherrystudio/embedjs-loader-sitemap@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-sitemap@npm:0.1.29" dependencies: @@ -569,7 +569,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-web@npm:0.1.29, @cherrystudio/embedjs-loader-web@npm:^0.1.28": +"@cherrystudio/embedjs-loader-web@npm:0.1.29, @cherrystudio/embedjs-loader-web@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-web@npm:0.1.29" dependencies: @@ -583,7 +583,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-loader-xml@npm:^0.1.28": +"@cherrystudio/embedjs-loader-xml@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-loader-xml@npm:0.1.29" dependencies: @@ -595,7 +595,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs-openai@npm:^0.1.28": +"@cherrystudio/embedjs-openai@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs-openai@npm:0.1.29" dependencies: @@ -616,7 +616,7 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/embedjs@npm:^0.1.28": +"@cherrystudio/embedjs@npm:^0.1.29": version: 0.1.29 resolution: "@cherrystudio/embedjs@npm:0.1.29" dependencies: @@ -632,14 +632,14 @@ __metadata: languageName: node linkType: hard -"@cherrystudio/mac-system-ocr@npm:^0.1.0": - version: 0.1.0 - resolution: "@cherrystudio/mac-system-ocr@npm:0.1.0" +"@cherrystudio/mac-system-ocr@npm:^0.2.0": + version: 0.2.0 + resolution: "@cherrystudio/mac-system-ocr@npm:0.2.0" dependencies: bindings: "npm:^1.5.0" node-api-headers: "npm:^1.0.1" node-gyp: "npm:latest" - checksum: 10c0/a8257e4c6015f93c630091b44643e3c89e1bd6c2ece836a2ecfa517938c60932f0b8040d126b458dd9e5708194ce9c39c9b288ea5d81d55a111a7f11c1c21c7e + checksum: 10c0/3ece86759b9120958aa9682b2e666f6fe495fb15c10113f350a3f124a7116e8db84d186e7efb7fdbd02437b39e6f0ff763e1784d6ffe8d7179b81b458055711a languageName: node linkType: hard @@ -2380,25 +2380,6 @@ __metadata: languageName: node linkType: hard -"@mapbox/node-pre-gyp@npm:^1.0.0": - version: 1.0.11 - resolution: "@mapbox/node-pre-gyp@npm:1.0.11" - dependencies: - detect-libc: "npm:^2.0.0" - https-proxy-agent: "npm:^5.0.0" - make-dir: "npm:^3.1.0" - node-fetch: "npm:^2.6.7" - nopt: "npm:^5.0.0" - npmlog: "npm:^5.0.1" - rimraf: "npm:^3.0.2" - semver: "npm:^7.3.5" - tar: "npm:^6.1.11" - bin: - node-pre-gyp: bin/node-pre-gyp - checksum: 10c0/2b24b93c31beca1c91336fa3b3769fda98e202fb7f9771f0f4062588d36dcc30fcf8118c36aa747fa7f7610d8cf601872bdaaf62ce7822bb08b545d1bbe086cc - languageName: node - linkType: hard - "@mistralai/mistralai@npm:^1.5.2": version: 1.5.2 resolution: "@mistralai/mistralai@npm:1.5.2" @@ -3497,13 +3478,6 @@ __metadata: languageName: node linkType: hard -"@types/pdf-parse@npm:^1.1.4": - version: 1.1.4 - resolution: "@types/pdf-parse@npm:1.1.4" - checksum: 10c0/1192e0a40bae935428a7f02a56d1313474f0d735820824e1d00f06a0330cf89cc18e34d63864202ae997fbd7806952fa68a47a1f0cbc81f8d03000f942543d5c - languageName: node - linkType: hard - "@types/plist@npm:^3.0.1": version: 3.0.5 resolution: "@types/plist@npm:3.0.5" @@ -3810,18 +3784,18 @@ __metadata: "@agentic/tavily": "npm:^7.3.3" "@ant-design/v5-patch-for-react-19": "npm:^1.0.3" "@anthropic-ai/sdk": "npm:^0.38.0" - "@cherrystudio/embedjs": "npm:^0.1.28" - "@cherrystudio/embedjs-libsql": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-csv": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-image": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-markdown": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-msoffice": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-pdf": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-sitemap": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-web": "npm:^0.1.28" - "@cherrystudio/embedjs-loader-xml": "npm:^0.1.28" - "@cherrystudio/embedjs-openai": "npm:^0.1.28" - "@cherrystudio/mac-system-ocr": "npm:^0.1.0" + "@cherrystudio/embedjs": "npm:^0.1.29" + "@cherrystudio/embedjs-libsql": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-csv": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-image": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-markdown": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-msoffice": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-pdf": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-sitemap": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-web": "npm:^0.1.29" + "@cherrystudio/embedjs-loader-xml": "npm:^0.1.29" + "@cherrystudio/embedjs-openai": "npm:^0.1.29" + "@cherrystudio/mac-system-ocr": "npm:^0.2.0" "@electron-toolkit/eslint-config-prettier": "npm:^3.0.0" "@electron-toolkit/eslint-config-ts": "npm:^3.0.0" "@electron-toolkit/preload": "npm:^3.0.0" @@ -3849,7 +3823,6 @@ __metadata: "@types/md5": "npm:^2.3.5" "@types/node": "npm:^18.19.9" "@types/pako": "npm:^1.0.2" - "@types/pdf-parse": "npm:^1.1.4" "@types/react": "npm:^19.0.12" "@types/react-dom": "npm:^19.0.4" "@types/react-infinite-scroll-component": "npm:^5.0.0" @@ -3862,6 +3835,7 @@ __metadata: axios: "npm:^1.7.3" babel-plugin-styled-components: "npm:^2.1.4" browser-image-compression: "npm:^2.0.2" + canvas: "npm:3.1.0" dayjs: "npm:^1.11.11" dexie: "npm:^4.0.8" dexie-react-hooks: "npm:^1.1.7" @@ -3897,8 +3871,8 @@ __metadata: officeparser: "npm:^4.1.1" openai: "patch:openai@npm%3A4.77.3#~/.yarn/patches/openai-npm-4.77.3-59c6d42e7a.patch" p-queue: "npm:^8.1.0" - pdf-parse: "npm:^1.1.1" pdf-to-img: "npm:^4.4.0" + pdfjs-dist: "npm:4.2.67" prettier: "npm:^3.5.3" proxy-agent: "npm:^6.5.0" rc-virtual-list: "npm:^3.18.5" @@ -4278,16 +4252,6 @@ __metadata: languageName: node linkType: hard -"are-we-there-yet@npm:^2.0.0": - version: 2.0.0 - resolution: "are-we-there-yet@npm:2.0.0" - dependencies: - delegates: "npm:^1.0.0" - readable-stream: "npm:^3.6.0" - checksum: 10c0/375f753c10329153c8d66dc95e8f8b6c7cc2aa66e05cb0960bd69092b10dae22900cacc7d653ad11d26b3ecbdbfe1e8bfb6ccf0265ba8077a7d979970f16b99c - languageName: node - linkType: hard - "are-we-there-yet@npm:^3.0.0": version: 3.0.1 resolution: "are-we-there-yet@npm:3.0.1" @@ -4964,18 +4928,6 @@ __metadata: languageName: node linkType: hard -"canvas@npm:^2.11.2": - version: 2.11.2 - resolution: "canvas@npm:2.11.2" - dependencies: - "@mapbox/node-pre-gyp": "npm:^1.0.0" - nan: "npm:^2.17.0" - node-gyp: "npm:latest" - simple-get: "npm:^3.0.3" - checksum: 10c0/943368798ad1b66b18633aa34b6181e1038dac5433fc9727cd07be35f0a633f572b60d9edb95f5ff90b6a9128e86d5312035f91a2934101c73185b15d906230a - languageName: node - linkType: hard - "caseless@npm:~0.12.0": version: 0.12.0 resolution: "caseless@npm:0.12.0" @@ -5254,7 +5206,7 @@ __metadata: languageName: node linkType: hard -"color-support@npm:^1.1.2, color-support@npm:^1.1.3": +"color-support@npm:^1.1.3": version: 1.1.3 resolution: "color-support@npm:1.1.3" bin: @@ -5738,15 +5690,6 @@ __metadata: languageName: node linkType: hard -"decompress-response@npm:^4.2.0": - version: 4.2.1 - resolution: "decompress-response@npm:4.2.1" - dependencies: - mimic-response: "npm:^2.0.0" - checksum: 10c0/5e4821be332e80e3639acee2441c41d245fc07ac3ee85a6f28893c10c079d66d9bf09e8d84bffeae5656a4625e09e9b93fb4a5705adbe6b07202eea64fae1c8d - languageName: node - linkType: hard - "decompress-response@npm:^6.0.0": version: 6.0.0 resolution: "decompress-response@npm:6.0.0" @@ -7812,23 +7755,6 @@ __metadata: languageName: node linkType: hard -"gauge@npm:^3.0.0": - version: 3.0.2 - resolution: "gauge@npm:3.0.2" - dependencies: - aproba: "npm:^1.0.3 || ^2.0.0" - color-support: "npm:^1.1.2" - console-control-strings: "npm:^1.0.0" - has-unicode: "npm:^2.0.1" - object-assign: "npm:^4.1.1" - signal-exit: "npm:^3.0.0" - string-width: "npm:^4.2.3" - strip-ansi: "npm:^6.0.1" - wide-align: "npm:^1.1.2" - checksum: 10c0/75230ccaf216471e31025c7d5fcea1629596ca20792de50c596eb18ffb14d8404f927cd55535aab2eeecd18d1e11bd6f23ec3c2e9878d2dda1dc74bccc34b913 - languageName: node - linkType: hard - "gauge@npm:^4.0.3": version: 4.0.4 resolution: "gauge@npm:4.0.4" @@ -9978,15 +9904,6 @@ __metadata: languageName: node linkType: hard -"make-dir@npm:^3.1.0": - version: 3.1.0 - resolution: "make-dir@npm:3.1.0" - dependencies: - semver: "npm:^6.0.0" - checksum: 10c0/56aaafefc49c2dfef02c5c95f9b196c4eb6988040cf2c712185c7fe5c99b4091591a7fc4d4eafaaefa70ff763a26f6ab8c3ff60b9e75ea19876f49b18667ecaa - languageName: node - linkType: hard - "make-fetch-happen@npm:^14.0.3": version: 14.0.3 resolution: "make-fetch-happen@npm:14.0.3" @@ -11087,13 +11004,6 @@ __metadata: languageName: node linkType: hard -"mimic-response@npm:^2.0.0": - version: 2.1.0 - resolution: "mimic-response@npm:2.1.0" - checksum: 10c0/717475c840f20deca87a16cb2f7561f9115f5de225ea2377739e09890c81aec72f43c81fd4984650c4044e66be5a846fa7a517ac7908f01009e1e624e19864d5 - languageName: node - linkType: hard - "mimic-response@npm:^3.1.0": version: 3.1.0 resolution: "mimic-response@npm:3.1.0" @@ -11375,15 +11285,6 @@ __metadata: languageName: node linkType: hard -"nan@npm:^2.17.0": - version: 2.22.2 - resolution: "nan@npm:2.22.2" - dependencies: - node-gyp: "npm:latest" - checksum: 10c0/971f963b8120631880fa47a389c71b00cadc1c1b00ef8f147782a3f4387d4fc8195d0695911272d57438c11562fb27b24c4ae5f8c05d5e4eeb4478ba51bb73c5 - languageName: node - linkType: hard - "nan@npm:~2.10.0": version: 2.10.0 resolution: "nan@npm:2.10.0" @@ -11714,18 +11615,6 @@ __metadata: languageName: node linkType: hard -"npmlog@npm:^5.0.1": - version: 5.0.1 - resolution: "npmlog@npm:5.0.1" - dependencies: - are-we-there-yet: "npm:^2.0.0" - console-control-strings: "npm:^1.1.0" - gauge: "npm:^3.0.0" - set-blocking: "npm:^2.0.0" - checksum: 10c0/489ba519031013001135c463406f55491a17fc7da295c18a04937fe3a4d523fd65e88dd418a28b967ab743d913fdeba1e29838ce0ad8c75557057c481f7d49fa - languageName: node - linkType: hard - "npmlog@npm:^6.0.0": version: 6.0.2 resolution: "npmlog@npm:6.0.2" @@ -11759,7 +11648,7 @@ __metadata: languageName: node linkType: hard -"object-assign@npm:^4, object-assign@npm:^4.0.1, object-assign@npm:^4.1.0, object-assign@npm:^4.1.1": +"object-assign@npm:^4, object-assign@npm:^4.0.1, object-assign@npm:^4.1.0": version: 4.1.1 resolution: "object-assign@npm:4.1.1" checksum: 10c0/1f4df9945120325d041ccf7b86f31e8bcc14e73d29171e37a7903050e96b81323784ec59f93f102ec635bcf6fa8034ba3ea0a8c7e69fa202b87ae3b6cec5a414 @@ -12402,7 +12291,7 @@ __metadata: languageName: node linkType: hard -"pdf-parse@npm:1.1.1, pdf-parse@npm:^1.1.1": +"pdf-parse@npm:1.1.1": version: 1.1.1 resolution: "pdf-parse@npm:1.1.1" dependencies: @@ -14405,7 +14294,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:^6.0.0, semver@npm:^6.2.0, semver@npm:^6.3.1": +"semver@npm:^6.2.0, semver@npm:^6.3.1": version: 6.3.1 resolution: "semver@npm:6.3.1" bin: @@ -14593,17 +14482,6 @@ __metadata: languageName: node linkType: hard -"simple-get@npm:^3.0.3": - version: 3.1.1 - resolution: "simple-get@npm:3.1.1" - dependencies: - decompress-response: "npm:^4.2.0" - once: "npm:^1.3.1" - simple-concat: "npm:^1.0.0" - checksum: 10c0/438c78844ea1b1e7268d13ee0b3a39c7d644183367aec916aed3b676b45d3037a61d9f975c200a49b42eb851f29f03745118af1e13c01e60a7b4044f2fd60be7 - languageName: node - linkType: hard - "simple-get@npm:^4.0.0": version: 4.0.1 resolution: "simple-get@npm:4.0.1" @@ -16269,7 +16147,7 @@ __metadata: languageName: node linkType: hard -"wide-align@npm:^1.1.0, wide-align@npm:^1.1.2, wide-align@npm:^1.1.5": +"wide-align@npm:^1.1.0, wide-align@npm:^1.1.5": version: 1.1.5 resolution: "wide-align@npm:1.1.5" dependencies: