From 777eb48e6e81c8d1b42020da61f3ceccd0f33aeb Mon Sep 17 00:00:00 2001 From: eeee0717 Date: Tue, 3 Jun 2025 21:29:08 +0800 Subject: [PATCH] rename ocr to preprocess --- src/main/ocr/DefaultOcrProvider.ts | 12 ---- src/main/ocr/OcrProviderFactory.ts | 29 --------- .../BasePreprocessProvider.ts} | 30 +++++----- .../preprocess/DefaultPreprocessProvider.ts | 12 ++++ .../Doc2xPreprocessProvider.ts} | 26 ++++---- .../{ocr => preprocess}/MacSysOcrProvider.ts | 10 ++-- .../MineruPreprocessProvider.ts} | 18 +++--- .../MistralPreprocessProvider.ts} | 20 +++---- .../PreprocessProvider.ts} | 14 ++--- .../preprocess/PreprocessProviderFactory.ts | 29 +++++++++ src/main/services/KnowledgeService.ts | 26 ++++---- ...ocrProviders.ts => preprocessProviders.ts} | 4 +- src/renderer/src/hooks/useOcr.ts | 45 -------------- src/renderer/src/hooks/usePreprocess.ts | 48 +++++++++++++++ src/renderer/src/i18n/locales/en-us.json | 13 ++++ .../src/pages/knowledge/KnowledgeContent.tsx | 2 +- .../components/KnowledgeSettings.tsx | 32 ++++------ .../src/pages/settings/SettingsPage.tsx | 4 +- .../ToolSettings/OcrSettings/index.tsx | 58 ------------------ .../PreprocessSettings.tsx} | 60 +++++++++---------- .../ToolSettings/PreprocessSettings/index.tsx | 58 ++++++++++++++++++ .../src/pages/settings/ToolSettings/index.tsx | 10 ++-- src/renderer/src/services/KnowledgeService.ts | 4 +- src/renderer/src/store/index.ts | 4 +- src/renderer/src/store/migrate.ts | 12 ++-- .../src/store/{ocr.ts => preprocess.ts} | 29 +++++---- src/renderer/src/types/index.ts | 10 ++-- 27 files changed, 316 insertions(+), 303 deletions(-) delete mode 100644 src/main/ocr/DefaultOcrProvider.ts delete mode 100644 src/main/ocr/OcrProviderFactory.ts rename src/main/{ocr/BaseOcrProvider.ts => preprocess/BasePreprocessProvider.ts} (86%) create mode 100644 src/main/preprocess/DefaultPreprocessProvider.ts rename src/main/{ocr/Doc2xOcrProvider.ts => preprocess/Doc2xPreprocessProvider.ts} (91%) rename src/main/{ocr => preprocess}/MacSysOcrProvider.ts (92%) rename src/main/{ocr/MineruOcrProvider.ts => preprocess/MineruPreprocessProvider.ts} (94%) rename src/main/{ocr/MistralOcrProvider.ts => preprocess/MistralPreprocessProvider.ts} (89%) rename src/main/{ocr/OcrProvider.ts => preprocess/PreprocessProvider.ts} (54%) create mode 100644 src/main/preprocess/PreprocessProviderFactory.ts rename src/renderer/src/config/{ocrProviders.ts => preprocessProviders.ts} (89%) delete mode 100644 src/renderer/src/hooks/useOcr.ts create mode 100644 src/renderer/src/hooks/usePreprocess.ts delete mode 100644 src/renderer/src/pages/settings/ToolSettings/OcrSettings/index.tsx rename src/renderer/src/pages/settings/ToolSettings/{OcrSettings/OcrProviderSettings.tsx => PreprocessSettings/PreprocessSettings.tsx} (66%) create mode 100644 src/renderer/src/pages/settings/ToolSettings/PreprocessSettings/index.tsx rename src/renderer/src/store/{ocr.ts => preprocess.ts} (58%) diff --git a/src/main/ocr/DefaultOcrProvider.ts b/src/main/ocr/DefaultOcrProvider.ts deleted file mode 100644 index 83c8d51c9..000000000 --- a/src/main/ocr/DefaultOcrProvider.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { FileMetadata, OcrProvider } from '@types' - -import BaseOcrProvider from './BaseOcrProvider' - -export default class DefaultOcrProvider extends BaseOcrProvider { - constructor(provider: OcrProvider) { - super(provider) - } - public parseFile(): Promise<{ processedFile: FileMetadata }> { - throw new Error('Method not implemented.') - } -} diff --git a/src/main/ocr/OcrProviderFactory.ts b/src/main/ocr/OcrProviderFactory.ts deleted file mode 100644 index 98cf7ca3b..000000000 --- a/src/main/ocr/OcrProviderFactory.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { isMac } from '@main/constant' -import { OcrProvider } from '@types' -import Logger from 'electron-log' - -import BaseOcrProvider from './BaseOcrProvider' -import DefaultOcrProvider from './DefaultOcrProvider' -import Doc2xOcrProvider from './Doc2xOcrProvider' -import MacSysOcrProvider from './MacSysOcrProvider' -import MineruOcrProvider from './MineruOcrProvider' -import MistralOcrProvider from './MistralOcrProvider' -export default class OcrProviderFactory { - static create(provider: OcrProvider): BaseOcrProvider { - switch (provider.id) { - case 'doc2x': - return new Doc2xOcrProvider(provider) - case 'mistral': - return new MistralOcrProvider(provider) - case 'system': - if (!isMac) { - Logger.warn('[OCR] System OCR provider is only available on macOS') - } - return new MacSysOcrProvider(provider) - case 'mineru': - return new MineruOcrProvider(provider) - default: - return new DefaultOcrProvider(provider) - } - } -} diff --git a/src/main/ocr/BaseOcrProvider.ts b/src/main/preprocess/BasePreprocessProvider.ts similarity index 86% rename from src/main/ocr/BaseOcrProvider.ts rename to src/main/preprocess/BasePreprocessProvider.ts index f941b6e96..60480e6af 100644 --- a/src/main/ocr/BaseOcrProvider.ts +++ b/src/main/preprocess/BasePreprocessProvider.ts @@ -3,47 +3,47 @@ import path from 'node:path' import { windowService } from '@main/services/WindowService' import { getFileExt } from '@main/utils/file' -import { FileMetadata, OcrProvider } from '@types' +import { FileMetadata, PreprocessProvider } from '@types' import { createCanvas, loadImage } from 'canvas' import { app } from 'electron' import { TypedArray } from 'pdfjs-dist/types/src/display/api' -export default abstract class BaseOcrProvider { - protected provider: OcrProvider +export default abstract class BasePreprocessProvider { + protected provider: PreprocessProvider private storageDir = path.join(app.getPath('userData'), 'Data', 'Files') - constructor(provider: OcrProvider) { + constructor(provider: PreprocessProvider) { if (!provider) { - throw new Error('Ocr provider is not set') + throw new Error('Preprocess provider is not set') } this.provider = provider } abstract parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata }> /** - * 检查文件是否已经被OCR处理过 - * 统一检测方法：如果 Data/Files/{file.id} 是目录，说明已被OCR处理 + * 检查文件是否已经被预处理过 + * 统一检测方法：如果 Data/Files/{file.id} 是目录，说明已被预处理 * @param file 文件信息 * @returns 如果已处理返回处理后的文件信息，否则返回null */ public async checkIfAlreadyProcessed(file: FileMetadata): Promise { try { // 检查 Data/Files/{file.id} 是否是目录 - const ocrDirPath = path.join(this.storageDir, file.id) + const preprocessDirPath = path.join(this.storageDir, file.id) - if (fs.existsSync(ocrDirPath)) { - const stats = await fs.promises.stat(ocrDirPath) + if (fs.existsSync(preprocessDirPath)) { + const stats = await fs.promises.stat(preprocessDirPath) - // 如果是目录，说明已经被OCR处理过 + // 如果是目录，说明已经被预处理过 if (stats.isDirectory()) { // 查找目录中的处理结果文件 - const files = await fs.promises.readdir(ocrDirPath) + const files = await fs.promises.readdir(preprocessDirPath) // 查找主要的处理结果文件（.md 或 .txt） const processedFile = files.find((fileName) => fileName.endsWith('.md') || fileName.endsWith('.txt')) if (processedFile) { - const processedFilePath = path.join(ocrDirPath, processedFile) + const processedFilePath = path.join(preprocessDirPath, processedFile) const processedStats = await fs.promises.stat(processedFilePath) const ext = getFileExt(processedFile) @@ -87,9 +87,9 @@ export default abstract class BaseOcrProvider { return document } - public async sendOcrProgress(sourceId: string, progress: number): Promise { + public async sendPreprocessProgress(sourceId: string, progress: number): Promise { const mainWindow = windowService.getMainWindow() - mainWindow?.webContents.send('file-ocr-progress', { + mainWindow?.webContents.send('file-preprocess-progress', { itemId: sourceId, progress: progress }) diff --git a/src/main/preprocess/DefaultPreprocessProvider.ts b/src/main/preprocess/DefaultPreprocessProvider.ts new file mode 100644 index 000000000..8413202d2 --- /dev/null +++ b/src/main/preprocess/DefaultPreprocessProvider.ts @@ -0,0 +1,12 @@ +import { FileMetadata, PreprocessProvider } from '@types' + +import BasePreprocessProvider from './BasePreprocessProvider' + +export default class DefaultPreprocessProvider extends BasePreprocessProvider { + constructor(provider: PreprocessProvider) { + super(provider) + } + public parseFile(): Promise<{ processedFile: FileMetadata }> { + throw new Error('Method not implemented.') + } +} diff --git a/src/main/ocr/Doc2xOcrProvider.ts b/src/main/preprocess/Doc2xPreprocessProvider.ts similarity index 91% rename from src/main/ocr/Doc2xOcrProvider.ts rename to src/main/preprocess/Doc2xPreprocessProvider.ts index d86778c76..d7adba738 100644 --- a/src/main/ocr/Doc2xOcrProvider.ts +++ b/src/main/preprocess/Doc2xPreprocessProvider.ts @@ -2,12 +2,12 @@ import fs from 'node:fs' import path from 'node:path' import { getFileDir } from '@main/utils/file' -import { FileMetadata, OcrProvider } from '@types' +import { FileMetadata, PreprocessProvider } from '@types' import AdmZip from 'adm-zip' import axios, { AxiosRequestConfig } from 'axios' import Logger from 'electron-log' -import BaseOcrProvider from './BaseOcrProvider' +import BasePreprocessProvider from './BasePreprocessProvider' type ApiResponse = { code: string @@ -30,8 +30,8 @@ type ParsedFileResponse = { url: string } -export default class Doc2xOcrProvider extends BaseOcrProvider { - constructor(provider: OcrProvider) { +export default class Doc2xPreprocessProvider extends BasePreprocessProvider { + constructor(provider: PreprocessProvider) { super(provider) } @@ -53,11 +53,11 @@ export default class Doc2xOcrProvider extends BaseOcrProvider { public async parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata }> { try { - Logger.info(`OCR processing started: ${file.path}`) + Logger.info(`Preprocess processing started: ${file.path}`) // 步骤1: 准备上传 const { uid, url } = await this.preupload() - Logger.info(`OCR preupload completed: uid=${uid}`) + Logger.info(`Preprocess preupload completed: uid=${uid}`) await this.validateFile(file.path) @@ -66,7 +66,7 @@ export default class Doc2xOcrProvider extends BaseOcrProvider { // 步骤3: 等待处理完成 await this.waitForProcessing(sourceId, uid) - Logger.info(`OCR parsing completed successfully for: ${file.path}`) + Logger.info(`Preprocess parsing completed successfully for: ${file.path}`) // 步骤4: 导出文件 const { path: outputPath } = await this.exportFile(file, uid) @@ -76,7 +76,9 @@ export default class Doc2xOcrProvider extends BaseOcrProvider { processedFile: this.createProcessedFileInfo(file, outputPath) } } catch (error) { - Logger.error(`OCR processing failed for ${file.path}: ${error instanceof Error ? error.message : String(error)}`) + Logger.error( + `Preprocess processing failed for ${file.path}: ${error instanceof Error ? error.message : String(error)}` + ) throw error } } @@ -121,13 +123,13 @@ export default class Doc2xOcrProvider extends BaseOcrProvider { while (true) { await this.delay(1000) const { status, progress } = await this.getStatus(uid) - await this.sendOcrProgress(sourceId, progress) - Logger.info(`OCR processing status: ${status}, progress: ${progress}%`) + await this.sendPreprocessProgress(sourceId, progress) + Logger.info(`Preprocess processing status: ${status}, progress: ${progress}%`) if (status === 'success') { return } else if (status === 'failed') { - throw new Error('OCR processing failed') + throw new Error('Preprocess processing failed') } } } @@ -211,7 +213,7 @@ export default class Doc2xOcrProvider extends BaseOcrProvider { } /** - * OCR文件 + * Preprocess文件 * @param uid 预上传响应的uid * @param filePath 文件路径 */ diff --git a/src/main/ocr/MacSysOcrProvider.ts b/src/main/preprocess/MacSysOcrProvider.ts similarity index 92% rename from src/main/ocr/MacSysOcrProvider.ts rename to src/main/preprocess/MacSysOcrProvider.ts index a565ebf95..78b52960f 100644 --- a/src/main/ocr/MacSysOcrProvider.ts +++ b/src/main/preprocess/MacSysOcrProvider.ts @@ -1,13 +1,13 @@ import { isMac } from '@main/constant' -import { FileMetadata, OcrProvider } from '@types' +import { FileMetadata, PreprocessProvider } from '@types' import Logger from 'electron-log' import * as fs from 'fs' import * as path from 'path' import { TextItem } from 'pdfjs-dist/types/src/display/api' -import BaseOcrProvider from './BaseOcrProvider' +import BasePreprocessProvider from './BasePreprocessProvider' -export default class MacSysOcrProvider extends BaseOcrProvider { +export default class MacSysOcrProvider extends BasePreprocessProvider { private readonly MIN_TEXT_LENGTH = 1000 private MacOCR: any @@ -32,7 +32,7 @@ export default class MacSysOcrProvider extends BaseOcrProvider { return level === 0 ? this.MacOCR.RECOGNITION_LEVEL_FAST : this.MacOCR.RECOGNITION_LEVEL_ACCURATE } - constructor(provider: OcrProvider) { + constructor(provider: PreprocessProvider) { super(provider) } @@ -61,7 +61,7 @@ export default class MacSysOcrProvider extends BaseOcrProvider { writeStream.write(ocrResult.text + '\n') // Update progress - await this.sendOcrProgress(sourceId, (pageNum / totalPages) * 100) + await this.sendPreprocessProgress(sourceId, (pageNum / totalPages) * 100) } } diff --git a/src/main/ocr/MineruOcrProvider.ts b/src/main/preprocess/MineruPreprocessProvider.ts similarity index 94% rename from src/main/ocr/MineruOcrProvider.ts rename to src/main/preprocess/MineruPreprocessProvider.ts index 1acfb0350..2fd394c13 100644 --- a/src/main/ocr/MineruOcrProvider.ts +++ b/src/main/preprocess/MineruPreprocessProvider.ts @@ -1,12 +1,12 @@ import fs from 'node:fs' import path from 'node:path' -import { FileMetadata, OcrProvider } from '@types' +import { FileMetadata, PreprocessProvider } from '@types' import AdmZip from 'adm-zip' import axios from 'axios' import Logger from 'electron-log' -import BaseOcrProvider from './BaseOcrProvider' +import BasePreprocessProvider from './BasePreprocessProvider' type ApiResponse = { code: number @@ -39,14 +39,14 @@ type ExtractResultResponse = { extract_result: ExtractFileResult[] } -export default class MineruOcrProvider extends BaseOcrProvider { - constructor(provider: OcrProvider) { +export default class MineruPreprocessProvider extends BasePreprocessProvider { + constructor(provider: PreprocessProvider) { super(provider) } public async parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata }> { try { - Logger.info(`MinerU OCR processing started: ${file.path}`) + Logger.info(`MinerU preprocess processing started: ${file.path}`) await this.validateFile(file.path) // 1. 获取上传URL并上传文件 @@ -65,8 +65,8 @@ export default class MineruOcrProvider extends BaseOcrProvider { processedFile: this.createProcessedFileInfo(file, outputPath) } } catch (error: any) { - Logger.error(`MinerU OCR processing failed for ${file.path}: ${error.message}`) - throw new Error(`OCR processing failed: ${error.message}`) + Logger.error(`MinerU preprocess processing failed for ${file.path}: ${error.message}`) + throw new Error(`preprocess processing failed: ${error.message}`) } } @@ -309,11 +309,11 @@ export default class MineruOcrProvider extends BaseOcrProvider { const progress = Math.round( (fileResult.extract_progress.extracted_pages / fileResult.extract_progress.total_pages) * 100 ) - await this.sendOcrProgress(sourceId, progress) + await this.sendPreprocessProgress(sourceId, progress) Logger.info(`File ${fileName} processing progress: ${progress}%`) } else { // 如果没有具体进度信息，发送一个通用进度 - await this.sendOcrProgress(sourceId, 50) + await this.sendPreprocessProgress(sourceId, 50) Logger.info(`File ${fileName} is still processing...`) } } diff --git a/src/main/ocr/MistralOcrProvider.ts b/src/main/preprocess/MistralPreprocessProvider.ts similarity index 89% rename from src/main/ocr/MistralOcrProvider.ts rename to src/main/preprocess/MistralPreprocessProvider.ts index 6bbbde3c9..9e7b598da 100644 --- a/src/main/ocr/MistralOcrProvider.ts +++ b/src/main/preprocess/MistralPreprocessProvider.ts @@ -6,19 +6,19 @@ import { Mistral } from '@mistralai/mistralai' import { DocumentURLChunk } from '@mistralai/mistralai/models/components/documenturlchunk' import { ImageURLChunk } from '@mistralai/mistralai/models/components/imageurlchunk' import { OCRResponse } from '@mistralai/mistralai/models/components/ocrresponse' -import { FileMetadata, FileTypes, OcrProvider, Provider } from '@types' +import { FileMetadata, FileTypes, PreprocessProvider, Provider } from '@types' import Logger from 'electron-log' import path from 'path' -import BaseOcrProvider from './BaseOcrProvider' +import BasePreprocessProvider from './BasePreprocessProvider' type PreuploadResponse = DocumentURLChunk | ImageURLChunk -export default class MistralOcrProvider extends BaseOcrProvider { +export default class MistralPreprocessProvider extends BasePreprocessProvider { private sdk: Mistral private fileService: MistralService - constructor(provider: OcrProvider) { + constructor(provider: PreprocessProvider) { super(provider) const clientManager = MistralClientManager.getInstance() const aiProvider: Provider = { @@ -36,7 +36,7 @@ export default class MistralOcrProvider extends BaseOcrProvider { private async preupload(file: FileMetadata): Promise { let document: PreuploadResponse - Logger.info(`OCR preupload started for local file: ${file.path}`) + Logger.info(`preprocess preupload started for local file: ${file.path}`) if (file.ext.toLowerCase() === '.pdf') { const uploadResponse = await this.fileService.uploadFile(file) @@ -45,12 +45,12 @@ export default class MistralOcrProvider extends BaseOcrProvider { Logger.error('File upload failed:', uploadResponse) throw new Error('Failed to upload file: ' + uploadResponse.displayName) } - await this.sendOcrProgress(file.id, 15) + await this.sendPreprocessProgress(file.id, 15) const fileUrl = await this.sdk.files.getSignedUrl({ fileId: uploadResponse.fileId }) Logger.info('Got signed URL:', fileUrl) - await this.sendOcrProgress(file.id, 20) + await this.sendPreprocessProgress(file.id, 20) document = { type: 'document_url', documentUrl: fileUrl.url @@ -78,16 +78,16 @@ export default class MistralOcrProvider extends BaseOcrProvider { includeImageBase64: true }) if (result) { - await this.sendOcrProgress(sourceId, 100) + await this.sendPreprocessProgress(sourceId, 100) const processedFile = this.convertFile(result, file) return { processedFile } } else { - throw new Error('OCR processing failed: OCR response is empty') + throw new Error('preprocess processing failed: OCR response is empty') } } catch (error) { - throw new Error('OCR processing failed: ' + error) + throw new Error('preprocess processing failed: ' + error) } } diff --git a/src/main/ocr/OcrProvider.ts b/src/main/preprocess/PreprocessProvider.ts similarity index 54% rename from src/main/ocr/OcrProvider.ts rename to src/main/preprocess/PreprocessProvider.ts index c9eb0c737..4232339a4 100644 --- a/src/main/ocr/OcrProvider.ts +++ b/src/main/preprocess/PreprocessProvider.ts @@ -1,19 +1,19 @@ -import { FileMetadata, OcrProvider as Provider } from '@types' +import { FileMetadata, PreprocessProvider as Provider } from '@types' -import BaseOcrProvider from './BaseOcrProvider' -import OcrProviderFactory from './OcrProviderFactory' +import BasePreprocessProvider from './BasePreprocessProvider' +import PreprocessProviderFactory from './PreprocessProviderFactory' -export default class OcrProvider { - private sdk: BaseOcrProvider +export default class PreprocessProvider { + private sdk: BasePreprocessProvider constructor(provider: Provider) { - this.sdk = OcrProviderFactory.create(provider) + this.sdk = PreprocessProviderFactory.create(provider) } public async parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata }> { return this.sdk.parseFile(sourceId, file) } /** - * 检查文件是否已经被OCR处理过 + * 检查文件是否已经被预处理过 * @param file 文件信息 * @returns 如果已处理返回处理后的文件信息，否则返回null */ diff --git a/src/main/preprocess/PreprocessProviderFactory.ts b/src/main/preprocess/PreprocessProviderFactory.ts new file mode 100644 index 000000000..ad17fc0f0 --- /dev/null +++ b/src/main/preprocess/PreprocessProviderFactory.ts @@ -0,0 +1,29 @@ +import { isMac } from '@main/constant' +import { PreprocessProvider } from '@types' +import Logger from 'electron-log' + +import BasePreprocessProvider from './BasePreprocessProvider' +import DefaultPreprocessProvider from './DefaultPreprocessProvider' +import Doc2xPreprocessProvider from './Doc2xPreprocessProvider' +import MacSysOcrProvider from './MacSysOcrProvider' +import MineruPreprocessProvider from './MineruPreprocessProvider' +import MistralPreprocessProvider from './MistralPreprocessProvider' +export default class PreprocessProviderFactory { + static create(provider: PreprocessProvider): BasePreprocessProvider { + switch (provider.id) { + case 'doc2x': + return new Doc2xPreprocessProvider(provider) + case 'mistral': + return new MistralPreprocessProvider(provider) + case 'system': + if (!isMac) { + Logger.warn('[OCR] System OCR provider is only available on macOS') + } + return new MacSysOcrProvider(provider) + case 'mineru': + return new MineruPreprocessProvider(provider) + default: + return new DefaultPreprocessProvider(provider) + } + } +} diff --git a/src/main/services/KnowledgeService.ts b/src/main/services/KnowledgeService.ts index e404db9c0..05eae32c3 100644 --- a/src/main/services/KnowledgeService.ts +++ b/src/main/services/KnowledgeService.ts @@ -23,7 +23,7 @@ import { SitemapLoader } from '@cherrystudio/embedjs-loader-sitemap' import { WebLoader } from '@cherrystudio/embedjs-loader-web' import Embeddings from '@main/embeddings/Embeddings' import { addFileLoader } from '@main/loader' -import OcrProvider from '@main/ocr/OcrProvider' +import PreprocessProvider from '@main/preprocess/PreprocessProvider' import Reranker from '@main/reranker/Reranker' import { windowService } from '@main/services/WindowService' import { getAllFiles } from '@main/utils/file' @@ -167,7 +167,7 @@ class KnowledgeService { { state: LoaderTaskItemState.PENDING, task: async () => { - // 添加OCR预处理逻辑 + // 添加预处理逻辑 const fileToProcess: FileMetadata = await this.preprocessing(file, base, item) // 使用处理后的文件进行加载 @@ -498,25 +498,25 @@ class KnowledgeService { item: KnowledgeItem ): Promise => { let fileToProcess: FileMetadata = file - - if (base.preprocessing && base.ocrProvider && file.ext.toLowerCase() === '.pdf') { + console.warn(`Preprocessing file`, JSON.stringify(base, null, 2)) + if (base.preprocessProvider && file.ext.toLowerCase() === '.pdf') { try { - const ocrProvider = new OcrProvider(base.ocrProvider) + const preprocessProvider = new PreprocessProvider(base.preprocessProvider) - // 首先检查文件是否已经被OCR处理过 - const alreadyProcessed = await ocrProvider.checkIfAlreadyProcessed(file) + // 首先检查文件是否已经被预处理过 + const alreadyProcessed = await preprocessProvider.checkIfAlreadyProcessed(file) if (alreadyProcessed) { - Logger.info(`File already OCR processed, using cached result: ${file.path}`) + Logger.info(`File already preprocess processed, using cached result: ${file.path}`) return alreadyProcessed } - // 执行OCR处理 - Logger.info(`Starting OCR processing for scanned PDF: ${file.path}`) - const { processedFile } = await ocrProvider.parseFile(item.id, file) + // 执行预处理 + Logger.info(`Starting preprocess processing for scanned PDF: ${file.path}`) + const { processedFile } = await preprocessProvider.parseFile(item.id, file) fileToProcess = processedFile } catch (err) { - Logger.error(`OCR processing failed: ${err}`) - // 如果OCR失败，使用原始文件 + Logger.error(`Preprocess processing failed: ${err}`) + // 如果预处理失败，使用原始文件 fileToProcess = file } } diff --git a/src/renderer/src/config/ocrProviders.ts b/src/renderer/src/config/preprocessProviders.ts similarity index 89% rename from src/renderer/src/config/ocrProviders.ts rename to src/renderer/src/config/preprocessProviders.ts index 43ae1b484..f8e1eaeab 100644 --- a/src/renderer/src/config/ocrProviders.ts +++ b/src/renderer/src/config/preprocessProviders.ts @@ -3,7 +3,7 @@ import MinerULogo from '@renderer/assets/images/ocr/mineru.jpg' import MacOSLogo from '@renderer/assets/images/providers/macos.svg' import MistralLogo from '@renderer/assets/images/providers/mistral.png' -export function getOcrProviderLogo(providerId: string) { +export function getPreprocessProviderLogo(providerId: string) { switch (providerId) { case 'doc2x': return Doc2xLogo @@ -18,7 +18,7 @@ export function getOcrProviderLogo(providerId: string) { } } -export const OCR_PROVIDER_CONFIG = { +export const PREPROCESS_PROVIDER_CONFIG = { doc2x: { websites: { official: 'https://doc2x.noedgeai.com', diff --git a/src/renderer/src/hooks/useOcr.ts b/src/renderer/src/hooks/useOcr.ts deleted file mode 100644 index 69a8f96ea..000000000 --- a/src/renderer/src/hooks/useOcr.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { RootState } from '@renderer/store' -import { - setDefaultOcrProvider as _setDefaultOcrProvider, - updateOcrProvider as _updateOcrProvider, - updateOcrProviders as _updateOcrProviders -} from '@renderer/store/ocr' -import { OcrProvider } from '@renderer/types' -import { useDispatch, useSelector } from 'react-redux' - -export const useOcrProvider = (id: string) => { - const dispatch = useDispatch() - const ocrProviders = useSelector((state: RootState) => state.ocr.providers) - const provider = ocrProviders.find((provider) => provider.id === id) - if (!provider) { - throw new Error(`ocr provider with id ${id} not found`) - } - const updateOcrProvider = (ocrProvider: OcrProvider) => { - dispatch(_updateOcrProvider(ocrProvider)) - } - return { provider, updateOcrProvider } -} - -export const useOcrProviders = () => { - const dispatch = useDispatch() - const ocrProviders = useSelector((state: RootState) => state.ocr.providers) - return { - ocrProviders, - updateOcrProviders: (ocrProviders: OcrProvider[]) => dispatch(_updateOcrProviders(ocrProviders)) - } -} - -export const useDefaultOcrProvider = () => { - const defaultProviderId = useSelector((state: RootState) => state.ocr.defaultProvider) - const { ocrProviders } = useOcrProviders() - const dispatch = useDispatch() - const provider = defaultProviderId ? ocrProviders.find((provider) => provider.id === defaultProviderId) : undefined - - const setDefaultOcrProvider = (ocrProvider: OcrProvider) => { - dispatch(_setDefaultOcrProvider(ocrProvider.id)) - } - const updateDefaultOcrProvider = (ocrProvider: OcrProvider) => { - dispatch(_updateOcrProvider(ocrProvider)) - } - return { provider, setDefaultOcrProvider, updateDefaultOcrProvider } -} diff --git a/src/renderer/src/hooks/usePreprocess.ts b/src/renderer/src/hooks/usePreprocess.ts new file mode 100644 index 000000000..5a4c6649b --- /dev/null +++ b/src/renderer/src/hooks/usePreprocess.ts @@ -0,0 +1,48 @@ +import { RootState } from '@renderer/store' +import { + setDefaultPreprocessProvider as _setDefaultPreprocessProvider, + updatePreprocessProvider as _updatePreprocessProvider, + updatePreprocessProviders as _updatePreprocessProviders +} from '@renderer/store/preprocess' +import { PreprocessProvider } from '@renderer/types' +import { useDispatch, useSelector } from 'react-redux' + +export const usePreprocessProvider = (id: string) => { + const dispatch = useDispatch() + const preprocessProviders = useSelector((state: RootState) => state.preprocess.providers) + const provider = preprocessProviders.find((provider) => provider.id === id) + if (!provider) { + throw new Error(`preprocess provider with id ${id} not found`) + } + const updatePreprocessProvider = (preprocessProvider: PreprocessProvider) => { + dispatch(_updatePreprocessProvider(preprocessProvider)) + } + return { provider, updatePreprocessProvider } +} + +export const usePreprocessProviders = () => { + const dispatch = useDispatch() + const preprocessProviders = useSelector((state: RootState) => state.preprocess.providers) + return { + preprocessProviders: preprocessProviders, + updatePreprocessProviders: (preprocessProviders: PreprocessProvider[]) => + dispatch(_updatePreprocessProviders(preprocessProviders)) + } +} + +export const useDefaultPreprocessProvider = () => { + const defaultProviderId = useSelector((state: RootState) => state.preprocess.defaultProvider) + const { preprocessProviders } = usePreprocessProviders() + const dispatch = useDispatch() + const provider = defaultProviderId + ? preprocessProviders.find((provider) => provider.id === defaultProviderId) + : undefined + + const setDefaultPreprocessProvider = (preprocessProvider: PreprocessProvider) => { + dispatch(_setDefaultPreprocessProvider(preprocessProvider.id)) + } + const updateDefaultPreprocessProvider = (preprocessProvider: PreprocessProvider) => { + dispatch(_updatePreprocessProvider(preprocessProvider)) + } + return { provider, setDefaultPreprocessProvider, updateDefaultPreprocessProvider } +} diff --git a/src/renderer/src/i18n/locales/en-us.json b/src/renderer/src/i18n/locales/en-us.json index 66da2e5ac..9d49a61c0 100644 --- a/src/renderer/src/i18n/locales/en-us.json +++ b/src/renderer/src/i18n/locales/en-us.json @@ -1728,6 +1728,19 @@ "min_confidence": "Minimum Confidence" } }, + "preprocess": { + "title": "Pre Process", + "provider": "Pre Process Provider", + "provider_placeholder": "Choose a Pre Process provider", + "mac_system_ocr_options": { + "mode": { + "title": "Recognition Mode", + "accurate": "Accurate", + "fast": "Fast" + }, + "min_confidence": "Minimum Confidence" + } + }, "websearch": { "blacklist": "Blacklist", "blacklist_description": "Results from the following websites will not appear in search results", diff --git a/src/renderer/src/pages/knowledge/KnowledgeContent.tsx b/src/renderer/src/pages/knowledge/KnowledgeContent.tsx index a8596c994..8ac5909aa 100644 --- a/src/renderer/src/pages/knowledge/KnowledgeContent.tsx +++ b/src/renderer/src/pages/knowledge/KnowledgeContent.tsx @@ -42,7 +42,7 @@ const KnowledgeContent: FC = ({ selectedBase }) => { useEffect(() => { const handlers = [ - window.electron.ipcRenderer.on('file-ocr-progress', (_, { itemId, progress }) => { + window.electron.ipcRenderer.on('file-preprocess-progress', (_, { itemId, progress }) => { setProgressMap((prev) => new Map(prev).set(itemId, progress)) }), diff --git a/src/renderer/src/pages/knowledge/components/KnowledgeSettings.tsx b/src/renderer/src/pages/knowledge/components/KnowledgeSettings.tsx index 33d3c6484..e420c432e 100644 --- a/src/renderer/src/pages/knowledge/components/KnowledgeSettings.tsx +++ b/src/renderer/src/pages/knowledge/components/KnowledgeSettings.tsx @@ -4,11 +4,11 @@ import { DEFAULT_KNOWLEDGE_DOCUMENT_COUNT } from '@renderer/config/constant' import { getEmbeddingMaxContext } from '@renderer/config/embedings' import { isEmbeddingModel, isRerankModel } from '@renderer/config/models' import { useKnowledge } from '@renderer/hooks/useKnowledge' -import { useOcrProviders } from '@renderer/hooks/useOcr' +import { usePreprocessProviders } from '@renderer/hooks/usePreprocess' import { useProviders } from '@renderer/hooks/useProvider' import { getModelUniqId } from '@renderer/services/ModelService' -import { KnowledgeBase, OcrProvider } from '@renderer/types' -import { Alert, Input, InputNumber, Modal, Select, Slider, Switch, Tabs, TabsProps, Tooltip } from 'antd' +import { KnowledgeBase, PreprocessProvider } from '@renderer/types' +import { Alert, Input, InputNumber, Modal, Select, Slider, Tabs, TabsProps, Tooltip } from 'antd' import { sortBy } from 'lodash' import { useState } from 'react' import { useTranslation } from 'react-i18next' @@ -23,8 +23,8 @@ interface Props extends ShowParams { } const PopupContainer: React.FC = ({ base: _base, resolve }) => { - const { ocrProviders } = useOcrProviders() - const [selectedProvider, setSelectedProvider] = useState(_base.ocrProvider) + const { preprocessProviders } = usePreprocessProviders() + const [selectedProvider, setSelectedProvider] = useState(_base.preprocessProvider) const [open, setOpen] = useState(true) const { t } = useTranslation() @@ -98,18 +98,6 @@ const PopupContainer: React.FC = ({ base: _base, resolve }) => { onChange={(e) => setNewBase({ ...newBase, name: e.target.value })} /> - -
- {t('knowledge.settings.preprocessing')} - - - -
- setNewBase({ ...newBase, preprocessing: checked })} - /> -
@@ -150,18 +138,18 @@ const PopupContainer: React.FC = ({ base: _base, resolve }) => { -
{t('settings.tool.ocr.provider')}
+
{t('settings.tool.preprocess.title')}
updateSelectedOcrProvider(value)} - placeholder={t('settings.tool.ocr.provider_placeholder')} - options={ocrProviders.map((p) => ({ - value: p.id, - label: p.name, - disabled: !isMac && p.id === 'system' // 在非 Mac 系统下禁用 system 选项 - }))} - /> -
- - - {selectedProvider && ( - - - - )} - - ) -} -export default KnowledgeSettings diff --git a/src/renderer/src/pages/settings/ToolSettings/OcrSettings/OcrProviderSettings.tsx b/src/renderer/src/pages/settings/ToolSettings/PreprocessSettings/PreprocessSettings.tsx similarity index 66% rename from src/renderer/src/pages/settings/ToolSettings/OcrSettings/OcrProviderSettings.tsx rename to src/renderer/src/pages/settings/ToolSettings/PreprocessSettings/PreprocessSettings.tsx index 2a56b3260..1be72340d 100644 --- a/src/renderer/src/pages/settings/ToolSettings/OcrSettings/OcrProviderSettings.tsx +++ b/src/renderer/src/pages/settings/ToolSettings/PreprocessSettings/PreprocessSettings.tsx @@ -1,8 +1,8 @@ import { ExportOutlined } from '@ant-design/icons' -import { getOcrProviderLogo, OCR_PROVIDER_CONFIG } from '@renderer/config/ocrProviders' -import { useOcrProvider } from '@renderer/hooks/useOcr' +import { getPreprocessProviderLogo, PREPROCESS_PROVIDER_CONFIG } from '@renderer/config/preprocessProviders' +import { usePreprocessProvider } from '@renderer/hooks/usePreprocess' import { formatApiKeys } from '@renderer/services/ApiService' -import { OcrProvider } from '@renderer/types' +import { PreprocessProvider } from '@renderer/types' import { hasObjectKey } from '@renderer/utils' import { Avatar, Divider, Flex, Input, InputNumber, Segmented } from 'antd' import Link from 'antd/es/typography/Link' @@ -22,29 +22,29 @@ import { } from '../..' interface Props { - provider: OcrProvider + provider: PreprocessProvider } -const OcrProviderSetting: FC = ({ provider: _provider }) => { - const { provider: ocrProvider, updateOcrProvider } = useOcrProvider(_provider.id) +const PreprocessProviderSettings: FC = ({ provider: _provider }) => { + const { provider: preprocessProvider, updatePreprocessProvider } = usePreprocessProvider(_provider.id) const { t } = useTranslation() - const [apiKey, setApiKey] = useState(ocrProvider.apiKey || '') - const [apiHost, setApiHost] = useState(ocrProvider.apiHost || '') - const [options, setOptions] = useState(ocrProvider.options || {}) + const [apiKey, setApiKey] = useState(preprocessProvider.apiKey || '') + const [apiHost, setApiHost] = useState(preprocessProvider.apiHost || '') + const [options, setOptions] = useState(preprocessProvider.options || {}) - const ocrProviderConfig = OCR_PROVIDER_CONFIG[ocrProvider.id] - const apiKeyWebsite = ocrProviderConfig?.websites?.apiKey - const officialWebsite = ocrProviderConfig?.websites?.official + const preprocessProviderConfig = PREPROCESS_PROVIDER_CONFIG[preprocessProvider.id] + const apiKeyWebsite = preprocessProviderConfig?.websites?.apiKey + const officialWebsite = preprocessProviderConfig?.websites?.official useEffect(() => { - setApiKey(ocrProvider.apiKey ?? '') - setApiHost(ocrProvider.apiHost ?? '') - setOptions(ocrProvider.options ?? {}) - }, [ocrProvider.apiKey, ocrProvider.apiHost, ocrProvider.options]) + setApiKey(preprocessProvider.apiKey ?? '') + setApiHost(preprocessProvider.apiHost ?? '') + setOptions(preprocessProvider.options ?? {}) + }, [preprocessProvider.apiKey, preprocessProvider.apiHost, preprocessProvider.options]) const onUpdateApiKey = () => { - if (apiKey !== ocrProvider.apiKey) { - updateOcrProvider({ ...ocrProvider, apiKey }) + if (apiKey !== preprocessProvider.apiKey) { + updatePreprocessProvider({ ...preprocessProvider, apiKey }) } } @@ -53,35 +53,35 @@ const OcrProviderSetting: FC = ({ provider: _provider }) => { if (trimmedHost.endsWith('/')) { trimmedHost = trimmedHost.slice(0, -1) } - if (trimmedHost !== ocrProvider.apiHost) { - updateOcrProvider({ ...ocrProvider, apiHost: trimmedHost }) + if (trimmedHost !== preprocessProvider.apiHost) { + updatePreprocessProvider({ ...preprocessProvider, apiHost: trimmedHost }) } else { - setApiHost(ocrProvider.apiHost || '') + setApiHost(preprocessProvider.apiHost || '') } } const onUpdateOptions = (key: string, value: any) => { const newOptions = { ...options, [key]: value } setOptions(newOptions) - updateOcrProvider({ ...ocrProvider, options: newOptions }) + updatePreprocessProvider({ ...preprocessProvider, options: newOptions }) } return ( <> - + - {ocrProvider.name} - {officialWebsite && ocrProviderConfig?.websites && ( - + {preprocessProvider.name} + {officialWebsite && preprocessProviderConfig?.websites && ( + )} - {hasObjectKey(ocrProvider, 'apiKey') && ( + {hasObjectKey(preprocessProvider, 'apiKey') && ( <> {t('settings.provider.api_key')} @@ -104,7 +104,7 @@ const OcrProviderSetting: FC = ({ provider: _provider }) => { )} - {hasObjectKey(ocrProvider, 'apiHost') && ( + {hasObjectKey(preprocessProvider, 'apiHost') && ( <> {t('settings.provider.api_host')} @@ -120,7 +120,7 @@ const OcrProviderSetting: FC = ({ provider: _provider }) => { )} - {hasObjectKey(ocrProvider, 'options') && ocrProvider.id === 'system' && ( + {hasObjectKey(preprocessProvider, 'options') && preprocessProvider.id === 'system' && ( <> @@ -165,4 +165,4 @@ const ProviderLogo = styled(Avatar)` border: 0.5px solid var(--color-border); ` -export default OcrProviderSetting +export default PreprocessProviderSettings diff --git a/src/renderer/src/pages/settings/ToolSettings/PreprocessSettings/index.tsx b/src/renderer/src/pages/settings/ToolSettings/PreprocessSettings/index.tsx new file mode 100644 index 000000000..23f8771e6 --- /dev/null +++ b/src/renderer/src/pages/settings/ToolSettings/PreprocessSettings/index.tsx @@ -0,0 +1,58 @@ +import { isMac } from '@renderer/config/constant' +import { useTheme } from '@renderer/context/ThemeProvider' +import { useDefaultPreprocessProvider, usePreprocessProviders } from '@renderer/hooks/usePreprocess' +import { PreprocessProvider } from '@renderer/types' +import { Select } from 'antd' +import { FC, useState } from 'react' +import { useTranslation } from 'react-i18next' + +import { SettingContainer, SettingDivider, SettingGroup, SettingRow, SettingRowTitle, SettingTitle } from '../..' +import PreprocessProviderSettings from './PreprocessSettings' + +const PreprocessSettings: FC = () => { + const { preprocessProviders } = usePreprocessProviders() + const { provider: defaultProvider, setDefaultPreprocessProvider } = useDefaultPreprocessProvider() + const { t } = useTranslation() + const [selectedProvider, setSelectedProvider] = useState(defaultProvider) + const { theme: themeMode } = useTheme() + + function updateSelectedPreprocessProvider(providerId: string) { + const provider = preprocessProviders.find((p) => p.id === providerId) + if (!provider) { + return + } + setDefaultPreprocessProvider(provider) + setSelectedProvider(provider) + } + + return ( + + + {t('settings.tool.preprocess.title')} + + + {t('settings.tool.preprocess.provider')} +
+