refactor: remove mac ocr

src/main/knowledge/ocr/BaseOcrProvider.ts

@@ -1,122 +0,0 @@
-import fs from 'node:fs'
-import path from 'node:path'
-
-import { windowService } from '@main/services/WindowService'
-import { getFileExt } from '@main/utils/file'
-import { FileMetadata, OcrProvider } from '@types'
-import { app } from 'electron'
-import pdfjs from 'pdfjs-dist'
-import { TypedArray } from 'pdfjs-dist/types/src/display/api'
-
-export default abstract class BaseOcrProvider {
-  protected provider: OcrProvider
-  public storageDir = path.join(app.getPath('userData'), 'Data', 'Files')
-
-  constructor(provider: OcrProvider) {
-    if (!provider) {
-      throw new Error('OCR provider is not set')
-    }
-    this.provider = provider
-  }
-  abstract parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata; quota?: number }>
-
-  /**
-   * 检查文件是否已经被预处理过
-   * 统一检测方法：如果 Data/Files/{file.id} 是目录，说明已被预处理
-   * @param file 文件信息
-   * @returns 如果已处理返回处理后的文件信息，否则返回null
-   */
-  public async checkIfAlreadyProcessed(file: FileMetadata): Promise<FileMetadata | null> {
-    try {
-      // 检查 Data/Files/{file.id} 是否是目录
-      const preprocessDirPath = path.join(this.storageDir, file.id)
-
-      if (fs.existsSync(preprocessDirPath)) {
-        const stats = await fs.promises.stat(preprocessDirPath)
-
-        // 如果是目录，说明已经被预处理过
-        if (stats.isDirectory()) {
-          // 查找目录中的处理结果文件
-          const files = await fs.promises.readdir(preprocessDirPath)
-
-          // 查找主要的处理结果文件（.md 或 .txt）
-          const processedFile = files.find((fileName) => fileName.endsWith('.md') || fileName.endsWith('.txt'))
-
-          if (processedFile) {
-            const processedFilePath = path.join(preprocessDirPath, processedFile)
-            const processedStats = await fs.promises.stat(processedFilePath)
-            const ext = getFileExt(processedFile)
-
-            return {
-              ...file,
-              name: file.name.replace(file.ext, ext),
-              path: processedFilePath,
-              ext: ext,
-              size: processedStats.size,
-              created_at: processedStats.birthtime.toISOString()
-            }
-          }
-        }
-      }
-
-      return null
-    } catch (error) {
-      // 如果检查过程中出现错误，返回null表示未处理
-      return null
-    }
-  }
-
-  /**
-   * 辅助方法：延迟执行
-   */
-  public delay = (ms: number): Promise<void> => {
-    return new Promise((resolve) => setTimeout(resolve, ms))
-  }
-
-  public async readPdf(
-    source: string | URL | TypedArray,
-    passwordCallback?: (fn: (password: string) => void, reason: string) => string
-  ) {
-    const documentLoadingTask = pdfjs.getDocument(source)
-    if (passwordCallback) {
-      documentLoadingTask.onPassword = passwordCallback
-    }
-
-    const document = await documentLoadingTask.promise
-    return document
-  }
-
-  public async sendOcrProgress(sourceId: string, progress: number): Promise<void> {
-    const mainWindow = windowService.getMainWindow()
-    mainWindow?.webContents.send('file-ocr-progress', {
-      itemId: sourceId,
-      progress: progress
-    })
-  }
-
-  /**
-   * 将文件移动到附件目录
-   * @param fileId 文件id
-   * @param filePaths 需要移动的文件路径数组
-   * @returns 移动后的文件路径数组
-   */
-  public moveToAttachmentsDir(fileId: string, filePaths: string[]): string[] {
-    const attachmentsPath = path.join(this.storageDir, fileId)
-    if (!fs.existsSync(attachmentsPath)) {
-      fs.mkdirSync(attachmentsPath, { recursive: true })
-    }
-
-    const movedPaths: string[] = []
-
-    for (const filePath of filePaths) {
-      if (fs.existsSync(filePath)) {
-        const fileName = path.basename(filePath)
-        const destPath = path.join(attachmentsPath, fileName)
-        fs.copyFileSync(filePath, destPath)
-        fs.unlinkSync(filePath) // 删除原文件，实现"移动"
-        movedPaths.push(destPath)
-      }
-    }
-    return movedPaths
-  }
-}

src/main/knowledge/ocr/DefaultOcrProvider.ts

@@ -1,12 +0,0 @@
-import { FileMetadata, OcrProvider } from '@types'
-
-import BaseOcrProvider from './BaseOcrProvider'
-
-export default class DefaultOcrProvider extends BaseOcrProvider {
-  constructor(provider: OcrProvider) {
-    super(provider)
-  }
-  public parseFile(): Promise<{ processedFile: FileMetadata }> {
-    throw new Error('Method not implemented.')
-  }
-}

src/main/knowledge/ocr/MacSysOcrProvider.ts

@@ -1,130 +0,0 @@
-import { loggerService } from '@logger'
-import { isMac } from '@main/constant'
-import { FileMetadata, OcrProvider } from '@types'
-import * as fs from 'fs'
-import * as path from 'path'
-import { TextItem } from 'pdfjs-dist/types/src/display/api'
-
-import BaseOcrProvider from './BaseOcrProvider'
-
-const logger = loggerService.withContext('MacSysOcrProvider')
-
-export default class MacSysOcrProvider extends BaseOcrProvider {
-  private readonly MIN_TEXT_LENGTH = 1000
-  private MacOCR: any
-
-  private async initMacOCR() {
-    if (!isMac) {
-      throw new Error('MacSysOcrProvider is only available on macOS')
-    }
-    if (!this.MacOCR) {
-      try {
-        // @ts-ignore This module is optional and only installed/available on macOS. Runtime checks prevent execution on other platforms.
-        const module = await import('@cherrystudio/mac-system-ocr')
-        this.MacOCR = module.default
-      } catch (error) {
-        logger.error('Failed to load mac-system-ocr:', error as Error)
-        throw error
-      }
-    }
-    return this.MacOCR
-  }
-
-  private getRecognitionLevel(level?: number) {
-    return level === 0 ? this.MacOCR.RECOGNITION_LEVEL_FAST : this.MacOCR.RECOGNITION_LEVEL_ACCURATE
-  }
-
-  constructor(provider: OcrProvider) {
-    super(provider)
-  }
-
-  private async processPages(
-    results: any,
-    totalPages: number,
-    sourceId: string,
-    writeStream: fs.WriteStream
-  ): Promise<void> {
-    await this.initMacOCR()
-    // TODO: 下个版本后面使用批处理，以及p-queue来优化
-    for (let i = 0; i < totalPages; i++) {
-      // Convert pages to buffers
-      const pageNum = i + 1
-      const pageBuffer = await results.getPage(pageNum)
-
-      // Process batch
-      const ocrResult = await this.MacOCR.recognizeFromBuffer(pageBuffer, {
-        ocrOptions: {
-          recognitionLevel: this.getRecognitionLevel(this.provider.options?.recognitionLevel),
-          minConfidence: this.provider.options?.minConfidence || 0.5
-        }
-      })
-
-      // Write results in order
-      writeStream.write(ocrResult.text + '\n')
-
-      // Update progress
-      await this.sendOcrProgress(sourceId, (pageNum / totalPages) * 100)
-    }
-  }
-
-  public async isScanPdf(buffer: Buffer): Promise<boolean> {
-    const doc = await this.readPdf(new Uint8Array(buffer))
-    const pageLength = doc.numPages
-    let counts = 0
-    const pagesToCheck = Math.min(pageLength, 10)
-    for (let i = 0; i < pagesToCheck; i++) {
-      const page = await doc.getPage(i + 1)
-      const pageData = await page.getTextContent()
-      const pageText = pageData.items.map((item) => (item as TextItem).str).join('')
-      counts += pageText.length
-      if (counts >= this.MIN_TEXT_LENGTH) {
-        return false
-      }
-    }
-    return true
-  }
-
-  public async parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata }> {
-    logger.info(`Starting OCR process for file: ${file.name}`)
-    if (file.ext === '.pdf') {
-      try {
-        const { pdf } = await import('@cherrystudio/pdf-to-img-napi')
-        const pdfBuffer = await fs.promises.readFile(file.path)
-        const results = await pdf(pdfBuffer, {
-          scale: 2
-        })
-        const totalPages = results.length
-
-        const baseDir = path.dirname(file.path)
-        const baseName = path.basename(file.path, path.extname(file.path))
-        const txtFileName = `${baseName}.txt`
-        const txtFilePath = path.join(baseDir, txtFileName)
-
-        const writeStream = fs.createWriteStream(txtFilePath)
-        await this.processPages(results, totalPages, sourceId, writeStream)
-
-        await new Promise<void>((resolve, reject) => {
-          writeStream.end(() => {
-            logger.info(`OCR process completed successfully for ${file.origin_name}`)
-            resolve()
-          })
-          writeStream.on('error', reject)
-        })
-        const movedPaths = this.moveToAttachmentsDir(file.id, [txtFilePath])
-        return {
-          processedFile: {
-            ...file,
-            name: txtFileName,
-            path: movedPaths[0],
-            ext: '.txt',
-            size: fs.statSync(movedPaths[0]).size
-          }
-        }
-      } catch (error) {
-        logger.error('Error during OCR process:', error as Error)
-        throw error
-      }
-    }
-    return { processedFile: file }
-  }
-}

src/main/knowledge/ocr/OcrProvider.ts

@@ -1,26 +0,0 @@
-import { FileMetadata, OcrProvider as Provider } from '@types'
-
-import BaseOcrProvider from './BaseOcrProvider'
-import OcrProviderFactory from './OcrProviderFactory'
-
-export default class OcrProvider {
-  private sdk: BaseOcrProvider
-  constructor(provider: Provider) {
-    this.sdk = OcrProviderFactory.create(provider)
-  }
-  public async parseFile(
-    sourceId: string,
-    file: FileMetadata
-  ): Promise<{ processedFile: FileMetadata; quota?: number }> {
-    return this.sdk.parseFile(sourceId, file)
-  }
-
-  /**
-   * 检查文件是否已经被预处理过
-   * @param file 文件信息
-   * @returns 如果已处理返回处理后的文件信息，否则返回null
-   */
-  public async checkIfAlreadyProcessed(file: FileMetadata): Promise<FileMetadata | null> {
-    return this.sdk.checkIfAlreadyProcessed(file)
-  }
-}

src/main/knowledge/ocr/OcrProviderFactory.ts

@@ -1,23 +0,0 @@
-import { loggerService } from '@logger'
-import { isMac } from '@main/constant'
-import { OcrProvider } from '@types'
-
-import BaseOcrProvider from './BaseOcrProvider'
-import DefaultOcrProvider from './DefaultOcrProvider'
-import MacSysOcrProvider from './MacSysOcrProvider'
-
-const logger = loggerService.withContext('OcrProviderFactory')
-
-export default class OcrProviderFactory {
-  static create(provider: OcrProvider): BaseOcrProvider {
-    switch (provider.id) {
-      case 'system':
-        if (!isMac) {
-          logger.warn('System OCR provider is only available on macOS')
-        }
-        return new MacSysOcrProvider(provider)
-      default:
-        return new DefaultOcrProvider(provider)
-    }
-  }
-}

src/main/knowledge/preprocess/BasePreprocessProvider.ts

@@ -1,17 +1,18 @@
 import fs from 'node:fs'
 import path from 'node:path'
 
+import { loggerService } from '@logger'
 import { windowService } from '@main/services/WindowService'
-import { getFileExt } from '@main/utils/file'
+import { getFileExt, getTempDir } from '@main/utils/file'
 import { FileMetadata, PreprocessProvider } from '@types'
-import { app } from 'electron'
-import pdfjs from 'pdfjs-dist'
-import { TypedArray } from 'pdfjs-dist/types/src/display/api'
+import { PDFDocument } from 'pdf-lib'
+
+const logger = loggerService.withContext('BasePreprocessProvider')
 
 export default abstract class BasePreprocessProvider {
   protected provider: PreprocessProvider
   protected userId?: string
-  public storageDir = path.join(app.getPath('userData'), 'Data', 'Files')
+  public storageDir = path.join(getTempDir(), 'preprocess')
 
   constructor(provider: PreprocessProvider, userId?: string) {
     if (!provider) {
@@ -19,7 +20,19 @@ export default abstract class BasePreprocessProvider {
     }
     this.provider = provider
     this.userId = userId
+    this.ensureDirectories()
   }
+
+  private ensureDirectories() {
+    try {
+      if (!fs.existsSync(this.storageDir)) {
+        fs.mkdirSync(this.storageDir, { recursive: true })
+      }
+    } catch (error) {
+      logger.error('Failed to create directories:', error as Error)
+    }
+  }
+
   abstract parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata; quota?: number }>
 
   abstract checkQuota(): Promise<number>
@@ -77,17 +90,11 @@ export default abstract class BasePreprocessProvider {
     return new Promise((resolve) => setTimeout(resolve, ms))
   }
 
-  public async readPdf(
-    source: string | URL | TypedArray,
-    passwordCallback?: (fn: (password: string) => void, reason: string) => string
-  ) {
-    const documentLoadingTask = pdfjs.getDocument(source)
-    if (passwordCallback) {
-      documentLoadingTask.onPassword = passwordCallback
+  public async readPdf(buffer: Buffer) {
+    const pdfDoc = await PDFDocument.load(buffer)
+    return {
+      numPages: pdfDoc.getPageCount()
     }
-
-    const document = await documentLoadingTask.promise
-    return document
   }
 
   public async sendPreprocessProgress(sourceId: string, progress: number): Promise<void> {

src/main/knowledge/preprocess/Doc2xPreprocessProvider.ts

@@ -39,7 +39,7 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
   private async validateFile(filePath: string): Promise<void> {
     const pdfBuffer = await fs.promises.readFile(filePath)
 
-    const doc = await this.readPdf(new Uint8Array(pdfBuffer))
+    const doc = await this.readPdf(pdfBuffer)
 
     // 文件页数小于1000页
     if (doc.numPages >= 1000) {

src/main/knowledge/preprocess/MineruPreprocessProvider.ts

@@ -115,7 +115,7 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider {
   private async validateFile(filePath: string): Promise<void> {
     const pdfBuffer = await fs.promises.readFile(filePath)
 
-    const doc = await this.readPdf(new Uint8Array(pdfBuffer))
+    const doc = await this.readPdf(pdfBuffer)
 
     // 文件页数小于600页
     if (doc.numPages >= 600) {
@@ -178,7 +178,7 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider {
     try {
       // 下载ZIP文件
       const response = await axios.get(zipUrl, { responseType: 'arraybuffer' })
-      fs.writeFileSync(zipPath, response.data)
+      fs.writeFileSync(zipPath, Buffer.from(response.data))
       logger.info(`Downloaded ZIP file: ${zipPath}`)
 
       // 确保提取目录存在