ouyang/cherry-studio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: replace axios and node fetch with electron's net module (#9212)

Browse Source 
* refactor: replace axios and node fetch with electron's net module for network requests in preprocess providers

- Updated Doc2xPreprocessProvider and MineruPreprocessProvider to use net.fetch instead of axios for making HTTP requests.
- Improved error handling for network responses across various methods.
- Removed unnecessary AxiosRequestConfig and related code to streamline the implementation.

* lint

* refactor(Doc2xPreprocessProvider): enhance file validation and upload process

- Added file size validation to prevent loading files larger than 300MB into memory.
- Implemented file size check before reading the PDF to ensure efficient memory usage.
- Updated the file upload method to use a stream, setting the 'Content-Length' header for better handling of large files.

* refactor(brave-search): update net.fetch calls to use url.toString()

- Modified all instances of net.fetch to use url.toString() for better URL handling.
- Ensured consistency in how URLs are passed to the fetch method across various functions.

* refactor(MCPService): improve URL handling in net.fetch calls

- Updated net.fetch to use url.toString() for better type handling of URLs.
- Ensured consistent URL processing across the MCPService class.

* feat(ProxyManager): integrate axios with fetch proxy support

- Added axios as a dependency to enable fetch proxy usage.
- Implemented logic to set axios's adapter to 'fetch' for proxy handling.
- Preserved original axios adapter for restoration when disabling the proxy.
This commit is contained in:
beyondkmp
2025-08-15 22:48:22 +08:00
committed by GitHub
parent 748ac600fa
commit 4a62bb6ad7
12 changed files with 182 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/knowledge/preprocess/Doc2xPreprocessProvider.ts
+92 -44
View File
@@ -5,7 +5,7 @@ import { loggerService } from '@logger'
import { fileStorage } from '@main/services/FileStorage'
import { FileMetadata, PreprocessProvider } from '@types'
import AdmZip from 'adm-zip'
import axios, { AxiosRequestConfig } from 'axios'
import { net } from 'electron'
import BasePreprocessProvider from './BasePreprocessProvider'
@@ -38,19 +38,24 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
}
private async validateFile(filePath: string): Promise<void> {
const pdfBuffer = await fs.promises.readFile(filePath)
// 首先检查文件大小，避免读取大文件到内存
const stats = await fs.promises.stat(filePath)
const fileSizeBytes = stats.size
// 文件大小小于300MB
if (fileSizeBytes >= 300 * 1024 * 1024) {
const fileSizeMB = Math.round(fileSizeBytes / (1024 * 1024))
throw new Error(`PDF file size (${fileSizeMB}MB) exceeds the limit of 300MB`)
}
// 只有在文件大小合理的情况下才读取文件内容检查页数
const pdfBuffer = await fs.promises.readFile(filePath)
const doc = await this.readPdf(pdfBuffer)
// 文件页数小于1000页
if (doc.numPages >= 1000) {
throw new Error(`PDF page count (${doc.numPages}) exceeds the limit of 1000 pages`)
}
// 文件大小小于300MB
if (pdfBuffer.length >= 300 * 1024 * 1024) {
const fileSizeMB = Math.round(pdfBuffer.length / (1024 * 1024))
throw new Error(`PDF file size (${fileSizeMB}MB) exceeds the limit of 300MB`)
}
}
public async parseFile(sourceId: string, file: FileMetadata): Promise<{ processedFile: FileMetadata }> {
@@ -160,11 +165,23 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
* @returns 预上传响应的url和uid
*/
private async preupload(): Promise<PreuploadResponse> {
const config = this.createAuthConfig()
const endpoint = `${this.provider.apiHost}/api/v2/parse/preupload`
try {
const { data } = await axios.post<ApiResponse<PreuploadResponse>>(endpoint, null, config)
const response = await net.fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.provider.apiKey}`
},
body: null
})
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const data = (await response.json()) as ApiResponse<PreuploadResponse>
if (data.code === 'success' && data.data) {
return data.data
@@ -178,17 +195,29 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
}
/**
* 上传文件
* 上传文件（使用流式上传）
* @param filePath 文件路径
* @param url 预上传响应的url
*/
private async putFile(filePath: string, url: string): Promise<void> {
try {
const fileStream = fs.createReadStream(filePath)
const response = await axios.put(url, fileStream)
// 获取文件大小用于设置 Content-Length
const stats = await fs.promises.stat(filePath)
const fileSize = stats.size
if (response.status !== 200) {
throw new Error(`HTTP status ${response.status}: ${response.statusText}`)
// 创建可读流
const fileStream = fs.createReadStream(filePath)
const response = await net.fetch(url, {
method: 'PUT',
body: fileStream as any, // TypeScript 类型转换，net.fetch 支持 ReadableStream
headers: {
'Content-Length': fileSize.toString()
}
})
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
} catch (error) {
logger.error(`Failed to upload file ${filePath}: ${error instanceof Error ? error.message : String(error)}`)
@@ -197,16 +226,25 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
}
private async getStatus(uid: string): Promise<StatusResponse> {
const config = this.createAuthConfig()
const endpoint = `${this.provider.apiHost}/api/v2/parse/status?uid=${uid}`
try {
const response = await axios.get<ApiResponse<StatusResponse>>(endpoint, config)
const response = await net.fetch(endpoint, {
method: 'GET',
headers: {
Authorization: `Bearer ${this.provider.apiKey}`
}
})
if (response.data.code === 'success' && response.data.data) {
return response.data.data
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const data = (await response.json()) as ApiResponse<StatusResponse>
if (data.code === 'success' && data.data) {
return data.data
} else {
throw new Error(`API returned error: ${response.data.message || JSON.stringify(response.data)}`)
throw new Error(`API returned error: ${data.message || JSON.stringify(data)}`)
}
} catch (error) {
logger.error(`Failed to get status for uid ${uid}: ${error instanceof Error ? error.message : String(error)}`)
@@ -221,13 +259,6 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
*/
private async convertFile(uid: string, filePath: string): Promise<void> {
const fileName = path.parse(filePath).name
const config = {
...this.createAuthConfig(),
headers: {
...this.createAuthConfig().headers,
'Content-Type': 'application/json'
}
}
const payload = {
uid,
@@ -239,10 +270,22 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
const endpoint = `${this.provider.apiHost}/api/v2/convert/parse`
try {
const response = await axios.post<ApiResponse<any>>(endpoint, payload, config)
const response = await net.fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${this.provider.apiKey}`
},
body: JSON.stringify(payload)
})
if (response.data.code !== 'success') {
throw new Error(`API returned error: ${response.data.message || JSON.stringify(response.data)}`)
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const data = (await response.json()) as ApiResponse<any>
if (data.code !== 'success') {
throw new Error(`API returned error: ${data.message || JSON.stringify(data)}`)
}
} catch (error) {
logger.error(`Failed to convert file ${filePath}: ${error instanceof Error ? error.message : String(error)}`)
@@ -256,16 +299,25 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
* @returns 解析后的文件信息
*/
private async getParsedFile(uid: string): Promise<ParsedFileResponse> {
const config = this.createAuthConfig()
const endpoint = `${this.provider.apiHost}/api/v2/convert/parse/result?uid=${uid}`
try {
const response = await axios.get<ApiResponse<ParsedFileResponse>>(endpoint, config)
const response = await net.fetch(endpoint, {
method: 'GET',
headers: {
Authorization: `Bearer ${this.provider.apiKey}`
}
})
if (response.status === 200 && response.data.data) {
return response.data.data
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const data = (await response.json()) as ApiResponse<ParsedFileResponse>
if (data.data) {
return data.data
} else {
throw new Error(`HTTP status ${response.status}: ${response.statusText}`)
throw new Error(`No data in response`)
}
} catch (error) {
logger.error(
@@ -295,8 +347,12 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
try {
// 下载文件
const response = await axios.get(url, { responseType: 'arraybuffer' })
fs.writeFileSync(zipPath, response.data)
const response = await net.fetch(url, { method: 'GET' })
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const arrayBuffer = await response.arrayBuffer()
fs.writeFileSync(zipPath, Buffer.from(arrayBuffer))
// 确保提取目录存在
if (!fs.existsSync(extractPath)) {
@@ -318,14 +374,6 @@ export default class Doc2xPreprocessProvider extends BasePreprocessProvider {
}
}
private createAuthConfig(): AxiosRequestConfig {
return {
headers: {
Authorization: `Bearer ${this.provider.apiKey}`
}
}
}
public checkQuota(): Promise<number> {
throw new Error('Method not implemented.')
}
src/main/knowledge/preprocess/MineruPreprocessProvider.ts
+11 -7
View File
@@ -5,7 +5,7 @@ import { loggerService } from '@logger'
import { fileStorage } from '@main/services/FileStorage'
import { FileMetadata, PreprocessProvider } from '@types'
import AdmZip from 'adm-zip'
import axios from 'axios'
import { net } from 'electron'
import BasePreprocessProvider from './BasePreprocessProvider'
@@ -95,7 +95,7 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider {
public async checkQuota() {
try {
const quota = await fetch(`${this.provider.apiHost}/api/v4/quota`, {
const quota = await net.fetch(`${this.provider.apiHost}/api/v4/quota`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
@@ -179,8 +179,12 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider {
try {
// 下载ZIP文件
const response = await axios.get(zipUrl, { responseType: 'arraybuffer' })
fs.writeFileSync(zipPath, Buffer.from(response.data))
const response = await net.fetch(zipUrl, { method: 'GET' })
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
}
const arrayBuffer = await response.arrayBuffer()
fs.writeFileSync(zipPath, Buffer.from(arrayBuffer))
logger.info(`Downloaded ZIP file: ${zipPath}`)
// 确保提取目录存在
@@ -236,7 +240,7 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider {
}
try {
const response = await fetch(endpoint, {
const response = await net.fetch(endpoint, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
@@ -271,7 +275,7 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider {
try {
const fileBuffer = await fs.promises.readFile(filePath)
const response = await fetch(uploadUrl, {
const response = await net.fetch(uploadUrl, {
method: 'PUT',
body: fileBuffer,
headers: {
@@ -316,7 +320,7 @@ export default class MineruPreprocessProvider extends BasePreprocessProvider {
const endpoint = `${this.provider.apiHost}/api/v4/extract-results/batch/${batchId}`
try {
const response = await fetch(endpoint, {
const response = await net.fetch(endpoint, {
method: 'GET',
headers: {
'Content-Type': 'application/json',