feat: remove faiss database (#10178)
This commit is contained in:
Chen Tao
@@ -1,63 +0,0 @@
|
||||
import { VoyageEmbeddings } from '@langchain/community/embeddings/voyage'
|
||||
import type { Embeddings } from '@langchain/core/embeddings'
|
||||
import { OllamaEmbeddings } from '@langchain/ollama'
|
||||
import { AzureOpenAIEmbeddings, OpenAIEmbeddings } from '@langchain/openai'
|
||||
import { ApiClient, SystemProviderIds } from '@types'
|
||||
|
||||
import { isJinaEmbeddingsModel, JinaEmbeddings } from './JinaEmbeddings'
|
||||
|
||||
export default class EmbeddingsFactory {
|
||||
static create({ embedApiClient, dimensions }: { embedApiClient: ApiClient; dimensions?: number }): Embeddings {
|
||||
const batchSize = 10
|
||||
const { model, provider, apiKey, apiVersion, baseURL } = embedApiClient
|
||||
if (provider === SystemProviderIds.ollama) {
|
||||
let baseUrl = baseURL
|
||||
if (baseURL.includes('v1/')) {
|
||||
baseUrl = baseURL.replace('v1/', '')
|
||||
}
|
||||
const headers = apiKey
|
||||
? {
|
||||
Authorization: `Bearer ${apiKey}`
|
||||
}
|
||||
: undefined
|
||||
return new OllamaEmbeddings({
|
||||
model: model,
|
||||
baseUrl,
|
||||
...headers
|
||||
})
|
||||
} else if (provider === SystemProviderIds.voyageai) {
|
||||
return new VoyageEmbeddings({
|
||||
modelName: model,
|
||||
apiKey,
|
||||
outputDimension: dimensions,
|
||||
batchSize
|
||||
})
|
||||
}
|
||||
if (isJinaEmbeddingsModel(model)) {
|
||||
return new JinaEmbeddings({
|
||||
model,
|
||||
apiKey,
|
||||
batchSize,
|
||||
dimensions,
|
||||
baseUrl: baseURL
|
||||
})
|
||||
}
|
||||
if (apiVersion !== undefined) {
|
||||
return new AzureOpenAIEmbeddings({
|
||||
azureOpenAIApiKey: apiKey,
|
||||
azureOpenAIApiVersion: apiVersion,
|
||||
azureOpenAIApiDeploymentName: model,
|
||||
azureOpenAIEndpoint: baseURL,
|
||||
dimensions,
|
||||
batchSize
|
||||
})
|
||||
}
|
||||
return new OpenAIEmbeddings({
|
||||
model,
|
||||
apiKey,
|
||||
dimensions,
|
||||
batchSize,
|
||||
configuration: { baseURL }
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
import { Embeddings, type EmbeddingsParams } from '@langchain/core/embeddings'
|
||||
import { chunkArray } from '@langchain/core/utils/chunk_array'
|
||||
import { getEnvironmentVariable } from '@langchain/core/utils/env'
|
||||
import { z } from 'zod'
|
||||
|
||||
const jinaModelSchema = z.union([
|
||||
z.literal('jina-clip-v2'),
|
||||
z.literal('jina-embeddings-v3'),
|
||||
z.literal('jina-colbert-v2'),
|
||||
z.literal('jina-clip-v1'),
|
||||
z.literal('jina-colbert-v1-en'),
|
||||
z.literal('jina-embeddings-v2-base-es'),
|
||||
z.literal('jina-embeddings-v2-base-code'),
|
||||
z.literal('jina-embeddings-v2-base-de'),
|
||||
z.literal('jina-embeddings-v2-base-zh'),
|
||||
z.literal('jina-embeddings-v2-base-en')
|
||||
])
|
||||
|
||||
type JinaModel = z.infer<typeof jinaModelSchema>
|
||||
|
||||
export const isJinaEmbeddingsModel = (model: string): model is JinaModel => {
|
||||
return jinaModelSchema.safeParse(model).success
|
||||
}
|
||||
|
||||
interface JinaEmbeddingsParams extends EmbeddingsParams {
|
||||
/** Model name to use */
|
||||
model: JinaModel
|
||||
|
||||
baseUrl?: string
|
||||
|
||||
/**
|
||||
* Timeout to use when making requests to Jina.
|
||||
*/
|
||||
timeout?: number
|
||||
|
||||
/**
|
||||
* The maximum number of documents to embed in a single request.
|
||||
*/
|
||||
batchSize?: number
|
||||
|
||||
/**
|
||||
* Whether to strip new lines from the input text.
|
||||
*/
|
||||
stripNewLines?: boolean
|
||||
|
||||
/**
|
||||
* The dimensions of the embedding.
|
||||
*/
|
||||
dimensions?: number
|
||||
|
||||
/**
|
||||
* Scales the embedding so its Euclidean (L2) norm becomes 1, preserving direction. Useful when downstream involves dot-product, classification, visualization..
|
||||
*/
|
||||
normalized?: boolean
|
||||
}
|
||||
|
||||
type JinaMultiModelInput =
|
||||
| {
|
||||
text: string
|
||||
image?: never
|
||||
}
|
||||
| {
|
||||
image: string
|
||||
text?: never
|
||||
}
|
||||
|
||||
type JinaEmbeddingsInput = string | JinaMultiModelInput
|
||||
|
||||
interface EmbeddingCreateParams {
|
||||
model: JinaEmbeddingsParams['model']
|
||||
|
||||
/**
|
||||
* input can be strings or JinaMultiModelInputs,if you want embed image,you should use JinaMultiModelInputs
|
||||
*/
|
||||
input: JinaEmbeddingsInput[]
|
||||
dimensions: number
|
||||
task?: 'retrieval.query' | 'retrieval.passage'
|
||||
}
|
||||
|
||||
interface EmbeddingResponse {
|
||||
model: string
|
||||
object: string
|
||||
usage: {
|
||||
total_tokens: number
|
||||
prompt_tokens: number
|
||||
}
|
||||
data: {
|
||||
object: string
|
||||
index: number
|
||||
embedding: number[]
|
||||
}[]
|
||||
}
|
||||
|
||||
interface EmbeddingErrorResponse {
|
||||
detail: string
|
||||
}
|
||||
|
||||
export class JinaEmbeddings extends Embeddings implements JinaEmbeddingsParams {
|
||||
model: JinaEmbeddingsParams['model'] = 'jina-clip-v2'
|
||||
|
||||
batchSize = 24
|
||||
|
||||
baseUrl = 'https://api.jina.ai/v1/embeddings'
|
||||
|
||||
stripNewLines = true
|
||||
|
||||
dimensions = 1024
|
||||
|
||||
apiKey: string
|
||||
|
||||
constructor(
|
||||
fields?: Partial<JinaEmbeddingsParams> & {
|
||||
apiKey?: string
|
||||
}
|
||||
) {
|
||||
const fieldsWithDefaults = { maxConcurrency: 2, ...fields }
|
||||
super(fieldsWithDefaults)
|
||||
|
||||
const apiKey =
|
||||
fieldsWithDefaults?.apiKey || getEnvironmentVariable('JINA_API_KEY') || getEnvironmentVariable('JINA_AUTH_TOKEN')
|
||||
|
||||
if (!apiKey) throw new Error('Jina API key not found')
|
||||
|
||||
this.apiKey = apiKey
|
||||
this.baseUrl = fieldsWithDefaults?.baseUrl ? `${fieldsWithDefaults?.baseUrl}embeddings` : this.baseUrl
|
||||
this.model = fieldsWithDefaults?.model ?? this.model
|
||||
this.dimensions = fieldsWithDefaults?.dimensions ?? this.dimensions
|
||||
this.batchSize = fieldsWithDefaults?.batchSize ?? this.batchSize
|
||||
this.stripNewLines = fieldsWithDefaults?.stripNewLines ?? this.stripNewLines
|
||||
}
|
||||
|
||||
private doStripNewLines(input: JinaEmbeddingsInput[]) {
|
||||
if (this.stripNewLines) {
|
||||
return input.map((i) => {
|
||||
if (typeof i === 'string') {
|
||||
return i.replace(/\n/g, ' ')
|
||||
}
|
||||
if (i.text) {
|
||||
return { text: i.text.replace(/\n/g, ' ') }
|
||||
}
|
||||
return i
|
||||
})
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
async embedDocuments(input: JinaEmbeddingsInput[]): Promise<number[][]> {
|
||||
const batches = chunkArray(this.doStripNewLines(input), this.batchSize)
|
||||
const batchRequests = batches.map((batch) => {
|
||||
const params = this.getParams(batch)
|
||||
return this.embeddingWithRetry(params)
|
||||
})
|
||||
|
||||
const batchResponses = await Promise.all(batchRequests)
|
||||
const embeddings: number[][] = []
|
||||
|
||||
for (let i = 0; i < batchResponses.length; i += 1) {
|
||||
const batch = batches[i]
|
||||
const batchResponse = batchResponses[i] || []
|
||||
for (let j = 0; j < batch.length; j += 1) {
|
||||
embeddings.push(batchResponse[j])
|
||||
}
|
||||
}
|
||||
|
||||
return embeddings
|
||||
}
|
||||
|
||||
async embedQuery(input: JinaEmbeddingsInput): Promise<number[]> {
|
||||
const params = this.getParams(this.doStripNewLines([input]), true)
|
||||
|
||||
const embeddings = (await this.embeddingWithRetry(params)) || [[]]
|
||||
return embeddings[0]
|
||||
}
|
||||
|
||||
private getParams(input: JinaEmbeddingsInput[], query?: boolean): EmbeddingCreateParams {
|
||||
return {
|
||||
model: this.model,
|
||||
input,
|
||||
dimensions: this.dimensions,
|
||||
task: query ? 'retrieval.query' : this.model === 'jina-clip-v2' ? undefined : 'retrieval.passage'
|
||||
}
|
||||
}
|
||||
|
||||
private async embeddingWithRetry(body: EmbeddingCreateParams) {
|
||||
const response = await fetch(this.baseUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${this.apiKey}`
|
||||
},
|
||||
body: JSON.stringify(body)
|
||||
})
|
||||
const embeddingData: EmbeddingResponse | EmbeddingErrorResponse = await response.json()
|
||||
if ('detail' in embeddingData && embeddingData.detail) {
|
||||
throw new Error(`${embeddingData.detail}`)
|
||||
}
|
||||
return (embeddingData as EmbeddingResponse).data.map(({ embedding }) => embedding)
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import type { Embeddings as BaseEmbeddings } from '@langchain/core/embeddings'
|
||||
import { TraceMethod } from '@mcp-trace/trace-core'
|
||||
import { ApiClient } from '@types'
|
||||
|
||||
import EmbeddingsFactory from './EmbeddingsFactory'
|
||||
|
||||
export default class TextEmbeddings {
|
||||
private sdk: BaseEmbeddings
|
||||
constructor({ embedApiClient, dimensions }: { embedApiClient: ApiClient; dimensions?: number }) {
|
||||
this.sdk = EmbeddingsFactory.create({
|
||||
embedApiClient,
|
||||
dimensions
|
||||
})
|
||||
}
|
||||
|
||||
@TraceMethod({ spanName: 'embedDocuments', tag: 'Embeddings' })
|
||||
public async embedDocuments(texts: string[]): Promise<number[][]> {
|
||||
return this.sdk.embedDocuments(texts)
|
||||
}
|
||||
|
||||
@TraceMethod({ spanName: 'embedQuery', tag: 'Embeddings' })
|
||||
public async embedQuery(text: string): Promise<number[]> {
|
||||
return this.sdk.embedQuery(text)
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
import { BaseDocumentLoader } from '@langchain/core/document_loaders/base'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { readTextFileWithAutoEncoding } from '@main/utils/file'
|
||||
import MarkdownIt from 'markdown-it'
|
||||
|
||||
export class MarkdownLoader extends BaseDocumentLoader {
|
||||
private path: string
|
||||
private md: MarkdownIt
|
||||
|
||||
constructor(path: string) {
|
||||
super()
|
||||
this.path = path
|
||||
this.md = new MarkdownIt()
|
||||
}
|
||||
public async load(): Promise<Document[]> {
|
||||
const content = await readTextFileWithAutoEncoding(this.path)
|
||||
return this.parseMarkdown(content)
|
||||
}
|
||||
|
||||
private parseMarkdown(content: string): Document[] {
|
||||
const tokens = this.md.parse(content, {})
|
||||
const documents: Document[] = []
|
||||
|
||||
let currentSection: {
|
||||
heading?: string
|
||||
level?: number
|
||||
content: string
|
||||
startLine?: number
|
||||
} = { content: '' }
|
||||
|
||||
let i = 0
|
||||
while (i < tokens.length) {
|
||||
const token = tokens[i]
|
||||
|
||||
if (token.type === 'heading_open') {
|
||||
// Save previous section if it has content
|
||||
if (currentSection.content.trim()) {
|
||||
documents.push(
|
||||
new Document({
|
||||
pageContent: currentSection.content.trim(),
|
||||
metadata: {
|
||||
source: this.path,
|
||||
heading: currentSection.heading || 'Introduction',
|
||||
level: currentSection.level || 0,
|
||||
startLine: currentSection.startLine || 0
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
// Start new section
|
||||
const level = parseInt(token.tag.slice(1)) // Extract number from h1, h2, etc.
|
||||
const headingContent = tokens[i + 1]?.content || ''
|
||||
|
||||
currentSection = {
|
||||
heading: headingContent,
|
||||
level: level,
|
||||
content: '',
|
||||
startLine: token.map?.[0] || 0
|
||||
}
|
||||
|
||||
// Skip heading_open, inline, heading_close tokens
|
||||
i += 3
|
||||
continue
|
||||
}
|
||||
|
||||
// Add token content to current section
|
||||
if (token.content) {
|
||||
currentSection.content += token.content
|
||||
}
|
||||
|
||||
// Add newlines for block tokens
|
||||
if (token.block && token.type !== 'heading_close') {
|
||||
currentSection.content += '\n'
|
||||
}
|
||||
|
||||
i++
|
||||
}
|
||||
|
||||
// Add the last section
|
||||
if (currentSection.content.trim()) {
|
||||
documents.push(
|
||||
new Document({
|
||||
pageContent: currentSection.content.trim(),
|
||||
metadata: {
|
||||
source: this.path,
|
||||
heading: currentSection.heading || 'Introduction',
|
||||
level: currentSection.level || 0,
|
||||
startLine: currentSection.startLine || 0
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
return documents
|
||||
}
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
import { BaseDocumentLoader } from '@langchain/core/document_loaders/base'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
|
||||
export class NoteLoader extends BaseDocumentLoader {
|
||||
private text: string
|
||||
private sourceUrl?: string
|
||||
constructor(
|
||||
public _text: string,
|
||||
public _sourceUrl?: string
|
||||
) {
|
||||
super()
|
||||
this.text = _text
|
||||
this.sourceUrl = _sourceUrl
|
||||
}
|
||||
|
||||
/**
|
||||
* A protected method that takes a `raw` string as a parameter and returns
|
||||
* a promise that resolves to an array containing the raw text as a single
|
||||
* element.
|
||||
* @param raw The raw text to be parsed.
|
||||
* @returns A promise that resolves to an array containing the raw text as a single element.
|
||||
*/
|
||||
protected async parse(raw: string): Promise<string[]> {
|
||||
return [raw]
|
||||
}
|
||||
|
||||
public async load(): Promise<Document[]> {
|
||||
const metadata = { source: this.sourceUrl || 'note' }
|
||||
const parsed = await this.parse(this.text)
|
||||
parsed.forEach((pageContent, i) => {
|
||||
if (typeof pageContent !== 'string') {
|
||||
throw new Error(`Expected string, at position ${i} got ${typeof pageContent}`)
|
||||
}
|
||||
})
|
||||
|
||||
return parsed.map(
|
||||
(pageContent, i) =>
|
||||
new Document({
|
||||
pageContent,
|
||||
metadata:
|
||||
parsed.length === 1
|
||||
? metadata
|
||||
: {
|
||||
...metadata,
|
||||
line: i + 1
|
||||
}
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
import { BaseDocumentLoader } from '@langchain/core/document_loaders/base'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { Innertube } from 'youtubei.js'
|
||||
|
||||
// ... (接口定义 YoutubeConfig 和 VideoMetadata 保持不变)
|
||||
|
||||
/**
|
||||
* Configuration options for the YoutubeLoader class. Includes properties
|
||||
* such as the videoId, language, and addVideoInfo.
|
||||
*/
|
||||
interface YoutubeConfig {
|
||||
videoId: string
|
||||
language?: string
|
||||
addVideoInfo?: boolean
|
||||
// 新增一个选项，用于控制输出格式
|
||||
transcriptFormat?: 'text' | 'srt'
|
||||
}
|
||||
|
||||
/**
|
||||
* Metadata of a YouTube video. Includes properties such as the source
|
||||
* (videoId), description, title, view_count, author, and category.
|
||||
*/
|
||||
interface VideoMetadata {
|
||||
source: string
|
||||
description?: string
|
||||
title?: string
|
||||
view_count?: number
|
||||
author?: string
|
||||
category?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* A document loader for loading data from YouTube videos. It uses the
|
||||
* youtubei.js library to fetch the transcript and video metadata.
|
||||
* @example
|
||||
* ```typescript
|
||||
* const loader = new YoutubeLoader({
|
||||
* videoId: "VIDEO_ID",
|
||||
* language: "en",
|
||||
* addVideoInfo: true,
|
||||
* transcriptFormat: "srt" // 获取 SRT 格式
|
||||
* });
|
||||
* const docs = await loader.load();
|
||||
* console.log(docs[0].pageContent);
|
||||
* ```
|
||||
*/
|
||||
export class YoutubeLoader extends BaseDocumentLoader {
|
||||
private videoId: string
|
||||
private language?: string
|
||||
private addVideoInfo: boolean
|
||||
// 新增格式化选项的私有属性
|
||||
private transcriptFormat: 'text' | 'srt'
|
||||
|
||||
constructor(config: YoutubeConfig) {
|
||||
super()
|
||||
this.videoId = config.videoId
|
||||
this.language = config?.language
|
||||
this.addVideoInfo = config?.addVideoInfo ?? false
|
||||
// 初始化格式化选项，默认为 'text' 以保持向后兼容
|
||||
this.transcriptFormat = config?.transcriptFormat ?? 'text'
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the videoId from a YouTube video URL.
|
||||
* @param url The URL of the YouTube video.
|
||||
* @returns The videoId of the YouTube video.
|
||||
*/
|
||||
private static getVideoID(url: string): string {
|
||||
const match = url.match(/.*(?:youtu.be\/|v\/|u\/\w\/|embed\/|watch\?v=)([^#&?]*).*/)
|
||||
if (match !== null && match[1].length === 11) {
|
||||
return match[1]
|
||||
} else {
|
||||
throw new Error('Failed to get youtube video id from the url')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new instance of the YoutubeLoader class from a YouTube video
|
||||
* URL.
|
||||
* @param url The URL of the YouTube video.
|
||||
* @param config Optional configuration options for the YoutubeLoader instance, excluding the videoId.
|
||||
* @returns A new instance of the YoutubeLoader class.
|
||||
*/
|
||||
static createFromUrl(url: string, config?: Omit<YoutubeConfig, 'videoId'>): YoutubeLoader {
|
||||
const videoId = YoutubeLoader.getVideoID(url)
|
||||
return new YoutubeLoader({ ...config, videoId })
|
||||
}
|
||||
|
||||
/**
|
||||
* [新增] 辅助函数：将毫秒转换为 SRT 时间戳格式 (HH:MM:SS,ms)
|
||||
* @param ms 毫秒数
|
||||
* @returns 格式化后的时间字符串
|
||||
*/
|
||||
private static formatTimestamp(ms: number): string {
|
||||
const totalSeconds = Math.floor(ms / 1000)
|
||||
const hours = Math.floor(totalSeconds / 3600)
|
||||
.toString()
|
||||
.padStart(2, '0')
|
||||
const minutes = Math.floor((totalSeconds % 3600) / 60)
|
||||
.toString()
|
||||
.padStart(2, '0')
|
||||
const seconds = (totalSeconds % 60).toString().padStart(2, '0')
|
||||
const milliseconds = (ms % 1000).toString().padStart(3, '0')
|
||||
return `${hours}:${minutes}:${seconds},${milliseconds}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the transcript and video metadata from the specified YouTube
|
||||
* video. It can return the transcript as plain text or in SRT format.
|
||||
* @returns An array of Documents representing the retrieved data.
|
||||
*/
|
||||
async load(): Promise<Document[]> {
|
||||
const metadata: VideoMetadata = {
|
||||
source: this.videoId
|
||||
}
|
||||
|
||||
try {
|
||||
const youtube = await Innertube.create({
|
||||
lang: this.language,
|
||||
retrieve_player: false
|
||||
})
|
||||
|
||||
const info = await youtube.getInfo(this.videoId)
|
||||
const transcriptData = await info.getTranscript()
|
||||
|
||||
if (!transcriptData.transcript.content?.body?.initial_segments) {
|
||||
throw new Error('Transcript segments not found in the response.')
|
||||
}
|
||||
|
||||
const segments = transcriptData.transcript.content.body.initial_segments
|
||||
|
||||
let pageContent: string
|
||||
|
||||
// 根据 transcriptFormat 选项决定如何格式化字幕
|
||||
if (this.transcriptFormat === 'srt') {
|
||||
// [修改] 将字幕片段格式化为 SRT 格式
|
||||
pageContent = segments
|
||||
.map((segment, index) => {
|
||||
const srtIndex = index + 1
|
||||
const startTime = YoutubeLoader.formatTimestamp(Number(segment.start_ms))
|
||||
const endTime = YoutubeLoader.formatTimestamp(Number(segment.end_ms))
|
||||
const text = segment.snippet?.text || '' // 使用 segment.snippet.text
|
||||
|
||||
return `${srtIndex}\n${startTime} --> ${endTime}\n${text}`
|
||||
})
|
||||
.join('\n\n') // 每个 SRT 块之间用两个换行符分隔
|
||||
} else {
|
||||
// [原始逻辑] 拼接为纯文本
|
||||
pageContent = segments.map((segment) => segment.snippet?.text || '').join(' ')
|
||||
}
|
||||
|
||||
if (this.addVideoInfo) {
|
||||
const basicInfo = info.basic_info
|
||||
metadata.description = basicInfo.short_description
|
||||
metadata.title = basicInfo.title
|
||||
metadata.view_count = basicInfo.view_count
|
||||
metadata.author = basicInfo.author
|
||||
}
|
||||
|
||||
const document = new Document({
|
||||
pageContent,
|
||||
metadata
|
||||
})
|
||||
|
||||
return [document]
|
||||
} catch (e: unknown) {
|
||||
throw new Error(`Failed to get YouTube video transcription: ${(e as Error).message}`)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,235 +0,0 @@
|
||||
import { DocxLoader } from '@langchain/community/document_loaders/fs/docx'
|
||||
import { EPubLoader } from '@langchain/community/document_loaders/fs/epub'
|
||||
import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf'
|
||||
import { PPTXLoader } from '@langchain/community/document_loaders/fs/pptx'
|
||||
import { CheerioWebBaseLoader } from '@langchain/community/document_loaders/web/cheerio'
|
||||
import { SitemapLoader } from '@langchain/community/document_loaders/web/sitemap'
|
||||
import { FaissStore } from '@langchain/community/vectorstores/faiss'
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { loggerService } from '@logger'
|
||||
import { UrlSource } from '@main/utils/knowledge'
|
||||
import { LoaderReturn } from '@shared/config/types'
|
||||
import { FileMetadata, FileTypes, KnowledgeBaseParams } from '@types'
|
||||
import { randomUUID } from 'crypto'
|
||||
import { JSONLoader } from 'langchain/document_loaders/fs/json'
|
||||
import { TextLoader } from 'langchain/document_loaders/fs/text'
|
||||
|
||||
import { SplitterFactory } from '../splitter'
|
||||
import { MarkdownLoader } from './MarkdownLoader'
|
||||
import { NoteLoader } from './NoteLoader'
|
||||
import { YoutubeLoader } from './YoutubeLoader'
|
||||
|
||||
const logger = loggerService.withContext('KnowledgeService File Loader')
|
||||
|
||||
type LoaderInstance =
|
||||
| TextLoader
|
||||
| PDFLoader
|
||||
| PPTXLoader
|
||||
| DocxLoader
|
||||
| JSONLoader
|
||||
| EPubLoader
|
||||
| CheerioWebBaseLoader
|
||||
| YoutubeLoader
|
||||
| SitemapLoader
|
||||
| NoteLoader
|
||||
| MarkdownLoader
|
||||
|
||||
/**
|
||||
* 为文档数组中的每个文档的 metadata 添加类型信息。
|
||||
*/
|
||||
function formatDocument(docs: Document[], type: string): Document[] {
|
||||
return docs.map((doc) => ({
|
||||
...doc,
|
||||
metadata: {
|
||||
...doc.metadata,
|
||||
type: type
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
/**
|
||||
* 通用文档处理管道
|
||||
*/
|
||||
async function processDocuments(
|
||||
base: KnowledgeBaseParams,
|
||||
vectorStore: FaissStore,
|
||||
docs: Document[],
|
||||
loaderType: string,
|
||||
splitterType?: string
|
||||
): Promise<LoaderReturn> {
|
||||
const formattedDocs = formatDocument(docs, loaderType)
|
||||
const splitter = SplitterFactory.create({
|
||||
chunkSize: base.chunkSize,
|
||||
chunkOverlap: base.chunkOverlap,
|
||||
...(splitterType && { type: splitterType })
|
||||
})
|
||||
|
||||
const splitterResults = await splitter.splitDocuments(formattedDocs)
|
||||
const ids = splitterResults.map(() => randomUUID())
|
||||
|
||||
await vectorStore.addDocuments(splitterResults, { ids })
|
||||
|
||||
return {
|
||||
entriesAdded: splitterResults.length,
|
||||
uniqueId: ids[0] || '',
|
||||
uniqueIds: ids,
|
||||
loaderType
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 通用加载器执行函数
|
||||
*/
|
||||
async function executeLoader(
|
||||
base: KnowledgeBaseParams,
|
||||
vectorStore: FaissStore,
|
||||
loaderInstance: LoaderInstance,
|
||||
loaderType: string,
|
||||
identifier: string,
|
||||
splitterType?: string
|
||||
): Promise<LoaderReturn> {
|
||||
const emptyResult: LoaderReturn = {
|
||||
entriesAdded: 0,
|
||||
uniqueId: '',
|
||||
uniqueIds: [],
|
||||
loaderType
|
||||
}
|
||||
|
||||
try {
|
||||
const docs = await loaderInstance.load()
|
||||
return await processDocuments(base, vectorStore, docs, loaderType, splitterType)
|
||||
} catch (error) {
|
||||
logger.error(`Error loading or processing ${identifier} with loader ${loaderType}: ${error}`)
|
||||
return emptyResult
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 文件扩展名到加载器的映射
|
||||
*/
|
||||
const FILE_LOADER_MAP: Record<string, { loader: new (path: string) => LoaderInstance; type: string }> = {
|
||||
'.pdf': { loader: PDFLoader, type: 'pdf' },
|
||||
'.txt': { loader: TextLoader, type: 'text' },
|
||||
'.pptx': { loader: PPTXLoader, type: 'pptx' },
|
||||
'.docx': { loader: DocxLoader, type: 'docx' },
|
||||
'.doc': { loader: DocxLoader, type: 'doc' },
|
||||
'.json': { loader: JSONLoader, type: 'json' },
|
||||
'.epub': { loader: EPubLoader, type: 'epub' },
|
||||
'.md': { loader: MarkdownLoader, type: 'markdown' }
|
||||
}
|
||||
|
||||
export async function addFileLoader(
|
||||
base: KnowledgeBaseParams,
|
||||
vectorStore: FaissStore,
|
||||
file: FileMetadata
|
||||
): Promise<LoaderReturn> {
|
||||
const fileExt = file.ext.toLowerCase()
|
||||
const loaderConfig = FILE_LOADER_MAP[fileExt]
|
||||
|
||||
if (!loaderConfig) {
|
||||
// 默认使用文本加载器
|
||||
const loaderInstance = new TextLoader(file.path)
|
||||
const type = fileExt.replace('.', '') || 'unknown'
|
||||
return executeLoader(base, vectorStore, loaderInstance, type, file.path)
|
||||
}
|
||||
|
||||
const loaderInstance = new loaderConfig.loader(file.path)
|
||||
return executeLoader(base, vectorStore, loaderInstance, loaderConfig.type, file.path)
|
||||
}
|
||||
|
||||
export async function addWebLoader(
|
||||
base: KnowledgeBaseParams,
|
||||
vectorStore: FaissStore,
|
||||
url: string,
|
||||
source: UrlSource
|
||||
): Promise<LoaderReturn> {
|
||||
let loaderInstance: CheerioWebBaseLoader | YoutubeLoader | undefined
|
||||
let splitterType: string | undefined
|
||||
|
||||
switch (source) {
|
||||
case 'normal':
|
||||
loaderInstance = new CheerioWebBaseLoader(url)
|
||||
break
|
||||
case 'youtube':
|
||||
loaderInstance = YoutubeLoader.createFromUrl(url, {
|
||||
addVideoInfo: true,
|
||||
transcriptFormat: 'srt'
|
||||
})
|
||||
splitterType = 'srt'
|
||||
break
|
||||
}
|
||||
|
||||
if (!loaderInstance) {
|
||||
return {
|
||||
entriesAdded: 0,
|
||||
uniqueId: '',
|
||||
uniqueIds: [],
|
||||
loaderType: source
|
||||
}
|
||||
}
|
||||
|
||||
return executeLoader(base, vectorStore, loaderInstance, source, url, splitterType)
|
||||
}
|
||||
|
||||
export async function addSitemapLoader(
|
||||
base: KnowledgeBaseParams,
|
||||
vectorStore: FaissStore,
|
||||
url: string
|
||||
): Promise<LoaderReturn> {
|
||||
const loaderInstance = new SitemapLoader(url)
|
||||
return executeLoader(base, vectorStore, loaderInstance, 'sitemap', url)
|
||||
}
|
||||
|
||||
export async function addNoteLoader(
|
||||
base: KnowledgeBaseParams,
|
||||
vectorStore: FaissStore,
|
||||
content: string,
|
||||
sourceUrl: string
|
||||
): Promise<LoaderReturn> {
|
||||
const loaderInstance = new NoteLoader(content, sourceUrl)
|
||||
return executeLoader(base, vectorStore, loaderInstance, 'note', sourceUrl)
|
||||
}
|
||||
|
||||
export async function addVideoLoader(
|
||||
base: KnowledgeBaseParams,
|
||||
vectorStore: FaissStore,
|
||||
files: FileMetadata[]
|
||||
): Promise<LoaderReturn> {
|
||||
const srtFile = files.find((f) => f.type === FileTypes.TEXT)
|
||||
const videoFile = files.find((f) => f.type === FileTypes.VIDEO)
|
||||
|
||||
const emptyResult: LoaderReturn = {
|
||||
entriesAdded: 0,
|
||||
uniqueId: '',
|
||||
uniqueIds: [],
|
||||
loaderType: 'video'
|
||||
}
|
||||
|
||||
if (!srtFile || !videoFile) {
|
||||
return emptyResult
|
||||
}
|
||||
|
||||
try {
|
||||
const loaderInstance = new TextLoader(srtFile.path)
|
||||
const originalDocs = await loaderInstance.load()
|
||||
|
||||
const docsWithVideoMeta = originalDocs.map(
|
||||
(doc) =>
|
||||
new Document({
|
||||
...doc,
|
||||
metadata: {
|
||||
...doc.metadata,
|
||||
video: {
|
||||
path: videoFile.path,
|
||||
name: videoFile.origin_name
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
return await processDocuments(base, vectorStore, docsWithVideoMeta, 'video', 'srt')
|
||||
} catch (error) {
|
||||
logger.error(`Error loading or processing file ${srtFile.path} with loader video: ${error}`)
|
||||
return emptyResult
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
import { BM25Retriever } from '@langchain/community/retrievers/bm25'
|
||||
import { FaissStore } from '@langchain/community/vectorstores/faiss'
|
||||
import { BaseRetriever } from '@langchain/core/retrievers'
|
||||
import { loggerService } from '@main/services/LoggerService'
|
||||
import { type KnowledgeBaseParams } from '@types'
|
||||
import { type Document } from 'langchain/document'
|
||||
import { EnsembleRetriever } from 'langchain/retrievers/ensemble'
|
||||
|
||||
const logger = loggerService.withContext('RetrieverFactory')
|
||||
export class RetrieverFactory {
|
||||
/**
|
||||
* 根据提供的参数创建一个 LangChain 检索器 (Retriever)。
|
||||
* @param base 知识库配置参数。
|
||||
* @param vectorStore 一个已初始化的向量存储实例。
|
||||
* @param documents 文档列表，用于初始化 BM25Retriever。
|
||||
* @returns 返回一个 BaseRetriever 实例。
|
||||
*/
|
||||
public createRetriever(base: KnowledgeBaseParams, vectorStore: FaissStore, documents: Document[]): BaseRetriever {
|
||||
const retrieverType = base.retriever?.mode ?? 'hybrid'
|
||||
const retrieverWeight = base.retriever?.weight ?? 0.5
|
||||
const searchK = base.documentCount ?? 5
|
||||
|
||||
logger.info(`Creating retriever of type: ${retrieverType} with k=${searchK}`)
|
||||
|
||||
switch (retrieverType) {
|
||||
case 'bm25':
|
||||
if (documents.length === 0) {
|
||||
throw new Error('BM25Retriever requires documents, but none were provided or found.')
|
||||
}
|
||||
logger.info('Create BM25 Retriever')
|
||||
return BM25Retriever.fromDocuments(documents, { k: searchK })
|
||||
|
||||
case 'hybrid': {
|
||||
if (documents.length === 0) {
|
||||
logger.warn('No documents provided for BM25 part of hybrid search. Falling back to vector search only.')
|
||||
return vectorStore.asRetriever(searchK)
|
||||
}
|
||||
|
||||
const vectorstoreRetriever = vectorStore.asRetriever(searchK)
|
||||
const bm25Retriever = BM25Retriever.fromDocuments(documents, { k: searchK })
|
||||
|
||||
logger.info('Create Hybrid Retriever')
|
||||
return new EnsembleRetriever({
|
||||
retrievers: [bm25Retriever, vectorstoreRetriever],
|
||||
weights: [retrieverWeight, 1 - retrieverWeight]
|
||||
})
|
||||
}
|
||||
|
||||
case 'vector':
|
||||
default:
|
||||
logger.info('Create Vector Retriever')
|
||||
return vectorStore.asRetriever(searchK)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,133 +0,0 @@
|
||||
import { Document } from '@langchain/core/documents'
|
||||
import { TextSplitter, TextSplitterParams } from 'langchain/text_splitter'
|
||||
|
||||
// 定义一个接口来表示解析后的单个字幕片段
|
||||
interface SrtSegment {
|
||||
text: string
|
||||
startTime: number // in seconds
|
||||
endTime: number // in seconds
|
||||
}
|
||||
|
||||
// 辅助函数：将 SRT 时间戳字符串 (HH:MM:SS,ms) 转换为秒
|
||||
function srtTimeToSeconds(time: string): number {
|
||||
const parts = time.split(':')
|
||||
const secondsAndMs = parts[2].split(',')
|
||||
const hours = parseInt(parts[0], 10)
|
||||
const minutes = parseInt(parts[1], 10)
|
||||
const seconds = parseInt(secondsAndMs[0], 10)
|
||||
const milliseconds = parseInt(secondsAndMs[1], 10)
|
||||
|
||||
return hours * 3600 + minutes * 60 + seconds + milliseconds / 1000
|
||||
}
|
||||
|
||||
export class SrtSplitter extends TextSplitter {
|
||||
constructor(fields?: Partial<TextSplitterParams>) {
|
||||
// 传入 chunkSize 和 chunkOverlap
|
||||
super(fields)
|
||||
}
|
||||
splitText(): Promise<string[]> {
|
||||
throw new Error('Method not implemented.')
|
||||
}
|
||||
|
||||
// 核心方法：重写 splitDocuments 来实现自定义逻辑
|
||||
async splitDocuments(documents: Document[]): Promise<Document[]> {
|
||||
const allChunks: Document[] = []
|
||||
|
||||
for (const doc of documents) {
|
||||
// 1. 解析 SRT 内容
|
||||
const segments = this.parseSrt(doc.pageContent)
|
||||
if (segments.length === 0) continue
|
||||
|
||||
// 2. 将字幕片段组合成块
|
||||
const chunks = this.mergeSegmentsIntoChunks(segments, doc.metadata)
|
||||
allChunks.push(...chunks)
|
||||
}
|
||||
|
||||
return allChunks
|
||||
}
|
||||
|
||||
// 辅助方法：解析整个 SRT 字符串
|
||||
private parseSrt(srt: string): SrtSegment[] {
|
||||
const segments: SrtSegment[] = []
|
||||
const blocks = srt.trim().split(/\n\n/)
|
||||
|
||||
for (const block of blocks) {
|
||||
const lines = block.split('\n')
|
||||
if (lines.length < 3) continue
|
||||
|
||||
const timeMatch = lines[1].match(/(\d{2}:\d{2}:\d{2},\d{3}) --> (\d{2}:\d{2}:\d{2},\d{3})/)
|
||||
if (!timeMatch) continue
|
||||
|
||||
const startTime = srtTimeToSeconds(timeMatch[1])
|
||||
const endTime = srtTimeToSeconds(timeMatch[2])
|
||||
const text = lines.slice(2).join(' ').trim()
|
||||
|
||||
segments.push({ text, startTime, endTime })
|
||||
}
|
||||
|
||||
return segments
|
||||
}
|
||||
|
||||
// 辅助方法：将解析后的片段合并成每 5 段一个块
|
||||
private mergeSegmentsIntoChunks(segments: SrtSegment[], baseMetadata: Record<string, any>): Document[] {
|
||||
const chunks: Document[] = []
|
||||
let currentChunkText = ''
|
||||
let currentChunkStartTime = 0
|
||||
let currentChunkEndTime = 0
|
||||
let segmentCount = 0
|
||||
|
||||
for (const segment of segments) {
|
||||
if (segmentCount === 0) {
|
||||
currentChunkStartTime = segment.startTime
|
||||
}
|
||||
|
||||
currentChunkText += (currentChunkText ? ' ' : '') + segment.text
|
||||
currentChunkEndTime = segment.endTime
|
||||
segmentCount++
|
||||
|
||||
// 当累积到 5 段时，创建一个新的 Document
|
||||
if (segmentCount === 5) {
|
||||
const metadata: Record<string, any> = {
|
||||
...baseMetadata,
|
||||
startTime: currentChunkStartTime,
|
||||
endTime: currentChunkEndTime
|
||||
}
|
||||
if (baseMetadata.source_url) {
|
||||
metadata.source_url_with_timestamp = `${baseMetadata.source_url}?t=${Math.floor(currentChunkStartTime)}s`
|
||||
}
|
||||
chunks.push(
|
||||
new Document({
|
||||
pageContent: currentChunkText,
|
||||
metadata
|
||||
})
|
||||
)
|
||||
|
||||
// 重置计数器和临时变量
|
||||
currentChunkText = ''
|
||||
currentChunkStartTime = 0
|
||||
currentChunkEndTime = 0
|
||||
segmentCount = 0
|
||||
}
|
||||
}
|
||||
|
||||
// 如果还有剩余的片段，创建最后一个 Document
|
||||
if (segmentCount > 0) {
|
||||
const metadata: Record<string, any> = {
|
||||
...baseMetadata,
|
||||
startTime: currentChunkStartTime,
|
||||
endTime: currentChunkEndTime
|
||||
}
|
||||
if (baseMetadata.source_url) {
|
||||
metadata.source_url_with_timestamp = `${baseMetadata.source_url}?t=${Math.floor(currentChunkStartTime)}s`
|
||||
}
|
||||
chunks.push(
|
||||
new Document({
|
||||
pageContent: currentChunkText,
|
||||
metadata
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
import { RecursiveCharacterTextSplitter, TextSplitter } from '@langchain/textsplitters'
|
||||
|
||||
import { SrtSplitter } from './SrtSplitter'
|
||||
|
||||
export type SplitterConfig = {
|
||||
chunkSize?: number
|
||||
chunkOverlap?: number
|
||||
type?: 'recursive' | 'srt' | string
|
||||
}
|
||||
export class SplitterFactory {
|
||||
/**
|
||||
* Creates a TextSplitter instance based on the provided configuration.
|
||||
* @param config - The configuration object specifying the splitter type and its parameters.
|
||||
* @returns An instance of a TextSplitter, or null if no splitting is required.
|
||||
*/
|
||||
public static create(config: SplitterConfig): TextSplitter {
|
||||
switch (config.type) {
|
||||
case 'srt':
|
||||
return new SrtSplitter({
|
||||
chunkSize: config.chunkSize,
|
||||
chunkOverlap: config.chunkOverlap
|
||||
})
|
||||
case 'recursive':
|
||||
default:
|
||||
return new RecursiveCharacterTextSplitter({
|
||||
chunkSize: config.chunkSize,
|
||||
chunkOverlap: config.chunkOverlap
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user