feat(WebSearch): support RAG for external websearch, improve feedback (#7446)

* feat(WebSearch, RAG): support RAG for external websearch

* refactor(WebSearch): handle content limit in service

* refactor: update migrate

* refactor: UI, constants, types

* refactor: migrate contentLimit to cutoffLimit

* refactor: update default rag document count

* refactor: add a helper function for merging references

* refactor: reference filtering

* feat: feedback for websearch phases

* feat: support cutoff by token

* refactor: add a warning and fix the bound of cutoff limit

* fix: not pass `dimensions` if it is not set by the user

* refactor: update i18n and error message

* refactor: improve UI

* fix: cutoff unit style

src/main/services/KnowledgeService.ts

@@ -16,13 +16,14 @@
 import * as fs from 'node:fs'
 import path from 'node:path'
 
-import { RAGApplication, RAGApplicationBuilder, TextLoader } from '@cherrystudio/embedjs'
+import { RAGApplication, RAGApplicationBuilder } from '@cherrystudio/embedjs'
 import type { ExtractChunkData } from '@cherrystudio/embedjs-interfaces'
 import { LibSqlDb } from '@cherrystudio/embedjs-libsql'
 import { SitemapLoader } from '@cherrystudio/embedjs-loader-sitemap'
 import { WebLoader } from '@cherrystudio/embedjs-loader-web'
 import Embeddings from '@main/embeddings/Embeddings'
 import { addFileLoader } from '@main/loader'
+import { NoteLoader } from '@main/loader/noteLoader'
 import Reranker from '@main/reranker/Reranker'
 import { windowService } from '@main/services/WindowService'
 import { getDataPath } from '@main/utils'
@@ -143,7 +144,7 @@ class KnowledgeService {
     this.getRagApplication(base)
   }
 
-  public reset = async (_: Electron.IpcMainInvokeEvent, { base }: { base: KnowledgeBaseParams }): Promise<void> => {
+  public reset = async (_: Electron.IpcMainInvokeEvent, base: KnowledgeBaseParams): Promise<void> => {
     const ragApplication = await this.getRagApplication(base)
     await ragApplication.reset()
   }
@@ -333,6 +334,7 @@ class KnowledgeService {
   ): LoaderTask {
     const { base, item, forceReload } = options
     const content = item.content as string
+    const sourceUrl = (item as any).sourceUrl
 
     const encoder = new TextEncoder()
     const contentBytes = encoder.encode(content)
@@ -342,7 +344,12 @@ class KnowledgeService {
           state: LoaderTaskItemState.PENDING,
           task: () => {
             const loaderReturn = ragApplication.addLoader(
-              new TextLoader({ text: content, chunkSize: base.chunkSize, chunkOverlap: base.chunkOverlap }),
+              new NoteLoader({
+                text: content,
+                sourceUrl,
+                chunkSize: base.chunkSize,
+                chunkOverlap: base.chunkOverlap
+              }),
               forceReload
             ) as Promise<LoaderReturn>