create transactions from AI receipt image recognition results
This commit is contained in:
@@ -30,6 +30,7 @@ Live Demo: [https://ezbookkeeping-demo.mayswind.net](https://ezbookkeeping-demo.
|
||||
- PWA support for native-like mobile experience
|
||||
- Dark mode
|
||||
- **AI-Powered Features**
|
||||
- Receipt image recognition
|
||||
- Supports MCP (Model Context Protocol) for AI integration
|
||||
- **Powerful Bookkeeping**
|
||||
- Two-level accounts and categories
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/mayswind/ezbookkeeping/pkg/datastore"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/duplicatechecker"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/exchangerates"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/llm"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/log"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/mail"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
@@ -90,6 +91,15 @@ func initializeSystem(c *core.CliContext) (*settings.Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = llm.InitializeLargeLanguageModelProvider(config)
|
||||
|
||||
if err != nil {
|
||||
if !isDisableBootLog {
|
||||
log.BootErrorf(c, "[initializer.initializeSystem] initializes large language model provider failed, because %s", err.Error())
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = uuid.InitializeUuidGenerator(config)
|
||||
|
||||
if err != nil {
|
||||
@@ -155,6 +165,9 @@ func getConfigWithoutSensitiveData(config *settings.Config) *settings.Config {
|
||||
clonedConfig.DatabaseConfig.DatabasePassword = "****"
|
||||
clonedConfig.SMTPConfig.SMTPPasswd = "****"
|
||||
clonedConfig.MinIOConfig.SecretAccessKey = "****"
|
||||
clonedConfig.OpenAIAPIKey = "****"
|
||||
clonedConfig.OpenAICompatibleAPIKey = "****"
|
||||
clonedConfig.OpenRouterAPIKey = "****"
|
||||
clonedConfig.SecretKey = "****"
|
||||
clonedConfig.AmapApplicationSecret = "****"
|
||||
|
||||
|
||||
@@ -396,6 +396,13 @@ func startWebServer(c *core.CliContext) error {
|
||||
apiV1Route.POST("/transaction/templates/move.json", bindApi(api.TransactionTemplates.TemplateMoveHandler))
|
||||
apiV1Route.POST("/transaction/templates/delete.json", bindApi(api.TransactionTemplates.TemplateDeleteHandler))
|
||||
|
||||
// Large Language Models
|
||||
if config.LLMProvider != "" {
|
||||
if config.TransactionFromAIImageRecognition {
|
||||
apiV1Route.POST("/llm/transactions/recognize_receipt_image.json", bindApi(api.LargeLanguageModels.RecognizeReceiptImageHandler))
|
||||
}
|
||||
}
|
||||
|
||||
// Exchange Rates
|
||||
apiV1Route.GET("/exchange_rates/latest.json", bindApi(api.ExchangeRates.LatestExchangeRateHandler))
|
||||
apiV1Route.POST("/exchange_rates/user_custom/update.json", bindApi(api.ExchangeRates.UserCustomExchangeRateUpdateHandler))
|
||||
|
||||
@@ -164,6 +164,53 @@ webdav_proxy = system
|
||||
# For "webdav" storage only, set to true to skip tls verification when connect webdav
|
||||
webdav_skip_tls_verify = false
|
||||
|
||||
[llm]
|
||||
# Large Language Model (LLM) provider, supports the following types: "openai", "openai_compatible", "openrouter", "ollama"
|
||||
llm_provider =
|
||||
|
||||
# For "openai" llm provider only, OpenAI API secret key, please visit https://platform.openai.com/api-keys for more information
|
||||
openai_api_key =
|
||||
|
||||
# For "openai" llm provider only, receipt image recognition model for creating transactions from images
|
||||
openai_receipt_image_recognition_model_id =
|
||||
|
||||
# For "openai_compatible" llm provider only, OpenAI compatible API base url, e.g. "https://api.openai.com/v1/"
|
||||
openai_compatible_base_url =
|
||||
|
||||
# For "openai_compatible" llm provider only, OpenAI compatible API secret key
|
||||
openai_compatible_api_key =
|
||||
|
||||
# For "openai_compatible" llm provider only, receipt image recognition model for creating transactions from images
|
||||
openai_compatible_receipt_image_recognition_model_id =
|
||||
|
||||
# For "openrouter" llm provider only, OpenRouter API key, please visit https://openrouter.ai/settings/keys for more information
|
||||
openrouter_api_key =
|
||||
|
||||
# For "openrouter" llm provider only, receipt image recognition model for creating transactions from images
|
||||
openrouter_receipt_image_recognition_model_id =
|
||||
|
||||
# For "ollama" llm provider only, Ollama server url, e.g. "http://127.0.0.1:11434/"
|
||||
ollama_server_url =
|
||||
|
||||
# For "ollama" llm provider only, receipt image recognition model for creating transactions from images
|
||||
ollama_receipt_image_recognition_model_id =
|
||||
|
||||
# Set to true to enable creating transactions from AI image recognition results, requires llm_provider and its related receipt image recognition model to be configured properly
|
||||
transaction_from_ai_image_recognition = false
|
||||
|
||||
# Maximum allowed AI recognition picture file size (1 - 4294967295 bytes)
|
||||
max_ai_recognition_picture_size = 10485760
|
||||
|
||||
# Requesting large language model api timeout (0 - 4294967295 milliseconds)
|
||||
# Set to 0 to disable timeout for requesting large language model api, default is 60000 (60 seconds)
|
||||
request_timeout = 60000
|
||||
|
||||
# Proxy for ezbookkeeping server requesting large language model api, supports "system" (use system proxy), "none" (do not use proxy), or proxy URL which starts with "http://", "https://" or "socks5://", default is "system"
|
||||
proxy = system
|
||||
|
||||
# Set to true to skip tls verification when request large language model api
|
||||
skip_tls_verify = false
|
||||
|
||||
[uuid]
|
||||
# Uuid generator type, supports "internal" currently
|
||||
generator_type = internal
|
||||
|
||||
345
pkg/api/large_language_models.go
Normal file
345
pkg/api/large_language_models.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/llm"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/log"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/models"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/services"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/templates"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/utils"
|
||||
)
|
||||
|
||||
// LargeLanguageModelsApi represents large language models api
|
||||
type LargeLanguageModelsApi struct {
|
||||
ApiUsingConfig
|
||||
transactionCategories *services.TransactionCategoryService
|
||||
transactionTags *services.TransactionTagService
|
||||
accounts *services.AccountService
|
||||
users *services.UserService
|
||||
}
|
||||
|
||||
// Initialize a large language models api singleton instance
|
||||
var (
|
||||
LargeLanguageModels = &LargeLanguageModelsApi{
|
||||
ApiUsingConfig: ApiUsingConfig{
|
||||
container: settings.Container,
|
||||
},
|
||||
transactionCategories: services.TransactionCategories,
|
||||
transactionTags: services.TransactionTags,
|
||||
accounts: services.Accounts,
|
||||
users: services.Users,
|
||||
}
|
||||
)
|
||||
|
||||
// RecognizeReceiptImageHandler returns the recognized receipt image result
|
||||
func (a *LargeLanguageModelsApi) RecognizeReceiptImageHandler(c *core.WebContext) (any, *errs.Error) {
|
||||
if a.CurrentConfig().LLMProvider == "" || !a.CurrentConfig().TransactionFromAIImageRecognition {
|
||||
return nil, errs.ErrLargeLanguageModelProviderNotEnabled
|
||||
}
|
||||
|
||||
utcOffset, err := c.GetClientTimezoneOffset()
|
||||
|
||||
if err != nil {
|
||||
log.Warnf(c, "[large_language_models.RecognizeReceiptImageHandler] cannot get client timezone offset, because %s", err.Error())
|
||||
return nil, errs.ErrClientTimezoneOffsetInvalid
|
||||
}
|
||||
|
||||
uid := c.GetCurrentUid()
|
||||
user, err := a.users.GetUserById(c, uid)
|
||||
|
||||
if err != nil {
|
||||
if !errs.IsCustomError(err) {
|
||||
log.Warnf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to get user for user \"uid:%d\", because %s", uid, err.Error())
|
||||
}
|
||||
|
||||
return false, errs.ErrUserNotFound
|
||||
}
|
||||
|
||||
if user.FeatureRestriction.Contains(core.USER_FEATURE_RESTRICTION_TYPE_CREATE_TRANSACTION_FROM_AI_IMAGE_RECOGNITION) {
|
||||
return false, errs.ErrNotPermittedToPerformThisAction
|
||||
}
|
||||
|
||||
form, err := c.MultipartForm()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to get multi-part form data for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrParameterInvalid
|
||||
}
|
||||
|
||||
imageFiles := form.File["image"]
|
||||
|
||||
if len(imageFiles) < 1 {
|
||||
log.Warnf(c, "[large_language_models.RecognizeReceiptImageHandler] there is no image in request for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrNoAIRecognitionImage
|
||||
}
|
||||
|
||||
if imageFiles[0].Size < 1 {
|
||||
log.Warnf(c, "[large_language_models.RecognizeReceiptImageHandler] the size of image in request is zero for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrAIRecognitionImageIsEmpty
|
||||
}
|
||||
|
||||
if imageFiles[0].Size > int64(a.CurrentConfig().MaxAIRecognitionPictureFileSize) {
|
||||
log.Warnf(c, "[large_language_models.RecognizeReceiptImageHandler] the upload file size \"%d\" exceeds the maximum size \"%d\" of image for user \"uid:%d\"", imageFiles[0].Size, a.CurrentConfig().MaxAIRecognitionPictureFileSize, uid)
|
||||
return nil, errs.ErrExceedMaxAIRecognitionImageFileSize
|
||||
}
|
||||
|
||||
fileExtension := utils.GetFileNameExtension(imageFiles[0].Filename)
|
||||
|
||||
if utils.GetImageContentType(fileExtension) == "" {
|
||||
log.Warnf(c, "[large_language_models.RecognizeReceiptImageHandler] the file extension \"%s\" of image in request is not supported for user \"uid:%d\"", fileExtension, uid)
|
||||
return nil, errs.ErrImageTypeNotSupported
|
||||
}
|
||||
|
||||
imageFile, err := imageFiles[0].Open()
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to get image file from request for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
defer imageFile.Close()
|
||||
|
||||
imageData, err := io.ReadAll(imageFile)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to read image file from request for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
accounts, err := a.accounts.GetAllAccountsByUid(c, uid)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to get all accounts for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
accountMap := a.accounts.GetVisibleAccountNameMapByList(accounts)
|
||||
accountNames := make([]string, 0, len(accounts))
|
||||
|
||||
for i := 0; i < len(accounts); i++ {
|
||||
if accounts[i].Hidden || accounts[i].Type == models.ACCOUNT_TYPE_MULTI_SUB_ACCOUNTS {
|
||||
continue
|
||||
}
|
||||
|
||||
accountNames = append(accountNames, accounts[i].Name)
|
||||
}
|
||||
|
||||
categories, err := a.transactionCategories.GetAllCategoriesByUid(c, uid, 0, -1)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to get categories for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
incomeCategoryMap := make(map[string]*models.TransactionCategory)
|
||||
incomeCategoryNames := make([]string, 0)
|
||||
|
||||
expenseCategoryMap := make(map[string]*models.TransactionCategory)
|
||||
expenseCategoryNames := make([]string, 0)
|
||||
|
||||
transferCategoryMap := make(map[string]*models.TransactionCategory)
|
||||
transferCategoryNames := make([]string, 0)
|
||||
|
||||
for i := 0; i < len(categories); i++ {
|
||||
category := categories[i]
|
||||
|
||||
if category.Hidden || category.ParentCategoryId == models.LevelOneTransactionCategoryParentId {
|
||||
continue
|
||||
}
|
||||
|
||||
if category.Type == models.CATEGORY_TYPE_INCOME {
|
||||
incomeCategoryMap[category.Name] = category
|
||||
incomeCategoryNames = append(incomeCategoryNames, category.Name)
|
||||
} else if category.Type == models.CATEGORY_TYPE_EXPENSE {
|
||||
expenseCategoryMap[category.Name] = category
|
||||
expenseCategoryNames = append(expenseCategoryNames, category.Name)
|
||||
} else if category.Type == models.CATEGORY_TYPE_TRANSFER {
|
||||
transferCategoryMap[category.Name] = category
|
||||
transferCategoryNames = append(transferCategoryNames, category.Name)
|
||||
}
|
||||
}
|
||||
|
||||
tags, err := a.transactionTags.GetAllTagsByUid(c, uid)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to get tags for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
tagMap := a.transactionTags.GetVisibleTagNameMapByList(tags)
|
||||
tagNames := make([]string, 0, len(tags))
|
||||
|
||||
for i := 0; i < len(tags); i++ {
|
||||
if tags[i].Hidden {
|
||||
continue
|
||||
}
|
||||
|
||||
tagNames = append(tagNames, tags[i].Name)
|
||||
}
|
||||
|
||||
systemPrompt, err := templates.GetTemplate(templates.SYSTEM_PROMPT_RECEIPT_IMAGE_RECOGNITION)
|
||||
|
||||
if err != nil {
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
systemPromptParams := map[string]any{
|
||||
"AllExpenseCategoryNames": strings.Join(expenseCategoryNames, "\n"),
|
||||
"AllIncomeCategoryNames": strings.Join(incomeCategoryNames, "\n"),
|
||||
"AllTransferCategoryNames": strings.Join(transferCategoryNames, "\n"),
|
||||
"AllAccountNames": strings.Join(accountNames, "\n"),
|
||||
"AllTagNames": strings.Join(tagNames, "\n"),
|
||||
}
|
||||
|
||||
var bodyBuffer bytes.Buffer
|
||||
err = systemPrompt.Execute(&bodyBuffer, systemPromptParams)
|
||||
|
||||
if err != nil {
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
llmRequest := &llm.LargeLanguageModelRequest{
|
||||
Stream: false,
|
||||
SystemPrompt: strings.ReplaceAll(bodyBuffer.String(), "\r\n", "\n"),
|
||||
UserPrompt: imageData,
|
||||
UserPromptType: llm.LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL,
|
||||
}
|
||||
|
||||
llmResponse, err := llm.Container.GetJsonResponseByReceiptImageRecognitionModel(c, c.GetCurrentUid(), a.CurrentConfig(), llmRequest)
|
||||
|
||||
if err != nil {
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
var result *models.RecognizedReceiptImageResult
|
||||
|
||||
if err := json.Unmarshal([]byte(llmResponse.Content), &result); err != nil {
|
||||
log.Errorf(c, "[large_language_models.RecognizeReceiptImageHandler] failed to unmarshal recognized receipt image result from llm response \"%s\" for user \"uid:%d\", because %s", llmResponse.Content, uid, err.Error())
|
||||
return nil, errs.Or(err, errs.ErrOperationFailed)
|
||||
}
|
||||
|
||||
return a.parseRecognizedReceiptImageResponse(c, uid, utcOffset, result, accountMap, expenseCategoryMap, incomeCategoryMap, transferCategoryMap, tagMap)
|
||||
}
|
||||
|
||||
func (a *LargeLanguageModelsApi) parseRecognizedReceiptImageResponse(c *core.WebContext, uid int64, utcOffset int16, recognizedResult *models.RecognizedReceiptImageResult, accountMap map[string]*models.Account, expenseCategoryMap map[string]*models.TransactionCategory, incomeCategoryMap map[string]*models.TransactionCategory, transferCategoryMap map[string]*models.TransactionCategory, tagMap map[string]*models.TransactionTag) (*models.RecognizedReceiptImageResponse, *errs.Error) {
|
||||
recognizedReceiptImageResponse := &models.RecognizedReceiptImageResponse{
|
||||
Type: models.TRANSACTION_TYPE_EXPENSE,
|
||||
}
|
||||
|
||||
if recognizedResult == nil {
|
||||
log.Errorf(c, "[large_language_models.parseRecognizedReceiptImageResponse] recoginzed result is null")
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
if recognizedResult.Type == "income" {
|
||||
recognizedReceiptImageResponse.Type = models.TRANSACTION_TYPE_INCOME
|
||||
|
||||
if len(recognizedResult.CategoryName) > 0 {
|
||||
category, exists := incomeCategoryMap[recognizedResult.CategoryName]
|
||||
|
||||
if exists {
|
||||
recognizedReceiptImageResponse.CategoryId = category.CategoryId
|
||||
}
|
||||
}
|
||||
} else if recognizedResult.Type == "expense" {
|
||||
recognizedReceiptImageResponse.Type = models.TRANSACTION_TYPE_EXPENSE
|
||||
|
||||
if len(recognizedResult.CategoryName) > 0 {
|
||||
category, exists := expenseCategoryMap[recognizedResult.CategoryName]
|
||||
|
||||
if exists {
|
||||
recognizedReceiptImageResponse.CategoryId = category.CategoryId
|
||||
}
|
||||
}
|
||||
} else if recognizedResult.Type == "transfer" {
|
||||
recognizedReceiptImageResponse.Type = models.TRANSACTION_TYPE_TRANSFER
|
||||
|
||||
if len(recognizedResult.CategoryName) > 0 {
|
||||
category, exists := transferCategoryMap[recognizedResult.CategoryName]
|
||||
|
||||
if exists {
|
||||
recognizedReceiptImageResponse.CategoryId = category.CategoryId
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Errorf(c, "[large_language_models.parseRecognizedReceiptImageResponse] recoginzed transaction type \"%s\" is invalid", recognizedResult.Type)
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
if len(recognizedResult.Time) > 0 {
|
||||
timestamp, err := utils.ParseFromLongDateTime(recognizedResult.Time, utcOffset)
|
||||
|
||||
if err != nil {
|
||||
log.Warnf(c, "[large_language_models.parseRecognizedReceiptImageResponse] recoginzed time \"%s\" is invalid", recognizedResult.Time)
|
||||
} else {
|
||||
recognizedReceiptImageResponse.Time = timestamp.Unix()
|
||||
}
|
||||
}
|
||||
|
||||
if len(recognizedResult.Amount) > 0 {
|
||||
amount, err := utils.ParseAmount(recognizedResult.Amount)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.parseRecognizedReceiptImageResponse] recoginzed amount \"%s\" is invalid", recognizedResult.Amount)
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
recognizedReceiptImageResponse.SourceAmount = amount
|
||||
|
||||
if recognizedReceiptImageResponse.Type == models.TRANSACTION_TYPE_TRANSFER && len(recognizedResult.DestinationAmount) > 0 {
|
||||
destinationAmount, err := utils.ParseAmount(recognizedResult.DestinationAmount)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[large_language_models.parseRecognizedReceiptImageResponse] recoginzed destination amount \"%s\" is invalid", recognizedResult.DestinationAmount)
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
recognizedReceiptImageResponse.DestinationAmount = destinationAmount
|
||||
}
|
||||
}
|
||||
|
||||
if len(recognizedResult.AccountName) > 0 {
|
||||
account, exists := accountMap[recognizedResult.AccountName]
|
||||
|
||||
if exists {
|
||||
recognizedReceiptImageResponse.SourceAccountId = account.AccountId
|
||||
}
|
||||
}
|
||||
|
||||
if len(recognizedResult.DestinationAccountName) > 0 {
|
||||
account, exists := accountMap[recognizedResult.DestinationAccountName]
|
||||
|
||||
if exists {
|
||||
recognizedReceiptImageResponse.DestinationAccountId = account.AccountId
|
||||
}
|
||||
}
|
||||
|
||||
if len(recognizedResult.TagNames) > 0 {
|
||||
tagIds := make([]string, 0, len(recognizedResult.TagNames))
|
||||
|
||||
for i := 0; i < len(recognizedResult.TagNames); i++ {
|
||||
tagName := recognizedResult.TagNames[i]
|
||||
tag, exists := tagMap[tagName]
|
||||
|
||||
if exists {
|
||||
tagIds = append(tagIds, utils.Int64ToString(tag.TagId))
|
||||
}
|
||||
}
|
||||
|
||||
recognizedReceiptImageResponse.TagIds = tagIds
|
||||
}
|
||||
|
||||
if len(recognizedResult.Description) > 0 {
|
||||
recognizedReceiptImageResponse.Comment = recognizedResult.Description
|
||||
}
|
||||
|
||||
return recognizedReceiptImageResponse, nil
|
||||
}
|
||||
@@ -47,6 +47,12 @@ func (a *ServerSettingsApi) ServerSettingsJavascriptHandler(c *core.WebContext)
|
||||
a.appendBooleanSetting(builder, "mcp", config.EnableMCPServer)
|
||||
}
|
||||
|
||||
if config.LLMProvider != "" {
|
||||
if config.TransactionFromAIImageRecognition {
|
||||
a.appendBooleanSetting(builder, "llmt", config.TransactionFromAIImageRecognition)
|
||||
}
|
||||
}
|
||||
|
||||
if config.LoginPageTips.Enabled {
|
||||
a.appendMultiLanguageTipSetting(builder, "lpt", config.LoginPageTips)
|
||||
}
|
||||
|
||||
@@ -76,19 +76,20 @@ type UserFeatureRestrictionType uint64
|
||||
|
||||
// User Feature Restriction Type
|
||||
const (
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_PASSWORD UserFeatureRestrictionType = 1
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_EMAIL UserFeatureRestrictionType = 2
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_PROFILE_BASIC_INFO UserFeatureRestrictionType = 3
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_AVATAR UserFeatureRestrictionType = 4
|
||||
USER_FEATURE_RESTRICTION_TYPE_REVOKE_OTHER_SESSION UserFeatureRestrictionType = 5
|
||||
USER_FEATURE_RESTRICTION_TYPE_ENABLE_2FA UserFeatureRestrictionType = 6
|
||||
USER_FEATURE_RESTRICTION_TYPE_DISABLE_2FA UserFeatureRestrictionType = 7
|
||||
USER_FEATURE_RESTRICTION_TYPE_FORGET_PASSWORD UserFeatureRestrictionType = 8
|
||||
USER_FEATURE_RESTRICTION_TYPE_IMPORT_TRANSACTION UserFeatureRestrictionType = 9
|
||||
USER_FEATURE_RESTRICTION_TYPE_EXPORT_TRANSACTION UserFeatureRestrictionType = 10
|
||||
USER_FEATURE_RESTRICTION_TYPE_CLEAR_ALL_DATA UserFeatureRestrictionType = 11
|
||||
USER_FEATURE_RESTRICTION_TYPE_SYNC_APPLICATION_SETTINGS UserFeatureRestrictionType = 12
|
||||
USER_FEATURE_RESTRICTION_TYPE_MCP_ACCESS UserFeatureRestrictionType = 13
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_PASSWORD UserFeatureRestrictionType = 1
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_EMAIL UserFeatureRestrictionType = 2
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_PROFILE_BASIC_INFO UserFeatureRestrictionType = 3
|
||||
USER_FEATURE_RESTRICTION_TYPE_UPDATE_AVATAR UserFeatureRestrictionType = 4
|
||||
USER_FEATURE_RESTRICTION_TYPE_REVOKE_OTHER_SESSION UserFeatureRestrictionType = 5
|
||||
USER_FEATURE_RESTRICTION_TYPE_ENABLE_2FA UserFeatureRestrictionType = 6
|
||||
USER_FEATURE_RESTRICTION_TYPE_DISABLE_2FA UserFeatureRestrictionType = 7
|
||||
USER_FEATURE_RESTRICTION_TYPE_FORGET_PASSWORD UserFeatureRestrictionType = 8
|
||||
USER_FEATURE_RESTRICTION_TYPE_IMPORT_TRANSACTION UserFeatureRestrictionType = 9
|
||||
USER_FEATURE_RESTRICTION_TYPE_EXPORT_TRANSACTION UserFeatureRestrictionType = 10
|
||||
USER_FEATURE_RESTRICTION_TYPE_CLEAR_ALL_DATA UserFeatureRestrictionType = 11
|
||||
USER_FEATURE_RESTRICTION_TYPE_SYNC_APPLICATION_SETTINGS UserFeatureRestrictionType = 12
|
||||
USER_FEATURE_RESTRICTION_TYPE_MCP_ACCESS UserFeatureRestrictionType = 13
|
||||
USER_FEATURE_RESTRICTION_TYPE_CREATE_TRANSACTION_FROM_AI_IMAGE_RECOGNITION UserFeatureRestrictionType = 14
|
||||
)
|
||||
|
||||
const userFeatureRestrictionTypeMinValue UserFeatureRestrictionType = USER_FEATURE_RESTRICTION_TYPE_UPDATE_PASSWORD
|
||||
|
||||
@@ -40,6 +40,7 @@ const (
|
||||
NormalSubcategoryConverter = 12
|
||||
NormalSubcategoryUserCustomExchangeRate = 13
|
||||
NormalSubcategoryModelContextProtocol = 14
|
||||
NormalSubcategoryLargeLanguageModel = 15
|
||||
)
|
||||
|
||||
// Error represents the specific error returned to user
|
||||
|
||||
11
pkg/errs/large_language_model.go
Normal file
11
pkg/errs/large_language_model.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package errs
|
||||
|
||||
import "net/http"
|
||||
|
||||
// Error codes related to large language model features
|
||||
var (
|
||||
ErrLargeLanguageModelProviderNotEnabled = NewNormalError(NormalSubcategoryLargeLanguageModel, 0, http.StatusBadRequest, "llm provider is not enabled")
|
||||
ErrNoAIRecognitionImage = NewNormalError(NormalSubcategoryLargeLanguageModel, 1, http.StatusBadRequest, "no image for AI recognition")
|
||||
ErrAIRecognitionImageIsEmpty = NewNormalError(NormalSubcategoryLargeLanguageModel, 2, http.StatusBadRequest, "image for AI recognition is empty")
|
||||
ErrExceedMaxAIRecognitionImageFileSize = NewNormalError(NormalSubcategoryLargeLanguageModel, 3, http.StatusBadRequest, "exceed the maximum size of image file for AI recognition")
|
||||
)
|
||||
@@ -24,4 +24,6 @@ var (
|
||||
ErrInvalidPasswordResetTokenExpiredTime = NewSystemError(SystemSubcategorySetting, 17, http.StatusInternalServerError, "invalid password reset token expired time")
|
||||
ErrInvalidExchangeRatesDataSource = NewSystemError(SystemSubcategorySetting, 18, http.StatusInternalServerError, "invalid exchange rates data source")
|
||||
ErrInvalidIpAddressPattern = NewSystemError(SystemSubcategorySetting, 19, http.StatusInternalServerError, "invalid ip address pattern")
|
||||
ErrInvalidLLMProvider = NewSystemError(SystemSubcategorySetting, 20, http.StatusInternalServerError, "invalid llm provider")
|
||||
ErrInvalidLLMModelId = NewSystemError(SystemSubcategorySetting, 21, http.StatusInternalServerError, "invalid llm model id")
|
||||
)
|
||||
|
||||
91
pkg/llm/http_large_language_model_provider.go
Normal file
91
pkg/llm/http_large_language_model_provider.go
Normal file
@@ -0,0 +1,91 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/log"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/utils"
|
||||
)
|
||||
|
||||
// HttpLargeLanguageModelProvider defines the structure of http large language model provider
|
||||
type HttpLargeLanguageModelProvider interface {
|
||||
// BuildTextualRequest returns the http request by the provider api definition
|
||||
BuildTextualRequest(c core.Context, uid int64, request *LargeLanguageModelRequest, modelId string, responseType LargeLanguageModelResponseFormat) (*http.Request, error)
|
||||
|
||||
// ParseTextualResponse returns the textual response entity by the provider api definition
|
||||
ParseTextualResponse(c core.Context, uid int64, body []byte, responseType LargeLanguageModelResponseFormat) (*LargeLanguageModelTextualResponse, error)
|
||||
|
||||
// GetReceiptImageRecognitionModelID returns the receipt image recognition model id if supported, otherwise returns empty string
|
||||
GetReceiptImageRecognitionModelID() string
|
||||
}
|
||||
|
||||
// CommonHttpLargeLanguageModelProvider defines the structure of common http large language model provider
|
||||
type CommonHttpLargeLanguageModelProvider struct {
|
||||
LargeLanguageModelProvider
|
||||
provider HttpLargeLanguageModelProvider
|
||||
}
|
||||
|
||||
// GetJsonResponseByReceiptImageRecognitionModel returns the json response from the OpenAI common compatible large language model provider
|
||||
func (p *CommonHttpLargeLanguageModelProvider) GetJsonResponseByReceiptImageRecognitionModel(c core.Context, uid int64, currentConfig *settings.Config, request *LargeLanguageModelRequest) (*LargeLanguageModelTextualResponse, error) {
|
||||
return p.getTextualResponse(c, uid, currentConfig, request, p.provider.GetReceiptImageRecognitionModelID(), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
}
|
||||
|
||||
func (p *CommonHttpLargeLanguageModelProvider) getTextualResponse(c core.Context, uid int64, currentConfig *settings.Config, request *LargeLanguageModelRequest, modelId string, responseType LargeLanguageModelResponseFormat) (*LargeLanguageModelTextualResponse, error) {
|
||||
if modelId == "" {
|
||||
return nil, errs.ErrInvalidLLMModelId
|
||||
}
|
||||
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
utils.SetProxyUrl(transport, currentConfig.LargeLanguageModelAPIProxy)
|
||||
|
||||
if currentConfig.LargeLanguageModelAPISkipTLSVerify {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
}
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
Timeout: time.Duration(currentConfig.LargeLanguageModelAPIRequestTimeout) * time.Millisecond,
|
||||
}
|
||||
|
||||
httpRequest, err := p.provider.BuildTextualRequest(c, uid, request, modelId, responseType)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[http_large_language_model_provider.getTextualResponse] failed to build requests for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
httpRequest.Header.Set("User-Agent", settings.GetUserAgent())
|
||||
|
||||
resp, err := client.Do(httpRequest)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[http_large_language_model_provider.getTextualResponse] failed to request large language model api for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
|
||||
log.Debugf(c, "[http_large_language_model_provider.getTextualResponse] response is %s", body)
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
log.Errorf(c, "[http_large_language_model_provider.getTextualResponse] failed to get large language model api response for user \"uid:%d\", because response code is %d", uid, resp.StatusCode)
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
return p.provider.ParseTextualResponse(c, uid, body, responseType)
|
||||
}
|
||||
|
||||
func newCommonHttpLargeLanguageModelProvider(provider HttpLargeLanguageModelProvider) *CommonHttpLargeLanguageModelProvider {
|
||||
return &CommonHttpLargeLanguageModelProvider{
|
||||
provider: provider,
|
||||
}
|
||||
}
|
||||
33
pkg/llm/large_language_model_data.go
Normal file
33
pkg/llm/large_language_model_data.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package llm
|
||||
|
||||
import "reflect"
|
||||
|
||||
type LargeLanguageModelRequestPromptType byte
|
||||
|
||||
// Large Language Model Request Prompt Type
|
||||
const (
|
||||
LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_TEXT LargeLanguageModelRequestPromptType = 0
|
||||
LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL LargeLanguageModelRequestPromptType = 1
|
||||
)
|
||||
|
||||
type LargeLanguageModelResponseFormat byte
|
||||
|
||||
// Large Language Model Response Format
|
||||
const (
|
||||
LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_TEXT LargeLanguageModelResponseFormat = 0
|
||||
LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON LargeLanguageModelResponseFormat = 1
|
||||
)
|
||||
|
||||
// LargeLanguageModelRequest represents a request to a large language model
|
||||
type LargeLanguageModelRequest struct {
|
||||
Stream bool
|
||||
SystemPrompt string
|
||||
UserPrompt []byte
|
||||
UserPromptType LargeLanguageModelRequestPromptType
|
||||
ResponseJsonObjectType reflect.Type
|
||||
}
|
||||
|
||||
// LargeLanguageModelTextualResponse represents a textual response from a large language model
|
||||
type LargeLanguageModelTextualResponse struct {
|
||||
Content string
|
||||
}
|
||||
12
pkg/llm/large_language_model_provider.go
Normal file
12
pkg/llm/large_language_model_provider.go
Normal file
@@ -0,0 +1,12 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
)
|
||||
|
||||
// LargeLanguageModelProvider defines the structure of large language model provider
|
||||
type LargeLanguageModelProvider interface {
|
||||
// GetJsonResponseByReceiptImageRecognitionModel returns the json response from the large language model provider by receipt image recognition model
|
||||
GetJsonResponseByReceiptImageRecognitionModel(c core.Context, uid int64, currentConfig *settings.Config, request *LargeLanguageModelRequest) (*LargeLanguageModelTextualResponse, error)
|
||||
}
|
||||
45
pkg/llm/large_language_model_provider_container.go
Normal file
45
pkg/llm/large_language_model_provider_container.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
)
|
||||
|
||||
// LargeLanguageModelProviderContainer contains the current large language model provider
|
||||
type LargeLanguageModelProviderContainer struct {
|
||||
current LargeLanguageModelProvider
|
||||
}
|
||||
|
||||
// Initialize a large language model provider container singleton instance
|
||||
var (
|
||||
Container = &LargeLanguageModelProviderContainer{}
|
||||
)
|
||||
|
||||
// InitializeLargeLanguageModelProvider initializes the current large language model provider according to the config
|
||||
func InitializeLargeLanguageModelProvider(config *settings.Config) error {
|
||||
if config.LLMProvider == settings.OpenAILLMProvider {
|
||||
Container.current = NewOpenAILargeLanguageModelProvider(config)
|
||||
return nil
|
||||
} else if config.LLMProvider == settings.OpenAICompatibleLLMProvider {
|
||||
Container.current = NewOpenAICompatibleLargeLanguageModelProvider(config)
|
||||
return nil
|
||||
} else if config.LLMProvider == settings.OpenRouterLLMProvider {
|
||||
Container.current = NewOpenRouterLargeLanguageModelProvider(config)
|
||||
return nil
|
||||
} else if config.LLMProvider == settings.OllamaLLMProvider {
|
||||
Container.current = NewOllamaLargeLanguageModelProvider(config)
|
||||
return nil
|
||||
}
|
||||
|
||||
return errs.ErrInvalidLLMProvider
|
||||
}
|
||||
|
||||
// GetJsonResponseByReceiptImageRecognitionModel returns the json response from the current large language model provider by receipt image recognition model
|
||||
func (l *LargeLanguageModelProviderContainer) GetJsonResponseByReceiptImageRecognitionModel(c core.Context, uid int64, currentConfig *settings.Config, request *LargeLanguageModelRequest) (*LargeLanguageModelTextualResponse, error) {
|
||||
if Container.current == nil {
|
||||
return nil, errs.ErrInvalidLLMProvider
|
||||
}
|
||||
|
||||
return l.current.GetJsonResponseByReceiptImageRecognitionModel(c, uid, currentConfig, request)
|
||||
}
|
||||
153
pkg/llm/ollama_large_language_model_provider.go
Normal file
153
pkg/llm/ollama_large_language_model_provider.go
Normal file
@@ -0,0 +1,153 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/log"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
)
|
||||
|
||||
const ollamaChatCompletionsPath = "api/chat"
|
||||
|
||||
// OllamaLargeLanguageModelProvider defines the structure of Ollama large language model provider
|
||||
type OllamaLargeLanguageModelProvider struct {
|
||||
CommonHttpLargeLanguageModelProvider
|
||||
OllamaServerURL string
|
||||
ReceiptImageRecognitionModelID string
|
||||
}
|
||||
|
||||
// BuildTextualRequest returns the http request by Ollama provider
|
||||
func (p *OllamaLargeLanguageModelProvider) BuildTextualRequest(c core.Context, uid int64, request *LargeLanguageModelRequest, modelId string, responseType LargeLanguageModelResponseFormat) (*http.Request, error) {
|
||||
requestBody, err := p.buildJsonRequestBody(c, uid, request, modelId, responseType)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpRequest, err := http.NewRequest("POST", p.getOllamaRequestUrl(), bytes.NewReader(requestBody))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpRequest.Header.Set("Content-Type", "application/json")
|
||||
|
||||
return httpRequest, nil
|
||||
}
|
||||
|
||||
// ParseTextualResponse returns the textual response by Ollama provider
|
||||
func (p *OllamaLargeLanguageModelProvider) ParseTextualResponse(c core.Context, uid int64, body []byte, responseType LargeLanguageModelResponseFormat) (*LargeLanguageModelTextualResponse, error) {
|
||||
responseBody := make(map[string]any)
|
||||
err := json.Unmarshal(body, &responseBody)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[ollama_large_language_model_provider.ParseTextualResponse] failed to parse response for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
message, ok := responseBody["message"].(map[string]any)
|
||||
|
||||
if !ok {
|
||||
log.Errorf(c, "[ollama_large_language_model_provider.ParseTextualResponse] no message found in response for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
content, ok := message["content"].(string)
|
||||
|
||||
if !ok {
|
||||
log.Errorf(c, "[ollama_large_language_model_provider.ParseTextualResponse] no content found in message for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
if responseType == LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON {
|
||||
if strings.HasPrefix(content, "```json") && strings.HasSuffix(content, "```") {
|
||||
content = strings.TrimPrefix(content, "```json")
|
||||
content = strings.TrimSuffix(content, "```")
|
||||
} else if strings.HasPrefix(content, "```") && strings.HasSuffix(content, "```") {
|
||||
content = strings.TrimPrefix(content, "```")
|
||||
content = strings.TrimSuffix(content, "```")
|
||||
}
|
||||
}
|
||||
|
||||
textualResponse := &LargeLanguageModelTextualResponse{
|
||||
Content: content,
|
||||
}
|
||||
|
||||
return textualResponse, nil
|
||||
}
|
||||
|
||||
// GetReceiptImageRecognitionModelID returns the receipt image recognition model id of Ollama provider
|
||||
func (p *OllamaLargeLanguageModelProvider) GetReceiptImageRecognitionModelID() string {
|
||||
return p.ReceiptImageRecognitionModelID
|
||||
}
|
||||
|
||||
func (p *OllamaLargeLanguageModelProvider) buildJsonRequestBody(c core.Context, uid int64, request *LargeLanguageModelRequest, modelId string, responseType LargeLanguageModelResponseFormat) ([]byte, error) {
|
||||
requestMessages := make([]any, 0)
|
||||
|
||||
if request.SystemPrompt != "" {
|
||||
requestMessages = append(requestMessages, map[string]string{
|
||||
"role": "system",
|
||||
"content": request.SystemPrompt,
|
||||
})
|
||||
}
|
||||
|
||||
if len(request.UserPrompt) > 0 {
|
||||
imageBase64Data := base64.StdEncoding.EncodeToString(request.UserPrompt)
|
||||
if request.UserPromptType == LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL {
|
||||
requestMessages = append(requestMessages, map[string]any{
|
||||
"role": "user",
|
||||
"content": "",
|
||||
"images": []string{imageBase64Data},
|
||||
})
|
||||
} else {
|
||||
requestMessages = append(requestMessages, map[string]string{
|
||||
"role": "user",
|
||||
"content": string(request.UserPrompt),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
requestBody := make(map[string]any)
|
||||
requestBody["model"] = modelId
|
||||
requestBody["stream"] = request.Stream
|
||||
requestBody["messages"] = requestMessages
|
||||
|
||||
if responseType == LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON {
|
||||
requestBody["format"] = "json"
|
||||
}
|
||||
|
||||
requestBodyBytes, err := json.Marshal(requestBody)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[ollama_large_language_model_provider.buildJsonRequestBody] failed to marshal request body for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
log.Debugf(c, "[ollama_large_language_model_provider.buildJsonRequestBody] request body is %s", requestBodyBytes)
|
||||
return requestBodyBytes, nil
|
||||
}
|
||||
|
||||
func (p *OllamaLargeLanguageModelProvider) getOllamaRequestUrl() string {
|
||||
url := p.OllamaServerURL
|
||||
|
||||
if url[len(url)-1] != '/' {
|
||||
url += "/"
|
||||
}
|
||||
|
||||
url += ollamaChatCompletionsPath
|
||||
return url
|
||||
}
|
||||
|
||||
// NewOllamaLargeLanguageModelProvider creates a new Ollama large language model provider instance
|
||||
func NewOllamaLargeLanguageModelProvider(config *settings.Config) LargeLanguageModelProvider {
|
||||
return newCommonHttpLargeLanguageModelProvider(&OllamaLargeLanguageModelProvider{
|
||||
OllamaServerURL: config.OllamaServerURL,
|
||||
ReceiptImageRecognitionModelID: config.OllamaReceiptImageRecognitionModelID,
|
||||
})
|
||||
}
|
||||
138
pkg/llm/ollama_large_language_model_provider_test.go
Normal file
138
pkg/llm/ollama_large_language_model_provider_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
)
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_buildJsonRequestBody_TextualUserPrompt(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{}
|
||||
|
||||
request := &LargeLanguageModelRequest{
|
||||
SystemPrompt: "You are a helpful assistant.",
|
||||
UserPrompt: []byte("Hello, how are you?"),
|
||||
}
|
||||
|
||||
bodyBytes, err := provider.buildJsonRequestBody(core.NewNullContext(), 0, request, "test", LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
|
||||
var body map[string]interface{}
|
||||
err = json.Unmarshal(bodyBytes, &body)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, "{\"format\":\"json\",\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"Hello, how are you?\",\"role\":\"user\"}],\"model\":\"test\",\"stream\":false}", string(bodyBytes))
|
||||
}
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_buildJsonRequestBody_ImageUserPrompt(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{}
|
||||
|
||||
request := &LargeLanguageModelRequest{
|
||||
SystemPrompt: "What's in this image?",
|
||||
UserPrompt: []byte("fakedata"),
|
||||
UserPromptType: LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL,
|
||||
}
|
||||
|
||||
bodyBytes, err := provider.buildJsonRequestBody(core.NewNullContext(), 0, request, "test", LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
|
||||
var body map[string]interface{}
|
||||
err = json.Unmarshal(bodyBytes, &body)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, "{\"format\":\"json\",\"messages\":[{\"content\":\"What's in this image?\",\"role\":\"system\"},{\"content\":\"\",\"images\":[\"ZmFrZWRhdGE=\"],\"role\":\"user\"}],\"model\":\"test\",\"stream\":false}", string(bodyBytes))
|
||||
}
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_ParseTextualResponse_ValidJsonResponse(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{}
|
||||
|
||||
response := `{
|
||||
"model": "test",
|
||||
"created_at": "2025-09-01T01:02:03.456789Z",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "This is a test response"
|
||||
}
|
||||
}`
|
||||
|
||||
result, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "This is a test response", result.Content)
|
||||
}
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_ParseTextualResponse_EmptyResponse(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{}
|
||||
|
||||
response := `{
|
||||
"model": "test",
|
||||
"created_at": "2025-09-01T01:02:03.456789Z",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": ""
|
||||
}
|
||||
}`
|
||||
|
||||
result, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "", result.Content)
|
||||
}
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_ParseTextualResponse_EmptyChoices(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{}
|
||||
|
||||
response := `{
|
||||
"model": "test",
|
||||
"created_at": "2025-09-01T01:02:03.456789Z",
|
||||
"message": {}
|
||||
}`
|
||||
|
||||
_, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.EqualError(t, err, "failed to request third party api")
|
||||
}
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_ParseTextualResponse_NoChoiceContent(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{}
|
||||
|
||||
response := `{
|
||||
"model": "test",
|
||||
"created_at": "2025-09-01T01:02:03.456789Z",
|
||||
"message": {
|
||||
"role": "assistant"
|
||||
}
|
||||
}`
|
||||
|
||||
_, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.EqualError(t, err, "failed to request third party api")
|
||||
}
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_ParseTextualResponse_InvalidJson(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{}
|
||||
|
||||
response := "error"
|
||||
|
||||
_, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.EqualError(t, err, "failed to request third party api")
|
||||
}
|
||||
|
||||
func TestOllamaLargeLanguageModelProvider_GetOllamaRequestUrl(t *testing.T) {
|
||||
provider := &OllamaLargeLanguageModelProvider{
|
||||
OllamaServerURL: "http://localhost:11434/",
|
||||
}
|
||||
url := provider.getOllamaRequestUrl()
|
||||
assert.Equal(t, "http://localhost:11434/api/chat", url)
|
||||
|
||||
provider = &OllamaLargeLanguageModelProvider{
|
||||
OllamaServerURL: "http://localhost:11434",
|
||||
}
|
||||
url = provider.getOllamaRequestUrl()
|
||||
assert.Equal(t, "http://localhost:11434/api/chat", url)
|
||||
|
||||
provider = &OllamaLargeLanguageModelProvider{
|
||||
OllamaServerURL: "http://example.com/ollama/",
|
||||
}
|
||||
url = provider.getOllamaRequestUrl()
|
||||
assert.Equal(t, "http://example.com/ollama/api/chat", url)
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/invopop/jsonschema"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/errs"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/log"
|
||||
)
|
||||
|
||||
// OpenAIChatCompletionsLargeLanguageModelProvider defines the structure of OpenAI chat completions compatible large language model provider
|
||||
type OpenAIChatCompletionsLargeLanguageModelProvider interface {
|
||||
// BuildChatCompletionsHttpRequest returns the chat completions http request
|
||||
BuildChatCompletionsHttpRequest(c core.Context, uid int64) (*http.Request, error)
|
||||
|
||||
// GetReceiptImageRecognitionModelID returns the receipt image recognition model id if supported, otherwise returns empty string
|
||||
GetReceiptImageRecognitionModelID() string
|
||||
}
|
||||
|
||||
// OpenAICommonChatCompletionsHttpLargeLanguageModelProvider defines the structure of OpenAI common compatible large language model provider based on chat completions api
|
||||
type OpenAICommonChatCompletionsHttpLargeLanguageModelProvider struct {
|
||||
CommonHttpLargeLanguageModelProvider
|
||||
provider OpenAIChatCompletionsLargeLanguageModelProvider
|
||||
}
|
||||
|
||||
// BuildTextualRequest returns the http request by OpenAI common compatible provider
|
||||
func (p *OpenAICommonChatCompletionsHttpLargeLanguageModelProvider) BuildTextualRequest(c core.Context, uid int64, request *LargeLanguageModelRequest, modelId string, responseType LargeLanguageModelResponseFormat) (*http.Request, error) {
|
||||
requestBody, err := p.buildJsonRequestBody(c, uid, request, modelId, responseType)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpRequest, err := p.provider.BuildChatCompletionsHttpRequest(c, uid)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
httpRequest.Body = io.NopCloser(bytes.NewReader(requestBody))
|
||||
httpRequest.Header.Set("Content-Type", "application/json")
|
||||
|
||||
return httpRequest, nil
|
||||
}
|
||||
|
||||
// ParseTextualResponse returns the textual response by OpenAI common compatible provider
|
||||
func (p *OpenAICommonChatCompletionsHttpLargeLanguageModelProvider) ParseTextualResponse(c core.Context, uid int64, body []byte, responseType LargeLanguageModelResponseFormat) (*LargeLanguageModelTextualResponse, error) {
|
||||
responseBody := make(map[string]any)
|
||||
err := json.Unmarshal(body, &responseBody)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[openai_common_compatible_large_language_model_provider.ParseTextualResponse] failed to parse response for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
choices, ok := responseBody["choices"].([]any)
|
||||
|
||||
if !ok || len(choices) < 1 {
|
||||
log.Errorf(c, "[openai_common_compatible_large_language_model_provider.ParseTextualResponse] no choices found in response for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
firstChoice, ok := choices[0].(map[string]any)
|
||||
|
||||
if !ok {
|
||||
log.Errorf(c, "[openai_common_compatible_large_language_model_provider.ParseTextualResponse] invalid choice format in response for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
message, ok := firstChoice["message"].(map[string]any)
|
||||
|
||||
if !ok {
|
||||
log.Errorf(c, "[openai_common_compatible_large_language_model_provider.ParseTextualResponse] no message found in choice for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
content, ok := message["content"].(string)
|
||||
|
||||
if !ok {
|
||||
log.Errorf(c, "[openai_common_compatible_large_language_model_provider.ParseTextualResponse] no content found in message for user \"uid:%d\"", uid)
|
||||
return nil, errs.ErrFailedToRequestRemoteApi
|
||||
}
|
||||
|
||||
if responseType == LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON {
|
||||
if strings.HasPrefix(content, "```json") && strings.HasSuffix(content, "```") {
|
||||
content = strings.TrimPrefix(content, "```json")
|
||||
content = strings.TrimSuffix(content, "```")
|
||||
} else if strings.HasPrefix(content, "```") && strings.HasSuffix(content, "```") {
|
||||
content = strings.TrimPrefix(content, "```")
|
||||
content = strings.TrimSuffix(content, "```")
|
||||
}
|
||||
}
|
||||
|
||||
textualResponse := &LargeLanguageModelTextualResponse{
|
||||
Content: content,
|
||||
}
|
||||
|
||||
return textualResponse, nil
|
||||
}
|
||||
|
||||
// GetReceiptImageRecognitionModelID returns the receipt image recognition model id of OpenAI common compatible provider
|
||||
func (p *OpenAICommonChatCompletionsHttpLargeLanguageModelProvider) GetReceiptImageRecognitionModelID() string {
|
||||
return p.provider.GetReceiptImageRecognitionModelID()
|
||||
}
|
||||
|
||||
func (p *OpenAICommonChatCompletionsHttpLargeLanguageModelProvider) buildJsonRequestBody(c core.Context, uid int64, request *LargeLanguageModelRequest, modelId string, responseType LargeLanguageModelResponseFormat) ([]byte, error) {
|
||||
requestMessages := make([]any, 0)
|
||||
|
||||
if request.SystemPrompt != "" {
|
||||
requestMessages = append(requestMessages, map[string]string{
|
||||
"role": "system",
|
||||
"content": request.SystemPrompt,
|
||||
})
|
||||
}
|
||||
|
||||
if len(request.UserPrompt) > 0 {
|
||||
if request.UserPromptType == LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL {
|
||||
imageBase64Data := "data:image/png;base64," + base64.StdEncoding.EncodeToString(request.UserPrompt)
|
||||
requestMessages = append(requestMessages, map[string]any{
|
||||
"role": "user",
|
||||
"content": []any{
|
||||
core.O{
|
||||
"type": "image_url",
|
||||
"image_url": core.O{
|
||||
"url": imageBase64Data,
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
} else {
|
||||
requestMessages = append(requestMessages, map[string]string{
|
||||
"role": "user",
|
||||
"content": string(request.UserPrompt),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
requestBody := make(map[string]any)
|
||||
requestBody["model"] = modelId
|
||||
requestBody["stream"] = request.Stream
|
||||
requestBody["messages"] = requestMessages
|
||||
|
||||
if responseType == LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON {
|
||||
if request.ResponseJsonObjectType != nil {
|
||||
schemeGenerator := jsonschema.Reflector{
|
||||
Anonymous: true,
|
||||
DoNotReference: true,
|
||||
ExpandedStruct: true,
|
||||
}
|
||||
|
||||
schema := schemeGenerator.ReflectFromType(request.ResponseJsonObjectType)
|
||||
schema.Version = ""
|
||||
|
||||
requestBody["response_format"] = core.O{
|
||||
"type": "json_schema",
|
||||
"json_schema": schema,
|
||||
}
|
||||
} else {
|
||||
requestBody["response_format"] = core.O{
|
||||
"type": "json_object",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
requestBodyBytes, err := json.Marshal(requestBody)
|
||||
|
||||
if err != nil {
|
||||
log.Errorf(c, "[openai_common_compatible_large_language_model_provider.buildJsonRequestBody] failed to marshal request body for user \"uid:%d\", because %s", uid, err.Error())
|
||||
return nil, errs.ErrOperationFailed
|
||||
}
|
||||
|
||||
log.Debugf(c, "[openai_common_compatible_large_language_model_provider.buildJsonRequestBody] request body is %s", requestBodyBytes)
|
||||
return requestBodyBytes, nil
|
||||
}
|
||||
|
||||
func newOpenAICommonChatCompletionsHttpLargeLanguageModelProvider(provider OpenAIChatCompletionsLargeLanguageModelProvider) LargeLanguageModelProvider {
|
||||
return newCommonHttpLargeLanguageModelProvider(&OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: provider,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
)
|
||||
|
||||
func TestOpenAICommonChatCompletionsHttpLargeLanguageModelProvider_buildJsonRequestBody_TextualUserPrompt(t *testing.T) {
|
||||
provider := &OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: &OpenAILargeLanguageModelProvider{},
|
||||
}
|
||||
|
||||
request := &LargeLanguageModelRequest{
|
||||
SystemPrompt: "You are a helpful assistant.",
|
||||
UserPrompt: []byte("Hello, how are you?"),
|
||||
}
|
||||
|
||||
bodyBytes, err := provider.buildJsonRequestBody(core.NewNullContext(), 0, request, "test", LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
|
||||
var body map[string]interface{}
|
||||
err = json.Unmarshal(bodyBytes, &body)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, "{\"messages\":[{\"content\":\"You are a helpful assistant.\",\"role\":\"system\"},{\"content\":\"Hello, how are you?\",\"role\":\"user\"}],\"model\":\"test\",\"response_format\":{\"type\":\"json_object\"},\"stream\":false}", string(bodyBytes))
|
||||
}
|
||||
|
||||
func TestOpenAICommonChatCompletionsHttpLargeLanguageModelProvider_buildJsonRequestBody_ImageUserPrompt(t *testing.T) {
|
||||
provider := &OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: &OpenAILargeLanguageModelProvider{},
|
||||
}
|
||||
|
||||
request := &LargeLanguageModelRequest{
|
||||
SystemPrompt: "What's in this image?",
|
||||
UserPrompt: []byte("fakedata"),
|
||||
UserPromptType: LARGE_LANGUAGE_MODEL_REQUEST_PROMPT_TYPE_IMAGE_URL,
|
||||
}
|
||||
|
||||
bodyBytes, err := provider.buildJsonRequestBody(core.NewNullContext(), 0, request, "test", LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
|
||||
var body map[string]interface{}
|
||||
err = json.Unmarshal(bodyBytes, &body)
|
||||
assert.Nil(t, err)
|
||||
|
||||
assert.Equal(t, "{\"messages\":[{\"content\":\"What's in this image?\",\"role\":\"system\"},{\"content\":[{\"image_url\":{\"url\":\"data:image/png;base64,ZmFrZWRhdGE=\"},\"type\":\"image_url\"}],\"role\":\"user\"}],\"model\":\"test\",\"response_format\":{\"type\":\"json_object\"},\"stream\":false}", string(bodyBytes))
|
||||
}
|
||||
|
||||
func TestOpenAICommonChatCompletionsHttpLargeLanguageModelProvider_ParseTextualResponse_ValidJsonResponse(t *testing.T) {
|
||||
provider := &OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: &OpenAILargeLanguageModelProvider{},
|
||||
}
|
||||
|
||||
response := `{
|
||||
"id": "test-123",
|
||||
"object": "chat.completion",
|
||||
"created": 1234567890,
|
||||
"model": "test",
|
||||
"usage": {
|
||||
"prompt_tokens": 13,
|
||||
"completion_tokens": 7,
|
||||
"total_tokens": 20
|
||||
},
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "This is a test response"
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
result, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "This is a test response", result.Content)
|
||||
}
|
||||
|
||||
func TestOpenAICommonChatCompletionsHttpLargeLanguageModelProvider_ParseTextualResponse_EmptyResponse(t *testing.T) {
|
||||
provider := &OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: &OpenAILargeLanguageModelProvider{},
|
||||
}
|
||||
|
||||
response := `{
|
||||
"id": "test-123",
|
||||
"object": "chat.completion",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": ""
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
result, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, "", result.Content)
|
||||
}
|
||||
|
||||
func TestOpenAICommonChatCompletionsHttpLargeLanguageModelProvider_ParseTextualResponse_EmptyChoices(t *testing.T) {
|
||||
provider := &OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: &OpenAILargeLanguageModelProvider{},
|
||||
}
|
||||
|
||||
response := `{
|
||||
"id": "test-123",
|
||||
"object": "chat.completion",
|
||||
"choices": []
|
||||
}`
|
||||
|
||||
_, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.EqualError(t, err, "failed to request third party api")
|
||||
}
|
||||
|
||||
func TestOpenAICommonChatCompletionsHttpLargeLanguageModelProvider_ParseTextualResponse_NoChoiceContent(t *testing.T) {
|
||||
provider := &OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: &OpenAILargeLanguageModelProvider{},
|
||||
}
|
||||
|
||||
response := `{
|
||||
"id": "chatcmpl-123",
|
||||
"object": "chat.completion",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant"
|
||||
}
|
||||
}
|
||||
]
|
||||
}`
|
||||
|
||||
_, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.EqualError(t, err, "failed to request third party api")
|
||||
}
|
||||
|
||||
func TestOpenAICommonChatCompletionsHttpLargeLanguageModelProvider_ParseTextualResponse_InvalidJson(t *testing.T) {
|
||||
provider := &OpenAICommonChatCompletionsHttpLargeLanguageModelProvider{
|
||||
provider: &OpenAILargeLanguageModelProvider{},
|
||||
}
|
||||
|
||||
response := "error"
|
||||
|
||||
_, err := provider.ParseTextualResponse(core.NewNullContext(), 0, []byte(response), LARGE_LANGUAGE_MODEL_RESPONSE_FORMAT_JSON)
|
||||
assert.EqualError(t, err, "failed to request third party api")
|
||||
}
|
||||
58
pkg/llm/openai_compatible_large_language_model_provider.go
Normal file
58
pkg/llm/openai_compatible_large_language_model_provider.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
)
|
||||
|
||||
const openAICompatibleChatCompletionsPath = "chat/completions"
|
||||
|
||||
// OpenAICompatibleLargeLanguageModelProvider defines the structure of OpenAI compatible large language model provider
|
||||
type OpenAICompatibleLargeLanguageModelProvider struct {
|
||||
OpenAIChatCompletionsLargeLanguageModelProvider
|
||||
OpenAICompatibleBaseURL string
|
||||
OpenAICompatibleAPIKey string
|
||||
ReceiptImageRecognitionModelID string
|
||||
}
|
||||
|
||||
// BuildChatCompletionsHttpRequest returns the chat completions http request by OpenAI compatible provider
|
||||
func (p *OpenAICompatibleLargeLanguageModelProvider) BuildChatCompletionsHttpRequest(c core.Context, uid int64) (*http.Request, error) {
|
||||
req, err := http.NewRequest("POST", p.getFinalChatCompletionsRequestUrl(), nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if p.OpenAICompatibleAPIKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+p.OpenAICompatibleAPIKey)
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// GetReceiptImageRecognitionModelID returns the receipt image recognition model id of OpenAI compatible provider
|
||||
func (p *OpenAICompatibleLargeLanguageModelProvider) GetReceiptImageRecognitionModelID() string {
|
||||
return p.ReceiptImageRecognitionModelID
|
||||
}
|
||||
|
||||
func (p *OpenAICompatibleLargeLanguageModelProvider) getFinalChatCompletionsRequestUrl() string {
|
||||
url := p.OpenAICompatibleBaseURL
|
||||
|
||||
if url[len(url)-1] != '/' {
|
||||
url += "/"
|
||||
}
|
||||
|
||||
url += openAICompatibleChatCompletionsPath
|
||||
return url
|
||||
}
|
||||
|
||||
// NewOpenAICompatibleLargeLanguageModelProvider creates a new OpenAI compatible large language model provider instance
|
||||
func NewOpenAICompatibleLargeLanguageModelProvider(config *settings.Config) LargeLanguageModelProvider {
|
||||
return newOpenAICommonChatCompletionsHttpLargeLanguageModelProvider(&OpenAICompatibleLargeLanguageModelProvider{
|
||||
OpenAICompatibleBaseURL: config.OpenAICompatibleBaseURL,
|
||||
OpenAICompatibleAPIKey: config.OpenAICompatibleAPIKey,
|
||||
ReceiptImageRecognitionModelID: config.OpenAICompatibleReceiptImageRecognitionModelID,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestOpenAICompatibleLargeLanguageModelProvider_GetFinalRequestUrl(t *testing.T) {
|
||||
provider := &OpenAICompatibleLargeLanguageModelProvider{
|
||||
OpenAICompatibleBaseURL: "https://api.example.com/v1/",
|
||||
}
|
||||
url := provider.getFinalChatCompletionsRequestUrl()
|
||||
assert.Equal(t, "https://api.example.com/v1/chat/completions", url)
|
||||
|
||||
provider = &OpenAICompatibleLargeLanguageModelProvider{
|
||||
OpenAICompatibleBaseURL: "https://api.example.com/v1",
|
||||
}
|
||||
url = provider.getFinalChatCompletionsRequestUrl()
|
||||
assert.Equal(t, "https://api.example.com/v1/chat/completions", url)
|
||||
|
||||
provider = &OpenAICompatibleLargeLanguageModelProvider{
|
||||
OpenAICompatibleBaseURL: "https://example.com/api",
|
||||
}
|
||||
url = provider.getFinalChatCompletionsRequestUrl()
|
||||
assert.Equal(t, "https://example.com/api/chat/completions", url)
|
||||
}
|
||||
43
pkg/llm/openai_large_language_model_provider.go
Normal file
43
pkg/llm/openai_large_language_model_provider.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
)
|
||||
|
||||
// OpenAILargeLanguageModelProvider defines the structure of OpenAI large language model provider
|
||||
type OpenAILargeLanguageModelProvider struct {
|
||||
OpenAIChatCompletionsLargeLanguageModelProvider
|
||||
OpenAIAPIKey string
|
||||
ReceiptImageRecognitionModelID string
|
||||
}
|
||||
|
||||
const openAIChatCompletionsUrl = "https://api.openai.com/v1/chat/completions"
|
||||
|
||||
// BuildChatCompletionsHttpRequest returns the chat completions http request by OpenAI provider
|
||||
func (p *OpenAILargeLanguageModelProvider) BuildChatCompletionsHttpRequest(c core.Context, uid int64) (*http.Request, error) {
|
||||
req, err := http.NewRequest("POST", openAIChatCompletionsUrl, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+p.OpenAIAPIKey)
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// GetReceiptImageRecognitionModelID returns the receipt image recognition model id of OpenAI provider
|
||||
func (p *OpenAILargeLanguageModelProvider) GetReceiptImageRecognitionModelID() string {
|
||||
return p.ReceiptImageRecognitionModelID
|
||||
}
|
||||
|
||||
// NewOpenAILargeLanguageModelProvider creates a new OpenAI large language model provider instance
|
||||
func NewOpenAILargeLanguageModelProvider(config *settings.Config) LargeLanguageModelProvider {
|
||||
return newOpenAICommonChatCompletionsHttpLargeLanguageModelProvider(&OpenAILargeLanguageModelProvider{
|
||||
OpenAIAPIKey: config.OpenAIAPIKey,
|
||||
ReceiptImageRecognitionModelID: config.OpenAIReceiptImageRecognitionModelID,
|
||||
})
|
||||
}
|
||||
45
pkg/llm/openrouter_large_language_model_provider.go
Normal file
45
pkg/llm/openrouter_large_language_model_provider.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package llm
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/mayswind/ezbookkeeping/pkg/core"
|
||||
"github.com/mayswind/ezbookkeeping/pkg/settings"
|
||||
)
|
||||
|
||||
// OpenRouterLargeLanguageModelProvider defines the structure of OpenRouter large language model provider
|
||||
type OpenRouterLargeLanguageModelProvider struct {
|
||||
OpenAIChatCompletionsLargeLanguageModelProvider
|
||||
OpenRouterAPIKey string
|
||||
ReceiptImageRecognitionModelID string
|
||||
}
|
||||
|
||||
const openRouterChatCompletionsUrl = "https://openrouter.ai/api/v1/chat/completions"
|
||||
|
||||
// BuildChatCompletionsHttpRequest returns the chat completions http request by OpenRouter provider
|
||||
func (p *OpenRouterLargeLanguageModelProvider) BuildChatCompletionsHttpRequest(c core.Context, uid int64) (*http.Request, error) {
|
||||
req, err := http.NewRequest("POST", openRouterChatCompletionsUrl, nil)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer "+p.OpenRouterAPIKey)
|
||||
req.Header.Set("HTTP-Referer", "https://ezbookkeeping.mayswind.net/")
|
||||
req.Header.Set("X-Title", "ezBookkeeping")
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// GetReceiptImageRecognitionModelID returns the receipt image recognition model id of OpenRouter provider
|
||||
func (p *OpenRouterLargeLanguageModelProvider) GetReceiptImageRecognitionModelID() string {
|
||||
return p.ReceiptImageRecognitionModelID
|
||||
}
|
||||
|
||||
// NewOpenRouterLargeLanguageModelProvider creates a new OpenRouter large language model provider instance
|
||||
func NewOpenRouterLargeLanguageModelProvider(config *settings.Config) LargeLanguageModelProvider {
|
||||
return newOpenAICommonChatCompletionsHttpLargeLanguageModelProvider(&OpenRouterLargeLanguageModelProvider{
|
||||
OpenRouterAPIKey: config.OpenRouterAPIKey,
|
||||
ReceiptImageRecognitionModelID: config.OpenRouterReceiptImageRecognitionModelID,
|
||||
})
|
||||
}
|
||||
27
pkg/models/large_language_model.go
Normal file
27
pkg/models/large_language_model.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package models
|
||||
|
||||
// RecognizedReceiptImageResponse represents a view-object of recognized receipt image response
|
||||
type RecognizedReceiptImageResponse struct {
|
||||
Type TransactionType `json:"type"`
|
||||
Time int64 `json:"time,omitempty"`
|
||||
CategoryId int64 `json:"categoryId,string,omitempty"`
|
||||
SourceAccountId int64 `json:"sourceAccountId,string,omitempty"`
|
||||
DestinationAccountId int64 `json:"destinationAccountId,string,omitempty"`
|
||||
SourceAmount int64 `json:"sourceAmount,omitempty"`
|
||||
DestinationAmount int64 `json:"destinationAmount,omitempty"`
|
||||
TagIds []string `json:"tagIds,omitempty"`
|
||||
Comment string `json:"comment,omitempty"`
|
||||
}
|
||||
|
||||
// RecognizedReceiptImageResult represents the result of recognized receipt image
|
||||
type RecognizedReceiptImageResult struct {
|
||||
Type string `json:"type,omitempty" jsonschema:"enum=income,enum=expense,enum=transfer" jsonschema_description:"Transaction type (income, expense, transfer)"`
|
||||
Time string `json:"time" jsonschema:"format=date-time" jsonschema_description:"Transaction time in long date time format (YYYY-MM-DD HH:mm:ss, e.g. 2023-01-01 12:00:00)"`
|
||||
Amount string `json:"amount,omitempty" jsonschema_description:"Transaction amount"`
|
||||
AccountName string `json:"account,omitempty" jsonschema_description:"Account name for the transaction"`
|
||||
CategoryName string `json:"category,omitempty" jsonschema_description:"Category name for the transaction"`
|
||||
TagNames []string `json:"tags,omitempty" jsonschema_description:"List of tags associated with the transaction (maximum 10 tags allowed)"`
|
||||
Description string `json:"description,omitempty" jsonschema_description:"Transaction description"`
|
||||
DestinationAmount string `json:"destination_amount,omitempty" jsonschema_description:"Destination amount for transfer transactions"`
|
||||
DestinationAccountName string `json:"destination_account,omitempty" jsonschema_description:"Destination account name for transfer transactions"`
|
||||
}
|
||||
@@ -66,6 +66,13 @@ const (
|
||||
WebDAVStorageType string = "webdav"
|
||||
)
|
||||
|
||||
const (
|
||||
OpenAILLMProvider string = "openai"
|
||||
OpenAICompatibleLLMProvider string = "openai_compatible"
|
||||
OpenRouterLLMProvider string = "openrouter"
|
||||
OllamaLLMProvider string = "ollama"
|
||||
)
|
||||
|
||||
// Uuid generator types
|
||||
const (
|
||||
InternalUuidGeneratorType string = "internal"
|
||||
@@ -140,6 +147,9 @@ const (
|
||||
|
||||
defaultWebDAVRequestTimeout uint32 = 10000 // 10 seconds
|
||||
|
||||
defaultAIRecognitionPictureMaxSize uint32 = 10485760 // 10MB
|
||||
defaultLargeLanguageModelAPIRequestTimeout uint32 = 60000 // 60 seconds
|
||||
|
||||
defaultInMemoryDuplicateCheckerCleanupInterval uint32 = 60 // 1 minutes
|
||||
defaultDuplicateSubmissionsInterval uint32 = 300 // 5 minutes
|
||||
|
||||
@@ -281,6 +291,23 @@ type Config struct {
|
||||
MinIOConfig *MinIOConfig
|
||||
WebDAVConfig *WebDAVConfig
|
||||
|
||||
// Large Language Model
|
||||
LLMProvider string
|
||||
OpenAIAPIKey string
|
||||
OpenAIReceiptImageRecognitionModelID string
|
||||
OpenAICompatibleBaseURL string
|
||||
OpenAICompatibleAPIKey string
|
||||
OpenAICompatibleReceiptImageRecognitionModelID string
|
||||
OpenRouterAPIKey string
|
||||
OpenRouterReceiptImageRecognitionModelID string
|
||||
OllamaServerURL string
|
||||
OllamaReceiptImageRecognitionModelID string
|
||||
TransactionFromAIImageRecognition bool
|
||||
MaxAIRecognitionPictureFileSize uint32
|
||||
LargeLanguageModelAPIRequestTimeout uint32
|
||||
LargeLanguageModelAPIProxy string
|
||||
LargeLanguageModelAPISkipTLSVerify bool
|
||||
|
||||
// Uuid
|
||||
UuidGeneratorType string
|
||||
UuidServerId uint8
|
||||
@@ -426,6 +453,12 @@ func LoadConfiguration(configFilePath string) (*Config, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = loadLLMConfiguration(config, cfgFile, "llm")
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = loadUuidConfiguration(config, cfgFile, "uuid")
|
||||
|
||||
if err != nil {
|
||||
@@ -751,6 +784,46 @@ func loadStorageConfiguration(config *Config, configFile *ini.File, sectionName
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadLLMConfiguration(config *Config, configFile *ini.File, sectionName string) error {
|
||||
llmProvider := getConfigItemStringValue(configFile, sectionName, "llm_provider")
|
||||
|
||||
if llmProvider == "" {
|
||||
config.LLMProvider = ""
|
||||
} else if llmProvider == OpenAILLMProvider {
|
||||
config.LLMProvider = OpenAILLMProvider
|
||||
} else if llmProvider == OpenAICompatibleLLMProvider {
|
||||
config.LLMProvider = OpenAICompatibleLLMProvider
|
||||
} else if llmProvider == OpenRouterLLMProvider {
|
||||
config.LLMProvider = OpenRouterLLMProvider
|
||||
} else if llmProvider == OllamaLLMProvider {
|
||||
config.LLMProvider = OllamaLLMProvider
|
||||
} else {
|
||||
return errs.ErrInvalidLLMProvider
|
||||
}
|
||||
|
||||
config.OpenAIAPIKey = getConfigItemStringValue(configFile, sectionName, "openai_api_key")
|
||||
config.OpenAIReceiptImageRecognitionModelID = getConfigItemStringValue(configFile, sectionName, "openai_receipt_image_recognition_model_id")
|
||||
|
||||
config.OpenAICompatibleBaseURL = getConfigItemStringValue(configFile, sectionName, "openai_compatible_base_url")
|
||||
config.OpenAICompatibleAPIKey = getConfigItemStringValue(configFile, sectionName, "openai_compatible_api_key")
|
||||
config.OpenAICompatibleReceiptImageRecognitionModelID = getConfigItemStringValue(configFile, sectionName, "openai_compatible_receipt_image_recognition_model_id")
|
||||
|
||||
config.OpenRouterAPIKey = getConfigItemStringValue(configFile, sectionName, "openrouter_api_key")
|
||||
config.OpenRouterReceiptImageRecognitionModelID = getConfigItemStringValue(configFile, sectionName, "openrouter_receipt_image_recognition_model_id")
|
||||
|
||||
config.OllamaServerURL = getConfigItemStringValue(configFile, sectionName, "ollama_server_url")
|
||||
config.OllamaReceiptImageRecognitionModelID = getConfigItemStringValue(configFile, sectionName, "ollama_receipt_image_recognition_model_id")
|
||||
|
||||
config.TransactionFromAIImageRecognition = getConfigItemBoolValue(configFile, sectionName, "transaction_from_ai_image_recognition", false)
|
||||
config.MaxAIRecognitionPictureFileSize = getConfigItemUint32Value(configFile, sectionName, "max_ai_recognition_picture_size", defaultAIRecognitionPictureMaxSize)
|
||||
|
||||
config.LargeLanguageModelAPIProxy = getConfigItemStringValue(configFile, sectionName, "proxy", "system")
|
||||
config.LargeLanguageModelAPIRequestTimeout = getConfigItemUint32Value(configFile, sectionName, "request_timeout", defaultLargeLanguageModelAPIRequestTimeout)
|
||||
config.LargeLanguageModelAPISkipTLSVerify = getConfigItemBoolValue(configFile, sectionName, "skip_tls_verify", false)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadUuidConfiguration(config *Config, configFile *ini.File, sectionName string) error {
|
||||
if getConfigItemStringValue(configFile, sectionName, "generator_type") == InternalUuidGeneratorType {
|
||||
config.UuidGeneratorType = InternalUuidGeneratorType
|
||||
|
||||
@@ -4,6 +4,7 @@ type KnownTemplate string
|
||||
|
||||
// Known templates
|
||||
const (
|
||||
TEMPLATE_VERIFY_EMAIL KnownTemplate = "email/verify_email"
|
||||
TEMPLATE_PASSWORD_RESET KnownTemplate = "email/password_reset"
|
||||
TEMPLATE_VERIFY_EMAIL KnownTemplate = "email/verify_email"
|
||||
TEMPLATE_PASSWORD_RESET KnownTemplate = "email/password_reset"
|
||||
SYSTEM_PROMPT_RECEIPT_IMAGE_RECOGNITION KnownTemplate = "prompt/receipt_image_recognition"
|
||||
)
|
||||
|
||||
181
src/components/mobile/AIImageRecognitionSheet.vue
Normal file
181
src/components/mobile/AIImageRecognitionSheet.vue
Normal file
@@ -0,0 +1,181 @@
|
||||
<template>
|
||||
<f7-sheet swipe-to-close swipe-handler=".swipe-handler" style="height:auto"
|
||||
:opened="show" @sheet:open="onSheetOpen" @sheet:closed="onSheetClosed">
|
||||
<f7-toolbar>
|
||||
<div class="swipe-handler"></div>
|
||||
<div class="left">
|
||||
<f7-link :class="{ 'disabled': loading || recognizing }" :text="tt('Choose from Library')" @click="showOpenImage"></f7-link>
|
||||
</div>
|
||||
<div class="right">
|
||||
<f7-link :class="{ 'disabled': loading || recognizing }" :text="tt('Take Photo')" @click="showCamera"></f7-link>
|
||||
</div>
|
||||
</f7-toolbar>
|
||||
<f7-page-content class="margin-top no-padding-top">
|
||||
<div class="padding-horizontal padding-bottom">
|
||||
<div class="image-container display-flex justify-content-center width-100 margin-bottom" style="height: 240px">
|
||||
<img height="240px" :src="imageSrc" v-if="imageSrc" />
|
||||
<div class="image-container-background display-flex justify-content-center align-items-center" v-if="!imageSrc">
|
||||
<span>{{ tt('Please select a receipt or transaction image first') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
<f7-button large fill color="primary"
|
||||
:class="{ 'disabled': loading || recognizing || !imageFile }"
|
||||
:text="tt('Recognize')"
|
||||
@click="confirm">
|
||||
</f7-button>
|
||||
<div class="margin-top text-align-center">
|
||||
<f7-link :class="{ 'disabled': loading || recognizing }" @click="cancel" :text="tt('Cancel')"></f7-link>
|
||||
</div>
|
||||
</div>
|
||||
</f7-page-content>
|
||||
|
||||
<input ref="imageInput" type="file" style="display: none" :accept="SUPPORTED_IMAGE_EXTENSIONS" @change="openImage($event)" />
|
||||
<input ref="cameraInput" type="file" style="display: none" :accept="SUPPORTED_IMAGE_EXTENSIONS" capture="environment" @change="openImage($event)" />
|
||||
</f7-sheet>
|
||||
</template>
|
||||
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, useTemplateRef } from 'vue';
|
||||
|
||||
import { useI18n } from '@/locales/helpers.ts';
|
||||
import { useI18nUIComponents, showLoading, hideLoading } from '@/lib/ui/mobile.ts';
|
||||
|
||||
import { useTransactionsStore } from '@/stores/transaction.ts';
|
||||
|
||||
import { KnownFileType } from '@/core/file.ts';
|
||||
import { SUPPORTED_IMAGE_EXTENSIONS } from '@/consts/file.ts';
|
||||
|
||||
import type { RecognizedReceiptImageResponse } from '@/models/large_language_model.ts';
|
||||
|
||||
import { compressJpgImage } from '@/lib/ui/common.ts';
|
||||
import logger from '@/lib/logger.ts';
|
||||
|
||||
defineProps<{
|
||||
show: boolean;
|
||||
}>();
|
||||
|
||||
const emit = defineEmits<{
|
||||
(e: 'update:show', value: boolean): void;
|
||||
(e: 'recognition:change', value: RecognizedReceiptImageResponse): void;
|
||||
}>();
|
||||
|
||||
const { tt } = useI18n();
|
||||
const { showToast } = useI18nUIComponents();
|
||||
|
||||
const transactionsStore = useTransactionsStore();
|
||||
|
||||
const imageInput = useTemplateRef<HTMLInputElement>('imageInput');
|
||||
const cameraInput = useTemplateRef<HTMLInputElement>('cameraInput');
|
||||
|
||||
const loading = ref<boolean>(false);
|
||||
const recognizing = ref<boolean>(false);
|
||||
const imageFile = ref<File | null>(null);
|
||||
const imageSrc = ref<string | undefined>(undefined);
|
||||
|
||||
function loadImage(file: File): void {
|
||||
compressJpgImage(file, 1280, 1280, 0.8).then(blob => {
|
||||
imageFile.value = KnownFileType.JPG.createFileFromBlob(blob, "image");
|
||||
imageSrc.value = URL.createObjectURL(blob);
|
||||
}).catch(error => {
|
||||
imageFile.value = null;
|
||||
imageSrc.value = undefined;
|
||||
logger.error('failed to compress image', error);
|
||||
showToast('Unable to load image');
|
||||
});
|
||||
}
|
||||
|
||||
function showOpenImage(): void {
|
||||
if (loading.value || recognizing.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
imageInput.value?.click();
|
||||
}
|
||||
|
||||
function showCamera(): void {
|
||||
if (loading.value || recognizing.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
cameraInput.value?.click();
|
||||
}
|
||||
|
||||
function openImage(event: Event): void {
|
||||
if (!event || !event.target) {
|
||||
return;
|
||||
}
|
||||
|
||||
const el = event.target as HTMLInputElement;
|
||||
|
||||
if (!el.files || !el.files.length || !el.files[0]) {
|
||||
return;
|
||||
}
|
||||
|
||||
const image = el.files[0] as File;
|
||||
|
||||
el.value = '';
|
||||
|
||||
loadImage(image);
|
||||
}
|
||||
|
||||
function confirm(): void {
|
||||
if (loading.value || recognizing.value || !imageFile.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
recognizing.value = true;
|
||||
showLoading(() => recognizing.value);
|
||||
|
||||
transactionsStore.recognizeReceiptImage({
|
||||
imageFile: imageFile.value
|
||||
}).then(response => {
|
||||
recognizing.value = false;
|
||||
hideLoading();
|
||||
emit('update:show', false);
|
||||
emit('recognition:change', response);
|
||||
}).catch(error => {
|
||||
recognizing.value = false;
|
||||
hideLoading();
|
||||
|
||||
if (!error.processed) {
|
||||
showToast(error.message || error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function cancel(): void {
|
||||
close();
|
||||
}
|
||||
|
||||
function close(): void {
|
||||
emit('update:show', false);
|
||||
loading.value = false;
|
||||
recognizing.value = false;
|
||||
imageFile.value = null;
|
||||
imageSrc.value = undefined;
|
||||
}
|
||||
|
||||
function onSheetOpen(): void {
|
||||
loading.value = false;
|
||||
recognizing.value = false;
|
||||
imageFile.value = null;
|
||||
imageSrc.value = undefined;
|
||||
}
|
||||
|
||||
function onSheetClosed(): void {
|
||||
close();
|
||||
}
|
||||
</script>
|
||||
|
||||
<style>
|
||||
.image-container {
|
||||
border: 1px solid var(--f7-page-master-border-color);
|
||||
}
|
||||
|
||||
.image-container-background {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background-color: var(--f7-page-bg-color);
|
||||
}
|
||||
</style>
|
||||
@@ -7,6 +7,7 @@ export const DEFAULT_API_TIMEOUT: number = 10000; // 10s
|
||||
export const DEFAULT_UPLOAD_API_TIMEOUT: number = 30000; // 30s
|
||||
export const DEFAULT_EXPORT_API_TIMEOUT: number = 180000; // 180s
|
||||
export const DEFAULT_IMPORT_API_TIMEOUT: number = 1800000; // 1800s
|
||||
export const DEFAULT_LLM_API_TIMEOUT: number = 600000; // 600s
|
||||
|
||||
export const GOOGLE_MAP_JAVASCRIPT_URL: string = 'https://maps.googleapis.com/maps/api/js';
|
||||
export const BAIDU_MAP_JAVASCRIPT_URL: string = 'https://api.map.baidu.com/api?v=3.0';
|
||||
|
||||
@@ -6,6 +6,7 @@ export class KnownFileType {
|
||||
public static readonly TSV = new KnownFileType('tsv', 'text/tab-separated-values');
|
||||
public static readonly MARKDOWN = new KnownFileType('md', 'text/markdown');
|
||||
public static readonly JS = new KnownFileType('js', 'application/javascript');
|
||||
public static readonly JPG = new KnownFileType('jpg', 'image/jpeg');
|
||||
|
||||
public readonly extension: string;
|
||||
public readonly contentType: string;
|
||||
@@ -37,6 +38,12 @@ export class KnownFileType {
|
||||
});
|
||||
}
|
||||
|
||||
public createFileFromBlob(blob: Blob, fileName: string): File {
|
||||
return new File([blob], this.formatFileName(fileName), {
|
||||
type: this.contentType,
|
||||
});
|
||||
}
|
||||
|
||||
public static parse(extension: string): KnownFileType | undefined {
|
||||
return KnownFileType.allInstancesByExtension[extension];
|
||||
}
|
||||
|
||||
@@ -35,6 +35,10 @@ export function isMCPServerEnabled(): boolean {
|
||||
return getServerSetting('mcp') === 1;
|
||||
}
|
||||
|
||||
export function isTransactionFromAIImageRecognitionEnabled(): boolean {
|
||||
return getServerSetting('llmt') === 1;
|
||||
}
|
||||
|
||||
export function getLoginPageTips(): Record<string, string>{
|
||||
return getServerSetting('lpt') as Record<string, string>;
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import {
|
||||
DEFAULT_UPLOAD_API_TIMEOUT,
|
||||
DEFAULT_EXPORT_API_TIMEOUT,
|
||||
DEFAULT_IMPORT_API_TIMEOUT,
|
||||
DEFAULT_LLM_API_TIMEOUT,
|
||||
GOOGLE_MAP_JAVASCRIPT_URL,
|
||||
BAIDU_MAP_JAVASCRIPT_URL,
|
||||
AMAP_JAVASCRIPT_URL
|
||||
@@ -134,6 +135,9 @@ import type {
|
||||
import type {
|
||||
UserApplicationCloudSettingsUpdateRequest
|
||||
} from '@/models/user_app_cloud_setting.ts';
|
||||
import type {
|
||||
RecognizedReceiptImageResponse
|
||||
} from '@/models/large_language_model.ts';
|
||||
|
||||
import {
|
||||
getCurrentToken,
|
||||
@@ -635,6 +639,13 @@ export default {
|
||||
deleteTransactionTemplate: (req: TransactionTemplateDeleteRequest): ApiResponsePromise<boolean> => {
|
||||
return axios.post<ApiResponse<boolean>>('v1/transaction/templates/delete.json', req);
|
||||
},
|
||||
recognizeReceiptImage: ({ imageFile }: { imageFile: File }): ApiResponsePromise<RecognizedReceiptImageResponse> => {
|
||||
return axios.postForm<ApiResponse<RecognizedReceiptImageResponse>>('v1/llm/transactions/recognize_receipt_image.json', {
|
||||
image: imageFile
|
||||
}, {
|
||||
timeout: DEFAULT_LLM_API_TIMEOUT
|
||||
});
|
||||
},
|
||||
getLatestExchangeRates: (param: { ignoreError?: boolean }): ApiResponsePromise<LatestExchangeRateResponse> => {
|
||||
return axios.get<ApiResponse<LatestExchangeRateResponse>>('v1/exchange_rates/latest.json', {
|
||||
ignoreError: !!param.ignoreError,
|
||||
|
||||
@@ -3,6 +3,7 @@ import Clipboard from 'clipboard';
|
||||
import { ThemeType } from '@/core/theme.ts';
|
||||
|
||||
import { type AmountColor, PresetAmountColor } from '@/core/color.ts';
|
||||
import { KnownFileType } from '@/core/file.ts';
|
||||
|
||||
import logger from '../logger.ts';
|
||||
|
||||
@@ -134,6 +135,64 @@ export function startDownloadFile(fileName: string, fileData: Blob): void {
|
||||
dataLink.click();
|
||||
}
|
||||
|
||||
export function compressJpgImage(file: File, maxWidth: number, maxHeight: number, quality: number): Promise<Blob> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
|
||||
reader.onload = (event) => {
|
||||
const img = new Image();
|
||||
|
||||
img.onload = () => {
|
||||
let width = img.width;
|
||||
let height = img.height;
|
||||
|
||||
if (width > maxWidth || height > maxHeight) {
|
||||
const scale = Math.min(maxWidth / width, maxHeight / height);
|
||||
width = Math.floor(width * scale);
|
||||
height = Math.floor(height * scale);
|
||||
}
|
||||
|
||||
const canvas = document.createElement('canvas');
|
||||
const ctx = canvas.getContext('2d');
|
||||
|
||||
if (!ctx) {
|
||||
reject(new Error('failed to get canvas context'));
|
||||
return;
|
||||
}
|
||||
|
||||
canvas.width = width;
|
||||
canvas.height = height;
|
||||
|
||||
ctx.drawImage(img, 0, 0, width, height);
|
||||
|
||||
canvas.toBlob((blob) => {
|
||||
if (blob) {
|
||||
resolve(blob);
|
||||
} else {
|
||||
reject(new Error('failed to compress image'));
|
||||
}
|
||||
}, KnownFileType.JPG.contentType, quality);
|
||||
};
|
||||
|
||||
img.onerror = (error) => {
|
||||
reject(error);
|
||||
};
|
||||
|
||||
if (event.target && event.target.result) {
|
||||
img.src = event.target.result as string;
|
||||
} else {
|
||||
reject(new Error('failed to read file'));
|
||||
}
|
||||
};
|
||||
|
||||
reader.onerror = (error) => {
|
||||
reject(error);
|
||||
};
|
||||
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
export function clearBrowserCaches(): Promise<void> {
|
||||
if (!window.caches) {
|
||||
logger.error('caches API is not supported in this browser');
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "Abfrageelemente dürfen nicht leer sein",
|
||||
"query items too much": "Zu viele Abfrageelemente",
|
||||
"query items have invalid item": "Ungültiges Element in Abfrageelementen",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Aktualisieren",
|
||||
"Clear": "Löschen",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Keine",
|
||||
"Unspecified": "Nicht angegeben",
|
||||
"Not set": "Nicht festgelegt",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Duplicate (With Time)",
|
||||
"Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)",
|
||||
"Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Kategorie",
|
||||
"Secondary Category": "Secondary Category",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "There are no query items",
|
||||
"query items too much": "There are too many query items",
|
||||
"query items have invalid item": "There is invalid item in query items",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Refresh",
|
||||
"Clear": "Clear",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "None",
|
||||
"Unspecified": "Unspecified",
|
||||
"Not set": "Not set",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Duplicate (With Time)",
|
||||
"Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)",
|
||||
"Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Category",
|
||||
"Secondary Category": "Secondary Category",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "--",
|
||||
"query items too much": "--",
|
||||
"query items have invalid item": "Hay un elemento no válido en los elementos de consulta",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Refrescar",
|
||||
"Clear": "Claro",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Ninguno",
|
||||
"Unspecified": "No especificado",
|
||||
"Not set": "No establecido",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Duplicate (With Time)",
|
||||
"Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)",
|
||||
"Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Categoría",
|
||||
"Secondary Category": "Secondary Category",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "Non ci sono elementi di query",
|
||||
"query items too much": "Ci sono troppi elementi di query",
|
||||
"query items have invalid item": "C'è un elemento non valido negli elementi di query",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Aggiorna",
|
||||
"Clear": "Pulisci",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Nessuno",
|
||||
"Unspecified": "Non specificato",
|
||||
"Not set": "Non impostato",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Duplica (con ora)",
|
||||
"Duplicate (With Geographic Location)": "Duplica (con posizione geografica)",
|
||||
"Duplicate (With Time and Geographic Location)": "Duplica (con ora e posizione geografica)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Categoria",
|
||||
"Secondary Category": "Categoria secondaria",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "クエリ項目がありません",
|
||||
"query items too much": "クエリ項目が多すぎます",
|
||||
"query items have invalid item": "クエリ項目に無効な項目があります",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "リフレッシュ",
|
||||
"Clear": "消去",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "なし",
|
||||
"Unspecified": "不特定",
|
||||
"Not set": "セットしていない",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "複製(時間含む)",
|
||||
"Duplicate (With Geographic Location)": "複製(地理座標を含む)",
|
||||
"Duplicate (With Time and Geographic Location)": "複製(時間と地理座標を含む)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "カテゴリ",
|
||||
"Secondary Category": "二次カテゴリ",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Wisselkoersgegevens voor basisvaluta kunnen niet worden bijgewerkt",
|
||||
"cannot delete exchange rate data for base currency": "Wisselkoersgegevens voor basisvaluta kunnen niet worden verwijderd",
|
||||
"mcp server is not enabled": "MCP-server is niet ingeschakeld",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "Geen zoekitems opgegeven",
|
||||
"query items too much": "Te veel zoekitems",
|
||||
"query items have invalid item": "Ongeldig item in zoekitems",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Vernieuwen",
|
||||
"Clear": "Wissen",
|
||||
"Generate": "Genereren",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Geen",
|
||||
"Unspecified": "Niet gespecificeerd",
|
||||
"Not set": "Niet ingesteld",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Dupliceren (met tijd)",
|
||||
"Duplicate (With Geographic Location)": "Dupliceren (met geografische locatie)",
|
||||
"Duplicate (With Time and Geographic Location)": "Dupliceren (met tijd en locatie)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Categorie",
|
||||
"Secondary Category": "Secundaire categorie",
|
||||
"Expense Category": "Uitgavecategorie",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Não é possível atualizar dados de taxa de câmbio para a moeda base",
|
||||
"cannot delete exchange rate data for base currency": "Não é possível excluir dados de taxa de câmbio para a moeda base",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "Não há itens de consulta",
|
||||
"query items too much": "Há muitos itens de consulta",
|
||||
"query items have invalid item": "Há item inválido nos itens de consulta",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Atualizar",
|
||||
"Clear": "Limpar",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Nenhum",
|
||||
"Unspecified": "Não especificado",
|
||||
"Not set": "Não definido",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Duplicar (Com Tempo)",
|
||||
"Duplicate (With Geographic Location)": "Duplicar (Com Localização Geográfica)",
|
||||
"Duplicate (With Time and Geographic Location)": "Duplicar (Com Tempo e Localização Geográfica)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Categoria",
|
||||
"Secondary Category": "Categoria Secundária",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "Нет элементов запроса",
|
||||
"query items too much": "Слишком много элементов запроса",
|
||||
"query items have invalid item": "В элементах запроса присутствует недопустимый элемент",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Обновить",
|
||||
"Clear": "Очистить",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Нет",
|
||||
"Unspecified": "Не указано",
|
||||
"Not set": "Не установлено",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Duplicate (With Time)",
|
||||
"Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)",
|
||||
"Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Категория",
|
||||
"Secondary Category": "Secondary Category",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "Елементи запиту не можуть бути порожніми",
|
||||
"query items too much": "Занадто багато елементів запиту",
|
||||
"query items have invalid item": "Запит містить недійсний елемент",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Оновити",
|
||||
"Clear": "Очистити",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Немає",
|
||||
"Unspecified": "Не вказано",
|
||||
"Not set": "Не встановлено",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Дублювати (з часом)",
|
||||
"Duplicate (With Geographic Location)": "Дублювати (з геолокацією)",
|
||||
"Duplicate (With Time and Geographic Location)": "Дублювати (з часом і геолокацією)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Категорія",
|
||||
"Secondary Category": "Вторинна категорія",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "Cannot update exchange rate data for base currency",
|
||||
"cannot delete exchange rate data for base currency": "Cannot delete exchange rate data for base currency",
|
||||
"mcp server is not enabled": "MCP Server is not enabled",
|
||||
"llm provider is not enabled": "Large Language Model provider is not enabled",
|
||||
"no image for AI recognition": "There is no image for AI recognition",
|
||||
"image for AI recognition is empty": "Image for AI recognition file is empty",
|
||||
"exceed the maximum size of image file for AI recognition": "The uploaded image for AI recognition exceeds the maximum allowed file size",
|
||||
"query items cannot be blank": "Không có mục truy vấn",
|
||||
"query items too much": "Có quá nhiều mục truy vấn",
|
||||
"query items have invalid item": "Có mục không hợp lệ trong các mục truy vấn",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "Làm mới",
|
||||
"Clear": "Xóa",
|
||||
"Generate": "Generate",
|
||||
"Recognize": "Recognize",
|
||||
"None": "Không có",
|
||||
"Unspecified": "Không xác định",
|
||||
"Not set": "Not set",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "Duplicate (With Time)",
|
||||
"Duplicate (With Geographic Location)": "Duplicate (With Geographic Location)",
|
||||
"Duplicate (With Time and Geographic Location)": "Duplicate (With Time and Geographic Location)",
|
||||
"AI Image Recognition": "AI Image Recognition",
|
||||
"Choose from Library": "Choose from Library",
|
||||
"Take Photo": "Take Photo",
|
||||
"Unable to load image": "Unable to load image",
|
||||
"Unable to recognize image": "Unable to recognize image",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "Drag and drop a receipt or transaction image here, or click to select one",
|
||||
"Release to load image": "Release to load image",
|
||||
"Please select a receipt or transaction image first": "Please select a receipt or transaction image first",
|
||||
"Category": "Danh mục",
|
||||
"Secondary Category": "Secondary Category",
|
||||
"Expense Category": "Expense Category",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "不能更新默认货币的汇率数据",
|
||||
"cannot delete exchange rate data for base currency": "不能删除默认货币的汇率数据",
|
||||
"mcp server is not enabled": "MCP 服务器没有启用",
|
||||
"llm provider is not enabled": "大语言模型服务提供者没有启用",
|
||||
"no image for AI recognition": "没有用于AI识别的图片",
|
||||
"image for AI recognition is empty": "用于AI识别的图片为空",
|
||||
"exceed the maximum size of image file for AI recognition": "用于AI识别的图片超出了允许的最大文件大小",
|
||||
"query items cannot be blank": "请求项目不能为空",
|
||||
"query items too much": "请求项目过多",
|
||||
"query items have invalid item": "请求项目中有非法项目",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "刷新",
|
||||
"Clear": "清除",
|
||||
"Generate": "生成",
|
||||
"Recognize": "识别",
|
||||
"None": "无",
|
||||
"Unspecified": "未指定",
|
||||
"Not set": "未设置",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "复制 (含时间)",
|
||||
"Duplicate (With Geographic Location)": "复制 (含地理位置)",
|
||||
"Duplicate (With Time and Geographic Location)": "复制 (含时间和地理位置)",
|
||||
"AI Image Recognition": "AI识图",
|
||||
"Choose from Library": "从图库选择",
|
||||
"Take Photo": "拍照",
|
||||
"Unable to load image": "无法加载图片",
|
||||
"Unable to recognize image": "无法识别图片",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "拖拽收据或交易图片到此处,或点击选择图片",
|
||||
"Release to load image": "释放以加载图片",
|
||||
"Please select a receipt or transaction image first": "请先选择收据或交易图片",
|
||||
"Category": "分类",
|
||||
"Secondary Category": "二级分类",
|
||||
"Expense Category": "支出分类",
|
||||
|
||||
@@ -1224,6 +1224,10 @@
|
||||
"cannot update exchange rate data for base currency": "不能更新基準貨幣的匯率資料",
|
||||
"cannot delete exchange rate data for base currency": "不能刪除基準貨幣的匯率資料",
|
||||
"mcp server is not enabled": "MCP 伺服器未啟用",
|
||||
"llm provider is not enabled": "大型語言模型服務提供者未啟用",
|
||||
"no image for AI recognition": "沒有用於AI識別的圖片檔案",
|
||||
"image for AI recognition is empty": "用於AI識別的圖片檔案為空",
|
||||
"exceed the maximum size of image file for AI recognition": "用於AI識別的圖片超出了允許的最大檔案大小",
|
||||
"query items cannot be blank": "查詢項目不能為空",
|
||||
"query items too much": "查詢項目過多",
|
||||
"query items have invalid item": "查詢項目中有非法項目",
|
||||
@@ -1389,6 +1393,7 @@
|
||||
"Refresh": "重新載入",
|
||||
"Clear": "清除",
|
||||
"Generate": "產生",
|
||||
"Recognize": "識別",
|
||||
"None": "無",
|
||||
"Unspecified": "未指定",
|
||||
"Not set": "未設置",
|
||||
@@ -1719,6 +1724,14 @@
|
||||
"Duplicate (With Time)": "複製 (含時間)",
|
||||
"Duplicate (With Geographic Location)": "複製 (含地理位置)",
|
||||
"Duplicate (With Time and Geographic Location)": "複製 (含時間和地理位置)",
|
||||
"AI Image Recognition": "AI識圖",
|
||||
"Choose from Library": "從相簿選擇",
|
||||
"Take Photo": "拍照",
|
||||
"Unable to load image": "無法載入圖片",
|
||||
"Unable to recognize image": "無法識別圖片",
|
||||
"Drag and drop a receipt or transaction image here, or click to select one": "將收據或交易圖片拖放到此處,或點擊以選擇圖片",
|
||||
"Release to load image": "放開以載入圖片",
|
||||
"Please select a receipt or transaction image first": "請先選擇收據或交易圖片",
|
||||
"Category": "分類",
|
||||
"Secondary Category": "次分類",
|
||||
"Expense Category": "支出分類",
|
||||
|
||||
@@ -79,6 +79,7 @@ import MapSheet from '@/components/mobile/MapSheet.vue';
|
||||
import TransactionTagSelectionSheet from '@/components/mobile/TransactionTagSelectionSheet.vue';
|
||||
import ScheduleFrequencySheet from '@/components/mobile/ScheduleFrequencySheet.vue';
|
||||
import AccountBalanceTrendsBarChart from '@/components/mobile/AccountBalanceTrendsBarChart.vue';
|
||||
import AIImageRecognitionSheet from '@/components/mobile/AIImageRecognitionSheet.vue';
|
||||
|
||||
import TextareaAutoSize from '@/directives/mobile/textareaAutoSize.ts';
|
||||
|
||||
@@ -170,8 +171,9 @@ app.component('InformationSheet', InformationSheet);
|
||||
app.component('NumberPadSheet', NumberPadSheet);
|
||||
app.component('MapSheet', MapSheet);
|
||||
app.component('TransactionTagSelectionSheet', TransactionTagSelectionSheet);
|
||||
app.component('AccountBalanceTrendsBarChart', AccountBalanceTrendsBarChart);
|
||||
app.component('ScheduleFrequencySheet', ScheduleFrequencySheet);
|
||||
app.component('AccountBalanceTrendsBarChart', AccountBalanceTrendsBarChart);
|
||||
app.component('AIImageRecognitionSheet', AIImageRecognitionSheet);
|
||||
|
||||
app.directive('TextareaAutoSize', TextareaAutoSize);
|
||||
|
||||
|
||||
11
src/models/large_language_model.ts
Normal file
11
src/models/large_language_model.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
export interface RecognizedReceiptImageResponse {
|
||||
readonly type: number;
|
||||
readonly time?: number;
|
||||
readonly categoryId?: string;
|
||||
readonly sourceAccountId?: string;
|
||||
readonly destinationAccountId?: string;
|
||||
readonly sourceAmount?: number;
|
||||
readonly destinationAmount?: number;
|
||||
readonly tagIds?: string[];
|
||||
readonly comment?: string;
|
||||
}
|
||||
@@ -33,6 +33,9 @@ import {
|
||||
import {
|
||||
type ExportTransactionDataRequest
|
||||
} from '@/models/data_management.ts';
|
||||
import type {
|
||||
RecognizedReceiptImageResponse
|
||||
} from '@/models/large_language_model.ts';
|
||||
|
||||
import {
|
||||
getUserTransactionDraft,
|
||||
@@ -1157,6 +1160,31 @@ export const useTransactionsStore = defineStore('transactions', () => {
|
||||
});
|
||||
}
|
||||
|
||||
function recognizeReceiptImage({ imageFile }: { imageFile: File }): Promise<RecognizedReceiptImageResponse> {
|
||||
return new Promise((resolve, reject) => {
|
||||
services.recognizeReceiptImage({ imageFile }).then(response => {
|
||||
const data = response.data;
|
||||
|
||||
if (!data || !data.success || !data.result) {
|
||||
reject({ message: 'Unable to recognize image' });
|
||||
return;
|
||||
}
|
||||
|
||||
resolve(data.result);
|
||||
}).catch(error => {
|
||||
logger.error('failed to recognize image', error);
|
||||
|
||||
if (error.response && error.response.data && error.response.data.errorMessage) {
|
||||
reject({ error: error.response.data });
|
||||
} else if (!error.processed) {
|
||||
reject({ message: 'Unable to recognize image' });
|
||||
} else {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function parseImportDsvFile({ fileType, fileEncoding, importFile }: { fileType: string, fileEncoding?: string, importFile: File }): Promise<string[][]> {
|
||||
return new Promise((resolve, reject) => {
|
||||
services.parseImportDsvFile({ fileType, fileEncoding, importFile }).then(response => {
|
||||
@@ -1370,6 +1398,7 @@ export const useTransactionsStore = defineStore('transactions', () => {
|
||||
getTransaction,
|
||||
saveTransaction,
|
||||
deleteTransaction,
|
||||
recognizeReceiptImage,
|
||||
parseImportDsvFile,
|
||||
parseImportTransaction,
|
||||
importTransactions,
|
||||
|
||||
@@ -63,11 +63,16 @@
|
||||
<v-btn class="ms-3" color="default" variant="outlined"
|
||||
:disabled="loading || !canAddTransaction" @click="add()">
|
||||
{{ tt('Add') }}
|
||||
<v-menu activator="parent" :open-on-hover="true" v-if="allTransactionTemplates && allTransactionTemplates.length">
|
||||
<v-menu activator="parent" :open-on-hover="true" v-if="isTransactionFromAIImageRecognitionEnabled() || (allTransactionTemplates && allTransactionTemplates.length)">
|
||||
<v-list>
|
||||
<v-list-item :title="template.name"
|
||||
<v-list-item key="AIImageRecognition"
|
||||
:title="tt('AI Image Recognition')"
|
||||
:prepend-icon="mdiMagicStaff"
|
||||
v-if="isTransactionFromAIImageRecognitionEnabled()"
|
||||
@click="addByRecognizingImage"></v-list-item>
|
||||
<v-list-item :key="template.id"
|
||||
:title="template.name"
|
||||
:prepend-icon="mdiTextBoxOutline"
|
||||
:key="template.id"
|
||||
v-for="template in allTransactionTemplates"
|
||||
@click="add(template)"></v-list-item>
|
||||
</v-list>
|
||||
@@ -620,6 +625,7 @@
|
||||
@error="onShowDateRangeError" />
|
||||
|
||||
<edit-dialog ref="editDialog" :type="TransactionEditPageType.Transaction" />
|
||||
<a-i-image-recognition-dialog ref="aiImageRecognitionDialog" />
|
||||
<import-dialog ref="importDialog" :persistent="true" />
|
||||
|
||||
<v-dialog width="800" v-model="showFilterAccountDialog">
|
||||
@@ -647,6 +653,7 @@ import PaginationButtons from '@/components/desktop/PaginationButtons.vue';
|
||||
import ConfirmDialog from '@/components/desktop/ConfirmDialog.vue';
|
||||
import SnackBar from '@/components/desktop/SnackBar.vue';
|
||||
import EditDialog from './list/dialogs/EditDialog.vue';
|
||||
import AIImageRecognitionDialog from './list/dialogs/AIImageRecognitionDialog.vue';
|
||||
import ImportDialog from './import/ImportDialog.vue';
|
||||
import AccountFilterSettingsCard from '@/views/desktop/common/cards/AccountFilterSettingsCard.vue';
|
||||
import CategoryFilterSettingsCard from '@/views/desktop/common/cards/CategoryFilterSettingsCard.vue';
|
||||
@@ -716,7 +723,7 @@ import {
|
||||
categoryTypeToTransactionType,
|
||||
transactionTypeToCategoryType
|
||||
} from '@/lib/category.ts';
|
||||
import { isDataExportingEnabled, isDataImportingEnabled } from '@/lib/server_settings.ts';
|
||||
import { isDataExportingEnabled, isDataImportingEnabled, isTransactionFromAIImageRecognitionEnabled } from '@/lib/server_settings.ts';
|
||||
import { startDownloadFile } from '@/lib/ui/common.ts';
|
||||
import { scrollToSelectedItem } from '@/lib/ui/desktop.ts';
|
||||
import logger from '@/lib/logger.ts';
|
||||
@@ -738,6 +745,7 @@ import {
|
||||
mdiMinusBoxMultipleOutline,
|
||||
mdiCloseBoxMultipleOutline,
|
||||
mdiPound,
|
||||
mdiMagicStaff,
|
||||
mdiTextBoxOutline
|
||||
} from '@mdi/js';
|
||||
|
||||
@@ -760,6 +768,7 @@ const props = defineProps<TransactionListProps>();
|
||||
type ConfirmDialogType = InstanceType<typeof ConfirmDialog>;
|
||||
type SnackBarType = InstanceType<typeof SnackBar>;
|
||||
type EditDialogType = InstanceType<typeof EditDialog>;
|
||||
type AIImageRecognitionDialogType = InstanceType<typeof AIImageRecognitionDialog>;
|
||||
type ImportDialogType = InstanceType<typeof ImportDialog>;
|
||||
|
||||
interface TransactionTemplateWithIcon {
|
||||
@@ -859,6 +868,7 @@ const tagFilterMenu = useTemplateRef<VMenu>('tagFilterMenu');
|
||||
const confirmDialog = useTemplateRef<ConfirmDialogType>('confirmDialog');
|
||||
const snackbar = useTemplateRef<SnackBarType>('snackbar');
|
||||
const editDialog = useTemplateRef<EditDialogType>('editDialog');
|
||||
const aiImageRecognitionDialog = useTemplateRef<AIImageRecognitionDialogType>('aiImageRecognitionDialog');
|
||||
const importDialog = useTemplateRef<ImportDialogType>('importDialog');
|
||||
|
||||
const activeTab = ref<string>('transactionPage');
|
||||
@@ -1597,6 +1607,33 @@ function add(template?: TransactionTemplate): void {
|
||||
});
|
||||
}
|
||||
|
||||
function addByRecognizingImage(): void {
|
||||
aiImageRecognitionDialog.value?.open().then(result => {
|
||||
editDialog.value?.open({
|
||||
time: result.time,
|
||||
type: result.type,
|
||||
categoryId: result.categoryId,
|
||||
accountId: result.sourceAccountId,
|
||||
destinationAccountId: result.destinationAccountId,
|
||||
amount: result.sourceAmount,
|
||||
destinationAmount: result.destinationAmount,
|
||||
tagIds: result.tagIds ? result.tagIds.join(',') : undefined,
|
||||
comment: result.comment,
|
||||
noTransactionDraft: true
|
||||
}).then(result => {
|
||||
if (result && result.message) {
|
||||
snackbar.value?.showMessage(result.message);
|
||||
}
|
||||
|
||||
reload(false, false);
|
||||
}).catch(error => {
|
||||
if (error) {
|
||||
snackbar.value?.showError(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function importTransaction(): void {
|
||||
importDialog.value?.open().then(() => {
|
||||
reload(false, false);
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
<template>
|
||||
<v-dialog width="800" :persistent="loading || recognizing || !!imageFile" v-model="showState">
|
||||
<v-card class="pa-2 pa-sm-4 pa-md-4">
|
||||
<template #title>
|
||||
<div class="d-flex align-center justify-center">
|
||||
<h4 class="text-h4">{{ tt('AI Image Recognition') }}</h4>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<v-card-text class="d-flex justify-center w-100 my-md-4 pt-0">
|
||||
<div class="w-100 border position-relative"
|
||||
@dragenter.prevent="onDragEnter"
|
||||
@dragover.prevent
|
||||
@dragleave.prevent="onDragLeave"
|
||||
@drop.prevent="onDrop">
|
||||
<div class="d-flex w-100 fill-height justify-center align-center justify-content-center"
|
||||
:class="{ 'dropzone': true, 'dropzone-dragover': isDragOver }" style="height: 480px">
|
||||
<h3 v-if="!imageFile && !isDragOver">{{ tt('Drag and drop a receipt or transaction image here, or click to select one') }}</h3>
|
||||
<h3 v-if="isDragOver">{{ tt('Release to load image') }}</h3>
|
||||
</div>
|
||||
<v-img height="480px" :class="{ 'cursor-pointer': !loading || !recognizing || !isDragOver }"
|
||||
:src="imageSrc" @click="showOpenImageDialog">
|
||||
<template #placeholder>
|
||||
<div class="w-100 fill-height bg-grey-200"></div>
|
||||
</template>
|
||||
</v-img>
|
||||
</div>
|
||||
</v-card-text>
|
||||
|
||||
<v-card-text class="overflow-y-visible">
|
||||
<div ref="buttonContainer" class="w-100 d-flex justify-center gap-4">
|
||||
<v-btn :disabled="loading || recognizing || !imageFile" @click="recognize">
|
||||
{{ tt('Recognize') }}
|
||||
<v-progress-circular indeterminate size="22" class="ms-2" v-if="recognizing"></v-progress-circular>
|
||||
</v-btn>
|
||||
<v-btn color="secondary" variant="tonal" :disabled="loading || recognizing"
|
||||
@click="cancel">{{ tt('Cancel') }}</v-btn>
|
||||
</div>
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<snack-bar ref="snackbar" />
|
||||
<input ref="imageInput" type="file" style="display: none" :accept="SUPPORTED_IMAGE_EXTENSIONS" @change="openImage($event)" />
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import SnackBar from '@/components/desktop/SnackBar.vue';
|
||||
|
||||
import { ref, useTemplateRef } from 'vue';
|
||||
|
||||
import { useI18n } from '@/locales/helpers.ts';
|
||||
|
||||
import { useTransactionsStore } from '@/stores/transaction.ts';
|
||||
|
||||
import { KnownFileType } from '@/core/file.ts';
|
||||
import { SUPPORTED_IMAGE_EXTENSIONS } from '@/consts/file.ts';
|
||||
|
||||
import type { RecognizedReceiptImageResponse } from '@/models/large_language_model.ts';
|
||||
|
||||
import { compressJpgImage } from '@/lib/ui/common.ts';
|
||||
import logger from '@/lib/logger.ts';
|
||||
|
||||
type SnackBarType = InstanceType<typeof SnackBar>;
|
||||
|
||||
const { tt } = useI18n();
|
||||
|
||||
const transactionsStore = useTransactionsStore();
|
||||
|
||||
const snackbar = useTemplateRef<SnackBarType>('snackbar');
|
||||
const imageInput = useTemplateRef<HTMLInputElement>('imageInput');
|
||||
|
||||
let resolveFunc: ((response: RecognizedReceiptImageResponse) => void) | null = null;
|
||||
let rejectFunc: ((reason?: unknown) => void) | null = null;
|
||||
|
||||
const showState = ref<boolean>(false);
|
||||
const loading = ref<boolean>(false);
|
||||
const recognizing = ref<boolean>(false);
|
||||
const imageFile = ref<File | null>(null);
|
||||
const imageSrc = ref<string | undefined>(undefined);
|
||||
const isDragOver = ref<boolean>(false);
|
||||
|
||||
function loadImage(file: File): void {
|
||||
compressJpgImage(file, 1280, 1280, 0.8).then(blob => {
|
||||
imageFile.value = KnownFileType.JPG.createFileFromBlob(blob, "image");
|
||||
imageSrc.value = URL.createObjectURL(blob);
|
||||
}).catch(error => {
|
||||
imageFile.value = null;
|
||||
imageSrc.value = undefined;
|
||||
logger.error('failed to compress image', error);
|
||||
snackbar.value?.showError('Unable to load image');
|
||||
});
|
||||
}
|
||||
|
||||
function open(): Promise<RecognizedReceiptImageResponse> {
|
||||
showState.value = true;
|
||||
loading.value = false;
|
||||
recognizing.value = false;
|
||||
imageFile.value = null;
|
||||
imageSrc.value = undefined;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
resolveFunc = resolve;
|
||||
rejectFunc = reject;
|
||||
});
|
||||
}
|
||||
|
||||
function showOpenImageDialog(): void {
|
||||
if (loading.value || recognizing.value || isDragOver.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
imageInput.value?.click();
|
||||
}
|
||||
|
||||
function openImage(event: Event): void {
|
||||
if (!event || !event.target) {
|
||||
return;
|
||||
}
|
||||
|
||||
const el = event.target as HTMLInputElement;
|
||||
|
||||
if (!el.files || !el.files.length || !el.files[0]) {
|
||||
return;
|
||||
}
|
||||
|
||||
const image = el.files[0] as File;
|
||||
|
||||
el.value = '';
|
||||
|
||||
loadImage(image);
|
||||
}
|
||||
|
||||
function recognize(): void {
|
||||
if (loading.value || recognizing.value || !imageFile.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
recognizing.value = true;
|
||||
|
||||
transactionsStore.recognizeReceiptImage({
|
||||
imageFile: imageFile.value
|
||||
}).then(response => {
|
||||
resolveFunc?.(response);
|
||||
showState.value = false;
|
||||
recognizing.value = false;
|
||||
}).catch(error => {
|
||||
recognizing.value = false;
|
||||
|
||||
if (!error.processed) {
|
||||
snackbar.value?.showError(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function cancel(): void {
|
||||
rejectFunc?.();
|
||||
showState.value = false;
|
||||
loading.value = false;
|
||||
recognizing.value = false;
|
||||
imageFile.value = null;
|
||||
imageSrc.value = undefined;
|
||||
}
|
||||
|
||||
function onDragEnter(): void {
|
||||
if (loading.value || recognizing.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
isDragOver.value = true;
|
||||
}
|
||||
|
||||
function onDragLeave(): void {
|
||||
isDragOver.value = false;
|
||||
}
|
||||
|
||||
function onDrop(event: DragEvent): void {
|
||||
if (loading.value || recognizing.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
isDragOver.value = false;
|
||||
|
||||
if (event.dataTransfer && event.dataTransfer.files && event.dataTransfer.files.length && event.dataTransfer.files[0]) {
|
||||
loadImage(event.dataTransfer.files[0] as File);
|
||||
}
|
||||
}
|
||||
|
||||
defineExpose({
|
||||
open
|
||||
});
|
||||
</script>
|
||||
|
||||
<style>
|
||||
.dropzone {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
pointer-events: none;
|
||||
border-radius: 8px;
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
.dropzone-dragover {
|
||||
border: 6px dashed rgba(var(--v-border-color),var(--v-border-opacity));
|
||||
}
|
||||
</style>
|
||||
@@ -188,7 +188,14 @@
|
||||
<f7-popover class="template-popover-menu" target-el="#homepage-add-button"
|
||||
v-model:opened="showTransactionTemplatePopover">
|
||||
<f7-list dividers v-if="allTransactionTemplates">
|
||||
<f7-list-item :title="template.name" :key="template.id"
|
||||
<f7-list-item key="AIImageRecognition" :title="tt('AI Image Recognition')"
|
||||
@click="showAIReceiptImageRecognitionSheet = true; showTransactionTemplatePopover = false"
|
||||
v-if="isTransactionFromAIImageRecognitionEnabled()">
|
||||
<template #media>
|
||||
<f7-icon f7="wand_stars"></f7-icon>
|
||||
</template>
|
||||
</f7-list-item>
|
||||
<f7-list-item :key="template.id" :title="template.name"
|
||||
:link="'/transaction/add?templateId=' + template.id"
|
||||
v-for="template in allTransactionTemplates">
|
||||
<template #media>
|
||||
@@ -197,11 +204,15 @@
|
||||
</f7-list-item>
|
||||
</f7-list>
|
||||
</f7-popover>
|
||||
|
||||
<a-i-image-recognition-sheet v-model:show="showAIReceiptImageRecognitionSheet"
|
||||
@recognition:change="onReceiptRecognitionChanged"/>
|
||||
</f7-page>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, computed } from 'vue';
|
||||
import type { Router } from 'framework7/types';
|
||||
|
||||
import { useI18n } from '@/locales/helpers.ts';
|
||||
import { useI18nUIComponents } from '@/lib/ui/mobile.ts';
|
||||
@@ -215,8 +226,14 @@ import { useOverviewStore } from '@/stores/overview.ts';
|
||||
import { DateRange } from '@/core/datetime.ts';
|
||||
import { TemplateType } from '@/core/template.ts';
|
||||
import { TransactionTemplate } from '@/models/transaction_template.ts';
|
||||
import type { RecognizedReceiptImageResponse } from '@/models/large_language_model.ts';
|
||||
|
||||
import { isUserLogined, isUserUnlocked } from '@/lib/userstate.ts';
|
||||
import { isTransactionFromAIImageRecognitionEnabled } from '@/lib/server_settings.ts';
|
||||
|
||||
const props = defineProps<{
|
||||
f7router: Router.Router;
|
||||
}>();
|
||||
|
||||
const { tt } = useI18n();
|
||||
const { showToast } = useI18nUIComponents();
|
||||
@@ -236,6 +253,7 @@ const overviewStore = useOverviewStore();
|
||||
|
||||
const loading = ref<boolean>(true);
|
||||
const showTransactionTemplatePopover = ref<boolean>(false);
|
||||
const showAIReceiptImageRecognitionSheet = ref<boolean>(false);
|
||||
|
||||
const allTransactionTemplates = computed<TransactionTemplate[]>(() => {
|
||||
const allTemplates = transactionTemplatesStore.allVisibleTemplates;
|
||||
@@ -243,7 +261,7 @@ const allTransactionTemplates = computed<TransactionTemplate[]>(() => {
|
||||
});
|
||||
|
||||
function openTransactionTemplatePopover(): void {
|
||||
if (allTransactionTemplates.value && allTransactionTemplates.value.length) {
|
||||
if (isTransactionFromAIImageRecognitionEnabled() || (allTransactionTemplates.value && allTransactionTemplates.value.length)) {
|
||||
showTransactionTemplatePopover.value = true;
|
||||
}
|
||||
}
|
||||
@@ -291,6 +309,48 @@ function reload(done?: () => void): void {
|
||||
});
|
||||
}
|
||||
|
||||
function onReceiptRecognitionChanged(result: RecognizedReceiptImageResponse): void {
|
||||
const params: string[] = [];
|
||||
|
||||
if (result.type) {
|
||||
params.push(`type=${result.type}`);
|
||||
}
|
||||
|
||||
if (result.time) {
|
||||
params.push(`time=${result.time}`);
|
||||
}
|
||||
|
||||
if (result.categoryId) {
|
||||
params.push(`categoryId=${result.categoryId}`);
|
||||
}
|
||||
|
||||
if (result.sourceAccountId) {
|
||||
params.push(`accountId=${result.sourceAccountId}`);
|
||||
}
|
||||
|
||||
if (result.destinationAccountId) {
|
||||
params.push(`destinationAccountId=${result.destinationAccountId}`);
|
||||
}
|
||||
|
||||
if (result.sourceAmount) {
|
||||
params.push(`amount=${result.sourceAmount}`);
|
||||
}
|
||||
|
||||
if (result.destinationAmount) {
|
||||
params.push(`destinationAmount=${result.destinationAmount}`);
|
||||
}
|
||||
|
||||
if (result.tagIds) {
|
||||
params.push(`tagIds=${result.tagIds.join(',')}`);
|
||||
}
|
||||
|
||||
if (result.comment) {
|
||||
params.push(`comment=${encodeURIComponent(result.comment)}`);
|
||||
}
|
||||
|
||||
props.f7router.navigate(`/transaction/add?${params.join('&')}`);
|
||||
}
|
||||
|
||||
function onPageAfterIn(): void {
|
||||
if (!loading.value) {
|
||||
reload();
|
||||
|
||||
44
templates/prompt/receipt_image_recognition.tmpl
Normal file
44
templates/prompt/receipt_image_recognition.tmpl
Normal file
@@ -0,0 +1,44 @@
|
||||
## Role
|
||||
You are a financial assistant.
|
||||
Your task is to extract structured transaction data from images provided by the user (such as receipts, transaction records, or vouchers).
|
||||
|
||||
## Output
|
||||
1. Format: JSON only
|
||||
2. No explanations, comments, or extra text outside JSON
|
||||
|
||||
## JSON Schema (with field descriptions)
|
||||
```
|
||||
{
|
||||
"type": "string (transaction type: expense | income | transfer)",
|
||||
"time": "string (transaction time, format: YYYY-MM-DD HH:mm:ss)",
|
||||
"amount": "string (transaction amount, numeric, up to 2 decimals)",
|
||||
"account": "string (source account name)",
|
||||
"category": "string (transaction category)",
|
||||
"tags": ["string (tag name, max 10 allowed)"],
|
||||
"description": "string (transaction description)",
|
||||
"destination_amount": "string (destination amount, numeric, up to 2 decimals, only for transfer)",
|
||||
"destination_account": "string (destination account name, only for transfer)"
|
||||
}
|
||||
```
|
||||
|
||||
## Important rules
|
||||
1. Only include fields you can confidently identify.
|
||||
2. If unsure about a value, omit the field (do not guess).
|
||||
3. If the image contains multiple items, please combine them into a single transaction.
|
||||
4. Always return valid JSON.
|
||||
|
||||
## Options
|
||||
### Expense categories:
|
||||
{{.AllExpenseCategoryNames}}
|
||||
|
||||
### Income categories:
|
||||
{{.AllIncomeCategoryNames}}
|
||||
|
||||
### Transfer categories:
|
||||
{{.AllTransferCategoryNames}}
|
||||
|
||||
### Account names:
|
||||
{{.AllAccountNames}}
|
||||
|
||||
### Tags:
|
||||
{{.AllTagNames}}
|
||||
Reference in New Issue
Block a user