refactor: refactor image relay (close #1068)

common/model-ratio.go

@@ -7,29 +7,6 @@ import (
 	"time"
 )
 
-var DalleSizeRatios = map[string]map[string]float64{
-	"dall-e-2": {
-		"256x256":   1,
-		"512x512":   1.125,
-		"1024x1024": 1.25,
-	},
-	"dall-e-3": {
-		"1024x1024": 1,
-		"1024x1792": 2,
-		"1792x1024": 2,
-	},
-}
-
-var DalleGenerationImageAmounts = map[string][2]int{
-	"dall-e-2": {1, 10},
-	"dall-e-3": {1, 1}, // OpenAI allows n=1 currently.
-}
-
-var DalleImagePromptLengthLimitations = map[string]int{
-	"dall-e-2": 1000,
-	"dall-e-3": 4000,
-}
-
 const (
 	USD2RMB = 7
 	USD     = 500 // $0.002 = 1 -> $1 = 500
@@ -94,14 +71,18 @@ var ModelRatio = map[string]float64{
 	"claude-2.0":              5.51,  // $11.02 / 1M tokens
 	"claude-2.1":              5.51,  // $11.02 / 1M tokens
 	// https://cloud.baidu.com/doc/WENXINWORKSHOP/s/hlrk4akp7
-	"ERNIE-Bot":                 0.8572,     // ￥0.012 / 1k tokens
+	"ERNIE-Bot":         0.8572,     // ￥0.012 / 1k tokens
-	"ERNIE-Bot-turbo":           0.5715,     // ￥0.008 / 1k tokens
+	"ERNIE-Bot-turbo":   0.5715,     // ￥0.008 / 1k tokens
-	"ERNIE-Bot-4":               0.12 * RMB, // ￥0.12 / 1k tokens
+	"ERNIE-Bot-4":       0.12 * RMB, // ￥0.12 / 1k tokens
-	"ERNIE-Bot-8k":              0.024 * RMB,
+	"ERNIE-Bot-8k":      0.024 * RMB,
-	"Embedding-V1":              0.1429, // ￥0.002 / 1k tokens
+	"Embedding-V1":      0.1429, // ￥0.002 / 1k tokens
-	"PaLM-2":                    1,
+	"PaLM-2":            1,
-	"gemini-pro":                1,      // $0.00025 / 1k characters -> $0.001 / 1k tokens
+	"gemini-pro":        1, // $0.00025 / 1k characters -> $0.001 / 1k tokens
-	"gemini-pro-vision":         1,      // $0.00025 / 1k characters -> $0.001 / 1k tokens
+	"gemini-pro-vision": 1, // $0.00025 / 1k characters -> $0.001 / 1k tokens
+	// https://open.bigmodel.cn/pricing
+	"glm-4":                     0.1 * RMB,
+	"glm-4v":                    0.1 * RMB,
+	"glm-3-turbo":               0.005 * RMB,
 	"chatglm_turbo":             0.3572, // ￥0.005 / 1k tokens
 	"chatglm_pro":               0.7143, // ￥0.01 / 1k tokens
 	"chatglm_std":               0.3572, // ￥0.005 / 1k tokens

relay/channel/openai/adaptor.go

@@ -76,7 +76,7 @@ func (a *Adaptor) DoRequest(c *gin.Context, meta *util.RelayMeta, requestBody io
 func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *util.RelayMeta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
 	if meta.IsStream {
 		var responseText string
-		err, responseText = StreamHandler(c, resp, meta.Mode)
+		err, responseText, _ = StreamHandler(c, resp, meta.Mode)
 		usage = ResponseText2Usage(responseText, meta.ActualModelName, meta.PromptTokens)
 	} else {
 		err, usage = Handler(c, resp, meta.PromptTokens, meta.ActualModelName)

relay/channel/openai/main.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 )
 
-func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.ErrorWithStatusCode, string) {
+func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.ErrorWithStatusCode, string, *model.Usage) {
 	responseText := ""
 	scanner := bufio.NewScanner(resp.Body)
 	scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) {
@@ -31,6 +31,7 @@ func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.E
 	})
 	dataChan := make(chan string)
 	stopChan := make(chan bool)
+	var usage *model.Usage
 	go func() {
 		for scanner.Scan() {
 			data := scanner.Text()
@@ -54,6 +55,9 @@ func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.E
 					for _, choice := range streamResponse.Choices {
 						responseText += choice.Delta.Content
 					}
+					if streamResponse.Usage != nil {
+						usage = streamResponse.Usage
+					}
 				case constant.RelayModeCompletions:
 					var streamResponse CompletionsStreamResponse
 					err := json.Unmarshal([]byte(data), &streamResponse)
@@ -86,9 +90,9 @@ func StreamHandler(c *gin.Context, resp *http.Response, relayMode int) (*model.E
 	})
 	err := resp.Body.Close()
 	if err != nil {
-		return ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), ""
+		return ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError), "", nil
 	}
-	return nil, responseText
+	return nil, responseText, usage
 }
 
 func Handler(c *gin.Context, resp *http.Response, promptTokens int, modelName string) (*model.ErrorWithStatusCode, *model.Usage) {

relay/channel/openai/model.go

@@ -132,6 +132,7 @@ type ChatCompletionsStreamResponse struct {
 	Created int64                                 `json:"created"`
 	Model   string                                `json:"model"`
 	Choices []ChatCompletionsStreamResponseChoice `json:"choices"`
+	Usage   *model.Usage                          `json:"usage"`
 }
 
 type CompletionsStreamResponse struct {

relay/channel/tencent/main.go

@@ -81,6 +81,7 @@ func responseTencent2OpenAI(response *ChatResponse) *openai.TextResponse {
 
 func streamResponseTencent2OpenAI(TencentResponse *ChatResponse) *openai.ChatCompletionsStreamResponse {
 	response := openai.ChatCompletionsStreamResponse{
+		Id:      fmt.Sprintf("chatcmpl-%s", helper.GetUUID()),
 		Object:  "chat.completion.chunk",
 		Created: helper.GetTimestamp(),
 		Model:   "tencent-hunyuan",

relay/channel/zhipu/adaptor.go

@@ -5,20 +5,35 @@ import (
 	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/songquanpeng/one-api/relay/channel"
+	"github.com/songquanpeng/one-api/relay/channel/openai"
 	"github.com/songquanpeng/one-api/relay/model"
 	"github.com/songquanpeng/one-api/relay/util"
 	"io"
 	"net/http"
+	"strings"
 )
 
 type Adaptor struct {
+	APIVersion string
 }
 
 func (a *Adaptor) Init(meta *util.RelayMeta) {
 
 }
 
+func (a *Adaptor) SetVersionByModeName(modelName string) {
+	if strings.HasPrefix(modelName, "glm-") {
+		a.APIVersion = "v4"
+	} else {
+		a.APIVersion = "v3"
+	}
+}
+
 func (a *Adaptor) GetRequestURL(meta *util.RelayMeta) (string, error) {
+	a.SetVersionByModeName(meta.ActualModelName)
+	if a.APIVersion == "v4" {
+		return fmt.Sprintf("%s/api/paas/v4/chat/completions", meta.BaseURL), nil
+	}
 	method := "invoke"
 	if meta.IsStream {
 		method = "sse-invoke"
@@ -37,6 +52,13 @@ func (a *Adaptor) ConvertRequest(c *gin.Context, relayMode int, request *model.G
 	if request == nil {
 		return nil, errors.New("request is nil")
 	}
+	if request.TopP >= 1 {
+		request.TopP = 0.99
+	}
+	a.SetVersionByModeName(request.Model)
+	if a.APIVersion == "v4" {
+		return request, nil
+	}
 	return ConvertRequest(*request), nil
 }
 
@@ -44,7 +66,19 @@ func (a *Adaptor) DoRequest(c *gin.Context, meta *util.RelayMeta, requestBody io
 	return channel.DoRequestHelper(a, c, meta, requestBody)
 }
 
+func (a *Adaptor) DoResponseV4(c *gin.Context, resp *http.Response, meta *util.RelayMeta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	if meta.IsStream {
+		err, _, usage = openai.StreamHandler(c, resp, meta.Mode)
+	} else {
+		err, usage = openai.Handler(c, resp, meta.PromptTokens, meta.ActualModelName)
+	}
+	return
+}
+
 func (a *Adaptor) DoResponse(c *gin.Context, resp *http.Response, meta *util.RelayMeta) (usage *model.Usage, err *model.ErrorWithStatusCode) {
+	if a.APIVersion == "v4" {
+		return a.DoResponseV4(c, resp, meta)
+	}
 	if meta.IsStream {
 		err, usage = StreamHandler(c, resp)
 	} else {

relay/channel/zhipu/constants.go

@@ -2,4 +2,5 @@ package zhipu
 
 var ModelList = []string{
 	"chatglm_turbo", "chatglm_pro", "chatglm_std", "chatglm_lite",
+	"glm-4", "glm-4v", "glm-3-turbo",
 }

relay/constant/image.go

@@ -0,0 +1,24 @@
+package constant
+
+var DalleSizeRatios = map[string]map[string]float64{
+	"dall-e-2": {
+		"256x256":   1,
+		"512x512":   1.125,
+		"1024x1024": 1.25,
+	},
+	"dall-e-3": {
+		"1024x1024": 1,
+		"1024x1792": 2,
+		"1792x1024": 2,
+	},
+}
+
+var DalleGenerationImageAmounts = map[string][2]int{
+	"dall-e-2": {1, 10},
+	"dall-e-3": {1, 1}, // OpenAI allows n=1 currently.
+}
+
+var DalleImagePromptLengthLimitations = map[string]int{
+	"dall-e-2": 1000,
+	"dall-e-3": 4000,
+}

relay/controller/helper.go

@@ -36,6 +36,65 @@ func getAndValidateTextRequest(c *gin.Context, relayMode int) (*relaymodel.Gener
 	return textRequest, nil
 }
 
+func getImageRequest(c *gin.Context, relayMode int) (*openai.ImageRequest, error) {
+	imageRequest := &openai.ImageRequest{}
+	err := common.UnmarshalBodyReusable(c, imageRequest)
+	if err != nil {
+		return nil, err
+	}
+	if imageRequest.N == 0 {
+		imageRequest.N = 1
+	}
+	if imageRequest.Size == "" {
+		imageRequest.Size = "1024x1024"
+	}
+	if imageRequest.Model == "" {
+		imageRequest.Model = "dall-e-2"
+	}
+	return imageRequest, nil
+}
+
+func validateImageRequest(imageRequest *openai.ImageRequest, meta *util.RelayMeta) *relaymodel.ErrorWithStatusCode {
+	// model validation
+	_, hasValidSize := constant.DalleSizeRatios[imageRequest.Model][imageRequest.Size]
+	if !hasValidSize {
+		return openai.ErrorWrapper(errors.New("size not supported for this image model"), "size_not_supported", http.StatusBadRequest)
+	}
+	// check prompt length
+	if imageRequest.Prompt == "" {
+		return openai.ErrorWrapper(errors.New("prompt is required"), "prompt_missing", http.StatusBadRequest)
+	}
+	if len(imageRequest.Prompt) > constant.DalleImagePromptLengthLimitations[imageRequest.Model] {
+		return openai.ErrorWrapper(errors.New("prompt is too long"), "prompt_too_long", http.StatusBadRequest)
+	}
+	// Number of generated images validation
+	if !isWithinRange(imageRequest.Model, imageRequest.N) {
+		// channel not azure
+		if meta.ChannelType != common.ChannelTypeAzure {
+			return openai.ErrorWrapper(errors.New("invalid value of n"), "n_not_within_range", http.StatusBadRequest)
+		}
+	}
+	return nil
+}
+
+func getImageCostRatio(imageRequest *openai.ImageRequest) (float64, error) {
+	if imageRequest == nil {
+		return 0, errors.New("imageRequest is nil")
+	}
+	imageCostRatio, hasValidSize := constant.DalleSizeRatios[imageRequest.Model][imageRequest.Size]
+	if !hasValidSize {
+		return 0, fmt.Errorf("size not supported for this image model: %s", imageRequest.Size)
+	}
+	if imageRequest.Quality == "hd" && imageRequest.Model == "dall-e-3" {
+		if imageRequest.Size == "1024x1024" {
+			imageCostRatio *= 2
+		} else {
+			imageCostRatio *= 1.5
+		}
+	}
+	return imageCostRatio, nil
+}
+
 func getPromptTokens(textRequest *relaymodel.GeneralOpenAIRequest, relayMode int) int {
 	switch relayMode {
 	case constant.RelayModeChatCompletions:

relay/controller/image.go

@@ -10,6 +10,7 @@ import (
 	"github.com/songquanpeng/one-api/common/logger"
 	"github.com/songquanpeng/one-api/model"
 	"github.com/songquanpeng/one-api/relay/channel/openai"
+	"github.com/songquanpeng/one-api/relay/constant"
 	relaymodel "github.com/songquanpeng/one-api/relay/model"
 	"github.com/songquanpeng/one-api/relay/util"
 	"io"
@@ -20,120 +21,65 @@ import (
 )
 
 func isWithinRange(element string, value int) bool {
-	if _, ok := common.DalleGenerationImageAmounts[element]; !ok {
+	if _, ok := constant.DalleGenerationImageAmounts[element]; !ok {
 		return false
 	}
-	min := common.DalleGenerationImageAmounts[element][0]
+	min := constant.DalleGenerationImageAmounts[element][0]
-	max := common.DalleGenerationImageAmounts[element][1]
+	max := constant.DalleGenerationImageAmounts[element][1]
 
 	return value >= min && value <= max
 }
 
 func RelayImageHelper(c *gin.Context, relayMode int) *relaymodel.ErrorWithStatusCode {
-	imageModel := "dall-e-2"
+	ctx := c.Request.Context()
-	imageSize := "1024x1024"
+	meta := util.GetRelayMeta(c)
-
+	imageRequest, err := getImageRequest(c, meta.Mode)
-	tokenId := c.GetInt("token_id")
-	channelType := c.GetInt("channel")
-	channelId := c.GetInt("channel_id")
-	userId := c.GetInt("id")
-	group := c.GetString("group")
-
-	var imageRequest openai.ImageRequest
-	err := common.UnmarshalBodyReusable(c, &imageRequest)
 	if err != nil {
-		return openai.ErrorWrapper(err, "bind_request_body_failed", http.StatusBadRequest)
+		logger.Errorf(ctx, "getImageRequest failed: %s", err.Error())
-	}
+		return openai.ErrorWrapper(err, "invalid_image_request", http.StatusBadRequest)
-
-	if imageRequest.N == 0 {
-		imageRequest.N = 1
-	}
-
-	// Size validation
-	if imageRequest.Size != "" {
-		imageSize = imageRequest.Size
-	}
-
-	// Model validation
-	if imageRequest.Model != "" {
-		imageModel = imageRequest.Model
-	}
-
-	imageCostRatio, hasValidSize := common.DalleSizeRatios[imageModel][imageSize]
-
-	// Check if model is supported
-	if hasValidSize {
-		if imageRequest.Quality == "hd" && imageModel == "dall-e-3" {
-			if imageSize == "1024x1024" {
-				imageCostRatio *= 2
-			} else {
-				imageCostRatio *= 1.5
-			}
-		}
-	} else {
-		return openai.ErrorWrapper(errors.New("size not supported for this image model"), "size_not_supported", http.StatusBadRequest)
-	}
-
-	// Prompt validation
-	if imageRequest.Prompt == "" {
-		return openai.ErrorWrapper(errors.New("prompt is required"), "prompt_missing", http.StatusBadRequest)
-	}
-
-	// Check prompt length
-	if len(imageRequest.Prompt) > common.DalleImagePromptLengthLimitations[imageModel] {
-		return openai.ErrorWrapper(errors.New("prompt is too long"), "prompt_too_long", http.StatusBadRequest)
-	}
-
-	// Number of generated images validation
-	if !isWithinRange(imageModel, imageRequest.N) {
-		// channel not azure
-		if channelType != common.ChannelTypeAzure {
-			return openai.ErrorWrapper(errors.New("invalid value of n"), "n_not_within_range", http.StatusBadRequest)
-		}
 	}
 
 	// map model name
-	modelMapping := c.GetString("model_mapping")
+	var isModelMapped bool
-	isModelMapped := false
+	meta.OriginModelName = imageRequest.Model
-	if modelMapping != "" {
+	imageRequest.Model, isModelMapped = util.GetMappedModelName(imageRequest.Model, meta.ModelMapping)
-		modelMap := make(map[string]string)
+	meta.ActualModelName = imageRequest.Model
-		err := json.Unmarshal([]byte(modelMapping), &modelMap)
+
-		if err != nil {
+	// model validation
-			return openai.ErrorWrapper(err, "unmarshal_model_mapping_failed", http.StatusInternalServerError)
+	bizErr := validateImageRequest(imageRequest, meta)
-		}
+	if bizErr != nil {
-		if modelMap[imageModel] != "" {
+		return bizErr
-			imageModel = modelMap[imageModel]
-			isModelMapped = true
-		}
 	}
-	baseURL := common.ChannelBaseURLs[channelType]
+
+	imageCostRatio, err := getImageCostRatio(imageRequest)
+	if err != nil {
+		return openai.ErrorWrapper(err, "get_image_cost_ratio_failed", http.StatusInternalServerError)
+	}
+
 	requestURL := c.Request.URL.String()
-	if c.GetString("base_url") != "" {
+	fullRequestURL := util.GetFullRequestURL(meta.BaseURL, requestURL, meta.ChannelType)
-		baseURL = c.GetString("base_url")
+	if meta.ChannelType == common.ChannelTypeAzure {
-	}
-	fullRequestURL := util.GetFullRequestURL(baseURL, requestURL, channelType)
-	if channelType == common.ChannelTypeAzure {
 		// https://learn.microsoft.com/en-us/azure/ai-services/openai/dall-e-quickstart?tabs=dalle3%2Ccommand-line&pivots=rest-api
 		apiVersion := util.GetAzureAPIVersion(c)
 		// https://{resource_name}.openai.azure.com/openai/deployments/dall-e-3/images/generations?api-version=2023-06-01-preview
-		fullRequestURL = fmt.Sprintf("%s/openai/deployments/%s/images/generations?api-version=%s", baseURL, imageModel, apiVersion)
+		fullRequestURL = fmt.Sprintf("%s/openai/deployments/%s/images/generations?api-version=%s", meta.BaseURL, imageRequest.Model, apiVersion)
 	}
 
 	var requestBody io.Reader
-	if isModelMapped || channelType == common.ChannelTypeAzure { // make Azure channel request body
+	if isModelMapped || meta.ChannelType == common.ChannelTypeAzure { // make Azure channel request body
 		jsonStr, err := json.Marshal(imageRequest)
 		if err != nil {
-			return openai.ErrorWrapper(err, "marshal_text_request_failed", http.StatusInternalServerError)
+			return openai.ErrorWrapper(err, "marshal_image_request_failed", http.StatusInternalServerError)
 		}
 		requestBody = bytes.NewBuffer(jsonStr)
 	} else {
 		requestBody = c.Request.Body
 	}
 
-	modelRatio := common.GetModelRatio(imageModel)
+	modelRatio := common.GetModelRatio(imageRequest.Model)
-	groupRatio := common.GetGroupRatio(group)
+	groupRatio := common.GetGroupRatio(meta.Group)
 	ratio := modelRatio * groupRatio
-	userQuota, err := model.CacheGetUserQuota(userId)
+	userQuota, err := model.CacheGetUserQuota(meta.UserId)
 
 	quota := int(ratio*imageCostRatio*1000) * imageRequest.N
 
@@ -146,7 +92,7 @@ func RelayImageHelper(c *gin.Context, relayMode int) *relaymodel.ErrorWithStatus
 		return openai.ErrorWrapper(err, "new_request_failed", http.StatusInternalServerError)
 	}
 	token := c.Request.Header.Get("Authorization")
-	if channelType == common.ChannelTypeAzure { // Azure authentication
+	if meta.ChannelType == common.ChannelTypeAzure { // Azure authentication
 		token = strings.TrimPrefix(token, "Bearer ")
 		req.Header.Set("api-key", token)
 	} else {
@@ -169,25 +115,25 @@ func RelayImageHelper(c *gin.Context, relayMode int) *relaymodel.ErrorWithStatus
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_request_body_failed", http.StatusInternalServerError)
 	}
-	var textResponse openai.ImageResponse
+	var imageResponse openai.ImageResponse
 
 	defer func(ctx context.Context) {
 		if resp.StatusCode != http.StatusOK {
 			return
 		}
-		err := model.PostConsumeTokenQuota(tokenId, quota)
+		err := model.PostConsumeTokenQuota(meta.TokenId, quota)
 		if err != nil {
 			logger.SysError("error consuming token remain quota: " + err.Error())
 		}
-		err = model.CacheUpdateUserQuota(userId)
+		err = model.CacheUpdateUserQuota(meta.UserId)
 		if err != nil {
 			logger.SysError("error update user quota cache: " + err.Error())
 		}
 		if quota != 0 {
 			tokenName := c.GetString("token_name")
 			logContent := fmt.Sprintf("模型倍率 %.2f，分组倍率 %.2f", modelRatio, groupRatio)
-			model.RecordConsumeLog(ctx, userId, channelId, 0, 0, imageModel, tokenName, quota, logContent)
+			model.RecordConsumeLog(ctx, meta.UserId, meta.ChannelId, 0, 0, imageRequest.Model, tokenName, quota, logContent)
-			model.UpdateUserUsedQuotaAndRequestCount(userId, quota)
+			model.UpdateUserUsedQuotaAndRequestCount(meta.UserId, quota)
 			channelId := c.GetInt("channel_id")
 			model.UpdateChannelUsedQuota(channelId, quota)
 		}
@@ -202,7 +148,7 @@ func RelayImageHelper(c *gin.Context, relayMode int) *relaymodel.ErrorWithStatus
 	if err != nil {
 		return openai.ErrorWrapper(err, "close_response_body_failed", http.StatusInternalServerError)
 	}
-	err = json.Unmarshal(responseBody, &textResponse)
+	err = json.Unmarshal(responseBody, &imageResponse)
 	if err != nil {
 		return openai.ErrorWrapper(err, "unmarshal_response_body_failed", http.StatusInternalServerError)
 	}

web/berry/src/views/Channel/type/Config.js

@@ -67,7 +67,7 @@ const typeConfig = {
   },
   16: {
     input: {
-      models: ["chatglm_turbo", "chatglm_pro", "chatglm_std", "chatglm_lite"],
+      models: ["glm-4", "glm-4v", "glm-3-turbo", "chatglm_turbo", "chatglm_pro", "chatglm_std", "chatglm_lite"],
     },
     modelGroup: "zhipu",
   },

web/default/src/pages/Channel/EditChannel.js

@@ -79,7 +79,7 @@ const EditChannel = () => {
           localModels = [...localModels, ...withInternetVersion];
           break;
         case 16:
-          localModels = ['chatglm_turbo', 'chatglm_pro', 'chatglm_std', 'chatglm_lite'];
+          localModels = ["glm-4", "glm-4v", "glm-3-turbo",'chatglm_turbo', 'chatglm_pro', 'chatglm_std', 'chatglm_lite'];
           break;
         case 18:
           localModels = [