hhyykk/ipms-sjy
1
1
Fork 0
mirror of https://gitee.com/hhyykk/ipms-sjy.git synced 2025-07-25 08:25:07 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

【代码优化】AI：MJ 生成图片的优化

Browse Source
This commit is contained in:
YunaiV
2024-06-25 19:57:37 +08:00
parent 88142ed74c
commit 098483d2be
23 changed files with 333 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/config/YudaoMidjourneyConfiguration.java
View File

@ -3,7 +3,6 @@ package cn.iocoder.yudao.module.ai.config;
import cn.iocoder.yudao.framework.ai.core.model.midjourney.MidjourneyConfig;
import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

11
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/AiImageController.http
View File

@ -29,12 +29,17 @@ Authorization: {{token}}
"style": "vivid"
}
### chat midjourney
### 生成图片：生成图片
POST {{baseUrl}}/admin-api/ai/image/midjourney
POST {{baseUrl}}/ai/image/midjourney/imagine
Content-Type: application/json
Authorization: {{token}}
{
"prompt": "Cute cartoon style mobile game scene, a colorful camping car with an outdoor table and chairs next to it on the road in a spring forest, the simple structure of the camper van, soft lighting, C4D rendering, 3d model in the style of a cartoon, cute shape, a pastel color scheme, closeup view from the side angle, high resolution, bright colors, a happy atmosphere."
"prompt": "中国旗袍",
"model": "midjourney",
"width": "1",
"height": "1",
"version": "6.0",
"base64Array": []
}

23
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/AiImageController.java
View File

@ -1,14 +1,14 @@
package cn.iocoder.yudao.module.ai.controller.admin.image;
import cn.hutool.core.util.ObjUtil;
import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.MidjourneyNotifyReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.AiImageDrawReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.AiImageMidjourneyImagineReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.AiImageRespVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.midjourney.AiImageMidjourneyImagineReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.image.AiImageDO;
import cn.iocoder.yudao.module.ai.service.image.AiImageService;
import io.swagger.v3.oas.annotations.Operation;
@ -63,19 +63,24 @@ public class AiImageController {
return success(true);
}
// ================ midjourney 接口 ================
// ================ midjourney 专属 ================
@Operation(summary = "Midjourney imagine（绘画）")
@Operation(summary = "【Midjourney】生成图片")
@PostMapping("/midjourney/imagine")
public CommonResult<Long> midjourneyImagine(@Validated @RequestBody AiImageMidjourneyImagineReqVO req) {
return success(imageService.midjourneyImagine(getLoginUserId(), req));
public CommonResult<Long> midjourneyImagine(@Validated @RequestBody AiImageMidjourneyImagineReqVO reqVO) {
if (true) {
imageService.midjourneySync();
return null;
}
Long imageId = imageService.midjourneyImagine(getLoginUserId(), reqVO);
return success(imageId);
}
@Operation(summary = "Midjourney 回调通知", description = "由 Midjourney Proxy 回调")
@Operation(summary = "Midjourney 生成图片的回调通知", description = "由 Midjourney Proxy 回调")
@PostMapping("/midjourney-notify")
@PermitAll
public void midjourneyNotify(@RequestBody MidjourneyNotifyReqVO notifyReqVO) {
imageService.midjourneyNotify(notifyReqVO);
public void midjourneyNotify(@RequestBody MidjourneyApi.Notify notify) {
imageService.midjourneyNotify(notify);
}
@Operation(summary = "Midjourney Action", description = "例如说：放大、缩小、U1、U2 等")

40
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/AiImageMidjourneyImagineReqVO.java
View File

@ -1,40 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import lombok.experimental.Accessors;
import java.util.List;
// TODO @fan：待定
/**
* midjourney req
*
* @author fansili
* @time 2024/4/28 17:42
* @since 1.0
*/
@Data
@Accessors(chain = true)
public class AiImageMidjourneyImagineReqVO {
@Schema(description = "提示词")
@NotNull(message = "提示词不能为空!")
private String prompt;
@Schema(description = "模型(midjourney、niji)")
private String model;
@Schema(description = "图片宽度 --ar 设置")
private Integer width;
@Schema(description = "图片高度 --ar 设置")
private Integer height;
@Schema(description = "版本号 --v 设置")
private String version;
@Schema(description = "垫图(参考图)base64数组")
private List<String> base64Array;
}

30
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/AiImageMidjourneyOperationsVO.java
View File

@ -1,30 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo;
import lombok.Data;
import lombok.experimental.Accessors;
/**
* mj 保存 components 记录
*
* "components": [
* {
* "custom_id": "MJ::JOB::upsample::1::5d32f4e8-8d2f-4bef-82d8-bf517e3c3660",
* "style": 2,
* "label": "U1",
* "type": 2
* },
* ]
*
* @author fansili
* @time 2024/5/8 14:44
* @since 1.0
*/
@Data
@Accessors(chain = true)
public class AiImageMidjourneyOperationsVO {
private String custom_id;
private String style;
private String label;
private String type;
}

6
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/AiImageRespVO.java
View File

@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo;
import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@ -46,14 +47,11 @@ public class AiImageRespVO {
@Schema(description = "绘制参数")
private Map<String, String> options;
@Schema(description = "绘画 response")
private MidjourneyNotifyReqVO response;
// TODO @fan：进度是百分比，还是一个数字哈？感觉这个可以统一成通用字段；
@Schema(description = "mj 进度")
private String progress;
@Schema(description = "mj buttons 按钮")
private List<MidjourneyNotifyReqVO.Button> buttons;
private List<MidjourneyApi.Button> buttons;
}

33
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/MidjourneyImagineReqVO.java
View File

@ -1,33 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
// TODO @fan：待定
/**
* Midjourney：Imagine 请求
*
* @author fansili
* @time 2024/5/30 14:02
* @since 1.0
*/
@Data
public class MidjourneyImagineReqVO {
@Schema(description = "垫图(参考图)base64数组", required = false)
private List<String> base64Array;
@Schema(description = "通知地址", required = false)
@NotNull(message = "回调地址不能为空!")
private String notifyHook;
@Schema(description = "提示词", required = true)
@NotNull(message = "提示词不能为空!")
private String prompt;
@Schema(description = "自定义参数", required = false)
private String state;
}

75
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/MidjourneyNotifyReqVO.java
View File

@ -1,75 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.List;
/**
* Midjourney Proxy 通知回调
*
* - Midjourney Proxy：通知回调 bean 是 com.github.novicezk.midjourney.support.Task
* - 毫秒 api 通知回调文档地址：https://gpt-best.apifox.cn/doc-3530863
*
* @author fansili
* @time 2024/5/31 10:37
* @since 1.0
*/
@Data
public class MidjourneyNotifyReqVO {
@Schema(description = "job id")
private String id;
@Schema(description = "任务类型 MidjourneyTaskActionEnum")
private String action;
@Schema(description = "任务状态 MidjourneyTaskStatusEnum")
private String status;
@Schema(description = "提示词")
private String prompt;
@Schema(description = "提示词-英文")
private String promptEn;
@Schema(description = "任务描述")
private String description;
@Schema(description = "自定义参数")
private String state;
@Schema(description = "提交时间")
private Long submitTime;
@Schema(description = "开始执行时间")
private Long startTime;
@Schema(description = "结束时间")
private Long finishTime;
@Schema(description = "图片url")
private String imageUrl;
@Schema(description = "任务进度")
private String progress;
@Schema(description = "失败原因")
private String failReason;
@Schema(description = "任务完成后的可执行按钮")
private List<Button> buttons;
@Data
public static class Button {
@Schema(description = "MJ::JOB::upsample::1::85a4b4c1-8835-46c5-a15c-aea34fad1862 动作标识")
private String customId;
@Schema(description = "图标 emoji")
private String emoji;
@Schema(description = "Make Variations 文本")
private String label;
@Schema(description = "类型，系统内部使用")
private String type;
@Schema(description = "样式: 2（Primary）、3（Green）")
private String style;
}
}

30
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/MidjourneySubmitRespVO.java
View File

@ -1,30 +0,0 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
import java.util.Map;
// TODO @fan：待定
/**
* Midjourney：Imagine 请求
*
* @author fansili
* @time 2024/5/30 14:02
* @since 1.0
*/
@Data
public class MidjourneySubmitRespVO {
@Schema(description = "状态码: 1(提交成功), 21(已存在), 22(排队中), other(错误)")
private String code;
@Schema(description = "描述")
private String description;
@Schema(description = "扩展字段")
private Map<String, Object> properties;
@Schema(description = "任务ID")
private String result;
}

38
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/controller/admin/image/vo/midjourney/AiImageMidjourneyImagineReqVO.java Normal file
View File

@ -0,0 +1,38 @@
package cn.iocoder.yudao.module.ai.controller.admin.image.vo.midjourney;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.NotNull;
import lombok.Data;
import java.util.List;
@Schema(description = "管理后台 - 绘画生成（Midjourney） Request VO")
@Data
public class AiImageMidjourneyImagineReqVO {
@Schema(description = "提示词", requiredMode = Schema.RequiredMode.REQUIRED, example = "中国神龙")
@NotEmpty(message = "提示词不能为空!")
private String prompt;
@Schema(description = "模型", requiredMode = Schema.RequiredMode.REQUIRED, example = "midjourney")
@NotEmpty(message = "模型不能为空")
private String model; // 参考 MidjourneyApi.ModelEnum
@Schema(description = "图片宽度", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "图片宽度不能为空")
private Integer width;
@Schema(description = "图片高度", requiredMode = Schema.RequiredMode.REQUIRED, example = "1")
@NotNull(message = "图片高度不能为空")
private Integer height;
@Schema(description = "版本号", requiredMode = Schema.RequiredMode.REQUIRED, example = "6.0")
@NotEmpty(message = "版本号不能为空")
private String version;
// TODO @fan：参考图，建议用 referImageUrl。
@Schema(description = "垫图(参考图)base64数组")
private List<String> base64Array;
}

23
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/dataobject/image/AiImageDO.java
View File

@ -1,8 +1,8 @@
package cn.iocoder.yudao.module.ai.dal.dataobject.image;
import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
import cn.iocoder.yudao.framework.common.util.json.JsonUtils;
import cn.iocoder.yudao.framework.mybatis.core.dataobject.BaseDO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.MidjourneyNotifyReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.model.AiChatModelDO;
import cn.iocoder.yudao.module.ai.enums.image.AiImageStatusEnum;
import cn.iocoder.yudao.module.system.api.user.dto.AdminUserRespDTO;
@ -28,8 +28,6 @@ import java.util.Map;
@Data
public class AiImageDO extends BaseDO {
// TODO @fan：1）使用 java 注释哈，不要注解。2）关联、枚举字段，要关联到对应类，参考 AiChatMessageDO 的注释
/**
* 编号
*/
@ -76,7 +74,7 @@ public class AiImageDO extends BaseDO {
*
* 枚举 {@link AiImageStatusEnum}
*/
private String status;
private Integer status;
/**
* 图片地址
@ -96,23 +94,11 @@ public class AiImageDO extends BaseDO {
@TableField(typeHandler = JacksonTypeHandler.class)
private Map<String, Object> options;
/**
* 绘画 response
*/
@TableField(typeHandler = JacksonTypeHandler.class)
private MidjourneyNotifyReqVO response;
// TODO @fan：这个建议 Double
/**
* mj 进度(10%、50%、100%)
*/
private String progress;
/**
* mj buttons 按钮
*/
@TableField(typeHandler = ButtonTypeHandler.class)
private List<MidjourneyNotifyReqVO.Button> buttons;
private List<MidjourneyApi.Button> buttons;
/**
* midjourney proxy 关联的 task id
@ -124,12 +110,11 @@ public class AiImageDO extends BaseDO {
*/
private String errorMessage;
// TODO @芋艿：看看是不是 MidjourneyNotifyReqVO.Button 搞到 MJ API 那
public static class ButtonTypeHandler extends AbstractJsonTypeHandler<Object> {
@Override
protected Object parse(String json) {
return JsonUtils.parseArray(json, MidjourneyNotifyReqVO.Button.class);
return JsonUtils.parseArray(json, MidjourneyApi.Button.class);
}
@Override

46
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/dal/mysql/image/AiImageMapper.java
View File

@ -1,13 +1,10 @@
package cn.iocoder.yudao.module.ai.dal.mysql.image;
import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.mybatis.core.mapper.BaseMapperX;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import cn.iocoder.yudao.module.ai.dal.dataobject.image.AiImageDO;
import cn.iocoder.yudao.module.ai.enums.image.AiImageStatusEnum;
import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper;
import org.apache.ibatis.annotations.Mapper;
import java.util.List;
@ -20,52 +17,19 @@ import java.util.List;
@Mapper
public interface AiImageMapper extends BaseMapperX<AiImageDO> {
// TODO @fan：这个建议，直接使用 update，service 拼接要改的状态哈
/**
* 更新 - 根据 messageId
*
* @param mjNonceId
* @param aiImageDO
*/
default void updateByMjNonce(Long mjNonceId, AiImageDO aiImageDO) {
// this.update(aiImageDO, new LambdaQueryWrapperX<AiImageDO>().eq(AiImageDO::getMjNonceId, mjNonceId));
return;
default AiImageDO selectByTaskId(String taskId) {
return this.selectOne(AiImageDO::getTaskId, taskId);
}
/**
* 查询 - 根据 job id
*
* @param id
* @return
*/
default AiImageDO selectByJobId(String id) {
return this.selectOne(new LambdaQueryWrapperX<AiImageDO>().eq(AiImageDO::getTaskId, id));
}
/**
* 查询 - page
*
* @param userId
* @param pageReqVO
* @return
*/
default PageResult<AiImageDO> selectPage(Long userId, PageParam pageReqVO) {
return selectPage(pageReqVO, new LambdaQueryWrapperX<AiImageDO>()
.eq(AiImageDO::getUserId, userId)
.orderByDesc(AiImageDO::getId));
}
/**
* 查询 - 根据 status 和 platform
*
* @return
*/
default List<AiImageDO> selectByStatusAndPlatform(AiImageStatusEnum statusEnum, AiPlatformEnum platformEnum) {
return this.selectList(new LambdaUpdateWrapper<AiImageDO>()
.eq(AiImageDO::getStatus, statusEnum.getStatus())
.eq(AiImageDO::getPlatform, platformEnum.getPlatform())
);
default List<AiImageDO> selectListByStatusAndPlatform(Integer status, String platform) {
return selectList(AiImageDO::getStatus, status,
AiImageDO::getPlatform, platform);
}
}

79
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/job/MidjourneyJob.java
View File

@ -1,79 +0,0 @@
package cn.iocoder.yudao.module.ai.job;
import cn.hutool.core.collection.CollUtil;
import cn.iocoder.yudao.framework.ai.core.enums.AiPlatformEnum;
import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
import cn.iocoder.yudao.framework.common.util.collection.CollectionUtils;
import cn.iocoder.yudao.framework.common.util.json.JsonUtils;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.framework.quartz.core.handler.JobHandler;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.MidjourneyNotifyReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.image.AiImageDO;
import cn.iocoder.yudao.module.ai.dal.mysql.image.AiImageMapper;
import cn.iocoder.yudao.module.ai.enums.image.AiImageStatusEnum;
import cn.iocoder.yudao.module.ai.service.image.AiImageService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* midjourney job 定时拉去 midjourney 绘制状态
*
* @author fansili
* @time 2024/6/5 14:55
* @since 1.0
*/
@Component
@Slf4j
public class MidjourneyJob implements JobHandler {
// TODO @fan：@Resource
@Autowired(required = false)
private MidjourneyApi midjourneyApi;
@Autowired
private AiImageMapper imageMapper;
@Autowired
private AiImageService imageService;
// TODO @fan：这个方法，建议实现到 AiImageService，例如说 midjourneySync，返回 int 同步数量。
@Override
public String execute(String param) {
// 1、获取 midjourney 平台，状态在 “进行中” 的 image
List<AiImageDO> imageList = imageMapper.selectByStatusAndPlatform(AiImageStatusEnum.IN_PROGRESS, AiPlatformEnum.MIDJOURNEY);
log.info("Midjourney 同步 - 任务数量 {}!", imageList.size());
if (CollUtil.isEmpty(imageList)) {
return "Midjourney 同步 - 数量为空!";
}
log.info("Midjourney 同步 - 开始...");
// 2、批量拉去 task 信息
List<MidjourneyApi.NotifyRequest> taskList = midjourneyApi
.listByCondition(CollectionUtils.convertSet(imageList, AiImageDO::getTaskId));
Map<String, MidjourneyApi.NotifyRequest> taskIdMap
= CollectionUtils.convertMap(taskList, MidjourneyApi.NotifyRequest::id);
// 3、更新 image 状态
List<AiImageDO> updateImageList = new ArrayList<>();
for (AiImageDO aiImageDO : imageList) {
// 3.1 排除掉空的情况
if (!taskIdMap.containsKey(aiImageDO.getTaskId())) {
log.warn("Midjourney 同步 - {} 任务为空!", aiImageDO.getTaskId());
continue;
}
// TODO @ 3.1 和 3.2 是不是融合下；get，然后判空，continue；
// 3.2 获取通知对象
MidjourneyApi.NotifyRequest notifyRequest = taskIdMap.get(aiImageDO.getTaskId());
// 3.2 构建更新对象
// TODO @fan：建议 List<MidjourneyNotifyReqVO> 作为 imageService 去更新；
// TODO @芋艿 BeanUtils.toBean 转换为 null
updateImageList.add(imageService.buildUpdateImage(aiImageDO.getId(),
JsonUtils.parseObject(JsonUtils.toJsonString(notifyRequest), MidjourneyNotifyReqVO.class)));
}
// 4、批了更新 updateImageList
imageMapper.updateBatch(updateImageList);
return "Midjourney 同步 - ".concat(String.valueOf(updateImageList.size())).concat(" 任务!");
}
}

28
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/job/image/MidjourneySyncJob.java Normal file
View File

@ -0,0 +1,28 @@
package cn.iocoder.yudao.module.ai.job.image;
import cn.iocoder.yudao.framework.quartz.core.handler.JobHandler;
import cn.iocoder.yudao.module.ai.service.image.AiImageService;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
/**
* Midjourney 同步 Job：定时拉去 midjourney 绘制状态
*
* @author fansili
*/
@Component
@Slf4j
public class MidjourneySyncJob implements JobHandler {
@Resource
private AiImageService imageService;
@Override
public String execute(String param) {
Integer count = imageService.midjourneySync();
log.info("[execute][同步 Midjourney ({}) 个]", count);
return String.format("同步 Midjourney %s 个", count);
}
}

46
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/image/AiImageService.java
View File

@ -1,10 +1,10 @@
package cn.iocoder.yudao.module.ai.service.image;
import cn.iocoder.yudao.framework.ai.core.model.midjourney.api.MidjourneyApi;
import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.MidjourneyNotifyReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.AiImageDrawReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.AiImageMidjourneyImagineReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.midjourney.AiImageMidjourneyImagineReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.image.AiImageDO;
/**
@ -40,15 +40,6 @@ public interface AiImageService {
*/
Long drawImage(Long userId, AiImageDrawReqVO drawReqVO);
/**
* Midjourney imagine（绘画）
*
* @param userId 用户编号
* @param imagineReqVO 绘制请求
* @return 绘画编号
*/
Long midjourneyImagine(Long userId, AiImageMidjourneyImagineReqVO imagineReqVO);
/**
* 删除【我的】绘画记录
*
@ -57,22 +48,30 @@ public interface AiImageService {
*/
void deleteImageMy(Long id, Long userId);
/**
* midjourney proxy - 回调通知
*
* @param notifyReqVO
* @return
*/
void midjourneyNotify(MidjourneyNotifyReqVO notifyReqVO);
// ================ midjourney 专属 ================
/**
* 构建 midjourney - 更新对象
* 【Midjourney】生成图片
*
* @param imageId
* @param notifyReqVO
* @return
* @param userId 用户编号
* @param reqVO 绘制请求
* @return 绘画编号
*/
AiImageDO buildUpdateImage(Long imageId, MidjourneyNotifyReqVO notifyReqVO);
Long midjourneyImagine(Long userId, AiImageMidjourneyImagineReqVO reqVO);
/**
* 【Midjourney】同步图片进展
*
* @return 同步成功数量
*/
Integer midjourneySync();
/**
* 【Midjourney】通知图片进展
*
* @param notify 通知
*/
void midjourneyNotify(MidjourneyApi.Notify notify);
/**
* midjourney - action(放大、缩小、U1、U2...)
@ -83,4 +82,5 @@ public interface AiImageService {
* @return
*/
void midjourneyAction(Long loginUserId, Long imageId, String customId);
}

223
yudao-module-ai/yudao-module-ai-biz/src/main/java/cn/iocoder/yudao/module/ai/service/image/AiImageServiceImpl.java
View File

@ -2,7 +2,7 @@ package cn.iocoder.yudao.module.ai.service.image;
import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.codec.Base64;
import cn.hutool.core.exceptions.ExceptionUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.ObjUtil;
import cn.hutool.core.util.StrUtil;
@ -14,8 +14,7 @@ import cn.iocoder.yudao.framework.common.pojo.PageParam;
import cn.iocoder.yudao.framework.common.pojo.PageResult;
import cn.iocoder.yudao.framework.common.util.object.BeanUtils;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.AiImageDrawReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.AiImageMidjourneyImagineReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.MidjourneyNotifyReqVO;
import cn.iocoder.yudao.module.ai.controller.admin.image.vo.midjourney.AiImageMidjourneyImagineReqVO;
import cn.iocoder.yudao.module.ai.dal.dataobject.image.AiImageDO;
import cn.iocoder.yudao.module.ai.dal.mysql.image.AiImageMapper;
import cn.iocoder.yudao.module.ai.enums.image.AiImageStatusEnum;
@ -29,15 +28,17 @@ import org.springframework.ai.image.ImagePrompt;
import org.springframework.ai.image.ImageResponse;
import org.springframework.ai.openai.OpenAiImageOptions;
import org.springframework.ai.stabilityai.api.StabilityAiImageOptions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import java.util.List;
import java.util.Map;
import static cn.iocoder.yudao.framework.common.exception.util.ServiceExceptionUtil.exception;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertMap;
import static cn.iocoder.yudao.framework.common.util.collection.CollectionUtils.convertSet;
import static cn.iocoder.yudao.module.ai.enums.ErrorCodeConstants.*;
/**
@ -51,12 +52,16 @@ public class AiImageServiceImpl implements AiImageService {
@Resource
private AiImageMapper imageMapper;
@Resource
private FileApi fileApi;
@Resource
private AiApiKeyService apiKeyService;
@Autowired(required = false)
@Resource
private MidjourneyApi midjourneyApi;
@Value("${ai.midjourney-proxy.notifyUrl:http://127.0.0.1:48080/admin-api/ai/image/midjourney-notify}")
private String midjourneyNotifyUrl;
@ -74,7 +79,7 @@ public class AiImageServiceImpl implements AiImageService {
public Long drawImage(Long userId, AiImageDrawReqVO drawReqVO) {
// 1. 保存数据库
AiImageDO image = BeanUtils.toBean(drawReqVO, AiImageDO.class).setUserId(userId).setPublicStatus(false)
.setWidth(drawReqVO.getWidth()).setHeight(drawReqVO.getHeight()).setStatus(AiImageStatusEnum.IN_PROGRESS.getStatus());
.setStatus(AiImageStatusEnum.IN_PROGRESS.getStatus());
imageMapper.insert(image);
// 2. 异步绘制，后续前端通过返回的 id 进行轮询结果
getSelf().executeDrawImage(image, drawReqVO);
@ -121,47 +126,6 @@ public class AiImageServiceImpl implements AiImageService {
throw new IllegalArgumentException("不支持的 AI 平台：" + draw.getPlatform());
}
@Override
@Transactional(rollbackFor = Exception.class)
public Long midjourneyImagine(Long userId, AiImageMidjourneyImagineReqVO req) {
// 1、构建 AiImageDO 并 保存
AiImageDO image = new AiImageDO()
.setUserId(userId)
.setPrompt(req.getPrompt())
.setPlatform(AiPlatformEnum.MIDJOURNEY.getPlatform())
.setModel(req.getModel())
.setWidth(req.getWidth())
.setHeight(req.getHeight())
.setStatus(AiImageStatusEnum.IN_PROGRESS.getStatus());
imageMapper.insert(image);
// 3、调用 MidjourneyProxy 提交任务
// 3.1、设置 midjourney 扩展参数
MidjourneyApi.ImagineRequest imagineRequest = new MidjourneyApi.ImagineRequest(null, midjourneyNotifyUrl, req.getPrompt(),
buildParams(req.getWidth(), req.getHeight(), req.getVersion(),
MidjourneyApi.ModelEnum.valueOfModel(req.getModel())));
// 3.2、提交绘画请求
// TODO @fan：5 这里，失败的情况，到底抛出异常，还是 RespVO，可以参考 OpenAI 的 API 封装
MidjourneyApi.SubmitResponse submitResponse = midjourneyApi.imagine(imagineRequest);
// 4、保存任务 id (状态码: 1(提交成功), 21(已存在), 22(排队中), other(错误))
if (!MidjourneyApi.SubmitCodeEnum.SUCCESS_CODES.contains(submitResponse.code())) {
if (submitResponse.description().contains("quota_not_enough")) {
throw exception(AI_IMAGE_SYSTEM_ACCOUNT_INSUFFICIENT_BALANCE, submitResponse.description());
}
throw exception(AI_IMAGE_MIDJOURNEY_SUBMIT_FAIL, submitResponse.description());
}
// 4.1、更新 taskId 和参数
imageMapper.updateById(new AiImageDO()
.setId(image.getId())
.setTaskId(submitResponse.result())
.setOptions(BeanUtil.beanToMap(req))
);
return image.getId();
}
@Override
public void deleteImageMy(Long id, Long userId) {
// 1. 校验是否存在
@ -173,50 +137,111 @@ public class AiImageServiceImpl implements AiImageService {
imageMapper.deleteById(id);
}
@Override
public void midjourneyNotify(MidjourneyNotifyReqVO notifyReqVO) {
// 1、根据 job id 查询关联的 image
AiImageDO image = imageMapper.selectByJobId(notifyReqVO.getId());
private AiImageDO validateImageExists(Long id) {
AiImageDO image = imageMapper.selectById(id);
if (image == null) {
log.warn("midjourneyNotify 回调的 jobId 不存在! jobId: {}", notifyReqVO.getId());
throw exception(AI_IMAGE_NOT_EXISTS);
}
// 2、转换状态
AiImageDO updateImage = buildUpdateImage(image.getId(), notifyReqVO);
// 3、更新 image 状态
imageMapper.updateById(updateImage);
return image;
}
public AiImageDO buildUpdateImage(Long imageId, MidjourneyNotifyReqVO notifyReqVO) {
// 1、转换状态
String imageStatus = null;
if (StrUtil.isNotBlank(notifyReqVO.getStatus())) {
MidjourneyApi.TaskStatusEnum taskStatusEnum = MidjourneyApi.TaskStatusEnum.valueOf(notifyReqVO.getStatus());
// ================ midjourney 专属 ================
@Override
@Transactional(rollbackFor = Exception.class)
public Long midjourneyImagine(Long userId, AiImageMidjourneyImagineReqVO reqVO) {
// 1. 保存数据库
AiImageDO image = BeanUtils.toBean(reqVO, AiImageDO.class).setUserId(userId).setPublicStatus(false)
.setStatus(AiImageStatusEnum.IN_PROGRESS.getStatus())
.setPlatform(AiPlatformEnum.MIDJOURNEY.getPlatform());
imageMapper.insert(image);
// 2. 调用 Midjourney Proxy 提交任务
MidjourneyApi.ImagineRequest imagineRequest = new MidjourneyApi.ImagineRequest(
null, midjourneyNotifyUrl, reqVO.getPrompt(),
MidjourneyApi.ImagineRequest.buildState(reqVO.getWidth(), reqVO.getHeight(), reqVO.getVersion(), reqVO.getModel()));
MidjourneyApi.SubmitResponse imagineResponse = midjourneyApi.imagine(imagineRequest);
// 3. 情况一【失败】：抛出业务异常
if (!MidjourneyApi.SubmitCodeEnum.SUCCESS_CODES.contains(imagineResponse.code())) {
String description = imagineResponse.description().contains("quota_not_enough") ?
"账户余额不足" : imagineResponse.description();
throw exception(AI_IMAGE_MIDJOURNEY_SUBMIT_FAIL, description);
}
// 4. 情况二【成功】：更新 taskId 和参数
imageMapper.updateById(new AiImageDO()
.setId(image.getId())
.setTaskId(imagineResponse.result())
.setOptions(BeanUtil.beanToMap(reqVO))
);
return image.getId();
}
@Override
public Integer midjourneySync() {
// 1.1 获取 Midjourney 平台，状态在 “进行中” 的 image
List<AiImageDO> imageList = imageMapper.selectListByStatusAndPlatform(
AiImageStatusEnum.IN_PROGRESS.getStatus(), AiPlatformEnum.MIDJOURNEY.getPlatform());
if (CollUtil.isEmpty(imageList)) {
return 0;
}
// 1.2 调用 Midjourney Proxy 获取任务进展
List<MidjourneyApi.Notify> taskList = midjourneyApi.getTaskList(convertSet(imageList, AiImageDO::getTaskId));
Map<String, MidjourneyApi.Notify> taskMap = convertMap(taskList, MidjourneyApi.Notify::id);
// 2. 逐个处理，更新进展
int count = 0;
for (AiImageDO image : imageList) {
MidjourneyApi.Notify notify = taskMap.get(image.getTaskId());
if (notify == null) {
log.error("[midjourneySync][image({}) 查询不到进展]", image);
continue;
}
count++;
updateMidjourneyStatus(image, notify);
}
return count;
}
@Override
public void midjourneyNotify(MidjourneyApi.Notify notify) {
// 1. 校验 image 存在
AiImageDO image = imageMapper.selectByTaskId(notify.id());
if (image == null) {
log.warn("[midjourneyNotify][回调任务({}) 不存在]", notify.id());
return;
}
// 2. 更新状态
updateMidjourneyStatus(image, notify);
}
private void updateMidjourneyStatus(AiImageDO image, MidjourneyApi.Notify notify) {
// 1. 转换状态
Integer status = null;
if (StrUtil.isNotBlank(notify.status())) {
MidjourneyApi.TaskStatusEnum taskStatusEnum = MidjourneyApi.TaskStatusEnum.valueOf(notify.status());
if (MidjourneyApi.TaskStatusEnum.SUCCESS == taskStatusEnum) {
imageStatus = AiImageStatusEnum.SUCCESS.getStatus();
status = AiImageStatusEnum.SUCCESS.getStatus();
} else if (MidjourneyApi.TaskStatusEnum.FAILURE == taskStatusEnum) {
imageStatus = AiImageStatusEnum.FAIL.getStatus();
status = AiImageStatusEnum.FAIL.getStatus();
}
}
// 2、上传图片
String filePath = null;
if (!StrUtil.isBlank(notifyReqVO.getImageUrl())) {
// 2. 上传图片
String picUrl = null;
if (StrUtil.isNotBlank(notify.imageUrl())) {
try {
filePath = fileApi.createFile(HttpUtil.downloadBytes(notifyReqVO.getImageUrl()));
picUrl = fileApi.createFile(HttpUtil.downloadBytes(notify.imageUrl()));
} catch (Exception e) {
log.warn("midjourneyNotify 图片上传失败! {} 异常：{}", notifyReqVO.getImageUrl(), ExceptionUtil.getMessage(e));
picUrl = notify.imageUrl();
log.warn("[updateMidjourneyStatus][图片({}) 地址({}) 上传失败]", image.getId(), notify.imageUrl(), e);
}
}
// 3、更新 image 状态
return new AiImageDO()
.setId(imageId)
.setStatus(imageStatus)
.setPicUrl(filePath)
.setProgress(notifyReqVO.getProgress())
.setResponse(notifyReqVO)
.setButtons(notifyReqVO.getButtons())
.setErrorMessage(notifyReqVO.getFailReason());
// 3. 更新 image 状态
imageMapper.updateById(new AiImageDO().setId(image.getId()).setStatus(status)
.setPicUrl(picUrl).setButtons(notify.buttons()).setErrorMessage(notify.failReason()));
}
@Override
@ -236,7 +261,6 @@ public class AiImageServiceImpl implements AiImageService {
// 5、新增 image 记录(根据 image 新增一个)
AiImageDO newImage = new AiImageDO();
newImage.setId(null);
newImage.setUserId(image.getUserId());
newImage.setPrompt(image.getPrompt());
@ -248,20 +272,15 @@ public class AiImageServiceImpl implements AiImageService {
newImage.setStatus(AiImageStatusEnum.IN_PROGRESS.getStatus());
newImage.setPublicStatus(image.getPublicStatus());
newImage.setPicUrl(null);
newImage.setProgress(null);
newImage.setButtons(null);
newImage.setOptions(image.getOptions());
newImage.setResponse(image.getResponse());
newImage.setTaskId(submitResponse.result());
newImage.setErrorMessage(null);
imageMapper.insert(newImage);
}
private static void validateCustomId(String customId, List<MidjourneyNotifyReqVO.Button> buttons) {
private static void validateCustomId(String customId, List<MidjourneyApi.Button> buttons) {
boolean isTrue = false;
for (MidjourneyNotifyReqVO.Button button : buttons) {
if (button.getCustomId().equals(customId)) {
for (MidjourneyApi.Button button : buttons) {
if (button.customId().equals(customId)) {
isTrue = true;
break;
}
@ -271,14 +290,6 @@ public class AiImageServiceImpl implements AiImageService {
}
}
private AiImageDO validateImageExists(Long id) {
AiImageDO image = imageMapper.selectById(id);
if (image == null) {
throw exception(AI_IMAGE_NOT_EXISTS);
}
return image;
}
/**
* 获得自身的代理对象，解决 AOP 生效问题
*
@ -288,28 +299,4 @@ public class AiImageServiceImpl implements AiImageService {
return SpringUtil.getBean(getClass());
}
// TODO @fan：这个是不是应该放在 MJ API 的封装里面搞哈？
/**
* 构建 Midjourney 自定义参数
*
* @param width
* @param height
* @param version
* @param model
* @return
*/
private String buildParams(Integer width, Integer height, String version, MidjourneyApi.ModelEnum model) {
StringBuilder params = new StringBuilder();
// --ar 来设置尺寸
params.append(String.format(" --ar %s:%s ", width, height));
// --niji 模型
if (MidjourneyApi.ModelEnum.NIJI == model) {
params.append(String.format(" --niji %s ", version));
} else {
// --v 版本
params.append(String.format(" --v %s ", version));
}
return params.toString();
}
}