no message

.drone.yml

@@ -34,22 +34,8 @@ steps:
       - pwd # 查看当前目录
       - mvn clean package -Dmaven.test.skip=true -P test
 
-      - ls -l /drone/src/kxs-gateway/target # 查看当前目录
-      - cp /drone/src/kxs-gateway/target/*.jar /drone/src/dist/ && cp -r /drone/src/kxs-gateway/target/lib /drone/src/dist/ && cp -r /drone/src/kxs-gateway/target/config /drone/src/dist/
-      # - cp -a ./package.json /nodeServer1 && cp -a ./bootstrap.js /nodeServer1
-      # - cp -a ./node_modules /nodeServer1
-      # - cp -a dist/* fwdocs
-
-  # - name: build-image # 步骤名称
-  #   image: plugins/docker # 使用镜像
-  #   depends_on: [build-project] # 依赖步骤
-  #   settings: # 当前设置
-  #     username: # 账号名称
-  #       from_secret: docker_username
-  #     password: # 账号密码
-  #       from_secret: docker_password
-  #     dockerfile: deploy/Dockerfile # Dockerfile地址， 注意是相对地址
-  #     repo: yxs970707/deploy-web-demo # 镜像名称
+      - ls -l /drone/src/kxs-product/target # 查看当前目录
+#      - cp /drone/src/kxs-gateway/target/*.jar /drone/src/dist/ && cp -r /drone/src/kxs-gateway/target/lib /drone/src/dist/ && cp -r /drone/src/kxs-gateway/target/config /drone/src/dist/ # 复制文件
 
 trigger:
   branch:
@@ -61,56 +47,60 @@ trigger:
 
 # ---
 
-# kind: pipeline
-# type: docker
-# name: deploy
+kind: pipeline
+type: docker
+name: deploy
 
-# depends_on: # 依赖build管道
-#   - build
+depends_on: # 依赖build管道
+  - build
 
-# clone:
-#   disable: true # 禁用拉取
+clone:
+  disable: true # 禁用拉取
 
-# steps:
-#   # - name: scp-project
-#   #   image: appleboy/drone-scp
-#   #   settings:
-#   #     host: 47.109.40.208
-#   #     username: root
-#   #     password: Ym*87540628
-#   #     port: 22
-#   #     command_timeout: 2m
-#   #     target: /home/deploy/web
-#   #     source:
-#   #       - release/*.tar.gz
+steps:
+  - name: scp-project
+    image: appleboy/drone-scp
+    settings:
+      host: 172.27.251.48
+      username: root
+      password: esG1rZb2ZvalCbOc
+      port: 22
+      command_timeout: 2m
+      target: /home/ABServer/webroot/kxs-gateway/
+      source:
+        - echo ====开始拷贝=======
+        - /drone/src/kxs-gateway/target/*.jar
+        - /drone/src/kxs-gateway/target/lib
+        - /drone/src/kxs-gateway/target/config
+        - echo ====结束拷贝=======
 
-#   - name: deploy-project
-#     image: appleboy/drone-ssh
-#     settings:
-#       host: 47.109.40.208
-#         # from_secret: server_host
-#       user: root
-#         # from_secret: server_username
-#       password: Ym*87540628
-#         # from_secret: server_password
-#       port: 22
-#       command_timeout: 2m
-#       script:
-#         - echo ====开始部署=======
-#         - cd /home/ABServer
-#         # - cp -a /home/ABServer/drone/dist/midwayjs/* /home/ABServer/webroot/nodeServer1
-#         # - if [[ "$(docker ps -q nodeServer1 2> /dev/null)" != "" ]];
-#         #     then
-#         #       xxx
-#         - docker restart nodeServer1
-#         # - docker restart nodeServer1
-#         - echo ====部署成功=======
+  # - name: deploy-project
+  #   image: appleboy/drone-ssh
+  #   settings:
+  #     host: 172.27.251.48
+  #       # from_secret: server_host
+  #     user: root
+  #       # from_secret: server_username
+  #     password: esG1rZb2ZvalCbOc
+  #       # from_secret: server_password
+  #     port: 22
+  #     command_timeout: 2m
+  #     script:
+  #       - echo ====开始部署=======
+  #       - cd /home/ABServer/webroot/
+  #       - scp -a /home/ABServer/drone/dist/midwayjs/* /home/ABServer/webroot/nodeServer1
+  #       # - if [[ "$(docker ps -q nodeServer1 2> /dev/null)" != "" ]];
+  #       #     then
+  #       #       xxx
+  #       - docker restart nodeServer1
+  #       # - docker restart nodeServer1
+  #       - echo ====部署成功=======
 
 
-# trigger:
-#   branch:
-#     include:
-#       - master
-#   event:
-#     include:
-#       - push
+trigger:
+  branch:
+    include:
+      - master
+  event:
+    include:
+      - push

db/Dockerfile

@@ -4,8 +4,8 @@ ENV TZ=Asia/Shanghai
 
 RUN ln -sf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-COPY kxs_sys.sql /docker-entrypoint-initdb.d
-
-COPY ./kxs_config.sql /docker-entrypoint-initdb.d
-
-COPY ./kxs_job.sql /docker-entrypoint-initdb.d
+#COPY kxs_sys.sql /docker-entrypoint-initdb.d
+#
+#COPY ./kxs_config.sql /docker-entrypoint-initdb.d
+#
+#COPY ./kxs_job.sql /docker-entrypoint-initdb.d

docker-compose.yml

@@ -13,7 +13,7 @@ services:
     ports:
       - 3306:3306
     volumes:
-      - ./data/mysql/conf:/etc/mysql/conf.d
+      - ./data/mysql/conf/my.cnf:/etc/my.cnf
       - ./data/mysql/logs:/logs
       - ./data/mysql/data:/var/lib/mysql
     networks:
@@ -35,6 +35,7 @@ services:
       - MYSQL_SERVICE_USER=root
       - MYSQL_SERVICE_PASSWORD=root
       - MYSQL_SERVICE_DB_NAME=kxs_config
+      - MYSQL_SERVICE_DB_PARAM=characterEncoding=utf8&connectTimeout=10000&socketTimeout=30000&autoReconnect=true&useSSL=false&allowPublicKeyRetrieval=true
       # 开启鉴权
       - NACOS_AUTH_ENABLE=true
       - NACOS_AUTH_IDENTITY_KEY=2222
@@ -156,6 +157,7 @@ services:
       # 以SEATA_IP作为host注册seata server
       SEATA_IP: kxs-seata
       SEATA_PORT: 8091
+      JMX_OPTS: -Xmx256m -Xms256m
     volumes:
       - "/usr/share/zoneinfo/Asia/Shanghai:/etc/localtime"        #设置系统时区
       - "/usr/share/zoneinfo/Asia/Shanghai:/etc/timezone"  #设置时区
@@ -166,5 +168,3 @@ services:
 
 networks:
   kxs_cloud_network:
-    name: kxs_cloud_network
-    driver: bridge

kxs-gateway/src/main/resources/logback-spring.xml

@@ -1,77 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Copyright (c) 2021 sky4cloud Authors. All Rights Reserved.
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
-
-<configuration debug="false" scan="false">
-	<springProperty scop="context" name="spring.application.name" source="spring.application.name" defaultValue=""/>
-	<property name="log.path" value="logs/${spring.application.name}"/>
-	<!-- 彩色日志格式 -->
-	<property name="CONSOLE_LOG_PATTERN"
-			  value="${CONSOLE_LOG_PATTERN:-%clr(%d{yyyy-MM-dd HH:mm:ss.SSS}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
-	<!-- 彩色日志依赖的渲染类 -->
-	<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter"/>
-	<conversionRule conversionWord="wex"
-					converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter"/>
-	<conversionRule conversionWord="wEx"
-					converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter"/>
-	<!-- Console log output -->
-	<appender name="console" class="ch.qos.logback.core.ConsoleAppender">
-		<encoder>
-			<pattern>${CONSOLE_LOG_PATTERN}</pattern>
-		</encoder>
-	</appender>
-
-	<!-- Log file debug output -->
-	<appender name="debug" class="ch.qos.logback.core.rolling.RollingFileAppender">
-		<file>${log.path}/debug.log</file>
-		<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-			<fileNamePattern>${log.path}/%d{yyyy-MM, aux}/debug.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
-			<maxFileSize>50MB</maxFileSize>
-			<maxHistory>30</maxHistory>
-		</rollingPolicy>
-		<encoder>
-			<pattern>${CONSOLE_LOG_PATTERN}</pattern>
-		</encoder>
-	</appender>
-
-	<!-- Log file error output -->
-	<appender name="error" class="ch.qos.logback.core.rolling.RollingFileAppender">
-		<file>${log.path}/error.log</file>
-		<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-			<fileNamePattern>${log.path}/%d{yyyy-MM}/error.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
-			<maxFileSize>50MB</maxFileSize>
-			<maxHistory>30</maxHistory>
-		</rollingPolicy>
-		<encoder>
-			<pattern>${CONSOLE_LOG_PATTERN}</pattern>
-		</encoder>
-		<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
-			<level>ERROR</level>
-		</filter>
-	</appender>
-
-	<!--nacos 心跳 INFO 屏蔽-->
-	<logger name="com.alibaba.nacos" level="OFF">
-		<appender-ref ref="error"/>
-	</logger>
-
-	<!-- Level: FATAL 0  ERROR 3  WARN 4  INFO 6  DEBUG 7 -->
-	<root level="INFO">
-		<appender-ref ref="console"/>
-		<appender-ref ref="debug"/>
-		<appender-ref ref="error"/>
-	</root>
-</configuration>

kxs-transfer/pom.xml

@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>com.kxs</groupId>
+        <artifactId>kxs-cloud</artifactId>
+        <version>1.1.0</version>
+    </parent>
+
+    <artifactId>kxs-transfer</artifactId>
+
+    <properties>
+        <maven.compiler.source>17</maven.compiler.source>
+        <maven.compiler.target>17</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.aliyun.dts</groupId>
+            <artifactId>dts-new-subscribe-sdk</artifactId>
+            <version>2.0.1</version>
+            <exclusions>
+            <exclusion>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-log4j12</artifactId>
+            </exclusion>
+        </exclusions>
+        </dependency>
+        <!--注册中心客户端-->
+        <dependency>
+            <groupId>com.alibaba.cloud</groupId>
+            <artifactId>spring-cloud-starter-alibaba-nacos-discovery</artifactId>
+        </dependency>
+        <!--配置中心客户端-->
+        <dependency>
+            <groupId>com.alibaba.cloud</groupId>
+            <artifactId>spring-cloud-starter-alibaba-nacos-config</artifactId>
+        </dependency>
+        <!--undertow容器-->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-undertow</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.kxs</groupId>
+            <artifactId>kxs-system-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.kxs</groupId>
+            <artifactId>kxs-product-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.kxs</groupId>
+            <artifactId>kxs-user-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.kxs</groupId>
+            <artifactId>kxs-store-api</artifactId>
+        </dependency>
+        <!--core 工具类-->
+        <dependency>
+            <groupId>com.kxs</groupId>
+            <artifactId>kxs-common-core</artifactId>
+        </dependency>
+        <!-- orm 模块-->
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>mybatis-plus-boot-starter</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.mysql</groupId>
+            <artifactId>mysql-connector-j</artifactId>
+        </dependency>
+        <!--多数据源-->
+        <dependency>
+            <groupId>com.baomidou</groupId>
+            <artifactId>dynamic-datasource-spring-boot3-starter</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>

kxs-transfer/src/main/java/com/kxs/transfer/api/KxsTransferApplication.java

@@ -0,0 +1,18 @@
+package com.kxs.transfer.api;
+
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+/**
+ * 数据迁移模块 写入
+ *
+ * @author 没秃顶的码农
+ * @date 2024-01-19
+ */
+@SpringBootApplication
+public class KxsTransferApplication {
+    public static void main(String[] args) {
+        SpringApplication.run(KxsTransferApplication.class, args);
+    }
+}

kxs-transfer/src/main/java/com/kxs/transfer/api/config/ConsumerAssignConfig.java

@@ -0,0 +1,91 @@
+package com.kxs.transfer.api.config;
+
+import com.aliyun.dts.subscribe.clients.ConsumerContext;
+import com.aliyun.dts.subscribe.clients.DTSConsumer;
+import com.aliyun.dts.subscribe.clients.DefaultDTSConsumer;
+import com.aliyun.dts.subscribe.clients.common.RecordListener;
+import com.aliyun.dts.subscribe.clients.record.OperationType;
+import com.aliyun.dts.subscribe.clients.recordprocessor.DbType;
+import com.aliyun.dts.subscribe.clients.recordprocessor.DefaultRecordPrintListener;
+import com.kxs.transfer.api.service.KxsCheckpointService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * 分布式 DTS 订阅管道配置
+ *
+ * @author 没秃顶的码农
+ * @date 2024-01-19
+ */
+@Configuration(proxyBeanMethods = false)
+@RequiredArgsConstructor
+public class ConsumerAssignConfig {
+
+    private final KxsCheckpointService kxsCheckpointService;
+
+    // kafka broker url
+    String brokerUrl = "your broker url";
+    // topic to consume, partition is 0
+    String topic = "your dts topic";
+    // user password and sid for auth
+    String sid = "your sid";
+    String userName = "your user name";
+    String password = "your password";
+    // initial checkpoint for first seek(a timestamp to set, eg 1566180200 if you want (Mon Aug 19 10:03:21 CST 2019))
+    String initCheckpoint = "start timestamp";
+    // when use subscribe mode, group config is required. kafka consumer group is enabled
+    ConsumerContext.ConsumerSubscribeMode subscribeMode = ConsumerContext.ConsumerSubscribeMode.ASSIGN;
+    // if force use config checkpoint when start. for checkpoint reset, only assign mode works
+    boolean isForceUseInitCheckpoint = true;
+
+    /**
+     * 初始化 初始化消费者
+     *
+     * @return {@link DTSConsumer}
+     */
+    @Bean
+    public DTSConsumer initDTSClient() {
+        ConsumerContext consumerContext = new ConsumerContext(brokerUrl, topic, sid, userName, password, initCheckpoint, subscribeMode);
+
+        //if this parameter is set, force to use the initCheckpoint to initial
+        consumerContext.setForceUseCheckpoint(isForceUseInitCheckpoint);
+
+        //add user store
+        consumerContext.setUserRegisteredStore(new UserMetaStore(kxsCheckpointService));
+
+        DTSConsumer dtsConsumer = new DefaultDTSConsumer(consumerContext);
+
+        dtsConsumer.addRecordListeners(buildRecordListener());
+        dtsConsumer.start();
+        return dtsConsumer;
+    }
+
+    public static Map<String, RecordListener> buildRecordListener() {
+        // user can impl their own listener
+        RecordListener mysqlRecordPrintListener = record -> {
+
+            OperationType operationType = record.getOperationType();
+
+            if(operationType.equals(OperationType.INSERT)
+                    || operationType.equals(OperationType.UPDATE)
+                    || operationType.equals(OperationType.DELETE)
+                    || operationType.equals(OperationType.DDL)
+                    || operationType.equals(OperationType.HEARTBEAT)) {
+
+                // consume record
+                RecordListener recordPrintListener = new DefaultRecordPrintListener(DbType.MySQL);
+
+                recordPrintListener.consume(record);
+
+                //commit method push the checkpoint update
+                record.commit("");
+            }
+        };
+        return Collections.singletonMap("mysqlRecordPrinter", mysqlRecordPrintListener);
+    }
+
+}

kxs-transfer/src/main/java/com/kxs/transfer/api/mapper/KxsCheckpointMapper.java

@@ -0,0 +1,19 @@
+package com.kxs.transfer.api.mapper;
+
+import com.baomidou.dynamic.datasource.annotation.DS;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+import com.kxs.transfer.api.model.KxsCheckpoint;
+import org.apache.ibatis.annotations.Mapper;
+
+/**
+ * 数据同步保存消费节点表(KxsCheckpoint)表数据库访问层
+ *
+ * @author 系统
+ * @since 2024-01-19 15:56:55
+ */
+@Mapper
+@DS("sys")
+public interface KxsCheckpointMapper extends BaseMapper<KxsCheckpoint> {
+
+}
+

kxs-transfer/src/main/java/com/kxs/transfer/api/model/KxsCheckpoint.java

@@ -0,0 +1,65 @@
+package com.kxs.transfer.api.model;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.math.BigDecimal;
+import java.time.LocalDateTime;
+
+import com.baomidou.mybatisplus.annotation.FieldFill;
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableField;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.extension.activerecord.Model;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+
+/**
+ * 数据同步保存消费节点表(KxsCheckpoint)表实体类
+ *
+ * @author 系统
+ * @since 2024-01-19 15:56:55
+ */
+@EqualsAndHashCode(callSuper = true)
+@Data
+public class KxsCheckpoint extends Model<KxsCheckpoint> implements Serializable {
+
+    @Serial
+    private static final long serialVersionUID = 1L;
+
+    
+    
+    /**
+     * 主键
+     */
+    @Schema(description = "主键")
+    @TableId(type = IdType.AUTO)
+    private Long id;
+
+
+    /**
+     * 组ID
+     */
+    @Schema(description = "组ID")
+    private String groupId;
+
+
+    /**
+     * 消费节点
+     */
+    @Schema(description = "消费节点")
+    private String checkpoint;
+
+    /**
+     * 新增时间
+     */
+    @TableField(fill = FieldFill.INSERT)
+    @Schema(description = "新增时间")
+    private LocalDateTime createTime;
+
+
+
+
+
+}
+

kxs-transfer/src/main/java/com/kxs/transfer/api/service/KxsCheckpointService.java

@@ -0,0 +1,15 @@
+package com.kxs.transfer.api.service;
+
+import com.baomidou.mybatisplus.extension.service.IService;
+import com.kxs.transfer.api.model.KxsCheckpoint;
+
+/**
+ * 数据同步保存消费节点表(KxsCheckpoint)表服务接口
+ *
+ * @author 系统
+ * @since 2024-01-19 15:56:55
+ */
+public interface KxsCheckpointService extends IService<KxsCheckpoint> {
+
+}
+

kxs-transfer/src/main/java/com/kxs/transfer/api/service/impl/KxsCheckpointServiceImpl.java

@@ -0,0 +1,19 @@
+package com.kxs.transfer.api.service.impl;
+
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import com.kxs.transfer.api.mapper.KxsCheckpointMapper;
+import com.kxs.transfer.api.model.KxsCheckpoint;
+import com.kxs.transfer.api.service.KxsCheckpointService;
+import org.springframework.stereotype.Service;
+
+/**
+ * 数据同步保存消费节点表(KxsCheckpoint)表服务实现类
+ *
+ * @author 系统
+ * @since 2024-01-19 15:56:55
+ */
+@Service("kxsCheckpointService")
+public class KxsCheckpointServiceImpl extends ServiceImpl<KxsCheckpointMapper, KxsCheckpoint> implements KxsCheckpointService {
+
+}
+

kxs-transfer/src/main/resources/application.yml

@@ -0,0 +1,22 @@
+
+server:
+  port: 6666
+
+spring:
+  application:
+    name: @artifactId@
+  cloud:
+    nacos:
+      username: @nacos.username@
+      password: @nacos.password@
+      discovery:
+        server-addr: ${NACOS_HOST:kxs-register}:${NACOS_PORT:8848}
+        watch:
+          enabled: true
+        watch-delay: 1000
+      config:
+        server-addr: ${spring.cloud.nacos.discovery.server-addr}
+  config:
+    import:
+      - optional:nacos:application-@profiles.active@.yml
+      - optional:nacos:${spring.application.name}-@profiles.active@.yml

pom.xml

@@ -80,6 +80,7 @@
         <module>kxs-store</module>
         <module>kxs-product</module>
         <module>kxs-visual</module>
+        <module>kxs-transfer</module>
     </modules>
 
     <dependencyManagement>