Canal 主要用途是基于MySQL 数据库增量日志解析，提供增量数据订阅和消费

部署canal 单机版

canal 的工作原理

MySQL 安装

mysql 主要用来存储canal 的一些信息

mysql 创建pvc 存储

---
apiVersion: v1
kind: PersistentVolume
metadata:name: mysql-pvnamespace: iot
spec:capacity:storage: 50GivolumeMode: FilesystemaccessModes:- ReadWriteManypersistentVolumeReclaimPolicy: RecyclestorageClassName: slownfs:path: /data/mysql-dataserver: ###IP 地址---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:name: mysql-pvcnamespace: iot
spec:accessModes:- ReadWriteManyvolumeMode: Filesystemresources:requests:storage: 50GistorageClassName: slow

apiVersion: v1
kind: Service
metadata:name: mysqlnamespace: iot
spec:type: NodePortports:- port: 3306selector:app: mysql
---
apiVersion: apps/v1
kind: Deployment
metadata:name: mysqlnamespace: iot
spec:selector:matchLabels:app: mysqlstrategy:type: Recreatetemplate:metadata:labels:app: mysqlspec:containers:- image: mysql:8.0.19name: mysqlenv:# 在实际中使用 secret- name: MYSQL_ROOT_PASSWORDvalue: bdtp@2022ports:- containerPort: 3306name: mysqlvolumeMounts:- name: mysql-datamountPath: /var/lib/mysqlvolumes:- name: mysql-datapersistentVolumeClaim:claimName: mysql-pvc

创建canal 数据库

create database canal_manager default character set utf8mb4 collate utf8mb4_unicode_ci;
CREATE USER 'canal'@'%' IDENTIFIED WITH mysql_native_password BY 'canal';
grant all on canal_manager.* to 'canal'@'%' ;
flush privileges;
#其中8.0远程连接要修改为mysql_native_password 认证连接，否则连接失败，如果不是mysql_native_password 则使用以下修改语句
## ALTER USER 'canal'@'%' IDENTIFIED WITH mysql_native_password BY 'canal';
导入sql 
https://github.com/alibaba/canal/blob/canal-1.1.6/admin/adminweb/src/main/resources/canal_manager.sql

canal-admin 创建

apiVersion: v1
kind: ConfigMap
metadata:name: canal-adminnamespace: iot
data:admin_user: "admin"admin_password: "123456"datasource_address: "mysql:3306"datasource_database: "canal_manager"datasource_username: "canal"datasource_password: "canal"---kind: Deployment
apiVersion: apps/v1
metadata:name: canal-adminnamespace: iotlabels:app.kubernetes.io/name: canal-admin
spec:replicas: 1selector:matchLabels:app.kubernetes.io/name: canal-admintemplate:metadata:name: canal-admincreationTimestamp: nulllabels:app.kubernetes.io/name: canal-adminspec:containers:- name: canal-adminimage: 'canal/canal-admin:v1.1.6'imagePullPolicy: IfNotPresentports:- name: webcontainerPort: 8089protocol: TCPenv:- name: server.portvalue: '8089'- name: canal.adminUservalueFrom:configMapKeyRef:name: canal-adminkey: admin_user- name: canal.adminPasswdvalueFrom:configMapKeyRef:name: canal-adminkey: admin_password- name: spring.datasource.addressvalueFrom:configMapKeyRef:name: canal-adminkey: datasource_address- name: spring.datasource.databasevalueFrom:configMapKeyRef:name: canal-adminkey: datasource_database- name: spring.datasource.usernamevalueFrom:configMapKeyRef:name: canal-adminkey: datasource_username- name: spring.datasource.passwordvalueFrom:configMapKeyRef:name: canal-adminkey: datasource_passwordresources:requests:cpu: 250mmemory: 256MilivenessProbe:httpGet:path: /port: 8089initialDelaySeconds: 10timeoutSeconds: 5periodSeconds: 30readinessProbe:httpGet:path: /port: 8089initialDelaySeconds: 10timeoutSeconds: 5periodSeconds: 30restartPolicy: AlwaysnodeSelector: {}affinity:podAntiAffinity:preferredDuringSchedulingIgnoredDuringExecution:- weight: 1podAffinityTerm:labelSelector:matchLabels:app.kubernetes.io/name: canal-adminnamespaces:- iottopologyKey: kubernetes.io/hostnamerevisionHistoryLimit: 10---kind: Service
apiVersion: v1
metadata:name: canal-adminnamespace: iot
spec:type: NodePortports:- protocol: TCPport: 8089targetPort: 8089selector:app.kubernetes.io/name: canal-admin

canal-server 部署

apiVersion: v1
kind: ConfigMap
metadata:name: canal-servernamespace: iot
data:admin_manager: "canal-admin:8089"admin_port: "11110"admin_user: "admin"admin_password: "6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9"   # 123456加密#admin_register_cluster: "local"  ##集群名称admin_register_auto: "true"admin_register_name: "iot-canal-server"  ##canal-server 自动注册名称---
kind: StatefulSet    ##canal-server 采用StatefulSet 方式部署
apiVersion: apps/v1
metadata:name: canal-servernamespace: iotlabels:app.kubernetes.io/name: canal-serverapp: canal-server
spec:replicas: 1selector:matchLabels:app.kubernetes.io/name: canal-serverapp: canal-servertemplate:metadata:name: canal-serverlabels:app.kubernetes.io/name: canal-serverapp: canal-serverspec:imagePullSecrets:- name: secretcontainers:- name: canal-serverimage: 'canal/canal-server:v1.1.6'imagePullPolicy: Alwaysports:- name: tcpcontainerPort: 11111protocol: TCPenv:- name: POD_NAMEvalueFrom:fieldRef:apiVersion: v1fieldPath: metadata.name- name: SERVICE_NAMEvalueFrom:fieldRef:apiVersion: v1fieldPath: 'metadata.labels[''app'']'- name: STS_NAMESPACEvalueFrom:fieldRef:apiVersion: v1fieldPath: metadata.namespace- name: canal.register.ipvalue: $(POD_NAME).$(SERVICE_NAME).$(STS_NAMESPACE)  ##  canal-server采用实例名，防止每次重启导致的IP不同，Instance实例选择时的变化- name: canal.admin.managervalueFrom:configMapKeyRef:name: canal-serverkey: admin_manager- name: canal.admin.portvalueFrom:configMapKeyRef:name: canal-serverkey: admin_port- name: canal.admin.uservalueFrom:configMapKeyRef:name: canal-serverkey: admin_user- name: canal.admin.passwdvalueFrom:configMapKeyRef:name: canal-serverkey: admin_password#- name: canal.admin.register.cluster#  valueFrom:#    configMapKeyRef:#      name: canal-server#      key: admin_register_cluster- name: canal.admin.register.autovalueFrom:configMapKeyRef:name: canal-serverkey: admin_register_auto- name: canal.admin.register.namevalueFrom:configMapKeyRef:name: canal-serverkey: admin_register_nameresources:requests:cpu: 250mmemory: 256MilivenessProbe:tcpSocket:port: 11112initialDelaySeconds: 10timeoutSeconds: 5periodSeconds: 30readinessProbe:tcpSocket:port: 11112initialDelaySeconds: 10timeoutSeconds: 5periodSeconds: 30restartPolicy: AlwaysnodeSelector: {}affinity:podAntiAffinity:preferredDuringSchedulingIgnoredDuringExecution:- weight: 1podAffinityTerm:labelSelector:matchLabels:app.kubernetes.io/name: canal-servernamespaces:- iottopologyKey: kubernetes.io/hostnameserviceName: canal-serverrevisionHistoryLimit: 10---
kind: Service
apiVersion: v1
metadata:name: canal-servernamespace: iot
spec:ports:- protocol: TCPport: 11110targetPort: 11110type: ClusterIPselector:app.kubernetes.io/name: canal-serverapp: canal-server

canlal 配置修改

canal主配置修改

运行正常之后，登录页面，默认是admin 123456 可以在页面修改账号与密码

可以看到默认已经注册了

点击操作—修改主配置-点击保存即可

# tcp, kafka, rocketMQ, rabbitMQ
canal.serverMode = kafka
# 默认是tcp模式，修改为kafka ######### 		     Kafka 		     #############
##################################################
kafka.bootstrap.servers = 172.19.0.64:9092
kafka.acks = all
kafka.compression.type = none
kafka.batch.size = 16384
kafka.linger.ms = 1
kafka.max.request.size = 1048576
kafka.buffer.memory = 33554432
kafka.max.in.flight.requests.per.connection = 1
kafka.retries = 0
kafka.sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username='用户名' password='密码';kafka.sasl.mechanism=PLAIN
kafka.security.protocol=PLAINTEXT
kafka.kerberos.enable = false
kafka.kerberos.krb5.file = "../conf/kerberos/krb5.conf"
kafka.kerberos.jaas.file = "../conf/kerberos/jaas.conf"

canal 创建instance

## 需要修改的部分，源库的ip数据库地址
canal.instance.master.address=127.0.0.1:3306# 源库的用户名与密码
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal# 匹配同步的数据库规则
# table regex
canal.instance.filter.regex=.*\\..*
# table black regex
canal.instance.filter.black.regex=# mq config  
# kafka topic 的名字，可以在kafka创建，与kafka topic对应，如果不填写，会自动创建
canal.mq.topic=example#备注： 需要在源库中创建用户名与密码并且同时具有同步的权限
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';
FLUSH PRIVILEGES;

kafka 查看消息

也可以查看instance 的日志