Add resource requests and limits to Kafka cluster configuration, and update KafkaUser to reference a new secret for password management. Also, enhance KafkaUser ACLs and MongoDB connector configuration for full document change streams.

Signed-off-by: zhenyus <zhenyus@mathmast.com>

cluster/manifests/freeleaps-data-platform/kafbat/kafka-user.yaml

@@ -18,6 +18,7 @@ spec:
         host: "*"
         operations:
           - Read
+          - Delete
       - resource:
           type: topic
           name: "*"

cluster/manifests/freeleaps-data-platform/kafka/connectors/freeleaps/freeleaps-mongo-cds.yaml

@@ -42,6 +42,10 @@ spec:
     connection.uri: mongodb+srv://freeadmin:0eMV0bt8oyaknA0m@freeleaps2.zmsmpos.mongodb.net/?retryWrites=true&w=majority
     database: freeleaps2
     pipeline: '[{"$match": {"operationType": {"$in": ["insert", "update", "delete", "replace"]}}}]'
+    # we need full document from cdc stream
+    change.stream.full.document: "whenAvailable"
+    # also we need full document before change
+    change.stream.full.document.before.change: "whenAvailable"
     startup.mode: copy_existing
     startup.mode.copy.existing.pipeline: "[]"
     topic.prefix: raw-cds-events

cluster/manifests/freeleaps-data-platform/kafka/freeleaps-kafka-cluster.yaml

@@ -18,6 +18,13 @@ spec:
         deleteClaim: false
         kraftMetadata: shared
         class: azure-disk-std-ssd-lrs
+  resources:
+    requests:
+      cpu: 500m
+      memory: 512Mi
+    limits:
+      cpu: "2"
+      memory: 2Gi
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaNodePool
@@ -39,6 +46,13 @@ spec:
         deleteClaim: false
         kraftMetadata: shared
         class: azure-disk-std-ssd-lrs
+  resources:
+    requests:
+      cpu: 500m
+      memory: 2Gi
+    limits:
+      cpu: "2"
+      memory: 4Gi
 ---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: Kafka
@@ -79,6 +93,15 @@ spec:
     topicOperator: {}
     userOperator: {}
 ---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: freeleaps-user-passwords
+  namespace: freeleaps-data-platform
+type: Opaque
+data:
+  client-passwords: eHpqbzFVZEE3eHM5ZDNEREhaME5nYVJvQndkaHh5QkU=
+---
 apiVersion: kafka.strimzi.io/v1beta2
 kind: KafkaUser
 metadata:
@@ -89,6 +112,11 @@ metadata:
 spec:
   authentication:
     type: scram-sha-512
+    password:
+      valueFrom:
+        secretKeyRef:
+          name: freeleaps-user-passwords
+          key: client-passwords
   authorization:
     type: simple
     acls:
@@ -105,4 +133,5 @@ spec:
           patternType: literal
         operations:
         - All
+        - Delete
         host: "*"