diff --git a/apps/gitea-webhook-ambassador/configs/configs.example.yaml b/apps/gitea-webhook-ambassador/configs/configs.example.yaml index 55ff13e6..734cacc8 100644 --- a/apps/gitea-webhook-ambassador/configs/configs.example.yaml +++ b/apps/gitea-webhook-ambassador/configs/configs.example.yaml @@ -47,4 +47,8 @@ worker: poolSize: 10 queueSize: 100 maxRetries: 3 - retryBackoff: 1 \ No newline at end of file + retryBackoff: 1 + +eventCleanup: + interval: 3600 + expireAfter: 7200 \ No newline at end of file diff --git a/apps/gitea-webhook-ambassador/main.go b/apps/gitea-webhook-ambassador/main.go index 7124add9..1c14578c 100644 --- a/apps/gitea-webhook-ambassador/main.go +++ b/apps/gitea-webhook-ambassador/main.go @@ -53,6 +53,11 @@ type Configuration struct { MaxRetries int `yaml:"maxRetries" default:"3" validate:"gte=0"` RetryBackoff int `yaml:"retryBackoff" default:"1" validate:"gt=0"` // seconds } `yaml:"worker"` + + EventCleanup struct { + Interval int `yaml:"interval" default:"3600"` // seconds + ExpireAfter int `yaml:"expireAfter" default:"7200"` // seconds + } `yaml:"eventCleanup"` } // ProjectConfig represents the configuration for a specific repository @@ -150,6 +155,9 @@ func main() { // Setup config file watcher for auto-reload setupConfigWatcher(*configFile) + // Start event cleanup goroutine + go cleanupEvents() + // Configure HTTP client with timeout configMutex.RLock() httpClient = &http.Client{ @@ -299,6 +307,12 @@ func loadConfig(file string) error { if newConfig.Worker.RetryBackoff == 0 { newConfig.Worker.RetryBackoff = 1 } + if newConfig.EventCleanup.Interval == 0 { + newConfig.EventCleanup.Interval = 3600 + } + if newConfig.EventCleanup.ExpireAfter == 0 { + newConfig.EventCleanup.ExpireAfter = 7200 + } // Handle legacy configuration format (where Projects is map[string]string) // This is to maintain backward compatibility with existing configs @@ -525,11 +539,7 @@ func handleWebhook(w http.ResponseWriter, r *http.Request) { } // Store in processed events with a TTL (we'll use a goroutine to remove after 1 hour) - processedEvents.Store(eventID, true) - go func(key string) { - time.Sleep(1 * time.Hour) - processedEvents.Delete(key) - }(eventID) + processedEvents.Store(eventID, time.Now()) // Check if we have a Jenkins job mapping for this repository configMutex.RLock() @@ -742,3 +752,25 @@ func logError(format string, v ...interface{}) { // Error level logs are always shown logger.Printf("[ERROR] "+format, v...) } + +func cleanupEvents() { + for { + configMutex.RLock() + interval := time.Duration(config.EventCleanup.Interval) * time.Second + expireAfter := time.Duration(config.EventCleanup.ExpireAfter) * time.Second + configMutex.RUnlock() + + time.Sleep(interval) + + now := time.Now() + processedEvents.Range(func(key, value interface{}) bool { + if timestamp, ok := value.(time.Time); ok { + if now.Sub(timestamp) > expireAfter { + processedEvents.Delete(key) + logDebug("Cleaned up expired event: %v", key) + } + } + return true + }) + } +} diff --git a/cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml b/cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml new file mode 100644 index 00000000..90e42dd1 --- /dev/null +++ b/cluster/manifests/freeleaps-controls-system/rbac/rbac.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: certificate-contributor + namespace: freeleaps-controls-system +rules: + - apiGroups: ["cert-manager.io"] + resources: ["certificates"] + verbs: ["*"] \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/doris/deploy.sh b/cluster/manifests/freeleaps-data-platform/doris/deploy.sh new file mode 100644 index 00000000..06ce5406 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/doris/deploy.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# 创建命名空间 +kubectl create namespace freeleaps-data-platform + +# 安装Doris Operator CRD +kubectl create -f https://raw.githubusercontent.com/apache/doris-operator/master/config/crd/bases/doris.apache.com_dorisclusters.yaml + +# 部署 Doris Operator +kubectl apply -f https://raw.githubusercontent.com/apache/doris-operator/master/config/operator/operator.yaml + +# 创建 Doris 集群配置 +kubectl apply -f doris-cluster.yaml -n freeleaps-data-platform + +# 检查部署状态 +kubectl get pods -n freeleaps-data-platform -l app=doris \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml b/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml new file mode 100644 index 00000000..9ee2bb52 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/doris/doris-cluster.yaml @@ -0,0 +1,57 @@ +apiVersion: doris.apache.com/v1 +kind: DorisCluster +metadata: + name: doris-cluster + namespace: freeleaps-data-platform +spec: + clusterDomain: "freeleaps.cluster" + + feSpec: + replicas: 1 + image: apache/doris:2.0.2 + resources: + requests: + cpu: "1" + memory: "2Gi" + limits: + cpu: "2" + memory: "4Gi" + service: + type: ClusterIP + configMap: + fe.conf: | + JAVA_OPTS="-Xmx2048m -XX:+UseG1GC" + + beSpec: + replicas: 1 + image: apache/doris:2.0.2 + storage: + storageSize: "50Gi" + storageClassName: "azure-disk-std-ssd-lrs" + resources: + requests: + cpu: "2" + memory: "4Gi" + limits: + cpu: "4" + memory: "8Gi" + storage: + storageSize: "50Gi" + storageClassName: "standard" + configMap: + be.conf: | + JAVA_OPTS="-Xmx8192m -XX:+UseG1GC" + BE_ADDR=${POD_IP}:9060 + BE_HTTP_PORT=8040 + BE_PORT=9060 + HEARTBEAT_SERVICE_PORT=9050 + BRPC_PORT=8060 + + feAddress: doris-cluster-fe-service + + monitoring: + enabled: true + prometheus: + serviceMonitor: + enabled: true + namespace: freeleaps-monitoring-system \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/kafka/kafka.sh b/cluster/manifests/freeleaps-data-platform/kafka/kafka.sh new file mode 100644 index 00000000..924fdfee --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/kafka/kafka.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# 查看kafka的pod +kubectl get pods -n freeleaps-data-platform +# 查看kafka的服务 +kubectl get svc -n freeleaps-data-platform + diff --git a/cluster/manifests/freeleaps-data-platform/kafka/values.yaml b/cluster/manifests/freeleaps-data-platform/kafka/values.yaml new file mode 100644 index 00000000..77dde670 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/kafka/values.yaml @@ -0,0 +1,75 @@ +global: + storageClass: "standard" + +nameOverride: "kafka" +fullnameOverride: "kafka" +namespaceOverride: "freeleaps-data-platform" + +kafka: + replicaCount: 1 + heapOpts: "-Xmx1024m -Xms1024m" + + resources: + requests: + memory: "1Gi" + cpu: "2" + limits: + memory: "2Gi" + cpu: "1" + + persistence: + enabled: true + size: 20Gi + mountPath: /bitnami/kafka + + config: + num.partitions: 1 + default.replication.factor: 1 + min.insync.replicas: 1 + auto.create.topics.enable: true + delete.topic.enable: true + log.retention.hours: 168 + log.retention.bytes: 1073741824 + + service: + type: ClusterIP + ports: + client: 9092 + internal: 9093 + + metrics: + kafka: + enabled: true + serviceMonitor: + enabled: true + namespace: freeleaps-data-platform + jmx: + enabled: true + +zookeeper: + enabled: true + replicaCount: 1 + + resources: + requests: + memory: "1Gi" + cpu: "1" + limits: + memory: "2Gi" + cpu: "1" + + persistence: + enabled: true + size: 8Gi + + service: + type: ClusterIP + port: 2181 + +serviceAccount: + create: true + name: "kafka" + +networkPolicy: + enabled: true + allowExternal: true \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/namespace.yaml b/cluster/manifests/freeleaps-data-platform/namespace.yaml new file mode 100644 index 00000000..e7df1b51 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: freeleaps-data-platform + labels: + name: freeleaps-data-platform diff --git a/cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml b/cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml new file mode 100644 index 00000000..0942c51c --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/pinot/certificate.yaml @@ -0,0 +1,13 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: pinot-dot-mathmast-dot-com + namespace: freeleaps-data-platform +spec: + commonName: pinot.mathmast.com + dnsNames: + - pinot.mathmast.com + issuerRef: + kind: ClusterIssuer + name: mathmast-dot-com + secretName: pinot-dot-mathmast-dot-com-tls \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/pinot/values.yaml b/cluster/manifests/freeleaps-data-platform/pinot/values.yaml new file mode 100644 index 00000000..e48c0151 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/pinot/values.yaml @@ -0,0 +1,42 @@ +cluster: + name: pinot-cluster + +namespaceOverride: "freeleaps-data-platform" + +controller: + replicaCount: 1 + persistence: + enabled: true + size: 20Gi + ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/rewrite-target: /$2 + cluster-issuer: mathmast-dot-com + hosts: + - host: pinot.freeleaps.com + paths: + - path: /pinot(/|$)(.*) + pathType: Prefix + port: 9000 + tls: + - secretName: pinot-dot-mathmast-dot-com-tls + hosts: + - pinot.freeleaps.com + +broker: + replicaCount: 1 + +server: + replicaCount: 1 + persistence: + enabled: true + size: 50Gi + +zookeeper: + enabled: false + external: + enabled: true + host: "kafka-zookeeper" + port: 2181 \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml b/cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml new file mode 100644 index 00000000..eb98fe38 --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/rbac/rbac.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: data-platform-contributor + namespace: freeleaps-data-platform +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] \ No newline at end of file diff --git a/cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml b/cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml new file mode 100644 index 00000000..b46fef1c --- /dev/null +++ b/cluster/manifests/freeleaps-data-platform/star-rocks/values.yaml @@ -0,0 +1,32 @@ +starrocks: + initPassword: + enabled: true + # 设置密码 secret，例如： + # kubectl create secret generic starrocks-root-pass --from-literal=password='g()()dpa$$word' + passwordSecret: starrocks-root-pass + + starrocksFESpec: + replicas: 3 + service: + type: LoadBalancer + resources: + requests: + cpu: 1 + memory: 1Gi + storageSpec: + name: fe + + starrocksBeSpec: + replicas: 3 + resources: + requests: + cpu: 1 + memory: 2Gi + storageSpec: + name: be + storageSize: 15Gi + + starrocksFeProxySpec: + enabled: true + service: + type: LoadBalancer \ No newline at end of file diff --git a/cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml b/cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml new file mode 100644 index 00000000..24039d4d --- /dev/null +++ b/cluster/manifests/freeleaps-monitoring-system/rbac/rbac.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-rules-contributor + namespace: freeleaps-monitoring-system +rules: + - apiGroups: ["monitoring.coreos.com"] + resources: ["prometheusrules"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: service-monitor-contributor + namespace: freeleaps-monitoring-system +rules: + - apiGroups: ["monitoring.coreos.com"] + resources: ["servicemonitors"] + verbs: ["*"] \ No newline at end of file diff --git a/cluster/manifests/helm-repos/REPO.list b/cluster/manifests/helm-repos/REPO.list index dcea5527..9806fb42 100644 --- a/cluster/manifests/helm-repos/REPO.list +++ b/cluster/manifests/helm-repos/REPO.list @@ -12,3 +12,5 @@ descheduler,https://kubernetes-sigs.github.io/descheduler/,force-update kubernetes-dashboard,https://kubernetes.github.io/dashboard/,force-update grafana,https://grafana.github.io/helm-charts,force-update fluent,https://fluent.github.io/helm-charts,force-update +pinot,https://raw.githubusercontent.com/apache/pinot/master/helm,force-update +starrocks,https://starrocks.github.io/starrocks-kubernetes-operator,force-update \ No newline at end of file