Merge remote-tracking branch 'origin/master' into icecheng/feature

2025-09-23 17:09:09 +08:00 · 2025-09-23 17:09:09 +08:00 · 38e210fe83
commit 38e210fe83
parent e8bcce7f97 43b43feb2d
27 changed files with 131 additions and 140 deletions
--- a/first-class-pipeline/src/com/freeleaps/devops/ImageBuilder.groovy
+++ b/first-class-pipeline/src/com/freeleaps/devops/ImageBuilder.groovy
@ -117,8 +117,8 @@ class ImageBuilder {
              steps.log.info("ImageBuilder", "Files in build context: ${filesInContext}")
              steps.log.info("ImageBuilder", "Set builder log level to plain...")
              steps.env.BUILDKIT_PROGRESS = "plain"
-              steps.log.info("ImageBuilder", "Set builder timeout to 10min...")
-              steps.env.BUILDKIT_TIMEOUT = "1800s"
+              steps.log.info("ImageBuilder", "Set builder timeout to 120min...")
+              steps.env.BUILDKIT_TIMEOUT = "7200s"
              steps.sh "docker buildx build --builder ${buildxBuilderName} --no-cache --platform ${architectures.join(",")} -t ${registry}/${repository}/${name}:${version} -f ${dockerfile} --push ${contextRoot}"
              steps.env.BUILD_IMAGE_REGISTRY = "${registry}"
              steps.env.BUILD_IMAGE_REPO = "${repository}"
--- a/first-class-pipeline/vars/executeFreeleapsPipeline.groovy
+++ b/first-class-pipeline/vars/executeFreeleapsPipeline.groovy
@ -640,7 +640,7 @@ def call(Closure closure) {
    agent any
    options {
      buildDiscarder(logRotator(numToKeepStr: '25'))
-      timeout(time: 120, unit: 'MINUTES')
+      timeout(time: 240, unit: 'MINUTES')
      parallelsAlwaysFailFast()
    }

--- a/freeleaps-devops-reconciler/helm-pkg/reconciler/values.prod.yaml
+++ b/freeleaps-devops-reconciler/helm-pkg/reconciler/values.prod.yaml
@ -3,7 +3,7 @@ reconciler:
  image:
    repository: freeleaps/reconciler
    pullPolicy: IfNotPresent
-    tag: snapshot-e5ba2fb
+    tag: snapshot-cf4af3e
    registry: docker.io
    name: reconciler
 imagePullSecrets: []
--- a/freeleaps/alpha/ci/freeleaps-service-hub/Jenkinsfile
+++ b/freeleaps/alpha/ci/freeleaps-service-hub/Jenkinsfile
@ -39,7 +39,7 @@ executeFreeleapsPipeline {
      dependenciesManager: 'pip',
      requirementsFile: 'requirements.txt',
      buildCacheEnabled: true,
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      lintEnabled: false,
      sastEnabled: false,
@ -59,7 +59,7 @@ executeFreeleapsPipeline {
      language: 'python',
      dependenciesManager: 'pip',
      requirementsFile: 'requirements.txt',
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      buildCacheEnabled: true,
      lintEnabled: false,
@ -80,7 +80,7 @@ executeFreeleapsPipeline {
      language: 'python',
      dependenciesManager: 'pip',
      requirementsFile: 'requirements.txt',
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      buildCacheEnabled: true,
      lintEnabled: false,
@ -101,7 +101,7 @@ executeFreeleapsPipeline {
      language: 'python',
      dependenciesManager: 'pip',
      requirementsFile: 'requirements.txt',
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      buildCacheEnabled: true,
      lintEnabled: false,
@ -122,7 +122,7 @@ executeFreeleapsPipeline {
      language: 'python',
      dependenciesManager: 'pip',
      requirementsFile: 'requirements.txt',
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      buildCacheEnabled: true,
      lintEnabled: false,
--- a/freeleaps/alpha/ci/freeleaps2-devsvc/Jenkinsfile
+++ b/freeleaps/alpha/ci/freeleaps2-devsvc/Jenkinsfile
@ -17,7 +17,7 @@ executeFreeleapsPipeline {
      dependenciesManager: 'pip',
      requirementsFile: 'requirements.txt',
      buildCacheEnabled: true,
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      lintEnabled: false,
      sastEnabled: false,
--- a/freeleaps/alpha/ci/freeleaps2-frontend/Jenkinsfile
+++ b/freeleaps/alpha/ci/freeleaps2-frontend/Jenkinsfile
@ -17,7 +17,7 @@ executeFreeleapsPipeline {
      dependenciesManager: 'pip',
      requirementsFile: 'chat/requirements.txt',
      buildCacheEnabled: true,
-      buildAgentImage: 'python:3.10-slim-bullseye',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      lintEnabled: false,
      sastEnabled: false,
@ -38,7 +38,7 @@ executeFreeleapsPipeline {
      dependenciesManager: 'pip',
      requirementsFile: 'freeleaps/requirements.txt',
      buildCacheEnabled: true,
-      buildAgentImage: 'python:3.10-slim-bullseye',
+      buildAgentImage: 'python:3.12-slim',
      buildArtifacts: ['.'],
      lintEnabled: false,
      sastEnabled: false,
@ -70,7 +70,7 @@ executeFreeleapsPipeline {
      imageBuilder: 'dind',
      dockerfilePath: 'Dockerfile',
      imageBuildRoot: '.',
-      imageReleaseArchitectures: ['linux/amd64', 'linux/arm64/v8'],
+      imageReleaseArchitectures: ['linux/amd64'],
      registryCredentialsId: 'freeleaps-devops-docker-hub-credentials',
      semanticReleaseEnabled: true
    ]
--- a/freeleaps/helm-pkg/authentication/templates/authentication/authentication-config.yaml
+++ b/freeleaps/helm-pkg/authentication/templates/authentication/authentication-config.yaml
@ -7,6 +7,7 @@ type: Opaque
 data:
  TZ: {{ .Values.authentication.configs.tz | b64enc | quote }}
  APP_NAME: {{ .Values.authentication.configs.appName | b64enc | quote }}
+  APP_ENV: {{ .Values.authentication.configs.appEnv | b64enc | quote }}
  DEVSVC_WEBAPI_URL_BASE: {{ .Values.authentication.configs.devsvcWebapiUrlBase | b64enc | quote }}
  NOTIFICATION_WEBAPI_URL_BASE: {{ .Values.authentication.configs.notificationWebapiUrlBase | b64enc | quote }}
  JWT_ALGORITHM: {{ .Values.authentication.configs.jwtAlgorithm | b64enc | quote }}
--- a/freeleaps/helm-pkg/authentication/templates/authentication/opentelemetry.yaml
+++ b/freeleaps/helm-pkg/authentication/templates/authentication/opentelemetry.yaml
@ -46,13 +46,10 @@ spec:
      filelog:
        include:
          - {{ .Values.logIngest.logPathPattern }}
-        start_at: end
+        start_at: beginning
        include_file_path: false
        include_file_name: false
-        operators:
-          - type: json_parser
-            parse_from: body
-            parse_to: attributes
+        operators: []
    processors:
      resource:
        attributes:
@ -78,13 +75,8 @@ spec:
        log_statements:
          - context: log
            statements:
-              - set(resource.attributes["application"], log.attributes["context"]["app"])
-              - set(resource.attributes["environment"], log.attributes["context"]["env"])
-              - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
-              - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
-              - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
-              - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
-              - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["application"], "authentication")
+              - set(resource.attributes["environment"], "{{ .Values.global.environment | default .Release.Namespace }}")
              - set(resource.attributes["body_json"], ParseJSON(log.body))
              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
--- a/freeleaps/helm-pkg/authentication/values.alpha.yaml
+++ b/freeleaps/helm-pkg/authentication/values.alpha.yaml
@ -1,7 +1,3 @@
-global:
-  registry: docker.io
-  repository: freeleaps
-  nodeSelector: {}
 dashboard:
  enabled: false
 logIngest:
@ -9,13 +5,15 @@ logIngest:
  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
  logPathPattern: /app/log/authentication/*.log
  logPath: /app/log/authentication
+global:
+  environment: alpha
 authentication:
  replicas: 1
  image:
    registry: docker.io
    repository: null
    name: authentication
-    tag: snapshot-c406f6d
+    tag: snapshot-3cba9e4
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
@ -78,6 +76,7 @@ authentication:
  configs:
    tz: UTC
    appName: authentication
+    appEnv: alpha
    devsvcWebapiUrlBase: http://devsvc-service.freeleaps-alpha.svc.freeleaps.cluster:8007/api/devsvc/
    notificationWebapiUrlBase: http://notification-service.freeleaps-alpha.svc.freeleaps.cluster:8003/api/notification/
    jwtAlgorithm: HS256
--- a/freeleaps/helm-pkg/authentication/values.prod.yaml
+++ b/freeleaps/helm-pkg/authentication/values.prod.yaml
@ -7,13 +7,20 @@ dashboard:
  name: freeleaps-prod-authentication-dashboard
  title: Authentication Service Dashboard (PROD)
  metricsPrefix: freeleaps_authentication
+logIngest:
+  enabled: true
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/authentication/*.log
+  logPath: /app/log/authentication
+global:
+  environment: prod
 authentication:
  replicas: 1
  image:
    registry: docker.io
    repository: null
    name: authentication
-    tag: snapshot-4a5bdb4
+    tag: 1.9.0
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
@ -64,6 +71,7 @@ authentication:
  configs:
    tz: UTC
    appName: authentication
+    appEnv: prod
    devsvcWebapiUrlBase: http://devsvc-service.freeleaps-prod.svc.freeleaps.cluster:8007/api/devsvc/
    notificationWebapiUrlBase: http://notification-service.freeleaps-prod.svc.freeleaps.cluster:8003/api/notification/
    jwtAlgorithm: HS256
--- a/freeleaps/helm-pkg/centralStorage/templates/central-storage/opentelemetry.yaml
+++ b/freeleaps/helm-pkg/centralStorage/templates/central-storage/opentelemetry.yaml
@ -46,15 +46,14 @@ spec:
      filelog:
        include:
          - {{ .Values.logIngest.logPathPattern }}
-        start_at: end
-        poll_interval: 1s
+        start_at: beginning
        include_file_path: false
        include_file_name: false
-        operators:
-          - type: json_parser
-            parse_from: body
-            parse_to: attributes
-      # Add container logs receiver
+        operators: []
+      # Add container logs receiver for stdout
+      k8s_cluster:
+        auth_type: serviceAccount
+        namespaces: [{{ .Release.Namespace }}]
    processors:
      resource:
        attributes:
@ -80,13 +79,8 @@ spec:
        log_statements:
          - context: log
            statements:
-              - set(resource.attributes["application"], log.attributes["context"]["app"])
-              - set(resource.attributes["environment"], log.attributes["context"]["env"])
-              - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
-              - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
-              - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
-              - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
-              - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["application"], "central-storage")
+              - set(resource.attributes["environment"], "{{ .Values.global.environment | default .Release.Namespace }}")
              - set(resource.attributes["body_json"], ParseJSON(log.body))
              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
@ -113,7 +107,7 @@ spec:
          level: info
      pipelines:
        logs:
-          receivers: [filelog]  # Only use filelog receiver to prevent duplicate collection
+          receivers: [filelog, k8s_cluster]  # Use both filelog and k8s_cluster receivers
          processors: [resource, transform, batch]
          exporters: [otlphttp/logs]
 {{- end }}
--- a/freeleaps/helm-pkg/centralStorage/values.alpha.yaml
+++ b/freeleaps/helm-pkg/centralStorage/values.alpha.yaml
@ -15,7 +15,7 @@ centralStorage:
    registry: docker.io
    repository: null
    name: central_storage
-    tag: snapshot-12a0d9a
+    tag: snapshot-3cba9e4
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
@ -90,22 +90,22 @@ centralStorage:
      kind: FreeleapsSecretStore
      name: freeleaps-main-secret-store
    target:
-      name: "freeleaps-central-storage-secrets"
-      creationPolicy: "Owner"
+      name: freeleaps-central-storage-secrets
+      creationPolicy: Owner
    refreshInterval: 30s
    data:
-      - key: mongodbUri
-        remoteRef:
-          key: "freeleaps-alpha-mongodb-uri"
-          type: Secret
-      - key: azureStorageDocumentApiKey
-        remoteRef:
-          key: "freeleaps-alpha-azure-storage-document-api-key"
-          type: Secret
-      - key: azureStorageDocumentApiEndpoint
-        remoteRef:
-          key: "freeleaps-alpha-azure-storage-document-api-endpoint"
-          type: Secret
+    - key: mongodbUri
+      remoteRef:
+        key: freeleaps-alpha-mongodb-uri
+        type: Secret
+    - key: azureStorageDocumentApiKey
+      remoteRef:
+        key: freeleaps-alpha-azure-storage-document-api-key
+        type: Secret
+    - key: azureStorageDocumentApiEndpoint
+      remoteRef:
+        key: freeleaps-alpha-azure-storage-document-api-endpoint
+        type: Secret
  vpa:
    minAllowed:
      enabled: false
--- a/freeleaps/helm-pkg/centralStorage/values.prod.yaml
+++ b/freeleaps/helm-pkg/centralStorage/values.prod.yaml
@ -18,7 +18,7 @@ centralStorage:
    registry: docker.io
    repository: null
    name: central_storage
-    tag: snapshot-046f9ff
+    tag: snapshot-ee519ca
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/chat/values.alpha.yaml
+++ b/freeleaps/helm-pkg/chat/values.alpha.yaml
@ -16,7 +16,7 @@ chat:
    registry: docker.io
    repository: null
    name: chat
-    tag: snapshot-cc66272
+    tag: snapshot-3069327
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/content/values.alpha.yaml
+++ b/freeleaps/helm-pkg/content/values.alpha.yaml
@ -15,7 +15,7 @@ content:
    registry: docker.io
    repository: null
    name: content
-    tag: snapshot-12a0d9a
+    tag: snapshot-3cba9e4
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
@ -90,18 +90,18 @@ content:
      kind: FreeleapsSecretStore
      name: freeleaps-main-secret-store
    target:
-      name: "freeleaps-content-alpha-secrets"
-      creationPolicy: "Owner"
+      name: freeleaps-content-alpha-secrets
+      creationPolicy: Owner
    refreshInterval: 30s
    data:
-      - key: mongodbUri
-        remoteRef:
-          key: "freeleaps-alpha-mongodb-uri"
-          type: Secret
-      - key: freeleapsWwwAsAzureClientSecret
-        remoteRef:
-          key: "freeleaps-alpha-www-as-azure-client-secret"
-          type: Secret
+    - key: mongodbUri
+      remoteRef:
+        key: freeleaps-alpha-mongodb-uri
+        type: Secret
+    - key: freeleapsWwwAsAzureClientSecret
+      remoteRef:
+        key: freeleaps-alpha-www-as-azure-client-secret
+        type: Secret
  vpa:
    minAllowed:
      enabled: false
--- a/freeleaps/helm-pkg/content/values.prod.yaml
+++ b/freeleaps/helm-pkg/content/values.prod.yaml
@ -13,7 +13,7 @@ content:
    registry: docker.io
    repository: null
    name: content
-    tag: snapshot-046f9ff
+    tag: 1.9.0
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/devops/values.alpha.yaml
+++ b/freeleaps/helm-pkg/devops/values.alpha.yaml
@ -10,7 +10,7 @@ devops:
    registry: docker.io
    repository: null
    name: devops
-    tag: snapshot-45b4ebd
+    tag: snapshot-3cba9e4
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/devops/values.prod.yaml
+++ b/freeleaps/helm-pkg/devops/values.prod.yaml
@ -10,7 +10,7 @@ devops:
    registry: docker.io
    repository: null
    name: devops
-    tag: snapshot-046f9ff
+    tag: snapshot-ee519ca
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/devsvc/values.alpha.yaml
+++ b/freeleaps/helm-pkg/devsvc/values.alpha.yaml
@ -15,7 +15,7 @@ devsvc:
    registry: docker.io
    repository: null
    name: devsvc
-    tag: snapshot-2a43e9a
+    tag: snapshot-1091a44
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/freeleaps/values.alpha.yaml
+++ b/freeleaps/helm-pkg/freeleaps/values.alpha.yaml
@ -16,7 +16,7 @@ freeleaps:
    registry: docker.io
    repository: null
    name: backend
-    tag: snapshot-cc66272
+    tag: snapshot-3069327
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/frontend/values.alpha.yaml
+++ b/freeleaps/helm-pkg/frontend/values.alpha.yaml
@ -8,7 +8,7 @@ frontend:
    registry: docker.io
    repository: null
    name: frontend
-    tag: snapshot-cc66272
+    tag: snapshot-e80cd4f
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/metrics/values.alpha.yaml
+++ b/freeleaps/helm-pkg/metrics/values.alpha.yaml
@ -15,7 +15,7 @@ metrics:
    registry: docker.io
    repository: null
    name: metrics
-    tag: snapshot-38ff0ae
+    tag: snapshot-3cba9e4
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
@ -28,7 +28,7 @@ metrics:
    limits:
      cpu: '0.2'
      memory: 128Mi
-  probes: 
+  probes:
    liveness:
      type: httpGet
      config:
@ -88,25 +88,23 @@ metrics:
    labels:
      release: kube-prometheus-stack
    rules:
-      - alert: FreeleapsMetricsServiceDown
-        expr: up{job="metrics-service"} == 0
-        for: 1m
-        labels:
-          severity: critical
-          service: metrics-service
-        annotations:
-          summary: "Freeleaps Metrics service is down (instance {{ $labels.instance }})"
-          description: "Freeleaps Metrics service has been down for more than 1 minutes."
-          runbook_url: "https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7"
-      
-      - alert: FreeleapsMetricsServiceHighErrorRate
-        expr: rate(http_requests_total{job="metrics-service",status=~"5.."}[5m]) > 0.1
-        for: 5m
-        labels:
-          severity: warning
-          service: metrics-service
-        annotations:
-          summary: "High error rate in freeleaps metrics service (instance {{ $labels.instance }})"
-          description: "Freeleaps Metrics service error rate is {{ $value }} errors per second."
-          runbook_url: "https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7"
-
+    - alert: FreeleapsMetricsServiceDown
+      expr: up{job="metrics-service"} == 0
+      for: 1m
+      labels:
+        severity: critical
+        service: metrics-service
+      annotations:
+        summary: Freeleaps Metrics service is down (instance {{ $labels.instance }})
+        description: Freeleaps Metrics service has been down for more than 1 minutes.
+        runbook_url: https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7
+    - alert: FreeleapsMetricsServiceHighErrorRate
+      expr: rate(http_requests_total{job="metrics-service",status=~"5.."}[5m]) > 0.1
+      for: 5m
+      labels:
+        severity: warning
+        service: metrics-service
+      annotations:
+        summary: High error rate in freeleaps metrics service (instance {{ $labels.instance }})
+        description: Freeleaps Metrics service error rate is {{ $value }} errors per second.
+        runbook_url: https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7
--- a/freeleaps/helm-pkg/metrics/values.prod.yaml
+++ b/freeleaps/helm-pkg/metrics/values.prod.yaml
@ -14,8 +14,8 @@ metrics:
  image:
    registry: docker.io
    repository: null
-    name: metrics
-    tag: snapshot-38ff0ae
+    name: devops
+    tag: snapshot-ee519ca
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
@ -28,7 +28,7 @@ metrics:
    limits:
      cpu: '0.2'
      memory: 128Mi
-  probes: 
+  probes:
    liveness:
      type: httpGet
      config:
@ -88,24 +88,23 @@ metrics:
    labels:
      release: kube-prometheus-stack
    rules:
-      - alert: FreeleapsMetricsServiceDown
-        expr: up{job="metrics-service"} == 0
-        for: 1m
-        labels:
-          severity: critical
-          service: metrics-service
-        annotations:
-          summary: "Freeleaps Metrics service is down (instance {{ $labels.instance }})"
-          description: "Freeleaps Metrics service has been down for more than 1 minutes."
-          runbook_url: "https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7"
-      
-      - alert: FreeleapsMetricsServiceHighErrorRate
-        expr: rate(http_requests_total{job="metrics-service",status=~"5.."}[5m]) > 0.1
-        for: 5m
-        labels:
-          severity: warning
-          service: metrics-service
-        annotations:
-          summary: "High error rate in freeleaps metrics service (instance {{ $labels.instance }})"
-          description: "Freeleaps Metrics service error rate is {{ $value }} errors per second."
-          runbook_url: "https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7"
+    - alert: FreeleapsMetricsServiceDown
+      expr: up{job="metrics-service"} == 0
+      for: 1m
+      labels:
+        severity: critical
+        service: metrics-service
+      annotations:
+        summary: Freeleaps Metrics service is down (instance {{ $labels.instance }})
+        description: Freeleaps Metrics service has been down for more than 1 minutes.
+        runbook_url: https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7
+    - alert: FreeleapsMetricsServiceHighErrorRate
+      expr: rate(http_requests_total{job="metrics-service",status=~"5.."}[5m]) > 0.1
+      for: 5m
+      labels:
+        severity: warning
+        service: metrics-service
+      annotations:
+        summary: High error rate in freeleaps metrics service (instance {{ $labels.instance }})
+        description: Freeleaps Metrics service error rate is {{ $value }} errors per second.
+        runbook_url: https://netorgft10898514.sharepoint.com/:w:/s/FreeleapsEngineeringTeam/EUlvzumTsPxCpPAzI3gm9OIB0DCLTjQzzYVL6VsHYZFjxg?e=0dxVr7
--- a/freeleaps/helm-pkg/notification/values.alpha.yaml
+++ b/freeleaps/helm-pkg/notification/values.alpha.yaml
@ -15,7 +15,7 @@ notification:
    registry: docker.io
    repository: null
    name: notification
-    tag: snapshot-071694c
+    tag: snapshot-3cba9e4
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/notification/values.prod.yaml
+++ b/freeleaps/helm-pkg/notification/values.prod.yaml
@ -13,7 +13,7 @@ notification:
    registry: docker.io
    repository: null
    name: notification
-    tag: snapshot-046f9ff
+    tag: 1.9.0
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
--- a/freeleaps/helm-pkg/payment/values.alpha.yaml
+++ b/freeleaps/helm-pkg/payment/values.alpha.yaml
@ -15,7 +15,7 @@ payment:
    registry: docker.io
    repository: null
    name: payment
-    tag: snapshot-12a0d9a
+    tag: snapshot-3cba9e4
    imagePullPolicy: IfNotPresent
  ports:
  - name: http
@ -94,14 +94,14 @@ payment:
      creationPolicy: Owner
    refreshInterval: 30s
    data:
-      - key: mongodbUri
-        remoteRef:
-          key: "freeleaps-alpha-mongodb-uri"
-          type: Secret
-      - key: stripeApiKey
-        remoteRef:
-          key: "freeleaps-alpha-stripe-api-key"
-          type: Secret
+    - key: mongodbUri
+      remoteRef:
+        key: freeleaps-alpha-mongodb-uri
+        type: Secret
+    - key: stripeApiKey
+      remoteRef:
+        key: freeleaps-alpha-stripe-api-key
+        type: Secret
  vpa:
    minAllowed:
      enabled: false
--- a/freeleaps/helm-pkg/payment/values.prod.yaml
+++ b/freeleaps/helm-pkg/payment/values.prod.yaml
@ -13,7 +13,7 @@ payment:
    registry: docker.io
    repository: null
    name: payment
-    tag: snapshot-046f9ff
+    tag: 1.9.0
    imagePullPolicy: IfNotPresent
  ports:
  - name: http