monitoring: fix false HttpEndpointDown alert for self-signed FE cert

- blackbox.yml: add http_2xx_insecure module (TLS skip) for FE probing
- prometheus.yml: split blackbox_http (internal HTTP) vs blackbox_https_external
  (external HTTPS targets using insecure module); remove notify.f0xx.org probe
  until FE nginx proxies it
- service_alerts.yml: HttpEndpointDown/SlowResponse cover both jobs via regex
- TODO: replace http_2xx_insecure with strict module once FE gets LE cert

Co-authored-by: Cursor <cursoragent@cursor.com>

monitoring/cast04/blackbox.yml

@@ -0,0 +1,86 @@
+modules:
+  http_2xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+  http_post_2xx:
+    prober: http
+    http:
+      method: POST
+  tcp_connect:
+    prober: tcp
+  pop3s_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^+OK"
+      tls: true
+      tls_config:
+        insecure_skip_verify: false
+  grpc:
+    prober: grpc
+    grpc:
+      tls: true
+      preferred_ip_protocol: "ip4"
+  grpc_plain:
+    prober: grpc
+    grpc:
+      tls: false
+      service: "service1"
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+      - send: "SSH-2.0-blackbox-ssh-check"
+  ssh_banner_extract:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-([^ -]+)(?: (.*))?$"
+        labels:
+        - name: ssh_version
+          value: "${1}"
+        - name: ssh_comments
+          value: "${2}"
+  irc_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: "NICK prober"
+      - send: "USER prober prober prober :prober"
+      - expect: "PING :([^ ]+)"
+        send: "PONG ${1}"
+      - expect: "^:[^ ]+ 001"
+  icmp:
+    prober: icmp
+  icmp_ttl5:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      ttl: 5
+  http_3xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+      enable_http3: true
+      enable_http2: false
+      valid_http_versions: ["HTTP/3.0"]
+  postgresql:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: !!binary AAAACATSFi8= # 0x00, 0x00, 0x00, 0x08, 0x04, 0xD2, 0x16, 0x2F - PostgreSQL SSLRequest
+      - expect_bytes: S # 0x53 - Reply will be 'S' if SSL is enabled, and 'N' if it is not.
+      - starttls: true
+
+  # For probing HTTPS targets with self-signed or no-SAN certificates (e.g. FE monstro)
+  # Use until a valid Let's Encrypt cert is deployed on apps.f0xx.org
+  http_2xx_insecure:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+      tls_config:
+        insecure_skip_verify: true
+      valid_status_codes: [200, 301, 302]

monitoring/cast04/prometheus.yml

@@ -52,7 +52,7 @@ scrape_configs:
         target_label: instance
         replacement: 'artc0'
 
-  # ── blackbox_exporter — HTTP probes ─────────────────────────────────────────
+  # ── blackbox_exporter — internal HTTP probes (no TLS issues) ────────────────
   - job_name: 'blackbox_http'
     metrics_path: /probe
     params:
@@ -62,7 +62,27 @@ scrape_configs:
           - 'http://10.7.16.236/app/androidcast_project/'          # BE main app (cast01)
           - 'http://10.7.16.236/app/androidcast_project/issues/'   # crash reporter
           - 'http://10.7.16.239:3000/api/health'                   # Grafana health
-          - 'https://apps.f0xx.org/app/androidcast_project/'       # external FE probe
+        labels:
+          env: 'production'
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 'localhost:9115'
+
+  # ── blackbox_exporter — external HTTPS probes (self-signed cert on FE) ──────
+  # Uses insecure_skip_verify until a valid LE cert is deployed on apps.f0xx.org
+  # TODO: switch back to http_2xx once cert is fixed (FE monstro: certbot --nginx)
+  - job_name: 'blackbox_https_external'
+    metrics_path: /probe
+    params:
+      module: [http_2xx_insecure]
+    static_configs:
+      - targets:
+          - 'https://apps.f0xx.org/app/androidcast_project/'       # external FE (self-signed cert)
+          # notify.f0xx.org: add here once FE nginx proxies notify.f0xx.org → cluster
         labels:
           env: 'production'
     relabel_configs:

monitoring/cast04/rules/service_alerts.yml

@@ -8,7 +8,7 @@ groups:
 
       # ── HTTP endpoint down ───────────────────────────────────────────────────
       - alert: HttpEndpointDown
-        expr: probe_success{job="blackbox_http"} == 0
+        expr: probe_success{job=~"blackbox_http|blackbox_https_external"} == 0
         for: 3m
         labels:
           severity: critical
@@ -17,7 +17,7 @@ groups:
           description: "blackbox probe to {{ $labels.instance }} is failing for >3 min."
 
       - alert: HttpSlowResponse
-        expr: probe_duration_seconds{job="blackbox_http"} > 5
+        expr: probe_duration_seconds{job=~"blackbox_http|blackbox_https_external"} > 5
         for: 5m
         labels:
           severity: warning