monitoring: fix false HttpEndpointDown alert for self-signed FE cert

- blackbox.yml: add http_2xx_insecure module (TLS skip) for FE probing
- prometheus.yml: split blackbox_http (internal HTTP) vs blackbox_https_external
  (external HTTPS targets using insecure module); remove notify.f0xx.org probe
  until FE nginx proxies it
- service_alerts.yml: HttpEndpointDown/SlowResponse cover both jobs via regex
- TODO: replace http_2xx_insecure with strict module once FE gets LE cert

Co-authored-by: Cursor <cursoragent@cursor.com>

monitoring/cast04/blackbox.yml

@@ -0,0 +1,86 @@
+modules:
+  http_2xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+  http_post_2xx:
+    prober: http
+    http:
+      method: POST
+  tcp_connect:
+    prober: tcp
+  pop3s_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^+OK"
+      tls: true
+      tls_config:
+        insecure_skip_verify: false
+  grpc:
+    prober: grpc
+    grpc:
+      tls: true
+      preferred_ip_protocol: "ip4"
+  grpc_plain:
+    prober: grpc
+    grpc:
+      tls: false
+      service: "service1"
+  ssh_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-"
+      - send: "SSH-2.0-blackbox-ssh-check"
+  ssh_banner_extract:
+    prober: tcp
+    timeout: 5s
+    tcp:
+      query_response:
+      - expect: "^SSH-2.0-([^ -]+)(?: (.*))?$"
+        labels:
+        - name: ssh_version
+          value: "${1}"
+        - name: ssh_comments
+          value: "${2}"
+  irc_banner:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: "NICK prober"
+      - send: "USER prober prober prober :prober"
+      - expect: "PING :([^ ]+)"
+        send: "PONG ${1}"
+      - expect: "^:[^ ]+ 001"
+  icmp:
+    prober: icmp
+  icmp_ttl5:
+    prober: icmp
+    timeout: 5s
+    icmp:
+      ttl: 5
+  http_3xx:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+      enable_http3: true
+      enable_http2: false
+      valid_http_versions: ["HTTP/3.0"]
+  postgresql:
+    prober: tcp
+    tcp:
+      query_response:
+      - send: !!binary AAAACATSFi8= # 0x00, 0x00, 0x00, 0x08, 0x04, 0xD2, 0x16, 0x2F - PostgreSQL SSLRequest
+      - expect_bytes: S # 0x53 - Reply will be 'S' if SSL is enabled, and 'N' if it is not.
+      - starttls: true
+
+  # For probing HTTPS targets with self-signed or no-SAN certificates (e.g. FE monstro)
+  # Use until a valid Let's Encrypt cert is deployed on apps.f0xx.org
+  http_2xx_insecure:
+    prober: http
+    http:
+      preferred_ip_protocol: "ip4"
+      tls_config:
+        insecure_skip_verify: true
+      valid_status_codes: [200, 301, 302]