Falco Reference: K8s Runtime Security, Custom Rules, Falcosidekick Alerts & Tuning

# Install Falco with Helm (eBPF driver — no kernel module needed):
helm repo add falcosecurity https://falcosecurity.github.io/charts
helm install falco falcosecurity/falco   --namespace falco --create-namespace   --set driver.kind=ebpf \              # use eBPF instead of kernel module
  --set falcosidekick.enabled=true \    # sidecar for routing alerts
  --set falcosidekick.config.slack.webhookurl="https://hooks.slack.com/..."

kubectl get pods -n falco               # one falco pod per node (DaemonSet)
kubectl logs -n falco daemonset/falco   # check for rule alerts

# Or install falco CLI for local testing:
# https://falco.org/docs/getting-started/installation/

# Falco ships with ~100 default rules. Key ones:

# 1. Shell spawned in container (attacker exec'd a shell):
# rule: Terminal shell in container
# condition: spawned_process and container and shell_procs and proc.tty != 0

# 2. Sensitive file read:
# rule: Read sensitive file untrusted
# covers: /etc/shadow, /etc/passwd, private keys, kubectl credentials

# 3. Privilege escalation attempts:
# rule: Launch Privileged Container
# rule: Container Run as Root User
# rule: Change thread namespace (nsenter, setns)

# 4. Unexpected network connections:
# rule: Unexpected outbound connection destination
# rule: Contact K8s API server from container
# rule: Exfiltration attempt: unexpected DNS

# 5. Crypto mining detection:
# rule: Detected ptrace
# rule: Launch Suspicious Network Tool in Container (nmap, nc, curl to mining pools)

# Check which rules are loaded:
kubectl exec -n falco daemonset/falco -- falco --list   # all rules + macros

# Test Falco is working (trigger a known alert):
kubectl run -it --rm test --image=busybox -- sh
# Falco should alert: "Terminal shell in container"

# Custom rule syntax:
# - rule: descriptive name
#   desc: what it detects
#   condition: syscall + filter expression
#   output: alert message (with field substitutions)
#   priority: EMERGENCY/ALERT/CRITICAL/ERROR/WARNING/NOTICE/INFORMATIONAL/DEBUG
#   tags: [process, network, filesystem, etc.]

# Detect outbound connection to unexpected port:
- rule: Outbound Connection on Unexpected Port
  desc: Detect container making outbound connection to port other than 80/443
  condition: >
    outbound and
    not (fd.sport in (80, 443, 8080, 8443)) and
    container
  output: >
    Unexpected outbound port (port=%fd.sport
    container=%container.name
    image=%container.image.repository
    proc=%proc.name
    pid=%proc.pid)
  priority: WARNING
  tags: [network, container]

# Detect write to /etc in a container:
- rule: Write to /etc in Container
  desc: Detect any write to /etc directory inside a container
  condition: >
    open_write and
    container and
    fd.name startswith /etc
  output: >
    Write to /etc in container
    (file=%fd.name container=%container.name image=%container.image.repository)
  priority: ERROR

# Whitelist a specific container for a rule:
- list: allowed_images_for_etc_write
  items: ["my-config-manager:v1.2"]

# Modify condition to exclude:
  condition: >
    open_write and container and fd.name startswith /etc
    and not container.image.repository in (allowed_images_for_etc_write)

# Falcosidekick routes alerts to 50+ outputs:
# Slack, Teams, PagerDuty, Elasticsearch, Loki, Kafka, NATS, webhooks, etc.

# Helm values for alert routing:
falcosidekick:
  config:
    slack:
      webhookurl: "https://hooks.slack.com/services/..."
      minimumpriority: warning       # only WARNING+ goes to Slack

    pagerduty:
      routingKey: "ROUTING_KEY"
      minimumpriority: critical      # only CRITICAL+ pages on-call

    elasticsearch:
      hostport: http://elasticsearch:9200
      index: falco-alerts
      minimumpriority: ""            # all alerts to Elasticsearch

    loki:
      hostport: http://loki:3100
      minimumpriority: ""

    webhook:
      address: https://my-siem.example.com/api/falco
      minimumpriority: warning

# Falcosidekick UI (built-in dashboard):
falcosidekick:
  webui:
    enabled: true     # exposes dashboard on :2802

# Port-forward to view:
kubectl port-forward -n falco svc/falco-falcosidekick-ui 2802

# Override rules without editing base rules (values.yaml):
customRules:
  custom-rules.yaml: |-
    # Disable a noisy default rule:
    - rule: Contact K8s API Server From Container
      condition: >
        (k8s_api_call and not proc.name in (kubectl, helm, argo))
      override:
        condition: replace

    # Append to an existing list (add trusted images):
    - list: trusted_shell_containers
      append: true
      items: ["my-debug-image"]

# Falco tuning tips:
# 1. Start in NOTICE/INFO mode — see what fires before blocking
# 2. Add specific workloads to allowlists (not broad image-name globs)
# 3. Use tags to group rules: enable/disable by tag
# 4. Check false positive rate before escalating priority to CRITICAL

# Performance check:
kubectl exec -n falco daemonset/falco -- falco --stats-interval 1 2>&1 | head -20
# Shows: events/sec, drops/sec (drops = performance bottleneck)

# Reload rules without restart:
kubectl exec -n falco daemonset/falco -- kill -1 $(pgrep falco)

# Upgrade Falco:
helm upgrade falco falcosecurity/falco -n falco --reuse-values