Datadog Synthetic Tests with Terraform

To simplify management of Datadog synthetic tests, I decided to use Terraform. After playing with it a bit, I came with following module structure which covers few cases that I needed to handle.

HTTP Status Monitoring

To cover HTTP status monitoring, I have created ux-monitoring module, with following files

resource "datadog_synthetics_test" "this" {
  for_each = var.sites

  type    = "api"
  subtype = "http"
  request_definition {
    method = "GET"
    url    = each.value.url
  }
  assertion {
    type     = "statusCode"
    operator = "is"
    target   = "200"
  }
  locations = ["aws:${var.region}"]
  options_list {
    tick_every = 900

    retry {
      count    = 2
      interval = 300
    }

    monitor_options {
      renotify_interval = 120
    }
  }
  name    = each.value.name
  message = "Notify @pagerduty"
  tags    = concat(["terraform:true"], each.value.tags)

  status = "live"
}

terraform {
  required_providers {
    datadog = {
      source = "DataDog/datadog"
    }
  }
}

variable "region" {
  type = string
}

variable "sites" {
  type = map(object({
    name         = string
    url          = string
    tags         = list(string)
  }))
}

Then to use the module, I simply did the following:

module "ux_monitoring" {
  source = "./modules/ux-monitoring"
  region = data.aws_region.current.name
  sites  = {
    "site-a" = {
      name = "site-a"
      url  = "https://site-a.example.com/dashboard"
      tags = ["cluster_name:cluster", "environment:dev", "project:example"]
    }

    "site-b" = {
      name = "site-b"
      url  = "https://site-b.example.com/dashboard/management"
      tags = ["cluster_name:cluster", "environment:dev", "project:example"]
    }

    "site-c" = {
      name = "site-c"
      url  = "https://site-c.example.com/dashboard"
      tags = ["cluster_name:cluster", "environment:dev", "project:example"]
    }
  }
}

Monitoring Certificate Validity

For certificates I created cert-monitoring module which contained following contents:

resource "datadog_synthetics_test" "this" {

  for_each = var.sites

  type    = "api"
  subtype = "ssl"
  request_definition {
    host = each.value.host
    port = 443
  }
  assertion {
    type     = "certificate"
    operator = "isInMoreThan"
    target   = "14"
  }
  locations = ["aws:${var.region}"]
  options_list {
    tick_every = 900

    retry {
      count    = 2
      interval = 300
    }

    monitor_options {
      renotify_interval = 120
    }
  }
  name    = each.value.name
  message = "Notify @pagerduty"
  tags    = concat(["terraform:true"], each.value.tags)

  status = "live"
}

terraform {
  required_providers {
    datadog = {
      source = "DataDog/datadog"
    }
  }
}

variable "region" {
  type = string
}

variable "sites" {
  type = map(object({
    name         = string
    host          = string
    tags         = list(string)
  }))
}

To use the module, we’re going to use the following code:

module "cert_monitoring" {
  source = "./modules/cert-monitoring"
  region = data.aws_region.current.name
  sites  = {
    "site-a" = {
      name = "${var.environment} rlx-side-portal"
      host = "site-a.example.com"
      tags = tags = ["cluster_name:cluster", "environment:dev", "project:example"]
    }
   
  }
}

API Monitoring

Some services had a nested JSON structure which contained a boolean status that I needed to check. For this case I have created a json-monitoring module with following code:

locals {
  site_defaults = {
    method       = "GET"
    status_code  = 200
    tick_every   = 900
    message      = "Notify @pagerduty"
    status       = "live"
    body         = ""
    content_type = "application/json"
  }

  sites = {for k, v in var.sites : k => merge(local.site_defaults, v)}
}

resource "datadog_synthetics_test" "this" {
  for_each = local.sites
  type     = "api"
  subtype  = "http"
  request_definition {
    method = each.value.method
    url    = each.value.url
    body   = each.value.body
  }
  request_headers = {
    Content-Type = each.value.content_type
  }
  assertion {
    type     = "statusCode"
    operator = "is"
    target   = each.value.status_code
  }
  assertion {
    type     = "body"
    operator = "validatesJSONPath"
    targetjsonpath {
      jsonpath    = each.value.jsonpath
      operator    = "is"
      targetvalue = each.value.targetvalue
    }
  }
  locations = ["aws:${var.region}"]
  options_list {
    tick_every = each.value.tick_every

    retry {
      count    = 2
      interval = 300
    }

    monitor_options {
      renotify_interval = 120
    }
  }
  name    = each.value.name
  message = each.value.message
  tags    = concat(["terraform:true"], each.value.tags)

  status = each.value.status
}

terraform {
  required_providers {
    datadog = {
      source = "DataDog/datadog"
    }
  }
}

variable "region" {
  type = string
}

variable "sites" {
  type = any
}

Usage of module:

module "json_monitoring" {
  source = "./modules/json-monitoring"

  region = data.aws_region.current.name
  sites  = {
    "site-a" = {
      name        = "Site-A Health API Check"
      url         = "https://site-a.example.com/management/v2/ServerStatus"
      jsonpath    = "entity.ready"
      targetvalue = "true"
      tags        = tags = ["cluster_name:cluster", "environment:dev", "project:example"]
    }

    "site-b" = {
      name        = "Site-B Health API Check"
      url         = "https://site-b.example.com/calculation/app/v2/get-status"
      jsonpath    = "service_status.ready"
      targetvalue = "true"
      tags        = ["environment:${var.environment}", "project:radar-live"]
      method      = "POST"
      body        = jsonencode({
        "check_service" : true

      })
    }
  }
}