To enable Datadog agent in our projects we need to add .cpu(512) and .memoryReservation(1024). Numbers have to be tweaked to the size of the Instance.
Then we would need to add the following piece of code in our CDK
To send logs to Datadog, we have to include another container into our task-definition. According to datadoghq.eu, we would need to use the firelens/fluent bit container, which you will see further down the text on this page.
Add this in the CDK stack where you add your container and task definition
val dataDogSite = CfnParameter(this, "datadog-site", CfnParameterProps.builder()
.description("Site URL")
.defaultValue("datadoghq.eu")
.build())
val datadogApiKey = CfnParameter(this, "datadog-api-key", CfnParameterProps.builder()
.description("API Key from DataDog")
.build())
val datadogAppKey = CfnParameter(this, "datadog-app-key", CfnParameterProps.builder()
.description("APP key from Datadog")
.build())
val container = fargateTaskDefinition.addContainer("container",
ContainerDefinitionOptions.builder()
.image(ContainerImage.fromEcrRepository(ecrRepo, imageTag))
.environment(mapOf(
"SERVER_SERVLET_CONTEXT_PATH" to SERVICE_ROUTING,
"DD_VERSION" to imageTag
))
.logging(FireLensLogDriver(FireLensLogDriverProps.builder()
.options(mapOf(
"Name" to "datadog",
"Host" to "http-intake.logs.datadoghq.eu",
"dd_service" to <SERVICE-NAME>,
"dd_source" to "java",
"dd_tags" to <EXTRA_TAGS>,
"dd_message_key" to "log",
"provider" to "ecs",
"apikey" to datadogApiKey.valueAsString))
.build())
)
.dockerLabels(mapOf(
"com.datadoghq.tags.env" to <ENVIRONMENT>,
"com.datadoghq.tags.service" to <SERVICENAME>,
"com.datadoghq.tags.version" to imageTag))
.cpu(512)
.memoryReservationMiB(1024)
.build())
// Traffic port
container.addPortMappings(PortMapping.builder()
.protocol(Protocol.TCP)
.containerPort(8080)
.hostPort(8080)
.build())
// Admin (health check) port
container.addPortMappings(PortMapping.builder()
.protocol(Protocol.TCP)
.containerPort(8090)
.hostPort(8090)
.build())
val datadogContainer = fargateTaskDefinition.addContainer("datadog-sidecar",
ContainerDefinitionOptions.builder()
.image(ContainerImage.fromRegistry("datadog/agent"))
.memoryReservationMiB(256)
.cpu(10)
.essential(true)
.logging(LogDriver.awsLogs(AwsLogDriverProps.builder()
.streamPrefix("server")
.logGroup(logGroup)
.build()))
.environment(
mutableMapOf(
"ECS_FARGATE" to "true",
"DD_SITE" to dataDogSite.valueAsString,
"DD_API_KEY" to datadogApiKey.valueAsString,
"DD_APM_ENABLED" to "true",
"DD_APM_NON_LOCAL_TRAFFIC" to "true",
"DD_DOCKER_LABELS_AS_TAGS" to "true",
"DD_DOGSTATSD_NON_LOCAL_TRAFFIC" to "true",
"DD_ENV" to namePrefix
// "LOG_LEVEL" to "debug"
)).build())
datadogContainer.addPortMappings(PortMapping.builder()
.protocol(Protocol.TCP)
.containerPort(8126)
.hostPort(8126)
.build())
datadogContainer.addPortMappings(PortMapping.builder()
.protocol(Protocol.UDP)
.containerPort(8125)
.hostPort(8125)
.build())
val firelens = fargateTaskDefinition.addFirelensLogRouter("firelens",
FirelensLogRouterDefinitionOptions.builder()
.image(ContainerImage.fromRegistry("amazon/aws-for-fluent-bit:latest"))
.cpu(10)
.memoryReservationMiB(256)
.logging(LogDriver.awsLogs(AwsLogDriverProps.builder()
.streamPrefix("$namePrefix-firelens")
.logGroup(logGroup)
.build()))
.firelensConfig(FirelensConfig.builder()
.type(FirelensLogRouterType.FLUENTBIT)
.options(FirelensOptions.builder()
.configFileType(FirelensConfigFileType.FILE)
.configFileValue("/fluent-bit/configs/parse-json.conf")
.enableEcsLogMetadata(true)
.build())
.build())
.build())
Add below to the application.properties
management.metrics.export.statsd.flavor=datadog
management.metrics.export.statsd.host=localhost
management.metrics.export.statsd.port=8125
<!-- Logging and metrics -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
<version>2.3.3.RELEASE</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-statsd</artifactId>
<version>1.5.5</version>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
<version>1.5.4</version>
</dependency>
<!-- End of Logging and metrics -->
.gitlab-ci.ymlwe would need to add the following line
- cdk deploy <STACK_NAME> --parameters cdkimagetag=$CI_COMMIT_SHORT_SHA --parameters datadogsite=$DATADOG_SITE --parameters datadogapikey=$DATADOG_API --parameters datadogappkey=$DATADOG_APP --require-approval never
Steps to enable datadog on a pure host using linux container.
a. Script that starts the containers
docker run -d --name dd-agent \
-e DD_ENV=environment \
-e DD_SITE=datadoghq.eu \
-e DD_API_KEY=datadogapikey \
-e DD_APM_ENABLED=true \
-e DD_LOGS_ENABLED=true \
-e DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL=true \
-e DD_CONTAINER_EXCLUDE="name:dd-agent" \
-e DD_PROCESS_AGENT_ENABLED=true \
-p 8126:8126/tcp \
-p 8125:8125/udp \
-v /var/run/docker.sock:/var/run/docker.sock:ro \
-v /proc:/host/proc:ro \
-v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro \
-v /opt/datadog-agent/run:/opt/datadog-agent/run:rw \
-v /etc/passwd:/etc/passwd:ro \
datadog/agent
docker-compose file
datadog-agent:
image: datadog/agent
environment:
DD_ENV: cs-local
DD_SITE: datadoghq.eu
DD_API_KEY: ${DD_API_KEY}
DD_APM_ENABLED: "true"
DD_LOGS_ENABLED: "true"
DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL: "true"
DD_CONTAINER_EXCLUDE: "name:datadog-agent"
DD_PROCESS_AGENT_ENABLED: "true"
ports:
- 8125:8125/udp
- 8126:8126/tcp
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- /proc:/host/proc:ro
- /sys/fs/cgroup/:/host/sys/fs/cgroup:ro
- /opt/datadog-agent/run:/opt/datadog-agent/run:rw
- /etc/passwd:/etc/passwd:ro
http://logback.qos.ch/manual/layouts.html
logFormat: "[%-5level] %d{ISO8601} [%-62thread] %+40(%logger{0}): %msg%n%ex{5}"
docker run -e DD_ENV=test -e DD_SERVICE=serviceName -e DD_VERSION=commitSha ...
DD_CONTAINER_EXCLUDE = “image:<IMAGE_NAME>”