GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-13 06:19:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Feat/integrations v0 mongo and nginx (#4763)

Browse Source 
* feat: flesh out pre-requisites for collecting mongodb logs and metrics

* chore: remove stale pipelines in bundled integrations

* chore: clean up 'collect metrics' step for mongodb

* feat: add instructions for collecting and parsing mongodb logs

* feat: add metrics and logs attributes to mongodb data collected list

* feat: nginx logs collection instructions and some other cleanup

* feat: add list of parsed log attributes to data collected list for nginx

* chore: do not run pipeline population integration test if no built-in integration has a pipeline
This commit is contained in:
Raj Kamal Singh 2024-03-28 19:57:07 +05:30 committed by GitHub
parent 5d5ff47d5e
commit 990fc83269
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 617 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
pkg/query-service/app/integrations/builtin_integrations/mongo/assets/pipelines/log-parser.json
View File

@ -1,33 +0,0 @@
{
"id": "parse-default-mongo-access-log",
"name": "Parse default mongo access log",
"alias": "parse-default-mongo-access-log",
"description": "Parse standard mongo access log",
"enabled": true,
"filter": {
"op": "AND",
"items": [
{
"key": {
"type": "tag",
"key": "source",
"dataType": "string"
},
"op": "=",
"value": "mongo"
}
]
},
"config": [
{
"type": "grok_parser",
"id": "parse-body-grok",
"enabled": true,
"orderId": 1,
"name": "Parse Body",
"parse_to": "attributes",
"pattern": "%{GREEDYDATA}",
"parse_from": "body"
}
]
}

117
pkg/query-service/app/integrations/builtin_integrations/mongo/config/collect-logs.md Normal file
View File

@ -0,0 +1,117 @@
### Collect MongoDB Logs
You can configure MongoDB logs collection by providing the required collector config to your collector.
#### Create collector config file
Save the following config for collecting MongoDB logs in a file named `mongodb-logs-collection-config.yaml`
```yaml
receivers:
filelog/mongodb:
include: ["${env:MONGODB_LOG_FILE}"]
operators:
# Parse structured mongodb logs
# For more details, see https://www.mongodb.com/docs/manual/reference/log-messages/#structured-logging
- type: json_parser
if: body matches '^\\s*{\\s*".*}\\s*$'
parse_from: body
parse_to: attributes
timestamp:
parse_from: attributes.t.$$date
layout: '2006-01-02T15:04:05.000-07:00'
layout_type: gotime
severity:
parse_from: attributes.s
overwrite_text: true
mapping:
debug:
- D1
- D2
- D3
- D4
- D5
info: I
warn: W
error: E
fatal: F
- type: flatten
if: attributes.attr != nil
field: attributes.attr
- type: move
if: attributes.msg != nil
from: attributes.msg
to: body
- type: move
if: attributes.c != nil
from: attributes.c
to: attributes.component
- type: move
if: attributes.id != nil
from: attributes.id
to: attributes.mongo_log_id
- type: remove
if: attributes.t != nil
field: attributes.t
- type: remove
if: attributes.s != nil
field: attributes.s
- type: add
field: attributes.source
value: mongodb
processors:
batch:
send_batch_size: 10000
send_batch_max_size: 11000
timeout: 10s
exporters:
# export to SigNoz cloud
otlp/mongodb-logs:
endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
tls:
insecure: false
headers:
"signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
# export to local collector
# otlp/mongodb-logs:
# endpoint: "localhost:4317"
# tls:
# insecure: true
service:
pipelines:
logs/mongodb:
receivers: [filelog/mongodb]
processors: [batch]
exporters: [otlp/mongodb-logs]
```
#### Set Environment Variables
Set the following environment variables in your otel-collector environment:
```bash
# path of MongoDB server log file. must be accessible by the otel collector
export MONGODB_LOG_FILE=/var/log/mongodb.log
# region specific SigNoz cloud ingestion endpoint
export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
# your SigNoz ingestion key
export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
```
#### Use collector config file
Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector
```bash
--config mongodb-logs-collection-config.yaml
```
Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.

62
pkg/query-service/app/integrations/builtin_integrations/mongo/config/configure-otel-collector.md → pkg/query-service/app/integrations/builtin_integrations/mongo/config/collect-metrics.md
View File

@ -1,19 +1,21 @@
### Configure otel collector ### Collect MongoDB Metrics
#### Save collector config file You can configure MongoDB metrics collection by providing the required collector config to your collector.
Save the following collector config in a file named `mongo-collector-config.yaml` #### Create collector config file
```bash Save the following config for collecting mongodb metrics in a file named `mongodb-metrics-collection-config.yaml`
```yaml
receivers: receivers:
mongodb: mongodb:
# - For standalone MongoDB deployments this is the hostname and port of the mongod instance # - For standalone MongoDB deployments this is the hostname and port of the mongod instance
# - For replica sets specify the hostnames and ports of the mongod instances that are in the replica set configuration. If the replica_set field is specified, nodes will be autodiscovered. # - For replica sets specify the hostnames and ports of the mongod instances that are in the replica set configuration. If the replica_set field is specified, nodes will be autodiscovered.
# - For a sharded MongoDB deployment, please specify a list of the mongos hosts. # - For a sharded MongoDB deployment, please specify a list of the mongos hosts.
hosts: hosts:
- endpoint: 127.0.0.1:27017 - endpoint: ${env:MONGODB_ENDPOINT}
# If authentication is required, the user can with clusterMonitor permissions can be provided here # If authentication is required, the user can with clusterMonitor permissions can be provided here
username: monitoring username: ${env:MONGODB_USERNAME}
# If authentication is required, the password can be provided here. # If authentication is required, the password can be provided here.
password: ${env:MONGODB_PASSWORD} password: ${env:MONGODB_PASSWORD}
collection_interval: 60s collection_interval: 60s
@ -46,18 +48,19 @@ processors:
hostname_sources: ["os"] hostname_sources: ["os"]
exporters: exporters:
# export to local collector
otlp/local:
endpoint: "localhost:4317"
tls:
insecure: true
# export to SigNoz cloud # export to SigNoz cloud
otlp/signoz: otlp/mongodb:
endpoint: "ingest.{region}.signoz.cloud:443" endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
tls: tls:
insecure: false insecure: false
headers: headers:
"signoz-access-token": "<SIGNOZ_INGESTION_KEY>" "signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
# export to local collector
# otlp/mongodb:
# endpoint: "localhost:4317"
# tls:
# insecure: true
service: service:
pipelines: pipelines:
@ -65,10 +68,37 @@ service:
receivers: [mongodb] receivers: [mongodb]
# note: remove this processor if the collector host is not running on the same host as the mongo instance # note: remove this processor if the collector host is not running on the same host as the mongo instance
processors: [resourcedetection/system] processors: [resourcedetection/system]
exporters: [otlp/local] exporters: [otlp/mongodb]
```
#### Set Environment Variables
Set the following environment variables in your otel-collector environment:
```bash
# MongoDB endpoint reachable from the otel collector"
export MONGODB_ENDPOINT="host:port"
# password for MongoDB monitoring user"
export MONGODB_USERNAME="monitoring"
# password for MongoDB monitoring user"
export MONGODB_PASSWORD="<PASSWORD>"
# region specific SigNoz cloud ingestion endpoint
export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
# your SigNoz ingestion key
export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
``` ```
#### Use collector config file #### Use collector config file
Run your collector with the added flag `--config mongo-collector-config.yaml` Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector
```bash
--config mongodb-metrics-collection-config.yaml
```
Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.

53
pkg/query-service/app/integrations/builtin_integrations/mongo/config/prerequisites.md
View File

@ -1,22 +1,41 @@
### Prepare mongo for monitoring ## Before You Begin
- Have a running mongodb instance To configure metrics and logs collection for MongoDB, you need the following.
- Have the monitoring user created
- Have the monitoring user granted the necessary permissions
Mongodb recommends to set up a least privilege user (LPU) with a `clusterMonitor` role in order to collect. ### Ensure MongoDB server is prepared for monitoring
Run the following command to create a user with the necessary permissions. - **Ensure that the MongoDB server is running a supported version**
MongoDB versions 4.4+ are supported.
You can use the following statement to determine server version
```js
db.version()
```
```bash - **If collecting metrics, ensure that there is a MongoDB user with required permissions**
use admin Mongodb recommends to set up a least privilege user (LPU) with a clusterMonitor role in order to collect metrics
db.createUser(
{
user: "monitoring",
pwd: "<PASSWORD>",
roles: ["clusterMonitor"]
}
);
```
Replace `<PASSWORD>` with a strong password and set is as env var `MONGODB_PASSWORD`. To create a monitoring user, run:
```js
use admin
db.createUser(
{
user: "monitoring",
pwd: "<PASSWORD>",
roles: ["clusterMonitor"]
}
);
```
### Ensure OTEL Collector is running with access to the MongoDB server
- **Ensure that an OTEL collector is running in your deployment environment**
If needed, please [install an OTEL Collector](https://signoz.io/docs/tutorial/opentelemetry-binary-usage-in-virtual-machine/)
If already installed, ensure that the collector version is v0.88.0 or newer.
Also ensure that you can provide config files to the collector and that you can set environment variables and command line flags used for running it.
- **Ensure that the OTEL collector can access the MongoDB server**
In order to collect metrics, the collector must be able to access the MongoDB server as a client using the monitoring user.
In order to collect logs, the collector must be able to read the MongoDB server log file.

226
pkg/query-service/app/integrations/builtin_integrations/mongo/integration.json
View File

@ -18,18 +18,20 @@
"instructions": "file://config/prerequisites.md" "instructions": "file://config/prerequisites.md"
}, },
{ {
"title": "Configure Otel Collector", "title": "Collect Metrics",
"instructions": "file://config/configure-otel-collector.md" "instructions": "file://config/collect-metrics.md"
},
{
"title": "Collect Logs",
"instructions": "file://config/collect-logs.md"
} }
], ],
"assets": { "assets": {
"logs": { "logs": {
"pipelines": [ "pipelines": []
"file://assets/pipelines/log-parser.json"
]
}, },
"dashboards": [ "dashboards": [
"file://assets/dashboards/overview.json" "file://assets/dashboards/overview.json"
], ],
"alerts": [] "alerts": []
}, },
@ -52,37 +54,207 @@
"data_collected": { "data_collected": {
"logs": [ "logs": [
{ {
"name": "Request Method", "name": "Timestamp",
"path": "attributes[\"http.request.method\"]", "path": "timestamp",
"type": "string", "type": "timestamp"
"description": "HTTP method"
}, },
{ {
"name": "Request Path", "name": "Severity Text",
"path": "attributes[\"url.path\"]", "path": "severity_text",
"type": "string", "type": "string"
"description": "path requested"
}, },
{ {
"name": "Response Status Code", "name": "Severity Number",
"path": "attributes[\"http.response.status_code\"]", "path": "severity_number",
"type": "int", "type": "number"
"description": "HTTP response code" },
{
"name": "MongoDB Component",
"path": "attributes.component",
"type": "string"
} }
], ],
"metrics": [ "metrics": [
{ {
"name": "http.server.request.duration", "description": "The number of cache operations of the instance.",
"type": "Histogram", "unit": "number",
"unit": "s", "type": "Sum",
"description": "Duration of HTTP server requests" "name": "mongodb_cache_operations"
}, },
{ {
"name": "http.server.active_requests", "description": "The number of collections.",
"type": "UpDownCounter", "unit": "number",
"unit": "{ request }", "type": "Sum",
"description": "Number of active HTTP server requests" "name": "mongodb_collection_count"
},
{
"description": "The size of the collection. Data compression does not affect this value.",
"unit": "Bytes",
"type": "Sum",
"name": "mongodb_data_size"
},
{
"description": "The number of connections.",
"unit": "number",
"type": "Sum",
"name": "mongodb_connection_count"
},
{
"description": "The number of extents.",
"unit": "number",
"type": "Sum",
"name": "mongodb_extent_count"
},
{
"description": "The time the global lock has been held.",
"unit": "ms",
"type": "Sum",
"name": "mongodb_global_lock_time"
},
{
"description": "The number of indexes.",
"unit": "number",
"type": "Sum",
"name": "mongodb_index_count"
},
{
"description": "Sum of the space allocated to all indexes in the database, including free index space.",
"unit": "Bytes",
"type": "Sum",
"name": "mongodb_index_size"
},
{
"description": "The amount of memory used.",
"unit": "Bytes",
"type": "Sum",
"name": "mongodb_memory_usage"
},
{
"description": "The number of objects.",
"unit": "number",
"type": "Sum",
"name": "mongodb_object_count"
},
{
"description": "The latency of operations.",
"unit": "us",
"type": "Gauge",
"name": "mongodb_operation_latency_time"
},
{
"description": "The number of operations executed.",
"unit": "number",
"type": "Sum",
"name": "mongodb_operation_count"
},
{
"description": "The number of replicated operations executed.",
"unit": "number",
"type": "Sum",
"name": "mongodb_operation_repl_count"
},
{
"description": "The total amount of storage allocated to this collection.",
"unit": "Bytes",
"type": "Sum",
"name": "mongodb_storage_size"
},
{
"description": "The number of existing databases.",
"unit": "number",
"type": "Sum",
"name": "mongodb_database_count"
},
{
"description": "The number of times an index has been accessed.",
"unit": "number",
"type": "Sum",
"name": "mongodb_index_access_count"
},
{
"description": "The number of document operations executed.",
"unit": "number",
"type": "Sum",
"name": "mongodb_document_operation_count"
},
{
"description": "The number of bytes received.",
"unit": "Bytes",
"type": "Sum",
"name": "mongodb_network_io_receive"
},
{
"description": "The number of by transmitted.",
"unit": "Bytes",
"type": "Sum",
"name": "mongodb_network_io_transmit"
},
{
"description": "The number of requests received by the server.",
"unit": "number",
"type": "Sum",
"name": "mongodb_network_request_count"
},
{
"description": "The total time spent performing operations.",
"unit": "ms",
"type": "Sum",
"name": "mongodb_operation_time"
},
{
"description": "The total number of active sessions.",
"unit": "number",
"type": "Sum",
"name": "mongodb_session_count"
},
{
"description": "The number of open cursors maintained for clients.",
"unit": "number",
"type": "Sum",
"name": "mongodb_cursor_count"
},
{
"description": "The number of cursors that have timed out.",
"unit": "number",
"type": "Sum",
"name": "mongodb_cursor_timeout_count"
},
{
"description": "Number of times the lock was acquired in the specified mode.",
"unit": "number",
"type": "Sum",
"name": "mongodb_lock_acquire_count"
},
{
"description": "Number of times the lock acquisitions encountered waits because the locks were held in a conflicting mode.",
"unit": "number",
"type": "Sum",
"name": "mongodb_lock_acquire_wait_count"
},
{
"description": "Cumulative wait time for the lock acquisitions.",
"unit": "microseconds",
"type": "Sum",
"name": "mongodb_lock_acquire_time"
},
{
"description": "Number of times the lock acquisitions encountered deadlocks.",
"unit": "number",
"type": "Sum",
"name": "mongodb_lock_deadlock_count"
},
{
"description": "The health status of the server.",
"unit": "number",
"type": "Gauge",
"name": "mongodb_health"
},
{
"description": "The amount of time that the server has been running.",
"unit": "ms",
"type": "Sum",
"name": "mongodb_uptime"
} }
] ]
} }
} }

5
pkg/query-service/app/integrations/builtin_integrations/mongo/overview.md
View File

@ -1,3 +1,6 @@
### Monitor MongoDB with SigNoz ### Monitor MongoDB with SigNoz
Collect key MongoDB metrics and parse your MongoDB logs Collect key MongoDB metrics and view them with an out of the box dashboard.
Collect and parse MongoDB logs to populate timestamp, severity, and other log attributes for better querying and aggregation.

62
pkg/query-service/app/integrations/builtin_integrations/nginx/assets/pipelines/log-parser.json
View File

@ -1,62 +0,0 @@
{
"id": "parse-default-nginx-access-log",
"name": "Parse default nginx access log",
"alias": "parse-default-nginx-access-log",
"description": "Parse standard nginx access log",
"enabled": true,
"filter": {
"op": "AND",
"items": [
{
"key": {
"type": "tag",
"key": "source",
"dataType": "string"
},
"op": "=",
"value": "nginx"
}
]
},
"config": [
{
"type": "grok_parser",
"id": "parse-body-grok",
"enabled": true,
"orderId": 1,
"name": "Parse Body",
"parse_to": "attributes",
"pattern": "%{IP:client.address} - %{USERNAME:enduser.id} \\[%{HTTPDATE:time.local}\\] \"((%{WORD:http.method} %{DATA:http.path}(\\?%{DATA:http.query})? %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version})|%{DATA})\" %{INT:http.response.status_code:int} %{INT:http.request.body.bytes:int} \"%{NOTSPACE:http.referer}\" \"%{DATA:http.user.agent}\" %{INT:http.request.bytes:int} %{NUMBER:http.request.time:float} \\[%{DATA:proxy.upstream.name}?\\] \\[%{DATA:proxy.alternative.upstream.name}?\\] ((%{IP:network.peer.address}:%{INT:network.peer.port:int})|%{DATA})? (%{INT:http.response.bytes:int}|-)? (%{NUMBER:http.response.time:float}|-)? (%{NUMBER:network.peer.status.code:int}|-)? %{NOTSPACE:request.id}",
"parse_from": "body"
},
{
"type": "severity_parser",
"id": "parse-sev",
"enabled": true,
"orderId": 2,
"name": "Set Severity",
"parse_from": "attributes[\"http.response.status_code\"]",
"mapping": {
"debug": [
"1xx"
],
"error": [
"4xx"
],
"fatal": [
"5xx"
],
"info": [
"2xx"
],
"trace": [
"trace"
],
"warn": [
"3xx"
]
},
"overwrite_text": true
}
]
}

139
pkg/query-service/app/integrations/builtin_integrations/nginx/config/collect-logs.md Normal file
View File

@ -0,0 +1,139 @@
### Collect Nginx Logs
You can configure Nginx logs collection by providing the required collector config to your collector.
#### Create collector config file
Save the following config for collecting Nginx logs in a file named `nginx-logs-collection-config.yaml`
```yaml
receivers:
filelog/nginx-access-logs:
include: ["${env:NGINX_ACCESS_LOG_FILE}"]
operators:
# Parse the default nginx access log format. Nginx defaults to the "combined" log format
# $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"
# For more details, see https://nginx.org/en/docs/http/ngx_http_log_module.html
- type: regex_parser
if: body matches '^(?P<remote_addr>[0-9\\.]+) - (?P<remote_user>[^\\s]+) \\[(?P<ts>.+)\\] "(?P<request_method>\\w+?) (?P<request_path>.+?)" (?P<status>[0-9]+) (?P<body_bytes_sent>[0-9]+) "(?P<http_referrer>.+?)" "(?P<http_user_agent>.+?)"$'
parse_from: body
parse_to: attributes
regex: '^(?P<remote_addr>[0-9\.]+) - (?P<remote_user>[^\s]+) \[(?P<ts>.+)\] "(?P<request_method>\w+?) (?P<request_path>.+?)" (?P<status>[0-9]+) (?P<body_bytes_sent>[0-9]+) "(?P<http_referrer>.+?)" "(?P<http_user_agent>.+?)"$'
timestamp:
parse_from: attributes.ts
layout: "02/Jan/2006:15:04:05 -0700"
layout_type: gotime
severity:
parse_from: attributes.status
overwrite_text: true
mapping:
debug: "1xx"
info:
- "2xx"
- "3xx"
warn: "4xx"
error: "5xx"
- type: remove
if: attributes.ts != nil
field: attributes.ts
- type: add
field: attributes.source
value: nginx
filelog/nginx-error-logs:
include: ["${env:NGINX_ERROR_LOG_FILE}"]
operators:
# Parse the default nginx error log format.
# YYYY/MM/DD HH:MM:SS [LEVEL] PID#TID: *CID MESSAGE
# For more details, see https://github.com/phusion/nginx/blob/master/src/core/ngx_log.c
- type: regex_parser
if: body matches '^(?P<ts>.+?) \\[(?P<log_level>\\w+)\\] (?P<pid>\\d+)#(?P<tid>\\d+). \\*(?P<cid>\\d+) (?P<message>.+)$'
parse_from: body
parse_to: attributes
regex: '^(?P<ts>.+?) \[(?P<log_level>\w+)\] (?P<pid>\d+)#(?P<tid>\d+). \*(?P<cid>\d+) (?P<message>.+)$'
timestamp:
parse_from: attributes.ts
layout: "2006/01/02 15:04:05"
layout_type: gotime
severity:
parse_from: attributes.log_level
overwrite_text: true
mapping:
debug: "debug"
info:
- "info"
- "notice"
warn: "warn"
error:
- "error"
- "crit"
- "alert"
fatal: "emerg"
- type: remove
if: attributes.ts != nil
field: attributes.ts
- type: move
if: attributes.message != nil
from: attributes.message
to: body
- type: add
field: attributes.source
value: nginx
processors:
batch:
send_batch_size: 10000
send_batch_max_size: 11000
timeout: 10s
exporters:
# export to SigNoz cloud
otlp/nginx-logs:
endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
tls:
insecure: false
headers:
"signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
# export to local collector
# otlp/nginx-logs:
# endpoint: "localhost:4317"
# tls:
# insecure: true
service:
pipelines:
logs/nginx:
receivers: [filelog/nginx-access-logs, filelog/nginx-error-logs]
processors: [batch]
exporters: [otlp/nginx-logs]
```
#### Set Environment Variables
Set the following environment variables in your otel-collector environment:
```bash
# path of Nginx access log file. must be accessible by the otel collector
export NGINX_ACCESS_LOG_FILE=/var/log/nginx/access.log;
# path of Nginx error log file. must be accessible by the otel collector
export NGINX_ERROR_LOG_FILE=/var/log/nginx/error.log
# region specific SigNoz cloud ingestion endpoint
export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
# your SigNoz ingestion key
export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
```
#### Use collector config file
Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector
```bash
--config nginx-logs-collection-config.yaml
```
Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.

1
pkg/query-service/app/integrations/builtin_integrations/nginx/config/configure-otel-collector.md
View File

@ -1 +0,0 @@
### Configure otel collector

1
pkg/query-service/app/integrations/builtin_integrations/nginx/config/prepare-nginx.md
View File

@ -1 +0,0 @@
### Prepare nginx for observability

19
pkg/query-service/app/integrations/builtin_integrations/nginx/config/prerequisites.md Normal file
View File

@ -0,0 +1,19 @@
## Before You Begin
To configure logs collection for Nginx, you need the following.
### Ensure Nginx server is running a supported version
Ensure that your Nginx server is running a version newer than 1.0.0
### Ensure OTEL Collector is running with access to the Nginx server
- **Ensure that an OTEL collector is running in your deployment environment**
If needed, please [install an OTEL Collector](https://signoz.io/docs/tutorial/opentelemetry-binary-usage-in-virtual-machine/)
If already installed, ensure that the collector version is v0.88.0 or newer.
Also ensure that you can provide config files to the collector and that you can set environment variables and command line flags used for running it.
- **Ensure that the OTEL collector can access the Nginx server**
In order to collect logs, the collector must be able to read Nginx server log files.

77
pkg/query-service/app/integrations/builtin_integrations/nginx/integration.json
View File

@ -15,19 +15,17 @@
"overview": "file://overview.md", "overview": "file://overview.md",
"configuration": [ "configuration": [
{ {
"title": "Prepare Nginx", "title": "Prerequisites",
"instructions": "file://config/prepare-nginx.md" "instructions": "file://config/prerequisites.md"
}, },
{ {
"title": "Configure Otel Collector", "title": "Collect Logs",
"instructions": "file://config/configure-otel-collector.md" "instructions": "file://config/collect-logs.md"
} }
], ],
"assets": { "assets": {
"logs": { "logs": {
"pipelines": [ "pipelines": []
"file://assets/pipelines/log-parser.json"
]
}, },
"dashboards": null, "dashboards": null,
"alerts": null "alerts": null
@ -50,38 +48,57 @@
}, },
"data_collected": { "data_collected": {
"logs": [ "logs": [
{
"name": "Timestamp",
"path": "timestamp",
"type": "timestamp"
},
{
"name": "Severity Text",
"path": "severity_text",
"type": "string"
},
{
"name": "Severity Number",
"path": "severity_number",
"type": "number"
},
{
"name": "Body Bytes Sent",
"path": "attributes.body_bytes_sent",
"type": "string"
},
{
"name": "Referrer",
"path": "attributes.http_referrer",
"type": "string"
},
{
"name": "User Agent",
"path": "attributes.http_user_agent",
"type": "string"
},
{ {
"name": "Request Method", "name": "Request Method",
"path": "attributes[\"http.request.method\"]", "path": "attributes.request_method",
"type": "string", "type": "string"
"description": "HTTP method"
}, },
{ {
"name": "Request Path", "name": "Request Path",
"path": "attributes[\"url.path\"]", "path": "attributes.request_path",
"type": "string", "type": "string"
"description": "path requested"
}, },
{ {
"name": "Response Status Code", "name": "Response Status Code",
"path": "attributes[\"http.response.status_code\"]", "path": "attributes.status",
"type": "int", "type": "string"
"description": "HTTP response code"
}
],
"metrics": [
{
"name": "http.server.request.duration",
"type": "Histogram",
"unit": "s",
"description": "Duration of HTTP server requests"
}, },
{ {
"name": "http.server.active_requests", "name": "Remote Address",
"type": "UpDownCounter", "path": "attributes.remote_addr",
"unit": "{ request }", "type": "string"
"description": "Number of active HTTP server requests"
} }
] ],
"metrics": []
} }
} }

2
pkg/query-service/app/integrations/builtin_integrations/nginx/overview.md
View File

@ -1,3 +1,3 @@
### Monitor Nginx with SigNoz ### Monitor Nginx with SigNoz
Parse your Nginx logs and collect key metrics. Collect and parse Nginx logs to populate timestamp, severity, and other log attributes for better querying and aggregation.

2
pkg/query-service/app/integrations/builtin_integrations/postgres/config/collect-logs.md
View File

@ -35,7 +35,7 @@ receivers:
- LOG - LOG
- NOTICE - NOTICE
- DETAIL - DETAIL
warning: WARNING warn: WARNING
error: ERROR error: ERROR
fatal: fatal:
- FATAL - FATAL

33
pkg/query-service/app/integrations/builtin_integrations/redis/assets/pipelines/log-parser.json
View File

@ -1,33 +0,0 @@
{
"id": "parse-default-redis-access-log",
"name": "Parse default redis access log",
"alias": "parse-default-redis-access-log",
"description": "Parse standard redis access log",
"enabled": true,
"filter": {
"op": "AND",
"items": [
{
"key": {
"type": "tag",
"key": "source",
"dataType": "string"
},
"op": "=",
"value": "redis"
}
]
},
"config": [
{
"type": "grok_parser",
"id": "parse-body-grok",
"enabled": true,
"orderId": 1,
"name": "Parse Body",
"parse_to": "attributes",
"pattern": "%{GREEDYDATA}",
"parse_from": "body"
}
]
}

2
pkg/query-service/app/integrations/builtin_integrations/redis/config/collect-logs.md
View File

@ -29,7 +29,7 @@ receivers:
info: info:
- '-' - '-'
- '*' - '*'
warning: '#' warn: '#'
on_error: send on_error: send
- type: move - type: move
if: attributes.message != nil if: attributes.message != nil

4
pkg/query-service/app/integrations/builtin_integrations/redis/integration.json
View File

@ -28,9 +28,7 @@
], ],
"assets": { "assets": {
"logs": { "logs": {
"pipelines": [ "pipelines": []
"file://assets/pipelines/log-parser.json"
]
}, },
"dashboards": [ "dashboards": [
"file://assets/dashboards/overview.json" "file://assets/dashboards/overview.json"

7
pkg/query-service/tests/integration/signoz_integrations_test.go
View File

@ -141,9 +141,14 @@ func TestLogPipelinesForInstalledSignozIntegrations(t *testing.T) {
break break
} }
} }
require.NotNil(testAvailableIntegration)
if testAvailableIntegration == nil {
// None of the built in integrations include a pipeline right now.
return
}
// Installing an integration should add its pipelines to pipelines list // Installing an integration should add its pipelines to pipelines list
require.NotNil(testAvailableIntegration)
require.False(testAvailableIntegration.IsInstalled) require.False(testAvailableIntegration.IsInstalled)
integrationsTB.RequestQSToInstallIntegration( integrationsTB.RequestQSToInstallIntegration(
testAvailableIntegration.Id, map[string]interface{}{}, testAvailableIntegration.Id, map[string]interface{}{},