Feat: QS: postgres integration: instructions for collecting and parsing logs (#4738)

* chore: offer metrics config instructions for signoz cloud only

* chore: some more cleanups

* chore: get log collection instructions started

* feat: flesh out log collection otel config for postgres

* chore: some cleanup

* chore: some more cleanup

* chore: some more cleanup

pkg/query-service/app/integrations/builtin_integrations/postgres/config/collect-logs.md

@@ -0,0 +1,109 @@
+### Collect Postgres Logs
+
+#### Create collector config file
+
+Save the following config for collecting postgres logs in a file named `postgres-logs-collection-config.yaml`
+
+```yaml
+receivers:
+  filelog/postgresql:
+    include: ["${env:POSTGRESQL_LOG_FILE}"]
+    operators:
+      # Parse default postgresql text log format.
+      # `log_line_prefix` postgres setting defaults to '%m [%p] ' which logs the timestamp and the process ID
+      # See https://www.postgresql.org/docs/current/runtime-config-logging.html#GUC-LOG-LINE-PREFIX for more details
+      - type: regex_parser
+        if: body matches '^(?P<ts>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}.?[0-9]*? [A-Z]*) \\[(?P<pid>[0-9]+)\\] (?P<log_level>[A-Z]*). (?P<message>.*)$'
+        parse_from: body
+        regex: '^(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.?[0-9]*? [A-Z]*) \[(?P<pid>[0-9]+)\] (?P<log_level>[A-Z]*). (?P<message>.*)$'
+        timestamp:
+          parse_from: attributes.ts
+          layout: '%Y-%m-%d %H:%M:%S %Z'
+        severity:
+          parse_from: attributes.log_level
+          mapping:
+            debug:
+              - DEBUG1
+              - DEBUG2
+              - DEBUG3
+              - DEBUG4
+              - DEBUG5
+            info:
+              - INFO
+              - LOG
+              - NOTICE
+              - DETAIL
+            warning: WARNING
+            error: ERROR
+            fatal:
+              - FATAL
+              - PANIC
+        on_error: send
+      - type: move
+        if: attributes.message != nil
+        from: attributes.message
+        to: body
+      - type: remove
+        if: attributes.log_level != nil
+        field: attributes.log_level
+      - type: remove
+        if: attributes.ts != nil
+        field: attributes.ts
+      - type: add
+        field: attributes.source
+        value: postgres
+
+processors:
+  batch:
+    send_batch_size: 10000
+    send_batch_max_size: 11000
+    timeout: 10s
+
+exporters:
+  # export to SigNoz cloud
+  otlp/postgres-logs:
+    endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
+    tls:
+      insecure: false
+    headers:
+      "signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
+
+  # export to local collector
+  # otlp/local:
+  #   endpoint: "localhost:4317"
+  #   tls:
+  #     insecure: true
+
+service:
+  pipelines:
+    postgresql:
+      receivers: [filelog/postgresql]
+      processors: [batch]
+      exporters: [otlp/postgresql-logs]
+```
+
+#### Set Environment Variables
+
+Set the following environment variables in your otel-collector environment:
+
+```bash
+
+# path of Postgres server log file. must be accessible by the otel collector
+export POSTGRESQL_LOG_FILE=/usr/local/var/log/postgres.log
+
+# region specific SigNoz cloud ingestion endpoint
+export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
+
+# your SigNoz ingestion key
+export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
+
+```
+
+#### Use collector config file
+
+Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector  
+```bash
+--config postgres-logs-collection-config.yaml
+```  
+Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.
+

pkg/query-service/app/integrations/builtin_integrations/postgres/config/collect-metrics.md

@@ -0,0 +1,101 @@
+### Collect Postgres Metrics
+
+You can configure Postgres metrics collection by providing the required collector config to your collector.
+
+#### Create collector config file
+
+Save the following config for collecting postgres metrics in a file named `postgres-metrics-collection-config.yaml`
+
+```yaml
+receivers:
+  postgresql:
+    # The endpoint of the postgresql server. Whether using TCP or Unix sockets, this value should be host:port. If transport is set to unix, the endpoint will internally be translated from host:port to /host.s.PGSQL.port
+    endpoint: ${env:POSTGRESQL_ENDPOINT}
+    # The frequency at which to collect metrics from the Postgres instance.
+    collection_interval: 60s
+    # The username used to access the postgres instance
+    username: ${env:POSTGRESQL_USERNAME}
+    # The password used to access the postgres instance
+    password: ${env:POSTGRESQL_PASSWORD}
+    # The list of databases for which the receiver will attempt to collect statistics. If an empty list is provided, the receiver will attempt to collect statistics for all non-template databases
+    databases: []
+    # # Defines the network to use for connecting to the server. Valid Values are `tcp` or `unix`
+    # transport: tcp
+    tls:
+      # set to false if SSL is enabled on the server
+      insecure: true
+    #   ca_file: /etc/ssl/certs/ca-certificates.crt
+    #   cert_file: /etc/ssl/certs/postgres.crt
+    #   key_file: /etc/ssl/certs/postgres.key
+    metrics:
+      postgresql.database.locks:
+        enabled: true
+      postgresql.deadlocks:
+        enabled: true
+      postgresql.sequential_scans:
+        enabled: true
+
+processors:
+  # enriches the data with additional host information
+  # see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#resource-detection-processor
+  resourcedetection/system:
+    # add additional detectors if needed
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+
+exporters:
+  # export to SigNoz cloud
+  otlp/postgres:
+    endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
+    tls:
+      insecure: false
+    headers:
+      "signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
+
+  # export to local collector
+  # otlp/local:
+  #   endpoint: "localhost:4317"
+  #   tls:
+  #     insecure: true
+
+service:
+  pipelines:
+    metrics/postgresql:
+      receivers: [postgresql]
+      # note: remove this processor if the collector host is not running on the same host as the postgres instance
+      processors: [resourcedetection/system]
+      exporters: [otlp/postgres]
+```
+
+#### Set Environment Variables
+
+Set the following environment variables in your otel-collector environment:
+
+```bash
+
+# password for Postgres monitoring user"
+export POSTGRESQL_USERNAME="monitoring"
+
+# password for Postgres monitoring user"
+export POSTGRESQL_PASSWORD="<PASSWORD>"
+
+# Postgres endpoint reachable from the otel collector"
+export POSTGRESQL_ENDPOINT="host:port"
+
+
+# region specific SigNoz cloud ingestion endpoint
+export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
+
+# your SigNoz ingestion key
+export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
+
+```
+
+#### Use collector config file
+
+Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector  
+```bash
+--config postgres-metrics-collection-config.yaml
+```  
+Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.

pkg/query-service/app/integrations/builtin_integrations/postgres/config/configure-otel-collector.md

@@ -1,146 +0,0 @@
-### Configure otel collector
-
-#### Create collector config file
-
-Save the collector config for monitoring postgres in a file named `postgres-collector-config.yaml`
-
-Use the following configuration for SigNoz cloud. See further below for configuration for self hosted SigNoz 
-
-```yaml
-receivers:
-  postgresql:
-    # The endpoint of the postgresql server. Whether using TCP or Unix sockets, this value should be host:port. If transport is set to unix, the endpoint will internally be translated from host:port to /host.s.PGSQL.port
-    endpoint: ${env:POSTGRESQL_ENDPOINT}
-    # The frequency at which to collect metrics from the Postgres instance.
-    collection_interval: 60s
-    # The username used to access the postgres instance
-    username: monitoring
-    # The password used to access the postgres instance
-    password: ${env:POSTGRESQL_PASSWORD}
-    # The list of databases for which the receiver will attempt to collect statistics. If an empty list is provided, the receiver will attempt to collect statistics for all non-template databases
-    databases: []
-    # # Defines the network to use for connecting to the server. Valid Values are `tcp` or `unix`
-    # transport: tcp
-    tls:
-      # set to false if SSL is enabled on the server
-      insecure: true
-    #   ca_file: /etc/ssl/certs/ca-certificates.crt
-    #   cert_file: /etc/ssl/certs/postgres.crt
-    #   key_file: /etc/ssl/certs/postgres.key
-    metrics:
-      postgresql.database.locks:
-        enabled: true
-      postgresql.deadlocks:
-        enabled: true
-      postgresql.sequential_scans:
-        enabled: true
-
-processors:
-  # enriches the data with additional host information
-  # see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#resource-detection-processor
-  resourcedetection/system:
-    # add additional detectors if needed
-    detectors: ["system"]
-    system:
-      hostname_sources: ["os"]
-
-exporters:
-  # export to SigNoz cloud
-  otlp/postgres:
-    endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
-    tls:
-      insecure: ${env:OTLP_DESTINATION_TLS_INSECURE}
-    headers:
-      "signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
-
-service:
-  pipelines:
-    metrics/postgresql:
-      receivers: [postgresql]
-      # note: remove this processor if the collector host is not running on the same host as the postgres instance
-      processors: [resourcedetection/system]
-      exporters: [otlp/postgres]
-```
-
-Use the following config if using self-hosted SigNoz. See the config above if using SigNoz cloud
-```yaml
-receivers:
-  postgresql:
-    # The endpoint of the postgresql server. Whether using TCP or Unix sockets, this value should be host:port. If transport is set to unix, the endpoint will internally be translated from host:port to /host.s.PGSQL.port
-    endpoint: ${env:POSTGRESQL_ENDPOINT}
-    # The frequency at which to collect metrics from the Postgres instance.
-    collection_interval: 60s
-    # The username used to access the postgres instance
-    username: monitoring
-    # The password used to access the postgres instance
-    password: ${env:POSTGRESQL_PASSWORD}
-    # The list of databases for which the receiver will attempt to collect statistics. If an empty list is provided, the receiver will attempt to collect statistics for all non-template databases
-    databases: []
-    # # Defines the network to use for connecting to the server. Valid Values are `tcp` or `unix`
-    # transport: tcp
-    tls:
-      # set to false if SSL is enabled on the server
-      insecure: true
-    #   ca_file: /etc/ssl/certs/ca-certificates.crt
-    #   cert_file: /etc/ssl/certs/postgres.crt
-    #   key_file: /etc/ssl/certs/postgres.key
-    metrics:
-      postgresql.database.locks:
-        enabled: true
-      postgresql.deadlocks:
-        enabled: true
-      postgresql.sequential_scans:
-        enabled: true
-
-processors:
-  # enriches the data with additional host information
-  # see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#resource-detection-processor
-  resourcedetection/system:
-    # add additional detectors if needed
-    detectors: ["system"]
-    system:
-      hostname_sources: ["os"]
-
-exporters:
-  otlp/postgres:
-    endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
-    tls:
-      insecure: ${env:OTLP_DESTINATION_TLS_INSECURE}
-
-service:
-  pipelines:
-    metrics/postgresql:
-      receivers: [postgresql]
-      # note: remove this processor if the collector host is not running on the same host as the postgres instance
-      processors: [resourcedetection/system]
-      exporters: [otlp/postgres]
-```
-
-
-#### Set Environment Variables
-
-Set the following environment variables in your otel-collector environment:
-
-```bash
-
-# password for postgres monitoring user"
-export POSTGRESQL_PASSWORD="password"
-
-# postgres endpoint reachable from the otel collector"
-export POSTGRESQL_ENDPOINT="host:port"
-
-# A reachable OTLP destination for collected metrics. Eg: localhost:4317 or signoz cloud ingestion endpoint
-export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
-
-# Set to true if using an endpoint without TLS
-export OTLP_DESTINATION_TLS_INSECURE="false"
-
-# your signoz ingestion key if using SigNoz cloud
-export SIGNOZ_INGESTION_KEY="key"
-
-```
-
-#### Use collector config file
-
-Make the `postgres-collector-config.yaml` file available to your otel collector and add the flag `--config postgres-collector-config.yaml` to the command for running your otel collector.    
-Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.

pkg/query-service/app/integrations/builtin_integrations/postgres/config/prerequisites.md

@@ -1,22 +1,40 @@
-### Prepare postgres for monitoring
+## Before You Begin  
 
-- Have a running postgresql instance
+To configure metrics and logs collection for a Postgres server, you need the following.
-- Have the monitoring user created
-- Have the monitoring user granted the necessary permissions
 
-This receiver supports PostgreSQL versions 9.6+
+### Ensure Postgres server is prepared for monitoring
 
-For PostgreSQL versions 10+, run:
+- **Ensure that the Postgres server is running a supported version**  
+  Postgres versions 9.6+ are supported.  
+  You can use the following SQL statement to determine server version  
+  ```SQL
+  SELECT version();
+  ```
 
-```sql
+- **If collecting metrics, ensure that there is a Postgres user with required permissions**  
-create user monitoring with password '<PASSWORD>';
+  To create a monitoring user for Postgres versions 10+, run:  
-grant pg_monitor to monitoring;
+  ```SQL
-grant SELECT ON pg_stat_database to monitoring;
+  create user monitoring with password '<PASSWORD>';
-```
+  grant pg_monitor to monitoring;
+  grant SELECT ON pg_stat_database to monitoring;
+  ```  
   
-For PostgreSQL versions >= 9.6 and <10, run:
+  To create a monitoring user for Postgres versions >= 9.6 and <10, run:  
+  ```SQL
+  create user monitoring with password '<PASSWORD>';
+  grant SELECT ON pg_stat_database to monitoring;
+  ```
     
-```sql
+
-create user monitoring with password '<PASSWORD>';
+### Ensure OTEL Collector is running with access to the Postgres server
-grant SELECT ON pg_stat_database to monitoring;
+
-```
+- **Ensure that an OTEL collector is running in your deployment environment**  
+  If needed, please [install an OTEL Collector](https://signoz.io/docs/tutorial/opentelemetry-binary-usage-in-virtual-machine/)  
+  If already installed, ensure that the collector version is v0.88.0 or newer.  
+
+  Also ensure that you can provide config files to the collector and that you can set environment variables and command line flags used for running it.  
+
+- **Ensure that the OTEL collector can access the Postgres server**  
+  In order to collect metrics, the collector must be able to access the Postgres server as a client using the monitoring user.  
+
+  In order to collect logs, the collector must be able to read the Postgres server log file.

pkg/query-service/app/integrations/builtin_integrations/postgres/integration.json

@@ -1,7 +1,7 @@
 {
   "id": "postgres",
   "title": "PostgreSQL",
-  "description": "Monitor postgres using logs and metrics.",
+  "description": "Monitor Postgres with metrics and logs",
   "author": {
     "name": "SigNoz",
     "email": "integrations@signoz.io",
@@ -18,8 +18,12 @@
       "instructions": "file://config/prerequisites.md"
     },
     {
-      "title": "Configure Otel Collector",
+      "title": "Collect Metrics",
-      "instructions": "file://config/configure-otel-collector.md"
+      "instructions": "file://config/collect-metrics.md"
+    },
+    {
+      "title": "Collect Logs",
+      "instructions": "file://config/collect-logs.md"
     }
   ],
   "assets": {
@@ -48,30 +52,48 @@
     }
   },
   "data_collected": {
-    "logs": [],
+    "logs": [
+      {
+        "name": "Process ID",
+        "path": "attributes.pid",
+        "type": "string"
+      }, {
+        "name": "Timestamp",
+        "path": "timestamp",
+        "type": "timestamp"
+      }, {
+        "name": "Severity Text",
+        "path": "severity_text",
+        "type": "string"
+      }, {
+        "name": "Severity Number",
+        "path": "severity_number",
+        "type": "number"
+      }
+    ],
     "metrics": [
       {
         "name": "postgresql.backends",
         "type": "sum",
-        "unit": "1",
+        "unit": "number",
         "description": "The number of backends."
       },
       {
         "name": "postgresql.bgwriter.buffers.allocated",
         "type": "sum",
-        "unit": "{buffers}",
+        "unit": "number",
         "description": "Number of buffers allocated."
       },
       {
         "name": "postgresql.bgwriter.buffers.writes",
         "type": "sum",
-        "unit": "{buffers}",
+        "unit": "number",
         "description": "Number of buffers written."
       },
       {
         "name": "postgresql.bgwriter.checkpoint.count",
         "type": "sum",
-        "unit": "{checkpoints}",
+        "unit": "number",
         "description": "The number of checkpoints performed."
       },
       {
@@ -83,133 +105,133 @@
       {
         "name": "postgresql.bgwriter.maxwritten",
         "type": "sum",
-        "unit": "1",
+        "unit": "number",
         "description": "Number of times the background writer stopped a cleaning scan because it had written too many buffers."
       },
       {
         "name": "postgresql.blocks_read",
         "type": "sum",
-        "unit": "1",
+        "unit": "number",
         "description": "The number of blocks read."
       },
       {
         "name": "postgresql.commits",
         "type": "sum",
-        "unit": "1",
+        "unit": "number",
         "description": "The number of commits."
       },
       {
         "name": "postgresql.connection.max",
         "type": "gauge",
-        "unit": "{connections}",
+        "unit": "number",
         "description": "Configured maximum number of client connections allowed"
       },
       {
         "name": "postgresql.database.count",
         "type": "sum",
-        "unit": "{databases}",
+        "unit": "number",
         "description": "Number of user databases."
       },
       {
         "name": "postgresql.database.locks",
         "type": "gauge",
-        "unit": "{lock}",
+        "unit": "number",
         "description": "The number of database locks."
       },
       {
         "name": "postgresql.db_size",
         "type": "sum",
-        "unit": "By",
+        "unit": "Bytes",
         "description": "The database disk usage."
       },
       {
         "name": "postgresql.deadlocks",
         "type": "sum",
-        "unit": "{deadlock}",
+        "unit": "number",
         "description": "The number of deadlocks."
       },
       {
         "name": "postgresql.index.scans",
         "type": "sum",
-        "unit": "{scans}",
+        "unit": "number",
         "description": "The number of index scans on a table."
       },
       {
         "name": "postgresql.index.size",
         "type": "gauge",
-        "unit": "By",
+        "unit": "Bytes",
         "description": "The size of the index on disk."
       },
       {
         "name": "postgresql.operations",
         "type": "sum",
-        "unit": "1",
+        "unit": "number",
         "description": "The number of db row operations."
       },
       {
         "name": "postgresql.replication.data_delay",
         "type": "gauge",
-        "unit": "By",
+        "unit": "Bytes",
         "description": "The amount of data delayed in replication."
       },
       {
         "name": "postgresql.rollbacks",
         "type": "sum",
-        "unit": "1",
+        "unit": "number",
         "description": "The number of rollbacks."
       },
       {
         "name": "postgresql.rows",
         "type": "sum",
-        "unit": "1",
+        "unit": "number",
         "description": "The number of rows in the database."
       },
       {
         "name": "postgresql.sequential_scans",
         "type": "sum",
-        "unit": "{sequential_scan}",
+        "unit": "number",
         "description": "The number of sequential scans."
       },
       {
         "name": "postgresql.table.count",
         "type": "sum",
-        "unit": "{table}",
+        "unit": "number",
         "description": "Number of user tables in a database."
       },
       {
         "name": "postgresql.table.size",
         "type": "sum",
-        "unit": "By",
+        "unit": "Bytes",
         "description": "Disk space used by a table."
       },
       {
         "name": "postgresql.table.vacuum.count",
         "type": "sum",
-        "unit": "{vacuums}",
+        "unit": "number",
         "description": "Number of times a table has manually been vacuumed."
       },
       {
         "name": "postgresql.temp_files",
         "type": "sum",
-        "unit": "{temp_file}",
+        "unit": "number",
         "description": "The number of temp files."
       },
       {
         "name": "postgresql.wal.age",
         "type": "gauge",
-        "unit": "s",
+        "unit": "seconds",
         "description": "Age of the oldest WAL file."
       },
       {
         "name": "postgresql.wal.delay",
         "type": "gauge",
-        "unit": "s",
+        "unit": "seconds",
         "description": "Time between flushing recent WAL locally and receiving notification that the standby server has completed an operation with it."
       },
       {
         "name": "postgresql.wal.lag",
         "type": "gauge",
-        "unit": "s",
+        "unit": "seconds",
         "description": "Time between flushing recent WAL locally and receiving notification that the standby server has completed an operation with it."
       }
     ]

pkg/query-service/app/integrations/builtin_integrations/postgres/overview.md

@@ -1,3 +1,5 @@
 ### Monitor Postgres with SigNoz
 
-Parse your Postgres logs and collect key metrics.
+Collect key Postgres metrics and view them with an out of the box dashboard.  
+
+Collect and parse Postgres logs to populate timestamp, severity, and other log attributes for better querying and aggregation.