signoz/pkg/query-service/common/query_range.go

package common

import (
	"math"
	"regexp"
	"sort"
	"time"
	"unicode"

	"go.signoz.io/signoz/pkg/query-service/constants"
	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
	"go.signoz.io/signoz/pkg/query-service/querycache"
	"go.signoz.io/signoz/pkg/query-service/utils/labels"
)

func AdjustedMetricTimeRange(start, end, step int64, mq v3.BuilderQuery) (int64, int64) {
	// align the start to the step interval
	start = start - (start % (step * 1000))
	// if the query is a rate query, we adjust the start time by one more step
	// so that we can calculate the rate for the first data point
	hasRunningDiff := false
	for _, fn := range mq.Functions {
		if fn.Name == v3.FunctionNameRunningDiff {
			hasRunningDiff = true
			break
		}
	}
	if (mq.AggregateOperator.IsRateOperator() || mq.TimeAggregation.IsRateOperator()) &&
		mq.Temporality != v3.Delta {
		start -= step * 1000
	}
	if hasRunningDiff {
		start -= step * 1000
	}
	// align the end to the nearest minute
	adjustStep := int64(math.Min(float64(step), 60))
	end = end - (end % (adjustStep * 1000))
	return start, end
}

func PastDayRoundOff() int64 {
	now := time.Now().UnixMilli()
	return int64(math.Floor(float64(now)/float64(time.Hour.Milliseconds()*24))) * time.Hour.Milliseconds() * 24
}

// start and end are in milliseconds
func MinAllowedStepInterval(start, end int64) int64 {
	step := (end - start) / constants.MaxAllowedPointsInTimeSeries / 1000
	if step < 60 {
		return step
	}
	// return the nearest lower multiple of 60
	return step - step%60
}

func GCD(a, b int64) int64 {
	for b != 0 {
		a, b = b, a%b
	}
	return a
}

func LCM(a, b int64) int64 {
	return (a * b) / GCD(a, b)
}

// LCMList computes the LCM of a list of int64 numbers.
func LCMList(nums []int64) int64 {
	if len(nums) == 0 {
		return 1
	}
	result := nums[0]
	for _, num := range nums[1:] {
		result = LCM(result, num)
	}
	return result
}

func NormalizeLabelName(name string) string {
	// See https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels

	// Regular expression to match non-alphanumeric characters except underscores
	reg := regexp.MustCompile(`[^a-zA-Z0-9_]`)

	// Replace all non-alphanumeric characters except underscores with underscores
	normalized := reg.ReplaceAllString(name, "_")

	// If the first character is not a letter or an underscore, prepend an underscore
	if len(normalized) > 0 && !unicode.IsLetter(rune(normalized[0])) && normalized[0] != '_' {
		normalized = "_" + normalized
	}

	return normalized
}

func GetSeriesFromCachedData(data []querycache.CachedSeriesData, start, end int64) []*v3.Series {
	series := make(map[uint64]*v3.Series)

	for _, cachedData := range data {
		for _, data := range cachedData.Data {
			h := labels.FromMap(data.Labels).Hash()

			if _, ok := series[h]; !ok {
				series[h] = &v3.Series{
					Labels:      data.Labels,
					LabelsArray: data.LabelsArray,
					Points:      make([]v3.Point, 0),
				}
			}

			for _, point := range data.Points {
				if point.Timestamp >= start && point.Timestamp <= end {
					series[h].Points = append(series[h].Points, point)
				}
			}
		}
	}

	newSeries := make([]*v3.Series, 0, len(series))
	for _, s := range series {
		s.SortPoints()
		s.RemoveDuplicatePoints()
		newSeries = append(newSeries, s)
	}
	return newSeries
}

// It is different from GetSeriesFromCachedData because doesn't remove a point if it is >= (start - (start % step*1000))
func GetSeriesFromCachedDataV2(data []querycache.CachedSeriesData, start, end, step int64) []*v3.Series {
	series := make(map[uint64]*v3.Series)

	for _, cachedData := range data {
		for _, data := range cachedData.Data {
			h := labels.FromMap(data.Labels).Hash()

			if _, ok := series[h]; !ok {
				series[h] = &v3.Series{
					Labels:      data.Labels,
					LabelsArray: data.LabelsArray,
					Points:      make([]v3.Point, 0),
				}
			}

			for _, point := range data.Points {
				if point.Timestamp >= (start-(start%(step*1000))) && point.Timestamp <= end {
					series[h].Points = append(series[h].Points, point)
				}
			}
		}
	}

	newSeries := make([]*v3.Series, 0, len(series))
	for _, s := range series {
		s.SortPoints()
		s.RemoveDuplicatePoints()
		newSeries = append(newSeries, s)
	}
	return newSeries
}

// filter series points for storing in cache
func FilterSeriesPoints(seriesList []*v3.Series, missStart, missEnd int64, stepInterval int64) ([]*v3.Series, int64, int64) {
	filteredSeries := make([]*v3.Series, 0)
	startTime := missStart
	endTime := missEnd

	stepMs := stepInterval * 1000

	// return empty series if the interval is not complete
	if missStart+stepMs > missEnd {
		return []*v3.Series{}, missStart, missEnd
	}

	// if the end time is not a complete aggregation window, then we will have to adjust the end time
	// to the previous complete aggregation window end
	endCompleteWindow := missEnd%stepMs == 0
	if !endCompleteWindow {
		endTime = missEnd - (missEnd % stepMs)
	}

	// if the start time is not a complete aggregation window, then we will have to adjust the start time
	// to the next complete aggregation window
	if missStart%stepMs != 0 {
		startTime = missStart + stepMs - (missStart % stepMs)
	}

	for _, series := range seriesList {
		// if data for the series is empty, then we will add it to the cache
		if len(series.Points) == 0 {
			filteredSeries = append(filteredSeries, &v3.Series{
				Labels:      series.Labels,
				LabelsArray: series.LabelsArray,
				Points:      make([]v3.Point, 0),
			})
			continue
		}

		// Sort the points based on timestamp
		sort.Slice(series.Points, func(i, j int) bool {
			return series.Points[i].Timestamp < series.Points[j].Timestamp
		})

		points := make([]v3.Point, len(series.Points))
		copy(points, series.Points)

		// Filter the first point that is not a complete aggregation window
		if series.Points[0].Timestamp < missStart {
			// Remove the first point
			points = points[1:]
		}

		// filter the last point if it is not a complete aggregation window
		// adding or condition to handle the end time is equal to a complete window end https://github.com/SigNoz/signoz/pull/7212#issuecomment-2703677190
		if (!endCompleteWindow && series.Points[len(series.Points)-1].Timestamp == missEnd-(missEnd%stepMs)) ||
			(endCompleteWindow && series.Points[len(series.Points)-1].Timestamp == missEnd) {
			// Remove the last point
			points = points[:len(points)-1]
		}

		// making sure that empty range doesn't enter the cache
		if len(points) > 0 {
			filteredSeries = append(filteredSeries, &v3.Series{
				Labels:      series.Labels,
				LabelsArray: series.LabelsArray,
				Points:      points,
			})
		}
	}

	return filteredSeries, startTime, endTime
}