GitHub-Proxy/signoz
1
0
Fork 0
mirror of https://git.mirrors.martin98.com/https://github.com/SigNoz/signoz synced 2025-08-11 01:19:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: enterprise edition (#1575)

Browse Source 
* feat: added license manager and feature flags
* feat: completed org domain api
* chore: checking in saml auth handler code
* feat: added signup with sso
* feat: added login support for admins
* feat: added pem support for certificate
* ci(build-workflow): 👷 include EE query-service
* fix: 🐛 update package name
* chore(ee): 🔧 LD_FLAGS related changes

Signed-off-by: Prashant Shahi <prashant@signoz.io>
Co-authored-by: Prashant Shahi <prashant@signoz.io>
Co-authored-by: nityanandagohain <nityanandagohain@gmail.com>
This commit is contained in:
Amol Umbark 2022-10-06 20:13:30 +05:30 committed by GitHub
parent 106033c296
commit 9c4521b34a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
89 changed files with 4163 additions and 675 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.dockerignore Normal file
View File

@ -0,0 +1,6 @@
.git
.github
.vscode
README.md
deploy
sample-apps

12
.github/workflows/build.yaml vendored
View File

@ -32,7 +32,17 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Build query-service image
- name: Build query-service image
shell: bash
run: |
make build-query-service-amd64
build-ee-query-service:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Build EE query-service image
shell: bash
run: |
make build-ee-query-service-amd64

7
.gitignore vendored
View File

@ -1,3 +1,4 @@
node_modules
yarn.lock
package.json
@ -43,8 +44,12 @@ pkg/query-service/signoz.db
pkg/query-service/tests/test-deploy/data/
ee/query-service/signoz.db
ee/query-service/tests/test-deploy/data/
# local data
*.db
/deploy/docker/clickhouse-setup/data/
/deploy/docker-swarm/clickhouse-setup/data/
bin/

54
Makefile
View File

@ -7,10 +7,12 @@ BUILD_VERSION ?= $(shell git describe --always --tags)
BUILD_HASH ?= $(shell git rev-parse --short HEAD)
BUILD_TIME ?= $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
BUILD_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
DEV_LICENSE_SIGNOZ_IO ?= https://staging-license.signoz.io/api/v1
# Internal variables or constants.
FRONTEND_DIRECTORY ?= frontend
QUERY_SERVICE_DIRECTORY ?= pkg/query-service
EE_QUERY_SERVICE_DIRECTORY ?= ee/query-service
STANDALONE_DIRECTORY ?= deploy/docker/clickhouse-setup
SWARM_DIRECTORY ?= deploy/docker-swarm/clickhouse-setup
LOCAL_GOOS ?= $(shell go env GOOS)
@ -21,15 +23,18 @@ DOCKER_TAG ?= latest
FRONTEND_DOCKER_IMAGE ?= frontend
QUERY_SERVICE_DOCKER_IMAGE ?= query-service
DEV_BUILD ?= ""
# Build-time Go variables
PACKAGE?=go.signoz.io/query-service
buildVersion=${PACKAGE}/version.buildVersion
buildHash=${PACKAGE}/version.buildHash
buildTime=${PACKAGE}/version.buildTime
gitBranch=${PACKAGE}/version.gitBranch
PACKAGE?=go.signoz.io/signoz
buildVersion=${PACKAGE}/pkg/query-service/version.buildVersion
buildHash=${PACKAGE}/pkg/query-service/version.buildHash
buildTime=${PACKAGE}/pkg/query-service/version.buildTime
gitBranch=${PACKAGE}/pkg/query-service/version.gitBranch
licenseSignozIo=${PACKAGE}/ee/query-service/constants.LicenseSignozIo
LD_FLAGS="-X ${buildHash}=${BUILD_HASH} -X ${buildTime}=${BUILD_TIME} -X ${buildVersion}=${BUILD_VERSION} -X ${gitBranch}=${BUILD_BRANCH}"
LD_FLAGS=-X ${buildHash}=${BUILD_HASH} -X ${buildTime}=${BUILD_TIME} -X ${buildVersion}=${BUILD_VERSION} -X ${gitBranch}=${BUILD_BRANCH}
DEV_LD_FLAGS=-X ${licenseSignozIo}=${DEV_LICENSE_SIGNOZ_IO}
all: build-push-frontend build-push-query-service
# Steps to build and push docker image of frontend
@ -40,7 +45,7 @@ build-frontend-amd64:
@echo "--> Building frontend docker image for amd64"
@echo "------------------"
@cd $(FRONTEND_DIRECTORY) && \
docker build -f Dockerfile --no-cache -t $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) \
docker build --file Dockerfile --no-cache -t $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) \
--build-arg TARGETPLATFORM="linux/amd64" .
# Step to build and push docker image of frontend(used in push pipeline)
@ -59,20 +64,43 @@ build-query-service-amd64:
@echo "------------------"
@echo "--> Building query-service docker image for amd64"
@echo "------------------"
@cd $(QUERY_SERVICE_DIRECTORY) && \
docker build -f Dockerfile --no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
--build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS=$(LD_FLAGS) .
@docker build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile \
--no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
--build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="$(LD_FLAGS)" .
# Step to build and push docker image of query in amd64 and arm64 (used in push pipeline)
build-push-query-service:
@echo "------------------"
@echo "--> Building and pushing query-service docker image"
@echo "------------------"
@cd $(QUERY_SERVICE_DIRECTORY) && \
docker buildx build --file Dockerfile --progress plane --no-cache \
--push --platform linux/arm64,linux/amd64 --build-arg LD_FLAGS=$(LD_FLAGS) \
@docker buildx build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile --progress plane --no-cache \
--push --platform linux/arm64,linux/amd64 --build-arg LD_FLAGS="$(LD_FLAGS)" \
--tag $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) .
# Step to build EE docker image of query service in amd64 (used in build pipeline)
build-ee-query-service-amd64:
@echo "------------------"
@echo "--> Building query-service docker image for amd64"
@echo "------------------"
@if [ $(DEV_BUILD) != "" ]; then \
docker build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \
--no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
--build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="${LD_FLAGS} ${DEV_LD_FLAGS}" .; \
else \
docker build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \
--no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
--build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="$(LD_FLAGS)" .; \
fi
# Step to build and push EE docker image of query in amd64 and arm64 (used in push pipeline)
build-push-ee-query-service:
@echo "------------------"
@echo "--> Building and pushing query-service docker image"
@echo "------------------"
@docker buildx build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \
--progress plane --no-cache --push --platform linux/arm64,linux/amd64 \
--build-arg LD_FLAGS="$(LD_FLAGS)" --tag $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) .
dev-setup:
mkdir -p /var/lib/signoz
sqlite3 /var/lib/signoz/signoz.db "VACUUM";

4
ee/query-service/.dockerignore Normal file
View File

@ -0,0 +1,4 @@
.vscode
README.md
signoz.db
bin

48
ee/query-service/Dockerfile Normal file
View File

@ -0,0 +1,48 @@
FROM golang:1.17-buster AS builder
# LD_FLAGS is passed as argument from Makefile. It will be empty, if no argument passed
ARG LD_FLAGS
ARG TARGETPLATFORM
ENV CGO_ENABLED=1
ENV GOPATH=/go
RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2)
# Prepare and enter src directory
WORKDIR /go/src/github.com/signoz/signoz
# Add the sources and proceed with build
ADD . .
RUN cd ee/query-service \
&& go build -tags timetzdata -a -o ./bin/query-service \
-ldflags "-linkmode external -extldflags '-static' -s -w $LD_FLAGS" \
&& chmod +x ./bin/query-service
# use a minimal alpine image
FROM alpine:3.7
# Add Maintainer Info
LABEL maintainer="signoz"
# add ca-certificates in case you need them
RUN apk update && apk add ca-certificates && rm -rf /var/cache/apk/*
# set working directory
WORKDIR /root
# copy the binary from builder
COPY --from=builder /go/src/github.com/signoz/signoz/ee/query-service/bin/query-service .
# copy prometheus YAML config
COPY pkg/query-service/config/prometheus.yml /root/config/prometheus.yml
# run the binary
ENTRYPOINT ["./query-service"]
CMD ["-config", "../config/prometheus.yml"]
# CMD ["./query-service -config /root/config/prometheus.yml"]
EXPOSE 8080

124
ee/query-service/app/api/api.go Normal file
View File

@ -0,0 +1,124 @@
package api
import (
"net/http"
"github.com/gorilla/mux"
"go.signoz.io/signoz/ee/query-service/dao"
"go.signoz.io/signoz/ee/query-service/interfaces"
"go.signoz.io/signoz/ee/query-service/license"
baseapp "go.signoz.io/signoz/pkg/query-service/app"
baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
rules "go.signoz.io/signoz/pkg/query-service/rules"
"go.signoz.io/signoz/pkg/query-service/version"
)
type APIHandlerOptions struct {
DataConnector interfaces.DataConnector
AppDao dao.ModelDao
RulesManager *rules.Manager
FeatureFlags baseint.FeatureLookup
LicenseManager *license.Manager
}
type APIHandler struct {
opts APIHandlerOptions
baseapp.APIHandler
}
// NewAPIHandler returns an APIHandler
func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) {
baseHandler, err := baseapp.NewAPIHandler(baseapp.APIHandlerOpts{
Reader: opts.DataConnector,
AppDao: opts.AppDao,
RuleManager: opts.RulesManager,
FeatureFlags: opts.FeatureFlags})
if err != nil {
return nil, err
}
ah := &APIHandler{
opts: opts,
APIHandler: *baseHandler,
}
return ah, nil
}
func (ah *APIHandler) FF() baseint.FeatureLookup {
return ah.opts.FeatureFlags
}
func (ah *APIHandler) RM() *rules.Manager {
return ah.opts.RulesManager
}
func (ah *APIHandler) LM() *license.Manager {
return ah.opts.LicenseManager
}
func (ah *APIHandler) AppDao() dao.ModelDao {
return ah.opts.AppDao
}
func (ah *APIHandler) CheckFeature(f string) bool {
err := ah.FF().CheckFeature(f)
return err == nil
}
// RegisterRoutes registers routes for this handler on the given router
func (ah *APIHandler) RegisterRoutes(router *mux.Router) {
// note: add ee override methods first
// routes available only in ee version
router.HandleFunc("/api/v1/licenses",
baseapp.AdminAccess(ah.listLicenses)).
Methods(http.MethodGet)
router.HandleFunc("/api/v1/licenses",
baseapp.AdminAccess(ah.applyLicense)).
Methods(http.MethodPost)
router.HandleFunc("/api/v1/featureFlags",
baseapp.OpenAccess(ah.getFeatureFlags)).
Methods(http.MethodGet)
router.HandleFunc("/api/v1/loginPrecheck",
baseapp.OpenAccess(ah.precheckLogin)).
Methods(http.MethodGet)
// paid plans specific routes
router.HandleFunc("/api/v1/complete/saml",
baseapp.OpenAccess(ah.receiveSAML)).
Methods(http.MethodPost)
router.HandleFunc("/api/v1/orgs/{orgId}/domains",
baseapp.AdminAccess(ah.listDomainsByOrg)).
Methods(http.MethodGet)
router.HandleFunc("/api/v1/domains",
baseapp.AdminAccess(ah.postDomain)).
Methods(http.MethodPost)
router.HandleFunc("/api/v1/domains/{id}",
baseapp.AdminAccess(ah.putDomain)).
Methods(http.MethodPut)
router.HandleFunc("/api/v1/domains/{id}",
baseapp.AdminAccess(ah.deleteDomain)).
Methods(http.MethodDelete)
// base overrides
router.HandleFunc("/api/v1/version", baseapp.OpenAccess(ah.getVersion)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/invite/{token}", baseapp.OpenAccess(ah.getInvite)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/register", baseapp.OpenAccess(ah.registerUser)).Methods(http.MethodPost)
router.HandleFunc("/api/v1/login", baseapp.OpenAccess(ah.loginUser)).Methods(http.MethodPost)
ah.APIHandler.RegisterRoutes(router)
}
func (ah *APIHandler) getVersion(w http.ResponseWriter, r *http.Request) {
version := version.GetVersion()
ah.WriteJSON(w, r, map[string]string{"version": version, "ee": "Y"})
}

297
ee/query-service/app/api/auth.go Normal file
View File

@ -0,0 +1,297 @@
package api
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/google/uuid"
"github.com/gorilla/mux"
"go.signoz.io/signoz/ee/query-service/constants"
"go.signoz.io/signoz/ee/query-service/model"
"go.signoz.io/signoz/pkg/query-service/auth"
baseauth "go.signoz.io/signoz/pkg/query-service/auth"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)
func parseRequest(r *http.Request, req interface{}) error {
defer r.Body.Close()
requestBody, err := ioutil.ReadAll(r.Body)
if err != nil {
return err
}
err = json.Unmarshal(requestBody, &req)
return err
}
// loginUser overrides base handler and considers SSO case.
func (ah *APIHandler) loginUser(w http.ResponseWriter, r *http.Request) {
req := basemodel.LoginRequest{}
err := parseRequest(r, &req)
if err != nil {
RespondError(w, model.BadRequest(err), nil)
return
}
ctx := context.Background()
if req.Email != "" && ah.CheckFeature(model.SSO) {
var apierr basemodel.BaseApiError
_, apierr = ah.AppDao().CanUsePassword(ctx, req.Email)
if apierr != nil && !apierr.IsNil() {
RespondError(w, apierr, nil)
}
}
// if all looks good, call auth
resp, err := auth.Login(ctx, &req)
if ah.HandleError(w, err, http.StatusUnauthorized) {
return
}
ah.WriteJSON(w, r, resp)
}
// registerUser registers a user and responds with a precheck
// so the front-end can decide the login method
func (ah *APIHandler) registerUser(w http.ResponseWriter, r *http.Request) {
if !ah.CheckFeature(model.SSO) {
ah.APIHandler.Register(w, r)
return
}
ctx := context.Background()
var req *baseauth.RegisterRequest
defer r.Body.Close()
requestBody, err := ioutil.ReadAll(r.Body)
if err != nil {
zap.S().Errorf("received no input in api\n", err)
RespondError(w, model.BadRequest(err), nil)
return
}
err = json.Unmarshal(requestBody, &req)
if err != nil {
zap.S().Errorf("received invalid user registration request", zap.Error(err))
RespondError(w, model.BadRequest(fmt.Errorf("failed to register user")), nil)
return
}
// get invite object
invite, err := baseauth.ValidateInvite(ctx, req)
if err != nil || invite == nil {
zap.S().Errorf("failed to validate invite token", err)
RespondError(w, model.BadRequest(basemodel.ErrSignupFailed{}), nil)
}
// get auth domain from email domain
domain, apierr := ah.AppDao().GetDomainByEmail(ctx, invite.Email)
if apierr != nil {
zap.S().Errorf("failed to get domain from email", apierr)
RespondError(w, model.InternalError(basemodel.ErrSignupFailed{}), nil)
}
precheckResp := &model.PrecheckResponse{
SSO: false,
IsUser: false,
}
if domain != nil && domain.SsoEnabled {
// so is enabled, create user and respond precheck data
user, apierr := baseauth.RegisterInvitedUser(ctx, req, true)
if apierr != nil {
RespondError(w, apierr, nil)
return
}
var precheckError basemodel.BaseApiError
precheckResp, precheckError = ah.AppDao().PrecheckLogin(ctx, user.Email, req.SourceUrl)
if precheckError != nil {
RespondError(w, precheckError, precheckResp)
}
} else {
// no-sso, validate password
if err := auth.ValidatePassword(req.Password); err != nil {
RespondError(w, model.InternalError(fmt.Errorf("password is not in a valid format")), nil)
return
}
_, registerError := baseauth.Register(ctx, req)
if !registerError.IsNil() {
RespondError(w, apierr, nil)
return
}
precheckResp.IsUser = true
}
ah.Respond(w, precheckResp)
}
// getInvite returns the invite object details for the given invite token. We do not need to
// protect this API because invite token itself is meant to be private.
func (ah *APIHandler) getInvite(w http.ResponseWriter, r *http.Request) {
token := mux.Vars(r)["token"]
sourceUrl := r.URL.Query().Get("ref")
ctx := context.Background()
inviteObject, err := baseauth.GetInvite(context.Background(), token)
if err != nil {
RespondError(w, model.BadRequest(err), nil)
return
}
resp := model.GettableInvitation{
InvitationResponseObject: inviteObject,
}
precheck, apierr := ah.AppDao().PrecheckLogin(ctx, inviteObject.Email, sourceUrl)
resp.Precheck = precheck
if apierr != nil {
RespondError(w, apierr, resp)
}
ah.WriteJSON(w, r, resp)
}
// PrecheckLogin enables browser login page to display appropriate
// login methods
func (ah *APIHandler) precheckLogin(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
email := r.URL.Query().Get("email")
sourceUrl := r.URL.Query().Get("ref")
resp, apierr := ah.AppDao().PrecheckLogin(ctx, email, sourceUrl)
if apierr != nil {
RespondError(w, apierr, resp)
}
ah.Respond(w, resp)
}
func (ah *APIHandler) receiveSAML(w http.ResponseWriter, r *http.Request) {
// this is the source url that initiated the login request
redirectUri := constants.GetDefaultSiteURL()
ctx := context.Background()
var apierr basemodel.BaseApiError
redirectOnError := func() {
ssoError := []byte("Login failed. Please contact your system administrator")
dst := make([]byte, base64.StdEncoding.EncodedLen(len(ssoError)))
base64.StdEncoding.Encode(dst, ssoError)
http.Redirect(w, r, fmt.Sprintf("%s?ssoerror=%s", redirectUri, string(dst)), http.StatusMovedPermanently)
}
if !ah.CheckFeature(model.SSO) {
zap.S().Errorf("[ReceiveSAML] sso requested but feature unavailable %s in org domain %s", model.SSO)
http.Redirect(w, r, fmt.Sprintf("%s?ssoerror=%s", redirectUri, "feature unavailable, please upgrade your billing plan to access this feature"), http.StatusMovedPermanently)
return
}
err := r.ParseForm()
if err != nil {
zap.S().Errorf("[ReceiveSAML] failed to process response - invalid response from IDP", err, r)
redirectOnError()
return
}
// the relay state is sent when a login request is submitted to
// Idp.
relayState := r.FormValue("RelayState")
zap.S().Debug("[ReceiveML] relay state", zap.String("relayState", relayState))
parsedState, err := url.Parse(relayState)
if err != nil || relayState == "" {
zap.S().Errorf("[ReceiveSAML] failed to process response - invalid response from IDP", err, r)
redirectOnError()
return
}
// upgrade redirect url from the relay state for better accuracy
redirectUri = fmt.Sprintf("%s://%s%s", parsedState.Scheme, parsedState.Host, "/login")
// derive domain id from relay state now
var domainIdStr string
for k, v := range parsedState.Query() {
if k == "domainId" && len(v) > 0 {
domainIdStr = strings.Replace(v[0], ":", "-", -1)
}
}
domainId, err := uuid.Parse(domainIdStr)
if err != nil {
zap.S().Errorf("[ReceiveSAML] failed to process request- failed to parse domain id ifrom relay", zap.Error(err))
redirectOnError()
return
}
domain, apierr := ah.AppDao().GetDomain(ctx, domainId)
if (apierr != nil) || domain == nil {
zap.S().Errorf("[ReceiveSAML] failed to process request- invalid domain", domainIdStr, zap.Error(apierr))
redirectOnError()
return
}
sp, err := domain.PrepareSamlRequest(parsedState)
if err != nil {
zap.S().Errorf("[ReceiveSAML] failed to prepare saml request for domain (%s): %v", domainId, err)
redirectOnError()
return
}
assertionInfo, err := sp.RetrieveAssertionInfo(r.FormValue("SAMLResponse"))
if err != nil {
zap.S().Errorf("[ReceiveSAML] failed to retrieve assertion info from saml response for organization (%s): %v", domainId, err)
redirectOnError()
return
}
if assertionInfo.WarningInfo.InvalidTime {
zap.S().Errorf("[ReceiveSAML] expired saml response for organization (%s): %v", domainId, err)
redirectOnError()
return
}
email := assertionInfo.NameID
// user email found, now start preparing jwt response
userPayload, baseapierr := ah.AppDao().GetUserByEmail(ctx, email)
if baseapierr != nil {
zap.S().Errorf("[ReceiveSAML] failed to find or register a new user for email %s and org %s", email, domainId, zap.Error(baseapierr.Err))
redirectOnError()
return
}
tokenStore, err := baseauth.GenerateJWTForUser(&userPayload.User)
if err != nil {
zap.S().Errorf("[ReceiveSAML] failed to generate access token for email %s and org %s", email, domainId, zap.Error(err))
redirectOnError()
return
}
userID := userPayload.User.Id
nextPage := fmt.Sprintf("%s?jwt=%s&usr=%s&refreshjwt=%s",
redirectUri,
tokenStore.AccessJwt,
userID,
tokenStore.RefreshJwt)
http.Redirect(w, r, nextPage, http.StatusMovedPermanently)
}

90
ee/query-service/app/api/domains.go Normal file
View File

@ -0,0 +1,90 @@
package api
import (
"context"
"encoding/json"
"fmt"
"net/http"
"github.com/google/uuid"
"github.com/gorilla/mux"
"go.signoz.io/signoz/ee/query-service/model"
)
func (ah *APIHandler) listDomainsByOrg(w http.ResponseWriter, r *http.Request) {
orgId := mux.Vars(r)["orgId"]
domains, apierr := ah.AppDao().ListDomains(context.Background(), orgId)
if apierr != nil {
RespondError(w, apierr, domains)
return
}
ah.Respond(w, domains)
}
func (ah *APIHandler) postDomain(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
req := model.OrgDomain{}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
RespondError(w, model.BadRequest(err), nil)
return
}
if err := req.ValidNew(); err != nil {
RespondError(w, model.BadRequest(err), nil)
return
}
if apierr := ah.AppDao().CreateDomain(ctx, &req); apierr != nil {
RespondError(w, apierr, nil)
return
}
ah.Respond(w, &req)
}
func (ah *APIHandler) putDomain(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
domainIdStr := mux.Vars(r)["id"]
domainId, err := uuid.Parse(domainIdStr)
if err != nil {
RespondError(w, model.BadRequest(err), nil)
return
}
req := model.OrgDomain{Id: domainId}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
RespondError(w, model.BadRequest(err), nil)
return
}
req.Id = domainId
if err := req.Valid(nil); err != nil {
RespondError(w, model.BadRequest(err), nil)
}
if apierr := ah.AppDao().UpdateDomain(ctx, &req); apierr != nil {
RespondError(w, apierr, nil)
return
}
ah.Respond(w, &req)
}
func (ah *APIHandler) deleteDomain(w http.ResponseWriter, r *http.Request) {
domainIdStr := mux.Vars(r)["id"]
domainId, err := uuid.Parse(domainIdStr)
if err != nil {
RespondError(w, model.BadRequest(fmt.Errorf("invalid domain id")), nil)
return
}
apierr := ah.AppDao().DeleteDomain(context.Background(), domainId)
if apierr != nil {
RespondError(w, apierr, nil)
return
}
ah.Respond(w, nil)
}

10
ee/query-service/app/api/featureFlags.go Normal file
View File

@ -0,0 +1,10 @@
package api
import (
"net/http"
)
func (ah *APIHandler) getFeatureFlags(w http.ResponseWriter, r *http.Request) {
featureSet := ah.FF().GetFeatureFlags()
ah.Respond(w, featureSet)
}

40
ee/query-service/app/api/license.go Normal file
View File

@ -0,0 +1,40 @@
package api
import (
"context"
"encoding/json"
"fmt"
"go.signoz.io/signoz/ee/query-service/model"
"net/http"
)
func (ah *APIHandler) listLicenses(w http.ResponseWriter, r *http.Request) {
licenses, apiError := ah.LM().GetLicenses(context.Background())
if apiError != nil {
RespondError(w, apiError, nil)
}
ah.Respond(w, licenses)
}
func (ah *APIHandler) applyLicense(w http.ResponseWriter, r *http.Request) {
ctx := context.Background()
var l model.License
if err := json.NewDecoder(r.Body).Decode(&l); err != nil {
RespondError(w, model.BadRequest(err), nil)
return
}
if l.Key == "" {
RespondError(w, model.BadRequest(fmt.Errorf("license key is required")), nil)
return
}
license, apiError := ah.LM().Activate(ctx, l.Key)
if apiError != nil {
RespondError(w, apiError, nil)
return
}
ah.Respond(w, license)
}

12
ee/query-service/app/api/response.go Normal file
View File

@ -0,0 +1,12 @@
package api
import (
"net/http"
baseapp "go.signoz.io/signoz/pkg/query-service/app"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
func RespondError(w http.ResponseWriter, apiErr basemodel.BaseApiError, data interface{}) {
baseapp.RespondError(w, apiErr, data)
}

28
ee/query-service/app/db/reader.go Normal file
View File

@ -0,0 +1,28 @@
package db
import (
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/jmoiron/sqlx"
basechr "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader"
)
type ClickhouseReader struct {
conn clickhouse.Conn
appdb *sqlx.DB
*basechr.ClickHouseReader
}
func NewDataConnector(localDB *sqlx.DB, promConfigPath string) *ClickhouseReader {
ch := basechr.NewReader(localDB, promConfigPath)
return &ClickhouseReader{
conn: ch.GetConn(),
appdb: localDB,
ClickHouseReader: ch,
}
}
func (r *ClickhouseReader) Start(readerReady chan bool) {
r.ClickHouseReader.Start(readerReady)
}

442
ee/query-service/app/server.go Normal file
View File

@ -0,0 +1,442 @@
package app
import (
"context"
"fmt"
"net"
"net/http"
_ "net/http/pprof" // http profiler
"os"
"time"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/jmoiron/sqlx"
"github.com/rs/cors"
"github.com/soheilhy/cmux"
"go.signoz.io/signoz/ee/query-service/app/api"
"go.signoz.io/signoz/ee/query-service/app/db"
"go.signoz.io/signoz/ee/query-service/dao"
"go.signoz.io/signoz/ee/query-service/interfaces"
licensepkg "go.signoz.io/signoz/ee/query-service/license"
"go.signoz.io/signoz/ee/query-service/usage"
"go.signoz.io/signoz/pkg/query-service/app/dashboards"
baseconst "go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/healthcheck"
basealm "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
pqle "go.signoz.io/signoz/pkg/query-service/pqlEngine"
rules "go.signoz.io/signoz/pkg/query-service/rules"
"go.signoz.io/signoz/pkg/query-service/telemetry"
"go.signoz.io/signoz/pkg/query-service/utils"
"go.uber.org/zap"
)
type ServerOptions struct {
PromConfigPath string
HTTPHostPort string
PrivateHostPort string
// alert specific params
DisableRules bool
RuleRepoURL string
}
// Server runs HTTP api service
type Server struct {
serverOptions *ServerOptions
conn net.Listener
ruleManager *rules.Manager
separatePorts bool
// public http router
httpConn net.Listener
httpServer *http.Server
// private http
privateConn net.Listener
privateHTTP *http.Server
// feature flags
featureLookup baseint.FeatureLookup
unavailableChannel chan healthcheck.Status
}
// HealthCheckStatus returns health check status channel a client can subscribe to
func (s Server) HealthCheckStatus() chan healthcheck.Status {
return s.unavailableChannel
}
// NewServer creates and initializes Server
func NewServer(serverOptions *ServerOptions) (*Server, error) {
modelDao, err := dao.InitDao("sqlite", baseconst.RELATIONAL_DATASOURCE_PATH)
if err != nil {
return nil, err
}
localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH)
if err != nil {
return nil, err
}
localDB.SetMaxOpenConns(10)
// initiate license manager
lm, err := licensepkg.StartManager("sqlite", localDB)
if err != nil {
return nil, err
}
// set license manager as feature flag provider in dao
modelDao.SetFlagProvider(lm)
readerReady := make(chan bool)
var reader interfaces.DataConnector
storage := os.Getenv("STORAGE")
if storage == "clickhouse" {
zap.S().Info("Using ClickHouse as datastore ...")
qb := db.NewDataConnector(localDB, serverOptions.PromConfigPath)
go qb.Start(readerReady)
reader = qb
} else {
return nil, fmt.Errorf("Storage type: %s is not supported in query service", storage)
}
<-readerReady
rm, err := makeRulesManager(serverOptions.PromConfigPath,
baseconst.GetAlertManagerApiPrefix(),
serverOptions.RuleRepoURL,
localDB,
reader,
serverOptions.DisableRules)
if err != nil {
return nil, err
}
// start the usagemanager
usageManager, err := usage.New("sqlite", localDB, lm.GetRepo(), reader.GetConn())
if err != nil {
return nil, err
}
err = usageManager.Start()
if err != nil {
return nil, err
}
telemetry.GetInstance().SetReader(reader)
apiOpts := api.APIHandlerOptions{
DataConnector: reader,
AppDao: modelDao,
RulesManager: rm,
FeatureFlags: lm,
LicenseManager: lm,
}
apiHandler, err := api.NewAPIHandler(apiOpts)
if err != nil {
return nil, err
}
s := &Server{
// logger: logger,
// tracer: tracer,
ruleManager: rm,
serverOptions: serverOptions,
unavailableChannel: make(chan healthcheck.Status),
}
httpServer, err := s.createPublicServer(apiHandler)
if err != nil {
return nil, err
}
s.httpServer = httpServer
privateServer, err := s.createPrivateServer(apiHandler)
if err != nil {
return nil, err
}
s.privateHTTP = privateServer
return s, nil
}
func (s *Server) createPrivateServer(apiHandler *api.APIHandler) (*http.Server, error) {
r := mux.NewRouter()
r.Use(setTimeoutMiddleware)
r.Use(s.analyticsMiddleware)
r.Use(loggingMiddlewarePrivate)
apiHandler.RegisterPrivateRoutes(r)
c := cors.New(cors.Options{
//todo(amol): find out a way to add exact domain or
// ip here for alert manager
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "PATCH"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type"},
})
handler := c.Handler(r)
handler = handlers.CompressHandler(handler)
return &http.Server{
Handler: handler,
}, nil
}
func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, error) {
r := mux.NewRouter()
r.Use(setTimeoutMiddleware)
r.Use(s.analyticsMiddleware)
r.Use(loggingMiddleware)
apiHandler.RegisterRoutes(r)
apiHandler.RegisterMetricsRoutes(r)
apiHandler.RegisterLogsRoutes(r)
c := cors.New(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "PATCH", "OPTIONS"},
AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "cache-control"},
})
handler := c.Handler(r)
handler = handlers.CompressHandler(handler)
return &http.Server{
Handler: handler,
}, nil
}
// loggingMiddleware is used for logging public api calls
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
route := mux.CurrentRoute(r)
path, _ := route.GetPathTemplate()
startTime := time.Now()
next.ServeHTTP(w, r)
zap.S().Info(path, "\ttimeTaken: ", time.Now().Sub(startTime))
})
}
// loggingMiddlewarePrivate is used for logging private api calls
// from internal services like alert manager
func loggingMiddlewarePrivate(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
route := mux.CurrentRoute(r)
path, _ := route.GetPathTemplate()
startTime := time.Now()
next.ServeHTTP(w, r)
zap.S().Info(path, "\tprivatePort: true", "\ttimeTaken: ", time.Now().Sub(startTime))
})
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
// WriteHeader(int) is not called if our response implicitly returns 200 OK, so
// we default to that status code.
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
// Flush implements the http.Flush interface.
func (lrw *loggingResponseWriter) Flush() {
lrw.ResponseWriter.(http.Flusher).Flush()
}
func (s *Server) analyticsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
route := mux.CurrentRoute(r)
path, _ := route.GetPathTemplate()
lrw := NewLoggingResponseWriter(w)
next.ServeHTTP(lrw, r)
data := map[string]interface{}{"path": path, "statusCode": lrw.statusCode}
if _, ok := telemetry.IgnoredPaths()[path]; !ok {
telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data)
}
})
}
func setTimeoutMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var cancel context.CancelFunc
// check if route is not excluded
url := r.URL.Path
if _, ok := baseconst.TimeoutExcludedRoutes[url]; !ok {
ctx, cancel = context.WithTimeout(r.Context(), baseconst.ContextTimeout*time.Second)
defer cancel()
}
r = r.WithContext(ctx)
next.ServeHTTP(w, r)
})
}
// initListeners initialises listeners of the server
func (s *Server) initListeners() error {
// listen on public port
var err error
publicHostPort := s.serverOptions.HTTPHostPort
if publicHostPort == "" {
return fmt.Errorf("baseconst.HTTPHostPort is required")
}
s.httpConn, err = net.Listen("tcp", publicHostPort)
if err != nil {
return err
}
zap.S().Info(fmt.Sprintf("Query server started listening on %s...", s.serverOptions.HTTPHostPort))
// listen on private port to support internal services
privateHostPort := s.serverOptions.PrivateHostPort
if privateHostPort == "" {
return fmt.Errorf("baseconst.PrivateHostPort is required")
}
s.privateConn, err = net.Listen("tcp", privateHostPort)
if err != nil {
return err
}
zap.S().Info(fmt.Sprintf("Query server started listening on private port %s...", s.serverOptions.PrivateHostPort))
return nil
}
// Start listening on http and private http port concurrently
func (s *Server) Start() error {
// initiate rule manager first
if !s.serverOptions.DisableRules {
s.ruleManager.Start()
} else {
zap.S().Info("msg: Rules disabled as rules.disable is set to TRUE")
}
err := s.initListeners()
if err != nil {
return err
}
var httpPort int
if port, err := utils.GetPort(s.httpConn.Addr()); err == nil {
httpPort = port
}
go func() {
zap.S().Info("Starting HTTP server", zap.Int("port", httpPort), zap.String("addr", s.serverOptions.HTTPHostPort))
switch err := s.httpServer.Serve(s.httpConn); err {
case nil, http.ErrServerClosed, cmux.ErrListenerClosed:
// normal exit, nothing to do
default:
zap.S().Error("Could not start HTTP server", zap.Error(err))
}
s.unavailableChannel <- healthcheck.Unavailable
}()
go func() {
zap.S().Info("Starting pprof server", zap.String("addr", baseconst.DebugHttpPort))
err = http.ListenAndServe(baseconst.DebugHttpPort, nil)
if err != nil {
zap.S().Error("Could not start pprof server", zap.Error(err))
}
}()
var privatePort int
if port, err := utils.GetPort(s.privateConn.Addr()); err == nil {
privatePort = port
}
fmt.Println("starting private http")
go func() {
zap.S().Info("Starting Private HTTP server", zap.Int("port", privatePort), zap.String("addr", s.serverOptions.PrivateHostPort))
switch err := s.privateHTTP.Serve(s.privateConn); err {
case nil, http.ErrServerClosed, cmux.ErrListenerClosed:
// normal exit, nothing to do
zap.S().Info("private http server closed")
default:
zap.S().Error("Could not start private HTTP server", zap.Error(err))
}
s.unavailableChannel <- healthcheck.Unavailable
}()
return nil
}
func makeRulesManager(
promConfigPath,
alertManagerURL string,
ruleRepoURL string,
db *sqlx.DB,
ch baseint.Reader,
disableRules bool) (*rules.Manager, error) {
// create engine
pqle, err := pqle.FromConfigPath(promConfigPath)
if err != nil {
return nil, fmt.Errorf("failed to create pql engine : %v", err)
}
// notifier opts
notifierOpts := basealm.NotifierOptions{
QueueCapacity: 10000,
Timeout: 1 * time.Second,
AlertManagerURLs: []string{alertManagerURL},
}
// create manager opts
managerOpts := &rules.ManagerOptions{
NotifierOpts: notifierOpts,
Queriers: &rules.Queriers{
PqlEngine: pqle,
Ch: ch.GetConn(),
},
RepoURL: ruleRepoURL,
DBConn: db,
Context: context.Background(),
Logger: nil,
DisableRules: disableRules,
}
// create Manager
manager, err := rules.NewManager(managerOpts)
if err != nil {
return nil, fmt.Errorf("rule manager error: %v", err)
}
zap.S().Info("rules manager is ready")
return manager, nil
}

28
ee/query-service/constants/constants.go Normal file
View File

@ -0,0 +1,28 @@
package constants
import (
"os"
)
const (
DefaultSiteURL = "https://localhost:3301"
)
var LicenseSignozIo = "https://license.signoz.io/api/v1"
func GetOrDefaultEnv(key string, fallback string) string {
v := os.Getenv(key)
if len(v) == 0 {
return fallback
}
return v
}
// constant functions that override env vars
// GetDefaultSiteURL returns default site url, primarily
// used to send saml request and allowing backend to
// handle http redirect
func GetDefaultSiteURL() string {
return GetOrDefaultEnv("SIGNOZ_SITE_URL", DefaultSiteURL)
}

18
ee/query-service/dao/factory.go Normal file
View File

@ -0,0 +1,18 @@
package dao
import (
"fmt"
"go.signoz.io/signoz/ee/query-service/dao/sqlite"
)
func InitDao(engine, path string) (ModelDao, error) {
switch engine {
case "sqlite":
return sqlite.InitDB(path)
default:
return nil, fmt.Errorf("qsdb type: %s is not supported in query service", engine)
}
}

33
ee/query-service/dao/interface.go Normal file
View File

@ -0,0 +1,33 @@
package dao
import (
"context"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"go.signoz.io/signoz/ee/query-service/model"
basedao "go.signoz.io/signoz/pkg/query-service/dao"
baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
type ModelDao interface {
basedao.ModelDao
// SetFlagProvider sets the feature lookup provider
SetFlagProvider(flags baseint.FeatureLookup)
DB() *sqlx.DB
// auth methods
PrecheckLogin(ctx context.Context, email, sourceUrl string) (*model.PrecheckResponse, basemodel.BaseApiError)
CanUsePassword(ctx context.Context, email string) (bool, basemodel.BaseApiError)
// org domain (auth domains) CRUD ops
ListDomains(ctx context.Context, orgId string) ([]model.OrgDomain, basemodel.BaseApiError)
GetDomain(ctx context.Context, id uuid.UUID) (*model.OrgDomain, basemodel.BaseApiError)
CreateDomain(ctx context.Context, d *model.OrgDomain) basemodel.BaseApiError
UpdateDomain(ctx context.Context, domain *model.OrgDomain) basemodel.BaseApiError
DeleteDomain(ctx context.Context, id uuid.UUID) basemodel.BaseApiError
GetDomainByEmail(ctx context.Context, email string) (*model.OrgDomain, basemodel.BaseApiError)
}

112
ee/query-service/dao/sqlite/auth.go Normal file
View File

@ -0,0 +1,112 @@
package sqlite
import (
"context"
"fmt"
"net/url"
"strings"
"go.signoz.io/signoz/ee/query-service/constants"
"go.signoz.io/signoz/ee/query-service/model"
baseconst "go.signoz.io/signoz/pkg/query-service/constants"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)
func (m *modelDao) CanUsePassword(ctx context.Context, email string) (bool, basemodel.BaseApiError) {
domain, apierr := m.GetDomainByEmail(ctx, email)
if apierr != nil {
return false, apierr
}
if domain != nil && domain.SsoEnabled {
// sso is enabled, check if the user has admin role
userPayload, baseapierr := m.GetUserByEmail(ctx, email)
if baseapierr != nil || userPayload == nil {
return false, baseapierr
}
if userPayload.Role != baseconst.AdminGroup {
return false, model.BadRequest(fmt.Errorf("auth method not supported"))
}
}
return true, nil
}
// PrecheckLogin is called when the login or signup page is loaded
// to check sso login is to be prompted
func (m *modelDao) PrecheckLogin(ctx context.Context, email, sourceUrl string) (*model.PrecheckResponse, basemodel.BaseApiError) {
// assume user is valid unless proven otherwise
resp := &model.PrecheckResponse{IsUser: true, CanSelfRegister: false}
// check if email is a valid user
userPayload, baseApiErr := m.GetUserByEmail(ctx, email)
if baseApiErr != nil {
return resp, baseApiErr
}
if userPayload == nil {
resp.IsUser = false
}
ssoAvailable := true
err := m.checkFeature(model.SSO)
if err != nil {
switch err.(type) {
case basemodel.ErrFeatureUnavailable:
// do nothing, just skip sso
ssoAvailable = false
default:
zap.S().Errorf("feature check failed", zap.String("featureKey", model.SSO), zap.Error(err))
return resp, model.BadRequest(err)
}
}
if ssoAvailable {
// find domain from email
orgDomain, apierr := m.GetDomainByEmail(ctx, email)
if apierr != nil {
var emailDomain string
emailComponents := strings.Split(email, "@")
if len(emailComponents) > 0 {
emailDomain = emailComponents[1]
}
zap.S().Errorf("failed to get org domain from email", zap.String("emailDomain", emailDomain), apierr.ToError())
return resp, apierr
}
if orgDomain != nil && orgDomain.SsoEnabled {
// saml is enabled for this domain, lets prepare sso url
if sourceUrl == "" {
sourceUrl = constants.GetDefaultSiteURL()
}
// parse source url that generated the login request
var err error
escapedUrl, _ := url.QueryUnescape(sourceUrl)
siteUrl, err := url.Parse(escapedUrl)
if err != nil {
zap.S().Errorf("failed to parse referer", err)
return resp, model.InternalError(fmt.Errorf("failed to generate login request"))
}
// build Idp URL that will authenticat the user
// the front-end will redirect user to this url
resp.SsoUrl, err = orgDomain.BuildSsoUrl(siteUrl)
if err != nil {
zap.S().Errorf("failed to prepare saml request for domain", zap.String("domain", orgDomain.Name), err)
return resp, model.InternalError(err)
}
// set SSO to true, as the url is generated correctly
resp.SSO = true
}
}
return resp, nil
}

183
ee/query-service/dao/sqlite/domain.go Normal file
View File

@ -0,0 +1,183 @@
package sqlite
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/google/uuid"
"go.signoz.io/signoz/ee/query-service/model"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)
// StoredDomain represents stored database record for org domain
type StoredDomain struct {
Id uuid.UUID `db:"id"`
Name string `db:"name"`
OrgId string `db:"org_id"`
Data string `db:"data"`
CreatedAt int64 `db:"created_at"`
UpdatedAt int64 `db:"updated_at"`
}
// GetDomain returns org domain for a given domain id
func (m *modelDao) GetDomain(ctx context.Context, id uuid.UUID) (*model.OrgDomain, basemodel.BaseApiError) {
stored := StoredDomain{}
err := m.DB().Get(&stored, `SELECT * FROM org_domains WHERE id=$1 LIMIT 1`, id)
if err != nil {
if err == sql.ErrNoRows {
return nil, model.BadRequest(fmt.Errorf("invalid domain id"))
}
return nil, model.InternalError(err)
}
domain := &model.OrgDomain{Id: stored.Id, Name: stored.Name, OrgId: stored.OrgId}
if err := domain.LoadConfig(stored.Data); err != nil {
return domain, model.InternalError(err)
}
return domain, nil
}
// ListDomains gets the list of auth domains by org id
func (m *modelDao) ListDomains(ctx context.Context, orgId string) ([]model.OrgDomain, basemodel.BaseApiError) {
domains := []model.OrgDomain{}
stored := []StoredDomain{}
err := m.DB().SelectContext(ctx, &stored, `SELECT * FROM org_domains WHERE org_id=$1`, orgId)
if err != nil {
if err == sql.ErrNoRows {
return []model.OrgDomain{}, nil
}
return nil, model.InternalError(err)
}
for _, s := range stored {
domain := model.OrgDomain{Id: s.Id, Name: s.Name, OrgId: s.OrgId}
if err := domain.LoadConfig(s.Data); err != nil {
zap.S().Errorf("ListDomains() failed", zap.Error(err))
}
domains = append(domains, domain)
}
return domains, nil
}
// CreateDomain creates a new auth domain
func (m *modelDao) CreateDomain(ctx context.Context, domain *model.OrgDomain) basemodel.BaseApiError {
if domain.Id == uuid.Nil {
domain.Id = uuid.New()
}
if domain.OrgId == "" || domain.Name == "" {
return model.BadRequest(fmt.Errorf("domain creation failed, missing fields: OrgId, Name "))
}
configJson, err := json.Marshal(domain)
if err != nil {
zap.S().Errorf("failed to unmarshal domain config", zap.Error(err))
return model.InternalError(fmt.Errorf("domain creation failed"))
}
_, err = m.DB().ExecContext(ctx,
"INSERT INTO org_domains (id, name, org_id, data, created_at, updated_at) VALUES ($1, $2, $3, $4, $5, $6)",
domain.Id,
domain.Name,
domain.OrgId,
configJson,
time.Now().Unix(),
time.Now().Unix())
if err != nil {
zap.S().Errorf("failed to insert domain in db", zap.Error(err))
return model.InternalError(fmt.Errorf("domain creation failed"))
}
return nil
}
// UpdateDomain updates stored config params for a domain
func (m *modelDao) UpdateDomain(ctx context.Context, domain *model.OrgDomain) basemodel.BaseApiError {
if domain.Id == uuid.Nil {
zap.S().Errorf("domain update failed", zap.Error(fmt.Errorf("OrgDomain.Id is null")))
return model.InternalError(fmt.Errorf("domain update failed"))
}
configJson, err := json.Marshal(domain)
if err != nil {
zap.S().Errorf("domain update failed", zap.Error(err))
return model.InternalError(fmt.Errorf("domain update failed"))
}
_, err = m.DB().ExecContext(ctx,
"UPDATE org_domains SET data = $1, updated_at = $2 WHERE id = $3",
configJson,
time.Now().Unix(),
domain.Id)
if err != nil {
zap.S().Errorf("domain update failed", zap.Error(err))
return model.InternalError(fmt.Errorf("domain update failed"))
}
return nil
}
// DeleteDomain deletes an org domain
func (m *modelDao) DeleteDomain(ctx context.Context, id uuid.UUID) basemodel.BaseApiError {
if id == uuid.Nil {
zap.S().Errorf("domain delete failed", zap.Error(fmt.Errorf("OrgDomain.Id is null")))
return model.InternalError(fmt.Errorf("domain delete failed"))
}
_, err := m.DB().ExecContext(ctx,
"DELETE FROM org_domains WHERE id = $1",
id)
if err != nil {
zap.S().Errorf("domain delete failed", zap.Error(err))
return model.InternalError(fmt.Errorf("domain delete failed"))
}
return nil
}
func (m *modelDao) GetDomainByEmail(ctx context.Context, email string) (*model.OrgDomain, basemodel.BaseApiError) {
if email == "" {
return nil, model.BadRequest(fmt.Errorf("could not find auth domain, missing fields: email "))
}
components := strings.Split(email, "@")
if len(components) < 2 {
return nil, model.BadRequest(fmt.Errorf("invalid email address"))
}
parsedDomain := components[1]
stored := StoredDomain{}
err := m.DB().Get(&stored, `SELECT * FROM org_domains WHERE name=$1 LIMIT 1`, parsedDomain)
if err != nil {
if err == sql.ErrNoRows {
return nil, nil
}
return nil, model.InternalError(err)
}
domain := &model.OrgDomain{Id: stored.Id, Name: stored.Name, OrgId: stored.OrgId}
if err := domain.LoadConfig(stored.Data); err != nil {
return domain, model.InternalError(err)
}
return domain, nil
}

63
ee/query-service/dao/sqlite/modelDao.go Normal file
View File

@ -0,0 +1,63 @@
package sqlite
import (
"fmt"
"github.com/jmoiron/sqlx"
basedao "go.signoz.io/signoz/pkg/query-service/dao"
basedsql "go.signoz.io/signoz/pkg/query-service/dao/sqlite"
baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
)
type modelDao struct {
*basedsql.ModelDaoSqlite
flags baseint.FeatureLookup
}
// SetFlagProvider sets the feature lookup provider
func (m *modelDao) SetFlagProvider(flags baseint.FeatureLookup) {
m.flags = flags
}
// CheckFeature confirms if a feature is available
func (m *modelDao) checkFeature(key string) error {
if m.flags == nil {
return fmt.Errorf("flag provider not set")
}
return m.flags.CheckFeature(key)
}
// InitDB creates and extends base model DB repository
func InitDB(dataSourceName string) (*modelDao, error) {
dao, err := basedsql.InitDB(dataSourceName)
if err != nil {
return nil, err
}
// set package variable so dependent base methods (e.g. AuthCache) will work
basedao.SetDB(dao)
m := &modelDao{ModelDaoSqlite: dao}
table_schema := `
PRAGMA foreign_keys = ON;
CREATE TABLE IF NOT EXISTS org_domains(
id TEXT PRIMARY KEY,
org_id TEXT NOT NULL,
name VARCHAR(50) NOT NULL UNIQUE,
created_at INTEGER NOT NULL,
updated_at INTEGER,
data TEXT NOT NULL,
FOREIGN KEY(org_id) REFERENCES organizations(id)
);`
_, err = m.DB().Exec(table_schema)
if err != nil {
return nil, fmt.Errorf("error in creating tables: %v", err.Error())
}
return m, nil
}
func (m *modelDao) DB() *sqlx.DB {
return m.ModelDaoSqlite.DB()
}

20
ee/query-service/integrations/signozio/response.go Normal file
View File

@ -0,0 +1,20 @@
package signozio
type status string
const (
statusSuccess status = "success"
statusError status = "error"
)
type ActivationResult struct {
Status status `json:"status"`
Data *ActivationResponse `json:"data,omitempty"`
ErrorType string `json:"errorType,omitempty"`
Error string `json:"error,omitempty"`
}
type ActivationResponse struct {
ActivationId string `json:"ActivationId"`
PlanDetails string `json:"PlanDetails"`
}

159
ee/query-service/integrations/signozio/signozio.go Normal file
View File

@ -0,0 +1,159 @@
package signozio
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"github.com/pkg/errors"
"go.signoz.io/signoz/ee/query-service/constants"
"go.signoz.io/signoz/ee/query-service/model"
"go.uber.org/zap"
)
var C *Client
const (
POST = "POST"
APPLICATION_JSON = "application/json"
)
type Client struct {
Prefix string
}
func New() *Client {
return &Client{
Prefix: constants.LicenseSignozIo,
}
}
func init() {
C = New()
}
// ActivateLicense sends key to license.signoz.io and gets activation data
func ActivateLicense(key, siteId string) (*ActivationResponse, *model.ApiError) {
licenseReq := map[string]string{
"key": key,
"siteId": siteId,
}
reqString, _ := json.Marshal(licenseReq)
httpResponse, err := http.Post(C.Prefix+"/licenses/activate", APPLICATION_JSON, bytes.NewBuffer(reqString))
if err != nil {
zap.S().Errorf("failed to connect to license.signoz.io", err)
return nil, model.BadRequest(fmt.Errorf("unable to connect with license.signoz.io, please check your network connection"))
}
httpBody, err := ioutil.ReadAll(httpResponse.Body)
if err != nil {
zap.S().Errorf("failed to read activation response from license.signoz.io", err)
return nil, model.BadRequest(fmt.Errorf("failed to read activation response from license.signoz.io"))
}
defer httpResponse.Body.Close()
// read api request result
result := ActivationResult{}
err = json.Unmarshal(httpBody, &result)
if err != nil {
zap.S().Errorf("failed to marshal activation response from license.signoz.io", err)
return nil, model.InternalError(errors.Wrap(err, "failed to marshal license activation response"))
}
switch httpResponse.StatusCode {
case 200, 201:
return result.Data, nil
case 400, 401:
return nil, model.BadRequest(fmt.Errorf(fmt.Sprintf("failed to activate: %s", result.Error)))
default:
return nil, model.InternalError(fmt.Errorf(fmt.Sprintf("failed to activate: %s", result.Error)))
}
}
// ValidateLicense validates the license key
func ValidateLicense(activationId string) (*ActivationResponse, *model.ApiError) {
validReq := map[string]string{
"activationId": activationId,
}
reqString, _ := json.Marshal(validReq)
response, err := http.Post(C.Prefix+"/licenses/validate", APPLICATION_JSON, bytes.NewBuffer(reqString))
if err != nil {
return nil, model.BadRequest(errors.Wrap(err, "unable to connect with license.signoz.io, please check your network connection"))
}
body, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, model.BadRequest(errors.Wrap(err, "failed to read validation response from license.signoz.io"))
}
defer response.Body.Close()
switch response.StatusCode {
case 200, 201:
a := ActivationResult{}
err = json.Unmarshal(body, &a)
if err != nil {
return nil, model.BadRequest(errors.Wrap(err, "failed to marshal license validation response"))
}
return a.Data, nil
case 400, 401:
return nil, model.BadRequest(errors.Wrap(fmt.Errorf(string(body)),
"bad request error received from license.signoz.io"))
default:
return nil, model.InternalError(errors.Wrap(fmt.Errorf(string(body)),
"internal error received from license.signoz.io"))
}
}
func NewPostRequestWithCtx(ctx context.Context, url string, contentType string, body io.Reader) (*http.Request, error) {
req, err := http.NewRequestWithContext(ctx, POST, url, body)
if err != nil {
return nil, err
}
req.Header.Add("Content-Type", contentType)
return req, err
}
// SendUsage reports the usage of signoz to license server
func SendUsage(ctx context.Context, usage *model.UsagePayload) *model.ApiError {
reqString, _ := json.Marshal(usage)
req, err := NewPostRequestWithCtx(ctx, C.Prefix+"/usage", APPLICATION_JSON, bytes.NewBuffer(reqString))
if err != nil {
return model.BadRequest(errors.Wrap(err, "unable to create http request"))
}
res, err := http.DefaultClient.Do(req)
if err != nil {
return model.BadRequest(errors.Wrap(err, "unable to connect with license.signoz.io, please check your network connection"))
}
body, err := io.ReadAll(res.Body)
if err != nil {
return model.BadRequest(errors.Wrap(err, "failed to read usage response from license.signoz.io"))
}
defer res.Body.Close()
switch res.StatusCode {
case 200, 201:
return nil
case 400, 401:
return model.BadRequest(errors.Wrap(fmt.Errorf(string(body)),
"bad request error received from license.signoz.io"))
default:
return model.InternalError(errors.Wrap(fmt.Errorf(string(body)),
"internal error received from license.signoz.io"))
}
}

12
ee/query-service/interfaces/connector.go Normal file
View File

@ -0,0 +1,12 @@
package interfaces
import (
baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
)
// Connector defines methods for interaction
// with o11y data. for example - clickhouse
type DataConnector interface {
Start(readerReady chan bool)
baseint.Reader
}

127
ee/query-service/license/db.go Normal file
View File

@ -0,0 +1,127 @@
package license
import (
"context"
"fmt"
"time"
"github.com/jmoiron/sqlx"
"go.signoz.io/signoz/ee/query-service/license/sqlite"
"go.signoz.io/signoz/ee/query-service/model"
"go.uber.org/zap"
)
// Repo is license repo. stores license keys in a secured DB
type Repo struct {
db *sqlx.DB
}
// NewLicenseRepo initiates a new license repo
func NewLicenseRepo(db *sqlx.DB) Repo {
return Repo{
db: db,
}
}
func (r *Repo) InitDB(engine string) error {
switch engine {
case "sqlite3", "sqlite":
return sqlite.InitDB(r.db)
default:
return fmt.Errorf("unsupported db")
}
}
func (r *Repo) GetLicenses(ctx context.Context) ([]model.License, error) {
licenses := []model.License{}
query := "SELECT key, activationId, planDetails, validationMessage FROM licenses"
err := r.db.Select(&licenses, query)
if err != nil {
return nil, fmt.Errorf("failed to get licenses from db: %v", err)
}
return licenses, nil
}
// GetActiveLicense fetches the latest active license from DB
func (r *Repo) GetActiveLicense(ctx context.Context) (*model.License, error) {
var err error
licenses := []model.License{}
query := "SELECT key, activationId, planDetails, validationMessage FROM licenses"
err = r.db.Select(&licenses, query)
if err != nil {
return nil, fmt.Errorf("failed to get active licenses from db: %v", err)
}
var active *model.License
for _, l := range licenses {
l.ParsePlan()
if active == nil &&
(l.ValidFrom != 0) &&
(l.ValidUntil == -1 || l.ValidUntil > time.Now().Unix()) {
active = &l
}
if active != nil &&
l.ValidFrom > active.ValidFrom &&
(l.ValidUntil == -1 || l.ValidUntil > time.Now().Unix()) {
active = &l
}
}
return active, nil
}
// InsertLicense inserts a new license in db
func (r *Repo) InsertLicense(ctx context.Context, l *model.License) error {
if l.Key == "" {
return fmt.Errorf("insert license failed: license key is required")
}
query := `INSERT INTO licenses
(key, planDetails, activationId, validationmessage)
VALUES ($1, $2, $3, $4)`
_, err := r.db.ExecContext(ctx,
query,
l.Key,
l.PlanDetails,
l.ActivationId,
l.ValidationMessage)
if err != nil {
zap.S().Errorf("error in inserting license data: ", zap.Error(err))
return fmt.Errorf("failed to insert license in db: %v", err)
}
return nil
}
// UpdatePlanDetails writes new plan details to the db
func (r *Repo) UpdatePlanDetails(ctx context.Context,
key,
planDetails string) error {
if key == "" {
return fmt.Errorf("Update Plan Details failed: license key is required")
}
query := `UPDATE licenses
SET planDetails = $1,
updatedAt = $2
WHERE key = $3`
_, err := r.db.ExecContext(ctx, query, planDetails, time.Now(), key)
if err != nil {
zap.S().Errorf("error in updating license: ", zap.Error(err))
return fmt.Errorf("failed to update license in db: %v", err)
}
return nil
}

295
ee/query-service/license/manager.go Normal file
View File

@ -0,0 +1,295 @@
package license
import (
"context"
"fmt"
"sync/atomic"
"time"
"github.com/jmoiron/sqlx"
"sync"
validate "go.signoz.io/signoz/ee/query-service/integrations/signozio"
"go.signoz.io/signoz/ee/query-service/model"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/telemetry"
"go.uber.org/zap"
)
var LM *Manager
// validate and update license every 24 hours
var validationFrequency = 24 * 60 * time.Minute
type Manager struct {
repo *Repo
mutex sync.Mutex
validatorRunning bool
// end the license validation, this is important to gracefully
// stopping validation and protect in-consistent updates
done chan struct{}
// terminated waits for the validate go routine to end
terminated chan struct{}
// last time the license was validated
lastValidated int64
// keep track of validation failure attempts
failedAttempts uint64
// keep track of active license and features
activeLicense *model.License
activeFeatures basemodel.FeatureSet
}
func StartManager(dbType string, db *sqlx.DB) (*Manager, error) {
if LM != nil {
return LM, nil
}
repo := NewLicenseRepo(db)
err := repo.InitDB(dbType)
if err != nil {
return nil, fmt.Errorf("failed to initiate license repo: %v", err)
}
m := &Manager{
repo: &repo,
}
if err := m.start(); err != nil {
return m, err
}
LM = m
return m, nil
}
// start loads active license in memory and initiates validator
func (lm *Manager) start() error {
err := lm.LoadActiveLicense()
return err
}
func (lm *Manager) Stop() {
close(lm.done)
<-lm.terminated
}
func (lm *Manager) SetActive(l *model.License) {
lm.mutex.Lock()
defer lm.mutex.Unlock()
if l == nil {
return
}
lm.activeLicense = l
lm.activeFeatures = l.FeatureSet
if !lm.validatorRunning {
// we want to make sure only one validator runs,
// we already have lock() so good to go
lm.validatorRunning = true
go lm.Validator(context.Background())
}
}
// LoadActiveLicense loads the most recent active licenseex
func (lm *Manager) LoadActiveLicense() error {
var err error
active, err := lm.repo.GetActiveLicense(context.Background())
if err != nil {
return err
}
if active != nil {
lm.SetActive(active)
} else {
zap.S().Info("No active license found.")
}
return nil
}
func (lm *Manager) GetLicenses(ctx context.Context) (response []model.License, apiError *model.ApiError) {
licenses, err := lm.repo.GetLicenses(ctx)
if err != nil {
return nil, model.InternalError(err)
}
for _, l := range licenses {
l.ParsePlan()
if l.Key == lm.activeLicense.Key {
l.IsCurrent = true
}
if l.ValidUntil == -1 {
// for subscriptions, there is no end-date as such
// but for showing user some validity we default one year timespan
l.ValidUntil = l.ValidFrom + 31556926
}
response = append(response, l)
}
return
}
// Validator validates license after an epoch of time
func (lm *Manager) Validator(ctx context.Context) {
defer close(lm.terminated)
tick := time.NewTicker(validationFrequency)
defer tick.Stop()
lm.Validate(ctx)
for {
select {
case <-lm.done:
return
default:
select {
case <-lm.done:
return
case <-tick.C:
lm.Validate(ctx)
}
}
}
}
// Validate validates the current active license
func (lm *Manager) Validate(ctx context.Context) (reterr error) {
zap.S().Info("License validation started")
if lm.activeLicense == nil {
return nil
}
defer func() {
lm.mutex.Lock()
lm.lastValidated = time.Now().Unix()
if reterr != nil {
zap.S().Errorf("License validation completed with error", reterr)
atomic.AddUint64(&lm.failedAttempts, 1)
telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_LICENSE_CHECK_FAILED,
map[string]interface{}{"err": reterr.Error()})
} else {
zap.S().Info("License validation completed with no errors")
}
lm.mutex.Unlock()
}()
response, apiError := validate.ValidateLicense(lm.activeLicense.ActivationId)
if apiError != nil {
zap.S().Errorf("failed to validate license", apiError)
return apiError.Err
}
if response.PlanDetails == lm.activeLicense.PlanDetails {
// license plan hasnt changed, nothing to do
return nil
}
if response.PlanDetails != "" {
// copy and replace the active license record
l := model.License{
Key: lm.activeLicense.Key,
CreatedAt: lm.activeLicense.CreatedAt,
PlanDetails: response.PlanDetails,
ValidationMessage: lm.activeLicense.ValidationMessage,
ActivationId: lm.activeLicense.ActivationId,
}
if err := l.ParsePlan(); err != nil {
zap.S().Errorf("failed to parse updated license", zap.Error(err))
return err
}
// updated plan is parsable, check if plan has changed
if lm.activeLicense.PlanDetails != response.PlanDetails {
err := lm.repo.UpdatePlanDetails(ctx, lm.activeLicense.Key, response.PlanDetails)
if err != nil {
// unexpected db write issue but we can let the user continue
// and wait for update to work in next cycle.
zap.S().Errorf("failed to validate license", zap.Error(err))
}
}
// activate the update license plan
lm.SetActive(&l)
}
return nil
}
// Activate activates a license key with signoz server
func (lm *Manager) Activate(ctx context.Context, key string) (licenseResponse *model.License, errResponse *model.ApiError) {
defer func() {
if errResponse != nil {
telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_LICENSE_ACT_FAILED,
map[string]interface{}{"err": errResponse.Err.Error()})
}
}()
response, apiError := validate.ActivateLicense(key, "")
if apiError != nil {
zap.S().Errorf("failed to activate license", zap.Error(apiError.Err))
return nil, apiError
}
l := &model.License{
Key: key,
ActivationId: response.ActivationId,
PlanDetails: response.PlanDetails,
}
// parse validity and features from the plan details
err := l.ParsePlan()
if err != nil {
zap.S().Errorf("failed to activate license", zap.Error(err))
return nil, model.InternalError(err)
}
// store the license before activating it
err = lm.repo.InsertLicense(ctx, l)
if err != nil {
zap.S().Errorf("failed to activate license", zap.Error(err))
return nil, model.InternalError(err)
}
// license is valid, activate it
lm.SetActive(l)
return l, nil
}
// CheckFeature will be internally used by backend routines
// for feature gating
func (lm *Manager) CheckFeature(featureKey string) error {
if _, ok := lm.activeFeatures[featureKey]; ok {
return nil
}
return basemodel.ErrFeatureUnavailable{Key: featureKey}
}
// GetFeatureFlags returns current active features
func (lm *Manager) GetFeatureFlags() basemodel.FeatureSet {
return lm.activeFeatures
}
// GetRepo return the license repo
func (lm *Manager) GetRepo() *Repo {
return lm.repo
}

37
ee/query-service/license/sqlite/init.go Normal file
View File

@ -0,0 +1,37 @@
package sqlite
import (
"fmt"
"github.com/jmoiron/sqlx"
)
func InitDB(db *sqlx.DB) error {
var err error
if db == nil {
return fmt.Errorf("invalid db connection")
}
table_schema := `CREATE TABLE IF NOT EXISTS licenses(
key TEXT PRIMARY KEY,
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
planDetails TEXT,
activationId TEXT,
validationMessage TEXT,
lastValidated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS sites(
uuid TEXT PRIMARY KEY,
alias VARCHAR(180) DEFAULT 'PROD',
url VARCHAR(300),
createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
`
_, err = db.Exec(table_schema)
if err != nil {
return fmt.Errorf("Error in creating licenses table: %s", err.Error())
}
return nil
}

90
ee/query-service/main.go Normal file
View File

@ -0,0 +1,90 @@
package main
import (
"context"
"flag"
"os"
"os/signal"
"syscall"
"go.signoz.io/signoz/ee/query-service/app"
"go.signoz.io/signoz/pkg/query-service/auth"
baseconst "go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/version"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
func initZapLog() *zap.Logger {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
config.EncoderConfig.TimeKey = "timestamp"
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, _ := config.Build()
return logger
}
func main() {
var promConfigPath string
// disables rule execution but allows change to the rule definition
var disableRules bool
// the url used to build link in the alert messages in slack and other systems
var ruleRepoURL string
flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)")
flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)")
flag.StringVar(&ruleRepoURL, "rules.repo-url", baseconst.AlertHelpPage, "(host address used to build rule link in alert messages)")
flag.Parse()
loggerMgr := initZapLog()
zap.ReplaceGlobals(loggerMgr)
defer loggerMgr.Sync() // flushes buffer, if any
logger := loggerMgr.Sugar()
version.PrintVersion()
serverOptions := &app.ServerOptions{
HTTPHostPort: baseconst.HTTPHostPort,
PromConfigPath: promConfigPath,
PrivateHostPort: baseconst.PrivateHostPort,
DisableRules: disableRules,
RuleRepoURL: ruleRepoURL,
}
// Read the jwt secret key
auth.JwtSecret = os.Getenv("SIGNOZ_JWT_SECRET")
if len(auth.JwtSecret) == 0 {
zap.S().Warn("No JWT secret key is specified.")
} else {
zap.S().Info("No JWT secret key set successfully.")
}
server, err := app.NewServer(serverOptions)
if err != nil {
logger.Fatal("Failed to create server", zap.Error(err))
}
if err := server.Start(); err != nil {
logger.Fatal("Could not start servers", zap.Error(err))
}
if err := auth.InitAuthCache(context.Background()); err != nil {
logger.Fatal("Failed to initialize auth cache", zap.Error(err))
}
signalsChannel := make(chan os.Signal, 1)
signal.Notify(signalsChannel, os.Interrupt, syscall.SIGTERM)
for {
select {
case status := <-server.HealthCheckStatus():
logger.Info("Received HealthCheck status: ", zap.Int("status", int(status)))
case <-signalsChannel:
logger.Fatal("Received OS Interrupt Signal ... ")
}
}
}

21
ee/query-service/model/auth.go Normal file
View File

@ -0,0 +1,21 @@
package model
import (
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
// PrecheckResponse contains login precheck response
type PrecheckResponse struct {
SSO bool `json:"sso"`
SsoUrl string `json:"ssoUrl"`
CanSelfRegister bool `json:"canSelfRegister"`
IsUser bool `json:"isUser"`
SsoError string `json:"ssoError"`
}
// GettableInvitation overrides base object and adds precheck into
// response
type GettableInvitation struct {
*basemodel.InvitationResponseObject
Precheck *PrecheckResponse `json:"precheck"`
}

142
ee/query-service/model/domain.go Normal file
View File

@ -0,0 +1,142 @@
package model
import (
"encoding/json"
"fmt"
"net/url"
"strings"
"github.com/google/uuid"
"github.com/pkg/errors"
saml2 "github.com/russellhaering/gosaml2"
"go.signoz.io/signoz/ee/query-service/saml"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
type SSOType string
const (
SAML SSOType = "SAML"
GoogleAuth SSOType = "GOOGLE_AUTH"
)
type SamlConfig struct {
SamlEntity string `json:"samlEntity"`
SamlIdp string `json:"samlIdp"`
SamlCert string `json:"samlCert"`
}
// OrgDomain identify org owned web domains for auth and other purposes
type OrgDomain struct {
Id uuid.UUID `json:"id"`
Name string `json:"name"`
OrgId string `json:"orgId"`
SsoEnabled bool `json:"ssoEnabled"`
SsoType SSOType `json:"ssoType"`
SamlConfig *SamlConfig `json:"samlConfig"`
Org *basemodel.Organization
}
// Valid is used a pipeline function to check if org domain
// loaded from db is valid
func (od *OrgDomain) Valid(err error) error {
if err != nil {
return err
}
if od.Id == uuid.Nil || od.OrgId == "" {
return fmt.Errorf("both id and orgId are required")
}
return nil
}
// ValidNew cheks if the org domain is valid for insertion in db
func (od *OrgDomain) ValidNew() error {
if od.OrgId == "" {
return fmt.Errorf("orgId is required")
}
if od.Name == "" {
return fmt.Errorf("name is required")
}
return nil
}
// LoadConfig loads config params from json text
func (od *OrgDomain) LoadConfig(jsondata string) error {
d := *od
err := json.Unmarshal([]byte(jsondata), &d)
if err != nil {
return errors.Wrap(err, "failed to marshal json to OrgDomain{}")
}
*od = d
return nil
}
func (od *OrgDomain) GetSAMLEntityID() string {
if od.SamlConfig != nil {
return od.SamlConfig.SamlEntity
}
return ""
}
func (od *OrgDomain) GetSAMLIdpURL() string {
if od.SamlConfig != nil {
return od.SamlConfig.SamlIdp
}
return ""
}
func (od *OrgDomain) GetSAMLCert() string {
if od.SamlConfig != nil {
return od.SamlConfig.SamlCert
}
return ""
}
// PrepareSamlRequest creates a request accordingly gosaml2
func (od *OrgDomain) PrepareSamlRequest(siteUrl *url.URL) (*saml2.SAMLServiceProvider, error) {
// this is the url Idp will call after login completion
acs := fmt.Sprintf("%s://%s/%s",
siteUrl.Scheme,
siteUrl.Host,
"api/v1/complete/saml")
// this is the address of the calling url, useful to redirect user
sourceUrl := fmt.Sprintf("%s://%s%s",
siteUrl.Scheme,
siteUrl.Host,
siteUrl.Path)
// ideally this should be some unique ID for each installation
// but since we dont have UI to support it, we default it to
// host. this issuer is an identifier of service provider (signoz)
// on id provider (e.g. azure, okta). Azure requires this id to be configured
// in their system, while others seem to not care about it.
// currently we default it to host from window.location (received from browser)
issuer := siteUrl.Host
return saml.PrepareRequest(issuer, acs, sourceUrl, od.GetSAMLEntityID(), od.GetSAMLIdpURL(), od.GetSAMLCert())
}
func (od *OrgDomain) BuildSsoUrl(siteUrl *url.URL) (ssoUrl string, err error) {
sp, err := od.PrepareSamlRequest(siteUrl)
if err != nil {
return "", err
}
fmtDomainId := strings.Replace(od.Id.String(), "-", ":", -1)
relayState := fmt.Sprintf("%s://%s%s?domainId=%s",
siteUrl.Scheme,
siteUrl.Host,
siteUrl.Path,
fmtDomainId)
return sp.BuildAuthURL(relayState)
}

91
ee/query-service/model/errors.go Normal file
View File

@ -0,0 +1,91 @@
package model
import (
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
type ApiError struct {
Typ basemodel.ErrorType
Err error
}
func (a *ApiError) Type() basemodel.ErrorType {
return a.Typ
}
func (a *ApiError) ToError() error {
if a != nil {
return a.Err
}
return a.Err
}
func (a *ApiError) Error() string {
return a.Err.Error()
}
func (a *ApiError) IsNil() bool {
return a == nil || a.Err == nil
}
// NewApiError returns a ApiError object of given type
func NewApiError(typ basemodel.ErrorType, err error) *ApiError {
return &ApiError{
Typ: typ,
Err: err,
}
}
// BadRequest returns a ApiError object of bad request
func BadRequest(err error) *ApiError {
return &ApiError{
Typ: basemodel.ErrorBadData,
Err: err,
}
}
// InternalError returns a ApiError object of internal type
func InternalError(err error) *ApiError {
return &ApiError{
Typ: basemodel.ErrorInternal,
Err: err,
}
}
var (
ErrorNone basemodel.ErrorType = ""
ErrorTimeout basemodel.ErrorType = "timeout"
ErrorCanceled basemodel.ErrorType = "canceled"
ErrorExec basemodel.ErrorType = "execution"
ErrorBadData basemodel.ErrorType = "bad_data"
ErrorInternal basemodel.ErrorType = "internal"
ErrorUnavailable basemodel.ErrorType = "unavailable"
ErrorNotFound basemodel.ErrorType = "not_found"
ErrorNotImplemented basemodel.ErrorType = "not_implemented"
ErrorUnauthorized basemodel.ErrorType = "unauthorized"
ErrorForbidden basemodel.ErrorType = "forbidden"
ErrorConflict basemodel.ErrorType = "conflict"
ErrorStreamingNotSupported basemodel.ErrorType = "streaming is not supported"
)
func init() {
ErrorNone = basemodel.ErrorNone
ErrorTimeout = basemodel.ErrorTimeout
ErrorCanceled = basemodel.ErrorCanceled
ErrorExec = basemodel.ErrorExec
ErrorBadData = basemodel.ErrorBadData
ErrorInternal = basemodel.ErrorInternal
ErrorUnavailable = basemodel.ErrorUnavailable
ErrorNotFound = basemodel.ErrorNotFound
ErrorNotImplemented = basemodel.ErrorNotImplemented
ErrorUnauthorized = basemodel.ErrorUnauthorized
ErrorForbidden = basemodel.ErrorForbidden
ErrorConflict = basemodel.ErrorConflict
ErrorStreamingNotSupported = basemodel.ErrorStreamingNotSupported
}
type ErrUnsupportedAuth struct{}
func (errUnsupportedAuth ErrUnsupportedAuth) Error() string {
return "this authentication method not supported"
}

91
ee/query-service/model/license.go Normal file
View File

@ -0,0 +1,91 @@
package model
import (
"encoding/base64"
"encoding/json"
"time"
"github.com/pkg/errors"
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
type License struct {
Key string `json:"key" db:"key"`
ActivationId string `json:"activationId" db:"activationId"`
CreatedAt time.Time `db:"created_at"`
// PlanDetails contains the encrypted plan info
PlanDetails string `json:"planDetails" db:"planDetails"`
// stores parsed license details
LicensePlan
FeatureSet basemodel.FeatureSet
// populated in case license has any errors
ValidationMessage string `db:"validationMessage"`
// used only for sending details to front-end
IsCurrent bool `json:"isCurrent"`
}
func (l *License) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
Key string `json:"key" db:"key"`
ActivationId string `json:"activationId" db:"activationId"`
ValidationMessage string `db:"validationMessage"`
IsCurrent bool `json:"isCurrent"`
PlanKey string `json:"planKey"`
ValidFrom time.Time `json:"ValidFrom"`
ValidUntil time.Time `json:"ValidUntil"`
Status string `json:"status"`
}{
Key: l.Key,
ActivationId: l.ActivationId,
IsCurrent: l.IsCurrent,
PlanKey: l.PlanKey,
ValidFrom: time.Unix(l.ValidFrom, 0),
ValidUntil: time.Unix(l.ValidUntil, 0),
Status: l.Status,
ValidationMessage: l.ValidationMessage,
})
}
type LicensePlan struct {
PlanKey string `json:"planKey"`
ValidFrom int64 `json:"validFrom"`
ValidUntil int64 `json:"validUntil"`
Status string `json:"status"`
}
func (l *License) ParsePlan() error {
l.LicensePlan = LicensePlan{}
planData, err := base64.StdEncoding.DecodeString(l.PlanDetails)
if err != nil {
return err
}
plan := LicensePlan{}
err = json.Unmarshal([]byte(planData), &plan)
if err != nil {
l.ValidationMessage = "failed to parse plan from license"
return errors.Wrap(err, "failed to parse plan from license")
}
l.LicensePlan = plan
l.ParseFeatures()
return nil
}
func (l *License) ParseFeatures() {
switch l.PlanKey {
case Pro:
l.FeatureSet = ProPlan
case Enterprise:
l.FeatureSet = EnterprisePlan
default:
l.FeatureSet = BasicPlan
}
}

27
ee/query-service/model/plans.go Normal file
View File

@ -0,0 +1,27 @@
package model
import (
basemodel "go.signoz.io/signoz/pkg/query-service/model"
)
const SSO = "SSO"
const Basic = "BASIC_PLAN"
const Pro = "PRO_PLAN"
const Enterprise = "ENTERPRISE_PLAN"
const DisableUpsell = "DISABLE_UPSELL"
var BasicPlan = basemodel.FeatureSet{
Basic: true,
SSO: false,
DisableUpsell: false,
}
var ProPlan = basemodel.FeatureSet{
Pro: true,
SSO: true,
}
var EnterprisePlan = basemodel.FeatureSet{
Enterprise: true,
SSO: true,
}

35
ee/query-service/model/usage.go Normal file
View File

@ -0,0 +1,35 @@
package model
import (
"time"
"github.com/google/uuid"
)
type UsageSnapshot struct {
CurrentLogSizeBytes uint64 `json:"currentLogSizeBytes"`
CurrentLogSizeBytesColdStorage uint64 `json:"currentLogSizeBytesColdStorage"`
CurrentSpansCount uint64 `json:"currentSpansCount"`
CurrentSpansCountColdStorage uint64 `json:"currentSpansCountColdStorage"`
CurrentSamplesCount uint64 `json:"currentSamplesCount"`
CurrentSamplesCountColdStorage uint64 `json:"currentSamplesCountColdStorage"`
}
type UsageBase struct {
Id uuid.UUID `json:"id" db:"id"`
InstallationId uuid.UUID `json:"installationId" db:"installation_id"`
ActivationId uuid.UUID `json:"activationId" db:"activation_id"`
CreatedAt time.Time `json:"createdAt" db:"created_at"`
FailedSyncRequest int `json:"failedSyncRequest" db:"failed_sync_request_count"`
}
type UsagePayload struct {
UsageBase
Metrics UsageSnapshot `json:"metrics"`
SnapshotDate time.Time `json:"snapshotDate"`
}
type Usage struct {
UsageBase
Snapshot string `db:"snapshot"`
}

107
ee/query-service/saml/request.go Normal file
View File

@ -0,0 +1,107 @@
package saml
import (
"crypto/x509"
"encoding/base64"
"encoding/pem"
"fmt"
"strings"
saml2 "github.com/russellhaering/gosaml2"
dsig "github.com/russellhaering/goxmldsig"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.uber.org/zap"
)
func LoadCertificateStore(certString string) (dsig.X509CertificateStore, error) {
certStore := &dsig.MemoryX509CertificateStore{
Roots: []*x509.Certificate{},
}
certData, err := base64.StdEncoding.DecodeString(certString)
if err != nil {
return certStore, fmt.Errorf(fmt.Sprintf("failed to read certificate: %v", err))
}
idpCert, err := x509.ParseCertificate(certData)
if err != nil {
return certStore, fmt.Errorf(fmt.Sprintf("failed to prepare saml request, invalid cert: %s", err.Error()))
}
certStore.Roots = append(certStore.Roots, idpCert)
return certStore, nil
}
func LoadCertFromPem(certString string) (dsig.X509CertificateStore, error) {
certStore := &dsig.MemoryX509CertificateStore{
Roots: []*x509.Certificate{},
}
block, _ := pem.Decode([]byte(certString))
if block == nil {
return certStore, fmt.Errorf("no valid pem cert found")
}
idpCert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return certStore, fmt.Errorf(fmt.Sprintf("failed to parse pem cert: %s", err.Error()))
}
certStore.Roots = append(certStore.Roots, idpCert)
return certStore, nil
}
// PrepareRequest prepares authorization URL (Idp Provider URL)
func PrepareRequest(issuer, acsUrl, audience, entity, idp, certString string) (*saml2.SAMLServiceProvider, error) {
var certStore dsig.X509CertificateStore
if certString == "" {
return nil, fmt.Errorf("invalid certificate data")
}
var err error
if strings.Contains(certString, "-----BEGIN CERTIFICATE-----") {
certStore, err = LoadCertFromPem(certString)
} else {
certStore, err = LoadCertificateStore(certString)
}
// certificate store can not be created, throw error
if err != nil {
return nil, err
}
randomKeyStore := dsig.RandomKeyStoreForTest()
// SIGNOZ_SAML_RETURN_URL env var would support overriding window.location
// as return destination after saml request is complete from IdP side.
// this var is also useful for development, as it is easy to override with backend endpoint
// e.g. http://localhost:8080/api/v1/complete/saml
acsUrl = constants.GetOrDefaultEnv("SIGNOZ_SAML_RETURN_URL", acsUrl)
sp := &saml2.SAMLServiceProvider{
IdentityProviderSSOURL: idp,
IdentityProviderIssuer: entity,
ServiceProviderIssuer: issuer,
AssertionConsumerServiceURL: acsUrl,
SignAuthnRequests: true,
AllowMissingAttributes: true,
// about cert stores -sender(signoz app) and receiver (idp)
// The random key (random key store) is sender cert. The public cert store(IDPCertificateStore) that you see on org domain is receiver cert (idp provided).
// At the moment, the library we use doesn't bother about sender cert and IdP too. It just adds additional layer of security, which we can explore in future versions
// The receiver (Idp) cert will be different for each org domain. Imagine cloud setup where each company setups their domain that integrates with their Idp.
// @signoz.io
// @next.io
// Each of above will have their own Idp setup and hence separate public cert to decrypt the response.
// The way SAML request travels is -
// SigNoz Backend -> IdP Login Screen -> SigNoz Backend -> SigNoz Frontend
// ---------------- | -------------------| -------------------------------------
// The dotted lines indicate request boundries. So if you notice, the response from Idp starts a new request. hence we need relay state to pass the context around.
IDPCertificateStore: certStore,
SPKeyStore: randomKeyStore,
}
zap.S().Debugf("SAML request:", sp)
return sp, nil
}

321
ee/query-service/usage/manager.go Normal file
View File

@ -0,0 +1,321 @@
package usage
import (
"context"
"encoding/json"
"fmt"
"sync/atomic"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"go.uber.org/zap"
licenseserver "go.signoz.io/signoz/ee/query-service/integrations/signozio"
"go.signoz.io/signoz/ee/query-service/license"
"go.signoz.io/signoz/ee/query-service/model"
"go.signoz.io/signoz/ee/query-service/usage/repository"
"go.signoz.io/signoz/pkg/query-service/utils/encryption"
)
const (
MaxRetries = 3
RetryInterval = 5 * time.Second
stateUnlocked uint32 = 0
stateLocked uint32 = 1
)
var (
// // collect usage every hour
// collectionFrequency = 1 * time.Hour
// // send usage every 24 hour
// uploadFrequency = 24 * time.Hour
// collect usage every hour
collectionFrequency = 5 * time.Second
// send usage every 24 hour
uploadFrequency = 30 * time.Second
locker = stateUnlocked
)
type Manager struct {
repository *repository.Repository
clickhouseConn clickhouse.Conn
licenseRepo *license.Repo
// end the usage routine, this is important to gracefully
// stopping usage reporting and protect in-consistent updates
done chan struct{}
// terminated waits for the UsageExporter go routine to end
terminated chan struct{}
}
func New(dbType string, db *sqlx.DB, licenseRepo *license.Repo, clickhouseConn clickhouse.Conn) (*Manager, error) {
repo := repository.New(db)
err := repo.Init(dbType)
if err != nil {
return nil, fmt.Errorf("failed to initiate usage repo: %v", err)
}
m := &Manager{
repository: repo,
clickhouseConn: clickhouseConn,
licenseRepo: licenseRepo,
}
return m, nil
}
// start loads collects and exports any exported snapshot and starts the exporter
func (lm *Manager) Start() error {
// compares the locker and stateUnlocked if both are same lock is applied else returns error
if !atomic.CompareAndSwapUint32(&locker, stateUnlocked, stateLocked) {
return fmt.Errorf("usage exporter is locked")
}
// check if license is present or not
license, err := lm.licenseRepo.GetActiveLicense(context.Background())
if err != nil {
return fmt.Errorf("failed to get active license")
}
if license == nil {
// we will not start the usage reporting if license is not present.
zap.S().Info("no license present, skipping usage reporting")
return nil
}
// upload previous snapshots if any
err = lm.UploadUsage(context.Background())
if err != nil {
return err
}
// collect snapshot if incase it wasn't collect in (t - collectionFrequency)
err = lm.CollectCurrentUsage(context.Background())
if err != nil {
return err
}
go lm.UsageExporter(context.Background())
return nil
}
// CollectCurrentUsage checks if needs to collect usage data
func (lm *Manager) CollectCurrentUsage(ctx context.Context) error {
// check the DB if anything exist where timestamp > t - collectionFrequency
ts := time.Now().Add(-collectionFrequency)
alreadyCreated, err := lm.repository.CheckSnapshotGtCreatedAt(ctx, ts)
if err != nil {
return err
}
if !alreadyCreated {
zap.S().Info("Collecting current usage")
exportError := lm.CollectAndStoreUsage(ctx)
if exportError != nil {
return exportError
}
} else {
zap.S().Info("Nothing to collect")
}
return nil
}
func (lm *Manager) UsageExporter(ctx context.Context) {
defer close(lm.terminated)
collectionTicker := time.NewTicker(collectionFrequency)
defer collectionTicker.Stop()
uploadTicker := time.NewTicker(uploadFrequency)
defer uploadTicker.Stop()
for {
select {
case <-lm.done:
return
case <-collectionTicker.C:
lm.CollectAndStoreUsage(ctx)
case <-uploadTicker.C:
lm.UploadUsage(ctx)
}
}
}
type TableSize struct {
Table string `ch:"table"`
DiskName string `ch:"disk_name"`
Rows uint64 `ch:"rows"`
UncompressedBytes uint64 `ch:"uncompressed_bytes"`
}
func (lm *Manager) CollectAndStoreUsage(ctx context.Context) error {
snap, err := lm.GetUsageFromClickHouse(ctx)
if err != nil {
return err
}
license, err := lm.licenseRepo.GetActiveLicense(ctx)
if err != nil {
return err
}
activationId, _ := uuid.Parse(license.ActivationId)
// TODO (nitya) : Add installation ID in the payload
payload := model.UsagePayload{
UsageBase: model.UsageBase{
ActivationId: activationId,
FailedSyncRequest: 0,
},
Metrics: *snap,
SnapshotDate: time.Now(),
}
err = lm.repository.InsertSnapshot(ctx, &payload)
if err != nil {
return err
}
return nil
}
func (lm *Manager) GetUsageFromClickHouse(ctx context.Context) (*model.UsageSnapshot, error) {
tableSizes := []TableSize{}
snap := model.UsageSnapshot{}
// get usage from clickhouse
query := `
SELECT
table,
disk_name,
sum(rows) as rows,
sum(data_uncompressed_bytes) AS uncompressed_bytes
FROM system.parts
WHERE active AND (database in ('signoz_logs', 'signoz_metrics', 'signoz_traces')) AND (table in ('logs','samples_v2', 'signoz_index_v2'))
GROUP BY
table,
disk_name
ORDER BY table
`
err := lm.clickhouseConn.Select(ctx, &tableSizes, query)
if err != nil {
return nil, err
}
for _, val := range tableSizes {
switch val.Table {
case "logs":
if val.DiskName == "default" {
snap.CurrentLogSizeBytes = val.UncompressedBytes
} else {
snap.CurrentLogSizeBytesColdStorage = val.UncompressedBytes
}
case "samples_v2":
if val.DiskName == "default" {
snap.CurrentSamplesCount = val.Rows
} else {
snap.CurrentSamplesCountColdStorage = val.Rows
}
case "signoz_index_v2":
if val.DiskName == "default" {
snap.CurrentSpansCount = val.Rows
} else {
snap.CurrentSpansCountColdStorage = val.Rows
}
}
}
return &snap, nil
}
func (lm *Manager) UploadUsage(ctx context.Context) error {
snapshots, err := lm.repository.GetSnapshotsNotSynced(ctx)
if err != nil {
return err
}
if len(snapshots) <= 0 {
zap.S().Info("no snapshots to upload, skipping.")
return nil
}
zap.S().Info("uploading snapshots")
for _, snap := range snapshots {
metricsBytes, err := encryption.Decrypt([]byte(snap.ActivationId.String()[:32]), []byte(snap.Snapshot))
if err != nil {
return err
}
metrics := model.UsageSnapshot{}
err = json.Unmarshal(metricsBytes, &metrics)
if err != nil {
return err
}
err = lm.UploadUsageWithExponentalBackOff(ctx, model.UsagePayload{
UsageBase: model.UsageBase{
Id: snap.Id,
InstallationId: snap.InstallationId,
ActivationId: snap.ActivationId,
FailedSyncRequest: snap.FailedSyncRequest,
},
SnapshotDate: snap.CreatedAt,
Metrics: metrics,
})
if err != nil {
return err
}
}
return nil
}
func (lm *Manager) UploadUsageWithExponentalBackOff(ctx context.Context, payload model.UsagePayload) error {
for i := 1; i <= MaxRetries; i++ {
apiErr := licenseserver.SendUsage(ctx, &payload)
if apiErr != nil && i == MaxRetries {
err := lm.repository.IncrementFailedRequestCount(ctx, payload.Id)
if err != nil {
zap.S().Errorf("failed to updated the failure count for snapshot in DB : ", zap.Error(err))
return err
}
zap.S().Errorf("retries stopped : %v", zap.Error(err))
// not returning error here since it is captured in the failed count
return nil
} else if apiErr != nil {
// sleeping for exponential backoff
sleepDuration := RetryInterval * time.Duration(i)
zap.S().Errorf("failed to upload snapshot retrying after %v secs : %v", sleepDuration.Seconds(), zap.Error(apiErr.Err))
time.Sleep(sleepDuration)
// update the failed request count
err := lm.repository.IncrementFailedRequestCount(ctx, payload.Id)
if err != nil {
zap.S().Errorf("failed to updated the failure count for snapshot in DB : %v", zap.Error(err))
return err
}
} else {
break
}
}
// update the database that it is synced
err := lm.repository.MoveToSynced(ctx, payload.Id)
if err != nil {
return err
}
return nil
}
func (lm *Manager) Stop() {
close(lm.done)
atomic.StoreUint32(&locker, stateUnlocked)
<-lm.terminated
}

126
ee/query-service/usage/repository/repository.go Normal file
View File

@ -0,0 +1,126 @@
package repository
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jmoiron/sqlx"
"go.uber.org/zap"
"go.signoz.io/signoz/ee/query-service/model"
"go.signoz.io/signoz/ee/query-service/usage/sqlite"
"go.signoz.io/signoz/pkg/query-service/utils/encryption"
)
const (
MaxFailedSyncCount = 9 // a snapshot will be ignored if the max failed count is greater than or equal to 9
)
// Repository is usage Repository which stores usage snapshot in a secured DB
type Repository struct {
db *sqlx.DB
}
// New initiates a new usage Repository
func New(db *sqlx.DB) *Repository {
return &Repository{
db: db,
}
}
func (r *Repository) Init(engine string) error {
switch engine {
case "sqlite3", "sqlite":
return sqlite.InitDB(r.db)
default:
return fmt.Errorf("unsupported db")
}
}
func (r *Repository) InsertSnapshot(ctx context.Context, usage *model.UsagePayload) error {
snapshotBytes, err := json.Marshal(usage.Metrics)
if err != nil {
return err
}
usage.Id = uuid.New()
encryptedSnapshot, err := encryption.Encrypt([]byte(usage.ActivationId.String()[:32]), snapshotBytes)
if err != nil {
return err
}
query := `INSERT INTO usage(id, activation_id, snapshot)
VALUES ($1, $2, $3)`
_, err = r.db.ExecContext(ctx,
query,
usage.Id,
usage.ActivationId,
string(encryptedSnapshot),
)
if err != nil {
zap.S().Errorf("error inserting usage data: %v", zap.Error(err))
return fmt.Errorf("failed to insert usage in db: %v", err)
}
return nil
}
func (r *Repository) MoveToSynced(ctx context.Context, id uuid.UUID) error {
query := `UPDATE usage
SET synced = 'true',
synced_at = $1
WHERE id = $2`
_, err := r.db.ExecContext(ctx, query, time.Now(), id)
if err != nil {
zap.S().Errorf("error in updating usage: %v", zap.Error(err))
return fmt.Errorf("failed to update usage in db: %v", err)
}
return nil
}
func (r *Repository) IncrementFailedRequestCount(ctx context.Context, id uuid.UUID) error {
query := `UPDATE usage SET failed_sync_request_count = failed_sync_request_count + 1 WHERE id = $1`
_, err := r.db.ExecContext(ctx, query, id)
if err != nil {
zap.S().Errorf("error in updating usage: %v", zap.Error(err))
return fmt.Errorf("failed to update usage in db: %v", err)
}
return nil
}
func (r *Repository) GetSnapshotsNotSynced(ctx context.Context) ([]*model.Usage, error) {
snapshots := []*model.Usage{}
query := `SELECT id,created_at, activation_id, snapshot, failed_sync_request_count from usage where synced!='true' and failed_sync_request_count < $1 order by created_at asc `
err := r.db.SelectContext(ctx, &snapshots, query, MaxFailedSyncCount)
if err != nil {
return nil, err
}
return snapshots, nil
}
// CheckSnapshotGtCreatedAt checks if there is any snapshot greater than the provided timestamp
func (r *Repository) CheckSnapshotGtCreatedAt(ctx context.Context, ts time.Time) (bool, error) {
var snapshots uint64
query := `SELECT count() from usage where created_at > '$1'`
err := r.db.QueryRowContext(ctx, query, ts).Scan(&snapshots)
if err != nil {
return false, err
}
return snapshots > 0, err
}

32
ee/query-service/usage/sqlite/init.go Normal file
View File

@ -0,0 +1,32 @@
package sqlite
import (
"fmt"
"github.com/jmoiron/sqlx"
)
func InitDB(db *sqlx.DB) error {
var err error
if db == nil {
return fmt.Errorf("invalid db connection")
}
table_schema := `CREATE TABLE IF NOT EXISTS usage(
id UUID PRIMARY KEY,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
activation_id UUID,
snapshot TEXT,
synced BOOLEAN DEFAULT 'false',
synced_at TIMESTAMP,
failed_sync_request_count INTEGER DEFAULT 0
);
`
_, err = db.Exec(table_schema)
if err != nil {
return fmt.Errorf("error in creating usage table: %v", err.Error())
}
return nil
}

26
pkg/query-service/go.mod → go.mod
View File

@ -1,4 +1,4 @@
module go.signoz.io/query-service
module go.signoz.io/signoz
go 1.17
@ -16,11 +16,13 @@ require (
github.com/minio/minio-go/v6 v6.0.57
github.com/oklog/oklog v0.3.2
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v0.9.0-pre1.0.20181001174001-0a8115f42e03
github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f
github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1
github.com/prometheus/prometheus v2.5.0+incompatible
github.com/prometheus/tsdb v0.0.0-20181003080831-0ce41118ed20
github.com/rs/cors v1.7.0
github.com/russellhaering/gosaml2 v0.8.0
github.com/russellhaering/goxmldsig v1.2.0
github.com/sethvargo/go-password v0.2.0
github.com/smartystreets/goconvey v1.6.4
github.com/soheilhy/cmux v0.1.4
go.uber.org/zap v1.16.0
@ -29,12 +31,16 @@ require (
)
require (
github.com/beevik/etree v1.1.0 // indirect
github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/klauspost/cpuid v1.2.3 // indirect
github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
github.com/minio/md5-simd v1.1.0 // indirect
github.com/minio/sha256-simd v0.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f // indirect
github.com/prometheus/client_golang v0.9.0-pre1.0.20181001174001-0a8115f42e03 // indirect
github.com/prometheus/tsdb v0.0.0-20181003080831-0ce41118ed20 // indirect
gopkg.in/ini.v1 v1.42.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)
@ -49,13 +55,13 @@ require (
github.com/aws/aws-sdk-go v1.27.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash v1.1.0
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c // indirect
github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c
github.com/go-logfmt/logfmt v0.5.0 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
@ -86,7 +92,6 @@ require (
github.com/hashicorp/serf v0.8.1-0.20161007004122-1d4fa605f6ff // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/kr/text v0.2.0 // indirect
github.com/lib/pq v1.10.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/miekg/dns v1.0.4 // indirect
@ -97,7 +102,7 @@ require (
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/oklog/ulid v0.3.1-0.20170117200651-66bb6560562f // indirect
github.com/opentracing/opentracing-go v1.1.0 // indirect
github.com/opentracing/opentracing-go v1.1.0
github.com/pascaldekloe/goe v0.1.0 // indirect
github.com/paulmach/orb v0.4.0 // indirect
github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible // indirect
@ -120,7 +125,7 @@ require (
go.uber.org/atomic v1.6.0 // indirect
go.uber.org/multierr v1.5.0 // indirect
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e // indirect
golang.org/x/net v0.0.0-20211013171255-e13a2654a71e
golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect
@ -133,10 +138,9 @@ require (
google.golang.org/grpc v1.41.0
google.golang.org/grpc/examples v0.0.0-20210803221256-6ba56c814be7 // indirect
google.golang.org/protobuf v1.27.1 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.0.0-20180628040859-072894a440bd // indirect
k8s.io/client-go v8.0.0+incompatible // indirect
)

22
pkg/query-service/go.sum → go.sum
View File

@ -70,6 +70,8 @@ github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9
github.com/aws/aws-sdk-go v1.13.44-0.20180507225419-00862f899353/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k=
github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk=
github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
github.com/beorn7/perks v0.0.0-20160229213445-3ac7bf7a47d1/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@ -292,10 +294,10 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC
github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w=
github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ=
github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@ -310,8 +312,9 @@ github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@ -320,6 +323,8 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU=
github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU=
@ -344,7 +349,6 @@ github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oe
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
@ -373,6 +377,7 @@ github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.1-0.20161029093637-248dadf4e906/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -394,8 +399,15 @@ github.com/prometheus/tsdb v0.0.0-20181003080831-0ce41118ed20 h1:Jh/eKJuru9z9u3r
github.com/prometheus/tsdb v0.0.0-20181003080831-0ce41118ed20/go.mod h1:lFf/o1J2a31WmWQbxYXfY1azJK5Xp5D8hwKMnVMBTGU=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russellhaering/gosaml2 v0.8.0 h1:rm1Gc09/UoEsKGTSFvg8VCHJLY3wrP4BWjC+1ov0qCo=
github.com/russellhaering/gosaml2 v0.8.0/go.mod h1:byViER/1YPUa0Puj9ROZblpoq2jsE7h/CJmitzX0geU=
github.com/russellhaering/goxmldsig v1.2.0 h1:Y6GTTc9Un5hCxSzVz4UIWQ/zuVwDvzJk80guqzwx6Vg=
github.com/russellhaering/goxmldsig v1.2.0/go.mod h1:gM4MDENBQf7M+V824SGfyIUVFWydB7n0KkEubVJl+Tw=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU=
@ -405,6 +417,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUt
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/segmentio/backo-go v1.0.0 h1:kbOAtGJY2DqOR0jfRkYEorx/b18RgtepGtY3+Cpe6qA=
github.com/segmentio/backo-go v1.0.0/go.mod h1:kJ9mm9YmoWSkk+oQ+5Cj8DEoRCX2JT6As4kEtIIOp1M=
github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI=
github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE=
github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=

11
pkg/query-service/Dockerfile
View File

@ -11,7 +11,7 @@ RUN export GOOS=$(echo ${TARGETPLATFORM} | cut -d / -f1) && \
export GOARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2)
# Prepare and enter src directory
WORKDIR /go/src/github.com/signoz/signoz/pkg/query-service
WORKDIR /go/src/github.com/signoz/signoz
# Cache dependencies
ADD go.mod .
@ -20,8 +20,10 @@ RUN go mod download -x
# Add the sources and proceed with build
ADD . .
RUN go build -tags timetzdata -a -ldflags "-linkmode external -extldflags '-static' -s -w $LD_FLAGS" -o ./bin/query-service ./main.go
RUN chmod +x ./bin/query-service
RUN cd pkg/query-service \
&& go build -tags timetzdata -a -o ./bin/query-service \
-ldflags "-linkmode external -extldflags '-static' -s -w $LD_FLAGS" \
&& chmod +x ./bin/query-service
# use a minimal alpine image
@ -39,7 +41,8 @@ WORKDIR /root
# copy the binary from builder
COPY --from=builder /go/src/github.com/signoz/signoz/pkg/query-service/bin/query-service .
COPY config/prometheus.yml /root/config/prometheus.yml
# copy prometheus YAML config
COPY pkg/query-service/config/prometheus.yml /root/config/prometheus.yml
# run the binary
ENTRYPOINT ["./query-service"]

10
pkg/query-service/app/clickhouseReader/reader.go
View File

@ -39,11 +39,11 @@ import (
"github.com/jmoiron/sqlx"
promModel "github.com/prometheus/common/model"
"go.signoz.io/query-service/app/logs"
"go.signoz.io/query-service/constants"
am "go.signoz.io/query-service/integrations/alertManager"
"go.signoz.io/query-service/model"
"go.signoz.io/query-service/utils"
"go.signoz.io/signoz/pkg/query-service/app/logs"
"go.signoz.io/signoz/pkg/query-service/constants"
am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/utils"
"go.uber.org/zap"
)

2
pkg/query-service/app/dashboards/model.go
View File

@ -10,7 +10,7 @@ import (
"github.com/google/uuid"
"github.com/gosimple/slug"
"github.com/jmoiron/sqlx"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)

2
pkg/query-service/app/dashboards/provision.go
View File

@ -5,7 +5,7 @@ import (
"io/ioutil"
"os"
"go.signoz.io/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.uber.org/zap"
)

631
pkg/query-service/app/http_handler.go
View File

File diff suppressed because it is too large Load Diff

4
pkg/query-service/app/logs/parser.go
View File

@ -7,8 +7,8 @@ import (
"strconv"
"strings"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
)
var operatorMapping = map[string]string{

2
pkg/query-service/app/logs/parser_test.go
View File

@ -4,7 +4,7 @@ import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/model"
)
var correctQueriesTest = []struct {

4
pkg/query-service/app/logs/validator.go
View File

@ -4,8 +4,8 @@ import (
"fmt"
"regexp"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
)
func ValidateUpdateFieldPayload(field *model.UpdateField) error {

4
pkg/query-service/app/metrics/query_builder.go
View File

@ -6,8 +6,8 @@ import (
"strings"
"github.com/SigNoz/govaluate"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)

2
pkg/query-service/app/metrics/query_builder_test.go
View File

@ -4,7 +4,7 @@ import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/model"
)
func TestBuildQuery(t *testing.T) {

6
pkg/query-service/app/parser.go
View File

@ -12,9 +12,9 @@ import (
"github.com/gorilla/mux"
promModel "github.com/prometheus/common/model"
"go.signoz.io/query-service/auth"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
)
var allowedFunctions = []string{"count", "ratePerSec", "sum", "avg", "min", "max", "p50", "p90", "p95", "p99"}

4
pkg/query-service/app/parser/metrics.go
View File

@ -6,8 +6,8 @@ import (
"net/http"
"strings"
"go.signoz.io/query-service/app/metrics"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/app/metrics"
"go.signoz.io/signoz/pkg/query-service/model"
)
func validateQueryRangeParamsV2(qp *model.QueryRangeParamsV2) error {

4
pkg/query-service/app/parser_test.go
View File

@ -8,8 +8,8 @@ import (
"github.com/smartystreets/assertions/should"
. "github.com/smartystreets/goconvey/convey"
"go.signoz.io/query-service/app/metrics"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/app/metrics"
"go.signoz.io/signoz/pkg/query-service/model"
)
func TestParseFilterSingleFilter(t *testing.T) {

28
pkg/query-service/app/server.go
View File

@ -15,17 +15,17 @@ import (
"github.com/rs/cors"
"github.com/soheilhy/cmux"
"go.signoz.io/query-service/app/clickhouseReader"
"go.signoz.io/query-service/app/dashboards"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/dao"
"go.signoz.io/query-service/healthcheck"
am "go.signoz.io/query-service/integrations/alertManager"
"go.signoz.io/query-service/interfaces"
pqle "go.signoz.io/query-service/pqlEngine"
"go.signoz.io/query-service/rules"
"go.signoz.io/query-service/telemetry"
"go.signoz.io/query-service/utils"
"go.signoz.io/signoz/pkg/query-service/app/clickhouseReader"
"go.signoz.io/signoz/pkg/query-service/app/dashboards"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/dao"
"go.signoz.io/signoz/pkg/query-service/healthcheck"
am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
"go.signoz.io/signoz/pkg/query-service/interfaces"
pqle "go.signoz.io/signoz/pkg/query-service/pqlEngine"
"go.signoz.io/signoz/pkg/query-service/rules"
"go.signoz.io/signoz/pkg/query-service/telemetry"
"go.signoz.io/signoz/pkg/query-service/utils"
"go.uber.org/zap"
)
@ -97,7 +97,11 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
}
telemetry.GetInstance().SetReader(reader)
apiHandler, err := NewAPIHandler(&reader, dao.DB(), rm)
apiHandler, err := NewAPIHandler(APIHandlerOpts{
Reader: reader,
AppDao: dao.DB(),
RuleManager: rm,
})
if err != nil {
return nil, err
}

249
pkg/query-service/auth/auth.go
View File

@ -8,9 +8,11 @@ import (
"github.com/golang-jwt/jwt"
"github.com/google/uuid"
"github.com/pkg/errors"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/dao"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/dao"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/utils"
"go.uber.org/zap"
"golang.org/x/crypto/bcrypt"
)
@ -119,7 +121,7 @@ func GetInvite(ctx context.Context, token string) (*model.InvitationResponseObje
}, nil
}
func validateInvite(ctx context.Context, req *RegisterRequest) (*model.InvitationObject, error) {
func ValidateInvite(ctx context.Context, req *RegisterRequest) (*model.InvitationObject, error) {
invitation, err := dao.DB().GetInviteFromEmail(ctx, req.Email)
if err != nil {
return nil, errors.Wrap(err.Err, "Failed to read from DB")
@ -207,65 +209,43 @@ type RegisterRequest struct {
Email string `json:"email"`
Password string `json:"password"`
InviteToken string `json:"token"`
// reference URL to track where the register request is coming from
SourceUrl string `json:"sourceUrl"`
}
// Register registers a new user. For the first register request, it doesn't need an invite token
// and also the first registration is an enforced ADMIN registration. Every subsequent request will
// need an invite token to go through.
func Register(ctx context.Context, req *RegisterRequest) *model.ApiError {
func RegisterFirstUser(ctx context.Context, req *RegisterRequest) (*model.User, *model.ApiError) {
zap.S().Debugf("Got a register request for email: %v\n", req.Email)
// TODO(Ahsan): We should optimize it, shouldn't make an extra DB call everytime to know if
// this is the first register request.
users, apiErr := dao.DB().GetUsers(ctx)
if apiErr != nil {
zap.S().Debugf("GetUser failed, err: %v\n", apiErr.Err)
return apiErr
if req.Email == "" {
return nil, model.BadRequest(model.ErrEmailRequired{})
}
var groupName, orgId string
// If there are no user, then this first user is granted Admin role. Also, an org is created
// based on the request. Any other user can't use any other org name, if they do then
// registration will fail because of foreign key violation while create user.
// TODO(Ahsan): We need to re-work this logic for the case of multi-tenant system.
if len(users) == 0 {
org, apiErr := dao.DB().CreateOrg(ctx, &model.Organization{Name: req.OrgName})
if apiErr != nil {
zap.S().Debugf("CreateOrg failed, err: %v\n", apiErr.Err)
return apiErr
}
groupName = constants.AdminGroup
orgId = org.Id
if req.Password == "" {
return nil, model.BadRequest(model.ErrPasswordRequired{})
}
if len(users) > 0 {
inv, err := validateInvite(ctx, req)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorUnauthorized}
}
org, apiErr := dao.DB().GetOrgByName(ctx, req.OrgName)
if apiErr != nil {
zap.S().Debugf("GetOrgByName failed, err: %v\n", apiErr.Err)
return apiErr
}
groupName := constants.AdminGroup
groupName = inv.Role
if org != nil {
orgId = org.Id
}
org, apierr := dao.DB().CreateOrg(ctx,
&model.Organization{Name: req.OrgName})
if apierr != nil {
zap.S().Debugf("CreateOrg failed, err: %v\n", zap.Error(apierr.ToError()))
return nil, apierr
}
group, apiErr := dao.DB().GetGroupByName(ctx, groupName)
if apiErr != nil {
zap.S().Debugf("GetGroupByName failed, err: %v\n", apiErr.Err)
return apiErr
return nil, apiErr
}
hash, err := passwordHash(req.Password)
var hash string
var err error
hash, err = passwordHash(req.Password)
if err != nil {
return &model.ApiError{Err: err, Typ: model.ErrorUnauthorized}
zap.S().Errorf("failed to generate password hash when registering a user", zap.Error(err))
return nil, model.InternalError(model.ErrSignupFailed{})
}
user := &model.User{
@ -276,17 +256,118 @@ func Register(ctx context.Context, req *RegisterRequest) *model.ApiError {
CreatedAt: time.Now().Unix(),
ProfilePirctureURL: "", // Currently unused
GroupId: group.Id,
OrgId: orgId,
OrgId: org.Id,
}
return dao.DB().CreateUser(ctx, user)
}
// RegisterInvitedUser handles registering a invited user
func RegisterInvitedUser(ctx context.Context, req *RegisterRequest, nopassword bool) (*model.User, *model.ApiError) {
if req.InviteToken == "" {
return nil, model.BadRequest(fmt.Errorf("invite token is required"))
}
if !nopassword && req.Password == "" {
return nil, model.BadRequest(model.ErrPasswordRequired{})
}
invite, err := ValidateInvite(ctx, req)
if err != nil {
zap.S().Errorf("failed to validate invite token", err)
return nil, model.BadRequest(model.ErrSignupFailed{})
}
// checking if user email already exists, this is defensive but
// required as delete invitation and user creation dont happen
// in the same transaction at the end of this function
userPayload, apierr := dao.DB().GetUserByEmail(ctx, invite.Email)
if apierr != nil {
zap.S().Debugf("failed to get user by email", apierr.Err)
return nil, apierr
}
if userPayload != nil {
// user already exists
return &userPayload.User, nil
}
if invite.OrgId == "" {
zap.S().Errorf("failed to find org in the invite")
return nil, model.InternalError(fmt.Errorf("invalid invite, org not found"))
}
if invite.Role == "" {
// if role is not provided, default to viewer
invite.Role = constants.ViewerGroup
}
group, apiErr := dao.DB().GetGroupByName(ctx, invite.Role)
if apiErr != nil {
zap.S().Debugf("GetGroupByName failed, err: %v\n", apiErr.Err)
return nil, model.InternalError(model.ErrSignupFailed{})
}
var hash string
// check if password is not empty, as for SSO case it can be
if req.Password != "" {
hash, err = passwordHash(req.Password)
if err != nil {
zap.S().Errorf("failed to generate password hash when registering a user", zap.Error(err))
return nil, model.InternalError(model.ErrSignupFailed{})
}
} else {
hash, err = passwordHash(utils.GeneratePassowrd())
if err != nil {
zap.S().Errorf("failed to generate password hash when registering a user", zap.Error(err))
return nil, model.InternalError(model.ErrSignupFailed{})
}
}
user := &model.User{
Id: uuid.NewString(),
Name: req.Name,
Email: req.Email,
Password: hash,
CreatedAt: time.Now().Unix(),
ProfilePirctureURL: "", // Currently unused
GroupId: group.Id,
OrgId: invite.OrgId,
}
// TODO(Ahsan): Ideally create user and delete invitation should happen in a txn.
_, apiErr = dao.DB().CreateUser(ctx, user)
user, apiErr = dao.DB().CreateUser(ctx, user)
if apiErr != nil {
zap.S().Debugf("CreateUser failed, err: %v\n", apiErr.Err)
return apiErr
return nil, apiErr
}
return dao.DB().DeleteInvitation(ctx, user.Email)
apiErr = dao.DB().DeleteInvitation(ctx, user.Email)
if apiErr != nil {
zap.S().Debugf("delete invitation failed, err: %v\n", apiErr.Err)
return nil, apiErr
}
return user, nil
}
// Register registers a new user. For the first register request, it doesn't need an invite token
// and also the first registration is an enforced ADMIN registration. Every subsequent request will
// need an invite token to go through.
func Register(ctx context.Context, req *RegisterRequest) (*model.User, *model.ApiError) {
users, err := dao.DB().GetUsers(ctx)
if err != nil {
return nil, model.InternalError(fmt.Errorf("failed to get user count"))
}
switch len(users) {
case 0:
return RegisterFirstUser(ctx, req)
default:
return RegisterInvitedUser(ctx, req, false)
}
}
// Login method returns access and refresh tokens on successful login, else it errors out.
@ -299,39 +380,15 @@ func Login(ctx context.Context, request *model.LoginRequest) (*model.LoginRespon
return nil, err
}
accessJwtExpiry := time.Now().Add(JwtExpiry).Unix()
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"id": user.Id,
"gid": user.GroupId,
"email": user.Email,
"exp": accessJwtExpiry,
})
accessJwt, err := token.SignedString([]byte(JwtSecret))
userjwt, err := GenerateJWTForUser(&user.User)
if err != nil {
return nil, errors.Errorf("failed to encode jwt: %v", err)
}
refreshJwtExpiry := time.Now().Add(JwtRefresh).Unix()
token = jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"id": user.Id,
"gid": user.GroupId,
"email": user.Email,
"exp": refreshJwtExpiry,
})
refreshJwt, err := token.SignedString([]byte(JwtSecret))
if err != nil {
return nil, errors.Errorf("failed to encode jwt: %v", err)
zap.S().Debugf("Failed to generate JWT against login creds, %v", err)
return nil, err
}
return &model.LoginResponse{
AccessJwt: accessJwt,
AccessJwtExpiry: accessJwtExpiry,
RefreshJwt: refreshJwt,
RefreshJwtExpiry: refreshJwtExpiry,
UserId: user.Id,
UserJwtObject: userjwt,
UserId: user.User.Id,
}, nil
}
@ -375,3 +432,35 @@ func passwordMatch(hash, password string) bool {
}
return true
}
func GenerateJWTForUser(user *model.User) (model.UserJwtObject, error) {
j := model.UserJwtObject{}
var err error
j.AccessJwtExpiry = time.Now().Add(JwtExpiry).Unix()
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"id": user.Id,
"gid": user.GroupId,
"email": user.Email,
"exp": j.AccessJwtExpiry,
})
j.AccessJwt, err = token.SignedString([]byte(JwtSecret))
if err != nil {
return j, errors.Errorf("failed to encode jwt: %v", err)
}
j.RefreshJwtExpiry = time.Now().Add(JwtRefresh).Unix()
token = jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
"id": user.Id,
"gid": user.GroupId,
"email": user.Email,
"exp": j.RefreshJwtExpiry,
})
j.RefreshJwt, err = token.SignedString([]byte(JwtSecret))
if err != nil {
return j, errors.Errorf("failed to encode jwt: %v", err)
}
return j, nil
}

2
pkg/query-service/auth/jwt.go
View File

@ -8,7 +8,7 @@ import (
jwtmiddleware "github.com/auth0/go-jwt-middleware"
"github.com/golang-jwt/jwt"
"github.com/pkg/errors"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
"google.golang.org/grpc/metadata"
)

6
pkg/query-service/auth/rbac.go
View File

@ -7,9 +7,9 @@ import (
"regexp"
"github.com/pkg/errors"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/dao"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/dao"
"go.signoz.io/signoz/pkg/query-service/model"
)
type Group struct {

4
pkg/query-service/auth/utils.go
View File

@ -5,8 +5,8 @@ import (
"encoding/hex"
"github.com/pkg/errors"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
)
var (

2
pkg/query-service/constants/constants.go
View File

@ -4,7 +4,7 @@ import (
"os"
"strconv"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/model"
)
const (

7
pkg/query-service/dao/factory.go
View File

@ -4,7 +4,7 @@ import (
"fmt"
"github.com/pkg/errors"
"go.signoz.io/query-service/dao/sqlite"
"go.signoz.io/signoz/pkg/query-service/dao/sqlite"
)
var db ModelDao
@ -24,6 +24,11 @@ func InitDao(engine, path string) error {
return nil
}
// SetDB is used by ee for setting modelDAO
func SetDB(m ModelDao) {
db = m
}
func DB() ModelDao {
if db == nil {
// Should never reach here

2
pkg/query-service/dao/interface.go
View File

@ -3,7 +3,7 @@ package dao
import (
"context"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/model"
)
type ModelDao interface {

11
pkg/query-service/dao/sqlite/connection.go
View File

@ -6,9 +6,9 @@ import (
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
"go.signoz.io/query-service/telemetry"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/telemetry"
"go.uber.org/zap"
)
@ -88,6 +88,11 @@ func InitDB(dataSourceName string) (*ModelDaoSqlite, error) {
return mds, nil
}
// DB returns database connection
func (mds *ModelDaoSqlite) DB() *sqlx.DB {
return mds.db
}
// initializeOrgPreferences initializes in-memory telemetry settings. It is planned to have
// multiple orgs in the system. In case of multiple orgs, there will be separate instance
// of in-memory telemetry for each of the org, having their own settings. As of now, we only

11
pkg/query-service/dao/sqlite/rbac.go
View File

@ -7,8 +7,8 @@ import (
"github.com/google/uuid"
"github.com/pkg/errors"
"go.signoz.io/query-service/model"
"go.signoz.io/query-service/telemetry"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/telemetry"
)
func (mds *ModelDaoSqlite) CreateInviteEntry(ctx context.Context,
@ -291,6 +291,13 @@ func (mds *ModelDaoSqlite) GetUser(ctx context.Context,
func (mds *ModelDaoSqlite) GetUserByEmail(ctx context.Context,
email string) (*model.UserPayload, *model.ApiError) {
if email == "" {
return nil, &model.ApiError{
Typ: model.ErrorBadData,
Err: fmt.Errorf("empty email address"),
}
}
users := []model.UserPayload{}
query := `select
u.id,

7
pkg/query-service/integrations/alertManager/manager.go
View File

@ -5,11 +5,12 @@ import (
"bytes"
"encoding/json"
"fmt"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/model"
"go.uber.org/zap"
"net/http"
neturl "net/url"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
)
const contentType = "application/json"

3
pkg/query-service/integrations/alertManager/model.go
View File

@ -2,8 +2,9 @@ package alertManager
import (
"fmt"
"go.signoz.io/query-service/utils/labels"
"time"
"go.signoz.io/signoz/pkg/query-service/utils/labels"
)
// Receiver configuration provides configuration on how to contact a receiver.

10
pkg/query-service/interfaces/featureLookup.go Normal file
View File

@ -0,0 +1,10 @@
package interfaces
import (
"go.signoz.io/signoz/pkg/query-service/model"
)
type FeatureLookup interface {
CheckFeature(f string) error
GetFeatureFlags() model.FeatureSet
}

4
pkg/query-service/interfaces/interface.go
View File

@ -7,8 +7,8 @@ import (
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/storage"
"github.com/prometheus/prometheus/util/stats"
am "go.signoz.io/query-service/integrations/alertManager"
"go.signoz.io/query-service/model"
am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
"go.signoz.io/signoz/pkg/query-service/model"
)
type Reader interface {

8
pkg/query-service/main.go
View File

@ -7,10 +7,10 @@ import (
"os/signal"
"syscall"
"go.signoz.io/query-service/app"
"go.signoz.io/query-service/auth"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/version"
"go.signoz.io/signoz/pkg/query-service/app"
"go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/version"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"

8
pkg/query-service/model/auth.go
View File

@ -32,12 +32,16 @@ type LoginRequest struct {
RefreshToken string `json:"refreshToken"`
}
type LoginResponse struct {
type UserJwtObject struct {
AccessJwt string `json:"accessJwt"`
AccessJwtExpiry int64 `json:"accessJwtExpiry"`
RefreshJwt string `json:"refreshJwt"`
RefreshJwtExpiry int64 `json:"refreshJwtExpiry"`
UserId string `json:"userId"`
}
type LoginResponse struct {
UserJwtObject
UserId string `json:"userId"`
}
type ChangePasswordRequest struct {

1
pkg/query-service/model/db.go
View File

@ -8,6 +8,7 @@ type Organization struct {
HasOptedUpdates bool `json:"hasOptedUpdates" db:"has_opted_updates"`
}
// InvitationObject represents the token object stored in the db
type InvitationObject struct {
Id string `json:"id" db:"id"`
Email string `json:"email" db:"email"`

36
pkg/query-service/model/errors.go Normal file
View File

@ -0,0 +1,36 @@
package model
import "fmt"
// custom errors related to registration
type ErrFeatureUnavailable struct {
Key string
}
func (errFeatureUnavailable ErrFeatureUnavailable) Error() string {
return fmt.Sprintf("feature unavailable: %s", errFeatureUnavailable.Key)
}
type ErrEmailRequired struct{}
func (errEmailRequired ErrEmailRequired) Error() string {
return "email is required"
}
type ErrPasswordRequired struct{}
func (errPasswordRequired ErrPasswordRequired) Error() string {
return "password is required"
}
type ErrSignupFailed struct{}
func (errSignupFailed ErrSignupFailed) Error() string {
return "failed to register user"
}
type ErrNoOrgFound struct{}
func (errNoOrgFound ErrNoOrgFound) Error() string {
return "no org found"
}

9
pkg/query-service/model/featureSet.go Normal file
View File

@ -0,0 +1,9 @@
package model
type FeatureSet map[string]bool
const Basic = "BASIC_PLAN"
var BasicPlan = FeatureSet{
Basic: true,
}

46
pkg/query-service/model/response.go
View File

@ -12,10 +12,40 @@ import (
"k8s.io/apimachinery/pkg/labels"
)
type BaseApiError interface {
Type() ErrorType
ToError() error
Error() string
IsNil() bool
}
type ApiError struct {
Typ ErrorType
Err error
}
func (a *ApiError) Type() ErrorType {
return a.Typ
}
func (a *ApiError) ToError() error {
if a != nil {
return a.Err
}
return a
}
func (a *ApiError) Error() string {
if a == nil || a.Err == nil {
return ""
}
return a.Err.Error()
}
func (a *ApiError) IsNil() bool {
return a == nil || a.Err == nil
}
type ErrorType string
const (
@ -34,6 +64,22 @@ const (
ErrorStreamingNotSupported ErrorType = "streaming is not supported"
)
// BadRequest returns a ApiError object of bad request
func BadRequest(err error) *ApiError {
return &ApiError{
Typ: ErrorBadData,
Err: err,
}
}
// InternalError returns a ApiError object of internal type
func InternalError(err error) *ApiError {
return &ApiError{
Typ: ErrorInternal,
Err: err,
}
}
type QueryDataV2 struct {
ResultType promql.ValueType `json:"resultType"`
Result promql.Value `json:"result"`

2
pkg/query-service/pqlEngine/engine.go
View File

@ -13,7 +13,7 @@ import (
pql "github.com/prometheus/prometheus/promql"
pstorage "github.com/prometheus/prometheus/storage"
premote "github.com/prometheus/prometheus/storage/remote"
"go.signoz.io/query-service/interfaces"
"go.signoz.io/signoz/pkg/query-service/interfaces"
)
type PqlEngine struct {

7
pkg/query-service/rules/alerting.go
View File

@ -3,12 +3,13 @@ package rules
import (
"encoding/json"
"fmt"
"github.com/pkg/errors"
"go.signoz.io/query-service/model"
"go.signoz.io/query-service/utils/labels"
"net/url"
"strings"
"time"
"github.com/pkg/errors"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/utils/labels"
)
// this file contains common structs and methods used by

11
pkg/query-service/rules/apiParams.go
View File

@ -4,14 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/pkg/errors"
"go.signoz.io/query-service/model"
"go.uber.org/zap"
"time"
"unicode/utf8"
"go.signoz.io/query-service/utils/times"
"go.signoz.io/query-service/utils/timestamp"
"github.com/pkg/errors"
"go.signoz.io/signoz/pkg/query-service/model"
"go.uber.org/zap"
"go.signoz.io/signoz/pkg/query-service/utils/times"
"go.signoz.io/signoz/pkg/query-service/utils/timestamp"
yaml "gopkg.in/yaml.v2"
)

11
pkg/query-service/rules/manager.go
View File

@ -4,13 +4,14 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/google/uuid"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/go-kit/log"
"go.uber.org/zap"
@ -19,9 +20,9 @@ import (
"github.com/pkg/errors"
// opentracing "github.com/opentracing/opentracing-go"
am "go.signoz.io/query-service/integrations/alertManager"
"go.signoz.io/query-service/model"
"go.signoz.io/query-service/utils/labels"
am "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/utils/labels"
)
// namespace for prom metrics
@ -241,7 +242,7 @@ func (m *Manager) EditRule(ruleStr string, id string) error {
func (m *Manager) editTask(rule *PostableRule, taskName string) error {
m.mtx.Lock()
defer m.mtx.Unlock()
zap.S().Debugf("msg:", "editing a rule task", "\t task name:", taskName)
newTask, err := m.prepareTask(false, rule, taskName)

155
pkg/query-service/rules/manager_test.go
View File

@ -1,155 +0,0 @@
package rules
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"go.signoz.io/query-service/app/clickhouseReader"
am "go.signoz.io/query-service/integrations/alertManager"
"go.signoz.io/query-service/model"
pqle "go.signoz.io/query-service/pqlEngine"
"go.signoz.io/query-service/utils/value"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"net/url"
"testing"
"time"
)
func initZapLog() *zap.Logger {
config := zap.NewDevelopmentConfig()
config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
config.EncoderConfig.TimeKey = "timestamp"
config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
logger, _ := config.Build()
return logger
}
func TestRules(t *testing.T) {
fmt.Println("starting test TestRules..")
loggerMgr := initZapLog()
zap.ReplaceGlobals(loggerMgr)
defer loggerMgr.Sync() // flushes buffer, if any
logger := loggerMgr.Sugar()
configFile := "../config/prometheus.yml"
// create engine
pqle, err := pqle.FromConfigPath(configFile)
if err != nil {
fmt.Println("failed to create pql:", err)
t.Errorf("failed to create pql engine : %v", err)
}
// create db conn
db, err := sqlx.Open("sqlite3", "../signoz.db")
if err != nil {
fmt.Println("failed to create db conn:", err)
t.Errorf("failed to create db conn: %v", err)
}
// create ch reader
ch := clickhouseReader.NewReader(db, configFile)
// notifier opts
notifierOpts := am.NotifierOptions{
QueueCapacity: 10000,
Timeout: 1 * time.Second,
AlertManagerURLs: []string{"http://localhost:9093/api/"},
}
externalURL, _ := url.Parse("http://signoz.io")
// create manager opts
managerOpts := &ManagerOptions{
NotifierOpts: notifierOpts,
Queriers: &Queriers{
PqlEngine: pqle,
Ch: ch,
},
ExternalURL: externalURL,
Conn: db,
Context: context.Background(),
Logger: nil,
}
// create Manager
manager, err := NewManager(managerOpts)
if err != nil {
fmt.Println("manager error:", err)
t.Errorf("manager error: %v", err)
}
fmt.Println("manager is ready:", manager)
manager.run()
// test rules
// create promql rule
/* promql rule
postableRule := PostableRule{
Alert: "test alert 1 - promql",
RuleType: RuleTypeProm,
EvalWindow: 5 * time.Minute,
Frequency: 30 * time.Second,
RuleCondition: RuleCondition{
CompositeMetricQuery: &model.CompositeMetricQuery{
QueryType: model.PROM,
PromQueries: map[string]*model.PromQuery{
"A": &model.PromQuery{Query: `sum(signoz_latency_count{span_kind="SPAN_KIND_SERVER"}) by (service_name) > 100`},
},
},
},
Labels: map[string]string{},
Annotations: map[string]string{},
}*/
// create builder rule
metricQuery := &model.MetricQuery{
QueryName: "A",
MetricName: "signoz_latency_count",
TagFilters: &model.FilterSet{Operation: "AND", Items: []model.FilterItem{
{Key: "span_kind", Value: "SPAN_KIND_SERVER", Operation: "neq"},
}},
GroupingTags: []string{"service_name"},
AggregateOperator: model.RATE_SUM,
Expression: "A",
}
postableRule := PostableRule{
Alert: "test alert 2 - builder",
RuleType: RuleTypeThreshold,
EvalWindow: 5 * time.Minute,
Frequency: 30 * time.Second,
RuleCondition: RuleCondition{
Target: value.Float64(500),
CompareOp: TargetIsMore,
CompositeMetricQuery: &model.CompositeMetricQuery{
QueryType: model.QUERY_BUILDER,
BuilderQueries: map[string]*model.MetricQuery{
"A": metricQuery,
},
},
},
Labels: map[string]string{"host": "server1"},
Annotations: map[string]string{},
}
err = manager.addTask(&postableRule, postableRule.Alert)
if err != nil {
fmt.Println("failed to add rule: ", err)
t.Errorf("failed to add rule")
}
signalsChannel := make(chan os.Signal, 1)
signal.Notify(signalsChannel, os.Interrupt, syscall.SIGTERM)
for {
select {
case <-signalsChannel:
logger.Fatal("Received OS Interrupt Signal ... ")
}
}
}

15
pkg/query-service/rules/promRule.go
View File

@ -3,18 +3,19 @@ package rules
import (
"context"
"fmt"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"go.uber.org/zap"
"sync"
"time"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"go.uber.org/zap"
plabels "github.com/prometheus/prometheus/pkg/labels"
pql "github.com/prometheus/prometheus/promql"
"go.signoz.io/query-service/model"
qslabels "go.signoz.io/query-service/utils/labels"
"go.signoz.io/query-service/utils/times"
"go.signoz.io/query-service/utils/timestamp"
"go.signoz.io/signoz/pkg/query-service/model"
qslabels "go.signoz.io/signoz/pkg/query-service/utils/labels"
"go.signoz.io/signoz/pkg/query-service/utils/times"
"go.signoz.io/signoz/pkg/query-service/utils/timestamp"
yaml "gopkg.in/yaml.v2"
)

2
pkg/query-service/rules/queriers.go
View File

@ -2,7 +2,7 @@ package rules
import (
"github.com/ClickHouse/clickhouse-go/v2"
pqle "go.signoz.io/query-service/pqlEngine"
pqle "go.signoz.io/signoz/pkg/query-service/pqlEngine"
)
// Queriers register the options for querying metrics or event sources

2
pkg/query-service/rules/resultTypes.go
View File

@ -5,7 +5,7 @@ import (
"fmt"
"strconv"
"go.signoz.io/query-service/utils/labels"
"go.signoz.io/signoz/pkg/query-service/utils/labels"
)
// common result format of query

3
pkg/query-service/rules/rule.go
View File

@ -2,8 +2,9 @@ package rules
import (
"context"
"go.signoz.io/query-service/utils/labels"
"time"
"go.signoz.io/signoz/pkg/query-service/utils/labels"
)
// A Rule encapsulates a vector expression which is evaluated at a specified

7
pkg/query-service/rules/ruleTask.go
View File

@ -3,12 +3,13 @@ package rules
import (
"context"
"fmt"
opentracing "github.com/opentracing/opentracing-go"
"go.signoz.io/query-service/utils/labels"
"go.uber.org/zap"
"sort"
"sync"
"time"
opentracing "github.com/opentracing/opentracing-go"
"go.signoz.io/signoz/pkg/query-service/utils/labels"
"go.uber.org/zap"
)
// RuleTask holds a rule (with composite queries)

2
pkg/query-service/rules/templates.go
View File

@ -14,7 +14,7 @@ import (
html_template "html/template"
text_template "text/template"
"go.signoz.io/query-service/utils/times"
"go.signoz.io/signoz/pkg/query-service/utils/times"
)
// this file contains all the methods and structs

17
pkg/query-service/rules/thresholdRule.go
View File

@ -3,21 +3,22 @@ package rules
import (
"context"
"fmt"
"go.uber.org/zap"
"math"
"reflect"
"sort"
"sync"
"time"
"go.uber.org/zap"
"github.com/ClickHouse/clickhouse-go/v2"
"go.signoz.io/query-service/app/metrics"
"go.signoz.io/query-service/constants"
qsmodel "go.signoz.io/query-service/model"
"go.signoz.io/query-service/utils/labels"
"go.signoz.io/query-service/utils/times"
"go.signoz.io/query-service/utils/timestamp"
"go.signoz.io/query-service/utils/value"
"go.signoz.io/signoz/pkg/query-service/app/metrics"
"go.signoz.io/signoz/pkg/query-service/constants"
qsmodel "go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/utils/labels"
"go.signoz.io/signoz/pkg/query-service/utils/times"
"go.signoz.io/signoz/pkg/query-service/utils/timestamp"
"go.signoz.io/signoz/pkg/query-service/utils/value"
yaml "gopkg.in/yaml.v2"
)

11
pkg/query-service/telemetry/telemetry.go
View File

@ -11,10 +11,10 @@ import (
"time"
ph "github.com/posthog/posthog-go"
"go.signoz.io/query-service/constants"
"go.signoz.io/query-service/interfaces"
"go.signoz.io/query-service/model"
"go.signoz.io/query-service/version"
"go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/interfaces"
"go.signoz.io/signoz/pkg/query-service/model"
"go.signoz.io/signoz/pkg/query-service/version"
"gopkg.in/segmentio/analytics-go.v3"
)
@ -27,6 +27,9 @@ const (
TELEMETRY_EVENT_HEART_BEAT = "Heart Beat"
TELEMETRY_EVENT_ORG_SETTINGS = "Org Settings"
DEFAULT_SAMPLING = 0.1
TELEMETRY_LICENSE_CHECK_FAILED = "License Check Failed"
TELEMETRY_LICENSE_UPDATED = "License Updated"
TELEMETRY_LICENSE_ACT_FAILED = "License Activation Failed"
)
const api_key = "4Gmoa4ixJAUHx2BpJxsjwA1bEfnwEeRz"

4
pkg/query-service/tests/auth_test.go
View File

@ -9,8 +9,8 @@ import (
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"go.signoz.io/query-service/auth"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/auth"
"go.signoz.io/signoz/pkg/query-service/model"
)
func invite(t *testing.T, email string) *model.InviteResponse {

2
pkg/query-service/tests/cold_storage_test.go
View File

@ -9,7 +9,7 @@ import (
"time"
"github.com/stretchr/testify/require"
"go.signoz.io/query-service/model"
"go.signoz.io/signoz/pkg/query-service/model"
)
const (

45
pkg/query-service/utils/encryption/encryption.go Normal file
View File

@ -0,0 +1,45 @@
package encryption
import (
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"errors"
"io"
)
func Encrypt(key, text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
b := base64.StdEncoding.EncodeToString(text)
ciphertext := make([]byte, aes.BlockSize+len(b))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
return nil, err
}
cfb := cipher.NewCFBEncrypter(block, iv)
cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
return ciphertext, nil
}
func Decrypt(key, text []byte) ([]byte, error) {
block, err := aes.NewCipher(key)
if err != nil {
return nil, err
}
if len(text) < aes.BlockSize {
return nil, errors.New("ciphertext too short")
}
iv := text[:aes.BlockSize]
text = text[aes.BlockSize:]
cfb := cipher.NewCFBDecrypter(block, iv)
cfb.XORKeyStream(text, text)
data, err := base64.StdEncoding.DecodeString(string(text))
if err != nil {
return nil, err
}
return data, nil
}

10
pkg/query-service/utils/pass.go Normal file
View File

@ -0,0 +1,10 @@
package utils
import (
"github.com/sethvargo/go-password/password"
)
func GeneratePassowrd() string {
res, _ := password.Generate(64, 10, 10, false, false)
return res
}