Merge branch 'master' into config-logging

go.mod

@@ -2,6 +2,7 @@ module git.rwth-aachen.de/acs/public/villas/web-backend-go
 
 require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
+	github.com/chenjiandongx/ginprom v0.0.0-20191022035802-6f3da3c84986
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/gin-gonic/gin v1.4.0
 	github.com/go-ini/ini v1.51.0 // indirect

go.sum

@@ -22,10 +22,15 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
 github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
+github.com/chenjiandongx/ginprom v0.0.0-20191022035802-6f3da3c84986 h1:6r9/zDxh+oMzqLWol3ByjfoZN85AaCTnXNvEfvc316U=
+github.com/chenjiandongx/ginprom v0.0.0-20191022035802-6f3da3c84986/go.mod h1:lINNCb1ZH3c0uL/9ApaQ8muR4QILsi0STj8Ojt8ZmwU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -147,6 +152,7 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
 github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -168,19 +174,26 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
 github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg=
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
 github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

k8s_test.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+kubectl -n services port-forward svc/postgres-postgresql 5432:5432 & PF=$!
+
+POSTGRES_PASS=$(kubectl -n villas-demo get secret postgres-credentials -o json | jq -r .data.password | base64 -d)
+
+go run start.go -dbhost localhost -dbname villas -dbuser villas -dbpass ${POSTGRES_PASS}
+
+kill $PF
+wait $PF

routes/metrics/metrics_endpoint.go

@@ -0,0 +1,103 @@
+package metrics
+
+import (
+	"github.com/chenjiandongx/ginprom"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/gin-gonic/gin"
+	"git.rwth-aachen.de/acs/public/villas/web-backend-go/database"
+	"github.com/jinzhu/gorm"
+)
+
+var (
+	SimulatorCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "simulators",
+			Help: "A counter for the total number of simulators",
+		},
+	)
+
+	SimulationModelCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "simulation_models",
+			Help: "A counter for the total number of simulation models",
+		},
+	)
+
+	FileCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "files",
+			Help: "A counter for the total number of files",
+		},
+	)
+
+	ScenarioCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "scenarios",
+			Help: "A counter for the total number of scenarios",
+		},
+	)
+
+	UserCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "users",
+			Help: "A counter for the total number of users",
+		},
+	)
+
+	DashboardCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Name: "dashboards",
+			Help: "A counter for the total number of dashboards",
+		},
+	)
+)
+
+func RegisterMetricsEndpoint(rg *gin.RouterGroup) {
+	// use prometheus metrics exporter middleware.
+	//
+	// ginprom.PromMiddleware() expects a ginprom.PromOpts{} poniter.
+	// It was used for filtering labels with regex. `nil` will pass every requests.
+	//
+	// ginprom promethues-labels: 
+	//   `status`, `endpoint`, `method`
+	//
+	// for example:
+	// 1). I want not to record the 404 status request. That's easy for it.
+	// ginprom.PromMiddleware(&ginprom.PromOpts{ExcludeRegexStatus: "404"})
+	//
+	// 2). And I wish ignore endpoint start with `/prefix`.
+	// ginprom.PromMiddleware(&ginprom.PromOpts{ExcludeRegexEndpoint: "^/prefix"})
+	r := gin.Default()
+	r.Use(ginprom.PromMiddleware(nil))
+
+	rg.GET("", ginprom.PromHandler(promhttp.Handler()))
+
+	// Register metrics
+	prometheus.MustRegister(
+		SimulatorCounter,
+		SimulationModelCounter,
+		FileCounter,
+		ScenarioCounter,
+		UserCounter,
+		DashboardCounter,
+	)
+}
+
+func InitCounters(db *gorm.DB) {
+	var simulators, simulation_models, files, scenarios, users, dashboards float64;
+
+	db.Model(&database.Simulator{}).Count(&simulators)
+	db.Model(&database.SimulationModel{}).Count(&simulation_models)
+	db.Model(&database.File{}).Count(&files)
+	db.Model(&database.Scenario{}).Count(&scenarios)
+	db.Model(&database.User{}).Count(&users)
+	db.Model(&database.Dashboard{}).Count(&dashboards)
+
+	SimulatorCounter.Add(simulators)
+	SimulationModelCounter.Add(simulation_models)
+	FileCounter.Add(files)
+	ScenarioCounter.Add(scenarios)
+	UserCounter.Add(users)
+	DashboardCounter.Add(dashboards)
+}

start.go

@@ -22,6 +22,7 @@ import (
 	"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/simulator"
 	"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/user"
 	"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/widget"
+	"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/metrics"
 )
 
 // @title VILLASweb Backend API
@@ -50,6 +51,8 @@ func main() {
 	docs.SwaggerInfo.Host = baseHost
 	docs.SwaggerInfo.BasePath = basePath
 
+	metrics.InitCounters(db)
+
 	r := gin.Default()
 
 	api := r.Group(basePath)
@@ -68,7 +71,8 @@ func main() {
 	file.RegisterFileEndpoints(api.Group("/files"))
 	user.RegisterUserEndpoints(api.Group("/users"))
 	simulator.RegisterSimulatorEndpoints(api.Group("/simulators"))
-	healthz.RegisterHealthzEndpoint(api.Group("/healthz"))
+	healthz.RegisterHealthzEndpoint(r.Group("/healthz"))
+	metrics.RegisterMetricsEndpoint(r.Group("/metrics"))
 
 	r.GET("swagger/*any", ginSwagger.WrapHandler(swaggerFiles.Handler))