Mirrors/VILLASweb-backend-go
1
0
Fork 0
mirror of https://git.rwth-aachen.de/acs/public/villas/web-backend-go/ synced 2025-03-30 00:00:12 +01:00
Code Issues Releases Wiki Activity

WIP: rework the demo data reading via config parameter(s)

Browse source
This commit is contained in:
Sonja Happ 2021-04-29 12:45:55 +02:00
parent e8673fad31
commit b1c08f16b0
4 changed files with 266 additions and 153 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
configuration/config.go
View file

@ -27,6 +27,7 @@ import (
"io/ioutil"
"log"
"os"
"sort"
"strings"
"gopkg.in/yaml.v3"
@ -50,11 +51,11 @@ func InitConfig() error {
dbUser = flag.String("db-user", "", "Username of database connection (default is <empty>)")
dbPass = flag.String("db-pass", "", "Password of database connection (default is <empty>)")
dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production
dbClear = flag.Bool("db-clear", false, "Set to true if you want to clear all DB tables upon startup. This parameter has to be used with great care, its effects cannot be reverted.")
amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)")
amqpUser = flag.String("amqp-user", "", "Username for AMQP broker")
amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker")
configFile = flag.String("config", "", "Path to YAML configuration file")
mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)")
port = flag.String("port", "4000", "Port of the backend (default is 4000)")
adminUser = flag.String("admin-user", "", "Initial admin username")
adminPass = flag.String("admin-pass", "", "Initial admin password")
@ -74,7 +75,7 @@ func InitConfig() error {
subTitle = flag.String("sub-title", "", "Sub-title shown in the frontend")
contactName = flag.String("contact-name", "Steffen Vogel", "Name of the administrative contact")
contactMail = flag.String("contact-mail", "svogel2@eonerc.rwth-aachen.de", "EMail of the administrative contact")
testDataPath = flag.String("test-data-path", "database/testdata.json", "The path to the test data json file (used in test mode)")
testDataPath = flag.String("test-data-path", "", "The path to a test data json file")
groupsPath = flag.String("groups-path", "configuration/groups.yaml", "The path to a YAML file that maps user groups to scenario IDs")
)
flag.Parse()
@ -88,7 +89,6 @@ func InitConfig() error {
"amqp.host": *amqpHost,
"amqp.user": *amqpUser,
"amqp.pass": *amqpPass,
"mode": *mode,
"port": *port,
"admin.user": *adminUser,
"admin.pass": *adminPass,
@ -107,6 +107,13 @@ func InitConfig() error {
"contact.mail": *contactMail,
"test.datapath": *testDataPath,
"groups.path": *groupsPath,
"config.file": *configFile,
}
if *dbClear == true {
static["db.clear"] = "true"
} else {
static["db.clear"] = "false"
}
if *s3NoSSL == true {
@ -152,20 +159,20 @@ func InitConfig() error {
return err
}
m, err := GlobalConfig.String("mode")
if err != nil {
return err
}
settings, _ := GlobalConfig.Settings()
if m != "test" {
settings, _ := GlobalConfig.Settings()
log.Print("All settings:")
for key, val := range settings {
// TODO password settings should be excluded!
log.Printf(" %s = %s \n", key, val)
keys := make([]string, 0, len(settings))
for k := range settings {
keys = append(keys, k)
}
sort.Strings(keys)
log.Print("All settings (except for PW and secrets):")
for _, k := range keys {
if !strings.Contains(k, "pass") && !strings.Contains(k, "secret") {
log.Printf(" %s = %s \n", k, settings[k])
}
}
return nil
}

40
database/database.go
View file

@ -1,4 +1,4 @@
/** Database package.
/** Package database
*
* @author Sonja Happ <sonja.happ@eonerc.rwth-aachen.de>
* @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC
@ -36,8 +36,8 @@ import (
var DBpool *gorm.DB // database used by backend
// Initialize connection to the database
func InitDB(cfg *config.Config) error {
// InitDB Initialize connection to the database
func InitDB(cfg *config.Config, dbClear string) error {
name, err := cfg.String("db.name")
if err != nil {
return err
@ -73,20 +73,25 @@ func InitDB(cfg *config.Config) error {
}
DBpool = db
// drop tables if parameter set
if dbClear == "true" {
DropTables()
log.Println("Database tables dropped")
}
MigrateModels()
log.Println("Database connection established")
return nil
}
// Connection pool to already opened DB
// GetDB Connection pool to already opened DB
func GetDB() *gorm.DB {
return DBpool
}
// Drop all the tables of the database
// TODO: Remove that function from the codebase and substitute the body
// to the Dummy*() where it is called
// DropTables drops all the tables of the database (use with care!)
func DropTables() {
DBpool.DropTableIfExists(&InfrastructureComponent{})
DBpool.DropTableIfExists(&Signal{})
@ -101,7 +106,7 @@ func DropTables() {
DBpool.DropTableIfExists("user_scenarios")
}
// AutoMigrate the models
// MigrateModels AutoMigrate the models
func MigrateModels() {
DBpool.AutoMigrate(&InfrastructureComponent{})
DBpool.AutoMigrate(&Signal{})
@ -115,26 +120,27 @@ func MigrateModels() {
}
// DBAddAdminUser adds a default admin user to the DB
func DBAddAdminUser(cfg *config.Config) (error, string) {
func DBAddAdminUser(cfg *config.Config) error {
DBpool.AutoMigrate(User{})
// Check if admin user exists in DB
var users []User
err := DBpool.Where("Role = ?", "Admin").Find(&users).Error
adminPW := ""
adminName := ""
if len(users) == 0 {
fmt.Println("No admin user found in DB, adding default admin user.")
name, err := cfg.String("admin.user")
if err != nil || name == "" {
name = "admin"
adminName, err = cfg.String("admin.user")
if err != nil || adminName == "" {
adminName = "admin"
}
adminPW, err = cfg.String("admin.pass")
if err != nil || adminPW == "" {
adminPW = generatePassword(16)
fmt.Printf(" Generated admin password: %s\n", adminPW)
fmt.Printf(" Generated admin password: %s for admin user %s\n", adminPW, adminName)
}
mail, err := cfg.String("admin.mail")
@ -145,14 +151,16 @@ func DBAddAdminUser(cfg *config.Config) (error, string) {
pwEnc, _ := bcrypt.GenerateFromPassword([]byte(adminPW), 10)
// create a copy of global test data
user := User{Username: name, Password: string(pwEnc),
user := User{Username: adminName, Password: string(pwEnc),
Role: "Admin", Mail: mail, Active: true}
// add admin user to DB
err = DBpool.Create(&user).Error
if err != nil {
return err
}
}
return err, adminPW
return nil
}
func generatePassword(Len int) string {

281
routes/register.go
View file

@ -102,41 +102,55 @@ func RegisterEndpoints(router *gin.Engine, api *gin.RouterGroup) {
}
// Read test data from JSON file (path set by ENV variable or command line param)
// ReadTestDataFromJson Reads test data from JSON file (path set by ENV variable or command line param)
func ReadTestDataFromJson(path string) error {
jsonFile, err := os.Open(path)
if err != nil {
return fmt.Errorf("error opening json file: %v", err)
}
log.Println("Successfully opened json data file", path)
_, err := os.Stat(path)
defer jsonFile.Close()
if err == nil {
byteValue, _ := ioutil.ReadAll(jsonFile)
jsonFile, err := os.Open(path)
if err != nil {
return fmt.Errorf("error opening json file: %v", err)
}
log.Println("Successfully opened json data file", path)
err = json.Unmarshal(byteValue, &GlobalTestData)
if err != nil {
return fmt.Errorf("error unmarshalling json: %v", err)
defer jsonFile.Close()
byteValue, _ := ioutil.ReadAll(jsonFile)
err = json.Unmarshal(byteValue, &GlobalTestData)
if err != nil {
return fmt.Errorf("error unmarshalling json: %v", err)
}
} else if os.IsNotExist(err) {
log.Println("Test data file does not exist, no test data added to DB:", path)
return nil
} else {
log.Println("Something is wrong with this file path:", path)
return nil
}
return nil
}
// Uses API endpoints to add test data to the backend; All endpoints have to be registered before invoking this function.
// AddTestData Uses API endpoints to add test data to the backend; All endpoints have to be registered before invoking this function.
func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) {
database.MigrateModels()
// add Admin user (defaults to User_0, xyz789)
err, adminpw := database.DBAddAdminUser(cfg)
var Admin = helper.Credentials{
Username: "admin",
Password: adminpw,
adminPW, err := cfg.String("admin.pass")
if err != nil {
log.Println("WARNING: cannot add test data: ", err)
return nil, nil
}
adminName, err := cfg.String("admin.user")
if err != nil {
log.Println("WARNING: cannot add test data: ", err)
return nil, nil
}
if err != nil {
return nil, err
var Admin = helper.Credentials{
Username: adminName,
Password: adminPW,
}
// authenticate as admin
@ -147,11 +161,22 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
basePath := "/api/v2"
db := database.GetDB()
// add users
for _, u := range GlobalTestData.Users {
code, resp, err := helper.TestEndpoint(router, token, basePath+"/users", "POST", helper.KeyModels{"user": u})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding user %v: %v", u.Username, err)
var x []user.User
err = db.Find(&x, "Username = ?", u.Username).Error
if err != nil {
return nil, err
}
if len(x) == 0 {
code, resp, err := helper.TestEndpoint(router, token, basePath+"/users", "POST", helper.KeyModels{"user": u})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding user %v: %v", u.Username, err)
}
}
}
@ -161,26 +186,44 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
for _, i := range GlobalTestData.ICs {
if (i.ManagedExternally && amqphost != "") || !i.ManagedExternally {
code, resp, err := helper.TestEndpoint(router, token, basePath+"/ic", "POST", helper.KeyModels{"ic": i})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding IC %v: %v", i.Name, err)
var x []infrastructure_component.InfrastructureComponent
err = db.Find(&x, "Name = ?", i.Name).Error
if err != nil {
return nil, err
}
if len(x) == 0 {
code, resp, err := helper.TestEndpoint(router, token, basePath+"/ic", "POST", helper.KeyModels{"ic": i})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding IC %v: %v", i.Name, err)
}
counterICs++
}
counterICs++
}
}
// add scenarios
for _, s := range GlobalTestData.Scenarios {
code, resp, err := helper.TestEndpoint(router, token, basePath+"/scenarios", "POST", helper.KeyModels{"scenario": s})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Scenario %v: %v", s.Name, err)
var x []scenario.Scenario
err = db.Find(&x, "Name = ?", s.Name).Error
if err != nil {
return nil, err
}
// add all users to the scenario
for _, u := range GlobalTestData.Users {
code, resp, err := helper.TestEndpoint(router, token, fmt.Sprintf("%v/scenarios/1/user?username="+u.Username, basePath), "PUT", nil)
if len(x) == 0 {
code, resp, err := helper.TestEndpoint(router, token, basePath+"/scenarios", "POST", helper.KeyModels{"scenario": s})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding user %v to scenario %v: %v", u.Username, s.Name, err)
return resp, fmt.Errorf("error adding Scenario %v: %v", s.Name, err)
}
// add all users to the scenario
for _, u := range GlobalTestData.Users {
code, resp, err := helper.TestEndpoint(router, token, fmt.Sprintf("%v/scenarios/1/user?username="+u.Username, basePath), "PUT", nil)
if code != http.StatusOK {
return resp, fmt.Errorf("error adding user %v to scenario %v: %v", u.Username, s.Name, err)
}
}
}
}
@ -190,11 +233,20 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
if len(GlobalTestData.Scenarios) > 0 && counterICs > 0 {
for _, c := range GlobalTestData.Configs {
c.ScenarioID = 1
c.ICID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/configs", "POST", helper.KeyModels{"config": c})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Config %v: %v", c.Name, err)
var x []component_configuration.ComponentConfiguration
err = db.Find(&x, "Name = ?", c.Name).Error
if err != nil {
return nil, err
}
if len(x) == 0 {
c.ScenarioID = 1
c.ICID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/configs", "POST", helper.KeyModels{"config": c})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Config %v: %v", c.Name, err)
}
}
configCounter++
}
@ -204,72 +256,114 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
dashboardCounter := 0
if len(GlobalTestData.Scenarios) > 0 {
for _, d := range GlobalTestData.Dashboards {
d.ScenarioID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/dashboards", "POST", helper.KeyModels{"dashboard": d})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Dashboard %v: %v", d.Name, err)
var x []dashboard.Dashboard
err = db.Find(&x, "Name = ?", d.Name).Error
if err != nil {
return nil, err
}
if len(x) == 0 {
d.ScenarioID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/dashboards", "POST", helper.KeyModels{"dashboard": d})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Dashboard %v: %v", d.Name, err)
}
}
dashboardCounter++
}
for _, r := range GlobalTestData.Results {
r.ScenarioID = 1
r.ResultFileIDs = []int64{}
code, resp, err := helper.TestEndpoint(router, token, basePath+"/results", "POST", helper.KeyModels{"result": r})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Result %v: %v", r.Description, err)
var x []result.Result
err = db.Find(&x, "Description = ?", r.Description).Error
if err != nil {
return nil, err
}
if len(x) == 0 {
r.ScenarioID = 1
r.ResultFileIDs = []int64{}
code, resp, err := helper.TestEndpoint(router, token, basePath+"/results", "POST", helper.KeyModels{"result": r})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Result %v: %v", r.Description, err)
}
}
}
// upload files
// upload readme file
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
fileWriter, _ := bodyWriter.CreateFormFile("file", "Readme.md")
fh, _ := os.Open("README.md")
defer fh.Close()
var x []file.File
err = db.Find(&x, "Name = ?", "Readme.md").Error
if err != nil {
return nil, err
}
// io copy
_, err = io.Copy(fileWriter, fh)
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
if len(x) == 0 {
// upload readme file
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
fileWriter, _ := bodyWriter.CreateFormFile("file", "Readme.md")
fh, _ := os.Open("README.md")
defer fh.Close()
// Create the request and add file to scenario
w1 := httptest.NewRecorder()
req1, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf)
req1.Header.Set("Content-Type", contentType)
req1.Header.Add("Authorization", "Bearer "+token)
router.ServeHTTP(w1, req1)
// io copy
_, err = io.Copy(fileWriter, fh)
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
// upload image file
bodyBuf = &bytes.Buffer{}
bodyWriter = multipart.NewWriter(bodyBuf)
fileWriter, _ = bodyWriter.CreateFormFile("file", "logo.png")
fh, _ = os.Open("doc/pictures/villas_web.png")
defer fh.Close()
// Create the request and add file to scenario
w1 := httptest.NewRecorder()
req1, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf)
req1.Header.Set("Content-Type", contentType)
req1.Header.Add("Authorization", "Bearer "+token)
router.ServeHTTP(w1, req1)
}
// io copy
_, err = io.Copy(fileWriter, fh)
contentType = bodyWriter.FormDataContentType()
bodyWriter.Close()
var y []file.File
err = db.Find(&y, "Name = ?", "logo.png").Error
if err != nil {
return nil, err
}
// Create the request and add a second file to scenario
w2 := httptest.NewRecorder()
req2, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf)
req2.Header.Set("Content-Type", contentType)
req2.Header.Add("Authorization", "Bearer "+token)
router.ServeHTTP(w2, req2)
if len(y) == 0 {
// upload image file
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
fileWriter, _ := bodyWriter.CreateFormFile("file", "logo.png")
fh, _ := os.Open("doc/pictures/villas_web.png")
defer fh.Close()
// io copy
_, err = io.Copy(fileWriter, fh)
contentType := bodyWriter.FormDataContentType()
bodyWriter.Close()
// Create the request and add a second file to scenario
w2 := httptest.NewRecorder()
req2, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf)
req2.Header.Set("Content-Type", contentType)
req2.Header.Add("Authorization", "Bearer "+token)
router.ServeHTTP(w2, req2)
}
}
// If there is at least one dashboard, add widgets
if dashboardCounter > 0 {
for _, w := range GlobalTestData.Widgets {
w.DashboardID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/widgets", "POST", helper.KeyModels{"widget": w})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Widget %v: %v", w.Name, err)
var x []widget.Widget
err = db.Find(&x, "Name = ?", w.Name).Error
if err != nil {
return nil, err
}
if len(x) == 0 {
w.DashboardID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/widgets", "POST", helper.KeyModels{"widget": w})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Widget %v: %v", w.Name, err)
}
}
}
}
@ -277,10 +371,19 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
// If there is at least one config, add signals
if configCounter > 0 {
for _, s := range GlobalTestData.Signals {
s.ConfigID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/signals", "POST", helper.KeyModels{"signal": s})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Signal %v: %v", s.Name, err)
var x []signal.Signal
err = db.Find(&x, "Name = ?", s.Name).Error
if err != nil {
return nil, err
}
if len(x) == 0 {
s.ConfigID = 1
code, resp, err := helper.TestEndpoint(router, token, basePath+"/signals", "POST", helper.KeyModels{"signal": s})
if code != http.StatusOK {
return resp, fmt.Errorf("error adding Signal %v: %v", s.Name, err)
}
}
}
}

63
start.go
View file

@ -35,34 +35,23 @@ import (
func addData(router *gin.Engine, cfg *config.Config) error {
if mode, err := cfg.String("mode"); err == nil && mode == "test" {
// test mode: drop all tables and add test data to DB
database.DropTables()
log.Println("Database tables dropped, using API to add test data")
testDataPath, err := cfg.String("test.datapath")
if err != nil {
return err
}
err = routes.ReadTestDataFromJson(testDataPath)
if err != nil {
log.Println("testdata could not be read from json")
return err
}
resp, err := routes.AddTestData(cfg, router)
if err != nil {
fmt.Println("error: testdata could not be added to DB:", err.Error(), "Response body: ", resp)
return err
}
} else {
// release mode: make sure that at least one admin user exists in DB
err, _ := database.DBAddAdminUser(cfg)
if err != nil {
fmt.Println("error: adding admin user failed:", err.Error())
return err
}
testDataPath, err := cfg.String("test.datapath")
if err != nil {
return err
}
err = routes.ReadTestDataFromJson(testDataPath)
if err != nil {
log.Println("testdata could not be read from json file")
return err
}
resp, err := routes.AddTestData(cfg, router)
if err != nil {
fmt.Println("error: testdata could not be added to DB:", err.Error(), "Response body: ", resp)
return err
}
return nil
}
@ -87,9 +76,9 @@ func main() {
log.Fatalf("Error during initialization of global configuration: %s, aborting.", err)
}
mode, err := configuration.GlobalConfig.String("mode")
dbClear, err := configuration.GlobalConfig.String("db.clear")
if err != nil {
log.Fatalf("Error reading mode from global configuration: %s, aborting.", err)
log.Fatalf("Error reading db.clear parameter from global configuration: %s, aborting.", err)
}
port, err := configuration.GlobalConfig.String("port")
@ -110,16 +99,14 @@ func main() {
}
// Init database
err = database.InitDB(configuration.GlobalConfig)
err = database.InitDB(configuration.GlobalConfig, dbClear)
if err != nil {
log.Fatalf("Error during initialization of database: %s, aborting.", err)
}
defer database.DBpool.Close()
// Init endpoints
if mode == "release" {
gin.SetMode(gin.ReleaseMode)
}
gin.SetMode(gin.ReleaseMode)
r := gin.Default()
api := r.Group("/api/v2")
@ -139,12 +126,20 @@ func main() {
}
}
// Add data to DB (if any)
// Make sure that at least one admin user exists in DB
err = database.DBAddAdminUser(configuration.GlobalConfig)
if err != nil {
fmt.Println("error: adding admin user failed:", err.Error())
log.Fatal(err)
}
// Add test/demo data to DB (if any)
err = addData(r, configuration.GlobalConfig)
if err != nil {
log.Fatal(err)
}
log.Println("Running...")
// Server at port 4000 to match frontend's redirect path
r.Run(":" + port)
}