From b1c08f16b061ce5b810c2e895df8b72075e8bac7 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 29 Apr 2021 12:45:55 +0200 Subject: [PATCH] WIP: rework the demo data reading via config parameter(s) --- configuration/config.go | 35 +++-- database/database.go | 40 +++--- routes/register.go | 281 +++++++++++++++++++++++++++------------- start.go | 63 +++++---- 4 files changed, 266 insertions(+), 153 deletions(-) diff --git a/configuration/config.go b/configuration/config.go index 07fb888..786da05 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -27,6 +27,7 @@ import ( "io/ioutil" "log" "os" + "sort" "strings" "gopkg.in/yaml.v3" @@ -50,11 +51,11 @@ func InitConfig() error { dbUser = flag.String("db-user", "", "Username of database connection (default is )") dbPass = flag.String("db-pass", "", "Password of database connection (default is )") dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production + dbClear = flag.Bool("db-clear", false, "Set to true if you want to clear all DB tables upon startup. This parameter has to be used with great care, its effects cannot be reverted.") amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)") amqpUser = flag.String("amqp-user", "", "Username for AMQP broker") amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker") configFile = flag.String("config", "", "Path to YAML configuration file") - mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)") port = flag.String("port", "4000", "Port of the backend (default is 4000)") adminUser = flag.String("admin-user", "", "Initial admin username") adminPass = flag.String("admin-pass", "", "Initial admin password") @@ -74,7 +75,7 @@ func InitConfig() error { subTitle = flag.String("sub-title", "", "Sub-title shown in the frontend") contactName = flag.String("contact-name", "Steffen Vogel", "Name of the administrative contact") contactMail = flag.String("contact-mail", "svogel2@eonerc.rwth-aachen.de", "EMail of the administrative contact") - testDataPath = flag.String("test-data-path", "database/testdata.json", "The path to the test data json file (used in test mode)") + testDataPath = flag.String("test-data-path", "", "The path to a test data json file") groupsPath = flag.String("groups-path", "configuration/groups.yaml", "The path to a YAML file that maps user groups to scenario IDs") ) flag.Parse() @@ -88,7 +89,6 @@ func InitConfig() error { "amqp.host": *amqpHost, "amqp.user": *amqpUser, "amqp.pass": *amqpPass, - "mode": *mode, "port": *port, "admin.user": *adminUser, "admin.pass": *adminPass, @@ -107,6 +107,13 @@ func InitConfig() error { "contact.mail": *contactMail, "test.datapath": *testDataPath, "groups.path": *groupsPath, + "config.file": *configFile, + } + + if *dbClear == true { + static["db.clear"] = "true" + } else { + static["db.clear"] = "false" } if *s3NoSSL == true { @@ -152,20 +159,20 @@ func InitConfig() error { return err } - m, err := GlobalConfig.String("mode") - if err != nil { - return err - } + settings, _ := GlobalConfig.Settings() - if m != "test" { - settings, _ := GlobalConfig.Settings() - log.Print("All settings:") - for key, val := range settings { - // TODO password settings should be excluded! - log.Printf(" %s = %s \n", key, val) + keys := make([]string, 0, len(settings)) + for k := range settings { + keys = append(keys, k) + } + sort.Strings(keys) + + log.Print("All settings (except for PW and secrets):") + for _, k := range keys { + if !strings.Contains(k, "pass") && !strings.Contains(k, "secret") { + log.Printf(" %s = %s \n", k, settings[k]) } } - return nil } diff --git a/database/database.go b/database/database.go index e278746..38f0736 100644 --- a/database/database.go +++ b/database/database.go @@ -1,4 +1,4 @@ -/** Database package. +/** Package database * * @author Sonja Happ * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC @@ -36,8 +36,8 @@ import ( var DBpool *gorm.DB // database used by backend -// Initialize connection to the database -func InitDB(cfg *config.Config) error { +// InitDB Initialize connection to the database +func InitDB(cfg *config.Config, dbClear string) error { name, err := cfg.String("db.name") if err != nil { return err @@ -73,20 +73,25 @@ func InitDB(cfg *config.Config) error { } DBpool = db + + // drop tables if parameter set + if dbClear == "true" { + DropTables() + log.Println("Database tables dropped") + } + MigrateModels() log.Println("Database connection established") return nil } -// Connection pool to already opened DB +// GetDB Connection pool to already opened DB func GetDB() *gorm.DB { return DBpool } -// Drop all the tables of the database -// TODO: Remove that function from the codebase and substitute the body -// to the Dummy*() where it is called +// DropTables drops all the tables of the database (use with care!) func DropTables() { DBpool.DropTableIfExists(&InfrastructureComponent{}) DBpool.DropTableIfExists(&Signal{}) @@ -101,7 +106,7 @@ func DropTables() { DBpool.DropTableIfExists("user_scenarios") } -// AutoMigrate the models +// MigrateModels AutoMigrate the models func MigrateModels() { DBpool.AutoMigrate(&InfrastructureComponent{}) DBpool.AutoMigrate(&Signal{}) @@ -115,26 +120,27 @@ func MigrateModels() { } // DBAddAdminUser adds a default admin user to the DB -func DBAddAdminUser(cfg *config.Config) (error, string) { +func DBAddAdminUser(cfg *config.Config) error { DBpool.AutoMigrate(User{}) // Check if admin user exists in DB var users []User err := DBpool.Where("Role = ?", "Admin").Find(&users).Error adminPW := "" + adminName := "" if len(users) == 0 { fmt.Println("No admin user found in DB, adding default admin user.") - name, err := cfg.String("admin.user") - if err != nil || name == "" { - name = "admin" + adminName, err = cfg.String("admin.user") + if err != nil || adminName == "" { + adminName = "admin" } adminPW, err = cfg.String("admin.pass") if err != nil || adminPW == "" { adminPW = generatePassword(16) - fmt.Printf(" Generated admin password: %s\n", adminPW) + fmt.Printf(" Generated admin password: %s for admin user %s\n", adminPW, adminName) } mail, err := cfg.String("admin.mail") @@ -145,14 +151,16 @@ func DBAddAdminUser(cfg *config.Config) (error, string) { pwEnc, _ := bcrypt.GenerateFromPassword([]byte(adminPW), 10) // create a copy of global test data - user := User{Username: name, Password: string(pwEnc), + user := User{Username: adminName, Password: string(pwEnc), Role: "Admin", Mail: mail, Active: true} // add admin user to DB err = DBpool.Create(&user).Error + if err != nil { + return err + } } - - return err, adminPW + return nil } func generatePassword(Len int) string { diff --git a/routes/register.go b/routes/register.go index 9c5105b..7f042ce 100644 --- a/routes/register.go +++ b/routes/register.go @@ -102,41 +102,55 @@ func RegisterEndpoints(router *gin.Engine, api *gin.RouterGroup) { } -// Read test data from JSON file (path set by ENV variable or command line param) +// ReadTestDataFromJson Reads test data from JSON file (path set by ENV variable or command line param) func ReadTestDataFromJson(path string) error { - jsonFile, err := os.Open(path) - if err != nil { - return fmt.Errorf("error opening json file: %v", err) - } - log.Println("Successfully opened json data file", path) + _, err := os.Stat(path) - defer jsonFile.Close() + if err == nil { - byteValue, _ := ioutil.ReadAll(jsonFile) + jsonFile, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening json file: %v", err) + } + log.Println("Successfully opened json data file", path) - err = json.Unmarshal(byteValue, &GlobalTestData) - if err != nil { - return fmt.Errorf("error unmarshalling json: %v", err) + defer jsonFile.Close() + + byteValue, _ := ioutil.ReadAll(jsonFile) + + err = json.Unmarshal(byteValue, &GlobalTestData) + if err != nil { + return fmt.Errorf("error unmarshalling json: %v", err) + } + } else if os.IsNotExist(err) { + log.Println("Test data file does not exist, no test data added to DB:", path) + return nil + } else { + log.Println("Something is wrong with this file path:", path) + return nil } return nil } -// Uses API endpoints to add test data to the backend; All endpoints have to be registered before invoking this function. +// AddTestData Uses API endpoints to add test data to the backend; All endpoints have to be registered before invoking this function. func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) { - database.MigrateModels() - // add Admin user (defaults to User_0, xyz789) - err, adminpw := database.DBAddAdminUser(cfg) - - var Admin = helper.Credentials{ - Username: "admin", - Password: adminpw, + adminPW, err := cfg.String("admin.pass") + if err != nil { + log.Println("WARNING: cannot add test data: ", err) + return nil, nil + } + adminName, err := cfg.String("admin.user") + if err != nil { + log.Println("WARNING: cannot add test data: ", err) + return nil, nil } - if err != nil { - return nil, err + var Admin = helper.Credentials{ + Username: adminName, + Password: adminPW, } // authenticate as admin @@ -147,11 +161,22 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) basePath := "/api/v2" + db := database.GetDB() + // add users for _, u := range GlobalTestData.Users { - code, resp, err := helper.TestEndpoint(router, token, basePath+"/users", "POST", helper.KeyModels{"user": u}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding user %v: %v", u.Username, err) + + var x []user.User + err = db.Find(&x, "Username = ?", u.Username).Error + if err != nil { + return nil, err + } + + if len(x) == 0 { + code, resp, err := helper.TestEndpoint(router, token, basePath+"/users", "POST", helper.KeyModels{"user": u}) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding user %v: %v", u.Username, err) + } } } @@ -161,26 +186,44 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) for _, i := range GlobalTestData.ICs { if (i.ManagedExternally && amqphost != "") || !i.ManagedExternally { - code, resp, err := helper.TestEndpoint(router, token, basePath+"/ic", "POST", helper.KeyModels{"ic": i}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding IC %v: %v", i.Name, err) + + var x []infrastructure_component.InfrastructureComponent + err = db.Find(&x, "Name = ?", i.Name).Error + if err != nil { + return nil, err + } + + if len(x) == 0 { + code, resp, err := helper.TestEndpoint(router, token, basePath+"/ic", "POST", helper.KeyModels{"ic": i}) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding IC %v: %v", i.Name, err) + } + counterICs++ } - counterICs++ } } // add scenarios for _, s := range GlobalTestData.Scenarios { - code, resp, err := helper.TestEndpoint(router, token, basePath+"/scenarios", "POST", helper.KeyModels{"scenario": s}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding Scenario %v: %v", s.Name, err) + + var x []scenario.Scenario + err = db.Find(&x, "Name = ?", s.Name).Error + if err != nil { + return nil, err } - // add all users to the scenario - for _, u := range GlobalTestData.Users { - code, resp, err := helper.TestEndpoint(router, token, fmt.Sprintf("%v/scenarios/1/user?username="+u.Username, basePath), "PUT", nil) + if len(x) == 0 { + code, resp, err := helper.TestEndpoint(router, token, basePath+"/scenarios", "POST", helper.KeyModels{"scenario": s}) if code != http.StatusOK { - return resp, fmt.Errorf("error adding user %v to scenario %v: %v", u.Username, s.Name, err) + return resp, fmt.Errorf("error adding Scenario %v: %v", s.Name, err) + } + + // add all users to the scenario + for _, u := range GlobalTestData.Users { + code, resp, err := helper.TestEndpoint(router, token, fmt.Sprintf("%v/scenarios/1/user?username="+u.Username, basePath), "PUT", nil) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding user %v to scenario %v: %v", u.Username, s.Name, err) + } } } } @@ -190,11 +233,20 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) if len(GlobalTestData.Scenarios) > 0 && counterICs > 0 { for _, c := range GlobalTestData.Configs { - c.ScenarioID = 1 - c.ICID = 1 - code, resp, err := helper.TestEndpoint(router, token, basePath+"/configs", "POST", helper.KeyModels{"config": c}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding Config %v: %v", c.Name, err) + + var x []component_configuration.ComponentConfiguration + err = db.Find(&x, "Name = ?", c.Name).Error + if err != nil { + return nil, err + } + + if len(x) == 0 { + c.ScenarioID = 1 + c.ICID = 1 + code, resp, err := helper.TestEndpoint(router, token, basePath+"/configs", "POST", helper.KeyModels{"config": c}) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding Config %v: %v", c.Name, err) + } } configCounter++ } @@ -204,72 +256,114 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) dashboardCounter := 0 if len(GlobalTestData.Scenarios) > 0 { for _, d := range GlobalTestData.Dashboards { - d.ScenarioID = 1 - code, resp, err := helper.TestEndpoint(router, token, basePath+"/dashboards", "POST", helper.KeyModels{"dashboard": d}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding Dashboard %v: %v", d.Name, err) + + var x []dashboard.Dashboard + err = db.Find(&x, "Name = ?", d.Name).Error + if err != nil { + return nil, err + } + + if len(x) == 0 { + d.ScenarioID = 1 + code, resp, err := helper.TestEndpoint(router, token, basePath+"/dashboards", "POST", helper.KeyModels{"dashboard": d}) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding Dashboard %v: %v", d.Name, err) + } } dashboardCounter++ } for _, r := range GlobalTestData.Results { - r.ScenarioID = 1 - r.ResultFileIDs = []int64{} - code, resp, err := helper.TestEndpoint(router, token, basePath+"/results", "POST", helper.KeyModels{"result": r}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding Result %v: %v", r.Description, err) + + var x []result.Result + err = db.Find(&x, "Description = ?", r.Description).Error + if err != nil { + return nil, err + } + + if len(x) == 0 { + r.ScenarioID = 1 + r.ResultFileIDs = []int64{} + code, resp, err := helper.TestEndpoint(router, token, basePath+"/results", "POST", helper.KeyModels{"result": r}) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding Result %v: %v", r.Description, err) + } } } // upload files - // upload readme file - bodyBuf := &bytes.Buffer{} - bodyWriter := multipart.NewWriter(bodyBuf) - fileWriter, _ := bodyWriter.CreateFormFile("file", "Readme.md") - fh, _ := os.Open("README.md") - defer fh.Close() + var x []file.File + err = db.Find(&x, "Name = ?", "Readme.md").Error + if err != nil { + return nil, err + } - // io copy - _, err = io.Copy(fileWriter, fh) - contentType := bodyWriter.FormDataContentType() - bodyWriter.Close() + if len(x) == 0 { + // upload readme file + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + fileWriter, _ := bodyWriter.CreateFormFile("file", "Readme.md") + fh, _ := os.Open("README.md") + defer fh.Close() - // Create the request and add file to scenario - w1 := httptest.NewRecorder() - req1, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf) - req1.Header.Set("Content-Type", contentType) - req1.Header.Add("Authorization", "Bearer "+token) - router.ServeHTTP(w1, req1) + // io copy + _, err = io.Copy(fileWriter, fh) + contentType := bodyWriter.FormDataContentType() + bodyWriter.Close() - // upload image file - bodyBuf = &bytes.Buffer{} - bodyWriter = multipart.NewWriter(bodyBuf) - fileWriter, _ = bodyWriter.CreateFormFile("file", "logo.png") - fh, _ = os.Open("doc/pictures/villas_web.png") - defer fh.Close() + // Create the request and add file to scenario + w1 := httptest.NewRecorder() + req1, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf) + req1.Header.Set("Content-Type", contentType) + req1.Header.Add("Authorization", "Bearer "+token) + router.ServeHTTP(w1, req1) + } - // io copy - _, err = io.Copy(fileWriter, fh) - contentType = bodyWriter.FormDataContentType() - bodyWriter.Close() + var y []file.File + err = db.Find(&y, "Name = ?", "logo.png").Error + if err != nil { + return nil, err + } - // Create the request and add a second file to scenario - w2 := httptest.NewRecorder() - req2, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf) - req2.Header.Set("Content-Type", contentType) - req2.Header.Add("Authorization", "Bearer "+token) - router.ServeHTTP(w2, req2) + if len(y) == 0 { + // upload image file + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + fileWriter, _ := bodyWriter.CreateFormFile("file", "logo.png") + fh, _ := os.Open("doc/pictures/villas_web.png") + defer fh.Close() + // io copy + _, err = io.Copy(fileWriter, fh) + contentType := bodyWriter.FormDataContentType() + bodyWriter.Close() + + // Create the request and add a second file to scenario + w2 := httptest.NewRecorder() + req2, _ := http.NewRequest("POST", basePath+"/files?scenarioID=1", bodyBuf) + req2.Header.Set("Content-Type", contentType) + req2.Header.Add("Authorization", "Bearer "+token) + router.ServeHTTP(w2, req2) + } } // If there is at least one dashboard, add widgets if dashboardCounter > 0 { for _, w := range GlobalTestData.Widgets { - w.DashboardID = 1 - code, resp, err := helper.TestEndpoint(router, token, basePath+"/widgets", "POST", helper.KeyModels{"widget": w}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding Widget %v: %v", w.Name, err) + + var x []widget.Widget + err = db.Find(&x, "Name = ?", w.Name).Error + if err != nil { + return nil, err + } + + if len(x) == 0 { + w.DashboardID = 1 + code, resp, err := helper.TestEndpoint(router, token, basePath+"/widgets", "POST", helper.KeyModels{"widget": w}) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding Widget %v: %v", w.Name, err) + } } } } @@ -277,10 +371,19 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) // If there is at least one config, add signals if configCounter > 0 { for _, s := range GlobalTestData.Signals { - s.ConfigID = 1 - code, resp, err := helper.TestEndpoint(router, token, basePath+"/signals", "POST", helper.KeyModels{"signal": s}) - if code != http.StatusOK { - return resp, fmt.Errorf("error adding Signal %v: %v", s.Name, err) + + var x []signal.Signal + err = db.Find(&x, "Name = ?", s.Name).Error + if err != nil { + return nil, err + } + + if len(x) == 0 { + s.ConfigID = 1 + code, resp, err := helper.TestEndpoint(router, token, basePath+"/signals", "POST", helper.KeyModels{"signal": s}) + if code != http.StatusOK { + return resp, fmt.Errorf("error adding Signal %v: %v", s.Name, err) + } } } } diff --git a/start.go b/start.go index f09f550..924b938 100644 --- a/start.go +++ b/start.go @@ -35,34 +35,23 @@ import ( func addData(router *gin.Engine, cfg *config.Config) error { - if mode, err := cfg.String("mode"); err == nil && mode == "test" { - // test mode: drop all tables and add test data to DB - database.DropTables() - log.Println("Database tables dropped, using API to add test data") - - testDataPath, err := cfg.String("test.datapath") - if err != nil { - return err - } - err = routes.ReadTestDataFromJson(testDataPath) - if err != nil { - log.Println("testdata could not be read from json") - return err - } - - resp, err := routes.AddTestData(cfg, router) - if err != nil { - fmt.Println("error: testdata could not be added to DB:", err.Error(), "Response body: ", resp) - return err - } - } else { - // release mode: make sure that at least one admin user exists in DB - err, _ := database.DBAddAdminUser(cfg) - if err != nil { - fmt.Println("error: adding admin user failed:", err.Error()) - return err - } + testDataPath, err := cfg.String("test.datapath") + if err != nil { + return err } + + err = routes.ReadTestDataFromJson(testDataPath) + if err != nil { + log.Println("testdata could not be read from json file") + return err + } + + resp, err := routes.AddTestData(cfg, router) + if err != nil { + fmt.Println("error: testdata could not be added to DB:", err.Error(), "Response body: ", resp) + return err + } + return nil } @@ -87,9 +76,9 @@ func main() { log.Fatalf("Error during initialization of global configuration: %s, aborting.", err) } - mode, err := configuration.GlobalConfig.String("mode") + dbClear, err := configuration.GlobalConfig.String("db.clear") if err != nil { - log.Fatalf("Error reading mode from global configuration: %s, aborting.", err) + log.Fatalf("Error reading db.clear parameter from global configuration: %s, aborting.", err) } port, err := configuration.GlobalConfig.String("port") @@ -110,16 +99,14 @@ func main() { } // Init database - err = database.InitDB(configuration.GlobalConfig) + err = database.InitDB(configuration.GlobalConfig, dbClear) if err != nil { log.Fatalf("Error during initialization of database: %s, aborting.", err) } defer database.DBpool.Close() // Init endpoints - if mode == "release" { - gin.SetMode(gin.ReleaseMode) - } + gin.SetMode(gin.ReleaseMode) r := gin.Default() api := r.Group("/api/v2") @@ -139,12 +126,20 @@ func main() { } } - // Add data to DB (if any) + // Make sure that at least one admin user exists in DB + err = database.DBAddAdminUser(configuration.GlobalConfig) + if err != nil { + fmt.Println("error: adding admin user failed:", err.Error()) + log.Fatal(err) + } + + // Add test/demo data to DB (if any) err = addData(r, configuration.GlobalConfig) if err != nil { log.Fatal(err) } + log.Println("Running...") // Server at port 4000 to match frontend's redirect path r.Run(":" + port) }