WIP: rework the demo data reading via config parameter(s)

configuration/config.go

@@ -27,6 +27,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"sort"
 	"strings"
 
 	"gopkg.in/yaml.v3"
@@ -50,11 +51,11 @@ func InitConfig() error {
 		dbUser                   = flag.String("db-user", "", "Username of database connection (default is <empty>)")
 		dbPass                   = flag.String("db-pass", "", "Password of database connection (default is <empty>)")
 		dbSSLMode                = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production
+		dbClear                  = flag.Bool("db-clear", false, "Set to true if you want to clear all DB tables upon startup. This parameter has to be used with great care, its effects cannot be reverted.")
 		amqpHost                 = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)")
 		amqpUser                 = flag.String("amqp-user", "", "Username for AMQP broker")
 		amqpPass                 = flag.String("amqp-pass", "", "Password for AMQP broker")
 		configFile               = flag.String("config", "", "Path to YAML configuration file")
-		mode                     = flag.String("mode", "release", "Select debug/release/test mode (default is release)")
 		port                     = flag.String("port", "4000", "Port of the backend (default is 4000)")
 		adminUser                = flag.String("admin-user", "", "Initial admin username")
 		adminPass                = flag.String("admin-pass", "", "Initial admin password")
@@ -74,7 +75,7 @@ func InitConfig() error {
 		subTitle                 = flag.String("sub-title", "", "Sub-title shown in the frontend")
 		contactName              = flag.String("contact-name", "Steffen Vogel", "Name of the administrative contact")
 		contactMail              = flag.String("contact-mail", "svogel2@eonerc.rwth-aachen.de", "EMail of the administrative contact")
-		testDataPath             = flag.String("test-data-path", "database/testdata.json", "The path to the test data json file (used in test mode)")
+		testDataPath             = flag.String("test-data-path", "", "The path to a test data json file")
 		groupsPath               = flag.String("groups-path", "configuration/groups.yaml", "The path to a YAML file that maps user groups to scenario IDs")
 	)
 	flag.Parse()
@@ -88,7 +89,6 @@ func InitConfig() error {
 		"amqp.host":                   *amqpHost,
 		"amqp.user":                   *amqpUser,
 		"amqp.pass":                   *amqpPass,
-		"mode":                        *mode,
 		"port":                        *port,
 		"admin.user":                  *adminUser,
 		"admin.pass":                  *adminPass,
@@ -107,6 +107,13 @@ func InitConfig() error {
 		"contact.mail":                *contactMail,
 		"test.datapath":               *testDataPath,
 		"groups.path":                 *groupsPath,
+		"config.file":                 *configFile,
+	}
+
+	if *dbClear == true {
+		static["db.clear"] = "true"
+	} else {
+		static["db.clear"] = "false"
 	}
 
 	if *s3NoSSL == true {
@@ -152,20 +159,20 @@ func InitConfig() error {
 		return err
 	}
 
-	m, err := GlobalConfig.String("mode")
-	if err != nil {
-		return err
-	}
-
-	if m != "test" {
 	settings, _ := GlobalConfig.Settings()
-		log.Print("All settings:")
-		for key, val := range settings {
-			// TODO password settings should be excluded!
-			log.Printf("   %s = %s \n", key, val)
-		}
-	}
 
+	keys := make([]string, 0, len(settings))
+	for k := range settings {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	log.Print("All settings (except for PW and secrets):")
+	for _, k := range keys {
+		if !strings.Contains(k, "pass") && !strings.Contains(k, "secret") {
+			log.Printf("   %s = %s \n", k, settings[k])
+		}
+	}
 	return nil
 }
 

database/database.go

@@ -1,4 +1,4 @@
-/** Database package.
+/** Package database
 *
 * @author Sonja Happ <sonja.happ@eonerc.rwth-aachen.de>
 * @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC
@@ -36,8 +36,8 @@ import (
 
 var DBpool *gorm.DB // database used by backend
 
-// Initialize connection to the database
+// InitDB Initialize connection to the database
-func InitDB(cfg *config.Config) error {
+func InitDB(cfg *config.Config, dbClear string) error {
 	name, err := cfg.String("db.name")
 	if err != nil {
 		return err
@@ -73,20 +73,25 @@ func InitDB(cfg *config.Config) error {
 	}
 
 	DBpool = db
+
+	// drop tables if parameter set
+	if dbClear == "true" {
+		DropTables()
+		log.Println("Database tables dropped")
+	}
+
 	MigrateModels()
 	log.Println("Database connection established")
 
 	return nil
 }
 
-// Connection pool to already opened DB
+// GetDB Connection pool to already opened DB
 func GetDB() *gorm.DB {
 	return DBpool
 }
 
-// Drop all the tables of the database
+// DropTables drops all the tables of the database (use with care!)
-// TODO: Remove that function from the codebase and substitute the body
-// to the Dummy*() where it is called
 func DropTables() {
 	DBpool.DropTableIfExists(&InfrastructureComponent{})
 	DBpool.DropTableIfExists(&Signal{})
@@ -101,7 +106,7 @@ func DropTables() {
 	DBpool.DropTableIfExists("user_scenarios")
 }
 
-// AutoMigrate the models
+// MigrateModels AutoMigrate the models
 func MigrateModels() {
 	DBpool.AutoMigrate(&InfrastructureComponent{})
 	DBpool.AutoMigrate(&Signal{})
@@ -115,26 +120,27 @@ func MigrateModels() {
 }
 
 // DBAddAdminUser adds a default admin user to the DB
-func DBAddAdminUser(cfg *config.Config) (error, string) {
+func DBAddAdminUser(cfg *config.Config) error {
 	DBpool.AutoMigrate(User{})
 
 	// Check if admin user exists in DB
 	var users []User
 	err := DBpool.Where("Role = ?", "Admin").Find(&users).Error
 	adminPW := ""
+	adminName := ""
 
 	if len(users) == 0 {
 		fmt.Println("No admin user found in DB, adding default admin user.")
 
-		name, err := cfg.String("admin.user")
+		adminName, err = cfg.String("admin.user")
-		if err != nil || name == "" {
+		if err != nil || adminName == "" {
-			name = "admin"
+			adminName = "admin"
 		}
 
 		adminPW, err = cfg.String("admin.pass")
 		if err != nil || adminPW == "" {
 			adminPW = generatePassword(16)
-			fmt.Printf("  Generated admin password: %s\n", adminPW)
+			fmt.Printf("  Generated admin password: %s for admin user %s\n", adminPW, adminName)
 		}
 
 		mail, err := cfg.String("admin.mail")
@@ -145,14 +151,16 @@ func DBAddAdminUser(cfg *config.Config) (error, string) {
 		pwEnc, _ := bcrypt.GenerateFromPassword([]byte(adminPW), 10)
 
 		// create a copy of global test data
-		user := User{Username: name, Password: string(pwEnc),
+		user := User{Username: adminName, Password: string(pwEnc),
 			Role: "Admin", Mail: mail, Active: true}
 
 		// add admin user to DB
 		err = DBpool.Create(&user).Error
+		if err != nil {
+			return err
 		}
-
+	}
-	return err, adminPW
+	return nil
 }
 
 func generatePassword(Len int) string {

routes/register.go

@@ -102,9 +102,13 @@ func RegisterEndpoints(router *gin.Engine, api *gin.RouterGroup) {
 
 }
 
-// Read test data from JSON file (path set by ENV variable or command line param)
+// ReadTestDataFromJson Reads test data from JSON file (path set by ENV variable or command line param)
 func ReadTestDataFromJson(path string) error {
 
+	_, err := os.Stat(path)
+
+	if err == nil {
+
 		jsonFile, err := os.Open(path)
 		if err != nil {
 			return fmt.Errorf("error opening json file: %v", err)
@@ -119,24 +123,34 @@ func ReadTestDataFromJson(path string) error {
 		if err != nil {
 			return fmt.Errorf("error unmarshalling json: %v", err)
 		}
+	} else if os.IsNotExist(err) {
+		log.Println("Test data file does not exist, no test data added to DB:", path)
+		return nil
+	} else {
+		log.Println("Something is wrong with this file path:", path)
+		return nil
+	}
 
 	return nil
 }
 
-// Uses API endpoints to add test data to the backend; All endpoints have to be registered before invoking this function.
+// AddTestData Uses API endpoints to add test data to the backend; All endpoints have to be registered before invoking this function.
 func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error) {
-	database.MigrateModels()
 
-	// add Admin user (defaults to User_0, xyz789)
+	adminPW, err := cfg.String("admin.pass")
-	err, adminpw := database.DBAddAdminUser(cfg)
+	if err != nil {
-
+		log.Println("WARNING: cannot add test data: ", err)
-	var Admin = helper.Credentials{
+		return nil, nil
-		Username: "admin",
+	}
-		Password: adminpw,
+	adminName, err := cfg.String("admin.user")
+	if err != nil {
+		log.Println("WARNING: cannot add test data: ", err)
+		return nil, nil
 	}
 
-	if err != nil {
+	var Admin = helper.Credentials{
-		return nil, err
+		Username: adminName,
+		Password: adminPW,
 	}
 
 	// authenticate as admin
@@ -147,13 +161,24 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 
 	basePath := "/api/v2"
 
+	db := database.GetDB()
+
 	// add users
 	for _, u := range GlobalTestData.Users {
+
+		var x []user.User
+		err = db.Find(&x, "Username = ?", u.Username).Error
+		if err != nil {
+			return nil, err
+		}
+
+		if len(x) == 0 {
 			code, resp, err := helper.TestEndpoint(router, token, basePath+"/users", "POST", helper.KeyModels{"user": u})
 			if code != http.StatusOK {
 				return resp, fmt.Errorf("error adding user %v: %v", u.Username, err)
 			}
 		}
+	}
 
 	// add infrastructure components
 	amqphost, _ := cfg.String("amqp.host")
@@ -161,6 +186,14 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 	for _, i := range GlobalTestData.ICs {
 
 		if (i.ManagedExternally && amqphost != "") || !i.ManagedExternally {
+
+			var x []infrastructure_component.InfrastructureComponent
+			err = db.Find(&x, "Name = ?", i.Name).Error
+			if err != nil {
+				return nil, err
+			}
+
+			if len(x) == 0 {
 				code, resp, err := helper.TestEndpoint(router, token, basePath+"/ic", "POST", helper.KeyModels{"ic": i})
 				if code != http.StatusOK {
 					return resp, fmt.Errorf("error adding IC %v: %v", i.Name, err)
@@ -168,9 +201,18 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 				counterICs++
 			}
 		}
+	}
 
 	// add scenarios
 	for _, s := range GlobalTestData.Scenarios {
+
+		var x []scenario.Scenario
+		err = db.Find(&x, "Name = ?", s.Name).Error
+		if err != nil {
+			return nil, err
+		}
+
+		if len(x) == 0 {
 			code, resp, err := helper.TestEndpoint(router, token, basePath+"/scenarios", "POST", helper.KeyModels{"scenario": s})
 			if code != http.StatusOK {
 				return resp, fmt.Errorf("error adding Scenario %v: %v", s.Name, err)
@@ -184,18 +226,28 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 				}
 			}
 		}
+	}
 
 	// If there is at least one scenario and one IC in the test data, add component configs
 	configCounter := 0
 	if len(GlobalTestData.Scenarios) > 0 && counterICs > 0 {
 
 		for _, c := range GlobalTestData.Configs {
+
+			var x []component_configuration.ComponentConfiguration
+			err = db.Find(&x, "Name = ?", c.Name).Error
+			if err != nil {
+				return nil, err
+			}
+
+			if len(x) == 0 {
 				c.ScenarioID = 1
 				c.ICID = 1
 				code, resp, err := helper.TestEndpoint(router, token, basePath+"/configs", "POST", helper.KeyModels{"config": c})
 				if code != http.StatusOK {
 					return resp, fmt.Errorf("error adding Config %v: %v", c.Name, err)
 				}
+			}
 			configCounter++
 		}
 	}
@@ -204,15 +256,32 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 	dashboardCounter := 0
 	if len(GlobalTestData.Scenarios) > 0 {
 		for _, d := range GlobalTestData.Dashboards {
+
+			var x []dashboard.Dashboard
+			err = db.Find(&x, "Name = ?", d.Name).Error
+			if err != nil {
+				return nil, err
+			}
+
+			if len(x) == 0 {
 				d.ScenarioID = 1
 				code, resp, err := helper.TestEndpoint(router, token, basePath+"/dashboards", "POST", helper.KeyModels{"dashboard": d})
 				if code != http.StatusOK {
 					return resp, fmt.Errorf("error adding Dashboard %v: %v", d.Name, err)
 				}
+			}
 			dashboardCounter++
 		}
 
 		for _, r := range GlobalTestData.Results {
+
+			var x []result.Result
+			err = db.Find(&x, "Description = ?", r.Description).Error
+			if err != nil {
+				return nil, err
+			}
+
+			if len(x) == 0 {
 				r.ScenarioID = 1
 				r.ResultFileIDs = []int64{}
 				code, resp, err := helper.TestEndpoint(router, token, basePath+"/results", "POST", helper.KeyModels{"result": r})
@@ -220,9 +289,17 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 					return resp, fmt.Errorf("error adding Result %v: %v", r.Description, err)
 				}
 			}
+		}
 
 		// upload files
 
+		var x []file.File
+		err = db.Find(&x, "Name = ?", "Readme.md").Error
+		if err != nil {
+			return nil, err
+		}
+
+		if len(x) == 0 {
 			// upload readme file
 			bodyBuf := &bytes.Buffer{}
 			bodyWriter := multipart.NewWriter(bodyBuf)
@@ -241,17 +318,25 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 			req1.Header.Set("Content-Type", contentType)
 			req1.Header.Add("Authorization", "Bearer "+token)
 			router.ServeHTTP(w1, req1)
+		}
 
+		var y []file.File
+		err = db.Find(&y, "Name = ?", "logo.png").Error
+		if err != nil {
+			return nil, err
+		}
+
+		if len(y) == 0 {
 			// upload image file
-		bodyBuf = &bytes.Buffer{}
+			bodyBuf := &bytes.Buffer{}
-		bodyWriter = multipart.NewWriter(bodyBuf)
+			bodyWriter := multipart.NewWriter(bodyBuf)
-		fileWriter, _ = bodyWriter.CreateFormFile("file", "logo.png")
+			fileWriter, _ := bodyWriter.CreateFormFile("file", "logo.png")
-		fh, _ = os.Open("doc/pictures/villas_web.png")
+			fh, _ := os.Open("doc/pictures/villas_web.png")
 			defer fh.Close()
 
 			// io copy
 			_, err = io.Copy(fileWriter, fh)
-		contentType = bodyWriter.FormDataContentType()
+			contentType := bodyWriter.FormDataContentType()
 			bodyWriter.Close()
 
 			// Create the request and add a second file to scenario
@@ -260,12 +345,20 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 			req2.Header.Set("Content-Type", contentType)
 			req2.Header.Add("Authorization", "Bearer "+token)
 			router.ServeHTTP(w2, req2)
-
+		}
 	}
 
 	// If there is at least one dashboard, add widgets
 	if dashboardCounter > 0 {
 		for _, w := range GlobalTestData.Widgets {
+
+			var x []widget.Widget
+			err = db.Find(&x, "Name = ?", w.Name).Error
+			if err != nil {
+				return nil, err
+			}
+
+			if len(x) == 0 {
 				w.DashboardID = 1
 				code, resp, err := helper.TestEndpoint(router, token, basePath+"/widgets", "POST", helper.KeyModels{"widget": w})
 				if code != http.StatusOK {
@@ -273,10 +366,19 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 				}
 			}
 		}
+	}
 
 	// If there is at least one config, add signals
 	if configCounter > 0 {
 		for _, s := range GlobalTestData.Signals {
+
+			var x []signal.Signal
+			err = db.Find(&x, "Name = ?", s.Name).Error
+			if err != nil {
+				return nil, err
+			}
+
+			if len(x) == 0 {
 				s.ConfigID = 1
 				code, resp, err := helper.TestEndpoint(router, token, basePath+"/signals", "POST", helper.KeyModels{"signal": s})
 				if code != http.StatusOK {
@@ -284,6 +386,7 @@ func AddTestData(cfg *config.Config, router *gin.Engine) (*bytes.Buffer, error)
 				}
 			}
 		}
+	}
 
 	return nil, nil
 }

start.go

@@ -35,18 +35,14 @@ import (
 
 func addData(router *gin.Engine, cfg *config.Config) error {
 
-	if mode, err := cfg.String("mode"); err == nil && mode == "test" {
-		// test mode: drop all tables and add test data to DB
-		database.DropTables()
-		log.Println("Database tables dropped, using API to add test data")
-
 	testDataPath, err := cfg.String("test.datapath")
 	if err != nil {
 		return err
 	}
+
 	err = routes.ReadTestDataFromJson(testDataPath)
 	if err != nil {
-			log.Println("testdata could not be read from json")
+		log.Println("testdata could not be read from json file")
 		return err
 	}
 
@@ -55,14 +51,7 @@ func addData(router *gin.Engine, cfg *config.Config) error {
 		fmt.Println("error: testdata could not be added to DB:", err.Error(), "Response body: ", resp)
 		return err
 	}
-	} else {
+
-		// release mode: make sure that at least one admin user exists in DB
-		err, _ := database.DBAddAdminUser(cfg)
-		if err != nil {
-			fmt.Println("error: adding admin user failed:", err.Error())
-			return err
-		}
-	}
 	return nil
 }
 
@@ -87,9 +76,9 @@ func main() {
 		log.Fatalf("Error during initialization of global configuration: %s, aborting.", err)
 	}
 
-	mode, err := configuration.GlobalConfig.String("mode")
+	dbClear, err := configuration.GlobalConfig.String("db.clear")
 	if err != nil {
-		log.Fatalf("Error reading mode from global configuration: %s, aborting.", err)
+		log.Fatalf("Error reading db.clear parameter from global configuration: %s, aborting.", err)
 	}
 
 	port, err := configuration.GlobalConfig.String("port")
@@ -110,16 +99,14 @@ func main() {
 	}
 
 	// Init database
-	err = database.InitDB(configuration.GlobalConfig)
+	err = database.InitDB(configuration.GlobalConfig, dbClear)
 	if err != nil {
 		log.Fatalf("Error during initialization of database: %s, aborting.", err)
 	}
 	defer database.DBpool.Close()
 
 	// Init endpoints
-	if mode == "release" {
 	gin.SetMode(gin.ReleaseMode)
-	}
 
 	r := gin.Default()
 	api := r.Group("/api/v2")
@@ -139,12 +126,20 @@ func main() {
 		}
 	}
 
-	// Add data to DB (if any)
+	// Make sure that at least one admin user exists in DB
+	err = database.DBAddAdminUser(configuration.GlobalConfig)
+	if err != nil {
+		fmt.Println("error: adding admin user failed:", err.Error())
+		log.Fatal(err)
+	}
+
+	// Add test/demo data to DB (if any)
 	err = addData(r, configuration.GlobalConfig)
 	if err != nil {
 		log.Fatal(err)
 	}
 
+	log.Println("Running...")
 	// Server at port 4000 to match frontend's redirect path
 	r.Run(":" + port)
 }