Merge branch 'master' into refactor-amqp-session

# Conflicts:
#	database/database.go
#	routes/healthz/healthz_endpoint.go
#	start.go

configuration/config.go

@@ -67,6 +67,7 @@ func InitConfig() error {
 		adminMail                = flag.String("admin-mail", "", "Initial admin mail address")
 		s3Bucket                 = flag.String("s3-bucket", "", "S3 Bucket for uploading files")
 		s3Endpoint               = flag.String("s3-endpoint", "", "Endpoint of S3 API for file uploads")
+		s3EndpointPublic         = flag.String("s3-endpoint-public", "", "Public endpoint address of S3 API for file uploads")
 		s3Region                 = flag.String("s3-region", "default", "S3 Region for file uploads")
 		s3NoSSL                  = flag.Bool("s3-nossl", false, "Use encrypted connections to the S3 API")
 		s3PathStyle              = flag.Bool("s3-pathstyle", false, "Use path-style S3 API")
@@ -83,8 +84,8 @@ func InitConfig() error {
 		testDataPath             = flag.String("test-data-path", "", "The path to a test data json file")
 		groupsPath               = flag.String("groups-path", "", "The path to a YAML file that maps user groups to scenario IDs")
 		apiUpdateInterval        = flag.String("api-update-interval", "10s" /* 10 sec */, "Interval in which API URL is queried for status updates of ICs")
-		rancherURL               = flag.String("rancher-url", "rancher.k8s.eonerc.rwth-aachen.de", "URL of Rancher instance that is used to deploy the backend")
+		k8sRancherURL            = flag.String("k8s-rancher-url", "https://rancher.k8s.eonerc.rwth-aachen.de", "URL of Rancher instance that is used to deploy the backend")
-		k8sCluster               = flag.String("k8s-cluster", "local", "Name of the Kubernetes cluster where the backend is deployed")
+		k8sClusterName           = flag.String("k8s-cluster-name", "local", "Name of the Kubernetes cluster where the backend is deployed")
 	)
 	flag.Parse()
 
@@ -103,6 +104,7 @@ func InitConfig() error {
 		"admin.mail":                  *adminMail,
 		"s3.bucket":                   *s3Bucket,
 		"s3.endpoint":                 *s3Endpoint,
+		"s3.endpoint-public":          *s3EndpointPublic,
 		"s3.region":                   *s3Region,
 		"jwt.secret":                  *jwtSecret,
 		"jwt.expires-after":           *jwtExpiresAfter,
@@ -117,8 +119,8 @@ func InitConfig() error {
 		"groups.path":                 *groupsPath,
 		"config.file":                 *configFile,
 		"apiupdateinterval":           *apiUpdateInterval,
-		"rancherURL":                  *rancherURL,
+		"k8s.rancher-url":             *k8sRancherURL,
-		"k8sCluster":                  *k8sCluster,
+		"k8s.cluster-name":            *k8sClusterName,
 	}
 
 	if *dbClear {

database/admin.go

@@ -0,0 +1,177 @@
+/** Package database
+*
+* @author Sonja Happ <sonja.happ@eonerc.rwth-aachen.de>
+* @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC
+* @license GNU General Public License (version 3)
+*
+* VILLASweb-backend-go
+*
+* This program is free software: you can redistribute it and/or modify
+* it under the terms of the GNU General Public License as published by
+* the Free Software Foundation, either version 3 of the License, or
+* any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*********************************************************************************/
+package database
+
+import (
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/zpatrick/go-config"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// AddAdminUser adds a default admin user to the DB
+func AddAdminUser(cfg *config.Config) (string, error) {
+	DBpool.AutoMigrate(User{})
+
+	updatedPW := false
+	generatedPW := false
+
+	adminName, _ := cfg.StringOr("admin.user", "admin")
+	adminMail, _ := cfg.StringOr("admin.mail", "admin@example.com")
+	adminPW, err := cfg.String("admin.pass")
+	if err == nil && adminPW != "" {
+		updatedPW = true
+	} else if err != nil || adminPW == "" {
+		adminPW = generatePassword(16)
+		generatedPW = true
+	}
+
+	adminPWEnc, err := bcrypt.GenerateFromPassword([]byte(adminPW), 10)
+	if err != nil {
+		return "", err
+	}
+
+	// Check if admin user exists in DB
+	var users []User
+	err = DBpool.Where("Username = ?", adminName).Find(&users).Error
+	if err != nil {
+		return "", err
+	}
+
+	if len(users) == 0 {
+		fmt.Println("No admin user found in DB, adding default admin user.")
+		if generatedPW {
+			fmt.Printf("  Generated admin password: %s for admin user %s\n", adminPW, adminName)
+		}
+
+		user := User{
+			Username: adminName,
+			Password: string(adminPWEnc),
+			Role:     "Admin",
+			Mail:     adminMail,
+			Active:   true,
+		}
+
+		// add admin user to DB
+		err = DBpool.Create(&user).Error
+		if err != nil {
+			return "", err
+		}
+	} else if updatedPW {
+		fmt.Println("Found existing admin user in DB, updating user from CLI parameters.")
+
+		user := users[0]
+
+		user.Password = string(adminPWEnc)
+		user.Role = "Admin"
+		user.Mail = adminMail
+		user.Active = true
+
+		err = DBpool.Model(user).Update(&user).Error
+		if err != nil {
+			return "", err
+		}
+	}
+
+	return adminPW, err
+}
+
+func generatePassword(Len int) string {
+	rand.Seed(time.Now().UnixNano())
+	chars := []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ" +
+		"abcdefghijklmnopqrstuvwxyz" +
+		"0123456789")
+
+	var b strings.Builder
+	for i := 0; i < Len; i++ {
+		b.WriteRune(chars[rand.Intn(len(chars))])
+	}
+
+	return b.String()
+}
+
+// add test users defined above
+func AddTestUsers() error {
+
+	testUsers := []User{User0, UserA, UserB, UserC}
+	DBpool.AutoMigrate(&User{})
+
+	for _, user := range testUsers {
+		err := DBpool.Create(&user).Error
+		if err != nil {
+			return err
+		}
+
+	}
+
+	return nil
+}
+
+// Credentials
+var StrPassword0 = "xyz789"
+var StrPasswordA = "abc123"
+var StrPasswordB = "bcd234"
+var StrPasswordC = "guestpw"
+
+// Hash passwords with bcrypt algorithm
+var bcryptCost = 10
+var pw0, _ = bcrypt.GenerateFromPassword([]byte(StrPassword0), bcryptCost)
+var pwA, _ = bcrypt.GenerateFromPassword([]byte(StrPasswordA), bcryptCost)
+var pwB, _ = bcrypt.GenerateFromPassword([]byte(StrPasswordB), bcryptCost)
+var pwC, _ = bcrypt.GenerateFromPassword([]byte(StrPasswordC), bcryptCost)
+
+var User0 = User{Username: "User_0", Password: string(pw0),
+	Role: "Admin", Mail: "User_0@example.com"}
+var UserA = User{Username: "User_A", Password: string(pwA),
+	Role: "User", Mail: "User_A@example.com", Active: true}
+var UserB = User{Username: "User_B", Password: string(pwB),
+	Role: "User", Mail: "User_B@example.com", Active: true}
+var UserC = User{Username: "User_C", Password: string(pwC),
+	Role: "Guest", Mail: "User_C@example.com", Active: true}
+
+type Credentials struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+var AdminCredentials = Credentials{
+	Username: User0.Username,
+	Password: StrPassword0,
+}
+
+var UserACredentials = Credentials{
+	Username: UserA.Username,
+	Password: StrPasswordA,
+}
+
+var UserBCredentials = Credentials{
+	Username: UserB.Username,
+	Password: StrPasswordB,
+}
+
+var GuestCredentials = Credentials{
+	Username: UserC.Username,
+	Password: StrPasswordC,
+}

database/database.go

@@ -24,11 +24,6 @@ package database
 import (
 	"fmt"
 	"log"
-	"math/rand"
-	"strings"
-	"time"
-
-	"golang.org/x/crypto/bcrypt"
 
 	"github.com/jinzhu/gorm"
 	_ "github.com/jinzhu/gorm/dialects/postgres"
@@ -123,125 +118,3 @@ func MigrateModels() {
 	DBpool.AutoMigrate(&Widget{})
 	DBpool.AutoMigrate(&Result{})
 }
-
-// DBAddAdminUser adds a default admin user to the DB
-func DBAddAdminUser(cfg *config.Config) (string, error) {
-	DBpool.AutoMigrate(User{})
-
-	// Check if admin user exists in DB
-	var users []User
-	DBpool.Where("Role = ?", "Admin").Find(&users)
-
-	adminPW := ""
-
-	if len(users) == 0 {
-		fmt.Println("No admin user found in DB, adding default admin user.")
-
-		adminName, err := cfg.String("admin.user")
-		if err != nil || adminName == "" {
-			adminName = "admin"
-		}
-
-		adminPW, err = cfg.String("admin.pass")
-		if err != nil || adminPW == "" {
-			adminPW = generatePassword(16)
-			fmt.Printf("  Generated admin password: %s for admin user %s\n", adminPW, adminName)
-		}
-
-		mail, err := cfg.String("admin.mail")
-		if err == nil || mail == "" {
-			mail = "admin@example.com"
-		}
-
-		pwEnc, _ := bcrypt.GenerateFromPassword([]byte(adminPW), 10)
-
-		// create a copy of global test data
-		user := User{Username: adminName, Password: string(pwEnc),
-			Role: "Admin", Mail: mail, Active: true}
-
-		// add admin user to DB
-		err = DBpool.Create(&user).Error
-		if err != nil {
-			return "", err
-		}
-	}
-	return adminPW, nil
-}
-
-func generatePassword(Len int) string {
-	rand.Seed(time.Now().UnixNano())
-	chars := []rune("ABCDEFGHIJKLMNOPQRSTUVWXYZ" +
-		"abcdefghijklmnopqrstuvwxyz" +
-		"0123456789")
-
-	var b strings.Builder
-	for i := 0; i < Len; i++ {
-		b.WriteRune(chars[rand.Intn(len(chars))])
-	}
-
-	return b.String()
-}
-
-// add test users defined above
-func AddTestUsers() error {
-
-	testUsers := []User{User0, UserA, UserB, UserC}
-	DBpool.AutoMigrate(&User{})
-
-	for _, user := range testUsers {
-		err := DBpool.Create(&user).Error
-		if err != nil {
-			return err
-		}
-
-	}
-
-	return nil
-}
-
-// Credentials
-var StrPassword0 = "xyz789"
-var StrPasswordA = "abc123"
-var StrPasswordB = "bcd234"
-var StrPasswordC = "guestpw"
-
-// Hash passwords with bcrypt algorithm
-var bcryptCost = 10
-var pw0, _ = bcrypt.GenerateFromPassword([]byte(StrPassword0), bcryptCost)
-var pwA, _ = bcrypt.GenerateFromPassword([]byte(StrPasswordA), bcryptCost)
-var pwB, _ = bcrypt.GenerateFromPassword([]byte(StrPasswordB), bcryptCost)
-var pwC, _ = bcrypt.GenerateFromPassword([]byte(StrPasswordC), bcryptCost)
-
-var User0 = User{Username: "User_0", Password: string(pw0),
-	Role: "Admin", Mail: "User_0@example.com"}
-var UserA = User{Username: "User_A", Password: string(pwA),
-	Role: "User", Mail: "User_A@example.com", Active: true}
-var UserB = User{Username: "User_B", Password: string(pwB),
-	Role: "User", Mail: "User_B@example.com", Active: true}
-var UserC = User{Username: "User_C", Password: string(pwC),
-	Role: "Guest", Mail: "User_C@example.com", Active: true}
-
-type Credentials struct {
-	Username string `json:"username"`
-	Password string `json:"password"`
-}
-
-var AdminCredentials = Credentials{
-	Username: User0.Username,
-	Password: StrPassword0,
-}
-
-var UserACredentials = Credentials{
-	Username: UserA.Username,
-	Password: StrPasswordA,
-}
-
-var UserBCredentials = Credentials{
-	Username: UserB.Username,
-	Password: StrPasswordB,
-}
-
-var GuestCredentials = Credentials{
-	Username: UserC.Username,
-	Password: StrPasswordC,
-}

routes/config/config_endpoint.go

@@ -47,14 +47,18 @@ type ConfigContact struct {
 	Mail string `json:"mail"`
 }
 
+type ConfigKubernetes struct {
+	RancherURL  string `json:"rancher_url"`
+	ClusterName string `json:"cluster_name"`
+}
+
 type Config struct {
 	Title          string               `json:"title"`
 	SubTitle       string               `json:"sub_title"`
 	Mode           string               `json:"mode"`
 	Contact        ConfigContact        `json:"contact"`
 	Authentication ConfigAuthentication `json:"authentication"`
-	RancherURL     string               `json:"rancherURL"`
+	Kubernetes     ConfigKubernetes     `json:"kubernetes"`
-	K8sCluster     string               `json:"k8sCluster"`
 }
 
 // getHealth godoc
@@ -79,8 +83,8 @@ func getConfig(c *gin.Context) {
 	resp.SubTitle, _ = cfg.String("sub-title")
 	resp.Contact.Name, _ = cfg.String("contact.name")
 	resp.Contact.Mail, _ = cfg.String("contact.mail")
-	resp.RancherURL, _ = cfg.String("rancherURL")
+	resp.Kubernetes.RancherURL, _ = cfg.String("k8s.rancher-url")
-	resp.K8sCluster, _ = cfg.String("k8sCluster")
+	resp.Kubernetes.ClusterName, _ = cfg.String("k8s.cluster-name")
 
 	c.JSON(200, resp)
 }

routes/file/file_s3.go

@@ -22,13 +22,16 @@
 package file
 
 import (
+	"errors"
 	"fmt"
 	"io"
+	"net/url"
 	"time"
 
 	"git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -83,7 +86,7 @@ func createS3Session() (*session.Session, error) {
 		},
 	)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create session: %s", err)
+		return nil, fmt.Errorf("failed to create session: %w", err)
 	}
 
 	return sess, nil
@@ -110,7 +113,7 @@ func (f *File) putS3(fileContent io.Reader) error {
 		Body:   fileContent,
 	})
 	if err != nil {
-		return fmt.Errorf("failed to upload file, %v", err)
+		return fmt.Errorf("failed to upload file: %w", err)
 	}
 
 	return nil
@@ -128,16 +131,21 @@ func (f *File) getS3Url() (string, error) {
 	svc := s3.New(sess)
 
 	req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
-		Bucket:              aws.String(bucket),
+		Bucket:                     aws.String(bucket),
-		Key:                 aws.String(f.Key),
+		Key:                        aws.String(f.Key),
-		ResponseContentType: aws.String(f.Type),
+		ResponseContentType:        aws.String(f.Type),
-		// ResponseContentDisposition: aws.String("attachment; filename=" + f.Name),
+		ResponseContentDisposition: aws.String("attachment; filename=" + f.Name),
 		// ResponseContentEncoding: aws.String(),
 		// ResponseContentLanguage: aws.String(),
 		// ResponseCacheControl:    aws.String(),
 		// ResponseExpires:         aws.String(),
 	})
 
+	err = updateS3Request(req)
+	if err != nil {
+		return "", err
+	}
+
 	urlStr, err := req.Presign(5 * 24 * 60 * time.Minute)
 	if err != nil {
 		return "", err
@@ -170,3 +178,35 @@ func (f *File) deleteS3() error {
 
 	return nil
 }
+
+// updateS3Request updates the request host to the public accessible S3
+// endpoint host so that presigned URLs are still valid when accessed
+// by the user
+func updateS3Request(req *request.Request) error {
+	epURL, err := getS3EndpointURL()
+	if err != nil {
+		return err
+	}
+
+	req.HTTPRequest.URL.Scheme = epURL.Scheme
+	req.HTTPRequest.URL.Host = epURL.Host
+
+	return nil
+}
+
+func getS3EndpointURL() (*url.URL, error) {
+	ep, err := configuration.GlobalConfig.String("s3.endpoint-public")
+	if err != nil {
+		ep, err = configuration.GlobalConfig.String("s3.endpoint")
+		if err != nil {
+			return nil, errors.New("missing s3.endpoint setting")
+		}
+	}
+
+	epURL, err := url.Parse(ep)
+	if err != nil {
+		return nil, err
+	}
+
+	return epURL, nil
+}

routes/healthz/healthz_endpoint.go

@@ -61,11 +61,8 @@ func getHealth(c *gin.Context) {
 	}
 
 	// check if connection to AMQP broker is alive if backend was started with AMQP client
-	url, err := configuration.GlobalConfig.StringOr("amqp.host", "not-set")
+	url, err := configuration.GlobalConfig.StringOr("amqp.host", "")
-	if err != nil && url == "not-set" {
+	if err != nil {
-		c.JSON(http.StatusOK, gin.H{})
-		return
-	} else if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{
 			"success:": false,
 			"message":  err.Error(),
@@ -73,7 +70,10 @@ func getHealth(c *gin.Context) {
 		return
 	}
 
-	if len(url) != 0 {
+	if url == "" {
+		c.Writer.WriteHeader(http.StatusNoContent)
+		return
+	} else {
 		if session != nil {
 			err = session.CheckConnection()
 			if err != nil {
@@ -91,7 +91,6 @@ func getHealth(c *gin.Context) {
 			})
 			return
 		}
-
 	}
 
 	// Send a 204 reponse

routes/user/user_test.go

@@ -97,7 +97,7 @@ func TestMain(m *testing.M) {
 func TestAuthenticate(t *testing.T) {
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// try to authenticate with non JSON body
@@ -194,7 +194,7 @@ func TestAuthenticateQueryToken(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin
@@ -215,7 +215,7 @@ func TestAddGetUser(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin
@@ -339,7 +339,7 @@ func TestUsersNotAllowedActions(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin
@@ -398,7 +398,7 @@ func TestGetAllUsers(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin
@@ -451,7 +451,7 @@ func TestModifyAddedUserAsUser(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin
@@ -606,7 +606,7 @@ func TestInvalidUserUpdate(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin
@@ -678,7 +678,7 @@ func TestModifyAddedUserAsAdmin(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin
@@ -795,7 +795,7 @@ func TestDeleteUser(t *testing.T) {
 
 	database.DropTables()
 	database.MigrateModels()
-	adminpw, err := database.DBAddAdminUser(configuration.GlobalConfig)
+	adminpw, err := database.AddAdminUser(configuration.GlobalConfig)
 	assert.NoError(t, err)
 
 	// authenticate as admin

routes/widget/widget_validators.go

@@ -103,11 +103,7 @@ func (r *addWidgetRequest) createWidget() Widget {
 func (r *updateWidgetRequest) updatedWidget(oldWidget Widget) Widget {
 	// Use the old Widget as a basis for the updated Widget `s`
 	s := oldWidget
-
+	s.Name = r.Widget.Name
-	if r.Widget.Name != "" {
-		s.Name = r.Widget.Name
-	}
-
 	s.Type = r.Widget.Type
 	s.Width = r.Widget.Width
 	s.Height = r.Widget.Height

start.go

@@ -117,13 +117,13 @@ func main() {
 	routes.RegisterEndpoints(r, api)
 
 	// Start AMQP client
-	AMQPhost, _ := configuration.GlobalConfig.String("amqp.host")
+	amqpHost, _ := configuration.GlobalConfig.String("amqp.host")
-	AMQPuser, _ := configuration.GlobalConfig.String("amqp.user")
+	amqpUser, _ := configuration.GlobalConfig.String("amqp.user")
-	AMQPpass, _ := configuration.GlobalConfig.String("amqp.pass")
+	amqpPass, _ := configuration.GlobalConfig.String("amqp.pass")
 
-	if AMQPhost != "" {
+	if amqpHost != "" {
 		// create amqp URL based on username, password and host
-		amqpurl := "amqp://" + AMQPuser + ":" + AMQPpass + "@" + AMQPhost
+		amqpurl := "amqp://" + amqpUser + ":" + amqpPass + "@" + amqpHost
 		session := helper.NewAMQPSession("villas-amqp-session", amqpurl, "villas", infrastructure_component.ProcessMessage)
 		healthz.SetAMQPSession(session)                  // healthz needs to know the amqp session to check the health of the backend
 		infrastructure_component.SetAMQPSession(session) // IC needs to know the session to send amqp messages
@@ -142,7 +142,7 @@ func main() {
 	}
 
 	// Make sure that at least one admin user exists in DB
-	_, err = database.DBAddAdminUser(configuration.GlobalConfig)
+	_, err = database.AddAdminUser(configuration.GlobalConfig)
 	if err != nil {
 		fmt.Println("error: adding admin user failed:", err.Error())
 		log.Fatal(err)