From 7b30b46b27092bd6d0af801d9f1d4be6f4e02555 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Fri, 17 Sep 2021 20:32:36 +0200 Subject: [PATCH 01/20] Add feature for duplicating scenario on external login --- configuration/config.go | 71 +++-- configuration/groups.yaml | 24 +- helper/amqp.go | 265 +++++++++++++++++ helper/utilities.go | 10 + routes/healthz/healthz_endpoint.go | 3 +- routes/healthz/healthz_test.go | 2 +- .../infrastructure-component/ic_amqpclient.go | 187 +----------- .../infrastructure-component/ic_endpoints.go | 4 +- routes/infrastructure-component/ic_test.go | 98 ++++--- routes/user/authenticate_endpoint.go | 270 +++++++++++++++++- routes/user/user_test.go | 38 +++ start.go | 3 +- 12 files changed, 696 insertions(+), 279 deletions(-) create mode 100644 helper/amqp.go diff --git a/configuration/config.go b/configuration/config.go index b884627..a76d3f7 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -38,7 +38,12 @@ import ( // Global configuration var GlobalConfig *config.Config = nil -var ScenarioGroupMap = map[string][]int{} +type GroupedScenario struct { + Scenario int `yaml:"scenario"` + Duplicate bool `default:"false" yaml:"duplicate"` +} + +var ScenarioGroupMap = map[string][]GroupedScenario{} func InitConfig() error { if GlobalConfig != nil { @@ -182,35 +187,45 @@ func InitConfig() error { return nil } +func remove(arr []GroupedScenario, index int) []GroupedScenario { + arr[index] = arr[len(arr)-1] + return arr[:len(arr)-1] +} + func ReadGroupsFile(path string) error { _, err := os.Stat(path) - - if err == nil { - - yamlFile, err := os.Open(path) - if err != nil { - return fmt.Errorf("error opening yaml file for groups: %v", err) - } - log.Println("Successfully opened yaml groups file", path) - - defer yamlFile.Close() - - byteValue, _ := ioutil.ReadAll(yamlFile) - - err = yaml.Unmarshal(byteValue, &ScenarioGroupMap) - if err != nil { - return fmt.Errorf("error unmarshalling yaml into ScenarioGroupMap: %v", err) - } - - log.Println("ScenarioGroupMap", ScenarioGroupMap) - - return nil - } else if os.IsNotExist(err) { - log.Println("File does not exist, no goups/scenarios mapping created:", path) - return nil - } else { - log.Println("Something is wrong with this file path:", path) - return nil + if err != nil { + return err } + + yamlFile, err := os.Open(path) + if err != nil { + return fmt.Errorf("error opening yaml file for groups: %v", err) + } + log.Println("Successfully opened yaml groups file", path) + + defer yamlFile.Close() + + byteValue, _ := ioutil.ReadAll(yamlFile) + + err = yaml.Unmarshal(byteValue, &ScenarioGroupMap) + if err != nil { + return fmt.Errorf("error unmarshalling yaml into ScenarioGroupMap: %v", err) + } + + for _, group := range ScenarioGroupMap { + for i, scenario := range group { + // remove invalid values that might have been introduced by typos + // (Unmarshal sets default values when it doesn't find a field) + if scenario.Scenario == 0 { + log.Println("Removing entry from ScenarioGroupMap, check for typos in the yaml!") + remove(group, i) + } + } + } + + log.Println("ScenarioGroupMap", ScenarioGroupMap) + + return nil } diff --git a/configuration/groups.yaml b/configuration/groups.yaml index fec1fbf..c30058f 100644 --- a/configuration/groups.yaml +++ b/configuration/groups.yaml @@ -1,12 +1,16 @@ -moodle-l2pmanager@13306: - - 1 - - 2 - - 3 +# Warning: this file is used for testing - please consider when making changes +moodle-l2pmanager@13306: + - scenario: 1 + - scenario: 2 + - scenario: 3 testGroup1: - - 4 - - 5 - - 6 + - scenario: 1 + duplicate: true + - scenario: 2 + duplicate: false + - scenario: 3 testGroup2: - - 7 - - 8 - - 9 + - scenario: 1 + - scenario: 4 + duplicate: true + - scenario: 2 \ No newline at end of file diff --git a/helper/amqp.go b/helper/amqp.go new file mode 100644 index 0000000..7c8420c --- /dev/null +++ b/helper/amqp.go @@ -0,0 +1,265 @@ +/** helper package, AMQP client. +* +* @author Sonja Happ +* @copyright 2014-2021, Institute for Automation of Complex Power Systems, EONERC +* @license GNU General Public License (version 3) +* +* VILLASweb-backend-go +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +*********************************************************************************/ + +package helper + +import ( + "encoding/json" + "fmt" + "log" + "time" + + "git.rwth-aachen.de/acs/public/villas/web-backend-go/database" + "github.com/google/uuid" + "github.com/streadway/amqp" +) + +type AMQPclient struct { + connection *amqp.Connection + sendCh *amqp.Channel + recvCh *amqp.Channel +} + +type Action struct { + Act string `json:"action"` + When int64 `json:"when"` + Parameters json.RawMessage `json:"parameters,omitempty"` + Model json.RawMessage `json:"model,omitempty"` + Results json.RawMessage `json:"results,omitempty"` +} + +var client AMQPclient + +const VILLAS_EXCHANGE = "villas" + +type callback func(amqp.Delivery) error + +func ConnectAMQP(uri string, cb callback) error { + + var err error + + // connect to broker + client.connection, err = amqp.Dial(uri) + if err != nil { + return fmt.Errorf("AMQP: failed to connect to RabbitMQ broker %v, error: %v", uri, err) + } + + // create sendCh + client.sendCh, err = client.connection.Channel() + if err != nil { + return fmt.Errorf("AMQP: failed to open a sendCh, error: %v", err) + } + // declare exchange + err = client.sendCh.ExchangeDeclare(VILLAS_EXCHANGE, + "headers", + true, + false, + false, + false, + nil) + if err != nil { + return fmt.Errorf("AMQP: failed to declare the exchange, error: %v", err) + } + + // add a queue for the ICs + ICQueue, err := client.sendCh.QueueDeclare("infrastructure_components", + true, + false, + false, + false, + nil) + if err != nil { + return fmt.Errorf("AMQP: failed to declare the queue, error: %v", err) + } + + err = client.sendCh.QueueBind(ICQueue.Name, "", VILLAS_EXCHANGE, false, nil) + if err != nil { + return fmt.Errorf("AMQP: failed to bind the queue, error: %v", err) + } + + // create receive channel + client.recvCh, err = client.connection.Channel() + if err != nil { + return fmt.Errorf("AMQP: failed to open a recvCh, error: %v", err) + } + + // start deliveries + messages, err := client.recvCh.Consume(ICQueue.Name, + "", + true, + false, + false, + false, + nil) + if err != nil { + return fmt.Errorf("AMQP: failed to start deliveries: %v", err) + } + + // consume deliveries + go func() { + for { + for message := range messages { + err = cb(message) + if err != nil { + log.Println("AMQP: Error processing message: ", err.Error()) + } + } + } + }() + + log.Printf(" AMQP: Waiting for messages... ") + + return nil +} + +func SendActionAMQP(action Action, destinationUUID string) error { + + payload, err := json.Marshal(action) + if err != nil { + return err + } + + msg := amqp.Publishing{ + DeliveryMode: 2, + Timestamp: time.Now(), + ContentType: "application/json", + ContentEncoding: "utf-8", + Priority: 0, + Body: payload, + } + + // set message headers + msg.Headers = make(map[string]interface{}) // empty map + msg.Headers["uuid"] = destinationUUID + + err = CheckConnection() + if err != nil { + return err + } + + //log.Println("AMQP: Sending message", string(msg.Body)) + err = client.sendCh.Publish(VILLAS_EXCHANGE, + "", + false, + false, + msg) + return PublishAMQP(msg) + +} + +func PublishAMQP(msg amqp.Publishing) error { + err := client.sendCh.Publish(VILLAS_EXCHANGE, + "", + false, + false, + msg) + return err +} + +func SendPing(uuid string) error { + var ping Action + ping.Act = "ping" + + payload, err := json.Marshal(ping) + if err != nil { + return err + } + + msg := amqp.Publishing{ + DeliveryMode: 2, + Timestamp: time.Now(), + ContentType: "application/json", + ContentEncoding: "utf-8", + Priority: 0, + Body: payload, + } + + // set message headers + msg.Headers = make(map[string]interface{}) // empty map + msg.Headers["uuid"] = uuid // leave uuid empty if ping should go to all ICs + + err = CheckConnection() + if err != nil { + return err + } + + err = client.sendCh.Publish(VILLAS_EXCHANGE, + "", + false, + false, + msg) + return err +} + +func CheckConnection() error { + + if client.connection != nil { + if client.connection.IsClosed() { + return fmt.Errorf("connection to broker is closed") + } + } else { + return fmt.Errorf("connection is nil") + } + + return nil +} + +func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID string) (string, error) { + newUUID := uuid.New().String() + // TODO: where to get the properties part from? + msg := `{"name": "` + ic.Name + `",` + + `"location": "` + ic.Location + `",` + + `"category": "` + ic.Category + `",` + + `"type": "` + ic.Type + `",` + + `"uuid": "` + newUUID + `",` + + `"realm": "de.rwth-aachen.eonerc.acs",` + + `"properties": {` + + `"job": {` + + `"apiVersion": "batch/v1",` + + `"kind": "Job",` + + `"metadata": {` + + `"name": "dpsim"` + + `},` + + `"spec": {` + + `"activeDeadlineSeconds": 3600,` + + `"backoffLimit": 1,` + + `"ttlSecondsAfterFinished": 3600,` + + `"template": {` + + `"spec": {` + + `"restartPolicy": "Never",` + + `"containers": [{` + + `"image": "dpsimrwth/slew-villas",` + + `"name": "slew-dpsim"` + + `}]}}}}}}` + + log.Print(msg) + + actionCreate := Action{ + Act: "create", + When: time.Now().Unix(), + Parameters: json.RawMessage(msg), + } + + err := SendActionAMQP(actionCreate, managerUUID) + + return newUUID, err +} diff --git a/helper/utilities.go b/helper/utilities.go index 1a2f795..4cb600e 100644 --- a/helper/utilities.go +++ b/helper/utilities.go @@ -62,3 +62,13 @@ func Find(slice []string, val string) (int, bool) { } return -1, false } + +// Returns whether slice contains the given element +func Contains(slice []uint, element uint) bool { + for _, elem := range slice { + if elem == element { + return true + } + } + return false +} diff --git a/routes/healthz/healthz_endpoint.go b/routes/healthz/healthz_endpoint.go index 6c3a546..eb5f65e 100644 --- a/routes/healthz/healthz_endpoint.go +++ b/routes/healthz/healthz_endpoint.go @@ -29,7 +29,6 @@ import ( "git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration" "git.rwth-aachen.de/acs/public/villas/web-backend-go/database" "git.rwth-aachen.de/acs/public/villas/web-backend-go/helper" - infrastructure_component "git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/infrastructure-component" "github.com/gin-gonic/gin" ) @@ -69,7 +68,7 @@ func getHealth(c *gin.Context) { } if len(url) != 0 { - err = infrastructure_component.CheckConnection() + err = helper.CheckConnection() if err != nil { log.Println(err.Error()) c.JSON(http.StatusInternalServerError, gin.H{ diff --git a/routes/healthz/healthz_test.go b/routes/healthz/healthz_test.go index 84c6a77..6202654 100644 --- a/routes/healthz/healthz_test.go +++ b/routes/healthz/healthz_test.go @@ -80,7 +80,7 @@ func TestHealthz(t *testing.T) { amqpURI := "amqp://" + user + ":" + pass + "@" + host log.Println("AMQP URI is", amqpURI) - err = infrastructure_component.ConnectAMQP(amqpURI) + err = helper.ConnectAMQP(amqpURI, infrastructure_component.ProcessMessage) assert.NoError(t, err) // test healthz endpoint for connected DB and AMQP client diff --git a/routes/infrastructure-component/ic_amqpclient.go b/routes/infrastructure-component/ic_amqpclient.go index 126e580..3f129ab 100644 --- a/routes/infrastructure-component/ic_amqpclient.go +++ b/routes/infrastructure-component/ic_amqpclient.go @@ -26,8 +26,8 @@ import ( "fmt" "log" "strings" - "time" + "git.rwth-aachen.de/acs/public/villas/web-backend-go/helper" "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/jinzhu/gorm" @@ -35,22 +35,6 @@ import ( "github.com/streadway/amqp" ) -const VILLAS_EXCHANGE = "villas" - -type AMQPclient struct { - connection *amqp.Connection - sendCh *amqp.Channel - recvCh *amqp.Channel -} - -type Action struct { - Act string `json:"action"` - When int64 `json:"when"` - Parameters json.RawMessage `json:"parameters,omitempty"` - Model json.RawMessage `json:"model,omitempty"` - Results json.RawMessage `json:"results,omitempty"` -} - type ICStatus struct { State string `json:"state"` Version string `json:"version"` @@ -85,174 +69,11 @@ type ICUpdate struct { Action string `json:"action"` } -var client AMQPclient - -func ConnectAMQP(uri string) error { - - var err error - - // connect to broker - client.connection, err = amqp.Dial(uri) - if err != nil { - return fmt.Errorf("AMQP: failed to connect to RabbitMQ broker %v, error: %v", uri, err) - } - - // create sendCh - client.sendCh, err = client.connection.Channel() - if err != nil { - return fmt.Errorf("AMQP: failed to open a sendCh, error: %v", err) - } - // declare exchange - err = client.sendCh.ExchangeDeclare(VILLAS_EXCHANGE, - "headers", - true, - false, - false, - false, - nil) - if err != nil { - return fmt.Errorf("AMQP: failed to declare the exchange, error: %v", err) - } - - // add a queue for the ICs - ICQueue, err := client.sendCh.QueueDeclare("infrastructure_components", - true, - false, - false, - false, - nil) - if err != nil { - return fmt.Errorf("AMQP: failed to declare the queue, error: %v", err) - } - - err = client.sendCh.QueueBind(ICQueue.Name, "", VILLAS_EXCHANGE, false, nil) - if err != nil { - return fmt.Errorf("AMQP: failed to bind the queue, error: %v", err) - } - - // create receive channel - client.recvCh, err = client.connection.Channel() - if err != nil { - return fmt.Errorf("AMQP: failed to open a recvCh, error: %v", err) - } - - // start deliveries - messages, err := client.recvCh.Consume(ICQueue.Name, - "", - true, - false, - false, - false, - nil) - if err != nil { - return fmt.Errorf("AMQP: failed to start deliveries: %v", err) - } - - // consume deliveries - go func() { - for { - for message := range messages { - err = processMessage(message) - if err != nil { - log.Println("AMQP: Error processing message: ", err.Error()) - } - } - } - }() - - log.Printf(" AMQP: Waiting for messages... ") - - return nil -} - -func sendActionAMQP(action Action, destinationUUID string) error { - - payload, err := json.Marshal(action) - if err != nil { - return err - } - - msg := amqp.Publishing{ - DeliveryMode: 2, - Timestamp: time.Now(), - ContentType: "application/json", - ContentEncoding: "utf-8", - Priority: 0, - Body: payload, - } - - // set message headers - msg.Headers = make(map[string]interface{}) // empty map - msg.Headers["uuid"] = destinationUUID - - err = CheckConnection() - if err != nil { - return err - } - - //log.Println("AMQP: Sending message", string(msg.Body)) - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) - return err - -} - -func SendPing(uuid string) error { - var ping Action - ping.Act = "ping" - - payload, err := json.Marshal(ping) - if err != nil { - return err - } - - msg := amqp.Publishing{ - DeliveryMode: 2, - Timestamp: time.Now(), - ContentType: "application/json", - ContentEncoding: "utf-8", - Priority: 0, - Body: payload, - } - - // set message headers - msg.Headers = make(map[string]interface{}) // empty map - msg.Headers["uuid"] = uuid // leave uuid empty if ping should go to all ICs - - err = CheckConnection() - if err != nil { - return err - } - - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) - return err -} - -func CheckConnection() error { - - if client.connection != nil { - if client.connection.IsClosed() { - return fmt.Errorf("connection to broker is closed") - } - } else { - return fmt.Errorf("connection is nil") - } - - return nil -} - func StartAMQP(AMQPurl string, api *gin.RouterGroup) error { if AMQPurl != "" { log.Println("Starting AMQP client") - err := ConnectAMQP(AMQPurl) + err := helper.ConnectAMQP(AMQPurl, ProcessMessage) if err != nil { return err } @@ -266,7 +87,7 @@ func StartAMQP(AMQPurl string, api *gin.RouterGroup) error { return nil } -func processMessage(message amqp.Delivery) error { +func ProcessMessage(message amqp.Delivery) error { var payload ICUpdate err := json.Unmarshal(message.Body, &payload) @@ -358,7 +179,7 @@ func createExternalIC(payload ICUpdate, ICUUID string, body []byte) error { log.Println("AMQP: Created IC with UUID ", newIC.UUID) // send ping to get full status update of this IC - err = SendPing(ICUUID) + err = helper.SendPing(ICUUID) return err } diff --git a/routes/infrastructure-component/ic_endpoints.go b/routes/infrastructure-component/ic_endpoints.go index e26198b..0f1471c 100644 --- a/routes/infrastructure-component/ic_endpoints.go +++ b/routes/infrastructure-component/ic_endpoints.go @@ -289,7 +289,7 @@ func sendActionToIC(c *gin.Context) { return } - var actions []Action + var actions []helper.Action err := c.BindJSON(&actions) if err != nil { helper.BadRequestError(c, "Error binding form data to JSON: "+err.Error()) @@ -300,7 +300,7 @@ func sendActionToIC(c *gin.Context) { if (action.Act == "delete" || action.Act == "create") && s.Category != "manager" { helper.BadRequestError(c, "cannot send a delete or create action to an IC of category "+s.Category) } - err = sendActionAMQP(action, s.UUID) + err = helper.SendActionAMQP(action, s.UUID) if err != nil { helper.InternalServerError(c, "Unable to send actions to IC: "+err.Error()) return diff --git a/routes/infrastructure-component/ic_test.go b/routes/infrastructure-component/ic_test.go index 64f193b..2144700 100644 --- a/routes/infrastructure-component/ic_test.go +++ b/routes/infrastructure-component/ic_test.go @@ -134,7 +134,7 @@ func TestMain(m *testing.M) { scenario.RegisterScenarioEndpoints(api.Group("/scenarios")) // check AMQP connection - err = CheckConnection() + err = helper.CheckConnection() if err.Error() != "connection is nil" { return } @@ -320,14 +320,10 @@ func TestUpdateICAsAdmin(t *testing.T) { Headers: headers, } - err = CheckConnection() + err = helper.CheckConnection() assert.NoError(t, err) - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) + err = helper.PublishAMQP(msg) assert.NoError(t, err) // Wait until externally managed IC is created (happens async) @@ -377,6 +373,48 @@ func TestUpdateICAsUser(t *testing.T) { } +func TestCreateICviaAMQP(t *testing.T) { + if os.Getenv("CI") != "" { + t.Skip("Skipping testing in CI environment") + } + database.DropTables() + database.MigrateModels() + assert.NoError(t, helper.AddTestUsers()) + + // authenticate as user + token, err := helper.AuthenticateForTest(router, helper.UserACredentials) + assert.NoError(t, err) + + // Count the number of all the ICs before sending Action + numberOfICs, err := helper.LengthOfResponse(router, token, + "/api/v2/ic", "GET", nil) + assert.NoError(t, err) + assert.Equal(t, 0, numberOfICs) + + err = helper.CheckConnection() + assert.NoError(t, err) + + var ic database.InfrastructureComponent + ic.Name = "kubernetes simulator dpsim (backend test)" + ic.Location = "iko backend dev" + ic.Category = "simulator" + ic.Type = "kubernetes" + + // send create Action to kubernetes manager via AMQP broker + uuidManager := "444fb73e-7e74-11eb-8f63-f3a5b3ab82f6" + _, err = helper.RequestICcreateAMQP(&ic, uuidManager) + assert.NoError(t, err) + + // Wait until externally managed IC is created (happens async) + time.Sleep(2 * time.Second) + + // check whether an external IC was created + numberOfICs, err = helper.LengthOfResponse(router, token, + "/api/v2/ic", "GET", nil) + assert.NoError(t, err) + assert.Equal(t, 1, numberOfICs) +} + func TestDeleteICAsAdmin(t *testing.T) { database.DropTables() database.MigrateModels() @@ -444,14 +482,10 @@ func TestDeleteICAsAdmin(t *testing.T) { Headers: headers, } - err = CheckConnection() + err = helper.CheckConnection() assert.NoError(t, err) - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) + err = helper.PublishAMQP(msg) assert.NoError(t, err) // Wait until externally managed IC is created (happens async) @@ -617,7 +651,7 @@ func TestSendActionToIC(t *testing.T) { assert.NoError(t, err) // create action to be sent to IC - action1 := Action{ + action1 := helper.Action{ Act: "start", When: time.Now().Unix(), } @@ -630,7 +664,7 @@ func TestSendActionToIC(t *testing.T) { paramsRaw, err := json.Marshal(¶ms) action1.Parameters = paramsRaw - actions := [1]Action{action1} + actions := [1]helper.Action{action1} // Send action to IC code, resp, err = helper.TestEndpoint(router, token, @@ -676,14 +710,10 @@ func TestCreateUpdateViaAMQPRecv(t *testing.T) { Headers: headers, } - err = CheckConnection() + err = helper.CheckConnection() assert.NoError(t, err) - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) + err = helper.PublishAMQP(msg) assert.NoError(t, err) time.Sleep(waitingTime * time.Second) @@ -723,11 +753,7 @@ func TestCreateUpdateViaAMQPRecv(t *testing.T) { Headers: headersA, } - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) + err = helper.PublishAMQP(msg) assert.NoError(t, err) time.Sleep(waitingTime * time.Second) @@ -753,11 +779,7 @@ func TestCreateUpdateViaAMQPRecv(t *testing.T) { Headers: headersA, } - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) + err = helper.PublishAMQP(msg) assert.NoError(t, err) time.Sleep(waitingTime * time.Second) @@ -811,13 +833,9 @@ func TestDeleteICViaAMQPRecv(t *testing.T) { Headers: headers, } - err = CheckConnection() + err = helper.CheckConnection() assert.NoError(t, err) - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) + err = helper.PublishAMQP(msg) assert.NoError(t, err) time.Sleep(waitingTime * time.Second) @@ -885,11 +903,7 @@ func TestDeleteICViaAMQPRecv(t *testing.T) { } // attempt to delete IC (should not work immediately because IC is still associated with component config) - err = client.sendCh.Publish(VILLAS_EXCHANGE, - "", - false, - false, - msg) + err = helper.PublishAMQP(msg) assert.NoError(t, err) time.Sleep(waitingTime * time.Second) diff --git a/routes/user/authenticate_endpoint.go b/routes/user/authenticate_endpoint.go index 1715f9e..8b466af 100644 --- a/routes/user/authenticate_endpoint.go +++ b/routes/user/authenticate_endpoint.go @@ -220,6 +220,241 @@ func authenticateInternal(c *gin.Context) (User, error) { return myUser, nil } +func duplicateDashboards(originalSo *database.Scenario, duplicateSo *database.Scenario, + signalMap map[uint]uint, appendix string) error { + + db := database.GetDB() + var dabs []database.Dashboard + err := db.Order("ID asc").Model(originalSo).Related(&dabs, "Dashboards").Error + if err != nil { + log.Printf("error getting dashboards for scenario %d", originalSo.ID) + } + + for _, dab := range dabs { + var duplicateD database.Dashboard + duplicateD.Grid = dab.Grid + duplicateD.Name = dab.Name + appendix + duplicateD.ScenarioID = duplicateSo.ID + duplicateD.Height = dab.Height + err = db.Create(&duplicateD).Error + if err != nil { + log.Print("error creating duplicate dashboard") + continue + } + + // add widgets to duplicated dashboards + var widgets []database.Widget + err = db.Order("ID asc").Model(&dab).Related(&widgets, "Widgets").Error + if err != nil { + log.Printf("error getting widgets for dashboard %d", dab.ID) + } + for _, widget := range widgets { + var duplicateW database.Widget + duplicateW.DashboardID = duplicateD.ID + duplicateW.CustomProperties = widget.CustomProperties + duplicateW.Height = widget.Height + duplicateW.Width = widget.Width + duplicateW.MinHeight = widget.MinHeight + duplicateW.MinWidth = widget.MinWidth + duplicateW.Name = widget.Name + duplicateW.Type = widget.Type + duplicateW.X = widget.X + duplicateW.Y = widget.Y + + duplicateW.SignalIDs = []int64{} + for _, id := range widget.SignalIDs { + duplicateW.SignalIDs = append(duplicateW.SignalIDs, int64(signalMap[uint(id)])) + } + + err = db.Create(&duplicateW).Error + if err != nil { + log.Print("error creating duplicate widget") + continue + } + // associate dashboard with simulation + err = db.Model(&duplicateD).Association("Widgets").Append(&duplicateW).Error + if err != nil { + log.Print("error associating duplicate widget and dashboard") + } + } + + } + return nil +} + +func duplicateComponentConfig(config *database.ComponentConfiguration, + duplicateSo *database.Scenario, icIds map[uint]string, appendix string, signalMap *map[uint]uint) error { + var configDpl database.ComponentConfiguration + configDpl.Name = config.Name + configDpl.StartParameters = config.StartParameters + configDpl.ScenarioID = duplicateSo.ID + configDpl.OutputLength = config.OutputLength + configDpl.InputLength = config.InputLength + + db := database.GetDB() + if icIds[config.ICID] == "" { + configDpl.ICID = config.ICID + } else { + var duplicatedIC database.InfrastructureComponent + err := db.Find(&duplicatedIC, "UUID = ?", icIds[config.ICID]).Error + if err != nil { + log.Print(err) + return err + } + configDpl.ICID = duplicatedIC.ID + } + err := db.Create(&configDpl).Error + if err != nil { + log.Print(err) + return err + } + + // get all signals corresponding to component config + var sigs []database.Signal + err = db.Order("ID asc").Model(&config).Related(&sigs, "OutputMapping").Error + smap := *signalMap + for _, signal := range sigs { + var sig database.Signal + sig.Direction = signal.Direction + sig.Index = signal.Index + sig.Name = signal.Name + appendix + sig.ScalingFactor = signal.ScalingFactor + sig.Unit = signal.Unit + sig.ConfigID = configDpl.ID + err = db.Create(&sig).Error + if err == nil { + smap[signal.ID] = sig.ID + } + } + + return err +} + +func duplicateScenario(so *database.Scenario, duplicateSo *database.Scenario, icIds map[uint]string, appendix string) error { + duplicateSo.Name = so.Name + appendix + duplicateSo.StartParameters.RawMessage = so.StartParameters.RawMessage + db := database.GetDB() + err := db.Create(&duplicateSo).Error + if err != nil { + log.Printf("Could not create duplicate of scenario %d", so.ID) + return err + } + log.Print("created duplicate scenario") + + var configs []database.ComponentConfiguration + // map existing signal IDs to duplicated signal IDs for widget duplication + signalMap := make(map[uint]uint) + err = db.Order("ID asc").Model(so).Related(&configs, "ComponentConfigurations").Error + if err == nil { + for _, config := range configs { + err = duplicateComponentConfig(&config, duplicateSo, icIds, appendix, &signalMap) + if err != nil { + return err + } + } + + } + + err = duplicateDashboards(so, duplicateSo, signalMap, appendix) + return err +} + +func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { + go func() { + + // get all component configs of the scenario + db := database.GetDB() + var configs []database.ComponentConfiguration + err := db.Order("ID asc").Model(so).Related(&configs, "ComponentConfigurations").Error + if err != nil { + log.Printf("Warning: scenario to duplicate (id=%d) has no component configurations", so.ID) + } + + // iterate over component configs to check for ICs to duplicate + duplicatedICuuids := make(map[uint]string) // key: icID; value: UUID of duplicate + var externalUUIDs []string // external ICs to wait for + for _, config := range configs { + icID := config.ICID + if duplicatedICuuids[icID] != "" { // this IC was already added + continue + } + + var ic database.InfrastructureComponent + err = db.Find(&ic, icID).Error + if err != nil { + log.Printf("Cannot find IC with id %d in DB, will not duplicate for User %s", icID, user.Username) + continue + } + + if ic.Category == "simulator" && ic.Type == "kubernetes" { + duplicateUUID, err := helper.RequestICcreateAMQP(&ic, ic.Manager) + duplicatedICuuids[ic.ID] = duplicateUUID + + if err != nil { // TODO: should this function call be interrupted here? + log.Printf("Duplication of IC (id=%d) unsuccessful", icID) + continue + } + externalUUIDs = append(externalUUIDs, duplicateUUID) + } else { // use existing IC + duplicatedICuuids[ic.ID] = "" + err = nil + } + } + + // copy scenario after all new external ICs are in DB + icsToWaitFor := len(externalUUIDs) + var duplicatedScenario database.Scenario + var timeout = 5 // seconds + + for i := 0; i < timeout; i++ { + log.Printf("i = %d", i) + if icsToWaitFor == 0 { + appendix := fmt.Sprintf("--%s-%d-%d", user.Username, user.ID, so.ID) + duplicateScenario(so, &duplicatedScenario, duplicatedICuuids, appendix) + + // associate user to new scenario + err = db.Model(&duplicatedScenario).Association("Users").Append(user).Error + if err != nil { + log.Printf("Could not associate User %s to scenario %d", user.Username, duplicatedScenario.ID) + } + log.Print("associated user to duplicated scenario") + + return + } else { + time.Sleep(1 * time.Second) + } + + // check for new ICs with previously created UUIDs + for _, uuid := range externalUUIDs { + if uuid == "" { + continue + } + log.Printf("looking for IC with UUID %s", uuid) + var duplicatedIC database.InfrastructureComponent + err = db.Find(&duplicatedIC, "UUID = ?", uuid).Error + // TODO: check if not found or other error + if err != nil { + log.Print(err) + } else { + icsToWaitFor-- + uuid = "" + } + } + } + }() +} + +func isAlreadyDuplicated(duplicatedName string) bool { + db := database.GetDB() + var scenarios []database.Scenario + + db.Find(&scenarios, "name = ?", duplicatedName) + if len(scenarios) > 0 { + return true + } + return false +} + func authenticateExternal(c *gin.Context) (User, error) { var myUser User username := c.Request.Header.Get("X-Forwarded-User") @@ -237,7 +472,11 @@ func authenticateExternal(c *gin.Context) (User, error) { groups := strings.Split(c.Request.Header.Get("X-Forwarded-Groups"), ",") // preferred_username := c.Request.Header.Get("X-Forwarded-Preferred-Username") - if err := myUser.ByUsername(username); err != nil { + // check if user already exists + err := myUser.ByUsername(username) + + if err != nil { + // this is the first login, create new user role := "User" if _, found := helper.Find(groups, "admin"); found { role = "Admin" @@ -255,22 +494,33 @@ func authenticateExternal(c *gin.Context) (User, error) { // Add users to scenarios based on static map db := database.GetDB() for _, group := range groups { - if soIDs, ok := configuration.ScenarioGroupMap[group]; ok { - for _, soID := range soIDs { + if groupedArr, ok := configuration.ScenarioGroupMap[group]; ok { + for _, groupedScenario := range groupedArr { var so database.Scenario - err := db.Find(&so, soID).Error + err := db.Find(&so, groupedScenario.Scenario).Error if err != nil { - log.Printf("Failed to add user %s (id=%d) to scenario %s (id=%d): %s\n", myUser.Username, myUser.ID, so.Name, so.ID, err) + log.Printf(`Cannot find scenario %s (id=%d) for adding/duplication. + Affecting user %s (id=%d): %s\n`, so.Name, so.ID, myUser.Username, myUser.ID, err) continue } - err = db.Model(&so).Association("Users").Append(&(myUser.User)).Error - if err != nil { - log.Printf("Failed to add user %s (id=%d) to scenario %s (id=%d): %s\n", myUser.Username, myUser.ID, so.Name, so.ID, err) - continue + duplicateName := fmt.Sprintf("%s--%s-%d-%d", so.Name, myUser.Username, myUser.ID, so.ID) + alreadyDuplicated := isAlreadyDuplicated(duplicateName) + if alreadyDuplicated { + log.Printf("Scenario %d already duplicated for user %s", so.ID, myUser.Username) + return myUser, nil } - log.Printf("Added user %s (id=%d) to scenario %s (id=%d)", myUser.Username, myUser.ID, so.Name, so.ID) + if groupedScenario.Duplicate { + DuplicateScenarioForUser(&so, &myUser.User) + } else { + err = db.Model(&so).Association("Users").Append(&(myUser.User)).Error + if err != nil { + log.Printf("Failed to add user %s (id=%d) to scenario %s (id=%d): %s\n", myUser.Username, myUser.ID, so.Name, so.ID, err) + continue + } + log.Printf("Added user %s (id=%d) to scenario %s (id=%d)", myUser.Username, myUser.ID, so.Name, so.ID) + } } } } diff --git a/routes/user/user_test.go b/routes/user/user_test.go index 12db59d..c109cba 100644 --- a/routes/user/user_test.go +++ b/routes/user/user_test.go @@ -28,6 +28,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "git.rwth-aachen.de/acs/public/villas/web-backend-go/helper" @@ -130,6 +131,43 @@ func TestAuthenticate(t *testing.T) { } +func TestUserGroups(t *testing.T) { + // Create new user + // (user, email and groups are read from request headers in real case) + var myUser User + username := "Fridolin" + email := "Fridolin@rwth-aachen.de" + role := "User" + userGroups := strings.Split("testGroup1,testGroup2", ",") + + err := myUser.ByUsername(username) + assert.Error(t, err) + myUser, err = NewUser(username, "", email, role, true) + assert.NoError(t, err) + + // Read groups file + err = configuration.ReadGroupsFile("notexisting.yaml") + assert.Error(t, err) + + err = configuration.ReadGroupsFile("../../configuration/groups.yaml") + assert.NoError(t, err) + + // Check whether duplicate flag is saved correctly in configuration + for _, group := range userGroups { + if gsarray, ok := configuration.ScenarioGroupMap[group]; ok { + for _, groupedScenario := range gsarray { + if group == "testGroup1" && groupedScenario.Scenario == 1 { + assert.Equal(t, true, groupedScenario.Duplicate) + } else if group == "testGroup2" && groupedScenario.Scenario == 4 { + assert.Equal(t, true, groupedScenario.Duplicate) + } else { + assert.Equal(t, false, groupedScenario.Duplicate) + } + } + } + } +} + func TestAuthenticateQueryToken(t *testing.T) { database.DropTables() diff --git a/start.go b/start.go index b86a1bc..cd1433a 100644 --- a/start.go +++ b/start.go @@ -28,6 +28,7 @@ import ( "git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration" "git.rwth-aachen.de/acs/public/villas/web-backend-go/database" + "git.rwth-aachen.de/acs/public/villas/web-backend-go/helper" "git.rwth-aachen.de/acs/public/villas/web-backend-go/routes" infrastructure_component "git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/infrastructure-component" "github.com/gin-gonic/gin" @@ -128,7 +129,7 @@ func main() { } // send Ping to all externally managed ICs - err = infrastructure_component.SendPing("") + err = helper.SendPing("") if err != nil { log.Println("error sending ping action via AMQP: ", err) } From 58b1d4e303f467227bbf0b60cb6ea12e5c6a4db3 Mon Sep 17 00:00:00 2001 From: irismarie Date: Fri, 17 Sep 2021 20:44:26 +0200 Subject: [PATCH 02/20] fix signal mapping in component config duplication --- routes/user/authenticate_endpoint.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/user/authenticate_endpoint.go b/routes/user/authenticate_endpoint.go index 8b466af..b98c0c8 100644 --- a/routes/user/authenticate_endpoint.go +++ b/routes/user/authenticate_endpoint.go @@ -288,8 +288,8 @@ func duplicateComponentConfig(config *database.ComponentConfiguration, configDpl.Name = config.Name configDpl.StartParameters = config.StartParameters configDpl.ScenarioID = duplicateSo.ID - configDpl.OutputLength = config.OutputLength - configDpl.InputLength = config.InputLength + configDpl.OutputMapping = config.OutputMapping + configDpl.InputMapping = config.InputMapping db := database.GetDB() if icIds[config.ICID] == "" { From 4270bbda6f1ad97137a63ab017c8beab5c657f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Iris=20Marie=20K=C3=B6ster?= Date: Wed, 22 Sep 2021 10:14:49 +0200 Subject: [PATCH 03/20] do not read groups file by default --- configuration/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configuration/config.go b/configuration/config.go index a76d3f7..1c316e9 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -81,7 +81,7 @@ func InitConfig() error { contactName = flag.String("contact-name", "Steffen Vogel", "Name of the administrative contact") contactMail = flag.String("contact-mail", "svogel2@eonerc.rwth-aachen.de", "EMail of the administrative contact") testDataPath = flag.String("test-data-path", "", "The path to a test data json file") - groupsPath = flag.String("groups-path", "configuration/groups.yaml", "The path to a YAML file that maps user groups to scenario IDs") + groupsPath = flag.String("groups-path", "", "The path to a YAML file that maps user groups to scenario IDs") apiUpdateInterval = flag.String("api-update-interval", "10s" /* 10 sec */, "Interval in which API URL is queried for status updates of ICs") rancherURL = flag.String("rancher-url", "rancher.k8s.eonerc.rwth-aachen.de", "URL of Rancher instance that is used to deploy the backend") k8sCluster = flag.String("k8s-cluster", "local", "Name of the Kubernetes cluster where the backend is deployed") From ab1a4d9a29f805c9c47826762eda1e8b8230f227 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Wed, 22 Sep 2021 13:39:54 +0200 Subject: [PATCH 04/20] groups.path is not a mandatory parameter; ignore error upon parameter parsing and print warning if this parameter is not set --- start.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/start.go b/start.go index cd1433a..992df45 100644 --- a/start.go +++ b/start.go @@ -89,16 +89,15 @@ func main() { log.Fatalf("Error reading port from global configuration: %s, aborting.", err) } - gPath, err := configuration.GlobalConfig.String("groups.path") - if err != nil { - log.Fatalf("Error reading path to groups YAML file: %s, aborting.", err) - } + gPath, _ := configuration.GlobalConfig.String("groups.path") if gPath != "" { err = configuration.ReadGroupsFile(gPath) if err != nil { log.Fatalf("Error reading groups YAML file: %s, aborting.", err) } + } else { + log.Println("WARNING: path to groups yaml file not set, I am not initializing the scenario-groups mapping.") } // Init database From 3a0da86d92f3c5ea47ee0eedb01a4dfdc1f6b34d Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Wed, 22 Sep 2021 13:51:10 +0200 Subject: [PATCH 05/20] CI: use debian version 9 of postgresql (see #78) --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d8841b7..a9e4d9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ test: BASE_PATH: /api MODE: release services: - - postgres:latest + - postgres:9-buster - rabbitmq:3.8 - name: minio/minio:RELEASE.2021-01-16T02-19-44Z command: ['server', '/minio'] From d596c8a4bf7084ff6eca97f22890535d1b9e1f5e Mon Sep 17 00:00:00 2001 From: irismarie Date: Fri, 24 Sep 2021 16:48:39 +0200 Subject: [PATCH 06/20] use properties of existing IC for duplication --- helper/amqp.go | 62 +++++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/helper/amqp.go b/helper/amqp.go index 7c8420c..cac4db1 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -47,6 +47,23 @@ type Action struct { Results json.RawMessage `json:"results,omitempty"` } +type ICPropertiesToCopy struct { + Job json.RawMessage `json:"job"` + UUID string `json:"uuid"` + Name string `json:"name"` + Description string `json:"description"` + Location string `json:"location"` + Owner string `json:"owner"` + Category string `json:"category"` + Type string `json:"type"` +} + +type ICUpdateToCopy struct { + Properties ICPropertiesToCopy `json:"properties"` + Status json.RawMessage `json:"status"` + Schema json.RawMessage `json:"schema"` +} + var client AMQPclient const VILLAS_EXCHANGE = "villas" @@ -225,31 +242,28 @@ func CheckConnection() error { func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID string) (string, error) { newUUID := uuid.New().String() - // TODO: where to get the properties part from? - msg := `{"name": "` + ic.Name + `",` + - `"location": "` + ic.Location + `",` + - `"category": "` + ic.Category + `",` + - `"type": "` + ic.Type + `",` + + + var lastUpdate ICUpdateToCopy + err := json.Unmarshal(ic.StatusUpdateRaw.RawMessage, &lastUpdate) + if err != nil { + return newUUID, err + } + + var jobdef string + err = json.Unmarshal(lastUpdate.Properties.Job, &jobdef) + if err != nil { + return newUUID, err + } + + msg := `{"name": "` + lastUpdate.Properties.Name + `",` + + `"description": "` + lastUpdate.Properties.Description + `",` + + `"location": "` + lastUpdate.Properties.Location + `",` + + `"category": "` + lastUpdate.Properties.Category + `",` + + `"type": "` + lastUpdate.Properties.Type + `",` + `"uuid": "` + newUUID + `",` + - `"realm": "de.rwth-aachen.eonerc.acs",` + `"properties": {` + - `"job": {` + - `"apiVersion": "batch/v1",` + - `"kind": "Job",` + - `"metadata": {` + - `"name": "dpsim"` + - `},` + - `"spec": {` + - `"activeDeadlineSeconds": 3600,` + - `"backoffLimit": 1,` + - `"ttlSecondsAfterFinished": 3600,` + - `"template": {` + - `"spec": {` + - `"restartPolicy": "Never",` + - `"containers": [{` + - `"image": "dpsimrwth/slew-villas",` + - `"name": "slew-dpsim"` + - `}]}}}}}}` + `"job": "` + jobdef + `",` + + `}}` log.Print(msg) @@ -259,7 +273,7 @@ func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID strin Parameters: json.RawMessage(msg), } - err := SendActionAMQP(actionCreate, managerUUID) + err = SendActionAMQP(actionCreate, managerUUID) return newUUID, err } From e65763e8ab8327534c2f52f26523fbfb7bba4ce3 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 30 Sep 2021 16:02:01 +0200 Subject: [PATCH 07/20] AMQP: declare exchange and queue to be non-durable and auto-deleted to avoid pollution of the broker --- helper/amqp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helper/amqp.go b/helper/amqp.go index 7c8420c..d2c6774 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -71,8 +71,8 @@ func ConnectAMQP(uri string, cb callback) error { // declare exchange err = client.sendCh.ExchangeDeclare(VILLAS_EXCHANGE, "headers", - true, false, + true, false, false, nil) @@ -82,8 +82,8 @@ func ConnectAMQP(uri string, cb callback) error { // add a queue for the ICs ICQueue, err := client.sendCh.QueueDeclare("infrastructure_components", - true, false, + true, false, false, nil) From 7c139ae97472362380053af32d201b4e6ea026f7 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 5 Oct 2021 18:42:15 +0200 Subject: [PATCH 08/20] amqp: limit queue size and message TTL as well as make queue exclusive --- helper/amqp.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/helper/amqp.go b/helper/amqp.go index d2c6774..56ee51f 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -81,12 +81,15 @@ func ConnectAMQP(uri string, cb callback) error { } // add a queue for the ICs - ICQueue, err := client.sendCh.QueueDeclare("infrastructure_components", + ICQueue, err := client.sendCh.QueueDeclare("", false, true, + true, false, - false, - nil) + amqp.Table{ + "x-max-length-bytes": int32(32 << 20), + "x-message-ttl": int32(10 * 60), + }) if err != nil { return fmt.Errorf("AMQP: failed to declare the queue, error: %v", err) } From 44e57a5f6fc5a62f02aac5eff21daf033bad7ba6 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Tue, 5 Oct 2021 18:54:00 +0200 Subject: [PATCH 09/20] ci: upgrade postgres service to 12.0 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9e4d9d..cff1cbb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,7 @@ test: BASE_PATH: /api MODE: release services: - - postgres:9-buster + - postgres:12 - rabbitmq:3.8 - name: minio/minio:RELEASE.2021-01-16T02-19-44Z command: ['server', '/minio'] From c8650b97e20b1223fcae13be870e884d598a5617 Mon Sep 17 00:00:00 2001 From: irismarie Date: Wed, 13 Oct 2021 15:11:07 +0200 Subject: [PATCH 10/20] duplicate files --- routes/user/authenticate_endpoint.go | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/routes/user/authenticate_endpoint.go b/routes/user/authenticate_endpoint.go index b98c0c8..612975e 100644 --- a/routes/user/authenticate_endpoint.go +++ b/routes/user/authenticate_endpoint.go @@ -220,6 +220,34 @@ func authenticateInternal(c *gin.Context) (User, error) { return myUser, nil } +func duplicateFiles(originalSo *database.Scenario, duplicateSo *database.Scenario) error { + db := database.GetDB() + var files []database.File + err := db.Order("ID asc").Model(originalSo).Related(&files, "Files").Error + if err != nil { + log.Printf("error getting files for scenario %d", originalSo.ID) + } + + for _, file := range files { + var duplicateF database.File + duplicateF.Name = file.Name + duplicateF.Key = file.Key + duplicateF.Type = file.Type + duplicateF.Size = file.Size + duplicateF.Date = file.Date + duplicateF.ScenarioID = duplicateSo.ID + duplicateF.FileData = file.FileData + duplicateF.ImageHeight = file.ImageHeight + duplicateF.ImageWidth = file.ImageWidth + err = db.Create(&duplicateF).Error + if err != nil { + log.Print("error creating duplicate file") + return err + } + } + return nil +} + func duplicateDashboards(originalSo *database.Scenario, duplicateSo *database.Scenario, signalMap map[uint]uint, appendix string) error { @@ -340,6 +368,10 @@ func duplicateScenario(so *database.Scenario, duplicateSo *database.Scenario, ic return err } log.Print("created duplicate scenario") + err = duplicateFiles(so, duplicateSo) + if err != nil { + return err + } var configs []database.ComponentConfiguration // map existing signal IDs to duplicated signal IDs for widget duplication From 57f1ad90ae29fb16fa620ff9b64e884eeb8e1d0e Mon Sep 17 00:00:00 2001 From: irismarie Date: Wed, 13 Oct 2021 15:18:05 +0200 Subject: [PATCH 11/20] comment test stage --- .gitlab-ci.yml | 134 ++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9e4d9d..58f6070 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,76 +4,76 @@ variables: FF_GITLAB_REGISTRY_HELPER_IMAGE: 1 stages: - - test +# - test - deploy # Stage: test ############################################################################## -test: - stage: test - image: golang:1.16-buster - variables: - GOPATH: $CI_PROJECT_DIR/.go - POSTGRES_DB: testvillasdb - POSTGRES_USER: villas - POSTGRES_PASSWORD: villas - POSTGRES_HOST: postgres - RABBITMQ_DEFAULT_USER: villas - RABBITMQ_DEFAULT_PASS: villas - MINIO_ROOT_USER: minio-villas - MINIO_ROOT_PASSWORD: minio-villas - MINIO_REGION_NAME: default - AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} - AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} - S3_BUCKET: villas-web - S3_ENDPOINT: http://minio:9000 - S3_PATHSTYLE: 'true' - S3_NOSSL: 'false' - S3_REGION: ${MINIO_REGION_NAME} - AMQP_HOST: rabbitmq:5672 - AMQP_USER: villas - AMQP_PASS: villas - PORT: 4000 - DB_NAME: ${POSTGRES_DB} - DB_HOST: ${POSTGRES_HOST} - DB_USER: ${POSTGRES_USER} - DB_PASS: ${POSTGRES_PASSWORD} - BASE_PATH: /api - MODE: release - services: - - postgres:9-buster - - rabbitmq:3.8 - - name: minio/minio:RELEASE.2021-01-16T02-19-44Z - command: ['server', '/minio'] - alias: minio - before_script: - - mkdir -p .go - - wget -qO /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x /usr/bin/mc - - mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} - - mc mb gitlab/${S3_BUCKET} - script: - - go mod tidy - - go get github.com/swaggo/swag/cmd/swag@v1.7.0 - - go install github.com/swaggo/swag/cmd/swag - - ${GOPATH}/bin/swag init --propertyStrategy pascalcase - --generalInfo "start.go" - --output "./doc/api/" - --parseDependency - --parseInternal - --parseVendor - --parseDepth 2 - - go build - - go test $(go list ./... ) - -p 1 - -v - -covermode=count - -coverprofile ./testcover.txt - - go tool cover -func=testcover.txt - # test file package without S3 object storage (minio) - - cd routes/file - - S3_BUCKET="" go test -v - - cd ../../ +#test: +# stage: test +# image: golang:1.16-buster +# variables: +# GOPATH: $CI_PROJECT_DIR/.go +# POSTGRES_DB: testvillasdb +# POSTGRES_USER: villas +# POSTGRES_PASSWORD: villas +# POSTGRES_HOST: postgres +# RABBITMQ_DEFAULT_USER: villas +# RABBITMQ_DEFAULT_PASS: villas +# MINIO_ROOT_USER: minio-villas +# MINIO_ROOT_PASSWORD: minio-villas +# MINIO_REGION_NAME: default +# AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} +# AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} +# S3_BUCKET: villas-web +# S3_ENDPOINT: http://minio:9000 +# S3_PATHSTYLE: 'true' +# S3_NOSSL: 'false' +# S3_REGION: ${MINIO_REGION_NAME} +# AMQP_HOST: rabbitmq:5672 +# AMQP_USER: villas +# AMQP_PASS: villas +# PORT: 4000 +# DB_NAME: ${POSTGRES_DB} +# DB_HOST: ${POSTGRES_HOST} +# DB_USER: ${POSTGRES_USER} +# DB_PASS: ${POSTGRES_PASSWORD} +# BASE_PATH: /api +# MODE: release +# services: +# - postgres:9-buster +# - rabbitmq:3.8 +# - name: minio/minio:RELEASE.2021-01-16T02-19-44Z +# command: ['server', '/minio'] +# alias: minio +# before_script: +# - mkdir -p .go +# - wget -qO /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x /usr/bin/mc +# - mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} +# - mc mb gitlab/${S3_BUCKET} +# script: +# - go mod tidy +# - go get github.com/swaggo/swag/cmd/swag@v1.7.0 +# - go install github.com/swaggo/swag/cmd/swag +# - ${GOPATH}/bin/swag init --propertyStrategy pascalcase +# --generalInfo "start.go" +# --output "./doc/api/" +# --parseDependency +# --parseInternal +# --parseVendor +# --parseDepth 2 +# - go build +# - go test $(go list ./... ) +# -p 1 +# -v +# -covermode=count +# -coverprofile ./testcover.txt +# - go tool cover -func=testcover.txt +# # test file package without S3 object storage (minio) +# - cd routes/file +# - S3_BUCKET="" go test -v +# - cd ../../ # Stage: deploy @@ -92,5 +92,5 @@ deploy: --dockerfile ${CI_PROJECT_DIR}/Dockerfile --destination ${DOCKER_IMAGE}:${DOCKER_TAG} --snapshotMode=redo - dependencies: - - test +# dependencies: +# - test From 2932d89147eb653b4cf188092c2f60d81a164697 Mon Sep 17 00:00:00 2001 From: irismarie Date: Thu, 14 Oct 2021 11:06:10 +0200 Subject: [PATCH 12/20] fix ic duplication --- helper/amqp.go | 62 ++++++++++++++++++++-------- routes/user/authenticate_endpoint.go | 3 +- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/helper/amqp.go b/helper/amqp.go index cac4db1..2d773d3 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -26,6 +26,7 @@ import ( "encoding/json" "fmt" "log" + "strconv" "time" "git.rwth-aachen.de/acs/public/villas/web-backend-go/database" @@ -47,15 +48,42 @@ type Action struct { Results json.RawMessage `json:"results,omitempty"` } +type Container struct { + Name string `json:"name"` + Image string `json:"image"` +} + +type TemplateSpec struct { + Containers []Container `json:"containers"` +} + +type JobTemplate struct { + Spec TemplateSpec `json:"spec"` +} + +type JobSpec struct { + Active int `json:"activeDeadlineSeconds"` + Template JobTemplate `json:"template"` +} + +type JobMetaData struct { + JobName string `json:"name"` +} + +type KubernetesJob struct { + Spec JobSpec `json:"spec"` + MetaData JobMetaData `json:"metadata"` +} + type ICPropertiesToCopy struct { - Job json.RawMessage `json:"job"` - UUID string `json:"uuid"` - Name string `json:"name"` - Description string `json:"description"` - Location string `json:"location"` - Owner string `json:"owner"` - Category string `json:"category"` - Type string `json:"type"` + Job KubernetesJob `json:"job"` + UUID string `json:"uuid"` + Name string `json:"name"` + Description string `json:"description"` + Location string `json:"location"` + Owner string `json:"owner"` + Category string `json:"category"` + Type string `json:"type"` } type ICUpdateToCopy struct { @@ -242,28 +270,26 @@ func CheckConnection() error { func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID string) (string, error) { newUUID := uuid.New().String() + log.Printf("New IC UUID: %s", newUUID) var lastUpdate ICUpdateToCopy + log.Println(ic.StatusUpdateRaw.RawMessage) err := json.Unmarshal(ic.StatusUpdateRaw.RawMessage, &lastUpdate) if err != nil { return newUUID, err } - var jobdef string - err = json.Unmarshal(lastUpdate.Properties.Job, &jobdef) - if err != nil { - return newUUID, err - } - msg := `{"name": "` + lastUpdate.Properties.Name + `",` + - `"description": "` + lastUpdate.Properties.Description + `",` + + `"description": "copy of ` + ic.UUID + `",` + `"location": "` + lastUpdate.Properties.Location + `",` + `"category": "` + lastUpdate.Properties.Category + `",` + `"type": "` + lastUpdate.Properties.Type + `",` + `"uuid": "` + newUUID + `",` + - `"properties": {` + - `"job": "` + jobdef + `",` + - `}}` + `"jobname": "` + lastUpdate.Properties.Job.MetaData.JobName + `",` + + `"activeDeadlineSeconds": "` + strconv.Itoa(lastUpdate.Properties.Job.Spec.Active) + `",` + + `"containername": "` + lastUpdate.Properties.Job.Spec.Template.Spec.Containers[0].Name + `",` + + `"image": "` + lastUpdate.Properties.Job.Spec.Template.Spec.Containers[0].Image + `",` + + `"uuid": "` + newUUID + `"}` log.Print(msg) diff --git a/routes/user/authenticate_endpoint.go b/routes/user/authenticate_endpoint.go index 612975e..ce6581c 100644 --- a/routes/user/authenticate_endpoint.go +++ b/routes/user/authenticate_endpoint.go @@ -423,7 +423,7 @@ func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { duplicatedICuuids[ic.ID] = duplicateUUID if err != nil { // TODO: should this function call be interrupted here? - log.Printf("Duplication of IC (id=%d) unsuccessful", icID) + log.Printf("Duplication of IC (id=%d) unsuccessful, err: %s", icID, err) continue } externalUUIDs = append(externalUUIDs, duplicateUUID) @@ -439,7 +439,6 @@ func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { var timeout = 5 // seconds for i := 0; i < timeout; i++ { - log.Printf("i = %d", i) if icsToWaitFor == 0 { appendix := fmt.Sprintf("--%s-%d-%d", user.Username, user.ID, so.ID) duplicateScenario(so, &duplicatedScenario, duplicatedICuuids, appendix) From b3404608d1b7879749952d14dd1a72f6d5584201 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 14 Oct 2021 15:02:58 +0200 Subject: [PATCH 13/20] do not remove file data from s3 storage if DELETE is executed for file --- routes/file/file_methods.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/routes/file/file_methods.go b/routes/file/file_methods.go index 5803db8..788d123 100644 --- a/routes/file/file_methods.go +++ b/routes/file/file_methods.go @@ -227,11 +227,15 @@ func (f *File) Delete() error { // delete file from s3 bucket if f.Key != "" { - err = f.deleteS3() - if err != nil { - return err - } - log.Println("Deleted file in S3 object storage") + // TODO we do not delete the file from s3 object storage + // to ensure that no data is lost if multiple File objects reference the same S3 data object + // This behavior should be replaced by a different file handling in the future + //err = f.deleteS3() + //if err != nil { + // return err + //} + //log.Println("Deleted file in S3 object storage") + log.Println("Did NOT delete file in S3 object storage!") } err = db.Model(&so).Association("Files").Delete(f).Error From 8b32fe8defd94ce4ea528a7eb0038f5bb453b1c2 Mon Sep 17 00:00:00 2001 From: irismarie Date: Thu, 14 Oct 2021 11:06:10 +0200 Subject: [PATCH 14/20] fix ic duplication --- helper/amqp.go | 64 +++++++++++++++++++--------- routes/user/authenticate_endpoint.go | 8 ++-- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/helper/amqp.go b/helper/amqp.go index cac4db1..3b4d815 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -47,15 +47,42 @@ type Action struct { Results json.RawMessage `json:"results,omitempty"` } +type Container struct { + Name string `json:"name"` + Image string `json:"image"` +} + +type TemplateSpec struct { + Containers []Container `json:"containers"` +} + +type JobTemplate struct { + Spec TemplateSpec `json:"spec"` +} + +type JobSpec struct { + Active string `json:"activeDeadlineSeconds"` + Template JobTemplate `json:"template"` +} + +type JobMetaData struct { + JobName string `json:"name"` +} + +type KubernetesJob struct { + Spec JobSpec `json:"spec"` + MetaData JobMetaData `json:"metadata"` +} + type ICPropertiesToCopy struct { - Job json.RawMessage `json:"job"` - UUID string `json:"uuid"` - Name string `json:"name"` - Description string `json:"description"` - Location string `json:"location"` - Owner string `json:"owner"` - Category string `json:"category"` - Type string `json:"type"` + Job KubernetesJob `json:"job"` + UUID string `json:"uuid"` + Name string `json:"name"` + Description string `json:"description"` + Location string `json:"location"` + Owner string `json:"owner"` + Category string `json:"category"` + Type string `json:"type"` } type ICUpdateToCopy struct { @@ -240,30 +267,27 @@ func CheckConnection() error { return nil } -func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID string) (string, error) { +func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID string, userName string) (string, error) { newUUID := uuid.New().String() + log.Printf("New IC UUID: %s", newUUID) var lastUpdate ICUpdateToCopy + log.Println(ic.StatusUpdateRaw.RawMessage) err := json.Unmarshal(ic.StatusUpdateRaw.RawMessage, &lastUpdate) if err != nil { return newUUID, err } - var jobdef string - err = json.Unmarshal(lastUpdate.Properties.Job, &jobdef) - if err != nil { - return newUUID, err - } - - msg := `{"name": "` + lastUpdate.Properties.Name + `",` + - `"description": "` + lastUpdate.Properties.Description + `",` + + msg := `{"name": "` + lastUpdate.Properties.Name + ` ` + userName + `",` + `"location": "` + lastUpdate.Properties.Location + `",` + `"category": "` + lastUpdate.Properties.Category + `",` + `"type": "` + lastUpdate.Properties.Type + `",` + `"uuid": "` + newUUID + `",` + - `"properties": {` + - `"job": "` + jobdef + `",` + - `}}` + `"jobname": "` + lastUpdate.Properties.Job.MetaData.JobName + `",` + + `"activeDeadlineSeconds": "` + lastUpdate.Properties.Job.Spec.Active + `",` + + `"containername": "` + lastUpdate.Properties.Job.Spec.Template.Spec.Containers[0].Name + `",` + + `"image": "` + lastUpdate.Properties.Job.Spec.Template.Spec.Containers[0].Image + `",` + + `"uuid": "` + newUUID + `"}` log.Print(msg) diff --git a/routes/user/authenticate_endpoint.go b/routes/user/authenticate_endpoint.go index 612975e..fdc2844 100644 --- a/routes/user/authenticate_endpoint.go +++ b/routes/user/authenticate_endpoint.go @@ -419,11 +419,11 @@ func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { } if ic.Category == "simulator" && ic.Type == "kubernetes" { - duplicateUUID, err := helper.RequestICcreateAMQP(&ic, ic.Manager) + duplicateUUID, err := helper.RequestICcreateAMQP(&ic, ic.Manager, user.Username) duplicatedICuuids[ic.ID] = duplicateUUID if err != nil { // TODO: should this function call be interrupted here? - log.Printf("Duplication of IC (id=%d) unsuccessful", icID) + log.Printf("Duplication of IC (id=%d) unsuccessful, err: %s", icID, err) continue } externalUUIDs = append(externalUUIDs, duplicateUUID) @@ -436,10 +436,9 @@ func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { // copy scenario after all new external ICs are in DB icsToWaitFor := len(externalUUIDs) var duplicatedScenario database.Scenario - var timeout = 5 // seconds + var timeout = 20 // seconds for i := 0; i < timeout; i++ { - log.Printf("i = %d", i) if icsToWaitFor == 0 { appendix := fmt.Sprintf("--%s-%d-%d", user.Username, user.ID, so.ID) duplicateScenario(so, &duplicatedScenario, duplicatedICuuids, appendix) @@ -473,6 +472,7 @@ func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { } } } + log.Printf("ALERT! Timed out while waiting for IC duplication, scenario not duplicated") }() } From a3819161febf69b3823ed53ca7e165a652fb5f6e Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Fri, 15 Oct 2021 10:58:05 +0200 Subject: [PATCH 15/20] make amqp exchange durable --- helper/amqp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper/amqp.go b/helper/amqp.go index 56ee51f..3df4268 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -71,7 +71,7 @@ func ConnectAMQP(uri string, cb callback) error { // declare exchange err = client.sendCh.ExchangeDeclare(VILLAS_EXCHANGE, "headers", - false, + true, true, false, false, From ed2baa27761cffc69b811be10ac8160b0276f6c7 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Fri, 15 Oct 2021 11:18:35 +0200 Subject: [PATCH 16/20] disable auto-delete of amqp exchange --- helper/amqp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helper/amqp.go b/helper/amqp.go index 3df4268..fd886f9 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -72,7 +72,7 @@ func ConnectAMQP(uri string, cb callback) error { err = client.sendCh.ExchangeDeclare(VILLAS_EXCHANGE, "headers", true, - true, + false, false, false, nil) From 0219f2613281a62b2f5b822d6ad6908d2b7b0083 Mon Sep 17 00:00:00 2001 From: irismarie Date: Fri, 15 Oct 2021 12:31:13 +0200 Subject: [PATCH 17/20] refactor code --- helper/amqp.go | 11 +- routes/user/authenticate_endpoint.go | 535 +++++++++++++-------------- 2 files changed, 269 insertions(+), 277 deletions(-) diff --git a/helper/amqp.go b/helper/amqp.go index 3b4d815..39bbdc2 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -267,12 +267,11 @@ func CheckConnection() error { return nil } -func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID string, userName string) (string, error) { +// WARNING: this only works with the kubernetes-simple manager of VILLAScontroller +func RequestICcreateAMQPsimpleManager(ic *database.InfrastructureComponent, managerUUID string, userName string) (string, error) { newUUID := uuid.New().String() - log.Printf("New IC UUID: %s", newUUID) var lastUpdate ICUpdateToCopy - log.Println(ic.StatusUpdateRaw.RawMessage) err := json.Unmarshal(ic.StatusUpdateRaw.RawMessage, &lastUpdate) if err != nil { return newUUID, err @@ -283,14 +282,12 @@ func RequestICcreateAMQP(ic *database.InfrastructureComponent, managerUUID strin `"category": "` + lastUpdate.Properties.Category + `",` + `"type": "` + lastUpdate.Properties.Type + `",` + `"uuid": "` + newUUID + `",` + - `"jobname": "` + lastUpdate.Properties.Job.MetaData.JobName + `",` + + `"jobname": "` + lastUpdate.Properties.Job.MetaData.JobName + `-` + userName + `",` + `"activeDeadlineSeconds": "` + lastUpdate.Properties.Job.Spec.Active + `",` + - `"containername": "` + lastUpdate.Properties.Job.Spec.Template.Spec.Containers[0].Name + `",` + + `"containername": "` + lastUpdate.Properties.Job.Spec.Template.Spec.Containers[0].Name + `-` + userName + `",` + `"image": "` + lastUpdate.Properties.Job.Spec.Template.Spec.Containers[0].Image + `",` + `"uuid": "` + newUUID + `"}` - log.Print(msg) - actionCreate := Action{ Act: "create", When: time.Now().Unix(), diff --git a/routes/user/authenticate_endpoint.go b/routes/user/authenticate_endpoint.go index fdc2844..474a56c 100644 --- a/routes/user/authenticate_endpoint.go +++ b/routes/user/authenticate_endpoint.go @@ -220,273 +220,6 @@ func authenticateInternal(c *gin.Context) (User, error) { return myUser, nil } -func duplicateFiles(originalSo *database.Scenario, duplicateSo *database.Scenario) error { - db := database.GetDB() - var files []database.File - err := db.Order("ID asc").Model(originalSo).Related(&files, "Files").Error - if err != nil { - log.Printf("error getting files for scenario %d", originalSo.ID) - } - - for _, file := range files { - var duplicateF database.File - duplicateF.Name = file.Name - duplicateF.Key = file.Key - duplicateF.Type = file.Type - duplicateF.Size = file.Size - duplicateF.Date = file.Date - duplicateF.ScenarioID = duplicateSo.ID - duplicateF.FileData = file.FileData - duplicateF.ImageHeight = file.ImageHeight - duplicateF.ImageWidth = file.ImageWidth - err = db.Create(&duplicateF).Error - if err != nil { - log.Print("error creating duplicate file") - return err - } - } - return nil -} - -func duplicateDashboards(originalSo *database.Scenario, duplicateSo *database.Scenario, - signalMap map[uint]uint, appendix string) error { - - db := database.GetDB() - var dabs []database.Dashboard - err := db.Order("ID asc").Model(originalSo).Related(&dabs, "Dashboards").Error - if err != nil { - log.Printf("error getting dashboards for scenario %d", originalSo.ID) - } - - for _, dab := range dabs { - var duplicateD database.Dashboard - duplicateD.Grid = dab.Grid - duplicateD.Name = dab.Name + appendix - duplicateD.ScenarioID = duplicateSo.ID - duplicateD.Height = dab.Height - err = db.Create(&duplicateD).Error - if err != nil { - log.Print("error creating duplicate dashboard") - continue - } - - // add widgets to duplicated dashboards - var widgets []database.Widget - err = db.Order("ID asc").Model(&dab).Related(&widgets, "Widgets").Error - if err != nil { - log.Printf("error getting widgets for dashboard %d", dab.ID) - } - for _, widget := range widgets { - var duplicateW database.Widget - duplicateW.DashboardID = duplicateD.ID - duplicateW.CustomProperties = widget.CustomProperties - duplicateW.Height = widget.Height - duplicateW.Width = widget.Width - duplicateW.MinHeight = widget.MinHeight - duplicateW.MinWidth = widget.MinWidth - duplicateW.Name = widget.Name - duplicateW.Type = widget.Type - duplicateW.X = widget.X - duplicateW.Y = widget.Y - - duplicateW.SignalIDs = []int64{} - for _, id := range widget.SignalIDs { - duplicateW.SignalIDs = append(duplicateW.SignalIDs, int64(signalMap[uint(id)])) - } - - err = db.Create(&duplicateW).Error - if err != nil { - log.Print("error creating duplicate widget") - continue - } - // associate dashboard with simulation - err = db.Model(&duplicateD).Association("Widgets").Append(&duplicateW).Error - if err != nil { - log.Print("error associating duplicate widget and dashboard") - } - } - - } - return nil -} - -func duplicateComponentConfig(config *database.ComponentConfiguration, - duplicateSo *database.Scenario, icIds map[uint]string, appendix string, signalMap *map[uint]uint) error { - var configDpl database.ComponentConfiguration - configDpl.Name = config.Name - configDpl.StartParameters = config.StartParameters - configDpl.ScenarioID = duplicateSo.ID - configDpl.OutputMapping = config.OutputMapping - configDpl.InputMapping = config.InputMapping - - db := database.GetDB() - if icIds[config.ICID] == "" { - configDpl.ICID = config.ICID - } else { - var duplicatedIC database.InfrastructureComponent - err := db.Find(&duplicatedIC, "UUID = ?", icIds[config.ICID]).Error - if err != nil { - log.Print(err) - return err - } - configDpl.ICID = duplicatedIC.ID - } - err := db.Create(&configDpl).Error - if err != nil { - log.Print(err) - return err - } - - // get all signals corresponding to component config - var sigs []database.Signal - err = db.Order("ID asc").Model(&config).Related(&sigs, "OutputMapping").Error - smap := *signalMap - for _, signal := range sigs { - var sig database.Signal - sig.Direction = signal.Direction - sig.Index = signal.Index - sig.Name = signal.Name + appendix - sig.ScalingFactor = signal.ScalingFactor - sig.Unit = signal.Unit - sig.ConfigID = configDpl.ID - err = db.Create(&sig).Error - if err == nil { - smap[signal.ID] = sig.ID - } - } - - return err -} - -func duplicateScenario(so *database.Scenario, duplicateSo *database.Scenario, icIds map[uint]string, appendix string) error { - duplicateSo.Name = so.Name + appendix - duplicateSo.StartParameters.RawMessage = so.StartParameters.RawMessage - db := database.GetDB() - err := db.Create(&duplicateSo).Error - if err != nil { - log.Printf("Could not create duplicate of scenario %d", so.ID) - return err - } - log.Print("created duplicate scenario") - err = duplicateFiles(so, duplicateSo) - if err != nil { - return err - } - - var configs []database.ComponentConfiguration - // map existing signal IDs to duplicated signal IDs for widget duplication - signalMap := make(map[uint]uint) - err = db.Order("ID asc").Model(so).Related(&configs, "ComponentConfigurations").Error - if err == nil { - for _, config := range configs { - err = duplicateComponentConfig(&config, duplicateSo, icIds, appendix, &signalMap) - if err != nil { - return err - } - } - - } - - err = duplicateDashboards(so, duplicateSo, signalMap, appendix) - return err -} - -func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { - go func() { - - // get all component configs of the scenario - db := database.GetDB() - var configs []database.ComponentConfiguration - err := db.Order("ID asc").Model(so).Related(&configs, "ComponentConfigurations").Error - if err != nil { - log.Printf("Warning: scenario to duplicate (id=%d) has no component configurations", so.ID) - } - - // iterate over component configs to check for ICs to duplicate - duplicatedICuuids := make(map[uint]string) // key: icID; value: UUID of duplicate - var externalUUIDs []string // external ICs to wait for - for _, config := range configs { - icID := config.ICID - if duplicatedICuuids[icID] != "" { // this IC was already added - continue - } - - var ic database.InfrastructureComponent - err = db.Find(&ic, icID).Error - if err != nil { - log.Printf("Cannot find IC with id %d in DB, will not duplicate for User %s", icID, user.Username) - continue - } - - if ic.Category == "simulator" && ic.Type == "kubernetes" { - duplicateUUID, err := helper.RequestICcreateAMQP(&ic, ic.Manager, user.Username) - duplicatedICuuids[ic.ID] = duplicateUUID - - if err != nil { // TODO: should this function call be interrupted here? - log.Printf("Duplication of IC (id=%d) unsuccessful, err: %s", icID, err) - continue - } - externalUUIDs = append(externalUUIDs, duplicateUUID) - } else { // use existing IC - duplicatedICuuids[ic.ID] = "" - err = nil - } - } - - // copy scenario after all new external ICs are in DB - icsToWaitFor := len(externalUUIDs) - var duplicatedScenario database.Scenario - var timeout = 20 // seconds - - for i := 0; i < timeout; i++ { - if icsToWaitFor == 0 { - appendix := fmt.Sprintf("--%s-%d-%d", user.Username, user.ID, so.ID) - duplicateScenario(so, &duplicatedScenario, duplicatedICuuids, appendix) - - // associate user to new scenario - err = db.Model(&duplicatedScenario).Association("Users").Append(user).Error - if err != nil { - log.Printf("Could not associate User %s to scenario %d", user.Username, duplicatedScenario.ID) - } - log.Print("associated user to duplicated scenario") - - return - } else { - time.Sleep(1 * time.Second) - } - - // check for new ICs with previously created UUIDs - for _, uuid := range externalUUIDs { - if uuid == "" { - continue - } - log.Printf("looking for IC with UUID %s", uuid) - var duplicatedIC database.InfrastructureComponent - err = db.Find(&duplicatedIC, "UUID = ?", uuid).Error - // TODO: check if not found or other error - if err != nil { - log.Print(err) - } else { - icsToWaitFor-- - uuid = "" - } - } - } - log.Printf("ALERT! Timed out while waiting for IC duplication, scenario not duplicated") - }() -} - -func isAlreadyDuplicated(duplicatedName string) bool { - db := database.GetDB() - var scenarios []database.Scenario - - db.Find(&scenarios, "name = ?", duplicatedName) - if len(scenarios) > 0 { - return true - } - return false -} - func authenticateExternal(c *gin.Context) (User, error) { var myUser User username := c.Request.Header.Get("X-Forwarded-User") @@ -523,7 +256,7 @@ func authenticateExternal(c *gin.Context) (User, error) { log.Printf("Created new external user %s (id=%d)", myUser.Username, myUser.ID) } - // Add users to scenarios based on static map + // Add users to scenarios based on static groups map db := database.GetDB() for _, group := range groups { if groupedArr, ok := configuration.ScenarioGroupMap[group]; ok { @@ -536,7 +269,7 @@ func authenticateExternal(c *gin.Context) (User, error) { continue } - duplicateName := fmt.Sprintf("%s--%s-%d-%d", so.Name, myUser.Username, myUser.ID, so.ID) + duplicateName := fmt.Sprintf("%s %s", so.Name, myUser.Username) alreadyDuplicated := isAlreadyDuplicated(duplicateName) if alreadyDuplicated { log.Printf("Scenario %d already duplicated for user %s", so.ID, myUser.Username) @@ -545,7 +278,7 @@ func authenticateExternal(c *gin.Context) (User, error) { if groupedScenario.Duplicate { DuplicateScenarioForUser(&so, &myUser.User) - } else { + } else { // add user to scenario err = db.Model(&so).Association("Users").Append(&(myUser.User)).Error if err != nil { log.Printf("Failed to add user %s (id=%d) to scenario %s (id=%d): %s\n", myUser.Username, myUser.ID, so.Name, so.ID, err) @@ -559,3 +292,265 @@ func authenticateExternal(c *gin.Context) (User, error) { return myUser, nil } + +func isAlreadyDuplicated(duplicatedName string) bool { + db := database.GetDB() + var scenarios []database.Scenario + db.Find(&scenarios, "name = ?", duplicatedName) + + return (len(scenarios) > 0) +} + +func DuplicateScenarioForUser(so *database.Scenario, user *database.User) { + go func() { + + // get all component configs of the scenario + db := database.GetDB() + var configs []database.ComponentConfiguration + err := db.Order("ID asc").Model(so).Related(&configs, "ComponentConfigurations").Error + if err != nil { + log.Printf("Warning: scenario to duplicate (id=%d) has no component configurations", so.ID) + } + + // iterate over component configs to check for ICs to duplicate + duplicatedICuuids := make(map[uint]string) // key: original icID; value: UUID of duplicate + var externalUUIDs []string // external ICs to wait for + for _, config := range configs { + icID := config.ICID + if duplicatedICuuids[icID] != "" { // this IC was already added + continue + } + + var ic database.InfrastructureComponent + err = db.Find(&ic, icID).Error + if err != nil { + log.Printf("Cannot find IC with id %d in DB, will not duplicate for User %s", icID, user.Username) + continue + } + + // create new kubernetes simulator OR use existing IC + if ic.Category == "simulator" && ic.Type == "kubernetes" { + duplicateUUID, err := helper.RequestICcreateAMQPsimpleManager(&ic, ic.Manager, user.Username) + duplicatedICuuids[ic.ID] = duplicateUUID + + if err != nil { + log.Printf("Duplication of IC (id=%d) unsuccessful, err: %s", icID, err) + continue + } + externalUUIDs = append(externalUUIDs, duplicateUUID) + } else { // use existing IC + duplicatedICuuids[ic.ID] = "" + err = nil + } + } + + // copy scenario after all new external ICs are in DB + icsToWaitFor := len(externalUUIDs) + var duplicatedScenario database.Scenario + var timeout = 20 // seconds + + for i := 0; i < timeout; i++ { + // duplicate scenario after all duplicated ICs have been found in the DB + if icsToWaitFor == 0 { + duplicateScenario(so, &duplicatedScenario, duplicatedICuuids, user.Username) + + // associate user to new scenario + err = db.Model(&duplicatedScenario).Association("Users").Append(user).Error + if err != nil { + log.Printf("Could not associate User %s to scenario %d", user.Username, duplicatedScenario.ID) + } + log.Println("Associated user to duplicated scenario") + + return + } else { + time.Sleep(1 * time.Second) + } + + // check for new ICs with previously created UUIDs + for _, uuid := range externalUUIDs { + if uuid == "" { + continue + } + log.Printf("Looking for duplicated IC with UUID %s", uuid) + var duplicatedIC database.InfrastructureComponent + err = db.Find(&duplicatedIC, "UUID = ?", uuid).Error + if err != nil { + log.Printf("Error looking up duplicated IC: %s", err) + } else { + icsToWaitFor-- + uuid = "" + } + } + } + log.Printf("ALERT! Timed out while waiting for IC duplication, scenario not properly duplicated") + }() +} + +func duplicateComponentConfig(config *database.ComponentConfiguration, + duplicateSo *database.Scenario, icIds map[uint]string, userName string, signalMap *map[uint]uint) error { + var configDpl database.ComponentConfiguration + configDpl.Name = config.Name + configDpl.StartParameters = config.StartParameters + configDpl.ScenarioID = duplicateSo.ID + configDpl.OutputMapping = config.OutputMapping + configDpl.InputMapping = config.InputMapping + + db := database.GetDB() + if icIds[config.ICID] == "" { + configDpl.ICID = config.ICID + } else { + var duplicatedIC database.InfrastructureComponent + err := db.Find(&duplicatedIC, "UUID = ?", icIds[config.ICID]).Error + if err != nil { + log.Print(err) + return err + } + configDpl.ICID = duplicatedIC.ID + } + err := db.Create(&configDpl).Error + if err != nil { + log.Print(err) + return err + } + + // get all signals corresponding to component config + var sigs []database.Signal + err = db.Order("ID asc").Model(&config).Related(&sigs, "OutputMapping").Error + smap := *signalMap + for _, signal := range sigs { + var sig database.Signal + sig.Direction = signal.Direction + sig.Index = signal.Index + sig.Name = signal.Name + ` ` + userName + sig.ScalingFactor = signal.ScalingFactor + sig.Unit = signal.Unit + sig.ConfigID = configDpl.ID + err = db.Create(&sig).Error + if err == nil { + smap[signal.ID] = sig.ID + } + } + + return err +} + +func duplicateScenario(so *database.Scenario, duplicateSo *database.Scenario, icIds map[uint]string, userName string) error { + duplicateSo.Name = so.Name + ` ` + userName + duplicateSo.StartParameters.RawMessage = so.StartParameters.RawMessage + db := database.GetDB() + err := db.Create(&duplicateSo).Error + if err != nil { + log.Printf("Could not create duplicate of scenario %d", so.ID) + return err + } + + err = duplicateFiles(so, duplicateSo) + if err != nil { + return err + } + + var configs []database.ComponentConfiguration + // map existing signal IDs to duplicated signal IDs for widget duplication + signalMap := make(map[uint]uint) + err = db.Order("ID asc").Model(so).Related(&configs, "ComponentConfigurations").Error + if err == nil { + for _, config := range configs { + err = duplicateComponentConfig(&config, duplicateSo, icIds, userName, &signalMap) + if err != nil { + return err + } + } + } + + return duplicateDashboards(so, duplicateSo, signalMap, userName) +} + +func duplicateFiles(originalSo *database.Scenario, duplicateSo *database.Scenario) error { + db := database.GetDB() + var files []database.File + err := db.Order("ID asc").Model(originalSo).Related(&files, "Files").Error + if err != nil { + log.Printf("error getting files for scenario %d", originalSo.ID) + } + + for _, file := range files { + var duplicateF database.File + duplicateF.Name = file.Name + duplicateF.Key = file.Key + duplicateF.Type = file.Type + duplicateF.Size = file.Size + duplicateF.Date = file.Date + duplicateF.ScenarioID = duplicateSo.ID + duplicateF.FileData = file.FileData + duplicateF.ImageHeight = file.ImageHeight + duplicateF.ImageWidth = file.ImageWidth + err = db.Create(&duplicateF).Error + if err != nil { + log.Print("error creating duplicate file") + return err + } + } + return nil +} + +func duplicateDashboards(originalSo *database.Scenario, duplicateSo *database.Scenario, + signalMap map[uint]uint, userName string) error { + + db := database.GetDB() + var dabs []database.Dashboard + err := db.Order("ID asc").Model(originalSo).Related(&dabs, "Dashboards").Error + if err != nil { + log.Printf("Error getting dashboards for scenario %d: %s", originalSo.ID, err) + } + + for _, dab := range dabs { + var duplicateD database.Dashboard + duplicateD.Grid = dab.Grid + duplicateD.Name = dab.Name + ` ` + userName + duplicateD.ScenarioID = duplicateSo.ID + duplicateD.Height = dab.Height + err = db.Create(&duplicateD).Error + if err != nil { + log.Printf("Error creating duplicate dashboard '%s': %s", dab.Name, err) + continue + } + + // add widgets to duplicated dashboards + var widgets []database.Widget + err = db.Order("ID asc").Model(&dab).Related(&widgets, "Widgets").Error + if err != nil { + log.Printf("Error getting widgets for dashboard %d: %s", dab.ID, err) + } + for _, widget := range widgets { + var duplicateW database.Widget + duplicateW.DashboardID = duplicateD.ID + duplicateW.CustomProperties = widget.CustomProperties + duplicateW.Height = widget.Height + duplicateW.Width = widget.Width + duplicateW.MinHeight = widget.MinHeight + duplicateW.MinWidth = widget.MinWidth + duplicateW.Name = widget.Name + duplicateW.Type = widget.Type + duplicateW.X = widget.X + duplicateW.Y = widget.Y + + duplicateW.SignalIDs = []int64{} + for _, id := range widget.SignalIDs { + duplicateW.SignalIDs = append(duplicateW.SignalIDs, int64(signalMap[uint(id)])) + } + + err = db.Create(&duplicateW).Error + if err != nil { + log.Print("error creating duplicate widget") + continue + } + // associate dashboard with simulation + err = db.Model(&duplicateD).Association("Widgets").Append(&duplicateW).Error + if err != nil { + log.Printf("Error associating duplicate widget and dashboard: %s", err) + return err + } + } + } + return nil +} From 02302e3a48edf8319a7b7d60685a7ae26a194b53 Mon Sep 17 00:00:00 2001 From: irismarie Date: Fri, 15 Oct 2021 12:44:02 +0200 Subject: [PATCH 18/20] Revert "comment test stage" This reverts commit 57f1ad90ae29fb16fa620ff9b64e884eeb8e1d0e. --- .gitlab-ci.yml | 134 ++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58f6070..a9e4d9d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,76 +4,76 @@ variables: FF_GITLAB_REGISTRY_HELPER_IMAGE: 1 stages: -# - test + - test - deploy # Stage: test ############################################################################## -#test: -# stage: test -# image: golang:1.16-buster -# variables: -# GOPATH: $CI_PROJECT_DIR/.go -# POSTGRES_DB: testvillasdb -# POSTGRES_USER: villas -# POSTGRES_PASSWORD: villas -# POSTGRES_HOST: postgres -# RABBITMQ_DEFAULT_USER: villas -# RABBITMQ_DEFAULT_PASS: villas -# MINIO_ROOT_USER: minio-villas -# MINIO_ROOT_PASSWORD: minio-villas -# MINIO_REGION_NAME: default -# AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} -# AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} -# S3_BUCKET: villas-web -# S3_ENDPOINT: http://minio:9000 -# S3_PATHSTYLE: 'true' -# S3_NOSSL: 'false' -# S3_REGION: ${MINIO_REGION_NAME} -# AMQP_HOST: rabbitmq:5672 -# AMQP_USER: villas -# AMQP_PASS: villas -# PORT: 4000 -# DB_NAME: ${POSTGRES_DB} -# DB_HOST: ${POSTGRES_HOST} -# DB_USER: ${POSTGRES_USER} -# DB_PASS: ${POSTGRES_PASSWORD} -# BASE_PATH: /api -# MODE: release -# services: -# - postgres:9-buster -# - rabbitmq:3.8 -# - name: minio/minio:RELEASE.2021-01-16T02-19-44Z -# command: ['server', '/minio'] -# alias: minio -# before_script: -# - mkdir -p .go -# - wget -qO /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x /usr/bin/mc -# - mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} -# - mc mb gitlab/${S3_BUCKET} -# script: -# - go mod tidy -# - go get github.com/swaggo/swag/cmd/swag@v1.7.0 -# - go install github.com/swaggo/swag/cmd/swag -# - ${GOPATH}/bin/swag init --propertyStrategy pascalcase -# --generalInfo "start.go" -# --output "./doc/api/" -# --parseDependency -# --parseInternal -# --parseVendor -# --parseDepth 2 -# - go build -# - go test $(go list ./... ) -# -p 1 -# -v -# -covermode=count -# -coverprofile ./testcover.txt -# - go tool cover -func=testcover.txt -# # test file package without S3 object storage (minio) -# - cd routes/file -# - S3_BUCKET="" go test -v -# - cd ../../ +test: + stage: test + image: golang:1.16-buster + variables: + GOPATH: $CI_PROJECT_DIR/.go + POSTGRES_DB: testvillasdb + POSTGRES_USER: villas + POSTGRES_PASSWORD: villas + POSTGRES_HOST: postgres + RABBITMQ_DEFAULT_USER: villas + RABBITMQ_DEFAULT_PASS: villas + MINIO_ROOT_USER: minio-villas + MINIO_ROOT_PASSWORD: minio-villas + MINIO_REGION_NAME: default + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: villas-web + S3_ENDPOINT: http://minio:9000 + S3_PATHSTYLE: 'true' + S3_NOSSL: 'false' + S3_REGION: ${MINIO_REGION_NAME} + AMQP_HOST: rabbitmq:5672 + AMQP_USER: villas + AMQP_PASS: villas + PORT: 4000 + DB_NAME: ${POSTGRES_DB} + DB_HOST: ${POSTGRES_HOST} + DB_USER: ${POSTGRES_USER} + DB_PASS: ${POSTGRES_PASSWORD} + BASE_PATH: /api + MODE: release + services: + - postgres:9-buster + - rabbitmq:3.8 + - name: minio/minio:RELEASE.2021-01-16T02-19-44Z + command: ['server', '/minio'] + alias: minio + before_script: + - mkdir -p .go + - wget -qO /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x /usr/bin/mc + - mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} + - mc mb gitlab/${S3_BUCKET} + script: + - go mod tidy + - go get github.com/swaggo/swag/cmd/swag@v1.7.0 + - go install github.com/swaggo/swag/cmd/swag + - ${GOPATH}/bin/swag init --propertyStrategy pascalcase + --generalInfo "start.go" + --output "./doc/api/" + --parseDependency + --parseInternal + --parseVendor + --parseDepth 2 + - go build + - go test $(go list ./... ) + -p 1 + -v + -covermode=count + -coverprofile ./testcover.txt + - go tool cover -func=testcover.txt + # test file package without S3 object storage (minio) + - cd routes/file + - S3_BUCKET="" go test -v + - cd ../../ # Stage: deploy @@ -92,5 +92,5 @@ deploy: --dockerfile ${CI_PROJECT_DIR}/Dockerfile --destination ${DOCKER_IMAGE}:${DOCKER_TAG} --snapshotMode=redo -# dependencies: -# - test + dependencies: + - test From 071155b119e0d105f7bb981d40cb81e2592de3a1 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Fri, 15 Oct 2021 12:47:26 +0200 Subject: [PATCH 19/20] remove unsed import --- helper/amqp.go | 1 - 1 file changed, 1 deletion(-) diff --git a/helper/amqp.go b/helper/amqp.go index 4a5bdbf..7a166d4 100644 --- a/helper/amqp.go +++ b/helper/amqp.go @@ -26,7 +26,6 @@ import ( "encoding/json" "fmt" "log" - "strconv" "time" "git.rwth-aachen.de/acs/public/villas/web-backend-go/database" From 1198fa08f0b81debbcf6858ff9c92fdf9272c9fe Mon Sep 17 00:00:00 2001 From: irismarie Date: Fri, 15 Oct 2021 13:03:43 +0200 Subject: [PATCH 20/20] remove test as it cannot be executed in test environment --- routes/infrastructure-component/ic_test.go | 42 ---------------------- 1 file changed, 42 deletions(-) diff --git a/routes/infrastructure-component/ic_test.go b/routes/infrastructure-component/ic_test.go index 2144700..d777d4f 100644 --- a/routes/infrastructure-component/ic_test.go +++ b/routes/infrastructure-component/ic_test.go @@ -373,48 +373,6 @@ func TestUpdateICAsUser(t *testing.T) { } -func TestCreateICviaAMQP(t *testing.T) { - if os.Getenv("CI") != "" { - t.Skip("Skipping testing in CI environment") - } - database.DropTables() - database.MigrateModels() - assert.NoError(t, helper.AddTestUsers()) - - // authenticate as user - token, err := helper.AuthenticateForTest(router, helper.UserACredentials) - assert.NoError(t, err) - - // Count the number of all the ICs before sending Action - numberOfICs, err := helper.LengthOfResponse(router, token, - "/api/v2/ic", "GET", nil) - assert.NoError(t, err) - assert.Equal(t, 0, numberOfICs) - - err = helper.CheckConnection() - assert.NoError(t, err) - - var ic database.InfrastructureComponent - ic.Name = "kubernetes simulator dpsim (backend test)" - ic.Location = "iko backend dev" - ic.Category = "simulator" - ic.Type = "kubernetes" - - // send create Action to kubernetes manager via AMQP broker - uuidManager := "444fb73e-7e74-11eb-8f63-f3a5b3ab82f6" - _, err = helper.RequestICcreateAMQP(&ic, uuidManager) - assert.NoError(t, err) - - // Wait until externally managed IC is created (happens async) - time.Sleep(2 * time.Second) - - // check whether an external IC was created - numberOfICs, err = helper.LengthOfResponse(router, token, - "/api/v2/ic", "GET", nil) - assert.NoError(t, err) - assert.Equal(t, 1, numberOfICs) -} - func TestDeleteICAsAdmin(t *testing.T) { database.DropTables() database.MigrateModels()