From 661f2049b6ca8d05dfe8a0d811fafa6a5aa3869d Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Sat, 28 Nov 2020 02:51:35 +0100 Subject: [PATCH 01/27] added support for storing files in S3 object storage --- configuration/config.go | 116 +++++++++++++++---------- database/models.go | 5 +- go.mod | 3 +- go.sum | 15 ++++ routes/file/file_methods.go | 109 ++++++++++++++--------- routes/file/file_s3.go | 166 ++++++++++++++++++++++++++++++++++++ 6 files changed, 325 insertions(+), 89 deletions(-) create mode 100644 routes/file/file_s3.go diff --git a/configuration/config.go b/configuration/config.go index 50e28d6..dcec201 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -38,59 +38,83 @@ func InitConfig() error { } var ( - dbHost = flag.String("db-host", "/var/run/postgresql", "Host of the PostgreSQL database (default is /var/run/postgresql for localhost DB on Ubuntu systems)") - dbName = flag.String("db-name", "villasdb", "Name of the database to use (default is villasdb)") - dbUser = flag.String("db-user", "", "Username of database connection (default is )") - dbPass = flag.String("db-pass", "", "Password of database connection (default is )") - dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production - amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)") - amqpUser = flag.String("amqp-user", "", "Username for AMQP broker") - amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker") - configFile = flag.String("config", "", "Path to YAML configuration file") - mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)") - port = flag.String("port", "4000", "Port of the backend (default is 4000)") - baseHost = flag.String("base-host", "localhost:4000", "The host at which the backend is hosted (default: localhost)") - basePath = flag.String("base-path", "/api/v2", "The path at which the API routes are located (default /api/v2)") - adminUser = flag.String("admin-user", "", "Initial admin username") - adminPass = flag.String("admin-pass", "", "Initial admin password") - adminMail = flag.String("admin-mail", "", "Initial admin mail address") + dbHost = flag.String("db-host", "/var/run/postgresql", "Host of the PostgreSQL database (default is /var/run/postgresql for localhost DB on Ubuntu systems)") + dbName = flag.String("db-name", "villasdb", "Name of the database to use (default is villasdb)") + dbUser = flag.String("db-user", "", "Username of database connection (default is )") + dbPass = flag.String("db-pass", "", "Password of database connection (default is )") + dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production + amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)") + amqpUser = flag.String("amqp-user", "", "Username for AMQP broker") + amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker") + configFile = flag.String("config", "", "Path to YAML configuration file") + mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)") + port = flag.String("port", "4000", "Port of the backend (default is 4000)") + baseHost = flag.String("base-host", "localhost:4000", "The host at which the backend is hosted (default: localhost)") + basePath = flag.String("base-path", "/api/v2", "The path at which the API routes are located (default /api/v2)") + adminUser = flag.String("admin-user", "", "Initial admin username") + adminPass = flag.String("admin-pass", "", "Initial admin password") + adminMail = flag.String("admin-mail", "", "Initial admin mail address") + s3Bucket = flag.String("s3-bucket", "", "S3 Bucket for uploading files") + s3Endpoint = flag.String("s3-endpoint", "", "Endpoint of S3 API for file uploads") + s3Region = flag.String("s3-region", "us-east-1", "S3 Region for file uploads") + s3NoSSL = flag.Bool("s3-nossl", false, "Use encrypted connections to the S3 API") + s3PathStyle = flag.Bool("s3-pathstyle", false, "Use path-style S3 API") ) flag.Parse() static := map[string]string{ - "db.host": *dbHost, - "db.name": *dbName, - "db.user": *dbUser, - "db.pass": *dbPass, - "db.ssl": *dbSSLMode, - "amqp.host": *amqpHost, - "amqp.user": *amqpUser, - "amqp.pass": *amqpPass, - "mode": *mode, - "port": *port, - "base.host": *baseHost, - "base.path": *basePath, - "admin.user": *adminUser, - "admin.pass": *adminPass, - "admin.mail": *adminMail, + "db.host": *dbHost, + "db.name": *dbName, + "db.user": *dbUser, + "db.pass": *dbPass, + "db.ssl": *dbSSLMode, + "amqp.host": *amqpHost, + "amqp.user": *amqpUser, + "amqp.pass": *amqpPass, + "mode": *mode, + "port": *port, + "base.host": *baseHost, + "base.path": *basePath, + "admin.user": *adminUser, + "admin.pass": *adminPass, + "admin.mail": *adminMail, + "s3.bucket": *s3Bucket, + "s3.endpoint": *s3Endpoint, + "s3.region": *s3Region, + } + + if *s3NoSSL { + static["s3.nossl"] = "true" + } else { + static["s3.nossl"] = "false" + } + + if *s3PathStyle { + static["s3.pathstyle"] = "true" + } else { + static["s3.pathstyle"] = "false" } mappings := map[string]string{ - "DB_HOST": "db.host", - "DB_NAME": "db.name", - "DB_USER": "db.user", - "DB_PASS": "db.pass", - "DB_SSLMODE": "db.ssl", - "AMQP_HOST": "amqp.host", - "AMQP_USER": "amqp.user", - "AMQP_PASS": "amqp.pass", - "BASE_HOST": "base.host", - "BASE_PATH": "base.path", - "MODE": "mode", - "PORT": "port", - "ADMIN_USER": "admin.user", - "ADMIN_PASS": "admin.pass", - "ADMIN_MAIL": "admin.mail", + "DB_HOST": "db.host", + "DB_NAME": "db.name", + "DB_USER": "db.user", + "DB_PASS": "db.pass", + "DB_SSLMODE": "db.ssl", + "AMQP_HOST": "amqp.host", + "AMQP_USER": "amqp.user", + "AMQP_PASS": "amqp.pass", + "BASE_HOST": "base.host", + "BASE_PATH": "base.path", + "MODE": "mode", + "PORT": "port", + "ADMIN_USER": "admin.user", + "ADMIN_PASS": "admin.pass", + "ADMIN_MAIL": "admin.mail", + "S3_BUCKET": "s3.bucket", + "S3_ENDPOINT": "s3.endpoint", + "S3_REGION": "s3.region", + "S3_NOSSL": "s3.nossl", } defaults := config.NewStatic(static) diff --git a/database/models.go b/database/models.go index 869d8f3..46c2470 100644 --- a/database/models.go +++ b/database/models.go @@ -22,9 +22,10 @@ package database import ( - "github.com/lib/pq" "time" + "github.com/lib/pq" + "github.com/jinzhu/gorm/dialects/postgres" ) @@ -197,6 +198,8 @@ type File struct { Model // Name of file Name string `json:"name" gorm:"not null"` + // Key of file in S3 bucket + Key string `json:"key"` // Type of file (MIME type) Type string `json:"type"` // Size of file (in byte) diff --git a/go.mod b/go.mod index 4385ce8..0e9c098 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module git.rwth-aachen.de/acs/public/villas/web-backend-go require ( github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 + github.com/aws/aws-sdk-go v1.35.35 github.com/chenjiandongx/ginprom v0.0.0-20191022035802-6f3da3c84986 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/gin-gonic/gin v1.4.0 @@ -19,7 +20,7 @@ require ( github.com/swaggo/gin-swagger v1.2.0 github.com/swaggo/swag v1.6.3 github.com/zpatrick/go-config v0.0.0-20191104215613-50bc2709703f - golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 gopkg.in/go-playground/validator.v9 v9.30.0 gopkg.in/ini.v1 v1.51.0 // indirect ) diff --git a/go.sum b/go.sum index 5870c78..5da25ee 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/aws/aws-sdk-go v1.35.35 h1:o/EbgEcIPWga7GWhJhb3tiaxqk4/goTdo5YEMdnVxgE= +github.com/aws/aws-sdk-go v1.35.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -105,6 +107,9 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= @@ -151,6 +156,7 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -209,6 +215,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -228,6 +236,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -248,10 +258,13 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -288,6 +301,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/routes/file/file_methods.go b/routes/file/file_methods.go index 1694ede..b9a874e 100644 --- a/routes/file/file_methods.go +++ b/routes/file/file_methods.go @@ -22,8 +22,7 @@ package file import ( - "git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/scenario" - "github.com/gin-gonic/gin" + "fmt" "image" _ "image/gif" _ "image/jpeg" @@ -33,10 +32,13 @@ import ( "mime/multipart" "net/http" "path/filepath" - "strconv" "strings" "time" + "git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration" + "git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/scenario" + "github.com/gin-gonic/gin" + "git.rwth-aachen.de/acs/public/villas/web-backend-go/database" ) @@ -62,20 +64,25 @@ func (f *File) save() error { func (f *File) download(c *gin.Context) error { - // create unique file name - filename := "file_" + strconv.FormatUint(uint64(f.ID), 10) + "_" + f.Name - // detect the content type of the file - contentType := http.DetectContentType(f.FileData) - //Seems this headers needed for some browsers (for example without this headers Chrome will download files as txt) - c.Header("Content-Description", "File Transfer") - c.Header("Content-Disposition", "attachment; filename="+filename) - c.Data(http.StatusOK, contentType, f.FileData) + if f.Key == "" { + // Seems this headers needed for some browsers (for example without this headers Chrome will download files as txt) + c.Header("Content-Description", "File Transfer") + c.Header("Content-Disposition", "attachment; filename="+f.Name) + c.Header("Expires", "") + c.Header("Cache-Control", "") + c.Data(http.StatusOK, f.Type, f.FileData) + } else { + url, err := f.getS3Url() + if err != nil { + return fmt.Errorf("Failed to presign S3 request: %s", err) + } + c.Redirect(http.StatusFound, url) + } return nil } func (f *File) register(fileHeader *multipart.FileHeader, scenarioID uint) error { - // Obtain properties of file f.Type = fileHeader.Header.Get("Content-Type") f.Name = filepath.Base(fileHeader.Filename) @@ -85,12 +92,21 @@ func (f *File) register(fileHeader *multipart.FileHeader, scenarioID uint) error // set file data fileContent, err := fileHeader.Open() + defer fileContent.Close() if err != nil { return err } - f.FileData, err = ioutil.ReadAll(fileContent) - defer fileContent.Close() + bucket, err := configuration.GolbalConfig.String("s3.bucket") + if bucket == "" { + f.FileData, err = ioutil.ReadAll(fileContent) + f.Key = "" + } else { + err := f.putS3(fileContent) + if err != nil { + return fmt.Errorf("Failed to upload to S3 bucket: %s", err) + } + } // Add image dimensions in case the file is an image if strings.Contains(f.Type, "image") || strings.Contains(f.Type, "Image") { @@ -100,13 +116,13 @@ func (f *File) register(fileHeader *multipart.FileHeader, scenarioID uint) error imageConfig, _, err := image.DecodeConfig(fileContent) if err != nil { - log.Println("Unable to decode image configuration: Dimensions of image file are not set: ", err) - } else { - f.ImageHeight = imageConfig.Height - f.ImageWidth = imageConfig.Width + return fmt.Errorf("Unable to decode image configuration: Dimensions of image file are not set: ", err) } + + f.ImageHeight = imageConfig.Height + f.ImageWidth = imageConfig.Width } else { - log.Println("Error on setting file reader back to start of file, dimensions not updated:", err) + return fmt.Errorf("Error on setting file reader back to start of file, dimensions not updated: ", err) } } @@ -134,49 +150,54 @@ func (f *File) update(fileHeader *multipart.FileHeader) error { // set file data fileContent, err := fileHeader.Open() + defer fileContent.Close() if err != nil { return err } - fileData, err := ioutil.ReadAll(fileContent) - defer fileContent.Close() + bucket, err := configuration.GolbalConfig.String("s3.bucket") + if bucket == "" { + f.FileData, err = ioutil.ReadAll(fileContent) + f.Key = "" + } else { + err := f.putS3(fileContent) + if err != nil { + return fmt.Errorf("Failed to upload to S3 bucket: %s", err) + } + } - fileType := fileHeader.Header.Get("Content-Type") - imageHeight := f.ImageHeight - imageWidth := f.ImageWidth + f.Type = fileHeader.Header.Get("Content-Type") + f.Size = uint(fileHeader.Size) + f.Date = time.Now().String() + f.Name = filepath.Base(fileHeader.Filename) // Update image dimensions in case the file is an image - if strings.Contains(fileType, "image") || strings.Contains(fileType, "Image") { + if strings.Contains(f.Type, "image") || strings.Contains(f.Type, "Image") { // set the file reader back to the start of the file _, err := fileContent.Seek(0, 0) if err == nil { imageConfig, _, err := image.DecodeConfig(fileContent) if err != nil { log.Println("Unable to decode image configuration: Dimensions of image file are not updated.", err) - } else { - imageHeight = imageConfig.Height - imageWidth = imageConfig.Width } + + f.ImageHeight = imageConfig.Height + f.ImageWidth = imageConfig.Width } else { log.Println("Error on setting file reader back to start of file, dimensions not updated::", err) } } else { - imageWidth = 0 - imageHeight = 0 + f.ImageWidth = 0 + f.ImageHeight = 0 } - db := database.GetDB() - err = db.Model(f).Updates(map[string]interface{}{ - "Size": uint(fileHeader.Size), - "FileData": fileData, - "Date": time.Now().String(), - "Name": filepath.Base(fileHeader.Filename), - "Type": fileType, - "ImageHeight": imageHeight, - "ImageWidth": imageWidth, - }).Error + // Add File object with parameters to DB + err = f.save() + if err != nil { + return err + } - return err + return nil } func (f *File) delete() error { @@ -189,6 +210,12 @@ func (f *File) delete() error { if err != nil { return err } + + // delete file from s3 bucket + if f.Key != "" { + f.deleteS3() + } + err = db.Model(&so).Association("Files").Delete(f).Error if err != nil { return err diff --git a/routes/file/file_s3.go b/routes/file/file_s3.go new file mode 100644 index 0000000..60d7dfb --- /dev/null +++ b/routes/file/file_s3.go @@ -0,0 +1,166 @@ +/** File package, S3 uploads. +* +* @author Steffen Vogel +* @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC +* @license GNU General Public License (version 3) +* +* VILLASweb-backend-go +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see . +*********************************************************************************/ +package file + +import ( + "fmt" + "io" + "time" + + "git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/google/uuid" +) + +// Global session +var s3Session *session.Session = nil + +func getS3Session() (*session.Session, error) { + if s3Session == nil { + var err error + s3Session, err = createS3Session() + if err != nil { + return nil, err + } + } + + return s3Session, nil +} + +func createS3Session() (*session.Session, error) { + endpoint, err := configuration.GolbalConfig.String("s3.endpoint") + region, err := configuration.GolbalConfig.String("s3.region") + pathStyle, err := configuration.GolbalConfig.Bool("s3.pathstyle") + nossl, err := configuration.GolbalConfig.Bool("s3.nossl") + + sess, err := session.NewSession( + &aws.Config{ + Region: aws.String(region), + Endpoint: aws.String(endpoint), + DisableSSL: aws.Bool(nossl), + S3ForcePathStyle: aws.Bool(pathStyle), + }, + ) + if err != nil { + return nil, err + } + + return sess, nil +} + +func (f *File) putS3(fileContent io.Reader) error { + + bucket, err := configuration.GolbalConfig.String("s3.bucket") + if err != nil || bucket == "" { + return fmt.Errorf("No S3 bucket configured") + } + + // The session the S3 Uploader will use + sess, err := getS3Session() + if err != nil { + return fmt.Errorf("Failed to create session: %s", err) + } + + // Create an uploader with the session and default options + uploader := s3manager.NewUploader(sess) + + f.Key = uuid.New().String() + f.FileData = nil + + // Upload the file to S3. + _, err = uploader.Upload(&s3manager.UploadInput{ + Bucket: aws.String(bucket), + Key: aws.String(f.Key), + Body: fileContent, + }) + if err != nil { + return fmt.Errorf("Failed to upload file, %v", err) + } + + return nil +} + +func (f *File) getS3Url() (string, error) { + bucket, err := configuration.GolbalConfig.String("s3.bucket") + if err != nil || bucket == "" { + return "", fmt.Errorf("No S3 bucket configured") + } + + // The session the S3 Uploader will use + sess, err := getS3Session() + if err != nil { + return "", fmt.Errorf("Failed to create session: %s", err) + } + + // Create S3 service client + svc := s3.New(sess) + + req, _ := svc.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(f.Key), + ResponseContentType: aws.String(f.Type), + ResponseContentDisposition: aws.String("attachment; filename=" + f.Name), + // ResponseContentEncoding: aws.String(), + // ResponseContentLanguage: aws.String(), + // ResponseCacheControl: aws.String(), + // ResponseExpires: aws.String(), + }) + + urlStr, err := req.Presign(5 * 24 * 60 * time.Minute) + if err != nil { + return "", err + } + + return urlStr, nil +} + +func (f *File) deleteS3() error { + bucket, err := configuration.GolbalConfig.String("s3.bucket") + if err != nil || bucket == "" { + return fmt.Errorf("No S3 bucket configured") + } + + // The session the S3 Uploader will use + sess, err := getS3Session() + if err != nil { + return fmt.Errorf("Failed to create session: %s", err) + } + + // Create S3 service client + svc := s3.New(sess) + + _, err = svc.DeleteObject(&s3.DeleteObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(f.Key), + }) + if err != nil { + return err + } + + f.Key = "" + + return nil +} From 4d9699fbcff115a4514950c35d5e2e17f3bb1bc0 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Wed, 13 Jan 2021 12:19:13 +0100 Subject: [PATCH 02/27] fix error messages --- routes/file/file_methods.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routes/file/file_methods.go b/routes/file/file_methods.go index f798aa9..3f5caea 100644 --- a/routes/file/file_methods.go +++ b/routes/file/file_methods.go @@ -117,13 +117,13 @@ func (f *File) Register(fileHeader *multipart.FileHeader, scenarioID uint) error imageConfig, _, err := image.DecodeConfig(fileContent) if err != nil { - return fmt.Errorf("Unable to decode image configuration: Dimensions of image file are not set: ", err) + return fmt.Errorf("unable to decode image configuration: Dimensions of image file are not set: %v", err) } f.ImageHeight = imageConfig.Height f.ImageWidth = imageConfig.Width } else { - return fmt.Errorf("Error on setting file reader back to start of file, dimensions not updated: ", err) + return fmt.Errorf("error on setting file reader back to start of file, dimensions not updated: %v", err) } } From cd0aa25f49f90216e9ab8311539fbc32b70f3845 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Wed, 13 Jan 2021 14:37:35 +0100 Subject: [PATCH 03/27] fix for file update method, remove redundant code, error formatting fixes --- go.sum | 3 +-- routes/file/file_endpoints.go | 4 ++-- routes/file/file_methods.go | 32 +++++++++++++++++---------- routes/file/file_s3.go | 41 +++++++++++++++-------------------- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/go.sum b/go.sum index 4f52e96..ed40b86 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,7 @@ github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -213,8 +214,6 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4= -golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= diff --git a/routes/file/file_endpoints.go b/routes/file/file_endpoints.go index df1ae77..e4eb792 100644 --- a/routes/file/file_endpoints.go +++ b/routes/file/file_endpoints.go @@ -97,14 +97,14 @@ func addFile(c *gin.Context) { } // Extract file from POST request form - file_header, err := c.FormFile("file") + fileHeader, err := c.FormFile("file") if err != nil { helper.BadRequestError(c, fmt.Sprintf("Get form error: %s", err.Error())) return } var newFile File - err = newFile.Register(file_header, so.ID) + err = newFile.Register(fileHeader, so.ID) if !helper.DBError(c, err) { c.JSON(http.StatusOK, gin.H{"file": newFile.File}) } diff --git a/routes/file/file_methods.go b/routes/file/file_methods.go index 3f5caea..77e8b1a 100644 --- a/routes/file/file_methods.go +++ b/routes/file/file_methods.go @@ -74,7 +74,7 @@ func (f *File) download(c *gin.Context) error { } else { url, err := f.getS3Url() if err != nil { - return fmt.Errorf("Failed to presign S3 request: %s", err) + return fmt.Errorf("failed to presign S3 request: %s", err) } c.Redirect(http.StatusFound, url) } @@ -93,10 +93,10 @@ func (f *File) Register(fileHeader *multipart.FileHeader, scenarioID uint) error // set file data fileContent, err := fileHeader.Open() - defer fileContent.Close() if err != nil { return err } + defer fileContent.Close() bucket, err := configuration.GolbalConfig.String("s3.bucket") if bucket == "" { @@ -105,7 +105,7 @@ func (f *File) Register(fileHeader *multipart.FileHeader, scenarioID uint) error } else { err := f.putS3(fileContent) if err != nil { - return fmt.Errorf("Failed to upload to S3 bucket: %s", err) + return fmt.Errorf("failed to upload to S3 bucket: %s", err) } } @@ -151,10 +151,10 @@ func (f *File) update(fileHeader *multipart.FileHeader) error { // set file data fileContent, err := fileHeader.Open() - defer fileContent.Close() if err != nil { return err } + defer fileContent.Close() bucket, err := configuration.GolbalConfig.String("s3.bucket") if bucket == "" { @@ -163,7 +163,7 @@ func (f *File) update(fileHeader *multipart.FileHeader) error { } else { err := f.putS3(fileContent) if err != nil { - return fmt.Errorf("Failed to upload to S3 bucket: %s", err) + return fmt.Errorf("failed to upload to S3 bucket: %s", err) } } @@ -193,12 +193,19 @@ func (f *File) update(fileHeader *multipart.FileHeader) error { } // Add File object with parameters to DB - err = f.save() - if err != nil { - return err - } + db := database.GetDB() + err = db.Model(f).Updates(map[string]interface{}{ + "Size": f.Size, + "FileData": f.FileData, + "Date": f.Date, + "Name": f.Name, + "Type": f.Type, + "ImageHeight": f.ImageHeight, + "ImageWidth": f.ImageWidth, + "Key": f.Key, + }).Error - return nil + return err } func (f *File) Delete() error { @@ -214,7 +221,10 @@ func (f *File) Delete() error { // delete file from s3 bucket if f.Key != "" { - f.deleteS3() + err = f.deleteS3() + if err != nil { + return err + } } err = db.Model(&so).Association("Files").Delete(f).Error diff --git a/routes/file/file_s3.go b/routes/file/file_s3.go index 60d7dfb..95ea322 100644 --- a/routes/file/file_s3.go +++ b/routes/file/file_s3.go @@ -38,16 +38,22 @@ import ( // Global session var s3Session *session.Session = nil -func getS3Session() (*session.Session, error) { +func getS3Session() (*session.Session, string, error) { + + bucket, err := configuration.GolbalConfig.String("s3.bucket") + if err != nil || bucket == "" { + return nil, "", fmt.Errorf("no S3 bucket configured: %s", err) + } + if s3Session == nil { var err error s3Session, err = createS3Session() if err != nil { - return nil, err + return nil, "", err } } - return s3Session, nil + return s3Session, bucket, nil } func createS3Session() (*session.Session, error) { @@ -65,7 +71,7 @@ func createS3Session() (*session.Session, error) { }, ) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create session: %s", err) } return sess, nil @@ -73,15 +79,10 @@ func createS3Session() (*session.Session, error) { func (f *File) putS3(fileContent io.Reader) error { - bucket, err := configuration.GolbalConfig.String("s3.bucket") - if err != nil || bucket == "" { - return fmt.Errorf("No S3 bucket configured") - } - // The session the S3 Uploader will use - sess, err := getS3Session() + sess, bucket, err := getS3Session() if err != nil { - return fmt.Errorf("Failed to create session: %s", err) + return err } // Create an uploader with the session and default options @@ -97,22 +98,18 @@ func (f *File) putS3(fileContent io.Reader) error { Body: fileContent, }) if err != nil { - return fmt.Errorf("Failed to upload file, %v", err) + return fmt.Errorf("failed to upload file, %v", err) } return nil } func (f *File) getS3Url() (string, error) { - bucket, err := configuration.GolbalConfig.String("s3.bucket") - if err != nil || bucket == "" { - return "", fmt.Errorf("No S3 bucket configured") - } // The session the S3 Uploader will use - sess, err := getS3Session() + sess, bucket, err := getS3Session() if err != nil { - return "", fmt.Errorf("Failed to create session: %s", err) + return "", err } // Create S3 service client @@ -138,15 +135,11 @@ func (f *File) getS3Url() (string, error) { } func (f *File) deleteS3() error { - bucket, err := configuration.GolbalConfig.String("s3.bucket") - if err != nil || bucket == "" { - return fmt.Errorf("No S3 bucket configured") - } // The session the S3 Uploader will use - sess, err := getS3Session() + sess, bucket, err := getS3Session() if err != nil { - return fmt.Errorf("Failed to create session: %s", err) + return err } // Create S3 service client From 81307ef7166feed41f281df325439600051dcf8d Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Wed, 13 Jan 2021 17:09:09 +0100 Subject: [PATCH 04/27] WIP: adapt CI for S3 storage --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 321f16c..6782519 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ services: - postgres:latest - rabbitmq:latest + - minio/minio:RELEASE.2021-01-08T21-18-21Z variables: DOCKER_TAG: ${CI_COMMIT_SHORT_SHA} @@ -14,6 +15,8 @@ variables: AMQP_HOST: rabbitmq:5672 AMQP_USER: villas AMQP_PASS: villas + S3_BUCKET: villas-web + S3_ENDPOINT: minio/minio:9000 PORT: 4000 GO_IMAGE: golang:1.13-buster From 133c3949dcbf90a1b5635ca4c4eb0960ae5554e2 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 14 Jan 2021 13:26:24 +0100 Subject: [PATCH 05/27] WIP: CI: add minio user and pw #50 --- .gitlab-ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6782519..2bcaeed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,8 +15,13 @@ variables: AMQP_HOST: rabbitmq:5672 AMQP_USER: villas AMQP_PASS: villas + MINIO_ROOT_USER: minio-villas + MINIO_ROOT_PASSWORD: minio-villas + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} S3_BUCKET: villas-web S3_ENDPOINT: minio/minio:9000 + PORT: 4000 GO_IMAGE: golang:1.13-buster From 9426e289dcea6564975016516dff16146595e81a Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 14 Jan 2021 13:57:59 +0100 Subject: [PATCH 06/27] WIP: CI fixes in minio service config --- .gitlab-ci.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2bcaeed..e79b28b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,9 @@ services: - postgres:latest - rabbitmq:latest - - minio/minio:RELEASE.2021-01-08T21-18-21Z + - name: minio/minio + command: ['server', '/minio'] + alias: minio variables: DOCKER_TAG: ${CI_COMMIT_SHORT_SHA} @@ -15,12 +17,13 @@ variables: AMQP_HOST: rabbitmq:5672 AMQP_USER: villas AMQP_PASS: villas - MINIO_ROOT_USER: minio-villas - MINIO_ROOT_PASSWORD: minio-villas - AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} - AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} + MINIO_ACCESS_KEY: minio-villas + MINIO_ACCESS_SECRET: minio-villas + MINIO_DOMAIN: localhost + AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY} + AWS_SECRET_ACCESS_KEY: ${MINIO_ACCESS_SECRET} S3_BUCKET: villas-web - S3_ENDPOINT: minio/minio:9000 + S3_ENDPOINT: http://minio:9000 PORT: 4000 GO_IMAGE: golang:1.13-buster From 2f796ed12069a0df0c67e2c6be09a366a1b1139a Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 14 Jan 2021 14:21:49 +0100 Subject: [PATCH 07/27] add S3 path style as environment variable and set it in CI config --- .gitlab-ci.yml | 2 +- configuration/config.go | 39 ++++++++++++++++++++------------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e79b28b..b3e4f07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: AWS_SECRET_ACCESS_KEY: ${MINIO_ACCESS_SECRET} S3_BUCKET: villas-web S3_ENDPOINT: http://minio:9000 - + S3_PATHSTYLE: 'true' PORT: 4000 GO_IMAGE: golang:1.13-buster diff --git a/configuration/config.go b/configuration/config.go index dcec201..e1ff52a 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -96,25 +96,26 @@ func InitConfig() error { } mappings := map[string]string{ - "DB_HOST": "db.host", - "DB_NAME": "db.name", - "DB_USER": "db.user", - "DB_PASS": "db.pass", - "DB_SSLMODE": "db.ssl", - "AMQP_HOST": "amqp.host", - "AMQP_USER": "amqp.user", - "AMQP_PASS": "amqp.pass", - "BASE_HOST": "base.host", - "BASE_PATH": "base.path", - "MODE": "mode", - "PORT": "port", - "ADMIN_USER": "admin.user", - "ADMIN_PASS": "admin.pass", - "ADMIN_MAIL": "admin.mail", - "S3_BUCKET": "s3.bucket", - "S3_ENDPOINT": "s3.endpoint", - "S3_REGION": "s3.region", - "S3_NOSSL": "s3.nossl", + "DB_HOST": "db.host", + "DB_NAME": "db.name", + "DB_USER": "db.user", + "DB_PASS": "db.pass", + "DB_SSLMODE": "db.ssl", + "AMQP_HOST": "amqp.host", + "AMQP_USER": "amqp.user", + "AMQP_PASS": "amqp.pass", + "BASE_HOST": "base.host", + "BASE_PATH": "base.path", + "MODE": "mode", + "PORT": "port", + "ADMIN_USER": "admin.user", + "ADMIN_PASS": "admin.pass", + "ADMIN_MAIL": "admin.mail", + "S3_BUCKET": "s3.bucket", + "S3_ENDPOINT": "s3.endpoint", + "S3_REGION": "s3.region", + "S3_NOSSL": "s3.nossl", + "S3_PATHSTYLE": "s3.pathstyle", } defaults := config.NewStatic(static) From e8610feb60e48d7ddea64042284a9b019e1bae1d Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 14 Jan 2021 14:37:35 +0100 Subject: [PATCH 08/27] CI: remove minio domain --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3e4f07..104aabf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,7 +19,6 @@ variables: AMQP_PASS: villas MINIO_ACCESS_KEY: minio-villas MINIO_ACCESS_SECRET: minio-villas - MINIO_DOMAIN: localhost AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY} AWS_SECRET_ACCESS_KEY: ${MINIO_ACCESS_SECRET} S3_BUCKET: villas-web From 7728ccf5fa362db2f849da6998539dd556dca88a Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 14 Jan 2021 15:13:12 +0100 Subject: [PATCH 09/27] CI: work on minio config --- .gitlab-ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 104aabf..7ac7948 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ services: - postgres:latest - rabbitmq:latest - - name: minio/minio + - name: minio/minio:RELEASE.2020-10-28T08-16-50Z.e0655e24f command: ['server', '/minio'] alias: minio @@ -17,13 +17,17 @@ variables: AMQP_HOST: rabbitmq:5672 AMQP_USER: villas AMQP_PASS: villas - MINIO_ACCESS_KEY: minio-villas - MINIO_ACCESS_SECRET: minio-villas + MINIO_ROOT_USER: minio-villas + MINIO_ROOT_ACCESS: minio-villas + MINIO_ACCESS_KEY: ${MINIO_ROOT_USER} + MINIO_ACCESS_SECRET: ${MINIO_ROOT_ACCESS} + MINIO_DOMAIN: localhost AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY} AWS_SECRET_ACCESS_KEY: ${MINIO_ACCESS_SECRET} S3_BUCKET: villas-web S3_ENDPOINT: http://minio:9000 S3_PATHSTYLE: 'true' + S3_NOSSL: 'true' PORT: 4000 GO_IMAGE: golang:1.13-buster From cc98eec16e7f7724f6680f951e3a251b54ab2942 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 16:02:50 +0100 Subject: [PATCH 10/27] CI: fix env var for minio creds --- .gitlab-ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ac7948..bdf6c73 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ services: - postgres:latest - rabbitmq:latest - - name: minio/minio:RELEASE.2020-10-28T08-16-50Z.e0655e24f + - name: minio/minio:RELEASE.2021-01-16T02-19-44Z command: ['server', '/minio'] alias: minio @@ -18,12 +18,10 @@ variables: AMQP_USER: villas AMQP_PASS: villas MINIO_ROOT_USER: minio-villas - MINIO_ROOT_ACCESS: minio-villas - MINIO_ACCESS_KEY: ${MINIO_ROOT_USER} - MINIO_ACCESS_SECRET: ${MINIO_ROOT_ACCESS} + MINIO_ROOT_PASSWORD: minio-villas MINIO_DOMAIN: localhost - AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY} - AWS_SECRET_ACCESS_KEY: ${MINIO_ACCESS_SECRET} + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} S3_BUCKET: villas-web S3_ENDPOINT: http://minio:9000 S3_PATHSTYLE: 'true' From 86fe71cbc770b2f683c261702dbd15727a8f41fb Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 16:09:44 +0100 Subject: [PATCH 11/27] CI: restrict services to test job --- .gitlab-ci.yml | 51 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bdf6c73..6c6b3c1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,32 +1,6 @@ -services: - - postgres:latest - - rabbitmq:latest - - name: minio/minio:RELEASE.2021-01-16T02-19-44Z - command: ['server', '/minio'] - alias: minio - variables: DOCKER_TAG: ${CI_COMMIT_SHORT_SHA} DOCKER_IMAGE: ${CI_REGISTRY_IMAGE} - POSTGRES_DB: testvillasdb - POSTGRES_USER: villas - POSTGRES_PASSWORD: villas - POSTGRES_HOST: postgres - RABBITMQ_DEFAULT_USER: villas - RABBITMQ_DEFAULT_PASS: villas - AMQP_HOST: rabbitmq:5672 - AMQP_USER: villas - AMQP_PASS: villas - MINIO_ROOT_USER: minio-villas - MINIO_ROOT_PASSWORD: minio-villas - MINIO_DOMAIN: localhost - AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} - AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} - S3_BUCKET: villas-web - S3_ENDPOINT: http://minio:9000 - S3_PATHSTYLE: 'true' - S3_NOSSL: 'true' - PORT: 4000 GO_IMAGE: golang:1.13-buster stages: @@ -57,6 +31,25 @@ test: stage: test image: ${GO_IMAGE} variables: + POSTGRES_DB: testvillasdb + POSTGRES_USER: villas + POSTGRES_PASSWORD: villas + POSTGRES_HOST: postgres + RABBITMQ_DEFAULT_USER: villas + RABBITMQ_DEFAULT_PASS: villas + MINIO_ROOT_USER: minio-villas + MINIO_ROOT_PASSWORD: minio-villas + MINIO_DOMAIN: localhost + AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} + AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: villas-web + S3_ENDPOINT: http://minio:9000 + S3_PATHSTYLE: 'true' + S3_NOSSL: 'true' + AMQP_HOST: rabbitmq:5672 + AMQP_USER: villas + AMQP_PASS: villas + PORT: 4000 DB_NAME: ${POSTGRES_DB} DB_HOST: ${POSTGRES_HOST} DB_USER: ${POSTGRES_USER} @@ -65,6 +58,12 @@ test: MODE: test ADMIN_USER: User_0 ADMIN_PASS: xyz789 + services: + - postgres:latest + - rabbitmq:latest + - name: minio/minio:RELEASE.2021-01-16T02-19-44Z + command: ['server', '/minio'] + alias: minio script: - go mod tidy - go test $(go list ./... ) From 6f5d01251e084c672ee0869375aee20546c6f7a1 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 16:13:19 +0100 Subject: [PATCH 12/27] CI: cache golang deps --- .gitlab-ci.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c6b3c1..2e7909c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,17 @@ variables: DOCKER_IMAGE: ${CI_REGISTRY_IMAGE} GO_IMAGE: golang:1.13-buster +# Template +.go: + image: ${GO_IMAGE} + variables: + GOPATH: $CI_PROJECT_DIR/.go + before_script: + - mkdir -p .go + cache: + paths: + - .go/pkg/mod/ + stages: - build - test @@ -13,7 +24,7 @@ stages: build: stage: build - image: ${GO_IMAGE} + extends: .go script: - go mod tidy - go install github.com/swaggo/swag/cmd/swag @@ -29,7 +40,7 @@ build: test: stage: test - image: ${GO_IMAGE} + extends: .go variables: POSTGRES_DB: testvillasdb POSTGRES_USER: villas From 55fa56a5b5704f836c0b3647716daf795d215904 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 16:25:31 +0100 Subject: [PATCH 13/27] CI: create default villas bucket --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2e7909c..7491ce7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -75,6 +75,10 @@ test: - name: minio/minio:RELEASE.2021-01-16T02-19-44Z command: ['server', '/minio'] alias: minio + before_script: + - wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/bin/mc && chmod +x /usr/bin/mc + - mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} + - mc mb gitlab/${S3_BUCKET} script: - go mod tidy - go test $(go list ./... ) From 3e2aadee39873fc52ad74aab211d804c70b78e37 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 16:27:24 +0100 Subject: [PATCH 14/27] CI: fix PATH for new GOPATH --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7491ce7..f462895 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,7 +28,7 @@ build: script: - go mod tidy - go install github.com/swaggo/swag/cmd/swag - - swag init -p pascalcase -g "start.go" -o "./doc/api/" + - ${GOPATH}/bin/swag init -p pascalcase -g "start.go" -o "./doc/api/" - go build artifacts: paths: From 1e301cb422ed85b945d19f3c0c119ed97c395f6a Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 17:38:42 +0100 Subject: [PATCH 15/27] tests: allow TestEndpoint() to follow redirects --- helper/test_utilities.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/helper/test_utilities.go b/helper/test_utilities.go index 0d2d1cb..6bb56be 100644 --- a/helper/test_utilities.go +++ b/helper/test_utilities.go @@ -25,11 +25,12 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/gin-gonic/gin" - "github.com/nsf/jsondiff" "log" "net/http" "net/http/httptest" + + "github.com/gin-gonic/gin" + "github.com/nsf/jsondiff" ) // data type used in testing @@ -169,7 +170,25 @@ func TestEndpoint(router *gin.Engine, token string, url string, } req.Header.Set("Content-Type", "application/json") req.Header.Add("Authorization", "Bearer "+token) - router.ServeHTTP(w, req) + + const maxRedirCount = 10 + var redirCount int + for redirCount = 0; redirCount < maxRedirCount; redirCount++ { + router.ServeHTTP(w, req) + + if w.Code == 301 || w.Code == 302 || w.Code == 307 || w.Code == 308 { + req.URL, err = w.Result().Location() + if err != nil { + return 0, nil, fmt.Errorf("Invalid location header") + } + } else { + break + } + } + + if redirCount == maxRedirCount { + return 0, nil, fmt.Errorf("Max redirection count exceeded") + } return w.Code, w.Body, nil } From 9b0f780ab32299aebddb871a39ea189910d9a291 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 18:21:34 +0100 Subject: [PATCH 16/27] renamed some files --- .gitlab-ci.yml | 2 +- helper/test_utilities.go | 42 ++++++++++++++++++++++++++-------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f462895..34d86bc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -76,7 +76,7 @@ test: command: ['server', '/minio'] alias: minio before_script: - - wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/bin/mc && chmod +x /usr/bin/mc + - wget -qO /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x /usr/bin/mc - mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} - mc mb gitlab/${S3_BUCKET} script: diff --git a/helper/test_utilities.go b/helper/test_utilities.go index 6bb56be..d0d28ba 100644 --- a/helper/test_utilities.go +++ b/helper/test_utilities.go @@ -171,25 +171,37 @@ func TestEndpoint(router *gin.Engine, token string, url string, req.Header.Set("Content-Type", "application/json") req.Header.Add("Authorization", "Bearer "+token) - const maxRedirCount = 10 - var redirCount int - for redirCount = 0; redirCount < maxRedirCount; redirCount++ { - router.ServeHTTP(w, req) + router.ServeHTTP(w, req) - if w.Code == 301 || w.Code == 302 || w.Code == 307 || w.Code == 308 { - req.URL, err = w.Result().Location() - if err != nil { - return 0, nil, fmt.Errorf("Invalid location header") - } - } else { - break + return handleRedirect(w, req) +} + +func handleRedirect(w *httptest.ResponseRecorder, req *http.Request) (int, *bytes.Buffer, error) { + if w.Code == http.StatusMovedPermanently || + w.Code == http.StatusFound || + w.Code == http.StatusTemporaryRedirect || + w.Code == http.StatusPermanentRedirect { + + // Follow external redirect + var err error + req.URL, err = w.Result().Location() + if err != nil { + return 0, nil, fmt.Errorf("Invalid location header") } + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return 0, nil, fmt.Errorf("Failed to follow redirect: %v", err) + } + + buf := new(bytes.Buffer) + buf.ReadFrom(resp.Body) + + return resp.StatusCode, buf, nil } - if redirCount == maxRedirCount { - return 0, nil, fmt.Errorf("Max redirection count exceeded") - } - + // No redirect return w.Code, w.Body, nil } From dd746295cc42a8361a61337138819156c98312ee Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 18:38:56 +0100 Subject: [PATCH 17/27] s3: fix broken pre-signed URL signature --- .gitlab-ci.yml | 1 - routes/file/file_s3.go | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34d86bc..87375aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,6 @@ test: RABBITMQ_DEFAULT_PASS: villas MINIO_ROOT_USER: minio-villas MINIO_ROOT_PASSWORD: minio-villas - MINIO_DOMAIN: localhost AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} S3_BUCKET: villas-web diff --git a/routes/file/file_s3.go b/routes/file/file_s3.go index 95ea322..b51f724 100644 --- a/routes/file/file_s3.go +++ b/routes/file/file_s3.go @@ -116,10 +116,10 @@ func (f *File) getS3Url() (string, error) { svc := s3.New(sess) req, _ := svc.GetObjectRequest(&s3.GetObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(f.Key), - ResponseContentType: aws.String(f.Type), - ResponseContentDisposition: aws.String("attachment; filename=" + f.Name), + Bucket: aws.String(bucket), + Key: aws.String(f.Key), + ResponseContentType: aws.String(f.Type), + // ResponseContentDisposition: aws.String("attachment; filename=" + f.Name), // ResponseContentEncoding: aws.String(), // ResponseContentLanguage: aws.String(), // ResponseCacheControl: aws.String(), From da6515ddddb157663b62b53db56e765334e9c2d4 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 20:34:28 +0100 Subject: [PATCH 18/27] S3: fix region inconsistency --- .gitlab-ci.yml | 2 ++ configuration/config.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 87375aa..d919b75 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,12 +50,14 @@ test: RABBITMQ_DEFAULT_PASS: villas MINIO_ROOT_USER: minio-villas MINIO_ROOT_PASSWORD: minio-villas + MINIO_REGION_NAME: default AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER} AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD} S3_BUCKET: villas-web S3_ENDPOINT: http://minio:9000 S3_PATHSTYLE: 'true' S3_NOSSL: 'true' + S3_REGION: ${MINIO_REGION_NAME} AMQP_HOST: rabbitmq:5672 AMQP_USER: villas AMQP_PASS: villas diff --git a/configuration/config.go b/configuration/config.go index e1ff52a..1da3044 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -56,7 +56,7 @@ func InitConfig() error { adminMail = flag.String("admin-mail", "", "Initial admin mail address") s3Bucket = flag.String("s3-bucket", "", "S3 Bucket for uploading files") s3Endpoint = flag.String("s3-endpoint", "", "Endpoint of S3 API for file uploads") - s3Region = flag.String("s3-region", "us-east-1", "S3 Region for file uploads") + s3Region = flag.String("s3-region", "default", "S3 Region for file uploads") s3NoSSL = flag.Bool("s3-nossl", false, "Use encrypted connections to the S3 API") s3PathStyle = flag.Bool("s3-pathstyle", false, "Use path-style S3 API") ) From 5627e80b27b471259d43e7f5a0d09a083903c4b7 Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 20:40:08 +0100 Subject: [PATCH 19/27] tests: create new request instead of updating old --- helper/test_utilities.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/helper/test_utilities.go b/helper/test_utilities.go index d0d28ba..6a95b7d 100644 --- a/helper/test_utilities.go +++ b/helper/test_utilities.go @@ -183,12 +183,17 @@ func handleRedirect(w *httptest.ResponseRecorder, req *http.Request) (int, *byte w.Code == http.StatusPermanentRedirect { // Follow external redirect - var err error - req.URL, err = w.Result().Location() + redirURL, err := w.Result().Location() if err != nil { return 0, nil, fmt.Errorf("Invalid location header") } + // TODO: resend orginal request body + req, err := http.NewRequest(req.Method, redirURL.String(), nil) + if err != nil { + return 0, nil, fmt.Errorf("Failed to create new request: %v", err) + } + client := &http.Client{} resp, err := client.Do(req) if err != nil { From 6e8f849eb92749c16704569c917905a8cf13f8ba Mon Sep 17 00:00:00 2001 From: Steffen Vogel Date: Mon, 18 Jan 2021 20:57:16 +0100 Subject: [PATCH 20/27] add skript to test and debug VILLASweb in our K8S cluster --- k8s_test.sh | 108 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 15 deletions(-) diff --git a/k8s_test.sh b/k8s_test.sh index 7093e5b..7806715 100644 --- a/k8s_test.sh +++ b/k8s_test.sh @@ -1,21 +1,99 @@ #!/bin/bash -kubectl -n services port-forward svc/postgres-postgresql 5432:5432 & PF1=$! -kubectl -n services port-forward svc/broker 5672:5672 & PF2=$! +set -e -NS=villas-demo +NS=${NS:-villas} +MK="minikube" +KCTL="kubectl -n${NS}" +HELM="helm -n${NS}" +# CHART="villas" +CHART="~/workspace/rwth/acs/public/catalogue/charts/villas" -POSTGRES_USER=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.username | base64 -d) -POSTGRES_PASS=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.password | base64 -d) -RABBITMQ_USER=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.username | base64 -d) -RABBITMQ_PASS=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.password | base64 -d) +CONFIG=$(mktemp) +NAME="villas" -go test ./routes/healthz -p 1 --args \ - -mode test \ - -db-host localhost -db-name villas \ - -db-user ${POSTGRES_USER} -db-pass ${POSTGRES_PASS} \ - -amqp-host localhost \ - -amqp-user ${RABBITMQ_USER} -amqp-pass ${RABBITMQ_PASS} +function cleanup() { + kill $PF1 $PF2 $PF3 + wait $PF1 $PF2 $PF3 -kill $PF1 $PF2 -wait $PF1 $PF2 + rm ${CONFIG} + + echo "Goodbye" +} +trap cleanup EXIT + +if [ -n "${USE_MINIKUBE}" ]; then + MK_START_OPTS="--addons=ingress" + if [ $(uname -s) == Darwin ]; then + # https://github.com/kubernetes/minikube/issues/7332 + MK_START_OPTS+="--vm=true" + fi + + ${MK} start ${MK_START_OPTS} + + kubectl -n kube-system expose deployment ingress-nginx-controller --type=LoadBalancer || true + + IP=$(minikube ip) + PORT=$(kubectl -n kube-system get service ingress-nginx-controller --output='jsonpath={.spec.ports[0].nodePort}') + + # Add pseudo hostname to /etc/hosts + echo "Please provide your root password for modifiying /etc/hosts" + sudo sed -in "/^${IP}.*/d" /etc/hosts + echo "${IP} minikube" | sudo tee -a /etc/hosts + + HELM_OPTS="--set web.backend.enabled=false \ + --set web.backend.external.ip=${IP} \ + --set web.backend.external.port=4000 \ + --set web.admin.password=admin \ + --set ingress.host=minikube" + + # Check if chart has already been deployed before + if helm get values villas > /dev/null; then + RABBITMQ_ERLANG_COOKIE=$(kubectl get secret --namespace default villas-broker -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode) + RABBITMQ_PASSWORD=$(kubectl get secret --namespace default villas-broker -o jsonpath="{.data.rabbitmq-password}" | base64 --decode) + + HELM_OPTS+=" --set broker.auth.password=${RABBITMQ_PASSWORD} --set broker.auth.erlangCookie=${RABBITMQ_ERLANG_COOKIE}" + fi + + ${HELM} upgrade \ + ${HELM_OPTS} \ + --install \ + --create-namespace \ + --wait \ + --repo https://packages.fein-aachen.org/helm/charts \ + ${NAME} ${CHART} +fi + +# Get backend config from cluster +${KCTL} get cm ${NAME}-web -o 'jsonpath={.data.config\.yaml}' > ${CONFIG} + +# Enable access of backend to db, broker and s3 +export DB_HOST=localhost +export DB_PASS=$(${KCTL} get secret ${NAME}-database -o 'jsonpath={.data.postgresql-password}' | base64 -d) + +export AMQP_HOST=localhost +export AMQP_PASS=$(${KCTL} get secret ${NAME}-broker -o 'jsonpath={.data.rabbitmq-password}' | base64 -d) + +export AWS_ACCESS_KEY_ID=$(${KCTL} get secret ${NAME}-s3 -o 'jsonpath={.data.accesskey}' | base64 -d) +export AWS_SECRET_ACCESS_KEY=$(${KCTL} get secret ${NAME}-s3 -o 'jsonpath={.data.secretkey}' | base64 -d) + +# Setup port forwards for backend +if [ -n "${USE_EXTERNAL_POSTGRESQL}" ]; then + kubectl -n postgresql port-forward svc/postgresql 5432:5432 & PF1=$! +else + ${KCTL} port-forward svc/${NAME}-database 5432:5432 & PF1=$! +fi +${KCTL} port-forward svc/${NAME}-broker 5672:5672 & PF2=$! + +sleep 2 + +if [ -n "${USE_MINIKUBE}" ]; then + # python -mwebbrowser http://minikube:${PORT} + echo + echo "===========================================" + echo "Access VILLASweb at http://minikube:${PORT}" + echo "===========================================" + echo +fi + +go run start.go -config ${CONFIG} From 79ca53ce1d4d5ee7606b7b4de52acfc3d0596370 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Tue, 19 Jan 2021 13:45:06 +0100 Subject: [PATCH 21/27] CI: use branch name as docker tag #53, try test with S3 SSL enabled, set brackend mode to release --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d919b75..2f3971a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DOCKER_TAG: ${CI_COMMIT_SHORT_SHA} + DOCKER_TAG: ${CI_COMMIT_BRANCH} DOCKER_IMAGE: ${CI_REGISTRY_IMAGE} GO_IMAGE: golang:1.13-buster @@ -56,7 +56,7 @@ test: S3_BUCKET: villas-web S3_ENDPOINT: http://minio:9000 S3_PATHSTYLE: 'true' - S3_NOSSL: 'true' + S3_NOSSL: 'false' S3_REGION: ${MINIO_REGION_NAME} AMQP_HOST: rabbitmq:5672 AMQP_USER: villas @@ -67,7 +67,7 @@ test: DB_USER: ${POSTGRES_USER} DB_PASS: ${POSTGRES_PASSWORD} BASE_PATH: /api - MODE: test + MODE: release ADMIN_USER: User_0 ADMIN_PASS: xyz789 services: From 2d3adbcc9b32c7576d51b3d42774e2710adb4a69 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Tue, 19 Jan 2021 14:10:38 +0100 Subject: [PATCH 22/27] Add some (temporary) debug output to test S3 --- routes/file/file_methods.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/routes/file/file_methods.go b/routes/file/file_methods.go index 77e8b1a..b0c5749 100644 --- a/routes/file/file_methods.go +++ b/routes/file/file_methods.go @@ -107,6 +107,7 @@ func (f *File) Register(fileHeader *multipart.FileHeader, scenarioID uint) error if err != nil { return fmt.Errorf("failed to upload to S3 bucket: %s", err) } + log.Println("Saved new file in S3 object storage") } // Add image dimensions in case the file is an image @@ -165,6 +166,8 @@ func (f *File) update(fileHeader *multipart.FileHeader) error { if err != nil { return fmt.Errorf("failed to upload to S3 bucket: %s", err) } + + log.Println("Updated file in S3 object storage") } f.Type = fileHeader.Header.Get("Content-Type") @@ -225,6 +228,7 @@ func (f *File) Delete() error { if err != nil { return err } + log.Println("Deleted file in S3 object storage") } err = db.Model(&so).Association("Files").Delete(f).Error From 5f73510679fa6d749c3f7801f0de143dbfb19d8d Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 21 Jan 2021 13:27:37 +0100 Subject: [PATCH 23/27] CI: run test in verbose mode --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2f3971a..6104cd0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,6 +84,7 @@ test: - go mod tidy - go test $(go list ./... ) -p 1 + -v -covermode=count -coverprofile ./testcover.txt - go tool cover -func=testcover.txt From 6db0373025469bbd20568b1ae9e35164ffcceb51 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 21 Jan 2021 15:22:24 +0100 Subject: [PATCH 24/27] improve handleRedirect method, add debug output --- helper/test_utilities.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/helper/test_utilities.go b/helper/test_utilities.go index 6a95b7d..a84e018 100644 --- a/helper/test_utilities.go +++ b/helper/test_utilities.go @@ -185,23 +185,27 @@ func handleRedirect(w *httptest.ResponseRecorder, req *http.Request) (int, *byte // Follow external redirect redirURL, err := w.Result().Location() if err != nil { - return 0, nil, fmt.Errorf("Invalid location header") + return 0, nil, fmt.Errorf("invalid location header") } - // TODO: resend orginal request body - req, err := http.NewRequest(req.Method, redirURL.String(), nil) + log.Println("redirecting request to", redirURL.String()) + + req, err := http.NewRequest(req.Method, redirURL.String(), req.Body) if err != nil { - return 0, nil, fmt.Errorf("Failed to create new request: %v", err) + return 0, nil, fmt.Errorf("handle redirect: failed to create new request: %v", err) } client := &http.Client{} resp, err := client.Do(req) if err != nil { - return 0, nil, fmt.Errorf("Failed to follow redirect: %v", err) + return 0, nil, fmt.Errorf("handle redirect: failed to follow redirect: %v", err) } buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) + _, err = buf.ReadFrom(resp.Body) + if err != nil { + return 0, nil, fmt.Errorf("handle redirect: failed to follow redirect: %v", err) + } return resp.StatusCode, buf, nil } From ddd43f4de7d1ecdf4d1daa668512d084553eadb6 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 21 Jan 2021 15:34:18 +0100 Subject: [PATCH 25/27] CI: disable kaniko cache, --- .gitlab-ci.yml | 2 -- configuration/config.go | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6104cd0..990e2ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -108,7 +108,5 @@ deploy: --dockerfile ${CI_PROJECT_DIR}/Dockerfile --destination ${DOCKER_IMAGE}:${DOCKER_TAG} --snapshotMode=redo - --cache=true - --cache-ttl=12h dependencies: - test diff --git a/configuration/config.go b/configuration/config.go index 1da3044..54ef502 100644 --- a/configuration/config.go +++ b/configuration/config.go @@ -83,13 +83,13 @@ func InitConfig() error { "s3.region": *s3Region, } - if *s3NoSSL { + if *s3NoSSL == true { static["s3.nossl"] = "true" } else { static["s3.nossl"] = "false" } - if *s3PathStyle { + if *s3PathStyle == true { static["s3.pathstyle"] = "true" } else { static["s3.pathstyle"] = "false" From 7dd719ba5f73049ea79975e77b86b436ef98f49d Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 21 Jan 2021 15:46:34 +0100 Subject: [PATCH 26/27] CI: add test job for file endpoints without S3 object storage --- .gitlab-ci.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 990e2ca..06f7e65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -91,6 +91,32 @@ test: dependencies: - build +test:files_postgres: +stage: test + extends: .go + variables: + POSTGRES_DB: testvillasdb + POSTGRES_USER: villas + POSTGRES_PASSWORD: villas + POSTGRES_HOST: postgres + PORT: 4000 + DB_NAME: ${POSTGRES_DB} + DB_HOST: ${POSTGRES_HOST} + DB_USER: ${POSTGRES_USER} + DB_PASS: ${POSTGRES_PASSWORD} + BASE_PATH: /api + MODE: release + ADMIN_USER: User_0 + ADMIN_PASS: xyz789 + services: + - postgres:latest + script: + - go mod tidy + - cd routes/file + - go test -v + dependencies: + - build + # Stage: deploy ############################################################################## From 86b7a702b5f2c53a2813720fc35dfceb3644bd10 Mon Sep 17 00:00:00 2001 From: Sonja Happ Date: Thu, 21 Jan 2021 15:48:28 +0100 Subject: [PATCH 27/27] CI: fix yaml file --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 06f7e65..8f9d37c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -92,7 +92,7 @@ test: - build test:files_postgres: -stage: test + stage: test extends: .go variables: POSTGRES_DB: testvillasdb