mirror of
https://git.rwth-aachen.de/acs/public/villas/web-backend-go/
synced 2025-03-30 00:00:12 +01:00
Merge branch 'large-files' into 'master'
S3 storage for large files See merge request acs/public/villas/web-backend-go!22
This commit is contained in:
Sonja Happ
commit
d11c1c5f61
10 changed files with 541 additions and 126 deletions
|
|
@ -1,22 +1,19 @@
|
|||
services:
|
||||
- postgres:latest
|
||||
- rabbitmq:latest
|
||||
|
||||
variables:
|
||||
DOCKER_TAG: ${CI_COMMIT_SHORT_SHA}
|
||||
DOCKER_TAG: ${CI_COMMIT_BRANCH}
|
||||
DOCKER_IMAGE: ${CI_REGISTRY_IMAGE}
|
||||
POSTGRES_DB: testvillasdb
|
||||
POSTGRES_USER: villas
|
||||
POSTGRES_PASSWORD: villas
|
||||
POSTGRES_HOST: postgres
|
||||
RABBITMQ_DEFAULT_USER: villas
|
||||
RABBITMQ_DEFAULT_PASS: villas
|
||||
AMQP_HOST: rabbitmq:5672
|
||||
AMQP_USER: villas
|
||||
AMQP_PASS: villas
|
||||
PORT: 4000
|
||||
GO_IMAGE: golang:1.13-buster
|
||||
|
||||
# Template
|
||||
.go:
|
||||
image: ${GO_IMAGE}
|
||||
variables:
|
||||
GOPATH: $CI_PROJECT_DIR/.go
|
||||
before_script:
|
||||
- mkdir -p .go
|
||||
cache:
|
||||
paths:
|
||||
- .go/pkg/mod/
|
||||
|
||||
stages:
|
||||
- build
|
||||
- test
|
||||
|
|
@ -27,11 +24,11 @@ stages:
|
|||
|
||||
build:
|
||||
stage: build
|
||||
image: ${GO_IMAGE}
|
||||
extends: .go
|
||||
script:
|
||||
- go mod tidy
|
||||
- go install github.com/swaggo/swag/cmd/swag
|
||||
- swag init -p pascalcase -g "start.go" -o "./doc/api/"
|
||||
- ${GOPATH}/bin/swag init -p pascalcase -g "start.go" -o "./doc/api/"
|
||||
- go build
|
||||
artifacts:
|
||||
paths:
|
||||
|
|
@ -43,26 +40,83 @@ build:
|
|||
|
||||
test:
|
||||
stage: test
|
||||
image: ${GO_IMAGE}
|
||||
extends: .go
|
||||
variables:
|
||||
POSTGRES_DB: testvillasdb
|
||||
POSTGRES_USER: villas
|
||||
POSTGRES_PASSWORD: villas
|
||||
POSTGRES_HOST: postgres
|
||||
RABBITMQ_DEFAULT_USER: villas
|
||||
RABBITMQ_DEFAULT_PASS: villas
|
||||
MINIO_ROOT_USER: minio-villas
|
||||
MINIO_ROOT_PASSWORD: minio-villas
|
||||
MINIO_REGION_NAME: default
|
||||
AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
|
||||
AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
|
||||
S3_BUCKET: villas-web
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_PATHSTYLE: 'true'
|
||||
S3_NOSSL: 'false'
|
||||
S3_REGION: ${MINIO_REGION_NAME}
|
||||
AMQP_HOST: rabbitmq:5672
|
||||
AMQP_USER: villas
|
||||
AMQP_PASS: villas
|
||||
PORT: 4000
|
||||
DB_NAME: ${POSTGRES_DB}
|
||||
DB_HOST: ${POSTGRES_HOST}
|
||||
DB_USER: ${POSTGRES_USER}
|
||||
DB_PASS: ${POSTGRES_PASSWORD}
|
||||
BASE_PATH: /api
|
||||
MODE: test
|
||||
MODE: release
|
||||
ADMIN_USER: User_0
|
||||
ADMIN_PASS: xyz789
|
||||
services:
|
||||
- postgres:latest
|
||||
- rabbitmq:latest
|
||||
- name: minio/minio:RELEASE.2021-01-16T02-19-44Z
|
||||
command: ['server', '/minio']
|
||||
alias: minio
|
||||
before_script:
|
||||
- wget -qO /usr/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x /usr/bin/mc
|
||||
- mc alias set gitlab http://minio:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD}
|
||||
- mc mb gitlab/${S3_BUCKET}
|
||||
script:
|
||||
- go mod tidy
|
||||
- go test $(go list ./... )
|
||||
-p 1
|
||||
-v
|
||||
-covermode=count
|
||||
-coverprofile ./testcover.txt
|
||||
- go tool cover -func=testcover.txt
|
||||
dependencies:
|
||||
- build
|
||||
|
||||
test:files_postgres:
|
||||
stage: test
|
||||
extends: .go
|
||||
variables:
|
||||
POSTGRES_DB: testvillasdb
|
||||
POSTGRES_USER: villas
|
||||
POSTGRES_PASSWORD: villas
|
||||
POSTGRES_HOST: postgres
|
||||
PORT: 4000
|
||||
DB_NAME: ${POSTGRES_DB}
|
||||
DB_HOST: ${POSTGRES_HOST}
|
||||
DB_USER: ${POSTGRES_USER}
|
||||
DB_PASS: ${POSTGRES_PASSWORD}
|
||||
BASE_PATH: /api
|
||||
MODE: release
|
||||
ADMIN_USER: User_0
|
||||
ADMIN_PASS: xyz789
|
||||
services:
|
||||
- postgres:latest
|
||||
script:
|
||||
- go mod tidy
|
||||
- cd routes/file
|
||||
- go test -v
|
||||
dependencies:
|
||||
- build
|
||||
|
||||
|
||||
# Stage: deploy
|
||||
##############################################################################
|
||||
|
|
@ -80,7 +134,5 @@ deploy:
|
|||
--dockerfile ${CI_PROJECT_DIR}/Dockerfile
|
||||
--destination ${DOCKER_IMAGE}:${DOCKER_TAG}
|
||||
--snapshotMode=redo
|
||||
--cache=true
|
||||
--cache-ttl=12h
|
||||
dependencies:
|
||||
- test
|
||||
|
|
|
|||
|
|
@ -38,59 +38,84 @@ func InitConfig() error {
|
|||
}
|
||||
|
||||
var (
|
||||
dbHost = flag.String("db-host", "/var/run/postgresql", "Host of the PostgreSQL database (default is /var/run/postgresql for localhost DB on Ubuntu systems)")
|
||||
dbName = flag.String("db-name", "villasdb", "Name of the database to use (default is villasdb)")
|
||||
dbUser = flag.String("db-user", "", "Username of database connection (default is <empty>)")
|
||||
dbPass = flag.String("db-pass", "", "Password of database connection (default is <empty>)")
|
||||
dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production
|
||||
amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)")
|
||||
amqpUser = flag.String("amqp-user", "", "Username for AMQP broker")
|
||||
amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker")
|
||||
configFile = flag.String("config", "", "Path to YAML configuration file")
|
||||
mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)")
|
||||
port = flag.String("port", "4000", "Port of the backend (default is 4000)")
|
||||
baseHost = flag.String("base-host", "localhost:4000", "The host at which the backend is hosted (default: localhost)")
|
||||
basePath = flag.String("base-path", "/api/v2", "The path at which the API routes are located (default /api/v2)")
|
||||
adminUser = flag.String("admin-user", "", "Initial admin username")
|
||||
adminPass = flag.String("admin-pass", "", "Initial admin password")
|
||||
adminMail = flag.String("admin-mail", "", "Initial admin mail address")
|
||||
dbHost = flag.String("db-host", "/var/run/postgresql", "Host of the PostgreSQL database (default is /var/run/postgresql for localhost DB on Ubuntu systems)")
|
||||
dbName = flag.String("db-name", "villasdb", "Name of the database to use (default is villasdb)")
|
||||
dbUser = flag.String("db-user", "", "Username of database connection (default is <empty>)")
|
||||
dbPass = flag.String("db-pass", "", "Password of database connection (default is <empty>)")
|
||||
dbSSLMode = flag.String("db-ssl-mode", "disable", "SSL mode of DB (default is disable)") // TODO: change default for production
|
||||
amqpHost = flag.String("amqp-host", "", "If set, use this as host for AMQP broker (default is disabled)")
|
||||
amqpUser = flag.String("amqp-user", "", "Username for AMQP broker")
|
||||
amqpPass = flag.String("amqp-pass", "", "Password for AMQP broker")
|
||||
configFile = flag.String("config", "", "Path to YAML configuration file")
|
||||
mode = flag.String("mode", "release", "Select debug/release/test mode (default is release)")
|
||||
port = flag.String("port", "4000", "Port of the backend (default is 4000)")
|
||||
baseHost = flag.String("base-host", "localhost:4000", "The host at which the backend is hosted (default: localhost)")
|
||||
basePath = flag.String("base-path", "/api/v2", "The path at which the API routes are located (default /api/v2)")
|
||||
adminUser = flag.String("admin-user", "", "Initial admin username")
|
||||
adminPass = flag.String("admin-pass", "", "Initial admin password")
|
||||
adminMail = flag.String("admin-mail", "", "Initial admin mail address")
|
||||
s3Bucket = flag.String("s3-bucket", "", "S3 Bucket for uploading files")
|
||||
s3Endpoint = flag.String("s3-endpoint", "", "Endpoint of S3 API for file uploads")
|
||||
s3Region = flag.String("s3-region", "default", "S3 Region for file uploads")
|
||||
s3NoSSL = flag.Bool("s3-nossl", false, "Use encrypted connections to the S3 API")
|
||||
s3PathStyle = flag.Bool("s3-pathstyle", false, "Use path-style S3 API")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
static := map[string]string{
|
||||
"db.host": *dbHost,
|
||||
"db.name": *dbName,
|
||||
"db.user": *dbUser,
|
||||
"db.pass": *dbPass,
|
||||
"db.ssl": *dbSSLMode,
|
||||
"amqp.host": *amqpHost,
|
||||
"amqp.user": *amqpUser,
|
||||
"amqp.pass": *amqpPass,
|
||||
"mode": *mode,
|
||||
"port": *port,
|
||||
"base.host": *baseHost,
|
||||
"base.path": *basePath,
|
||||
"admin.user": *adminUser,
|
||||
"admin.pass": *adminPass,
|
||||
"admin.mail": *adminMail,
|
||||
"db.host": *dbHost,
|
||||
"db.name": *dbName,
|
||||
"db.user": *dbUser,
|
||||
"db.pass": *dbPass,
|
||||
"db.ssl": *dbSSLMode,
|
||||
"amqp.host": *amqpHost,
|
||||
"amqp.user": *amqpUser,
|
||||
"amqp.pass": *amqpPass,
|
||||
"mode": *mode,
|
||||
"port": *port,
|
||||
"base.host": *baseHost,
|
||||
"base.path": *basePath,
|
||||
"admin.user": *adminUser,
|
||||
"admin.pass": *adminPass,
|
||||
"admin.mail": *adminMail,
|
||||
"s3.bucket": *s3Bucket,
|
||||
"s3.endpoint": *s3Endpoint,
|
||||
"s3.region": *s3Region,
|
||||
}
|
||||
|
||||
if *s3NoSSL == true {
|
||||
static["s3.nossl"] = "true"
|
||||
} else {
|
||||
static["s3.nossl"] = "false"
|
||||
}
|
||||
|
||||
if *s3PathStyle == true {
|
||||
static["s3.pathstyle"] = "true"
|
||||
} else {
|
||||
static["s3.pathstyle"] = "false"
|
||||
}
|
||||
|
||||
mappings := map[string]string{
|
||||
"DB_HOST": "db.host",
|
||||
"DB_NAME": "db.name",
|
||||
"DB_USER": "db.user",
|
||||
"DB_PASS": "db.pass",
|
||||
"DB_SSLMODE": "db.ssl",
|
||||
"AMQP_HOST": "amqp.host",
|
||||
"AMQP_USER": "amqp.user",
|
||||
"AMQP_PASS": "amqp.pass",
|
||||
"BASE_HOST": "base.host",
|
||||
"BASE_PATH": "base.path",
|
||||
"MODE": "mode",
|
||||
"PORT": "port",
|
||||
"ADMIN_USER": "admin.user",
|
||||
"ADMIN_PASS": "admin.pass",
|
||||
"ADMIN_MAIL": "admin.mail",
|
||||
"DB_HOST": "db.host",
|
||||
"DB_NAME": "db.name",
|
||||
"DB_USER": "db.user",
|
||||
"DB_PASS": "db.pass",
|
||||
"DB_SSLMODE": "db.ssl",
|
||||
"AMQP_HOST": "amqp.host",
|
||||
"AMQP_USER": "amqp.user",
|
||||
"AMQP_PASS": "amqp.pass",
|
||||
"BASE_HOST": "base.host",
|
||||
"BASE_PATH": "base.path",
|
||||
"MODE": "mode",
|
||||
"PORT": "port",
|
||||
"ADMIN_USER": "admin.user",
|
||||
"ADMIN_PASS": "admin.pass",
|
||||
"ADMIN_MAIL": "admin.mail",
|
||||
"S3_BUCKET": "s3.bucket",
|
||||
"S3_ENDPOINT": "s3.endpoint",
|
||||
"S3_REGION": "s3.region",
|
||||
"S3_NOSSL": "s3.nossl",
|
||||
"S3_PATHSTYLE": "s3.pathstyle",
|
||||
}
|
||||
|
||||
defaults := config.NewStatic(static)
|
||||
|
|
|
|||
|
|
@ -22,9 +22,10 @@
|
|||
package database
|
||||
|
||||
import (
|
||||
"github.com/lib/pq"
|
||||
"time"
|
||||
|
||||
"github.com/lib/pq"
|
||||
|
||||
"github.com/jinzhu/gorm/dialects/postgres"
|
||||
)
|
||||
|
||||
|
|
@ -199,6 +200,8 @@ type File struct {
|
|||
Model
|
||||
// Name of file
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
// Key of file in S3 bucket
|
||||
Key string `json:"key"`
|
||||
// Type of file (MIME type)
|
||||
Type string `json:"type"`
|
||||
// Size of file (in byte)
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -2,6 +2,7 @@ module git.rwth-aachen.de/acs/public/villas/web-backend-go
|
|||
|
||||
require (
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
|
||||
github.com/aws/aws-sdk-go v1.35.35
|
||||
github.com/chenjiandongx/ginprom v0.0.0-20191022035802-6f3da3c84986
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible
|
||||
github.com/gin-gonic/gin v1.4.0
|
||||
|
|
@ -19,7 +20,7 @@ require (
|
|||
github.com/swaggo/gin-swagger v1.2.0
|
||||
github.com/swaggo/swag v1.6.3
|
||||
github.com/zpatrick/go-config v0.0.0-20191104215613-50bc2709703f
|
||||
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
|
||||
gopkg.in/go-playground/validator.v9 v9.30.0
|
||||
gopkg.in/ini.v1 v1.51.0 // indirect
|
||||
)
|
||||
|
|
|
|||
19
go.sum
19
go.sum
|
|
@ -18,6 +18,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
|
|||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
|
||||
github.com/aws/aws-sdk-go v1.35.35 h1:o/EbgEcIPWga7GWhJhb3tiaxqk4/goTdo5YEMdnVxgE=
|
||||
github.com/aws/aws-sdk-go v1.35.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
|
|
@ -105,6 +107,10 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
|
|||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M=
|
||||
github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
|
||||
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
|
||||
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
|
||||
|
|
@ -151,6 +157,7 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ
|
|||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
|
|
@ -207,8 +214,8 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
|
|||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708 h1:pXVtWnwHkrWD9ru3sDxY/qFK/bfc0egRovX91EjWjf4=
|
||||
golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
|
|
@ -228,6 +235,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn
|
|||
golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
|
@ -248,10 +257,14 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
|
||||
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
|
|
@ -288,6 +301,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
|
|||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
|
|
@ -25,11 +25,12 @@ import (
|
|||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/nsf/jsondiff"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/nsf/jsondiff"
|
||||
)
|
||||
|
||||
// data type used in testing
|
||||
|
|
@ -169,8 +170,47 @@ func TestEndpoint(router *gin.Engine, token string, url string,
|
|||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+token)
|
||||
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
return handleRedirect(w, req)
|
||||
}
|
||||
|
||||
func handleRedirect(w *httptest.ResponseRecorder, req *http.Request) (int, *bytes.Buffer, error) {
|
||||
if w.Code == http.StatusMovedPermanently ||
|
||||
w.Code == http.StatusFound ||
|
||||
w.Code == http.StatusTemporaryRedirect ||
|
||||
w.Code == http.StatusPermanentRedirect {
|
||||
|
||||
// Follow external redirect
|
||||
redirURL, err := w.Result().Location()
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("invalid location header")
|
||||
}
|
||||
|
||||
log.Println("redirecting request to", redirURL.String())
|
||||
|
||||
req, err := http.NewRequest(req.Method, redirURL.String(), req.Body)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("handle redirect: failed to create new request: %v", err)
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("handle redirect: failed to follow redirect: %v", err)
|
||||
}
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
_, err = buf.ReadFrom(resp.Body)
|
||||
if err != nil {
|
||||
return 0, nil, fmt.Errorf("handle redirect: failed to follow redirect: %v", err)
|
||||
}
|
||||
|
||||
return resp.StatusCode, buf, nil
|
||||
}
|
||||
|
||||
// No redirect
|
||||
return w.Code, w.Body, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
108
k8s_test.sh
108
k8s_test.sh
|
|
@ -1,21 +1,99 @@
|
|||
#!/bin/bash
|
||||
|
||||
kubectl -n services port-forward svc/postgres-postgresql 5432:5432 & PF1=$!
|
||||
kubectl -n services port-forward svc/broker 5672:5672 & PF2=$!
|
||||
set -e
|
||||
|
||||
NS=villas-demo
|
||||
NS=${NS:-villas}
|
||||
MK="minikube"
|
||||
KCTL="kubectl -n${NS}"
|
||||
HELM="helm -n${NS}"
|
||||
# CHART="villas"
|
||||
CHART="~/workspace/rwth/acs/public/catalogue/charts/villas"
|
||||
|
||||
POSTGRES_USER=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.username | base64 -d)
|
||||
POSTGRES_PASS=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.password | base64 -d)
|
||||
RABBITMQ_USER=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.username | base64 -d)
|
||||
RABBITMQ_PASS=$(kubectl -n ${NS} get secret postgres-credentials -o json | jq -r .data.password | base64 -d)
|
||||
CONFIG=$(mktemp)
|
||||
NAME="villas"
|
||||
|
||||
go test ./routes/healthz -p 1 --args \
|
||||
-mode test \
|
||||
-db-host localhost -db-name villas \
|
||||
-db-user ${POSTGRES_USER} -db-pass ${POSTGRES_PASS} \
|
||||
-amqp-host localhost \
|
||||
-amqp-user ${RABBITMQ_USER} -amqp-pass ${RABBITMQ_PASS}
|
||||
function cleanup() {
|
||||
kill $PF1 $PF2 $PF3
|
||||
wait $PF1 $PF2 $PF3
|
||||
|
||||
kill $PF1 $PF2
|
||||
wait $PF1 $PF2
|
||||
rm ${CONFIG}
|
||||
|
||||
echo "Goodbye"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
if [ -n "${USE_MINIKUBE}" ]; then
|
||||
MK_START_OPTS="--addons=ingress"
|
||||
if [ $(uname -s) == Darwin ]; then
|
||||
# https://github.com/kubernetes/minikube/issues/7332
|
||||
MK_START_OPTS+="--vm=true"
|
||||
fi
|
||||
|
||||
${MK} start ${MK_START_OPTS}
|
||||
|
||||
kubectl -n kube-system expose deployment ingress-nginx-controller --type=LoadBalancer || true
|
||||
|
||||
IP=$(minikube ip)
|
||||
PORT=$(kubectl -n kube-system get service ingress-nginx-controller --output='jsonpath={.spec.ports[0].nodePort}')
|
||||
|
||||
# Add pseudo hostname to /etc/hosts
|
||||
echo "Please provide your root password for modifiying /etc/hosts"
|
||||
sudo sed -in "/^${IP}.*/d" /etc/hosts
|
||||
echo "${IP} minikube" | sudo tee -a /etc/hosts
|
||||
|
||||
HELM_OPTS="--set web.backend.enabled=false \
|
||||
--set web.backend.external.ip=${IP} \
|
||||
--set web.backend.external.port=4000 \
|
||||
--set web.admin.password=admin \
|
||||
--set ingress.host=minikube"
|
||||
|
||||
# Check if chart has already been deployed before
|
||||
if helm get values villas > /dev/null; then
|
||||
RABBITMQ_ERLANG_COOKIE=$(kubectl get secret --namespace default villas-broker -o jsonpath="{.data.rabbitmq-erlang-cookie}" | base64 --decode)
|
||||
RABBITMQ_PASSWORD=$(kubectl get secret --namespace default villas-broker -o jsonpath="{.data.rabbitmq-password}" | base64 --decode)
|
||||
|
||||
HELM_OPTS+=" --set broker.auth.password=${RABBITMQ_PASSWORD} --set broker.auth.erlangCookie=${RABBITMQ_ERLANG_COOKIE}"
|
||||
fi
|
||||
|
||||
${HELM} upgrade \
|
||||
${HELM_OPTS} \
|
||||
--install \
|
||||
--create-namespace \
|
||||
--wait \
|
||||
--repo https://packages.fein-aachen.org/helm/charts \
|
||||
${NAME} ${CHART}
|
||||
fi
|
||||
|
||||
# Get backend config from cluster
|
||||
${KCTL} get cm ${NAME}-web -o 'jsonpath={.data.config\.yaml}' > ${CONFIG}
|
||||
|
||||
# Enable access of backend to db, broker and s3
|
||||
export DB_HOST=localhost
|
||||
export DB_PASS=$(${KCTL} get secret ${NAME}-database -o 'jsonpath={.data.postgresql-password}' | base64 -d)
|
||||
|
||||
export AMQP_HOST=localhost
|
||||
export AMQP_PASS=$(${KCTL} get secret ${NAME}-broker -o 'jsonpath={.data.rabbitmq-password}' | base64 -d)
|
||||
|
||||
export AWS_ACCESS_KEY_ID=$(${KCTL} get secret ${NAME}-s3 -o 'jsonpath={.data.accesskey}' | base64 -d)
|
||||
export AWS_SECRET_ACCESS_KEY=$(${KCTL} get secret ${NAME}-s3 -o 'jsonpath={.data.secretkey}' | base64 -d)
|
||||
|
||||
# Setup port forwards for backend
|
||||
if [ -n "${USE_EXTERNAL_POSTGRESQL}" ]; then
|
||||
kubectl -n postgresql port-forward svc/postgresql 5432:5432 & PF1=$!
|
||||
else
|
||||
${KCTL} port-forward svc/${NAME}-database 5432:5432 & PF1=$!
|
||||
fi
|
||||
${KCTL} port-forward svc/${NAME}-broker 5672:5672 & PF2=$!
|
||||
|
||||
sleep 2
|
||||
|
||||
if [ -n "${USE_MINIKUBE}" ]; then
|
||||
# python -mwebbrowser http://minikube:${PORT}
|
||||
echo
|
||||
echo "==========================================="
|
||||
echo "Access VILLASweb at http://minikube:${PORT}"
|
||||
echo "==========================================="
|
||||
echo
|
||||
fi
|
||||
|
||||
go run start.go -config ${CONFIG}
|
||||
|
|
|
|||
|
|
@ -97,14 +97,14 @@ func addFile(c *gin.Context) {
|
|||
}
|
||||
|
||||
// Extract file from POST request form
|
||||
file_header, err := c.FormFile("file")
|
||||
fileHeader, err := c.FormFile("file")
|
||||
if err != nil {
|
||||
helper.BadRequestError(c, fmt.Sprintf("Get form error: %s", err.Error()))
|
||||
return
|
||||
}
|
||||
|
||||
var newFile File
|
||||
err = newFile.Register(file_header, so.ID)
|
||||
err = newFile.Register(fileHeader, so.ID)
|
||||
if !helper.DBError(c, err) {
|
||||
c.JSON(http.StatusOK, gin.H{"file": newFile.File})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,8 +22,7 @@
|
|||
package file
|
||||
|
||||
import (
|
||||
"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/scenario"
|
||||
"github.com/gin-gonic/gin"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/gif"
|
||||
_ "image/jpeg"
|
||||
|
|
@ -33,10 +32,13 @@ import (
|
|||
"mime/multipart"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration"
|
||||
"git.rwth-aachen.de/acs/public/villas/web-backend-go/routes/scenario"
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"git.rwth-aachen.de/acs/public/villas/web-backend-go/database"
|
||||
)
|
||||
|
||||
|
|
@ -62,14 +64,20 @@ func (f *File) save() error {
|
|||
|
||||
func (f *File) download(c *gin.Context) error {
|
||||
|
||||
// create unique file name
|
||||
filename := "file_" + strconv.FormatUint(uint64(f.ID), 10) + "_" + f.Name
|
||||
// detect the content type of the file
|
||||
contentType := http.DetectContentType(f.FileData)
|
||||
//Seems this headers needed for some browsers (for example without this headers Chrome will download files as txt)
|
||||
c.Header("Content-Description", "File Transfer")
|
||||
c.Header("Content-Disposition", "attachment; filename="+filename)
|
||||
c.Data(http.StatusOK, contentType, f.FileData)
|
||||
if f.Key == "" {
|
||||
// Seems this headers needed for some browsers (for example without this headers Chrome will download files as txt)
|
||||
c.Header("Content-Description", "File Transfer")
|
||||
c.Header("Content-Disposition", "attachment; filename="+f.Name)
|
||||
c.Header("Expires", "")
|
||||
c.Header("Cache-Control", "")
|
||||
c.Data(http.StatusOK, f.Type, f.FileData)
|
||||
} else {
|
||||
url, err := f.getS3Url()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to presign S3 request: %s", err)
|
||||
}
|
||||
c.Redirect(http.StatusFound, url)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -88,10 +96,20 @@ func (f *File) Register(fileHeader *multipart.FileHeader, scenarioID uint) error
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.FileData, err = ioutil.ReadAll(fileContent)
|
||||
defer fileContent.Close()
|
||||
|
||||
bucket, err := configuration.GolbalConfig.String("s3.bucket")
|
||||
if bucket == "" {
|
||||
f.FileData, err = ioutil.ReadAll(fileContent)
|
||||
f.Key = ""
|
||||
} else {
|
||||
err := f.putS3(fileContent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload to S3 bucket: %s", err)
|
||||
}
|
||||
log.Println("Saved new file in S3 object storage")
|
||||
}
|
||||
|
||||
// Add image dimensions in case the file is an image
|
||||
if strings.Contains(f.Type, "image") || strings.Contains(f.Type, "Image") {
|
||||
// set the file reader back to the start of the file
|
||||
|
|
@ -100,13 +118,13 @@ func (f *File) Register(fileHeader *multipart.FileHeader, scenarioID uint) error
|
|||
|
||||
imageConfig, _, err := image.DecodeConfig(fileContent)
|
||||
if err != nil {
|
||||
log.Println("Unable to decode image configuration: Dimensions of image file are not set: ", err)
|
||||
} else {
|
||||
f.ImageHeight = imageConfig.Height
|
||||
f.ImageWidth = imageConfig.Width
|
||||
return fmt.Errorf("unable to decode image configuration: Dimensions of image file are not set: %v", err)
|
||||
}
|
||||
|
||||
f.ImageHeight = imageConfig.Height
|
||||
f.ImageWidth = imageConfig.Width
|
||||
} else {
|
||||
log.Println("Error on setting file reader back to start of file, dimensions not updated:", err)
|
||||
return fmt.Errorf("error on setting file reader back to start of file, dimensions not updated: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -137,43 +155,57 @@ func (f *File) update(fileHeader *multipart.FileHeader) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileData, err := ioutil.ReadAll(fileContent)
|
||||
defer fileContent.Close()
|
||||
|
||||
fileType := fileHeader.Header.Get("Content-Type")
|
||||
imageHeight := f.ImageHeight
|
||||
imageWidth := f.ImageWidth
|
||||
bucket, err := configuration.GolbalConfig.String("s3.bucket")
|
||||
if bucket == "" {
|
||||
f.FileData, err = ioutil.ReadAll(fileContent)
|
||||
f.Key = ""
|
||||
} else {
|
||||
err := f.putS3(fileContent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload to S3 bucket: %s", err)
|
||||
}
|
||||
|
||||
log.Println("Updated file in S3 object storage")
|
||||
}
|
||||
|
||||
f.Type = fileHeader.Header.Get("Content-Type")
|
||||
f.Size = uint(fileHeader.Size)
|
||||
f.Date = time.Now().String()
|
||||
f.Name = filepath.Base(fileHeader.Filename)
|
||||
|
||||
// Update image dimensions in case the file is an image
|
||||
if strings.Contains(fileType, "image") || strings.Contains(fileType, "Image") {
|
||||
if strings.Contains(f.Type, "image") || strings.Contains(f.Type, "Image") {
|
||||
// set the file reader back to the start of the file
|
||||
_, err := fileContent.Seek(0, 0)
|
||||
if err == nil {
|
||||
imageConfig, _, err := image.DecodeConfig(fileContent)
|
||||
if err != nil {
|
||||
log.Println("Unable to decode image configuration: Dimensions of image file are not updated.", err)
|
||||
} else {
|
||||
imageHeight = imageConfig.Height
|
||||
imageWidth = imageConfig.Width
|
||||
}
|
||||
|
||||
f.ImageHeight = imageConfig.Height
|
||||
f.ImageWidth = imageConfig.Width
|
||||
} else {
|
||||
log.Println("Error on setting file reader back to start of file, dimensions not updated::", err)
|
||||
}
|
||||
} else {
|
||||
imageWidth = 0
|
||||
imageHeight = 0
|
||||
f.ImageWidth = 0
|
||||
f.ImageHeight = 0
|
||||
}
|
||||
|
||||
// Add File object with parameters to DB
|
||||
db := database.GetDB()
|
||||
err = db.Model(f).Updates(map[string]interface{}{
|
||||
"Size": uint(fileHeader.Size),
|
||||
"FileData": fileData,
|
||||
"Date": time.Now().String(),
|
||||
"Name": filepath.Base(fileHeader.Filename),
|
||||
"Type": fileType,
|
||||
"ImageHeight": imageHeight,
|
||||
"ImageWidth": imageWidth,
|
||||
"Size": f.Size,
|
||||
"FileData": f.FileData,
|
||||
"Date": f.Date,
|
||||
"Name": f.Name,
|
||||
"Type": f.Type,
|
||||
"ImageHeight": f.ImageHeight,
|
||||
"ImageWidth": f.ImageWidth,
|
||||
"Key": f.Key,
|
||||
}).Error
|
||||
|
||||
return err
|
||||
|
|
@ -189,6 +221,16 @@ func (f *File) Delete() error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// delete file from s3 bucket
|
||||
if f.Key != "" {
|
||||
err = f.deleteS3()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("Deleted file in S3 object storage")
|
||||
}
|
||||
|
||||
err = db.Model(&so).Association("Files").Delete(f).Error
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
|||
159
routes/file/file_s3.go
Normal file
159
routes/file/file_s3.go
Normal file
|
|
@ -0,0 +1,159 @@
|
|||
/** File package, S3 uploads.
|
||||
*
|
||||
* @author Steffen Vogel <svogel2@eonerc.rwth-aachen.de>
|
||||
* @copyright 2014-2019, Institute for Automation of Complex Power Systems, EONERC
|
||||
* @license GNU General Public License (version 3)
|
||||
*
|
||||
* VILLASweb-backend-go
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*********************************************************************************/
|
||||
package file
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"git.rwth-aachen.de/acs/public/villas/web-backend-go/configuration"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Global session
|
||||
var s3Session *session.Session = nil
|
||||
|
||||
func getS3Session() (*session.Session, string, error) {
|
||||
|
||||
bucket, err := configuration.GolbalConfig.String("s3.bucket")
|
||||
if err != nil || bucket == "" {
|
||||
return nil, "", fmt.Errorf("no S3 bucket configured: %s", err)
|
||||
}
|
||||
|
||||
if s3Session == nil {
|
||||
var err error
|
||||
s3Session, err = createS3Session()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
}
|
||||
|
||||
return s3Session, bucket, nil
|
||||
}
|
||||
|
||||
func createS3Session() (*session.Session, error) {
|
||||
endpoint, err := configuration.GolbalConfig.String("s3.endpoint")
|
||||
region, err := configuration.GolbalConfig.String("s3.region")
|
||||
pathStyle, err := configuration.GolbalConfig.Bool("s3.pathstyle")
|
||||
nossl, err := configuration.GolbalConfig.Bool("s3.nossl")
|
||||
|
||||
sess, err := session.NewSession(
|
||||
&aws.Config{
|
||||
Region: aws.String(region),
|
||||
Endpoint: aws.String(endpoint),
|
||||
DisableSSL: aws.Bool(nossl),
|
||||
S3ForcePathStyle: aws.Bool(pathStyle),
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create session: %s", err)
|
||||
}
|
||||
|
||||
return sess, nil
|
||||
}
|
||||
|
||||
func (f *File) putS3(fileContent io.Reader) error {
|
||||
|
||||
// The session the S3 Uploader will use
|
||||
sess, bucket, err := getS3Session()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create an uploader with the session and default options
|
||||
uploader := s3manager.NewUploader(sess)
|
||||
|
||||
f.Key = uuid.New().String()
|
||||
f.FileData = nil
|
||||
|
||||
// Upload the file to S3.
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(f.Key),
|
||||
Body: fileContent,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload file, %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *File) getS3Url() (string, error) {
|
||||
|
||||
// The session the S3 Uploader will use
|
||||
sess, bucket, err := getS3Session()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Create S3 service client
|
||||
svc := s3.New(sess)
|
||||
|
||||
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(f.Key),
|
||||
ResponseContentType: aws.String(f.Type),
|
||||
// ResponseContentDisposition: aws.String("attachment; filename=" + f.Name),
|
||||
// ResponseContentEncoding: aws.String(),
|
||||
// ResponseContentLanguage: aws.String(),
|
||||
// ResponseCacheControl: aws.String(),
|
||||
// ResponseExpires: aws.String(),
|
||||
})
|
||||
|
||||
urlStr, err := req.Presign(5 * 24 * 60 * time.Minute)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return urlStr, nil
|
||||
}
|
||||
|
||||
func (f *File) deleteS3() error {
|
||||
|
||||
// The session the S3 Uploader will use
|
||||
sess, bucket, err := getS3Session()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create S3 service client
|
||||
svc := s3.New(sess)
|
||||
|
||||
_, err = svc.DeleteObject(&s3.DeleteObjectInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Key: aws.String(f.Key),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.Key = ""
|
||||
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue