From 0286574692e0f98149eb9692efcbc6f1a9e37f5d Mon Sep 17 00:00:00 2001 From: Jonas Plum Date: Fri, 24 Dec 2021 00:22:08 +0100 Subject: [PATCH] Add backup and restore test (#1) * Add backup and restore test * Update arango binaries --- .github/workflows/ci.yml | 5 +- backup.go | 1 + database/busdb/busdb.go | 4 + database/db.go | 50 ++++++--- dev/docker-compose.yml | 6 +- dev/nginx.conf | 33 ++++++ restore.go | 9 +- test/server_test.go | 233 +++++++++++++++++++++++++++++++++++++++ test/test.go | 22 +++- 9 files changed, 336 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec55287..b5f4db0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,9 @@ jobs: working-directory: dev - name: Install ArangoDB run: | - curl -OL https://download.arangodb.com/arangodb34/DEBIAN/Release.key + curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key sudo apt-key add Release.key - sudo apt-add-repository 'deb https://download.arangodb.com/arangodb34/DEBIAN/ /' + sudo apt-add-repository 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /' sudo apt-get update -y && sudo apt-get -y install arangodb3 - run: go test -coverprofile=cover.out -coverpkg=./... ./... - run: go tool cover -func=cover.out @@ -46,6 +46,7 @@ jobs: with: { name: ui, path: ui/dist, retention-days: 1 } build: + if: github.event_name != 'pull_request' name: Build runs-on: ubuntu-latest needs: [ build-npm, test ] diff --git a/backup.go b/backup.go index 141994c..317059b 100644 --- a/backup.go +++ b/backup.go @@ -59,6 +59,7 @@ func backupS3(catalystStorage *storage.Storage, archive *zip.Writer) error { if err != nil { return err } + for _, bucket := range buckets.Buckets { objects, err := catalystStorage.S3().ListObjectsV2(&s3.ListObjectsV2Input{ Bucket: bucket.Name, diff --git a/database/busdb/busdb.go b/database/busdb/busdb.go index b9680c2..069bcc1 100644 --- a/database/busdb/busdb.go +++ b/database/busdb/busdb.go @@ -180,3 +180,7 @@ func (c Collection) ReplaceDocument(ctx context.Context, key string, document in func (c Collection) RemoveDocument(ctx context.Context, formatInt string) (driver.DocumentMeta, error) { return c.internal.RemoveDocument(ctx, formatInt) } + +func (c Collection) Truncate(ctx context.Context) error { + return c.internal.Truncate(ctx) +} diff --git a/database/db.go b/database/db.go index 079f6c7..6c46b2c 100644 --- a/database/db.go +++ b/database/db.go @@ -45,8 +45,8 @@ type Database struct { tickettypeCollection *busdb.Collection jobCollection *busdb.Collection - relatedCollection *busdb.Collection - containsCollection *busdb.Collection + relatedCollection *busdb.Collection + // containsCollection *busdb.Collection } type Config struct { @@ -66,6 +66,7 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo if err != nil { return nil, err } + client, err := driver.NewClient(driver.ClientConfig{ Connection: conn, Authentication: driver.BasicAuthentication(config.User, config.Password), @@ -76,58 +77,58 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo hooks.DatabaseAfterConnect(ctx, client, name) - db, err := setupDB(ctx, client, name) + arangoDB, err := SetupDB(ctx, client, name) if err != nil { return nil, fmt.Errorf("DB setup failed: %w", err) } - if err = migrations.PerformMigrations(ctx, db); err != nil { + if err = migrations.PerformMigrations(ctx, arangoDB); err != nil { return nil, fmt.Errorf("migrations failed: %w", err) } - ticketCollection, err := db.Collection(ctx, TicketCollectionName) + ticketCollection, err := arangoDB.Collection(ctx, TicketCollectionName) if err != nil { return nil, err } - templateCollection, err := db.Collection(ctx, TemplateCollectionName) + templateCollection, err := arangoDB.Collection(ctx, TemplateCollectionName) if err != nil { return nil, err } - playbookCollection, err := db.Collection(ctx, PlaybookCollectionName) + playbookCollection, err := arangoDB.Collection(ctx, PlaybookCollectionName) if err != nil { return nil, err } - relatedCollection, err := db.Collection(ctx, RelatedTicketsCollectionName) + relatedCollection, err := arangoDB.Collection(ctx, RelatedTicketsCollectionName) if err != nil { return nil, err } - automationCollection, err := db.Collection(ctx, AutomationCollectionName) + automationCollection, err := arangoDB.Collection(ctx, AutomationCollectionName) if err != nil { return nil, err } - userdataCollection, err := db.Collection(ctx, UserDataCollectionName) + userdataCollection, err := arangoDB.Collection(ctx, UserDataCollectionName) if err != nil { return nil, err } - userCollection, err := db.Collection(ctx, UserCollectionName) + userCollection, err := arangoDB.Collection(ctx, UserCollectionName) if err != nil { return nil, err } - tickettypeCollection, err := db.Collection(ctx, TicketTypeCollectionName) + tickettypeCollection, err := arangoDB.Collection(ctx, TicketTypeCollectionName) if err != nil { return nil, err } - jobCollection, err := db.Collection(ctx, JobCollectionName) + jobCollection, err := arangoDB.Collection(ctx, JobCollectionName) if err != nil { return nil, err } - hookedDB, err := busdb.NewDatabase(ctx, db, bus) + hookedDB, err := busdb.NewDatabase(ctx, arangoDB, bus) if err != nil { return nil, err } - return &Database{ + db := &Database{ BusDatabase: hookedDB, bus: bus, Index: index, @@ -141,10 +142,12 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo userCollection: busdb.NewCollection(userCollection, hookedDB), tickettypeCollection: busdb.NewCollection(tickettypeCollection, hookedDB), jobCollection: busdb.NewCollection(jobCollection, hookedDB), - }, nil + } + + return db, nil } -func setupDB(ctx context.Context, client driver.Client, dbName string) (driver.Database, error) { +func SetupDB(ctx context.Context, client driver.Client, dbName string) (driver.Database, error) { databaseExists, err := client.DatabaseExists(ctx, dbName) if err != nil { return nil, err @@ -175,3 +178,16 @@ func setupDB(ctx context.Context, client driver.Client, dbName string) (driver.D return db, nil } + +func (db *Database) Truncate(ctx context.Context) { + db.templateCollection.Truncate(ctx) + db.ticketCollection.Truncate(ctx) + db.playbookCollection.Truncate(ctx) + db.automationCollection.Truncate(ctx) + db.userdataCollection.Truncate(ctx) + db.userCollection.Truncate(ctx) + db.tickettypeCollection.Truncate(ctx) + db.jobCollection.Truncate(ctx) + db.relatedCollection.Truncate(ctx) + // db.containsCollection.Truncate(ctx) +} diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 4308c8a..48a5bd3 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -4,7 +4,7 @@ services: image: nginx:1.21 volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro - ports: [ "80:80", "8529:8529", "9000:9000", "9001:9001", "9002:9002" ] + ports: [ "80:80", "8529:8529", "9000:9000", "9001:9001", "9002:9002", "9003:9003" ] arangodb: image: arangodb/arangodb:3.8.1 @@ -18,14 +18,14 @@ services: # A9RysEsPJni8RaHeg_K0FKXQNfBrUyw- minio: - image: minio/minio + image: minio/minio:RELEASE.2021-12-10T23-03-39Z environment: MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: minio123 command: server /data -console-address ":9003" postgres: - image: postgres + image: postgres:13 environment: POSTGRES_DB: keycloak POSTGRES_USER: keycloak diff --git a/dev/nginx.conf b/dev/nginx.conf index fcb5a48..eb648cf 100644 --- a/dev/nginx.conf +++ b/dev/nginx.conf @@ -53,6 +53,17 @@ http { server_name _; location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + resolver 127.0.0.11 valid=30s; set $upstream_minio minio; proxy_pass http://$upstream_minio:9000; @@ -76,6 +87,28 @@ http { proxy_set_header X-Forwarded-Server $host; } } + + server { + listen 9003 default_server; + server_name _; + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Host $http_host; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + resolver 127.0.0.11 valid=30s; + set $upstream_minio minio; + proxy_pass http://$upstream_minio:9003; + } + } } stream { diff --git a/restore.go b/restore.go index 276cc3c..c906d61 100644 --- a/restore.go +++ b/restore.go @@ -15,6 +15,7 @@ import ( "path" "strings" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/gin-gonic/gin" @@ -102,7 +103,11 @@ func restoreS3(catalystStorage *storage.Storage, p string) error { func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir fs.FS) error { _, err := catalystStorage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String(entry.Name())}) if err != nil { - return err + awsError, ok := err.(awserr.Error) + if !ok || (awsError.Code() != s3.ErrCodeBucketAlreadyExists && awsError.Code() != s3.ErrCodeBucketAlreadyOwnedByYou) { + return err + } + return nil } uploader := catalystStorage.Uploader() @@ -127,7 +132,7 @@ func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir } func unzip(archive *zip.Reader, dir string) error { - return fs.WalkDir(archive, "arango", func(p string, d fs.DirEntry, err error) error { + return fs.WalkDir(archive, ".", func(p string, d fs.DirEntry, err error) error { if err != nil { return err } diff --git a/test/server_test.go b/test/server_test.go index d12b1dc..93c6bb3 100644 --- a/test/server_test.go +++ b/test/server_test.go @@ -1,17 +1,29 @@ package test import ( + "archive/zip" "bytes" + "context" "encoding/json" "io" + "log" + "mime/multipart" "net/http" "net/http/httptest" "reflect" + "regexp" "testing" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/SecurityBrewery/catalyst" "github.com/SecurityBrewery/catalyst/database/busdb" + "github.com/SecurityBrewery/catalyst/generated/models" + "github.com/SecurityBrewery/catalyst/pointer" ) func TestService(t *testing.T) { @@ -80,6 +92,227 @@ func TestService(t *testing.T) { } } +func TestBackupAndRestore(t *testing.T) { + gin.SetMode(gin.TestMode) + log.SetFlags(log.LstdFlags | log.Lshortfile) + + type args struct { + method string + url string + data interface{} + } + type want struct { + status int + // body interface{} + } + tests := []struct { + name string + // args args + want want + }{ + {name: "Backup", want: want{status: http.StatusOK}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, _, server, err := Catalyst(t) + if err != nil { + t.Fatal(err) + } + + if err := SetupTestData(ctx, server.DB); err != nil { + t.Fatal(err) + } + + createFile(ctx, server) + + server.Server.Use(func(context *gin.Context) { + busdb.SetContext(context, Bob) + }) + + zipB := assertBackup(t, server) + + assertZipFile(t, readZipFile(t, zipB)) + + clearAllDatabases(server) + _, err = server.DB.UserCreateSetupAPIKey(ctx, "test") + if err != nil { + log.Fatal(err) + } + + deleteAllBuckets(t, server) + + assertRestore(t, zipB, server) + + assertTicketExists(t, server) + + assertFileExists(t, server) + }) + } +} + +func assertBackup(t *testing.T, server *catalyst.Server) []byte { + // setup request + req := httptest.NewRequest(http.MethodGet, "/api/backup/create", nil) + req.Header.Set("PRIVATE-TOKEN", "test") + + // run request + backupRequestRecorder := httptest.NewRecorder() + server.Server.ServeHTTP(backupRequestRecorder, req) + backupResult := backupRequestRecorder.Result() + + // assert results + assert.Equal(t, http.StatusOK, backupResult.StatusCode) + + zipBuf := &bytes.Buffer{} + if _, err := io.Copy(zipBuf, backupResult.Body); err != nil { + t.Fatal(err) + } + assert.NoError(t, backupResult.Body.Close()) + + return zipBuf.Bytes() +} + +func assertZipFile(t *testing.T, r *zip.Reader) { + var names []string + for _, f := range r.File { + names = append(names, f.Name) + } + + if !includes(t, names, "minio/catalyst-8125/test.txt") { + t.Error("Minio file missing") + } + + for _, p := range []string{ + "arango/ENCRYPTION", "arango/automations_.*.data.json.gz", "arango/automations_.*.structure.json", "arango/dump.json", "arango/jobs_.*.data.json.gz", "arango/jobs_.*.structure.json", "arango/logs_.*.data.json.gz", "arango/logs_.*.structure.json", "arango/migrations_.*.data.json.gz", "arango/migrations_.*.structure.json", "arango/playbooks_.*.data.json.gz", "arango/playbooks_.*.structure.json", "arango/related_.*.data.json.gz", "arango/related_.*.structure.json", "arango/templates_.*.data.json.gz", "arango/templates_.*.structure.json", "arango/tickets_.*.data.json.gz", "arango/tickets_.*.structure.json", "arango/tickettypes_.*.data.json.gz", "arango/tickettypes_.*.structure.json", "arango/userdata_.*.data.json.gz", "arango/userdata_.*.structure.json", "arango/users_.*.data.json.gz", "arango/users_.*.structure.json", + } { + if !includes(t, names, p) { + t.Errorf("Arango file missing: %s", p) + } + } +} + +func clearAllDatabases(server *catalyst.Server) { + server.DB.Truncate(context.Background()) +} + +func deleteAllBuckets(t *testing.T, server *catalyst.Server) { + buckets, err := server.Storage.S3().ListBuckets(&s3.ListBucketsInput{}) + for _, bucket := range buckets.Buckets { + server.Storage.S3().DeleteBucket(&s3.DeleteBucketInput{ + Bucket: bucket.Name, + }) + } + + if err != nil { + t.Fatal(err) + } +} + +func assertRestore(t *testing.T, zipB []byte, server *catalyst.Server) { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + fileWriter, err := bodyWriter.CreateFormFile("backup", "backup.zip") + if err != nil { + log.Fatal(err) + } + + _, err = fileWriter.Write(zipB) + if err != nil { + log.Fatal(err) + } + + assert.NoError(t, bodyWriter.Close()) + + req := httptest.NewRequest(http.MethodPost, "/api/backup/restore", bodyBuf) + req.Header.Set("PRIVATE-TOKEN", "test") + req.Header.Set("Content-Type", bodyWriter.FormDataContentType()) + + // run request + restoreRequestRecorder := httptest.NewRecorder() + server.Server.ServeHTTP(restoreRequestRecorder, req) + restoreResult := restoreRequestRecorder.Result() + + if !assert.Equal(t, http.StatusOK, restoreResult.StatusCode) { + t.FailNow() + } +} + +func createFile(ctx context.Context, server *catalyst.Server) { + buf := bytes.NewBufferString("test text") + + server.Storage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String("catalyst-8125")}) + + if _, err := server.Storage.Uploader().Upload(&s3manager.UploadInput{Body: buf, Bucket: pointer.String("catalyst-8125"), Key: pointer.String("test.txt")}); err != nil { + log.Fatal(err) + } + + if _, err := server.DB.LinkFiles(ctx, 8125, []*models.File{{Key: "test.txt", Name: "test.txt"}}); err != nil { + log.Fatal(err) + } +} + +func assertTicketExists(t *testing.T, server *catalyst.Server) { + req := httptest.NewRequest(http.MethodGet, "/api/tickets/8125", nil) + req.Header.Set("PRIVATE-TOKEN", "test") + + // run request + backupRequestRecorder := httptest.NewRecorder() + server.Server.ServeHTTP(backupRequestRecorder, req) + backupResult := backupRequestRecorder.Result() + + // assert results + assert.Equal(t, http.StatusOK, backupResult.StatusCode) + + zipBuf := &bytes.Buffer{} + if _, err := io.Copy(zipBuf, backupResult.Body); err != nil { + t.Fatal(err) + } + assert.NoError(t, backupResult.Body.Close()) + + var ticket models.Ticket + assert.NoError(t, json.Unmarshal(zipBuf.Bytes(), &ticket)) + + assert.Equal(t, "phishing from selenafadel@von.com detected", ticket.Name) +} + +func assertFileExists(t *testing.T, server *catalyst.Server) { + obj, err := server.Storage.S3().GetObject(&s3.GetObjectInput{ + Bucket: aws.String("catalyst-8125"), + Key: aws.String("test.txt"), + }) + assert.NoError(t, err) + + b, err := io.ReadAll(obj.Body) + assert.NoError(t, err) + + assert.Equal(t, "test text", string(b)) +} + +func includes(t *testing.T, names []string, s string) bool { + for _, name := range names { + match, err := regexp.MatchString(s, name) + if err != nil { + t.Fatal(err) + } + + if match { + return true + } + } + return false +} + +func readZipFile(t *testing.T, b []byte) *zip.Reader { + buf := bytes.NewReader(b) + + zr, err := zip.NewReader(buf, int64(buf.Len())) + if err != nil { + t.Fatal(string(b), err) + } + + return zr +} + func jsonEqual(t *testing.T, got io.Reader, want interface{}) { var j, j2 interface{} c, err := io.ReadAll(got) diff --git a/test/test.go b/test/test.go index e6f1929..229a798 100644 --- a/test/test.go +++ b/test/test.go @@ -36,7 +36,8 @@ func Context() context.Context { func Config(ctx context.Context) (*catalyst.Config, error) { config := &catalyst.Config{ - IndexPath: "index.bleve", + InitialAPIKey: "test", + IndexPath: "index.bleve", DB: &database.Config{ Host: "http://localhost:8529", User: "root", @@ -147,8 +148,8 @@ func DB(t *testing.T) (context.Context, *catalyst.Config, *bus.Bus, *index.Index _, err = db.JobCreate(ctx, "99cd67131b48", &models.JobForm{ Automation: "hash.sha1", - Payload: "test", - Origin: nil, + Payload: "test", + Origin: nil, }) if err != nil { return nil, nil, nil, nil, nil, nil, nil, err @@ -188,6 +189,21 @@ func Server(t *testing.T) (context.Context, *catalyst.Config, *bus.Bus, *index.I return ctx, config, rbus, catalystIndex, catalystStorage, db, catalystService, catalystServer, cleanup, err } +func Catalyst(t *testing.T) (context.Context, *catalyst.Config, *catalyst.Server, error) { + ctx := Context() + + config, err := Config(ctx) + if err != nil { + t.Fatal(err) + } + config.DB.Name = cleanName(t) + + c, err := catalyst.New(&hooks.Hooks{ + DatabaseAfterConnectFuncs: []func(ctx context.Context, client driver.Client, name string){Clear}, + }, config) + return ctx, config, c, err +} + func cleanName(t *testing.T) string { name := t.Name() name = strings.ReplaceAll(name, " ", "")