Add backup and restore test (#1)

* Add backup and restore test
* Update arango binaries

.github/workflows/ci.yml

@@ -26,9 +26,9 @@ jobs:
         working-directory: dev
       - name: Install ArangoDB
         run: |
-          curl -OL https://download.arangodb.com/arangodb34/DEBIAN/Release.key
+          curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key
           sudo apt-key add Release.key
-          sudo apt-add-repository 'deb https://download.arangodb.com/arangodb34/DEBIAN/ /'
+          sudo apt-add-repository 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /'
           sudo apt-get update -y && sudo apt-get -y install arangodb3
       - run: go test -coverprofile=cover.out -coverpkg=./... ./...
       - run: go tool cover -func=cover.out
@@ -46,6 +46,7 @@ jobs:
         with: { name: ui, path: ui/dist, retention-days: 1 }
 
   build:
+    if: github.event_name != 'pull_request'
     name: Build
     runs-on: ubuntu-latest
     needs: [ build-npm, test ]

backup.go

@@ -59,6 +59,7 @@ func backupS3(catalystStorage *storage.Storage, archive *zip.Writer) error {
 	if err != nil {
 		return err
 	}
+
 	for _, bucket := range buckets.Buckets {
 		objects, err := catalystStorage.S3().ListObjectsV2(&s3.ListObjectsV2Input{
 			Bucket: bucket.Name,

database/busdb/busdb.go

@@ -180,3 +180,7 @@ func (c Collection) ReplaceDocument(ctx context.Context, key string, document in
 func (c Collection) RemoveDocument(ctx context.Context, formatInt string) (driver.DocumentMeta, error) {
 	return c.internal.RemoveDocument(ctx, formatInt)
 }
+
+func (c Collection) Truncate(ctx context.Context) error {
+	return c.internal.Truncate(ctx)
+}

database/db.go

@@ -46,7 +46,7 @@ type Database struct {
 	jobCollection        *busdb.Collection
 
 	relatedCollection *busdb.Collection
-	containsCollection *busdb.Collection
+	// containsCollection *busdb.Collection
 }
 
 type Config struct {
@@ -66,6 +66,7 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo
 	if err != nil {
 		return nil, err
 	}
+
 	client, err := driver.NewClient(driver.ClientConfig{
 		Connection:     conn,
 		Authentication: driver.BasicAuthentication(config.User, config.Password),
@@ -76,58 +77,58 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo
 
 	hooks.DatabaseAfterConnect(ctx, client, name)
 
-	db, err := setupDB(ctx, client, name)
+	arangoDB, err := SetupDB(ctx, client, name)
 	if err != nil {
 		return nil, fmt.Errorf("DB setup failed: %w", err)
 	}
 
-	if err = migrations.PerformMigrations(ctx, db); err != nil {
+	if err = migrations.PerformMigrations(ctx, arangoDB); err != nil {
 		return nil, fmt.Errorf("migrations failed: %w", err)
 	}
 
-	ticketCollection, err := db.Collection(ctx, TicketCollectionName)
+	ticketCollection, err := arangoDB.Collection(ctx, TicketCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	templateCollection, err := db.Collection(ctx, TemplateCollectionName)
+	templateCollection, err := arangoDB.Collection(ctx, TemplateCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	playbookCollection, err := db.Collection(ctx, PlaybookCollectionName)
+	playbookCollection, err := arangoDB.Collection(ctx, PlaybookCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	relatedCollection, err := db.Collection(ctx, RelatedTicketsCollectionName)
+	relatedCollection, err := arangoDB.Collection(ctx, RelatedTicketsCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	automationCollection, err := db.Collection(ctx, AutomationCollectionName)
+	automationCollection, err := arangoDB.Collection(ctx, AutomationCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	userdataCollection, err := db.Collection(ctx, UserDataCollectionName)
+	userdataCollection, err := arangoDB.Collection(ctx, UserDataCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	userCollection, err := db.Collection(ctx, UserCollectionName)
+	userCollection, err := arangoDB.Collection(ctx, UserCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	tickettypeCollection, err := db.Collection(ctx, TicketTypeCollectionName)
+	tickettypeCollection, err := arangoDB.Collection(ctx, TicketTypeCollectionName)
 	if err != nil {
 		return nil, err
 	}
-	jobCollection, err := db.Collection(ctx, JobCollectionName)
+	jobCollection, err := arangoDB.Collection(ctx, JobCollectionName)
 	if err != nil {
 		return nil, err
 	}
 
-	hookedDB, err := busdb.NewDatabase(ctx, db, bus)
+	hookedDB, err := busdb.NewDatabase(ctx, arangoDB, bus)
 	if err != nil {
 		return nil, err
 	}
 
-	return &Database{
+	db := &Database{
 		BusDatabase:          hookedDB,
 		bus:                  bus,
 		Index:                index,
@@ -141,10 +142,12 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo
 		userCollection:       busdb.NewCollection(userCollection, hookedDB),
 		tickettypeCollection: busdb.NewCollection(tickettypeCollection, hookedDB),
 		jobCollection:        busdb.NewCollection(jobCollection, hookedDB),
-	}, nil
+	}
+
+	return db, nil
 }
 
-func setupDB(ctx context.Context, client driver.Client, dbName string) (driver.Database, error) {
+func SetupDB(ctx context.Context, client driver.Client, dbName string) (driver.Database, error) {
 	databaseExists, err := client.DatabaseExists(ctx, dbName)
 	if err != nil {
 		return nil, err
@@ -175,3 +178,16 @@ func setupDB(ctx context.Context, client driver.Client, dbName string) (driver.D
 
 	return db, nil
 }
+
+func (db *Database) Truncate(ctx context.Context) {
+	db.templateCollection.Truncate(ctx)
+	db.ticketCollection.Truncate(ctx)
+	db.playbookCollection.Truncate(ctx)
+	db.automationCollection.Truncate(ctx)
+	db.userdataCollection.Truncate(ctx)
+	db.userCollection.Truncate(ctx)
+	db.tickettypeCollection.Truncate(ctx)
+	db.jobCollection.Truncate(ctx)
+	db.relatedCollection.Truncate(ctx)
+	// db.containsCollection.Truncate(ctx)
+}

dev/docker-compose.yml

@@ -4,7 +4,7 @@ services:
     image: nginx:1.21
     volumes:
       - ./nginx.conf:/etc/nginx/nginx.conf:ro
-    ports: [ "80:80", "8529:8529", "9000:9000", "9001:9001", "9002:9002" ]
+    ports: [ "80:80", "8529:8529", "9000:9000", "9001:9001", "9002:9002", "9003:9003" ]
 
   arangodb:
     image: arangodb/arangodb:3.8.1
@@ -18,14 +18,14 @@ services:
       # A9RysEsPJni8RaHeg_K0FKXQNfBrUyw-
 
   minio:
-    image: minio/minio
+    image: minio/minio:RELEASE.2021-12-10T23-03-39Z
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
     command: server /data -console-address ":9003"
 
   postgres:
-    image: postgres
+    image: postgres:13
     environment:
       POSTGRES_DB: keycloak
       POSTGRES_USER: keycloak

dev/nginx.conf

@@ -53,6 +53,17 @@ http {
     server_name _;
 
     location / {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
+
+      proxy_connect_timeout 300;
+      # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
       resolver        127.0.0.11 valid=30s;
       set             $upstream_minio minio;
       proxy_pass      http://$upstream_minio:9000;
@@ -76,6 +87,28 @@ http {
       proxy_set_header X-Forwarded-Server $host;
     }
   }
+
+  server {
+    listen       9003 default_server;
+    server_name _;
+
+    location / {
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header Host $http_host;
+
+      proxy_connect_timeout 300;
+      # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+      proxy_http_version 1.1;
+      proxy_set_header Connection "";
+      chunked_transfer_encoding off;
+
+      resolver        127.0.0.11 valid=30s;
+      set             $upstream_minio minio;
+      proxy_pass      http://$upstream_minio:9003;
+    }
+  }
 }
 
 stream {

restore.go

@@ -15,6 +15,7 @@ import (
 	"path"
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/gin-gonic/gin"
@@ -102,8 +103,12 @@ func restoreS3(catalystStorage *storage.Storage, p string) error {
 func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir fs.FS) error {
 	_, err := catalystStorage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String(entry.Name())})
 	if err != nil {
+		awsError, ok := err.(awserr.Error)
+		if !ok || (awsError.Code() != s3.ErrCodeBucketAlreadyExists && awsError.Code() != s3.ErrCodeBucketAlreadyOwnedByYou) {
 			return err
 		}
+		return nil
+	}
 
 	uploader := catalystStorage.Uploader()
 
@@ -127,7 +132,7 @@ func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir
 }
 
 func unzip(archive *zip.Reader, dir string) error {
-	return fs.WalkDir(archive, "arango", func(p string, d fs.DirEntry, err error) error {
+	return fs.WalkDir(archive, ".", func(p string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}

test/server_test.go

@@ -1,17 +1,29 @@
 package test
 
 import (
+	"archive/zip"
 	"bytes"
+	"context"
 	"encoding/json"
 	"io"
+	"log"
+	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"regexp"
 	"testing"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/gin-gonic/gin"
+	"github.com/stretchr/testify/assert"
 
+	"github.com/SecurityBrewery/catalyst"
 	"github.com/SecurityBrewery/catalyst/database/busdb"
+	"github.com/SecurityBrewery/catalyst/generated/models"
+	"github.com/SecurityBrewery/catalyst/pointer"
 )
 
 func TestService(t *testing.T) {
@@ -80,6 +92,227 @@ func TestService(t *testing.T) {
 	}
 }
 
+func TestBackupAndRestore(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+
+	type args struct {
+		method string
+		url    string
+		data   interface{}
+	}
+	type want struct {
+		status int
+		// body   interface{}
+	}
+	tests := []struct {
+		name string
+		// args args
+		want want
+	}{
+		{name: "Backup", want: want{status: http.StatusOK}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx, _, server, err := Catalyst(t)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if err := SetupTestData(ctx, server.DB); err != nil {
+				t.Fatal(err)
+			}
+
+			createFile(ctx, server)
+
+			server.Server.Use(func(context *gin.Context) {
+				busdb.SetContext(context, Bob)
+			})
+
+			zipB := assertBackup(t, server)
+
+			assertZipFile(t, readZipFile(t, zipB))
+
+			clearAllDatabases(server)
+			_, err = server.DB.UserCreateSetupAPIKey(ctx, "test")
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			deleteAllBuckets(t, server)
+
+			assertRestore(t, zipB, server)
+
+			assertTicketExists(t, server)
+
+			assertFileExists(t, server)
+		})
+	}
+}
+
+func assertBackup(t *testing.T, server *catalyst.Server) []byte {
+	// setup request
+	req := httptest.NewRequest(http.MethodGet, "/api/backup/create", nil)
+	req.Header.Set("PRIVATE-TOKEN", "test")
+
+	// run request
+	backupRequestRecorder := httptest.NewRecorder()
+	server.Server.ServeHTTP(backupRequestRecorder, req)
+	backupResult := backupRequestRecorder.Result()
+
+	// assert results
+	assert.Equal(t, http.StatusOK, backupResult.StatusCode)
+
+	zipBuf := &bytes.Buffer{}
+	if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
+		t.Fatal(err)
+	}
+	assert.NoError(t, backupResult.Body.Close())
+
+	return zipBuf.Bytes()
+}
+
+func assertZipFile(t *testing.T, r *zip.Reader) {
+	var names []string
+	for _, f := range r.File {
+		names = append(names, f.Name)
+	}
+
+	if !includes(t, names, "minio/catalyst-8125/test.txt") {
+		t.Error("Minio file missing")
+	}
+
+	for _, p := range []string{
+		"arango/ENCRYPTION", "arango/automations_.*.data.json.gz", "arango/automations_.*.structure.json", "arango/dump.json", "arango/jobs_.*.data.json.gz", "arango/jobs_.*.structure.json", "arango/logs_.*.data.json.gz", "arango/logs_.*.structure.json", "arango/migrations_.*.data.json.gz", "arango/migrations_.*.structure.json", "arango/playbooks_.*.data.json.gz", "arango/playbooks_.*.structure.json", "arango/related_.*.data.json.gz", "arango/related_.*.structure.json", "arango/templates_.*.data.json.gz", "arango/templates_.*.structure.json", "arango/tickets_.*.data.json.gz", "arango/tickets_.*.structure.json", "arango/tickettypes_.*.data.json.gz", "arango/tickettypes_.*.structure.json", "arango/userdata_.*.data.json.gz", "arango/userdata_.*.structure.json", "arango/users_.*.data.json.gz", "arango/users_.*.structure.json",
+	} {
+		if !includes(t, names, p) {
+			t.Errorf("Arango file missing: %s", p)
+		}
+	}
+}
+
+func clearAllDatabases(server *catalyst.Server) {
+	server.DB.Truncate(context.Background())
+}
+
+func deleteAllBuckets(t *testing.T, server *catalyst.Server) {
+	buckets, err := server.Storage.S3().ListBuckets(&s3.ListBucketsInput{})
+	for _, bucket := range buckets.Buckets {
+		server.Storage.S3().DeleteBucket(&s3.DeleteBucketInput{
+			Bucket: bucket.Name,
+		})
+	}
+
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func assertRestore(t *testing.T, zipB []byte, server *catalyst.Server) {
+	bodyBuf := &bytes.Buffer{}
+	bodyWriter := multipart.NewWriter(bodyBuf)
+	fileWriter, err := bodyWriter.CreateFormFile("backup", "backup.zip")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	_, err = fileWriter.Write(zipB)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	assert.NoError(t, bodyWriter.Close())
+
+	req := httptest.NewRequest(http.MethodPost, "/api/backup/restore", bodyBuf)
+	req.Header.Set("PRIVATE-TOKEN", "test")
+	req.Header.Set("Content-Type", bodyWriter.FormDataContentType())
+
+	// run request
+	restoreRequestRecorder := httptest.NewRecorder()
+	server.Server.ServeHTTP(restoreRequestRecorder, req)
+	restoreResult := restoreRequestRecorder.Result()
+
+	if !assert.Equal(t, http.StatusOK, restoreResult.StatusCode) {
+		t.FailNow()
+	}
+}
+
+func createFile(ctx context.Context, server *catalyst.Server) {
+	buf := bytes.NewBufferString("test text")
+
+	server.Storage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String("catalyst-8125")})
+
+	if _, err := server.Storage.Uploader().Upload(&s3manager.UploadInput{Body: buf, Bucket: pointer.String("catalyst-8125"), Key: pointer.String("test.txt")}); err != nil {
+		log.Fatal(err)
+	}
+
+	if _, err := server.DB.LinkFiles(ctx, 8125, []*models.File{{Key: "test.txt", Name: "test.txt"}}); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func assertTicketExists(t *testing.T, server *catalyst.Server) {
+	req := httptest.NewRequest(http.MethodGet, "/api/tickets/8125", nil)
+	req.Header.Set("PRIVATE-TOKEN", "test")
+
+	// run request
+	backupRequestRecorder := httptest.NewRecorder()
+	server.Server.ServeHTTP(backupRequestRecorder, req)
+	backupResult := backupRequestRecorder.Result()
+
+	// assert results
+	assert.Equal(t, http.StatusOK, backupResult.StatusCode)
+
+	zipBuf := &bytes.Buffer{}
+	if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
+		t.Fatal(err)
+	}
+	assert.NoError(t, backupResult.Body.Close())
+
+	var ticket models.Ticket
+	assert.NoError(t, json.Unmarshal(zipBuf.Bytes(), &ticket))
+
+	assert.Equal(t, "phishing from selenafadel@von.com detected", ticket.Name)
+}
+
+func assertFileExists(t *testing.T, server *catalyst.Server) {
+	obj, err := server.Storage.S3().GetObject(&s3.GetObjectInput{
+		Bucket: aws.String("catalyst-8125"),
+		Key:    aws.String("test.txt"),
+	})
+	assert.NoError(t, err)
+
+	b, err := io.ReadAll(obj.Body)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "test text", string(b))
+}
+
+func includes(t *testing.T, names []string, s string) bool {
+	for _, name := range names {
+		match, err := regexp.MatchString(s, name)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if match {
+			return true
+		}
+	}
+	return false
+}
+
+func readZipFile(t *testing.T, b []byte) *zip.Reader {
+	buf := bytes.NewReader(b)
+
+	zr, err := zip.NewReader(buf, int64(buf.Len()))
+	if err != nil {
+		t.Fatal(string(b), err)
+	}
+
+	return zr
+}
+
 func jsonEqual(t *testing.T, got io.Reader, want interface{}) {
 	var j, j2 interface{}
 	c, err := io.ReadAll(got)

test/test.go

@@ -36,6 +36,7 @@ func Context() context.Context {
 
 func Config(ctx context.Context) (*catalyst.Config, error) {
 	config := &catalyst.Config{
+		InitialAPIKey: "test",
 		IndexPath:     "index.bleve",
 		DB: &database.Config{
 			Host:     "http://localhost:8529",
@@ -188,6 +189,21 @@ func Server(t *testing.T) (context.Context, *catalyst.Config, *bus.Bus, *index.I
 	return ctx, config, rbus, catalystIndex, catalystStorage, db, catalystService, catalystServer, cleanup, err
 }
 
+func Catalyst(t *testing.T) (context.Context, *catalyst.Config, *catalyst.Server, error) {
+	ctx := Context()
+
+	config, err := Config(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	config.DB.Name = cleanName(t)
+
+	c, err := catalyst.New(&hooks.Hooks{
+		DatabaseAfterConnectFuncs: []func(ctx context.Context, client driver.Client, name string){Clear},
+	}, config)
+	return ctx, config, c, err
+}
+
 func cleanName(t *testing.T) string {
 	name := t.Name()
 	name = strings.ReplaceAll(name, " ", "")