CSEC_PUBLIC/catalyst
1
0
Fork 0
mirror of https://github.com/SecurityBrewery/catalyst.git synced 2025-12-06 15:22:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add backup and restore test (#1)

Browse Source 
* Add backup and restore test
* Update arango binaries
This commit is contained in:
Jonas Plum
2021-12-24 00:22:08 +01:00
committed by GitHub
parent c27e61b875
commit 0286574692
9 changed files with 336 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.github/workflows/ci.yml vendored
View File

@@ -26,9 +26,9 @@ jobs:
working-directory: dev
- name: Install ArangoDB
run: |
curl -OL https://download.arangodb.com/arangodb34/DEBIAN/Release.key
curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key
sudo apt-key add Release.key
sudo apt-add-repository 'deb https://download.arangodb.com/arangodb34/DEBIAN/ /'
sudo apt-add-repository 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /'
sudo apt-get update -y && sudo apt-get -y install arangodb3
- run: go test -coverprofile=cover.out -coverpkg=./... ./...
- run: go tool cover -func=cover.out
@@ -46,6 +46,7 @@ jobs:
with: { name: ui, path: ui/dist, retention-days: 1 }
build:
if: github.event_name != 'pull_request'
name: Build
runs-on: ubuntu-latest
needs: [ build-npm, test ]

1
backup.go
View File

@@ -59,6 +59,7 @@ func backupS3(catalystStorage *storage.Storage, archive *zip.Writer) error {
if err != nil {
return err
}
for _, bucket := range buckets.Buckets {
objects, err := catalystStorage.S3().ListObjectsV2(&s3.ListObjectsV2Input{
Bucket: bucket.Name,

4
database/busdb/busdb.go
View File

@@ -180,3 +180,7 @@ func (c Collection) ReplaceDocument(ctx context.Context, key string, document in
func (c Collection) RemoveDocument(ctx context.Context, formatInt string) (driver.DocumentMeta, error) {
return c.internal.RemoveDocument(ctx, formatInt)
}
func (c Collection) Truncate(ctx context.Context) error {
return c.internal.Truncate(ctx)
}

48
database/db.go
View File

@@ -46,7 +46,7 @@ type Database struct {
jobCollection *busdb.Collection
relatedCollection *busdb.Collection
containsCollection *busdb.Collection
// containsCollection *busdb.Collection
}
type Config struct {
@@ -66,6 +66,7 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo
if err != nil {
return nil, err
}
client, err := driver.NewClient(driver.ClientConfig{
Connection: conn,
Authentication: driver.BasicAuthentication(config.User, config.Password),
@@ -76,58 +77,58 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo
hooks.DatabaseAfterConnect(ctx, client, name)
db, err := setupDB(ctx, client, name)
arangoDB, err := SetupDB(ctx, client, name)
if err != nil {
return nil, fmt.Errorf("DB setup failed: %w", err)
}
if err = migrations.PerformMigrations(ctx, db); err != nil {
if err = migrations.PerformMigrations(ctx, arangoDB); err != nil {
return nil, fmt.Errorf("migrations failed: %w", err)
}
ticketCollection, err := db.Collection(ctx, TicketCollectionName)
ticketCollection, err := arangoDB.Collection(ctx, TicketCollectionName)
if err != nil {
return nil, err
}
templateCollection, err := db.Collection(ctx, TemplateCollectionName)
templateCollection, err := arangoDB.Collection(ctx, TemplateCollectionName)
if err != nil {
return nil, err
}
playbookCollection, err := db.Collection(ctx, PlaybookCollectionName)
playbookCollection, err := arangoDB.Collection(ctx, PlaybookCollectionName)
if err != nil {
return nil, err
}
relatedCollection, err := db.Collection(ctx, RelatedTicketsCollectionName)
relatedCollection, err := arangoDB.Collection(ctx, RelatedTicketsCollectionName)
if err != nil {
return nil, err
}
automationCollection, err := db.Collection(ctx, AutomationCollectionName)
automationCollection, err := arangoDB.Collection(ctx, AutomationCollectionName)
if err != nil {
return nil, err
}
userdataCollection, err := db.Collection(ctx, UserDataCollectionName)
userdataCollection, err := arangoDB.Collection(ctx, UserDataCollectionName)
if err != nil {
return nil, err
}
userCollection, err := db.Collection(ctx, UserCollectionName)
userCollection, err := arangoDB.Collection(ctx, UserCollectionName)
if err != nil {
return nil, err
}
tickettypeCollection, err := db.Collection(ctx, TicketTypeCollectionName)
tickettypeCollection, err := arangoDB.Collection(ctx, TicketTypeCollectionName)
if err != nil {
return nil, err
}
jobCollection, err := db.Collection(ctx, JobCollectionName)
jobCollection, err := arangoDB.Collection(ctx, JobCollectionName)
if err != nil {
return nil, err
}
hookedDB, err := busdb.NewDatabase(ctx, db, bus)
hookedDB, err := busdb.NewDatabase(ctx, arangoDB, bus)
if err != nil {
return nil, err
}
return &Database{
db := &Database{
BusDatabase: hookedDB,
bus: bus,
Index: index,
@@ -141,10 +142,12 @@ func New(ctx context.Context, index *index.Index, bus *bus.Bus, hooks *hooks.Hoo
userCollection: busdb.NewCollection(userCollection, hookedDB),
tickettypeCollection: busdb.NewCollection(tickettypeCollection, hookedDB),
jobCollection: busdb.NewCollection(jobCollection, hookedDB),
}, nil
}
func setupDB(ctx context.Context, client driver.Client, dbName string) (driver.Database, error) {
return db, nil
}
func SetupDB(ctx context.Context, client driver.Client, dbName string) (driver.Database, error) {
databaseExists, err := client.DatabaseExists(ctx, dbName)
if err != nil {
return nil, err
@@ -175,3 +178,16 @@ func setupDB(ctx context.Context, client driver.Client, dbName string) (driver.D
return db, nil
}
func (db *Database) Truncate(ctx context.Context) {
db.templateCollection.Truncate(ctx)
db.ticketCollection.Truncate(ctx)
db.playbookCollection.Truncate(ctx)
db.automationCollection.Truncate(ctx)
db.userdataCollection.Truncate(ctx)
db.userCollection.Truncate(ctx)
db.tickettypeCollection.Truncate(ctx)
db.jobCollection.Truncate(ctx)
db.relatedCollection.Truncate(ctx)
// db.containsCollection.Truncate(ctx)
}

6
dev/docker-compose.yml
View File

@@ -4,7 +4,7 @@ services:
image: nginx:1.21
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf:ro
ports: [ "80:80", "8529:8529", "9000:9000", "9001:9001", "9002:9002" ]
ports: [ "80:80", "8529:8529", "9000:9000", "9001:9001", "9002:9002", "9003:9003" ]
arangodb:
image: arangodb/arangodb:3.8.1
@@ -18,14 +18,14 @@ services:
# A9RysEsPJni8RaHeg_K0FKXQNfBrUyw-
minio:
image: minio/minio
image: minio/minio:RELEASE.2021-12-10T23-03-39Z
environment:
MINIO_ROOT_USER: minio
MINIO_ROOT_PASSWORD: minio123
command: server /data -console-address ":9003"
postgres:
image: postgres
image: postgres:13
environment:
POSTGRES_DB: keycloak
POSTGRES_USER: keycloak

33
dev/nginx.conf
View File

@@ -53,6 +53,17 @@ http {
server_name _;
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
resolver 127.0.0.11 valid=30s;
set $upstream_minio minio;
proxy_pass http://$upstream_minio:9000;
@@ -76,6 +87,28 @@ http {
proxy_set_header X-Forwarded-Server $host;
}
}
server {
listen 9003 default_server;
server_name _;
location / {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_connect_timeout 300;
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
proxy_http_version 1.1;
proxy_set_header Connection "";
chunked_transfer_encoding off;
resolver 127.0.0.11 valid=30s;
set $upstream_minio minio;
proxy_pass http://$upstream_minio:9003;
}
}
}
stream {

7
restore.go
View File

@@ -15,6 +15,7 @@ import (
"path"
"strings"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/gin-gonic/gin"
@@ -102,8 +103,12 @@ func restoreS3(catalystStorage *storage.Storage, p string) error {
func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir fs.FS) error {
_, err := catalystStorage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String(entry.Name())})
if err != nil {
awsError, ok := err.(awserr.Error)
if !ok || (awsError.Code() != s3.ErrCodeBucketAlreadyExists && awsError.Code() != s3.ErrCodeBucketAlreadyOwnedByYou) {
return err
}
return nil
}
uploader := catalystStorage.Uploader()
@@ -127,7 +132,7 @@ func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir
}
func unzip(archive *zip.Reader, dir string) error {
return fs.WalkDir(archive, "arango", func(p string, d fs.DirEntry, err error) error {
return fs.WalkDir(archive, ".", func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

233
test/server_test.go
View File

@@ -1,17 +1,29 @@
package test
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"io"
"log"
"mime/multipart"
"net/http"
"net/http/httptest"
"reflect"
"regexp"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/assert"
"github.com/SecurityBrewery/catalyst"
"github.com/SecurityBrewery/catalyst/database/busdb"
"github.com/SecurityBrewery/catalyst/generated/models"
"github.com/SecurityBrewery/catalyst/pointer"
)
func TestService(t *testing.T) {
@@ -80,6 +92,227 @@ func TestService(t *testing.T) {
}
}
func TestBackupAndRestore(t *testing.T) {
gin.SetMode(gin.TestMode)
log.SetFlags(log.LstdFlags | log.Lshortfile)
type args struct {
method string
url string
data interface{}
}
type want struct {
status int
// body interface{}
}
tests := []struct {
name string
// args args
want want
}{
{name: "Backup", want: want{status: http.StatusOK}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx, _, server, err := Catalyst(t)
if err != nil {
t.Fatal(err)
}
if err := SetupTestData(ctx, server.DB); err != nil {
t.Fatal(err)
}
createFile(ctx, server)
server.Server.Use(func(context *gin.Context) {
busdb.SetContext(context, Bob)
})
zipB := assertBackup(t, server)
assertZipFile(t, readZipFile(t, zipB))
clearAllDatabases(server)
_, err = server.DB.UserCreateSetupAPIKey(ctx, "test")
if err != nil {
log.Fatal(err)
}
deleteAllBuckets(t, server)
assertRestore(t, zipB, server)
assertTicketExists(t, server)
assertFileExists(t, server)
})
}
}
func assertBackup(t *testing.T, server *catalyst.Server) []byte {
// setup request
req := httptest.NewRequest(http.MethodGet, "/api/backup/create", nil)
req.Header.Set("PRIVATE-TOKEN", "test")
// run request
backupRequestRecorder := httptest.NewRecorder()
server.Server.ServeHTTP(backupRequestRecorder, req)
backupResult := backupRequestRecorder.Result()
// assert results
assert.Equal(t, http.StatusOK, backupResult.StatusCode)
zipBuf := &bytes.Buffer{}
if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
t.Fatal(err)
}
assert.NoError(t, backupResult.Body.Close())
return zipBuf.Bytes()
}
func assertZipFile(t *testing.T, r *zip.Reader) {
var names []string
for _, f := range r.File {
names = append(names, f.Name)
}
if !includes(t, names, "minio/catalyst-8125/test.txt") {
t.Error("Minio file missing")
}
for _, p := range []string{
"arango/ENCRYPTION", "arango/automations_.*.data.json.gz", "arango/automations_.*.structure.json", "arango/dump.json", "arango/jobs_.*.data.json.gz", "arango/jobs_.*.structure.json", "arango/logs_.*.data.json.gz", "arango/logs_.*.structure.json", "arango/migrations_.*.data.json.gz", "arango/migrations_.*.structure.json", "arango/playbooks_.*.data.json.gz", "arango/playbooks_.*.structure.json", "arango/related_.*.data.json.gz", "arango/related_.*.structure.json", "arango/templates_.*.data.json.gz", "arango/templates_.*.structure.json", "arango/tickets_.*.data.json.gz", "arango/tickets_.*.structure.json", "arango/tickettypes_.*.data.json.gz", "arango/tickettypes_.*.structure.json", "arango/userdata_.*.data.json.gz", "arango/userdata_.*.structure.json", "arango/users_.*.data.json.gz", "arango/users_.*.structure.json",
} {
if !includes(t, names, p) {
t.Errorf("Arango file missing: %s", p)
}
}
}
func clearAllDatabases(server *catalyst.Server) {
server.DB.Truncate(context.Background())
}
func deleteAllBuckets(t *testing.T, server *catalyst.Server) {
buckets, err := server.Storage.S3().ListBuckets(&s3.ListBucketsInput{})
for _, bucket := range buckets.Buckets {
server.Storage.S3().DeleteBucket(&s3.DeleteBucketInput{
Bucket: bucket.Name,
})
}
if err != nil {
t.Fatal(err)
}
}
func assertRestore(t *testing.T, zipB []byte, server *catalyst.Server) {
bodyBuf := &bytes.Buffer{}
bodyWriter := multipart.NewWriter(bodyBuf)
fileWriter, err := bodyWriter.CreateFormFile("backup", "backup.zip")
if err != nil {
log.Fatal(err)
}
_, err = fileWriter.Write(zipB)
if err != nil {
log.Fatal(err)
}
assert.NoError(t, bodyWriter.Close())
req := httptest.NewRequest(http.MethodPost, "/api/backup/restore", bodyBuf)
req.Header.Set("PRIVATE-TOKEN", "test")
req.Header.Set("Content-Type", bodyWriter.FormDataContentType())
// run request
restoreRequestRecorder := httptest.NewRecorder()
server.Server.ServeHTTP(restoreRequestRecorder, req)
restoreResult := restoreRequestRecorder.Result()
if !assert.Equal(t, http.StatusOK, restoreResult.StatusCode) {
t.FailNow()
}
}
func createFile(ctx context.Context, server *catalyst.Server) {
buf := bytes.NewBufferString("test text")
server.Storage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String("catalyst-8125")})
if _, err := server.Storage.Uploader().Upload(&s3manager.UploadInput{Body: buf, Bucket: pointer.String("catalyst-8125"), Key: pointer.String("test.txt")}); err != nil {
log.Fatal(err)
}
if _, err := server.DB.LinkFiles(ctx, 8125, []*models.File{{Key: "test.txt", Name: "test.txt"}}); err != nil {
log.Fatal(err)
}
}
func assertTicketExists(t *testing.T, server *catalyst.Server) {
req := httptest.NewRequest(http.MethodGet, "/api/tickets/8125", nil)
req.Header.Set("PRIVATE-TOKEN", "test")
// run request
backupRequestRecorder := httptest.NewRecorder()
server.Server.ServeHTTP(backupRequestRecorder, req)
backupResult := backupRequestRecorder.Result()
// assert results
assert.Equal(t, http.StatusOK, backupResult.StatusCode)
zipBuf := &bytes.Buffer{}
if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
t.Fatal(err)
}
assert.NoError(t, backupResult.Body.Close())
var ticket models.Ticket
assert.NoError(t, json.Unmarshal(zipBuf.Bytes(), &ticket))
assert.Equal(t, "phishing from selenafadel@von.com detected", ticket.Name)
}
func assertFileExists(t *testing.T, server *catalyst.Server) {
obj, err := server.Storage.S3().GetObject(&s3.GetObjectInput{
Bucket: aws.String("catalyst-8125"),
Key: aws.String("test.txt"),
})
assert.NoError(t, err)
b, err := io.ReadAll(obj.Body)
assert.NoError(t, err)
assert.Equal(t, "test text", string(b))
}
func includes(t *testing.T, names []string, s string) bool {
for _, name := range names {
match, err := regexp.MatchString(s, name)
if err != nil {
t.Fatal(err)
}
if match {
return true
}
}
return false
}
func readZipFile(t *testing.T, b []byte) *zip.Reader {
buf := bytes.NewReader(b)
zr, err := zip.NewReader(buf, int64(buf.Len()))
if err != nil {
t.Fatal(string(b), err)
}
return zr
}
func jsonEqual(t *testing.T, got io.Reader, want interface{}) {
var j, j2 interface{}
c, err := io.ReadAll(got)

16
test/test.go
View File

@@ -36,6 +36,7 @@ func Context() context.Context {
func Config(ctx context.Context) (*catalyst.Config, error) {
config := &catalyst.Config{
InitialAPIKey: "test",
IndexPath: "index.bleve",
DB: &database.Config{
Host: "http://localhost:8529",
@@ -188,6 +189,21 @@ func Server(t *testing.T) (context.Context, *catalyst.Config, *bus.Bus, *index.I
return ctx, config, rbus, catalystIndex, catalystStorage, db, catalystService, catalystServer, cleanup, err
}
func Catalyst(t *testing.T) (context.Context, *catalyst.Config, *catalyst.Server, error) {
ctx := Context()
config, err := Config(ctx)
if err != nil {
t.Fatal(err)
}
config.DB.Name = cleanName(t)
c, err := catalyst.New(&hooks.Hooks{
DatabaseAfterConnectFuncs: []func(ctx context.Context, client driver.Client, name string){Clear},
}, config)
return ctx, config, c, err
}
func cleanName(t *testing.T) string {
name := t.Name()
name = strings.ReplaceAll(name, " ", "")