Uplift (#1025)

.github/workflows/ci.yml

@@ -15,12 +15,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
-        with: { go-version: '1.19' }
+        with: { go-version: '1.21' }
       - run: |
           mkdir -p ui/dist/img
           touch ui/dist/index.html ui/dist/favicon.ico ui/dist/manifest.json ui/dist/img/fake.png
       - uses: golangci/golangci-lint-action@v3
-        with: { version: 'v1.52' }
+        with: { version: 'v1.54' }
 
   test:
     name: Test
@@ -30,18 +30,12 @@ jobs:
       - uses: actions/setup-node@v3
         with: { node-version: '14', cache: 'yarn', cache-dependency-path: 'ui/yarn.lock' }
       - uses: actions/setup-go@v4
-        with: { go-version: '1.19', cache: true }
+        with: { go-version: '1.21' }
       - run: |
           mkdir -p ui/dist/img
           touch ui/dist/index.html ui/dist/favicon.ico ui/dist/manifest.json ui/dist/img/fake.png
       - run: docker compose up --quiet-pull --detach
         working-directory: dev
-      - name: Install ArangoDB
-        run: |
-          curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key
-          sudo apt-key add Release.key
-          sudo apt-add-repository 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /'
-          sudo apt-get update -y && sudo apt-get -y install arangodb3
       - run: go test -coverprofile=cover.out -coverpkg=./... ./...
       - run: go tool cover -func=cover.out
       - uses: codecov/codecov-action@v3
@@ -55,7 +49,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
-        with: { go-version: '1.18' }
+        with: { go-version: '1.21' }
       - uses: actions/setup-node@v3
         with: { node-version: '14' }
       # run UI
@@ -109,7 +103,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
-        with: { go-version: '1.19', cache: true }
+        with: { go-version: '1.21' }
       - uses: actions/download-artifact@v3
         with: { name: ui, path: ui/dist }
       - name: Version

.gitignore

@@ -81,6 +81,7 @@ dist
 node_modules
 
 profile.cov
+cover.out
 
 generated/caql/parser/*.interp
 generated/caql/parser/*.tokens

.golangci.yml

@@ -9,7 +9,6 @@ linters:
     - asciicheck
     - containedctx
     - decorder
-    - depguard
     - dogsled
     - durationcheck
     - errchkjson
@@ -71,6 +70,7 @@ linters:
     - nestif
 
     # disable
+    - depguard
     - dupl
     - exhaustivestruct
     - funlen

Makefile

@@ -0,0 +1,9 @@
+.PHONY: lint
+lint:
+	golangci-lint run  ./...
+
+.PHONY: fmt
+fmt:
+	gci write -s standard -s default -s "prefix(github.com/SecurityBrewery/catalyst)" .
+	# gofumpt -l -w .
+	# wsl --fix ./...

backup.go

@@ -1,155 +0,0 @@
-package catalyst
-
-import (
-	"archive/zip"
-	"bytes"
-	"io"
-	"io/fs"
-	"net/http"
-	"os"
-	"os/exec"
-	"path"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/service/s3"
-
-	"github.com/SecurityBrewery/catalyst/database"
-	"github.com/SecurityBrewery/catalyst/generated/api"
-	"github.com/SecurityBrewery/catalyst/storage"
-)
-
-func backupHandler(catalystStorage *storage.Storage, c *database.Config) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Disposition", "attachment; filename=backup.zip")
-		w.Header().Set("Content-Type", "application/zip")
-		err := Backup(catalystStorage, c, w)
-		if err != nil {
-			api.JSONError(w, err)
-		}
-	}
-}
-
-type WriterAtBuffer struct {
-	bytes.Buffer
-}
-
-func (fw WriterAtBuffer) WriteAt(p []byte, _ int64) (n int, err error) {
-	return fw.Write(p)
-}
-
-func Backup(catalystStorage *storage.Storage, c *database.Config, writer io.Writer) error {
-	archive := zip.NewWriter(writer)
-	defer archive.Close()
-
-	err := archive.SetComment(GetVersion())
-	if err != nil {
-		return err
-	}
-
-	// S3
-	if err := backupS3(catalystStorage, archive); err != nil {
-		return err
-	}
-
-	// Arango
-	return backupArango(c, archive)
-}
-
-func backupS3(catalystStorage *storage.Storage, archive *zip.Writer) error {
-	buckets, err := catalystStorage.S3().ListBuckets(nil)
-	if err != nil {
-		return err
-	}
-
-	for _, bucket := range buckets.Buckets {
-		objects, err := catalystStorage.S3().ListObjectsV2(&s3.ListObjectsV2Input{
-			Bucket: bucket.Name,
-		})
-		if err != nil {
-			return err
-		}
-
-		for _, content := range objects.Contents {
-			rbuf := &WriterAtBuffer{}
-			_, err := catalystStorage.Downloader().Download(rbuf, &s3.GetObjectInput{
-				Bucket: bucket.Name,
-				Key:    content.Key,
-			})
-			if err != nil {
-				return err
-			}
-
-			a, err := archive.Create(path.Join("minio", *bucket.Name, *content.Key))
-			if err != nil {
-				return err
-			}
-
-			if _, err := io.Copy(a, rbuf); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func backupArango(c *database.Config, archive *zip.Writer) error {
-	dir, err := os.MkdirTemp("", "catalyst-backup")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(dir)
-
-	if err := arangodump(dir, c); err != nil {
-		return err
-	}
-
-	return zipDump(dir, archive)
-}
-
-func zipDump(dir string, archive *zip.Writer) error {
-	fsys := os.DirFS(dir)
-
-	return fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if d.IsDir() {
-			return nil
-		}
-
-		a, err := archive.Create(path.Join("arango", p))
-		if err != nil {
-			return err
-		}
-
-		f, err := fsys.Open(p)
-		if err != nil {
-			return err
-		}
-
-		if _, err := io.Copy(a, f); err != nil {
-			return err
-		}
-
-		return nil
-	})
-}
-
-func arangodump(dir string, config *database.Config) error {
-	host := strings.Replace(config.Host, "http", "tcp", 1)
-
-	name := config.Name
-	if config.Name == "" {
-		name = database.Name
-	}
-	args := []string{
-		"--output-directory", dir, "--server.endpoint", host,
-		"--server.username", config.User, "--server.password", config.Password,
-		"--server.database", name,
-	}
-	cmd := exec.Command("arangodump", args...)
-
-	return cmd.Run()
-}

go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/alecthomas/kong-yaml v0.2.0
 	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220816024939-bc8df83d7b9d
 	github.com/arangodb/go-driver v1.6.0
-	github.com/aws/aws-sdk-go v1.45.7
+	github.com/aws/aws-sdk-go v1.45.28
-	github.com/blevesearch/bleve/v2 v2.3.9
+	github.com/blevesearch/bleve/v2 v2.3.10
 	github.com/coreos/go-oidc/v3 v3.6.0
 	github.com/docker/docker v17.12.0-ce-rc1.0.20201201034508-7d75c1d40d88+incompatible
 	github.com/go-chi/chi/v5 v5.0.10
@@ -22,36 +22,36 @@ require (
 	github.com/jonas-plum/maut v0.0.0-20221105155335-ed984fd96915
 	github.com/mingrammer/commonregex v1.0.1
 	github.com/stretchr/testify v1.8.4
-	github.com/tidwall/gjson v1.16.0
+	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/sjson v1.2.5
 	github.com/tus/tusd v1.13.0
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
-	golang.org/x/oauth2 v0.12.0
+	golang.org/x/oauth2 v0.13.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.5.1 // indirect
-	github.com/RoaringBitmap/roaring v1.2.3 // indirect
+	github.com/RoaringBitmap/roaring v1.6.0 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/bits-and-blooms/bitset v1.2.1 // indirect
+	github.com/bits-and-blooms/bitset v1.10.0 // indirect
-	github.com/blevesearch/bleve_index_api v1.0.5 // indirect
+	github.com/blevesearch/bleve_index_api v1.0.6 // indirect
-	github.com/blevesearch/geo v0.1.17 // indirect
+	github.com/blevesearch/geo v0.1.18 // indirect
 	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
 	github.com/blevesearch/gtreap v0.1.1 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
-	github.com/blevesearch/scorch_segment_api/v2 v2.1.5 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect
 	github.com/blevesearch/segment v0.9.1 // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
 	github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
 	github.com/blevesearch/vellum v1.0.10 // indirect
-	github.com/blevesearch/zapx/v11 v11.3.9 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.10 // indirect
-	github.com/blevesearch/zapx/v12 v12.3.9 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.10 // indirect
-	github.com/blevesearch/zapx/v13 v13.3.9 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.10 // indirect
-	github.com/blevesearch/zapx/v14 v14.3.9 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.10 // indirect
-	github.com/blevesearch/zapx/v15 v15.3.12 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.13 // indirect
 	github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect
 	github.com/containerd/containerd v1.6.8 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -62,7 +62,7 @@ require (
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
+	github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect
@@ -79,14 +79,14 @@ require (
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	go.etcd.io/bbolt v1.3.7 // indirect
-	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
-	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect
 	google.golang.org/grpc v1.57.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect

go.sum

@@ -770,8 +770,8 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
+github.com/RoaringBitmap/roaring v1.6.0 h1:dc7kRiroETgJcHhWX6BerXkZz2b3JgLGg9nTURJL/og=
-github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
+github.com/RoaringBitmap/roaring v1.6.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
 github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
 github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
@@ -802,29 +802,29 @@ github.com/arangodb/go-driver v1.6.0/go.mod h1:HQmdGkvNMVBTE3SIPSQ8T/ZddC6iwNsfM
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
 github.com/aws/aws-sdk-go v1.45.1/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.45.7 h1:k4QsvWZhm8409TYeRuTV1P6+j3lLKoe+giFA/j3VAps=
+github.com/aws/aws-sdk-go v1.45.28 h1:p2ATcaK6ffSw4yZ2UAGzgRyRXwKyOJY6ZCiKqj5miJE=
-github.com/aws/aws-sdk-go v1.45.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.45.28/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
-github.com/bits-and-blooms/bitset v1.2.1 h1:M+/hrU9xlMp7t4TyTDQW97d3tRPVuKFC6zBEK16QnXY=
+github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=
-github.com/bits-and-blooms/bitset v1.2.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
+github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
-github.com/blevesearch/bleve/v2 v2.3.9 h1:pUMvK0mxAexqasZcVj8lazmWnEW5XiV0tASIqANiNTQ=
+github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg=
-github.com/blevesearch/bleve/v2 v2.3.9/go.mod h1:1PibElcjlQMQHF9uS9mRv58ODQgj4pCWHA1Wfd+qagU=
+github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA=
-github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw=
+github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8=
-github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
+github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
-github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA=
+github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw=
-github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
+github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
 github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
 github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
 github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
 github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.5 h1:1g713kpCQZ8u4a3stRGBfrwVOuGRnmxOVU5MQkUPrHU=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA=
-github.com/blevesearch/scorch_segment_api/v2 v2.1.5/go.mod h1:f2nOkKS1HcjgIWZgDAErgBdxmr2eyt0Kn7IY+FU1Xe4=
+github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
@@ -833,16 +833,16 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG
 github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
 github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
 github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
-github.com/blevesearch/zapx/v11 v11.3.9 h1:y3ijS4h4MJdmQ07MHASxat4owAixreK2xdo76w9ncrw=
+github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
-github.com/blevesearch/zapx/v11 v11.3.9/go.mod h1:jcAYnQwlr+LqD2vLjDWjWiZDXDXGFqPbpPDRTd3XmS4=
+github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
-github.com/blevesearch/zapx/v12 v12.3.9 h1:MXGLlZ03oxXH3DMJTZaBaRj2xb6t4wQVZeZK/wu1M6w=
+github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
-github.com/blevesearch/zapx/v12 v12.3.9/go.mod h1:QXCMwmOkdLnMDgTN1P4CcuX5F851iUOtOwXbw0HMBYs=
+github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
-github.com/blevesearch/zapx/v13 v13.3.9 h1:+VAz9V0VmllHXlZV4DCvfYj0nqaZHgF3MeEHwOyRBwQ=
+github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
-github.com/blevesearch/zapx/v13 v13.3.9/go.mod h1:s+WjNp4WSDtrBVBpa37DUOd7S/Gr/jTZ7ST/MbCVj/0=
+github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
-github.com/blevesearch/zapx/v14 v14.3.9 h1:wuqxATgsTCNHM9xsOFOeFp8H2heZ/gMX/tsl9lRK8U4=
+github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
-github.com/blevesearch/zapx/v14 v14.3.9/go.mod h1:MWZ4v8AzFBRurhDzkLvokFW8ljcq9Evm27mkWe8OGbM=
+github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
-github.com/blevesearch/zapx/v15 v15.3.12 h1:w/kU9aHyfMDEdwHGZzCiakC3HZ9z5gYlXaALDC4Dct8=
+github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
-github.com/blevesearch/zapx/v15 v15.3.12/go.mod h1:tx53gDJS/7Oa3Je820cmVurqCuJ4dqdAy1kiDMV/IUo=
+github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
 github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f h1:gOO/tNZMjjvTKZWpY7YnXC72ULNLErRtp94LountVE8=
 github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
@@ -951,8 +951,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
-github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
+github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I=
-github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
+github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
@@ -1238,12 +1238,13 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
+github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
-github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tus/tusd v1.13.0 h1:W7rtb1XPSpde/GPZAgdfUS3vus2Jt2KmckS6OUd3CU8=
@@ -1304,8 +1305,8 @@ golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0
 golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
 golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
 golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
-golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -1432,8 +1433,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
 golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
-golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1466,8 +1467,8 @@ golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4
 golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
 golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
 golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
-golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
+golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
+golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1582,8 +1583,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1776,8 +1777,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
-google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
 google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
 google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=

restore.go

@@ -1,176 +0,0 @@
-package catalyst
-
-import (
-	"archive/zip"
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"io/fs"
-	"net/http"
-	"os"
-	"os/exec"
-	"path"
-	"strings"
-
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-
-	"github.com/SecurityBrewery/catalyst/database"
-	"github.com/SecurityBrewery/catalyst/generated/api"
-	"github.com/SecurityBrewery/catalyst/generated/pointer"
-	"github.com/SecurityBrewery/catalyst/storage"
-)
-
-func restoreHandler(catalystStorage *storage.Storage, db *database.Database, c *database.Config) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		uf, header, err := r.FormFile("backup")
-		if err != nil {
-			api.JSONError(w, err)
-
-			return
-		}
-
-		if err = Restore(r.Context(), catalystStorage, db, c, uf, header.Size); err != nil {
-			api.JSONError(w, err)
-
-			return
-		}
-	}
-}
-
-func Restore(ctx context.Context, catalystStorage *storage.Storage, db *database.Database, c *database.Config, r io.Reader, size int64) error {
-	b, err := io.ReadAll(r)
-	if err != nil {
-		return err
-	}
-
-	ra := bytes.NewReader(b)
-	fsys, err := zip.NewReader(ra, size)
-	if err != nil {
-		return err
-	}
-
-	if fsys.Comment != GetVersion() {
-		return fmt.Errorf("wrong version, got: %s, want: %s", fsys.Comment, GetVersion())
-	}
-
-	dir, err := os.MkdirTemp("", "catalyst-restore")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(dir)
-
-	if err = unzip(fsys, dir); err != nil {
-		return err
-	}
-
-	if err := restoreS3(catalystStorage, path.Join(dir, "minio")); err != nil {
-		return err
-	}
-
-	if err := arangorestore(path.Join(dir, "arango"), c); err != nil {
-		return err
-	}
-
-	return db.IndexRebuild(ctx)
-}
-
-func restoreS3(catalystStorage *storage.Storage, p string) error {
-	minioDir := os.DirFS(p)
-
-	entries, err := fs.ReadDir(minioDir, ".")
-	if err != nil {
-		// directory might not exist
-		return nil
-	}
-
-	for _, entry := range entries {
-		if err := restoreBucket(catalystStorage, entry, minioDir); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir fs.FS) error {
-	_, err := catalystStorage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String(entry.Name())})
-	if err != nil {
-		var awsError awserr.Error
-		if errors.As(err, &awsError) && (awsError.Code() == s3.ErrCodeBucketAlreadyExists || awsError.Code() == s3.ErrCodeBucketAlreadyOwnedByYou) {
-			return nil
-		}
-
-		return err
-	}
-
-	uploader := catalystStorage.Uploader()
-
-	f, err := minioDir.Open(entry.Name())
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	err = fs.WalkDir(minioDir, ".", func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return nil
-		}
-		_, err = uploader.Upload(&s3manager.UploadInput{Body: f, Bucket: pointer.String(entry.Name()), Key: pointer.String(path)})
-
-		return err
-	})
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func unzip(archive *zip.Reader, dir string) error {
-	return fs.WalkDir(archive, ".", func(p string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return err
-		}
-
-		if d.IsDir() {
-			_ = os.MkdirAll(path.Join(dir, p), os.ModePerm)
-
-			return nil
-		}
-
-		f, err := archive.Open(p)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-
-		b, err := io.ReadAll(f)
-		if err != nil {
-			return err
-		}
-
-		return os.WriteFile(path.Join(dir, p), b, os.ModePerm)
-	})
-}
-
-func arangorestore(dir string, config *database.Config) error {
-	host := strings.Replace(config.Host, "http", "tcp", 1)
-
-	name := config.Name
-	if config.Name == "" {
-		name = database.Name
-	}
-	args := []string{
-		"--batch-size", "524288",
-		"--input-directory", dir, "--server.endpoint", host,
-		"--server.username", config.User, "--server.password", config.Password,
-		"--server.database", name,
-	}
-	cmd := exec.Command("arangorestore", args...)
-
-	return cmd.Run()
-}

server.go

@@ -74,7 +74,7 @@ func New(hooks *hooks.Hooks, config *Config) (*Server, error) {
 		return nil, fmt.Errorf("failed to create authenticator: %w", err)
 	}
 
-	apiServer, err := setupAPI(authenticator, catalystService, catalystStorage, catalystDatabase, config.DB, catalystBus, config)
+	apiServer, err := setupAPI(authenticator, catalystService, catalystStorage, catalystDatabase, catalystBus, config)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create api server: %w", err)
 	}
@@ -88,7 +88,7 @@ func New(hooks *hooks.Hooks, config *Config) (*Server, error) {
 	}, nil
 }
 
-func setupAPI(authenticator *maut.Authenticator, catalystService *service.Service, catalystStorage *storage.Storage, catalystDatabase *database.Database, dbConfig *database.Config, bus *bus.Bus, config *Config) (chi.Router, error) {
+func setupAPI(authenticator *maut.Authenticator, catalystService *service.Service, catalystStorage *storage.Storage, catalystDatabase *database.Database, bus *bus.Bus, config *Config) (chi.Router, error) {
 	middlewares := []func(next http.Handler) http.Handler{
 		authenticator.Authenticate(),
 		authenticator.AuthorizeBlockedUser(),
@@ -97,7 +97,6 @@ func setupAPI(authenticator *maut.Authenticator, catalystService *service.Servic
 	// create server
 	apiServer := api.NewServer(catalystService, permissionAuth(authenticator), middlewares...)
 	apiServer.Mount("/files", fileServer(authenticator, catalystDatabase, bus, catalystStorage, config))
-	apiServer.Mount("/backup", backupServer(authenticator, catalystStorage, catalystDatabase, dbConfig))
 
 	server := chi.NewRouter()
 	server.Use(middleware.RequestID, middleware.RealIP, middleware.Logger, middleware.Recoverer)
@@ -130,11 +129,3 @@ func fileServer(authenticator *maut.Authenticator, catalystDatabase *database.Da
 
 	return server
 }
-
-func backupServer(authenticator *maut.Authenticator, catalystStorage *storage.Storage, catalystDatabase *database.Database, dbConfig *database.Config) *chi.Mux {
-	server := chi.NewRouter()
-	server.With(authenticator.AuthorizePermission("backup:create")).Get("/create", backupHandler(catalystStorage, dbConfig))
-	server.With(authenticator.AuthorizePermission("backup:restore")).Post("/restore", restoreHandler(catalystStorage, catalystDatabase, dbConfig))
-
-	return server
-}

storage/storage.go

@@ -20,7 +20,7 @@ type Storage struct {
 type Config struct {
 	Host     string
 	User     string
-	Region	 string
+	Region   string
 	Password string
 }
 

test/backup_test.go

@@ -1,262 +0,0 @@
-package test
-
-import (
-	"archive/zip"
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"log"
-	"mime/multipart"
-	"net/http"
-	"net/http/httptest"
-	"regexp"
-	"runtime"
-	"testing"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/SecurityBrewery/catalyst"
-	"github.com/SecurityBrewery/catalyst/generated/model"
-	"github.com/SecurityBrewery/catalyst/generated/pointer"
-)
-
-func TestBackupAndRestore(t *testing.T) {
-	t.Parallel()
-
-	log.SetFlags(log.LstdFlags | log.Lshortfile)
-
-	if runtime.GOARCH == "arm64" {
-		t.Skip("test does not run on arm")
-	}
-
-	type want struct {
-		status int
-	}
-	tests := []struct {
-		name string
-		want want
-	}{
-		{name: "Backup", want: want{status: http.StatusOK}},
-	}
-	for _, tt := range tests {
-		tt := tt
-		t.Run(tt.name, func(t *testing.T) {
-			t.Parallel()
-
-			ctx, _, server, err := Catalyst(t)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if err := SetupTestData(ctx, server.DB); err != nil {
-				t.Fatal(err)
-			}
-
-			createFile(ctx, server)
-
-			zipB := assertBackup(t, server)
-
-			assertZipFile(t, readZipFile(t, zipB))
-
-			clearAllDatabases(server)
-			_, err = server.DB.UserCreateSetupAPIKey(ctx, "test")
-			if err != nil {
-				log.Fatal(err)
-			}
-
-			deleteAllBuckets(t, server)
-
-			assertRestore(t, zipB, server)
-
-			assertTicketExists(t, server)
-
-			assertFileExists(t, server)
-		})
-	}
-}
-
-func assertBackup(t *testing.T, server *catalyst.Server) []byte {
-	t.Helper()
-
-	// setup request
-	req := httptest.NewRequest(http.MethodGet, "/api/backup/create", nil)
-	req.Header.Set("PRIVATE-TOKEN", "test")
-
-	// run request
-	backupRequestRecorder := httptest.NewRecorder()
-	server.Server.ServeHTTP(backupRequestRecorder, req)
-	backupResult := backupRequestRecorder.Result()
-
-	// assert results
-	assert.Equal(t, http.StatusOK, backupResult.StatusCode)
-
-	zipBuf := &bytes.Buffer{}
-	if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
-		t.Fatal(err)
-	}
-	assert.NoError(t, backupResult.Body.Close())
-
-	return zipBuf.Bytes()
-}
-
-func assertZipFile(t *testing.T, r *zip.Reader) {
-	t.Helper()
-
-	var names []string
-	for _, f := range r.File {
-		names = append(names, f.Name)
-	}
-
-	if !includes(t, names, "minio/catalyst-8125/test.txt") {
-		t.Error("Minio file missing")
-	}
-
-	for _, p := range []string{
-		"arango/ENCRYPTION", "arango/automations_.*.data.json.gz", "arango/automations_.*.structure.json", "arango/dump.json", "arango/jobs_.*.data.json.gz", "arango/jobs_.*.structure.json", "arango/logs_.*.data.json.gz", "arango/logs_.*.structure.json", "arango/migrations_.*.data.json.gz", "arango/migrations_.*.structure.json", "arango/playbooks_.*.data.json.gz", "arango/playbooks_.*.structure.json", "arango/related_.*.data.json.gz", "arango/related_.*.structure.json", "arango/templates_.*.data.json.gz", "arango/templates_.*.structure.json", "arango/tickets_.*.data.json.gz", "arango/tickets_.*.structure.json", "arango/tickettypes_.*.data.json.gz", "arango/tickettypes_.*.structure.json", "arango/userdata_.*.data.json.gz", "arango/userdata_.*.structure.json", "arango/users_.*.data.json.gz", "arango/users_.*.structure.json",
-	} {
-		if !includes(t, names, p) {
-			t.Errorf("Arango file missing: %s", p)
-		}
-	}
-}
-
-func clearAllDatabases(server *catalyst.Server) {
-	server.DB.Truncate(context.Background())
-}
-
-func deleteAllBuckets(t *testing.T, server *catalyst.Server) {
-	t.Helper()
-
-	buckets, err := server.Storage.S3().ListBuckets(&s3.ListBucketsInput{})
-	for _, bucket := range buckets.Buckets {
-		_, _ = server.Storage.S3().DeleteBucket(&s3.DeleteBucketInput{
-			Bucket: bucket.Name,
-		})
-	}
-
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
-func assertRestore(t *testing.T, zipB []byte, server *catalyst.Server) {
-	t.Helper()
-
-	bodyBuf := &bytes.Buffer{}
-	bodyWriter := multipart.NewWriter(bodyBuf)
-	fileWriter, err := bodyWriter.CreateFormFile("backup", "backup.zip")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	_, err = fileWriter.Write(zipB)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	assert.NoError(t, bodyWriter.Close())
-
-	req := httptest.NewRequest(http.MethodPost, "/api/backup/restore", bodyBuf)
-	req.Header.Set("PRIVATE-TOKEN", "test")
-	req.Header.Set("Content-Type", bodyWriter.FormDataContentType())
-
-	// run request
-	restoreRequestRecorder := httptest.NewRecorder()
-	server.Server.ServeHTTP(restoreRequestRecorder, req)
-	restoreResult := restoreRequestRecorder.Result()
-
-	if !assert.Equal(t, http.StatusOK, restoreResult.StatusCode) {
-		b, _ := io.ReadAll(restoreResult.Body)
-		log.Println(string(b))
-		t.FailNow()
-	}
-}
-
-func createFile(ctx context.Context, server *catalyst.Server) {
-	buf := bytes.NewBufferString("test text")
-
-	_, _ = server.Storage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String("catalyst-8125")})
-
-	if _, err := server.Storage.Uploader().Upload(&s3manager.UploadInput{Body: buf, Bucket: pointer.String("catalyst-8125"), Key: pointer.String("test.txt")}); err != nil {
-		log.Fatal(err)
-	}
-
-	if _, err := server.DB.AddFile(ctx, 8125, &model.File{Key: "test.txt", Name: "test.txt"}); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func assertTicketExists(t *testing.T, server *catalyst.Server) {
-	t.Helper()
-
-	req := httptest.NewRequest(http.MethodGet, "/api/tickets/8125", nil)
-	req.Header.Set("PRIVATE-TOKEN", "test")
-
-	// run request
-	backupRequestRecorder := httptest.NewRecorder()
-	server.Server.ServeHTTP(backupRequestRecorder, req)
-	backupResult := backupRequestRecorder.Result()
-
-	// assert results
-	assert.Equal(t, http.StatusOK, backupResult.StatusCode)
-
-	zipBuf := &bytes.Buffer{}
-	if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
-		t.Fatal(err)
-	}
-	assert.NoError(t, backupResult.Body.Close())
-
-	var ticket model.Ticket
-	assert.NoError(t, json.Unmarshal(zipBuf.Bytes(), &ticket))
-
-	assert.Equal(t, "phishing from selenafadel@von.com detected", ticket.Name)
-}
-
-func assertFileExists(t *testing.T, server *catalyst.Server) {
-	t.Helper()
-
-	obj, err := server.Storage.S3().GetObject(&s3.GetObjectInput{
-		Bucket: aws.String("catalyst-8125"),
-		Key:    aws.String("test.txt"),
-	})
-	assert.NoError(t, err)
-
-	b, err := io.ReadAll(obj.Body)
-	assert.NoError(t, err)
-
-	assert.Equal(t, "test text", string(b))
-}
-
-func includes(t *testing.T, names []string, s string) bool {
-	t.Helper()
-
-	for _, name := range names {
-		match, err := regexp.MatchString(s, name)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if match {
-			return true
-		}
-	}
-
-	return false
-}
-
-func readZipFile(t *testing.T, b []byte) *zip.Reader {
-	t.Helper()
-
-	buf := bytes.NewReader(b)
-
-	zr, err := zip.NewReader(buf, int64(buf.Len()))
-	if err != nil {
-		t.Fatal(string(b), err)
-	}
-
-	return zr
-}