mirror of
https://github.com/SecurityBrewery/catalyst.git
synced 2025-12-07 07:42:45 +01:00
Uplift (#1025)
This commit is contained in:
Jonas Plum
GitHub
16
.github/workflows/ci.yml
vendored
16
.github/workflows/ci.yml
vendored
@@ -15,12 +15,12 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v4
|
||||
with: { go-version: '1.19' }
|
||||
with: { go-version: '1.21' }
|
||||
- run: |
|
||||
mkdir -p ui/dist/img
|
||||
touch ui/dist/index.html ui/dist/favicon.ico ui/dist/manifest.json ui/dist/img/fake.png
|
||||
- uses: golangci/golangci-lint-action@v3
|
||||
with: { version: 'v1.52' }
|
||||
with: { version: 'v1.54' }
|
||||
|
||||
test:
|
||||
name: Test
|
||||
@@ -30,18 +30,12 @@ jobs:
|
||||
- uses: actions/setup-node@v3
|
||||
with: { node-version: '14', cache: 'yarn', cache-dependency-path: 'ui/yarn.lock' }
|
||||
- uses: actions/setup-go@v4
|
||||
with: { go-version: '1.19', cache: true }
|
||||
with: { go-version: '1.21' }
|
||||
- run: |
|
||||
mkdir -p ui/dist/img
|
||||
touch ui/dist/index.html ui/dist/favicon.ico ui/dist/manifest.json ui/dist/img/fake.png
|
||||
- run: docker compose up --quiet-pull --detach
|
||||
working-directory: dev
|
||||
- name: Install ArangoDB
|
||||
run: |
|
||||
curl -OL https://download.arangodb.com/arangodb38/DEBIAN/Release.key
|
||||
sudo apt-key add Release.key
|
||||
sudo apt-add-repository 'deb https://download.arangodb.com/arangodb38/DEBIAN/ /'
|
||||
sudo apt-get update -y && sudo apt-get -y install arangodb3
|
||||
- run: go test -coverprofile=cover.out -coverpkg=./... ./...
|
||||
- run: go tool cover -func=cover.out
|
||||
- uses: codecov/codecov-action@v3
|
||||
@@ -55,7 +49,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v4
|
||||
with: { go-version: '1.18' }
|
||||
with: { go-version: '1.21' }
|
||||
- uses: actions/setup-node@v3
|
||||
with: { node-version: '14' }
|
||||
# run UI
|
||||
@@ -109,7 +103,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/setup-go@v4
|
||||
with: { go-version: '1.19', cache: true }
|
||||
with: { go-version: '1.21' }
|
||||
- uses: actions/download-artifact@v3
|
||||
with: { name: ui, path: ui/dist }
|
||||
- name: Version
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -81,6 +81,7 @@ dist
|
||||
node_modules
|
||||
|
||||
profile.cov
|
||||
cover.out
|
||||
|
||||
generated/caql/parser/*.interp
|
||||
generated/caql/parser/*.tokens
|
||||
|
||||
@@ -9,7 +9,6 @@ linters:
|
||||
- asciicheck
|
||||
- containedctx
|
||||
- decorder
|
||||
- depguard
|
||||
- dogsled
|
||||
- durationcheck
|
||||
- errchkjson
|
||||
@@ -71,6 +70,7 @@ linters:
|
||||
- nestif
|
||||
|
||||
# disable
|
||||
- depguard
|
||||
- dupl
|
||||
- exhaustivestruct
|
||||
- funlen
|
||||
|
||||
9
Makefile
Normal file
9
Makefile
Normal file
@@ -0,0 +1,9 @@
|
||||
.PHONY: lint
|
||||
lint:
|
||||
golangci-lint run ./...
|
||||
|
||||
.PHONY: fmt
|
||||
fmt:
|
||||
gci write -s standard -s default -s "prefix(github.com/SecurityBrewery/catalyst)" .
|
||||
# gofumpt -l -w .
|
||||
# wsl --fix ./...
|
||||
155
backup.go
155
backup.go
@@ -1,155 +0,0 @@
|
||||
package catalyst
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
|
||||
"github.com/SecurityBrewery/catalyst/database"
|
||||
"github.com/SecurityBrewery/catalyst/generated/api"
|
||||
"github.com/SecurityBrewery/catalyst/storage"
|
||||
)
|
||||
|
||||
func backupHandler(catalystStorage *storage.Storage, c *database.Config) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=backup.zip")
|
||||
w.Header().Set("Content-Type", "application/zip")
|
||||
err := Backup(catalystStorage, c, w)
|
||||
if err != nil {
|
||||
api.JSONError(w, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type WriterAtBuffer struct {
|
||||
bytes.Buffer
|
||||
}
|
||||
|
||||
func (fw WriterAtBuffer) WriteAt(p []byte, _ int64) (n int, err error) {
|
||||
return fw.Write(p)
|
||||
}
|
||||
|
||||
func Backup(catalystStorage *storage.Storage, c *database.Config, writer io.Writer) error {
|
||||
archive := zip.NewWriter(writer)
|
||||
defer archive.Close()
|
||||
|
||||
err := archive.SetComment(GetVersion())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// S3
|
||||
if err := backupS3(catalystStorage, archive); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Arango
|
||||
return backupArango(c, archive)
|
||||
}
|
||||
|
||||
func backupS3(catalystStorage *storage.Storage, archive *zip.Writer) error {
|
||||
buckets, err := catalystStorage.S3().ListBuckets(nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, bucket := range buckets.Buckets {
|
||||
objects, err := catalystStorage.S3().ListObjectsV2(&s3.ListObjectsV2Input{
|
||||
Bucket: bucket.Name,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, content := range objects.Contents {
|
||||
rbuf := &WriterAtBuffer{}
|
||||
_, err := catalystStorage.Downloader().Download(rbuf, &s3.GetObjectInput{
|
||||
Bucket: bucket.Name,
|
||||
Key: content.Key,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
a, err := archive.Create(path.Join("minio", *bucket.Name, *content.Key))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(a, rbuf); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func backupArango(c *database.Config, archive *zip.Writer) error {
|
||||
dir, err := os.MkdirTemp("", "catalyst-backup")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
if err := arangodump(dir, c); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return zipDump(dir, archive)
|
||||
}
|
||||
|
||||
func zipDump(dir string, archive *zip.Writer) error {
|
||||
fsys := os.DirFS(dir)
|
||||
|
||||
return fs.WalkDir(fsys, ".", func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
a, err := archive.Create(path.Join("arango", p))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := fsys.Open(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(a, f); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func arangodump(dir string, config *database.Config) error {
|
||||
host := strings.Replace(config.Host, "http", "tcp", 1)
|
||||
|
||||
name := config.Name
|
||||
if config.Name == "" {
|
||||
name = database.Name
|
||||
}
|
||||
args := []string{
|
||||
"--output-directory", dir, "--server.endpoint", host,
|
||||
"--server.username", config.User, "--server.password", config.Password,
|
||||
"--server.database", name,
|
||||
}
|
||||
cmd := exec.Command("arangodump", args...)
|
||||
|
||||
return cmd.Run()
|
||||
}
|
||||
40
go.mod
40
go.mod
@@ -9,8 +9,8 @@ require (
|
||||
github.com/alecthomas/kong-yaml v0.2.0
|
||||
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20220816024939-bc8df83d7b9d
|
||||
github.com/arangodb/go-driver v1.6.0
|
||||
github.com/aws/aws-sdk-go v1.45.7
|
||||
github.com/blevesearch/bleve/v2 v2.3.9
|
||||
github.com/aws/aws-sdk-go v1.45.28
|
||||
github.com/blevesearch/bleve/v2 v2.3.10
|
||||
github.com/coreos/go-oidc/v3 v3.6.0
|
||||
github.com/docker/docker v17.12.0-ce-rc1.0.20201201034508-7d75c1d40d88+incompatible
|
||||
github.com/go-chi/chi/v5 v5.0.10
|
||||
@@ -22,36 +22,36 @@ require (
|
||||
github.com/jonas-plum/maut v0.0.0-20221105155335-ed984fd96915
|
||||
github.com/mingrammer/commonregex v1.0.1
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/tidwall/gjson v1.16.0
|
||||
github.com/tidwall/gjson v1.17.0
|
||||
github.com/tidwall/sjson v1.2.5
|
||||
github.com/tus/tusd v1.13.0
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
|
||||
golang.org/x/oauth2 v0.12.0
|
||||
golang.org/x/oauth2 v0.13.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
|
||||
github.com/Microsoft/go-winio v0.5.1 // indirect
|
||||
github.com/RoaringBitmap/roaring v1.2.3 // indirect
|
||||
github.com/RoaringBitmap/roaring v1.6.0 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/bits-and-blooms/bitset v1.2.1 // indirect
|
||||
github.com/blevesearch/bleve_index_api v1.0.5 // indirect
|
||||
github.com/blevesearch/geo v0.1.17 // indirect
|
||||
github.com/bits-and-blooms/bitset v1.10.0 // indirect
|
||||
github.com/blevesearch/bleve_index_api v1.0.6 // indirect
|
||||
github.com/blevesearch/geo v0.1.18 // indirect
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
|
||||
github.com/blevesearch/gtreap v0.1.1 // indirect
|
||||
github.com/blevesearch/mmap-go v1.0.4 // indirect
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.5 // indirect
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect
|
||||
github.com/blevesearch/segment v0.9.1 // indirect
|
||||
github.com/blevesearch/snowballstem v0.9.0 // indirect
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
|
||||
github.com/blevesearch/vellum v1.0.10 // indirect
|
||||
github.com/blevesearch/zapx/v11 v11.3.9 // indirect
|
||||
github.com/blevesearch/zapx/v12 v12.3.9 // indirect
|
||||
github.com/blevesearch/zapx/v13 v13.3.9 // indirect
|
||||
github.com/blevesearch/zapx/v14 v14.3.9 // indirect
|
||||
github.com/blevesearch/zapx/v15 v15.3.12 // indirect
|
||||
github.com/blevesearch/zapx/v11 v11.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v12 v12.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v13 v13.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v14 v14.3.10 // indirect
|
||||
github.com/blevesearch/zapx/v15 v15.3.13 // indirect
|
||||
github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f // indirect
|
||||
github.com/containerd/containerd v1.6.8 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
@@ -62,7 +62,7 @@ require (
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
|
||||
github.com/golang/geo v0.0.0-20230421003525-6adc56603217 // indirect
|
||||
github.com/golang/protobuf v1.5.3 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/gorilla/securecookie v1.1.1 // indirect
|
||||
@@ -79,14 +79,14 @@ require (
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
go.etcd.io/bbolt v1.3.7 // indirect
|
||||
golang.org/x/crypto v0.13.0 // indirect
|
||||
golang.org/x/net v0.15.0 // indirect
|
||||
golang.org/x/sys v0.12.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
golang.org/x/crypto v0.14.0 // indirect
|
||||
golang.org/x/net v0.17.0 // indirect
|
||||
golang.org/x/sys v0.13.0 // indirect
|
||||
google.golang.org/appengine v1.6.8 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230807174057-1744710a1577 // indirect
|
||||
google.golang.org/grpc v1.57.0 // indirect
|
||||
google.golang.org/protobuf v1.31.0 // indirect
|
||||
|
||||
78
go.sum
78
go.sum
@@ -770,8 +770,8 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0
|
||||
github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
|
||||
github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
|
||||
github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY=
|
||||
github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
|
||||
github.com/RoaringBitmap/roaring v1.6.0 h1:dc7kRiroETgJcHhWX6BerXkZz2b3JgLGg9nTURJL/og=
|
||||
github.com/RoaringBitmap/roaring v1.6.0/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE=
|
||||
github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
|
||||
github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
|
||||
github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
|
||||
@@ -802,29 +802,29 @@ github.com/arangodb/go-driver v1.6.0/go.mod h1:HQmdGkvNMVBTE3SIPSQ8T/ZddC6iwNsfM
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
|
||||
github.com/aws/aws-sdk-go v1.45.1/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
|
||||
github.com/aws/aws-sdk-go v1.45.7 h1:k4QsvWZhm8409TYeRuTV1P6+j3lLKoe+giFA/j3VAps=
|
||||
github.com/aws/aws-sdk-go v1.45.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
|
||||
github.com/aws/aws-sdk-go v1.45.28 h1:p2ATcaK6ffSw4yZ2UAGzgRyRXwKyOJY6ZCiKqj5miJE=
|
||||
github.com/aws/aws-sdk-go v1.45.28/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
|
||||
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
|
||||
github.com/bits-and-blooms/bitset v1.2.1 h1:M+/hrU9xlMp7t4TyTDQW97d3tRPVuKFC6zBEK16QnXY=
|
||||
github.com/bits-and-blooms/bitset v1.2.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
|
||||
github.com/blevesearch/bleve/v2 v2.3.9 h1:pUMvK0mxAexqasZcVj8lazmWnEW5XiV0tASIqANiNTQ=
|
||||
github.com/blevesearch/bleve/v2 v2.3.9/go.mod h1:1PibElcjlQMQHF9uS9mRv58ODQgj4pCWHA1Wfd+qagU=
|
||||
github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw=
|
||||
github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
|
||||
github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA=
|
||||
github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
|
||||
github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88=
|
||||
github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
|
||||
github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg=
|
||||
github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA=
|
||||
github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8=
|
||||
github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms=
|
||||
github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw=
|
||||
github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo=
|
||||
github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M=
|
||||
github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y=
|
||||
github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk=
|
||||
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
|
||||
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.5 h1:1g713kpCQZ8u4a3stRGBfrwVOuGRnmxOVU5MQkUPrHU=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.5/go.mod h1:f2nOkKS1HcjgIWZgDAErgBdxmr2eyt0Kn7IY+FU1Xe4=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA=
|
||||
github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc=
|
||||
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
|
||||
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
|
||||
github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
|
||||
@@ -833,16 +833,16 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG
|
||||
github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ=
|
||||
github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI=
|
||||
github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k=
|
||||
github.com/blevesearch/zapx/v11 v11.3.9 h1:y3ijS4h4MJdmQ07MHASxat4owAixreK2xdo76w9ncrw=
|
||||
github.com/blevesearch/zapx/v11 v11.3.9/go.mod h1:jcAYnQwlr+LqD2vLjDWjWiZDXDXGFqPbpPDRTd3XmS4=
|
||||
github.com/blevesearch/zapx/v12 v12.3.9 h1:MXGLlZ03oxXH3DMJTZaBaRj2xb6t4wQVZeZK/wu1M6w=
|
||||
github.com/blevesearch/zapx/v12 v12.3.9/go.mod h1:QXCMwmOkdLnMDgTN1P4CcuX5F851iUOtOwXbw0HMBYs=
|
||||
github.com/blevesearch/zapx/v13 v13.3.9 h1:+VAz9V0VmllHXlZV4DCvfYj0nqaZHgF3MeEHwOyRBwQ=
|
||||
github.com/blevesearch/zapx/v13 v13.3.9/go.mod h1:s+WjNp4WSDtrBVBpa37DUOd7S/Gr/jTZ7ST/MbCVj/0=
|
||||
github.com/blevesearch/zapx/v14 v14.3.9 h1:wuqxATgsTCNHM9xsOFOeFp8H2heZ/gMX/tsl9lRK8U4=
|
||||
github.com/blevesearch/zapx/v14 v14.3.9/go.mod h1:MWZ4v8AzFBRurhDzkLvokFW8ljcq9Evm27mkWe8OGbM=
|
||||
github.com/blevesearch/zapx/v15 v15.3.12 h1:w/kU9aHyfMDEdwHGZzCiakC3HZ9z5gYlXaALDC4Dct8=
|
||||
github.com/blevesearch/zapx/v15 v15.3.12/go.mod h1:tx53gDJS/7Oa3Je820cmVurqCuJ4dqdAy1kiDMV/IUo=
|
||||
github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk=
|
||||
github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ=
|
||||
github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s=
|
||||
github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs=
|
||||
github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8=
|
||||
github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk=
|
||||
github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU=
|
||||
github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns=
|
||||
github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ=
|
||||
github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg=
|
||||
github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
|
||||
github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f h1:gOO/tNZMjjvTKZWpY7YnXC72ULNLErRtp94LountVE8=
|
||||
github.com/bmizerany/pat v0.0.0-20210406213842-e4b6760bdd6f/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c=
|
||||
@@ -951,8 +951,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo=
|
||||
github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
|
||||
github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I=
|
||||
github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
|
||||
github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
|
||||
@@ -1238,12 +1238,13 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl
|
||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
|
||||
github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
|
||||
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
github.com/tus/tusd v1.13.0 h1:W7rtb1XPSpde/GPZAgdfUS3vus2Jt2KmckS6OUd3CU8=
|
||||
@@ -1304,8 +1305,8 @@ golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0
|
||||
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
|
||||
golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
|
||||
golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
|
||||
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
|
||||
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
@@ -1432,8 +1433,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
|
||||
golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
|
||||
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
|
||||
golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -1466,8 +1467,8 @@ golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4
|
||||
golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
|
||||
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
|
||||
golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
|
||||
golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4=
|
||||
golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4=
|
||||
golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
|
||||
golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -1582,8 +1583,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -1776,8 +1777,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
|
||||
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
|
||||
|
||||
176
restore.go
176
restore.go
@@ -1,176 +0,0 @@
|
||||
package catalyst
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws/awserr"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
|
||||
"github.com/SecurityBrewery/catalyst/database"
|
||||
"github.com/SecurityBrewery/catalyst/generated/api"
|
||||
"github.com/SecurityBrewery/catalyst/generated/pointer"
|
||||
"github.com/SecurityBrewery/catalyst/storage"
|
||||
)
|
||||
|
||||
func restoreHandler(catalystStorage *storage.Storage, db *database.Database, c *database.Config) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
uf, header, err := r.FormFile("backup")
|
||||
if err != nil {
|
||||
api.JSONError(w, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if err = Restore(r.Context(), catalystStorage, db, c, uf, header.Size); err != nil {
|
||||
api.JSONError(w, err)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Restore(ctx context.Context, catalystStorage *storage.Storage, db *database.Database, c *database.Config, r io.Reader, size int64) error {
|
||||
b, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ra := bytes.NewReader(b)
|
||||
fsys, err := zip.NewReader(ra, size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if fsys.Comment != GetVersion() {
|
||||
return fmt.Errorf("wrong version, got: %s, want: %s", fsys.Comment, GetVersion())
|
||||
}
|
||||
|
||||
dir, err := os.MkdirTemp("", "catalyst-restore")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
if err = unzip(fsys, dir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := restoreS3(catalystStorage, path.Join(dir, "minio")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := arangorestore(path.Join(dir, "arango"), c); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return db.IndexRebuild(ctx)
|
||||
}
|
||||
|
||||
func restoreS3(catalystStorage *storage.Storage, p string) error {
|
||||
minioDir := os.DirFS(p)
|
||||
|
||||
entries, err := fs.ReadDir(minioDir, ".")
|
||||
if err != nil {
|
||||
// directory might not exist
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
if err := restoreBucket(catalystStorage, entry, minioDir); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func restoreBucket(catalystStorage *storage.Storage, entry fs.DirEntry, minioDir fs.FS) error {
|
||||
_, err := catalystStorage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String(entry.Name())})
|
||||
if err != nil {
|
||||
var awsError awserr.Error
|
||||
if errors.As(err, &awsError) && (awsError.Code() == s3.ErrCodeBucketAlreadyExists || awsError.Code() == s3.ErrCodeBucketAlreadyOwnedByYou) {
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
uploader := catalystStorage.Uploader()
|
||||
|
||||
f, err := minioDir.Open(entry.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
err = fs.WalkDir(minioDir, ".", func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
_, err = uploader.Upload(&s3manager.UploadInput{Body: f, Bucket: pointer.String(entry.Name()), Key: pointer.String(path)})
|
||||
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func unzip(archive *zip.Reader, dir string) error {
|
||||
return fs.WalkDir(archive, ".", func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if d.IsDir() {
|
||||
_ = os.MkdirAll(path.Join(dir, p), os.ModePerm)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
f, err := archive.Open(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
b, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(path.Join(dir, p), b, os.ModePerm)
|
||||
})
|
||||
}
|
||||
|
||||
func arangorestore(dir string, config *database.Config) error {
|
||||
host := strings.Replace(config.Host, "http", "tcp", 1)
|
||||
|
||||
name := config.Name
|
||||
if config.Name == "" {
|
||||
name = database.Name
|
||||
}
|
||||
args := []string{
|
||||
"--batch-size", "524288",
|
||||
"--input-directory", dir, "--server.endpoint", host,
|
||||
"--server.username", config.User, "--server.password", config.Password,
|
||||
"--server.database", name,
|
||||
}
|
||||
cmd := exec.Command("arangorestore", args...)
|
||||
|
||||
return cmd.Run()
|
||||
}
|
||||
13
server.go
13
server.go
@@ -74,7 +74,7 @@ func New(hooks *hooks.Hooks, config *Config) (*Server, error) {
|
||||
return nil, fmt.Errorf("failed to create authenticator: %w", err)
|
||||
}
|
||||
|
||||
apiServer, err := setupAPI(authenticator, catalystService, catalystStorage, catalystDatabase, config.DB, catalystBus, config)
|
||||
apiServer, err := setupAPI(authenticator, catalystService, catalystStorage, catalystDatabase, catalystBus, config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create api server: %w", err)
|
||||
}
|
||||
@@ -88,7 +88,7 @@ func New(hooks *hooks.Hooks, config *Config) (*Server, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func setupAPI(authenticator *maut.Authenticator, catalystService *service.Service, catalystStorage *storage.Storage, catalystDatabase *database.Database, dbConfig *database.Config, bus *bus.Bus, config *Config) (chi.Router, error) {
|
||||
func setupAPI(authenticator *maut.Authenticator, catalystService *service.Service, catalystStorage *storage.Storage, catalystDatabase *database.Database, bus *bus.Bus, config *Config) (chi.Router, error) {
|
||||
middlewares := []func(next http.Handler) http.Handler{
|
||||
authenticator.Authenticate(),
|
||||
authenticator.AuthorizeBlockedUser(),
|
||||
@@ -97,7 +97,6 @@ func setupAPI(authenticator *maut.Authenticator, catalystService *service.Servic
|
||||
// create server
|
||||
apiServer := api.NewServer(catalystService, permissionAuth(authenticator), middlewares...)
|
||||
apiServer.Mount("/files", fileServer(authenticator, catalystDatabase, bus, catalystStorage, config))
|
||||
apiServer.Mount("/backup", backupServer(authenticator, catalystStorage, catalystDatabase, dbConfig))
|
||||
|
||||
server := chi.NewRouter()
|
||||
server.Use(middleware.RequestID, middleware.RealIP, middleware.Logger, middleware.Recoverer)
|
||||
@@ -130,11 +129,3 @@ func fileServer(authenticator *maut.Authenticator, catalystDatabase *database.Da
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
func backupServer(authenticator *maut.Authenticator, catalystStorage *storage.Storage, catalystDatabase *database.Database, dbConfig *database.Config) *chi.Mux {
|
||||
server := chi.NewRouter()
|
||||
server.With(authenticator.AuthorizePermission("backup:create")).Get("/create", backupHandler(catalystStorage, dbConfig))
|
||||
server.With(authenticator.AuthorizePermission("backup:restore")).Post("/restore", restoreHandler(catalystStorage, catalystDatabase, dbConfig))
|
||||
|
||||
return server
|
||||
}
|
||||
|
||||
@@ -1,262 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/aws/aws-sdk-go/service/s3/s3manager"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/SecurityBrewery/catalyst"
|
||||
"github.com/SecurityBrewery/catalyst/generated/model"
|
||||
"github.com/SecurityBrewery/catalyst/generated/pointer"
|
||||
)
|
||||
|
||||
func TestBackupAndRestore(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||
|
||||
if runtime.GOARCH == "arm64" {
|
||||
t.Skip("test does not run on arm")
|
||||
}
|
||||
|
||||
type want struct {
|
||||
status int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
want want
|
||||
}{
|
||||
{name: "Backup", want: want{status: http.StatusOK}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
tt := tt
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, _, server, err := Catalyst(t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := SetupTestData(ctx, server.DB); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
createFile(ctx, server)
|
||||
|
||||
zipB := assertBackup(t, server)
|
||||
|
||||
assertZipFile(t, readZipFile(t, zipB))
|
||||
|
||||
clearAllDatabases(server)
|
||||
_, err = server.DB.UserCreateSetupAPIKey(ctx, "test")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
deleteAllBuckets(t, server)
|
||||
|
||||
assertRestore(t, zipB, server)
|
||||
|
||||
assertTicketExists(t, server)
|
||||
|
||||
assertFileExists(t, server)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func assertBackup(t *testing.T, server *catalyst.Server) []byte {
|
||||
t.Helper()
|
||||
|
||||
// setup request
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/backup/create", nil)
|
||||
req.Header.Set("PRIVATE-TOKEN", "test")
|
||||
|
||||
// run request
|
||||
backupRequestRecorder := httptest.NewRecorder()
|
||||
server.Server.ServeHTTP(backupRequestRecorder, req)
|
||||
backupResult := backupRequestRecorder.Result()
|
||||
|
||||
// assert results
|
||||
assert.Equal(t, http.StatusOK, backupResult.StatusCode)
|
||||
|
||||
zipBuf := &bytes.Buffer{}
|
||||
if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assert.NoError(t, backupResult.Body.Close())
|
||||
|
||||
return zipBuf.Bytes()
|
||||
}
|
||||
|
||||
func assertZipFile(t *testing.T, r *zip.Reader) {
|
||||
t.Helper()
|
||||
|
||||
var names []string
|
||||
for _, f := range r.File {
|
||||
names = append(names, f.Name)
|
||||
}
|
||||
|
||||
if !includes(t, names, "minio/catalyst-8125/test.txt") {
|
||||
t.Error("Minio file missing")
|
||||
}
|
||||
|
||||
for _, p := range []string{
|
||||
"arango/ENCRYPTION", "arango/automations_.*.data.json.gz", "arango/automations_.*.structure.json", "arango/dump.json", "arango/jobs_.*.data.json.gz", "arango/jobs_.*.structure.json", "arango/logs_.*.data.json.gz", "arango/logs_.*.structure.json", "arango/migrations_.*.data.json.gz", "arango/migrations_.*.structure.json", "arango/playbooks_.*.data.json.gz", "arango/playbooks_.*.structure.json", "arango/related_.*.data.json.gz", "arango/related_.*.structure.json", "arango/templates_.*.data.json.gz", "arango/templates_.*.structure.json", "arango/tickets_.*.data.json.gz", "arango/tickets_.*.structure.json", "arango/tickettypes_.*.data.json.gz", "arango/tickettypes_.*.structure.json", "arango/userdata_.*.data.json.gz", "arango/userdata_.*.structure.json", "arango/users_.*.data.json.gz", "arango/users_.*.structure.json",
|
||||
} {
|
||||
if !includes(t, names, p) {
|
||||
t.Errorf("Arango file missing: %s", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func clearAllDatabases(server *catalyst.Server) {
|
||||
server.DB.Truncate(context.Background())
|
||||
}
|
||||
|
||||
func deleteAllBuckets(t *testing.T, server *catalyst.Server) {
|
||||
t.Helper()
|
||||
|
||||
buckets, err := server.Storage.S3().ListBuckets(&s3.ListBucketsInput{})
|
||||
for _, bucket := range buckets.Buckets {
|
||||
_, _ = server.Storage.S3().DeleteBucket(&s3.DeleteBucketInput{
|
||||
Bucket: bucket.Name,
|
||||
})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertRestore(t *testing.T, zipB []byte, server *catalyst.Server) {
|
||||
t.Helper()
|
||||
|
||||
bodyBuf := &bytes.Buffer{}
|
||||
bodyWriter := multipart.NewWriter(bodyBuf)
|
||||
fileWriter, err := bodyWriter.CreateFormFile("backup", "backup.zip")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = fileWriter.Write(zipB)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
assert.NoError(t, bodyWriter.Close())
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/backup/restore", bodyBuf)
|
||||
req.Header.Set("PRIVATE-TOKEN", "test")
|
||||
req.Header.Set("Content-Type", bodyWriter.FormDataContentType())
|
||||
|
||||
// run request
|
||||
restoreRequestRecorder := httptest.NewRecorder()
|
||||
server.Server.ServeHTTP(restoreRequestRecorder, req)
|
||||
restoreResult := restoreRequestRecorder.Result()
|
||||
|
||||
if !assert.Equal(t, http.StatusOK, restoreResult.StatusCode) {
|
||||
b, _ := io.ReadAll(restoreResult.Body)
|
||||
log.Println(string(b))
|
||||
t.FailNow()
|
||||
}
|
||||
}
|
||||
|
||||
func createFile(ctx context.Context, server *catalyst.Server) {
|
||||
buf := bytes.NewBufferString("test text")
|
||||
|
||||
_, _ = server.Storage.S3().CreateBucket(&s3.CreateBucketInput{Bucket: pointer.String("catalyst-8125")})
|
||||
|
||||
if _, err := server.Storage.Uploader().Upload(&s3manager.UploadInput{Body: buf, Bucket: pointer.String("catalyst-8125"), Key: pointer.String("test.txt")}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := server.DB.AddFile(ctx, 8125, &model.File{Key: "test.txt", Name: "test.txt"}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertTicketExists(t *testing.T, server *catalyst.Server) {
|
||||
t.Helper()
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/tickets/8125", nil)
|
||||
req.Header.Set("PRIVATE-TOKEN", "test")
|
||||
|
||||
// run request
|
||||
backupRequestRecorder := httptest.NewRecorder()
|
||||
server.Server.ServeHTTP(backupRequestRecorder, req)
|
||||
backupResult := backupRequestRecorder.Result()
|
||||
|
||||
// assert results
|
||||
assert.Equal(t, http.StatusOK, backupResult.StatusCode)
|
||||
|
||||
zipBuf := &bytes.Buffer{}
|
||||
if _, err := io.Copy(zipBuf, backupResult.Body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assert.NoError(t, backupResult.Body.Close())
|
||||
|
||||
var ticket model.Ticket
|
||||
assert.NoError(t, json.Unmarshal(zipBuf.Bytes(), &ticket))
|
||||
|
||||
assert.Equal(t, "phishing from selenafadel@von.com detected", ticket.Name)
|
||||
}
|
||||
|
||||
func assertFileExists(t *testing.T, server *catalyst.Server) {
|
||||
t.Helper()
|
||||
|
||||
obj, err := server.Storage.S3().GetObject(&s3.GetObjectInput{
|
||||
Bucket: aws.String("catalyst-8125"),
|
||||
Key: aws.String("test.txt"),
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
b, err := io.ReadAll(obj.Body)
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.Equal(t, "test text", string(b))
|
||||
}
|
||||
|
||||
func includes(t *testing.T, names []string, s string) bool {
|
||||
t.Helper()
|
||||
|
||||
for _, name := range names {
|
||||
match, err := regexp.MatchString(s, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if match {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func readZipFile(t *testing.T, b []byte) *zip.Reader {
|
||||
t.Helper()
|
||||
|
||||
buf := bytes.NewReader(b)
|
||||
|
||||
zr, err := zip.NewReader(buf, int64(buf.Len()))
|
||||
if err != nil {
|
||||
t.Fatal(string(b), err)
|
||||
}
|
||||
|
||||
return zr
|
||||
}
|
||||
Reference in New Issue
Block a user