CSEC_PUBLIC/catalyst
1
0
Fork 0
mirror of https://github.com/SecurityBrewery/catalyst.git synced 2025-12-06 23:32:47 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add test for jobs (#34)

Browse Source
This commit is contained in:
Jonas Plum
2022-02-27 18:33:50 +01:00
committed by GitHub
parent fd18458f3d
commit ffba7b4f5f
18 changed files with 181 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
bus/request.go
View File

@@ -26,11 +26,11 @@ func (b *Bus) PublishRequest(user, f string, ids []driver.DocumentID) error {
func (b *Bus) SubscribeRequest(f func(msg *RequestMsg)) error {
return b.safeSubscribe(b.config.requestKey, ChannelRequest, func(c *emitter.Client, m emitter.Message) {
var msg RequestMsg
if err := json.Unmarshal(m.Payload(), &msg); err != nil {
msg := &RequestMsg{}
if err := json.Unmarshal(m.Payload(), msg); err != nil {
log.Println(err)
return
}
go f(&msg)
go f(msg)
})
}

6
bus/result.go
View File

@@ -23,11 +23,11 @@ func (b *Bus) PublishResult(automation string, data map[string]interface{}, targ
func (b *Bus) SubscribeResult(f func(msg *ResultMsg)) error {
return b.safeSubscribe(b.config.resultBusKey, channelResult, func(c *emitter.Client, m emitter.Message) {
var msg ResultMsg
if err := json.Unmarshal(m.Payload(), &msg); err != nil {
msg := &ResultMsg{}
if err := json.Unmarshal(m.Payload(), msg); err != nil {
log.Println(err)
return
}
go f(&msg)
go f(msg)
})
}

2
database/artifact.go
View File

@@ -78,7 +78,7 @@ func (db *Database) EnrichArtifact(ctx context.Context, id int64, name string, e
query := `LET d = DOCUMENT(@@collection, @ID)
` + ticketFilterQuery + `
FOR a IN d.artifacts
FOR a IN NOT_NULL(d.artifacts, [])
FILTER a.name == @name
LET enrichments = NOT_NULL(a.enrichments, {})
LET newenrichments = MERGE(enrichments, ZIP( [@enrichmentname], [@enrichment]) )

18
definition/jobs.yaml
View File

@@ -13,7 +13,7 @@ paths:
schema: { type: array, items: { $ref: "#/definitions/JobResponse" } }
examples:
test:
- id: "99cd67131b48"
- id: "b81c2366-ea37-43d2-b61b-03afdc21d985"
automation: "hash.sha1"
payload: "test"
status: "created"
@@ -23,9 +23,13 @@ paths:
summary: "Start a new job"
operationId: "runJob"
parameters:
- { name: "job", in: "body", description: "New job", required: true, schema: { $ref: "#/definitions/JobForm" }, x-example: { automation: "hash.sha1", message: { payload: "test" } } }
- { name: "job", in: "body", description: "New job", required: true, schema: { $ref: "#/definitions/JobForm" }, x-example: { automation: "hash.sha1", payload: "test" } }
responses:
"204": { description: "successful operation" }
"200":
description: "successful operation"
schema: { $ref: "#/definitions/JobResponse" }
examples:
test: { id: "87390749-2125-4a87-91c5-da7e3f9bebf1", automation: "hash.sha1", payload: "test", status: "created" }
security: [ { roles: [ "job:write" ] } ]
/jobs/{id}:
@@ -34,27 +38,27 @@ paths:
summary: "Get a single job"
operationId: "getJob"
parameters:
- { name: "id", in: "path", description: "Job ID", required: true, type: string, x-example: "99cd67131b48" }
- { name: "id", in: "path", description: "Job ID", required: true, type: string, x-example: "b81c2366-ea37-43d2-b61b-03afdc21d985" }
responses:
"200":
description: "successful operation"
schema: { $ref: "#/definitions/JobResponse" }
examples:
test: { id: "99cd67131b48", automation: "hash.sha1", payload: "test", status: "created" }
test: { id: "b81c2366-ea37-43d2-b61b-03afdc21d985", automation: "hash.sha1", payload: "test", status: "created" }
security: [ { roles: [ "job:read" ] } ]
put:
tags: [ "jobs" ]
summary: "Update an existing job"
operationId: "updateJob"
parameters:
- { name: "id", in: "path", description: "Job ID", required: true, type: string, x-example: "99cd67131b48" }
- { name: "id", in: "path", description: "Job ID", required: true, type: string, x-example: "b81c2366-ea37-43d2-b61b-03afdc21d985" }
- { name: "job", in: "body", description: "Job object that needs to be added", required: true, schema: { $ref: "#/definitions/JobUpdate" }, x-example: { status: "failed", running: false } }
responses:
"200":
description: "successful operation"
schema: { $ref: "#/definitions/JobResponse" }
examples:
test: { id: "99cd67131b48", automation: "hash.sha1", payload: "test", status: "failed" }
test: { id: "b81c2366-ea37-43d2-b61b-03afdc21d985", automation: "hash.sha1", payload: "test", status: "failed" }
security: [ { roles: [ "job:write" ] } ]

1
dev/docker-compose.yml
View File

@@ -38,7 +38,6 @@ services:
keycloak:
image: quay.io/keycloak/keycloak:14.0.0
platform: linux/amd64
environment:
DB_VENDOR: POSTGRES
DB_ADDR: postgres

5
generated/api/api.go
View File

@@ -39,7 +39,7 @@ type Service interface {
CurrentUserData(context.Context) (*model.UserDataResponse, error)
UpdateCurrentUserData(context.Context, *model.UserData) (*model.UserDataResponse, error)
ListJobs(context.Context) ([]*model.JobResponse, error)
RunJob(context.Context, *model.JobForm) error
RunJob(context.Context, *model.JobForm) (*model.JobResponse, error)
GetJob(context.Context, string) (*model.JobResponse, error)
UpdateJob(context.Context, string, *model.JobUpdate) (*model.JobResponse, error)
GetLogs(context.Context, string) ([]*model.LogEntry, error)
@@ -346,7 +346,8 @@ func (s *server) runJobHandler(w http.ResponseWriter, r *http.Request) {
return
}
response(w, nil, s.service.RunJob(r.Context(), jobP))
result, err := s.service.RunJob(r.Context(), jobP)
response(w, result, err)
}
func (s *server) getJobHandler(w http.ResponseWriter, r *http.Request) {

16
generated/api/test_api.go
View File

@@ -95,34 +95,34 @@ var Tests = []struct {
Args: Args{Method: "Get", URL: "/jobs"},
Want: Want{
Status: 200,
Body: []interface{}{map[string]interface{}{"automation": "hash.sha1", "id": "99cd67131b48", "payload": "test", "status": "created"}},
Body: []interface{}{map[string]interface{}{"automation": "hash.sha1", "id": "b81c2366-ea37-43d2-b61b-03afdc21d985", "payload": "test", "status": "created"}},
},
},
{
Name: "RunJob",
Args: Args{Method: "Post", URL: "/jobs", Data: map[string]interface{}{"automation": "hash.sha1", "message": map[string]interface{}{"payload": "test"}}},
Args: Args{Method: "Post", URL: "/jobs", Data: map[string]interface{}{"automation": "hash.sha1", "payload": "test"}},
Want: Want{
Status: 204,
Body: nil,
Status: 200,
Body: map[string]interface{}{"automation": "hash.sha1", "id": "87390749-2125-4a87-91c5-da7e3f9bebf1", "payload": "test", "status": "created"},
},
},
{
Name: "GetJob",
Args: Args{Method: "Get", URL: "/jobs/99cd67131b48"},
Args: Args{Method: "Get", URL: "/jobs/b81c2366-ea37-43d2-b61b-03afdc21d985"},
Want: Want{
Status: 200,
Body: map[string]interface{}{"automation": "hash.sha1", "id": "99cd67131b48", "payload": "test", "status": "created"},
Body: map[string]interface{}{"automation": "hash.sha1", "id": "b81c2366-ea37-43d2-b61b-03afdc21d985", "payload": "test", "status": "created"},
},
},
{
Name: "UpdateJob",
Args: Args{Method: "Put", URL: "/jobs/99cd67131b48", Data: map[string]interface{}{"running": false, "status": "failed"}},
Args: Args{Method: "Put", URL: "/jobs/b81c2366-ea37-43d2-b61b-03afdc21d985", Data: map[string]interface{}{"running": false, "status": "failed"}},
Want: Want{
Status: 200,
Body: map[string]interface{}{"automation": "hash.sha1", "id": "99cd67131b48", "payload": "test", "status": "failed"},
Body: map[string]interface{}{"automation": "hash.sha1", "id": "b81c2366-ea37-43d2-b61b-03afdc21d985", "payload": "test", "status": "failed"},
},
},

28
generated/catalyst.json
View File

@@ -556,7 +556,7 @@
"test" : {
"example" : [ {
"automation" : "hash.sha1",
"id" : "99cd67131b48",
"id" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"payload" : "test",
"status" : "created"
} ]
@@ -585,8 +585,22 @@
"required" : true
},
"responses" : {
"204" : {
"content" : { },
"200" : {
"content" : {
"application/json" : {
"schema" : {
"$ref" : "#/components/schemas/JobResponse"
}
},
"test" : {
"example" : {
"automation" : "hash.sha1",
"id" : "87390749-2125-4a87-91c5-da7e3f9bebf1",
"payload" : "test",
"status" : "created"
}
}
},
"description" : "successful operation"
}
},
@@ -603,7 +617,7 @@
"operationId" : "getJob",
"parameters" : [ {
"description" : "Job ID",
"example" : "99cd67131b48",
"example" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"in" : "path",
"name" : "id",
"required" : true,
@@ -622,7 +636,7 @@
"test" : {
"example" : {
"automation" : "hash.sha1",
"id" : "99cd67131b48",
"id" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"payload" : "test",
"status" : "created"
}
@@ -641,7 +655,7 @@
"operationId" : "updateJob",
"parameters" : [ {
"description" : "Job ID",
"example" : "99cd67131b48",
"example" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"in" : "path",
"name" : "id",
"required" : true,
@@ -671,7 +685,7 @@
"test" : {
"example" : {
"automation" : "hash.sha1",
"id" : "99cd67131b48",
"id" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"payload" : "test",
"status" : "failed"
}

23
generated/catalyst.yml
View File

@@ -1791,7 +1791,7 @@ paths:
examples:
test:
- automation: hash.sha1
id: 99cd67131b48
id: b81c2366-ea37-43d2-b61b-03afdc21d985
payload: test
status: created
schema:
@@ -1815,11 +1815,18 @@ paths:
$ref: '#/definitions/JobForm'
x-example:
automation: hash.sha1
message:
payload: test
payload: test
responses:
"204":
"200":
description: successful operation
examples:
test:
automation: hash.sha1
id: 87390749-2125-4a87-91c5-da7e3f9bebf1
payload: test
status: created
schema:
$ref: '#/definitions/JobResponse'
security:
- roles:
- job:write
@@ -1835,14 +1842,14 @@ paths:
name: id
required: true
type: string
x-example: 99cd67131b48
x-example: b81c2366-ea37-43d2-b61b-03afdc21d985
responses:
"200":
description: successful operation
examples:
test:
automation: hash.sha1
id: 99cd67131b48
id: b81c2366-ea37-43d2-b61b-03afdc21d985
payload: test
status: created
schema:
@@ -1861,7 +1868,7 @@ paths:
name: id
required: true
type: string
x-example: 99cd67131b48
x-example: b81c2366-ea37-43d2-b61b-03afdc21d985
- description: Job object that needs to be added
in: body
name: job
@@ -1877,7 +1884,7 @@ paths:
examples:
test:
automation: hash.sha1
id: 99cd67131b48
id: b81c2366-ea37-43d2-b61b-03afdc21d985
payload: test
status: failed
schema:

28
generated/community.json
View File

@@ -324,7 +324,7 @@
"test" : {
"example" : [ {
"automation" : "hash.sha1",
"id" : "99cd67131b48",
"id" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"payload" : "test",
"status" : "created"
} ]
@@ -353,8 +353,22 @@
"required" : true
},
"responses" : {
"204" : {
"content" : { },
"200" : {
"content" : {
"application/json" : {
"schema" : {
"$ref" : "#/components/schemas/JobResponse"
}
},
"test" : {
"example" : {
"automation" : "hash.sha1",
"id" : "87390749-2125-4a87-91c5-da7e3f9bebf1",
"payload" : "test",
"status" : "created"
}
}
},
"description" : "successful operation"
}
},
@@ -371,7 +385,7 @@
"operationId" : "getJob",
"parameters" : [ {
"description" : "Job ID",
"example" : "99cd67131b48",
"example" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"in" : "path",
"name" : "id",
"required" : true,
@@ -390,7 +404,7 @@
"test" : {
"example" : {
"automation" : "hash.sha1",
"id" : "99cd67131b48",
"id" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"payload" : "test",
"status" : "created"
}
@@ -409,7 +423,7 @@
"operationId" : "updateJob",
"parameters" : [ {
"description" : "Job ID",
"example" : "99cd67131b48",
"example" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"in" : "path",
"name" : "id",
"required" : true,
@@ -439,7 +453,7 @@
"test" : {
"example" : {
"automation" : "hash.sha1",
"id" : "99cd67131b48",
"id" : "b81c2366-ea37-43d2-b61b-03afdc21d985",
"payload" : "test",
"status" : "failed"
}

23
generated/community.yml
View File

@@ -1520,7 +1520,7 @@ paths:
examples:
test:
- automation: hash.sha1
id: 99cd67131b48
id: b81c2366-ea37-43d2-b61b-03afdc21d985
payload: test
status: created
schema:
@@ -1544,11 +1544,18 @@ paths:
$ref: '#/definitions/JobForm'
x-example:
automation: hash.sha1
message:
payload: test
payload: test
responses:
"204":
"200":
description: successful operation
examples:
test:
automation: hash.sha1
id: 87390749-2125-4a87-91c5-da7e3f9bebf1
payload: test
status: created
schema:
$ref: '#/definitions/JobResponse'
security:
- roles:
- job:write
@@ -1564,14 +1571,14 @@ paths:
name: id
required: true
type: string
x-example: 99cd67131b48
x-example: b81c2366-ea37-43d2-b61b-03afdc21d985
responses:
"200":
description: successful operation
examples:
test:
automation: hash.sha1
id: 99cd67131b48
id: b81c2366-ea37-43d2-b61b-03afdc21d985
payload: test
status: created
schema:
@@ -1590,7 +1597,7 @@ paths:
name: id
required: true
type: string
x-example: 99cd67131b48
x-example: b81c2366-ea37-43d2-b61b-03afdc21d985
- description: Job object that needs to be added
in: body
name: job
@@ -1606,7 +1613,7 @@ paths:
examples:
test:
automation: hash.sha1
id: 99cd67131b48
id: b81c2366-ea37-43d2-b61b-03afdc21d985
payload: test
status: failed
schema:

12
service/job.go
View File

@@ -26,12 +26,20 @@ func (s *Service) ListJobs(ctx context.Context) ([]*model.JobResponse, error) {
return s.database.JobList(ctx)
}
func (s *Service) RunJob(ctx context.Context, form *model.JobForm) (err error) {
func (s *Service) RunJob(ctx context.Context, form *model.JobForm) (doc *model.JobResponse, err error) {
msgContext := &model.Context{}
newJobID := uuid.NewString()
defer s.publishRequest(ctx, err, "RunJob", jobID(newJobID))
return s.bus.PublishJob(newJobID, form.Automation, form.Payload, msgContext, form.Origin)
err = s.bus.PublishJob(newJobID, form.Automation, form.Payload, msgContext, form.Origin)
return &model.JobResponse{
Automation: form.Automation,
ID: newJobID,
Origin: form.Origin,
Payload: form.Payload,
Status: "published",
}, err
}
func (s *Service) GetJob(ctx context.Context, id string) (*model.JobResponse, error) {

5
test/backup_test.go
View File

@@ -11,6 +11,7 @@ import (
"net/http"
"net/http/httptest"
"regexp"
"runtime"
"testing"
"github.com/aws/aws-sdk-go/aws"
@@ -26,6 +27,10 @@ import (
func TestBackupAndRestore(t *testing.T) {
log.SetFlags(log.LstdFlags | log.Lshortfile)
if runtime.GOARCH == "arm64" {
t.Skip("test does not run on arm")
}
type want struct {
status int
}

65
test/job_test.go
View File

@@ -3,47 +3,80 @@ package test
import (
"bytes"
"encoding/json"
"io"
"log"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/go-chi/chi"
"github.com/stretchr/testify/assert"
"github.com/tidwall/gjson"
"github.com/SecurityBrewery/catalyst/generated/model"
)
func TestJob(t *testing.T) {
_, _, _, _, _, _, _, server, cleanup, err := Server(t)
log.SetFlags(log.LstdFlags | log.Lshortfile)
_, _, server, err := Catalyst(t)
if err != nil {
t.Fatal(err)
}
defer cleanup()
// server.ConfigureRoutes()
w := httptest.NewRecorder()
// setup request
var req *http.Request
b, err := json.Marshal(model.JobForm{
Automation: "hash.sha1",
Origin: nil,
Payload: nil,
Payload: map[string]interface{}{"default": "test"},
})
if err != nil {
t.Fatal(err)
}
result := request(t, server.Server, http.MethodPost, "/api/jobs", bytes.NewBuffer(b))
id := gjson.GetBytes(result, "id").String()
req = httptest.NewRequest(http.MethodPost, "/jobs", bytes.NewBuffer(b))
start := time.Now()
for {
time.Sleep(5 * time.Second)
if time.Since(start) > time.Minute {
t.Fatal("job did not complete within a minute")
}
job := request(t, server.Server, http.MethodGet, "/api/jobs/"+id, nil)
status := gjson.GetBytes(job, "status").String()
if status != "completed" {
continue
}
output := gjson.GetBytes(job, "output.hash").String()
assert.Equal(t, "a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", output)
break
}
}
func request(t *testing.T, server chi.Router, method, url string, data io.Reader) []byte {
w := httptest.NewRecorder()
// setup request
req := httptest.NewRequest(method, url, data)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("PRIVATE-TOKEN", "test")
// run request
server.ServeHTTP(w, req)
result := w.Result()
// assert results
if result.StatusCode != http.StatusNoContent {
t.Fatalf("Status got = %v, want %v", result.Status, http.StatusNoContent)
b, err := io.ReadAll(result.Body)
if err != nil {
t.Fatal(err)
}
// if tt.want.status != http.StatusNoContent {
// jsonEqual(t, result.Body, tt.want.body)
// }
if result.StatusCode != http.StatusOK {
t.Fatalf("Status got = %v: %v, want %v", result.Status, string(b), http.StatusOK)
}
return b
}

14
test/server_test.go
View File

@@ -67,13 +67,13 @@ func TestServer(t *testing.T) {
t.Fatalf("Status got = %v (%s), want %v", result.Status, msg, tt.Want.Status)
}
if tt.Want.Status != http.StatusNoContent {
jsonEqual(t, result.Body, tt.Want.Body)
jsonEqual(t, tt.Name, result.Body, tt.Want.Body)
}
})
}
}
func jsonEqual(t *testing.T, got io.Reader, want interface{}) {
func jsonEqual(t *testing.T, name string, got io.Reader, want interface{}) {
var gotObject, wantObject interface{}
// load bytes
@@ -86,7 +86,15 @@ func jsonEqual(t *testing.T, got io.Reader, want interface{}) {
t.Fatal(err)
}
fields := []string{"secret"}
var fields []string
if name == "CreateUser" {
fields = append(fields, "secret")
}
if name == "RunJob" {
fields = append(fields, "id", "status")
}
for _, field := range fields {
gField := gjson.GetBytes(wantBytes, field)
if gField.Exists() && gjson.GetBytes(gotBytes, field).Exists() {

3
test/test.go
View File

@@ -144,7 +144,7 @@ func DB(t *testing.T) (context.Context, *catalyst.Config, *bus.Bus, *index.Index
return nil, nil, nil, nil, nil, nil, nil, err
}
_, err = db.JobCreate(ctx, "99cd67131b48", &model.JobForm{
_, err = db.JobCreate(ctx, "b81c2366-ea37-43d2-b61b-03afdc21d985", &model.JobForm{
Automation: "hash.sha1",
Payload: "test",
Origin: nil,
@@ -201,6 +201,7 @@ func Catalyst(t *testing.T) (context.Context, *catalyst.Config, *catalyst.Server
t.Fatal(err)
}
config.DB.Name = cleanName(t)
config.IndexPath = cleanName(t) + ".bleve"
c, err := catalyst.New(&hooks.Hooks{
DatabaseAfterConnectFuncs: []func(ctx context.Context, client driver.Client, name string){Clear},

2
test/user_test.go
View File

@@ -61,7 +61,7 @@ func TestUser(t *testing.T) {
t.Fatalf("Status got = %v, want %v", result.Status, tt.want.status)
}
if tt.want.status != http.StatusNoContent {
jsonEqual(t, result.Body, tt.want.body)
jsonEqual(t, tt.name, result.Body, tt.want.body)
}
})
}

4
ui/src/client/api.ts
View File

@@ -3303,7 +3303,7 @@ export const JobsApiFp = function(configuration?: Configuration) {
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
async runJob(job: JobForm, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<void>> {
async runJob(job: JobForm, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<JobResponse>> {
const localVarAxiosArgs = await localVarAxiosParamCreator.runJob(job, options);
return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
},
@@ -3355,7 +3355,7 @@ export const JobsApiFactory = function (configuration?: Configuration, basePath?
* @param {*} [options] Override http request option.
* @throws {RequiredError}
*/
runJob(job: JobForm, options?: any): AxiosPromise<void> {
runJob(job: JobForm, options?: any): AxiosPromise<JobResponse> {
return localVarFp.runJob(job, options).then((request) => request(axios, basePath));
},
/**