package database import ( "context" "encoding/json" "errors" "fmt" "sort" "strconv" "strings" "sync" "github.com/arangodb/go-driver" "github.com/xeipuuv/gojsonschema" "github.com/SecurityBrewery/catalyst/bus" "github.com/SecurityBrewery/catalyst/caql" "github.com/SecurityBrewery/catalyst/database/busdb" "github.com/SecurityBrewery/catalyst/generated/model" "github.com/SecurityBrewery/catalyst/generated/time" "github.com/SecurityBrewery/catalyst/index" ) func toTicket(ticketForm *model.TicketForm) (interface{}, error) { playbooks, err := toPlaybooks(ticketForm.Playbooks) if err != nil { return nil, err } ticket := &model.Ticket{ Artifacts: ticketForm.Artifacts, Comments: ticketForm.Comments, Details: ticketForm.Details, Files: ticketForm.Files, Name: ticketForm.Name, Owner: ticketForm.Owner, Playbooks: playbooks, Read: ticketForm.Read, References: ticketForm.References, Status: ticketForm.Status, Type: ticketForm.Type, Write: ticketForm.Write, // ID: ticketForm.ID, // Created: ticketForm.Created, // Modified: ticketForm.Modified, // Schema: ticketForm.Schema, } if ticketForm.Created != nil { ticket.Created = *ticketForm.Created } else { ticket.Created = time.Now().UTC() } if ticketForm.Modified != nil { ticket.Modified = *ticketForm.Modified } else { ticket.Modified = time.Now().UTC() } if ticketForm.Schema != nil { ticket.Schema = *ticketForm.Schema } else { ticket.Schema = "{}" } if ticketForm.Status == "" { ticket.Status = "open" } if ticketForm.ID != nil { return &busdb.Keyed{Key: strconv.FormatInt(*ticketForm.ID, 10), Doc: ticket}, nil } return ticket, nil } func toTicketResponses(tickets []*model.TicketSimpleResponse) ([]*model.TicketResponse, error) { var extendedTickets []*model.TicketResponse for _, simple := range tickets { tr, err := toTicketResponse(simple) if err != nil { return nil, err } extendedTickets = append(extendedTickets, tr) } return extendedTickets, nil } func toTicketResponse(ticket *model.TicketSimpleResponse) (*model.TicketResponse, error) { playbooks, err := toPlaybookResponses(ticket.Playbooks) if err != nil { return nil, err } return &model.TicketResponse{ ID: ticket.ID, Artifacts: ticket.Artifacts, Comments: ticket.Comments, Created: ticket.Created, Details: ticket.Details, Files: ticket.Files, Modified: ticket.Modified, Name: ticket.Name, Owner: ticket.Owner, Playbooks: playbooks, Read: ticket.Read, References: ticket.References, Schema: ticket.Schema, Status: ticket.Status, Type: ticket.Type, Write: ticket.Write, }, nil } func toTicketSimpleResponse(key string, ticket *model.Ticket) (*model.TicketSimpleResponse, error) { id, err := strconv.ParseInt(key, 10, 64) if err != nil { return nil, err } return &model.TicketSimpleResponse{ Artifacts: ticket.Artifacts, Comments: ticket.Comments, Created: ticket.Created, Details: ticket.Details, Files: ticket.Files, ID: id, Modified: ticket.Modified, Name: ticket.Name, Owner: ticket.Owner, Playbooks: ticket.Playbooks, Read: ticket.Read, References: ticket.References, Schema: ticket.Schema, Status: ticket.Status, Type: ticket.Type, Write: ticket.Write, }, nil } func toTicketWithTickets(ticketResponse *model.TicketResponse, tickets []*model.TicketSimpleResponse, logs []*model.LogEntry) *model.TicketWithTickets { return &model.TicketWithTickets{ Artifacts: ticketResponse.Artifacts, Comments: ticketResponse.Comments, Created: ticketResponse.Created, Details: ticketResponse.Details, Files: ticketResponse.Files, ID: ticketResponse.ID, Modified: ticketResponse.Modified, Name: ticketResponse.Name, Owner: ticketResponse.Owner, Playbooks: ticketResponse.Playbooks, Read: ticketResponse.Read, References: ticketResponse.References, Schema: ticketResponse.Schema, Status: ticketResponse.Status, Type: ticketResponse.Type, Write: ticketResponse.Write, Logs: logs, Tickets: tickets, } } func toPlaybookResponses(playbooks map[string]*model.Playbook) (map[string]*model.PlaybookResponse, error) { pr := map[string]*model.PlaybookResponse{} var err error for k, v := range playbooks { pr[k], err = toPlaybookResponse(v) if err != nil { return nil, err } } return pr, nil } func toPlaybookResponse(playbook *model.Playbook) (*model.PlaybookResponse, error) { graph, err := playbookGraph(playbook) if err != nil { return nil, err } re := &model.PlaybookResponse{ Name: playbook.Name, Tasks: map[string]*model.TaskResponse{}, } results, err := graph.Toposort() if err != nil { return nil, err } i := 0 for _, taskID := range results { rootTask, err := toTaskResponse(playbook, taskID, i, graph) if err != nil { return nil, err } re.Tasks[taskID] = rootTask i++ } return re, nil } func (db *Database) TicketBatchCreate(ctx context.Context, ticketForms []*model.TicketForm) ([]*model.TicketResponse, error) { update, err := db.Hooks.IngestionFilter(ctx, db.Index) if err != nil { return nil, err } var dbTickets []interface{} for _, ticketForm := range ticketForms { ticket, err := toTicket(ticketForm) if err != nil { return nil, err } if err := validate(ticket, model.TicketSchema); err != nil { return nil, err } dbTickets = append(dbTickets, ticket) } ticketFilterQuery, ticketFilterVars, err := db.Hooks.TicketWriteFilter(ctx) if err != nil { return nil, err } query := `FOR d IN @tickets ` + ticketFilterQuery + ` LET updates = ` + update + ` LET newdoc = LENGTH(updates) != 0 ? APPLY("MERGE_RECURSIVE", APPEND([d], updates)) : d LET keyeddoc = HAS(newdoc, "id") ? MERGE(newdoc, {"_key": TO_STRING(newdoc.id)}) : newdoc LET noiddoc = UNSET(keyeddoc, "id") INSERT noiddoc INTO @@collection RETURN NEW` apiTickets, _, err := db.ticketListQuery(ctx, query, mergeMaps(map[string]interface{}{ "tickets": dbTickets, }, ticketFilterVars), busdb.CreateOperation) if err != nil { return nil, err } if err = batchIndex(db.Index, apiTickets); err != nil { return nil, err } var ids []driver.DocumentID for _, apiTicket := range apiTickets { ids = append(ids, driver.NewDocumentID(TicketCollectionName, fmt.Sprint(apiTicket.ID))) } go db.bus.PublishDatabaseUpdate(ids, bus.DatabaseEntryUpdated) ticketResponses, err := toTicketResponses(apiTickets) if err != nil { return nil, err } for _, ticketResponse := range ticketResponses { for playbookID := range ticketResponse.Playbooks { if err := runRootTask(ticketResponse, playbookID, db); err != nil { return nil, err } } } return ticketResponses, nil } func (db *Database) IndexRebuild(ctx context.Context) error { if err := db.Index.Truncate(); err != nil { return err } tickets, _, err := db.ticketListQuery(ctx, "FOR d IN @@collection RETURN d", nil, busdb.ReadOperation) if err != nil { return err } return batchIndex(db.Index, tickets) } func batchIndex(index *index.Index, tickets []*model.TicketSimpleResponse) error { var wg sync.WaitGroup var batch []*model.TicketSimpleResponse for _, ticket := range tickets { batch = append(batch, ticket) if len(batch) > 100 { wg.Add(1) go func(docs []*model.TicketSimpleResponse) { index.Index(docs) wg.Done() }(batch) batch = []*model.TicketSimpleResponse{} } } wg.Wait() return nil } func (db *Database) TicketGet(ctx context.Context, ticketID int64) (*model.TicketWithTickets, error) { ticketFilterQuery, ticketFilterVars, err := db.Hooks.TicketReadFilter(ctx) if err != nil { return nil, err } return db.ticketGetQuery(ctx, ticketID, `LET d = DOCUMENT(@@collection, @ID) `+ticketFilterQuery+` RETURN d`, ticketFilterVars, busdb.ReadOperation) } func (db *Database) ticketGetQuery(ctx context.Context, ticketID int64, query string, bindVars map[string]interface{}, operation *busdb.Operation) (*model.TicketWithTickets, error) { if bindVars == nil { bindVars = map[string]interface{}{} } bindVars["@collection"] = TicketCollectionName if ticketID != 0 { bindVars["ID"] = fmt.Sprint(ticketID) } cur, _, err := db.Query(ctx, query, bindVars, operation) if err != nil { return nil, err } defer cur.Close() ticket := model.Ticket{} meta, err := cur.ReadDocument(ctx, &ticket) if err != nil { return nil, err } ticketSimpleResponse, err := toTicketSimpleResponse(meta.Key, &ticket) if err != nil { return nil, err } // index go db.Index.Index([]*model.TicketSimpleResponse{ticketSimpleResponse}) ticketFilterQuery, ticketFilterVars, err := db.Hooks.TicketReadFilter(ctx) if err != nil { return nil, err } // tickets ticketsQuery := `FOR vertex, edge IN OUTBOUND DOCUMENT(@@tickets, @ID) GRAPH @graph FILTER IS_SAME_COLLECTION(@@collection, vertex) FILTER vertex != null LET d = DOCUMENT(@@collection, edge["_to"]) ` + ticketFilterQuery + ` RETURN d` outTickets, _, err := db.ticketListQuery(ctx, ticketsQuery, mergeMaps(map[string]interface{}{ "ID": fmt.Sprint(ticketID), "graph": TicketArtifactsGraphName, "@tickets": TicketCollectionName, }, ticketFilterVars), busdb.ReadOperation) if err != nil { return nil, err } ticketsQuery = `FOR vertex, edge IN INBOUND DOCUMENT(@@tickets, @ID) GRAPH @graph FILTER IS_SAME_COLLECTION(@@collection, vertex) FILTER vertex != null LET d = DOCUMENT(@@collection, edge["_from"]) ` + ticketFilterQuery + ` RETURN d` inTickets, _, err := db.ticketListQuery(ctx, ticketsQuery, mergeMaps(map[string]interface{}{ "ID": fmt.Sprint(ticketID), "graph": TicketArtifactsGraphName, "@tickets": TicketCollectionName, }, ticketFilterVars), busdb.ReadOperation) if err != nil { return nil, err } var artifactNames []string for _, artifact := range ticketSimpleResponse.Artifacts { artifactNames = append(artifactNames, artifact.Name) } ticketsQuery = `FOR d IN @@collection FILTER d._key != @ID ` + ticketFilterQuery + ` FOR a IN NOT_NULL(d.artifacts, []) FILTER POSITION(@artifacts, a.name) RETURN d` sameArtifactTickets, _, err := db.ticketListQuery(ctx, ticketsQuery, mergeMaps(map[string]interface{}{ "ID": fmt.Sprint(ticketID), "artifacts": artifactNames, }, ticketFilterVars), busdb.ReadOperation) if err != nil { return nil, err } tickets := append(outTickets, inTickets...) tickets = append(tickets, sameArtifactTickets...) sort.Slice(tickets, func(i, j int) bool { return tickets[i].ID < tickets[j].ID }) ticketResponse, err := toTicketResponse(ticketSimpleResponse) if err != nil { return nil, err } logs, err := db.LogList(ctx, fmt.Sprintf("%s/%d", TicketCollectionName, ticketID)) if err != nil { return nil, err } return toTicketWithTickets(ticketResponse, tickets, logs), nil } func (db *Database) TicketUpdate(ctx context.Context, ticketID int64, ticket *model.Ticket) (*model.TicketWithTickets, error) { ticketFilterQuery, ticketFilterVars, err := db.Hooks.TicketWriteFilter(ctx) if err != nil { return nil, err } query := `LET d = DOCUMENT(@@collection, @ID) ` + ticketFilterQuery + ` REPLACE d WITH @ticket IN @@collection RETURN NEW` ticket.Modified = time.Now().UTC() // TODO make setable? return db.ticketGetQuery(ctx, ticketID, query, mergeMaps(map[string]interface{}{"ticket": ticket}, ticketFilterVars), &busdb.Operation{ Type: bus.DatabaseEntryUpdated, Ids: []driver.DocumentID{ driver.NewDocumentID(TicketCollectionName, strconv.FormatInt(ticketID, 10)), }, }) } func (db *Database) TicketDelete(ctx context.Context, ticketID int64) error { _, err := db.TicketGet(ctx, ticketID) if err != nil { return err } _, err = db.ticketCollection.RemoveDocument(ctx, strconv.FormatInt(ticketID, 10)) if err != nil { return err } return nil } func (db *Database) TicketList(ctx context.Context, ticketType string, query string, sorts []string, desc []bool, offset, count int64) (*model.TicketList, error) { binVars := map[string]interface{}{} parser := &caql.Parser{Searcher: db.Index, Prefix: "d."} var typeString = "" if ticketType != "" { typeString = "FILTER d.type == @type " binVars["type"] = ticketType } var filterString = "" if query != "" { queryTree, err := parser.Parse(query) if err != nil { return nil, errors.New("invalid filter query: syntax error") } filterString, err = queryTree.String() if err != nil { return nil, fmt.Errorf("invalid filter query: %w", err) } filterString = "FILTER " + filterString } documentCount, err := db.TicketCount(ctx, typeString, filterString, binVars) if err != nil { return nil, err } sortQ := sortQuery(sorts, desc, binVars) binVars["offset"] = offset binVars["count"] = count ticketFilterQuery, ticketFilterVars, err := db.Hooks.TicketReadFilter(ctx) if err != nil { return nil, err } q := `FOR d IN @@collection ` + ticketFilterQuery + ` ` + sortQ + ` ` + typeString + ` ` + filterString + ` LIMIT @offset, @count SORT d._key ASC RETURN d` // RETURN KEEP(d, "_key", "id", "name", "type", "created")` ticketList, _, err := db.ticketListQuery(ctx, q, mergeMaps(binVars, ticketFilterVars), busdb.ReadOperation) return &model.TicketList{ Count: documentCount, Tickets: ticketList, }, err // return map[string]interface{}{"tickets": ticketList, "count": documentCount}, err } func (db *Database) ticketListQuery(ctx context.Context, query string, bindVars map[string]interface{}, operation *busdb.Operation) ([]*model.TicketSimpleResponse, *model.LogEntry, error) { if bindVars == nil { bindVars = map[string]interface{}{} } bindVars["@collection"] = TicketCollectionName cursor, logEntry, err := db.Query(ctx, query, bindVars, operation) if err != nil { return nil, nil, err } defer cursor.Close() var docs []*model.TicketSimpleResponse for { doc := model.Ticket{} meta, err := cursor.ReadDocument(ctx, &doc) if driver.IsNoMoreDocuments(err) { break } else if err != nil { return nil, nil, err } resp, err := toTicketSimpleResponse(meta.Key, &doc) if err != nil { return nil, nil, err } docs = append(docs, resp) } return docs, logEntry, nil } func (db *Database) TicketCount(ctx context.Context, typequery, filterquery string, bindVars map[string]interface{}) (int, error) { if bindVars == nil { bindVars = map[string]interface{}{} } bindVars["@collection"] = TicketCollectionName ticketFilterQuery, ticketFilterVars, err := db.Hooks.TicketReadFilter(ctx) if err != nil { return 0, err } countQuery := `RETURN LENGTH(FOR d IN @@collection ` + ticketFilterQuery + " " + typequery + " " + filterquery + ` RETURN 1)` cursor, _, err := db.Query(ctx, countQuery, mergeMaps(bindVars, ticketFilterVars), busdb.ReadOperation) if err != nil { return 0, err } documentCount := 0 _, err = cursor.ReadDocument(ctx, &documentCount) if err != nil { return 0, err } cursor.Close() return documentCount, nil } func sortQuery(paramsSort []string, paramsDesc []bool, bindVars map[string]interface{}) string { sort := "" if len(paramsSort) > 0 { var sorts []string for i, column := range paramsSort { colsort := fmt.Sprintf("d.@column%d", i) if len(paramsDesc) > i && paramsDesc[i] { colsort += " DESC" } sorts = append(sorts, colsort) bindVars[fmt.Sprintf("column%d", i)] = column } sort = "SORT " + strings.Join(sorts, ", ") } return sort } func mergeMaps(a map[string]interface{}, b map[string]interface{}) map[string]interface{} { merged := map[string]interface{}{} for k, v := range a { merged[k] = v } for k, v := range b { merged[k] = v } return merged } func validate(e interface{}, schema *gojsonschema.Schema) error { b, err := json.Marshal(e) if err != nil { return err } res, err := schema.Validate(gojsonschema.NewStringLoader(string(b))) if err != nil { return err } if len(res.Errors()) > 0 { var l []string for _, e := range res.Errors() { l = append(l, e.String()) } return fmt.Errorf("validation failed: %v", strings.Join(l, ", ")) } return nil }