Improve bus (#3)

* Improve bus
* Add ticket log

database/artifact.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/arangodb/go-driver"
 
+	"github.com/SecurityBrewery/catalyst/bus"
 	"github.com/SecurityBrewery/catalyst/database/busdb"
 	"github.com/SecurityBrewery/catalyst/generated/models"
 	"github.com/SecurityBrewery/catalyst/time"
@@ -60,11 +61,10 @@ func (db *Database) ArtifactUpdate(ctx context.Context, id int64, name string, a
 		"name":        name,
 		"artifact":    artifact,
 	}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: fmt.Sprintf("Update artifact %s", name),
 	})
 }
 
@@ -92,10 +92,9 @@ func (db *Database) EnrichArtifact(ctx context.Context, id int64, name string, e
 		"enrichmentname": enrichment.Name,
 		"enrichment":     enrichment,
 	}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: fmt.Sprintf("Run %s on artifact", enrichment.Name),
 	})
 }

database/busdb/busdb.go

@@ -33,22 +33,13 @@ func NewDatabase(ctx context.Context, internal driver.Database, b *bus.Bus) (*Bu
 	}, nil
 }
 
-type OperationType int
-
-const (
-	Create OperationType = iota
-	Read                 = iota
-	Update               = iota
-)
-
 type Operation struct {
-	OperationType OperationType
-	Ids           []driver.DocumentID
-	Msg           string
+	Type bus.DatabaseUpdateType
+	Ids  []driver.DocumentID
 }
 
-var CreateOperation = &Operation{OperationType: Create}
-var ReadOperation = &Operation{OperationType: Read}
+var CreateOperation = &Operation{Type: bus.DatabaseEntryCreated}
+var ReadOperation = &Operation{Type: bus.DatabaseEntryRead}
 
 func (db BusDatabase) Query(ctx context.Context, query string, vars map[string]interface{}, operation *Operation) (driver.Cursor, *models.LogEntry, error) {
 	cur, err := db.internal.Query(ctx, query, vars)
@@ -59,8 +50,8 @@ func (db BusDatabase) Query(ctx context.Context, query string, vars map[string]i
 	var logs *models.LogEntry
 
 	switch {
-	case operation.OperationType == Update:
-		if err := db.LogAndNotify(ctx, operation.Ids, operation.Msg); err != nil {
+	case operation.Type == bus.DatabaseEntryCreated, operation.Type == bus.DatabaseEntryUpdated:
+		if err := db.bus.PublishDatabaseUpdate(operation.Ids, operation.Type); err != nil {
 			return nil, nil, err
 		}
 	}
@@ -68,19 +59,6 @@ func (db BusDatabase) Query(ctx context.Context, query string, vars map[string]i
 	return cur, logs, err
 }
 
-func (db BusDatabase) LogAndNotify(ctx context.Context, ids []driver.DocumentID, msg string) error {
-	var logEntries []*models.LogEntry
-	for _, i := range ids {
-		logEntries = append(logEntries, &models.LogEntry{Reference: i.String(), Message: msg})
-	}
-
-	if err := db.LogBatchCreate(ctx, logEntries); err != nil {
-		return err
-	}
-
-	return db.bus.PublishUpdate(ids)
-}
-
 func (db BusDatabase) Remove(ctx context.Context) error {
 	return db.internal.Remove(ctx)
 }
@@ -104,7 +82,7 @@ func (c Collection) CreateDocument(ctx, newctx context.Context, key string, docu
 		return meta, err
 	}
 
-	err = c.db.LogAndNotify(ctx, []driver.DocumentID{meta.ID}, "Document created")
+	err = c.db.bus.PublishDatabaseUpdate([]driver.DocumentID{meta.ID}, bus.DatabaseEntryCreated)
 	if err != nil {
 		return meta, err
 	}
@@ -117,7 +95,7 @@ func (c Collection) CreateEdge(ctx, newctx context.Context, edge *driver.EdgeDoc
 		return meta, err
 	}
 
-	err = c.db.LogAndNotify(ctx, []driver.DocumentID{meta.ID}, "Document created")
+	err = c.db.bus.PublishDatabaseUpdate([]driver.DocumentID{meta.ID}, bus.DatabaseEntryCreated)
 	if err != nil {
 		return meta, err
 	}
@@ -138,7 +116,7 @@ func (c Collection) CreateEdges(ctx context.Context, edges []*driver.EdgeDocumen
 		ids = append(ids, meta.ID)
 	}
 
-	err = c.db.LogAndNotify(ctx, ids, "Document created")
+	err = c.db.bus.PublishDatabaseUpdate(ids, bus.DatabaseEntryCreated)
 	if err != nil {
 		return metas, err
 	}
@@ -160,7 +138,7 @@ func (c Collection) UpdateDocument(ctx context.Context, key string, update inter
 		return meta, err
 	}
 
-	return meta, c.db.bus.PublishUpdate([]driver.DocumentID{meta.ID})
+	return meta, c.db.bus.PublishDatabaseUpdate([]driver.DocumentID{meta.ID}, bus.DatabaseEntryUpdated)
 }
 
 func (c Collection) ReplaceDocument(ctx context.Context, key string, document interface{}) (driver.DocumentMeta, error) {
@@ -169,7 +147,7 @@ func (c Collection) ReplaceDocument(ctx context.Context, key string, document in
 		return meta, err
 	}
 
-	return meta, c.db.bus.PublishUpdate([]driver.DocumentID{meta.ID})
+	return meta, c.db.bus.PublishDatabaseUpdate([]driver.DocumentID{meta.ID}, bus.DatabaseEntryUpdated)
 }
 
 func (c Collection) RemoveDocument(ctx context.Context, formatInt string) (driver.DocumentMeta, error) {

database/busdb/log.go

@@ -3,6 +3,8 @@ package busdb
 import (
 	"context"
 	"errors"
+	"github.com/SecurityBrewery/catalyst/bus"
+	"strings"
 
 	"github.com/arangodb/go-driver"
 
@@ -12,15 +14,16 @@ import (
 
 const LogCollectionName = "logs"
 
-func (db *BusDatabase) LogCreate(ctx context.Context, reference, message string) (*models.LogEntry, error) {
+func (db *BusDatabase) LogCreate(ctx context.Context, logType, reference, message string) (*models.LogEntry, error) {
 	user, ok := UserFromContext(ctx)
 	if !ok {
 		return nil, errors.New("no user in context")
 	}
 
 	logentry := &models.LogEntry{
+		Type:      logType,
 		Reference: reference,
-		Created:   time.Now(),
+		Created:   time.Now().UTC(),
 		Creator:   user.ID,
 		Message:   message,
 	}
@@ -31,27 +34,18 @@ func (db *BusDatabase) LogCreate(ctx context.Context, reference, message string)
 		return nil, err
 	}
 
-	return &doc, db.bus.PublishUpdate([]driver.DocumentID{driver.DocumentID(logentry.Reference)})
+	return &doc, nil
 }
 
-func (db *BusDatabase) LogBatchCreate(ctx context.Context, logEntryForms []*models.LogEntry) error {
-	user, ok := UserFromContext(ctx)
-	if !ok {
-		return errors.New("no user in context")
-	}
-
+func (db *BusDatabase) LogBatchCreate(ctx context.Context, logentries []*models.LogEntry) error {
 	var ids []driver.DocumentID
-	var logentries []*models.LogEntry
-	for _, logEntryForm := range logEntryForms {
-		logentry := &models.LogEntry{
-			Reference: logEntryForm.Reference,
-			Created:   time.Now(),
-			Creator:   user.ID,
-			Message:   logEntryForm.Message,
+	for _, entry := range logentries {
+		if strings.HasPrefix(entry.Reference, "tickets/") {
+			ids = append(ids, driver.DocumentID(entry.Reference))
 		}
-
-		logentries = append(logentries, logentry)
-		ids = append(ids, driver.DocumentID(logentry.Reference))
+	}
+	if ids != nil {
+		go db.bus.PublishDatabaseUpdate(ids, bus.DatabaseEntryCreated)
 	}
 
 	_, errs, err := db.logCollection.CreateDocuments(ctx, logentries)
@@ -63,7 +57,7 @@ func (db *BusDatabase) LogBatchCreate(ctx context.Context, logEntryForms []*mode
 		return err
 	}
 
-	return db.bus.PublishUpdate(ids)
+	return nil
 }
 
 func (db *BusDatabase) LogList(ctx context.Context, reference string) ([]*models.LogEntry, error) {

database/job.go

@@ -11,6 +11,7 @@ import (
 	"github.com/docker/docker/client"
 	"github.com/xeipuuv/gojsonschema"
 
+	"github.com/SecurityBrewery/catalyst/bus"
 	"github.com/SecurityBrewery/catalyst/caql"
 	"github.com/SecurityBrewery/catalyst/database/busdb"
 	"github.com/SecurityBrewery/catalyst/generated/models"
@@ -144,11 +145,10 @@ func (db *Database) JobLogAppend(ctx context.Context, id string, logLine string)
 		"ID":          id,
 		"logline":     logLine,
 	}, &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%s", JobCollectionName, id)),
 		},
-		Msg: fmt.Sprintf("Append logline"),
 	})
 	if err != nil {
 		return err
@@ -166,11 +166,10 @@ func (db *Database) JobComplete(ctx context.Context, id string, out interface{})
 		"ID":          id,
 		"out":         out,
 	}, &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%s", JobCollectionName, id)),
 		},
-		Msg: fmt.Sprintf("Set output"),
 	})
 	if err != nil {
 		return err

database/relationships.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/arangodb/go-driver"
 
+	"github.com/SecurityBrewery/catalyst/bus"
 	"github.com/SecurityBrewery/catalyst/database/busdb"
 )
 
@@ -37,12 +38,11 @@ func (db *Database) RelatedRemove(ctx context.Context, id, id2 int64) error {
 		"id":          driver.DocumentID(TicketCollectionName + "/" + strconv.Itoa(int(id))),
 		"id2":         driver.DocumentID(TicketCollectionName + "/" + strconv.Itoa(int(id2))),
 	}, &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(TicketCollectionName + "/" + strconv.Itoa(int(id))),
 			driver.DocumentID(TicketCollectionName + "/" + strconv.Itoa(int(id2))),
 		},
-		Msg: "Removed ticket/artifact relation",
 	})
 	return err
 }

database/ticket.go

@@ -13,6 +13,7 @@ import (
 	"github.com/arangodb/go-driver"
 	"github.com/xeipuuv/gojsonschema"
 
+	"github.com/SecurityBrewery/catalyst/bus"
 	"github.com/SecurityBrewery/catalyst/caql"
 	"github.com/SecurityBrewery/catalyst/database/busdb"
 	"github.com/SecurityBrewery/catalyst/generated/models"
@@ -133,7 +134,7 @@ func toTicketSimpleResponse(key string, ticket *models.Ticket) (*models.TicketSi
 	}, nil
 }
 
-func toTicketWithTickets(ticketResponse *models.TicketResponse, tickets []*models.TicketSimpleResponse) *models.TicketWithTickets {
+func toTicketWithTickets(ticketResponse *models.TicketResponse, tickets []*models.TicketSimpleResponse, logs []*models.LogEntry) *models.TicketWithTickets {
 	return &models.TicketWithTickets{
 		Artifacts:  ticketResponse.Artifacts,
 		Comments:   ticketResponse.Comments,
@@ -152,6 +153,7 @@ func toTicketWithTickets(ticketResponse *models.TicketResponse, tickets []*model
 		Type:       ticketResponse.Type,
 		Write:      ticketResponse.Write,
 
+		Logs:    logs,
 		Tickets: tickets,
 	}
 }
@@ -244,9 +246,8 @@ func (db *Database) TicketBatchCreate(ctx context.Context, ticketForms []*models
 	for _, apiTicket := range apiTickets {
 		ids = append(ids, driver.NewDocumentID(TicketCollectionName, fmt.Sprint(apiTicket.ID)))
 	}
-	if err := db.BusDatabase.LogAndNotify(ctx, ids, "Ticket created"); err != nil {
-		return nil, err
-	}
+
+	go db.bus.PublishDatabaseUpdate(ids, bus.DatabaseEntryUpdated)
 
 	ticketResponses, err := toTicketResponses(apiTickets)
 	if err != nil {
@@ -405,7 +406,12 @@ func (db *Database) ticketGetQuery(ctx context.Context, ticketID int64, query st
 		return nil, err
 	}
 
-	return toTicketWithTickets(ticketResponse, tickets), nil
+	logs, err := db.LogList(ctx, fmt.Sprintf("%s/%d", TicketCollectionName, ticketID))
+	if err != nil {
+		return nil, err
+	}
+
+	return toTicketWithTickets(ticketResponse, tickets, logs), nil
 }
 
 func (db *Database) TicketUpdate(ctx context.Context, ticketID int64, ticket *models.Ticket) (*models.TicketWithTickets, error) {
@@ -420,10 +426,9 @@ func (db *Database) TicketUpdate(ctx context.Context, ticketID int64, ticket *mo
 	RETURN NEW`
 	ticket.Modified = time.Now().UTC() // TODO make setable?
 	return db.ticketGetQuery(ctx, ticketID, query, mergeMaps(map[string]interface{}{"ticket": ticket}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update, Ids: []driver.DocumentID{
+		Type: bus.DatabaseEntryUpdated, Ids: []driver.DocumentID{
 			driver.NewDocumentID(TicketCollectionName, strconv.FormatInt(ticketID, 10)),
 		},
-		Msg: "Ticket updated",
 	})
 }
 

database/ticket_field.go

@@ -9,6 +9,7 @@ import (
 	"github.com/iancoleman/strcase"
 	"github.com/mingrammer/commonregex"
 
+	"github.com/SecurityBrewery/catalyst/bus"
 	"github.com/SecurityBrewery/catalyst/database/busdb"
 	"github.com/SecurityBrewery/catalyst/generated/models"
 	"github.com/SecurityBrewery/catalyst/pointer"
@@ -34,11 +35,10 @@ func (db *Database) AddArtifact(ctx context.Context, id int64, artifact *models.
 	UPDATE d WITH { "modified": @now, "artifacts": PUSH(NOT_NULL(d.artifacts, []), @artifact) } IN @@collection
 	RETURN NEW`
 	return db.ticketGetQuery(ctx, id, query, mergeMaps(map[string]interface{}{"artifact": artifact, "now": time.Now().UTC()}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: "Add artifact",
 	})
 }
 
@@ -74,11 +74,10 @@ func (db *Database) RemoveArtifact(ctx context.Context, id int64, name string) (
 	UPDATE d WITH { "modified": @now, "artifacts": newartifacts } IN @@collection
 	RETURN NEW`
 	return db.ticketGetQuery(ctx, id, query, mergeMaps(map[string]interface{}{"name": name, "now": time.Now().UTC()}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: "Remove artifact",
 	})
 }
 
@@ -93,11 +92,10 @@ func (db *Database) SetTemplate(ctx context.Context, id int64, schema string) (*
 	UPDATE d WITH { "schema": @schema } IN @@collection
 	RETURN NEW`
 	return db.ticketGetQuery(ctx, id, query, mergeMaps(map[string]interface{}{"schema": schema}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: "Set Template",
 	})
 }
 
@@ -125,11 +123,10 @@ func (db *Database) AddComment(ctx context.Context, id int64, comment *models.Co
 	UPDATE d WITH { "modified": @now, "comments": PUSH(NOT_NULL(d.comments, []), @comment) } IN @@collection
 	RETURN NEW`
 	return db.ticketGetQuery(ctx, id, query, mergeMaps(map[string]interface{}{"comment": comment, "now": time.Now().UTC()}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: "Add comment",
 	})
 }
 
@@ -144,11 +141,10 @@ func (db *Database) RemoveComment(ctx context.Context, id int64, commentID int64
 	UPDATE d WITH { "modified": @now, "comments": REMOVE_NTH(d.comments, @commentID) } IN @@collection
 	RETURN NEW`
 	return db.ticketGetQuery(ctx, id, query, mergeMaps(map[string]interface{}{"commentID": commentID, "now": time.Now().UTC()}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: "Remove comment",
 	})
 }
 
@@ -163,11 +159,10 @@ func (db *Database) SetReferences(ctx context.Context, id int64, references []*m
 	UPDATE d WITH { "modified": @now, "references": @references } IN @@collection
 	RETURN NEW`
 	return db.ticketGetQuery(ctx, id, query, mergeMaps(map[string]interface{}{"references": references, "now": time.Now().UTC()}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: "Changed references",
 	})
 }
 
@@ -182,11 +177,10 @@ func (db *Database) LinkFiles(ctx context.Context, id int64, files []*models.Fil
 	UPDATE d WITH { "modified": @now, "files": @files } IN @@collection
 	RETURN NEW`
 	return db.ticketGetQuery(ctx, id, query, mergeMaps(map[string]interface{}{"files": files, "now": time.Now().UTC()}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.DocumentID(fmt.Sprintf("%s/%d", TicketCollectionName, id)),
 		},
-		Msg: "Linked files",
 	})
 }
 
@@ -224,11 +218,10 @@ func (db *Database) AddTicketPlaybook(ctx context.Context, id int64, playbookTem
 		"playbookID": findName(parentTicket.Playbooks, playbookID),
 		"now":        time.Now().UTC(),
 	}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.NewDocumentID(TicketCollectionName, fmt.Sprintf("%d", id)),
 		},
-		Msg: "Added playbook",
 	})
 	if err != nil {
 		return nil, err
@@ -284,10 +277,9 @@ func (db *Database) RemoveTicketPlaybook(ctx context.Context, id int64, playbook
 		"playbookID": playbookID,
 		"now":        time.Now().UTC(),
 	}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.NewDocumentID(TicketCollectionName, fmt.Sprintf("%d", id)),
 		},
-		Msg: fmt.Sprintf("Removed playbook %s", playbookID),
 	})
 }

database/ticket_task.go

@@ -9,6 +9,7 @@ import (
 	"github.com/arangodb/go-driver"
 	"github.com/google/uuid"
 
+	"github.com/SecurityBrewery/catalyst/bus"
 	"github.com/SecurityBrewery/catalyst/database/busdb"
 	"github.com/SecurityBrewery/catalyst/generated/models"
 	"github.com/SecurityBrewery/catalyst/time"
@@ -74,11 +75,10 @@ func (db *Database) TaskComplete(ctx context.Context, id int64, playbookID strin
 		"closed":     time.Now().UTC(),
 		"now":        time.Now().UTC(),
 	}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.NewDocumentID(TicketCollectionName, fmt.Sprintf("%d", id)),
 		},
-		Msg: fmt.Sprintf("Completed task %s in playbook %s", taskID, playbookID),
 	})
 	if err != nil {
 		return nil, err
@@ -136,11 +136,10 @@ func (db *Database) TaskUpdate(ctx context.Context, id int64, playbookID string,
 		"task":       task,
 		"now":        time.Now().UTC(),
 	}, ticketFilterVars), &busdb.Operation{
-		OperationType: busdb.Update,
+		Type: bus.DatabaseEntryUpdated,
 		Ids: []driver.DocumentID{
 			driver.NewDocumentID(TicketCollectionName, fmt.Sprintf("%d", id)),
 		},
-		Msg: fmt.Sprintf("Saved task %s in playbook %s", taskID, playbookID),
 	})
 	if err != nil {
 		return nil, err