CSEC_PUBLIC/catalyst
1
0
Fork 0
mirror of https://github.com/SecurityBrewery/catalyst.git synced 2026-01-25 07:23:28 +01:00
Code Issues Packages Projects Releases Wiki Activity

Release catalyst

Browse Source
This commit is contained in:
Jonas Plum
2021-12-13 00:39:15 +01:00
commit 15cf0ebd49
339 changed files with 111677 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
caql/blevebuilder.go Normal file
View File

@@ -0,0 +1,182 @@
package caql
import (
"errors"
"fmt"
"strconv"
"github.com/SecurityBrewery/catalyst/generated/caql/parser"
)
var TooComplexError = errors.New("unsupported features for index queries, use advanced search instead")
type bleveBuilder struct {
*parser.BaseCAQLParserListener
stack []string
err error
}
// push is a helper function for pushing new node to the listener Stack.
func (s *bleveBuilder) push(i string) {
s.stack = append(s.stack, i)
}
// pop is a helper function for poping a node from the listener Stack.
func (s *bleveBuilder) pop() (n string) {
// Check that we have nodes in the stack.
size := len(s.stack)
if size < 1 {
panic(ErrStack)
}
// Pop the last value from the Stack.
n, s.stack = s.stack[size-1], s.stack[:size-1]
return
}
func (s *bleveBuilder) binaryPop() (interface{}, interface{}) {
right, left := s.pop(), s.pop()
return left, right
}
// ExitExpression is called when production expression is exited.
func (s *bleveBuilder) ExitExpression(ctx *parser.ExpressionContext) {
switch {
case ctx.Value_literal() != nil:
// pass
case ctx.Reference() != nil:
// pass
case ctx.Operator_unary() != nil:
s.err = TooComplexError
return
case ctx.T_PLUS() != nil:
fallthrough
case ctx.T_MINUS() != nil:
fallthrough
case ctx.T_TIMES() != nil:
fallthrough
case ctx.T_DIV() != nil:
fallthrough
case ctx.T_MOD() != nil:
s.err = TooComplexError
return
case ctx.T_RANGE() != nil:
s.err = TooComplexError
return
case ctx.T_LT() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s:<%s", left, right))
case ctx.T_GT() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s:>%s", left, right))
case ctx.T_LE() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s:<=%s", left, right))
case ctx.T_GE() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s:>=%s", left, right))
case ctx.T_IN() != nil && ctx.GetEq_op() == nil:
s.err = TooComplexError
return
case ctx.T_EQ() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s:%s", left, right))
case ctx.T_NE() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("-%s:%s", left, right))
case ctx.T_ALL() != nil && ctx.GetEq_op() != nil:
fallthrough
case ctx.T_ANY() != nil && ctx.GetEq_op() != nil:
fallthrough
case ctx.T_NONE() != nil && ctx.GetEq_op() != nil:
s.err = TooComplexError
return
case ctx.T_ALL() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
fallthrough
case ctx.T_ANY() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
fallthrough
case ctx.T_NONE() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
s.err = TooComplexError
return
case ctx.T_LIKE() != nil:
s.err = errors.New("index queries are like queries by default")
return
case ctx.T_REGEX_MATCH() != nil:
left, right := s.binaryPop()
if ctx.T_NOT() != nil {
s.err = TooComplexError
return
} else {
s.push(fmt.Sprintf("%s:/%s/", left, right))
}
case ctx.T_REGEX_NON_MATCH() != nil:
s.err = errors.New("index query cannot contain regex non matches, use advanced search instead")
return
case ctx.T_AND() != nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s %s", left, right))
case ctx.T_OR() != nil:
s.err = errors.New("index query cannot contain OR, use advanced search instead")
return
case ctx.T_QUESTION() != nil && len(ctx.AllExpression()) == 3:
s.err = errors.New("index query cannot contain ternary operations, use advanced search instead")
return
case ctx.T_QUESTION() != nil && len(ctx.AllExpression()) == 2:
s.err = errors.New("index query cannot contain ternary operations, use advanced search instead")
return
default:
panic("unknown expression")
}
}
// ExitReference is called when production reference is exited.
func (s *bleveBuilder) ExitReference(ctx *parser.ReferenceContext) {
switch {
case ctx.DOT() != nil:
reference := s.pop()
s.push(fmt.Sprintf("%s.%s", reference, ctx.T_STRING().GetText()))
case ctx.T_STRING() != nil:
s.push(ctx.T_STRING().GetText())
case ctx.Compound_value() != nil:
s.err = TooComplexError
return
case ctx.Function_call() != nil:
s.err = TooComplexError
return
case ctx.T_OPEN() != nil:
s.err = TooComplexError
return
case ctx.T_ARRAY_OPEN() != nil:
s.err = TooComplexError
return
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}
// ExitValue_literal is called when production value_literal is exited.
func (s *bleveBuilder) ExitValue_literal(ctx *parser.Value_literalContext) {
if ctx.T_QUOTED_STRING() != nil {
st, err := unquote(ctx.GetText())
if err != nil {
panic(err)
}
s.push(strconv.Quote(st))
} else {
s.push(ctx.GetText())
}
}

50
caql/blevebuilder_test.go Normal file
View File

@@ -0,0 +1,50 @@
package caql
import (
"testing"
)
func TestBleveBuilder(t *testing.T) {
tests := []struct {
name string
saql string
wantBleve string
wantParseErr bool
wantRebuildErr bool
}{
{name: "Search 1", saql: `"Bob"`, wantBleve: `"Bob"`},
{name: "Search 2", saql: `"Bob" AND title == 'Name'`, wantBleve: `"Bob" title:"Name"`},
{name: "Search 3", saql: `"Bob" OR title == 'Name'`, wantRebuildErr: true},
{name: "Search 4", saql: `title == 'malware' AND 'wannacry'`, wantBleve: `title:"malware" "wannacry"`},
}
for _, tt := range tests {
parser := &Parser{}
t.Run(tt.name, func(t *testing.T) {
expr, err := parser.Parse(tt.saql)
if (err != nil) != tt.wantParseErr {
t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantParseErr)
if expr != nil {
t.Error(expr.String())
}
return
}
if err != nil {
return
}
got, err := expr.BleveString()
if (err != nil) != tt.wantRebuildErr {
t.Error(expr.String())
t.Errorf("String() error = %v, wantErr %v", err, tt.wantParseErr)
return
}
if err != nil {
return
}
if got != tt.wantBleve {
t.Errorf("String() got = %v, want %v", got, tt.wantBleve)
}
})
}
}

317
caql/builder.go Normal file
View File

@@ -0,0 +1,317 @@
package caql
import (
"fmt"
"strconv"
"strings"
"github.com/SecurityBrewery/catalyst/generated/caql/parser"
)
type Searcher interface {
Search(term string) (ids []string, err error)
}
type aqlBuilder struct {
*parser.BaseCAQLParserListener
searcher Searcher
stack []string
prefix string
}
// push is a helper function for pushing new node to the listener Stack.
func (s *aqlBuilder) push(i string) {
s.stack = append(s.stack, i)
}
// pop is a helper function for poping a node from the listener Stack.
func (s *aqlBuilder) pop() (n string) {
// Check that we have nodes in the stack.
size := len(s.stack)
if size < 1 {
panic(ErrStack)
}
// Pop the last value from the Stack.
n, s.stack = s.stack[size-1], s.stack[:size-1]
return
}
func (s *aqlBuilder) binaryPop() (string, string) {
right, left := s.pop(), s.pop()
return left, right
}
// ExitExpression is called when production expression is exited.
func (s *aqlBuilder) ExitExpression(ctx *parser.ExpressionContext) {
switch {
case ctx.Value_literal() != nil:
if ctx.GetParent().GetParent() == nil {
s.push(s.toBoolString(s.pop()))
}
case ctx.Reference() != nil:
ref := s.pop()
if ref == "d.id" {
s.push("d._key")
} else {
s.push(ref)
}
// pass
case ctx.Operator_unary() != nil:
s.push(s.toBoolString(s.pop()))
case ctx.T_PLUS() != nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s + %s", left, right))
case ctx.T_MINUS() != nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s - %s", left, right))
case ctx.T_TIMES() != nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s * %s", left, right))
case ctx.T_DIV() != nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s / %s", left, right))
case ctx.T_MOD() != nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s %% %s", left, right))
case ctx.T_RANGE() != nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s..%s", left, right))
case ctx.T_LT() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s < %s", left, right))
case ctx.T_GT() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s > %s", left, right))
case ctx.T_LE() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s <= %s", left, right))
case ctx.T_GE() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s >= %s", left, right))
case ctx.T_IN() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
if ctx.T_NOT() != nil {
s.push(fmt.Sprintf("%s NOT IN %s", left, right))
} else {
s.push(fmt.Sprintf("%s IN %s", left, right))
}
case ctx.T_EQ() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s == %s", left, right))
case ctx.T_NE() != nil && ctx.GetEq_op() == nil:
left, right := s.binaryPop()
s.push(fmt.Sprintf("%s != %s", left, right))
case ctx.T_ALL() != nil && ctx.GetEq_op() != nil:
right, left := s.pop(), s.pop()
s.push(fmt.Sprintf("%s ALL %s %s", left, ctx.GetEq_op().GetText(), right))
case ctx.T_ANY() != nil && ctx.GetEq_op() != nil:
right, left := s.pop(), s.pop()
s.push(fmt.Sprintf("%s ANY %s %s", left, ctx.GetEq_op().GetText(), right))
case ctx.T_NONE() != nil && ctx.GetEq_op() != nil:
right, left := s.pop(), s.pop()
s.push(fmt.Sprintf("%s NONE %s %s", left, ctx.GetEq_op().GetText(), right))
case ctx.T_ALL() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
right, left := s.pop(), s.pop()
s.push(fmt.Sprintf("%s ALL IN %s", left, right))
case ctx.T_ANY() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
right, left := s.pop(), s.pop()
s.push(fmt.Sprintf("%s ANY IN %s", left, right))
case ctx.T_NONE() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
right, left := s.pop(), s.pop()
s.push(fmt.Sprintf("%s NONE IN %s", left, right))
case ctx.T_LIKE() != nil:
left, right := s.binaryPop()
if ctx.T_NOT() != nil {
s.push(fmt.Sprintf("%s NOT LIKE %s", left, right))
} else {
s.push(fmt.Sprintf("%s LIKE %s", left, right))
}
case ctx.T_REGEX_MATCH() != nil:
left, right := s.binaryPop()
if ctx.T_NOT() != nil {
s.push(fmt.Sprintf("%s NOT =~ %s", left, right))
} else {
s.push(fmt.Sprintf("%s =~ %s", left, right))
}
case ctx.T_REGEX_NON_MATCH() != nil:
left, right := s.binaryPop()
if ctx.T_NOT() != nil {
s.push(fmt.Sprintf("%s NOT !~ %s", left, right))
} else {
s.push(fmt.Sprintf("%s !~ %s", left, right))
}
case ctx.T_AND() != nil:
left, right := s.binaryPop()
left = s.toBoolString(left)
right = s.toBoolString(right)
s.push(fmt.Sprintf("%s AND %s", left, right))
case ctx.T_OR() != nil:
left, right := s.binaryPop()
left = s.toBoolString(left)
right = s.toBoolString(right)
s.push(fmt.Sprintf("%s OR %s", left, right))
case ctx.T_QUESTION() != nil && len(ctx.AllExpression()) == 3:
right, middle, left := s.pop(), s.pop(), s.pop()
s.push(fmt.Sprintf("%s ? %s : %s", left, middle, right))
case ctx.T_QUESTION() != nil && len(ctx.AllExpression()) == 2:
right, left := s.pop(), s.pop()
s.push(fmt.Sprintf("%s ? : %s", left, right))
default:
panic("unknown expression")
}
}
func (s *aqlBuilder) toBoolString(v string) string {
_, err := unquote(v)
if err == nil {
ids, err := s.searcher.Search(v)
if err != nil {
panic("invalid search " + err.Error())
}
return fmt.Sprintf(`d._key IN ["%s"]`, strings.Join(ids, `","`))
}
return v
}
// ExitOperator_unary is called when production operator_unary is exited.
func (s *aqlBuilder) ExitOperator_unary(ctx *parser.Operator_unaryContext) {
value := s.pop()
switch {
case ctx.T_PLUS() != nil:
s.push(value)
case ctx.T_MINUS() != nil:
s.push(fmt.Sprintf("-%s", value))
case ctx.T_NOT() != nil:
s.push(fmt.Sprintf("NOT %s", value))
default:
panic(fmt.Sprintf("unexpected operation: %s", ctx.GetText()))
}
}
// ExitReference is called when production reference is exited.
func (s *aqlBuilder) ExitReference(ctx *parser.ReferenceContext) {
switch {
case ctx.DOT() != nil:
reference := s.pop()
if s.prefix != "" && !strings.HasPrefix(reference, s.prefix) {
reference = s.prefix + reference
}
s.push(fmt.Sprintf("%s.%s", reference, ctx.T_STRING().GetText()))
case ctx.T_STRING() != nil:
reference := ctx.T_STRING().GetText()
if s.prefix != "" && !strings.HasPrefix(reference, s.prefix) {
reference = s.prefix + reference
}
s.push(reference)
case ctx.Compound_value() != nil:
// pass
case ctx.Function_call() != nil:
// pass
case ctx.T_OPEN() != nil:
s.push(fmt.Sprintf("(%s)", s.pop()))
case ctx.T_ARRAY_OPEN() != nil:
key := s.pop()
reference := s.pop()
s.push(fmt.Sprintf("%s[%s]", reference, key))
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}
// ExitCompound_value is called when production compound_value is exited.
func (s *aqlBuilder) ExitCompound_value(ctx *parser.Compound_valueContext) {
// pass
}
// ExitFunction_call is called when production function_call is exited.
func (s *aqlBuilder) ExitFunction_call(ctx *parser.Function_callContext) {
var array []string
for range ctx.AllExpression() {
// prepend element
array = append([]string{s.pop()}, array...)
}
parameter := strings.Join(array, ", ")
if !stringSliceContains(functionNames, strings.ToUpper(ctx.T_STRING().GetText())) {
panic("unknown function")
}
s.push(fmt.Sprintf("%s(%s)", strings.ToUpper(ctx.T_STRING().GetText()), parameter))
}
// ExitValue_literal is called when production value_literal is exited.
func (s *aqlBuilder) ExitValue_literal(ctx *parser.Value_literalContext) {
if ctx.T_QUOTED_STRING() != nil {
st, err := unquote(ctx.GetText())
if err != nil {
panic(err)
}
s.push(strconv.Quote(st))
} else {
s.push(ctx.GetText())
}
}
// ExitArray is called when production array is exited.
func (s *aqlBuilder) ExitArray(ctx *parser.ArrayContext) {
var elements []string
for range ctx.AllExpression() {
// elements = append(elements, s.pop())
elements = append([]string{s.pop()}, elements...)
}
s.push("[" + strings.Join(elements, ", ") + "]")
}
// ExitObject is called when production object is exited.
func (s *aqlBuilder) ExitObject(ctx *parser.ObjectContext) {
var elements []string
for range ctx.AllObject_element() {
key, value := s.pop(), s.pop()
elements = append([]string{fmt.Sprintf("%s: %v", key, value)}, elements...)
}
// s.push(object)
s.push("{" + strings.Join(elements, ", ") + "}")
}
// ExitObject_element is called when production object_element is exited.
func (s *aqlBuilder) ExitObject_element(ctx *parser.Object_elementContext) {
switch {
case ctx.T_STRING() != nil:
s.push(ctx.GetText())
s.push(ctx.GetText())
case ctx.Object_element_name() != nil, ctx.T_ARRAY_OPEN() != nil:
key, value := s.pop(), s.pop()
s.push(key)
s.push(value)
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}
// ExitObject_element_name is called when production object_element_name is exited.
func (s *aqlBuilder) ExitObject_element_name(ctx *parser.Object_element_nameContext) {
switch {
case ctx.T_STRING() != nil:
s.push(ctx.T_STRING().GetText())
case ctx.T_QUOTED_STRING() != nil:
s.push(ctx.T_QUOTED_STRING().GetText())
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}

8
caql/errors.go Normal file
View File

@@ -0,0 +1,8 @@
package caql
import "errors"
var (
ErrStack = errors.New("unexpected operator stack")
ErrUndefined = errors.New("variable not defined")
)

750
caql/function.go Normal file
View File

@@ -0,0 +1,750 @@
package caql
import (
"errors"
"fmt"
"math"
"math/rand"
"sort"
"strings"
"unicode/utf8"
"github.com/imdario/mergo"
"github.com/SecurityBrewery/catalyst/generated/caql/parser"
)
func (s *aqlInterpreter) function(ctx *parser.Function_callContext) {
switch strings.ToUpper(ctx.T_STRING().GetText()) {
default:
s.appendErrors(errors.New("unknown function"))
// Array https://www.arangodb.com/docs/stable/aql/functions-array.html
case "APPEND":
u := false
if len(ctx.AllExpression()) == 3 {
u = s.pop().(bool)
}
seen := map[interface{}]bool{}
values, anyArray := s.pop().([]interface{}), s.pop().([]interface{})
if u {
for _, e := range anyArray {
seen[e] = true
}
}
for _, e := range values {
_, ok := seen[e]
if !ok || !u {
seen[e] = true
anyArray = append(anyArray, e)
}
}
s.push(anyArray)
case "COUNT_DISTINCT", "COUNT_UNIQUE":
count := 0
seen := map[interface{}]bool{}
array := s.pop().([]interface{})
for _, e := range array {
_, ok := seen[e]
if !ok {
seen[e] = true
count += 1
}
}
s.push(float64(count))
case "FIRST":
array := s.pop().([]interface{})
if len(array) == 0 {
s.push(nil)
} else {
s.push(array[0])
}
// case "FLATTEN":
// case "INTERLEAVE":
case "INTERSECTION":
iset := New(s.pop().([]interface{})...)
for i := 1; i < len(ctx.AllExpression()); i++ {
iset = iset.Intersection(New(s.pop().([]interface{})...))
}
s.push(iset.Values())
// case "JACCARD":
case "LAST":
array := s.pop().([]interface{})
if len(array) == 0 {
s.push(nil)
} else {
s.push(array[len(array)-1])
}
case "COUNT", "LENGTH":
switch v := s.pop().(type) {
case nil:
s.push(float64(0))
case bool:
if v {
s.push(float64(1))
} else {
s.push(float64(0))
}
case float64:
s.push(float64(len(fmt.Sprint(v))))
case string:
s.push(float64(utf8.RuneCountInString(v)))
case []interface{}:
s.push(float64(len(v)))
case map[string]interface{}:
s.push(float64(len(v)))
default:
panic("unknown type")
}
case "MINUS":
var sets []*Set
for i := 0; i < len(ctx.AllExpression()); i++ {
sets = append(sets, New(s.pop().([]interface{})...))
}
iset := sets[len(sets)-1]
// for i := len(sets)-1; i > 0; i-- {
for i := 0; i < len(sets)-1; i++ {
iset = iset.Minus(sets[i])
}
s.push(iset.Values())
case "NTH":
pos := s.pop().(float64)
array := s.pop().([]interface{})
if int(pos) >= len(array) || pos < 0 {
s.push(nil)
} else {
s.push(array[int64(pos)])
}
// case "OUTERSECTION":
// array := s.pop().([]interface{})
// union := New(array...)
// intersection := New(s.pop().([]interface{})...)
// for i := 1; i < len(ctx.AllExpression()); i++ {
// array = s.pop().([]interface{})
// union = union.Union(New(array...))
// intersection = intersection.Intersection(New(array...))
// }
// s.push(union.Minus(intersection).Values())
case "POP":
array := s.pop().([]interface{})
s.push(array[:len(array)-1])
case "POSITION", "CONTAINS_ARRAY":
returnIndex := false
if len(ctx.AllExpression()) == 3 {
returnIndex = s.pop().(bool)
}
search := s.pop()
array := s.pop().([]interface{})
for idx, e := range array {
if e == search {
if returnIndex {
s.push(float64(idx))
} else {
s.push(true)
}
}
}
if returnIndex {
s.push(float64(-1))
} else {
s.push(false)
}
case "PUSH":
u := false
if len(ctx.AllExpression()) == 3 {
u = s.pop().(bool)
}
element := s.pop()
array := s.pop().([]interface{})
if u && contains(array, element) {
s.push(array)
} else {
s.push(append(array, element))
}
case "REMOVE_NTH":
position := s.pop().(float64)
anyArray := s.pop().([]interface{})
if position < 0 {
position = float64(len(anyArray) + int(position))
}
result := []interface{}{}
for idx, e := range anyArray {
if idx != int(position) {
result = append(result, e)
}
}
s.push(result)
case "REPLACE_NTH":
defaultPaddingValue := ""
if len(ctx.AllExpression()) == 4 {
defaultPaddingValue = s.pop().(string)
}
replaceValue := s.pop().(string)
position := s.pop().(float64)
anyArray := s.pop().([]interface{})
if position < 0 {
position = float64(len(anyArray) + int(position))
if position < 0 {
position = 0
}
}
switch {
case int(position) < len(anyArray):
anyArray[int(position)] = replaceValue
case int(position) == len(anyArray):
anyArray = append(anyArray, replaceValue)
default:
if defaultPaddingValue == "" {
panic("missing defaultPaddingValue")
}
for len(anyArray) < int(position) {
anyArray = append(anyArray, defaultPaddingValue)
}
anyArray = append(anyArray, replaceValue)
}
s.push(anyArray)
case "REMOVE_VALUE":
limit := math.Inf(1)
if len(ctx.AllExpression()) == 3 {
limit = s.pop().(float64)
}
value := s.pop()
array := s.pop().([]interface{})
result := []interface{}{}
for idx, e := range array {
if e != value || float64(idx) > limit {
result = append(result, e)
}
}
s.push(result)
case "REMOVE_VALUES":
values := s.pop().([]interface{})
array := s.pop().([]interface{})
result := []interface{}{}
for _, e := range array {
if !contains(values, e) {
result = append(result, e)
}
}
s.push(result)
case "REVERSE":
array := s.pop().([]interface{})
var reverse []interface{}
for _, e := range array {
reverse = append([]interface{}{e}, reverse...)
}
s.push(reverse)
case "SHIFT":
s.push(s.pop().([]interface{})[1:])
case "SLICE":
length := float64(-1)
full := true
if len(ctx.AllExpression()) == 3 {
length = s.pop().(float64)
full = false
}
start := int64(s.pop().(float64))
array := s.pop().([]interface{})
if start < 0 {
start = int64(len(array)) + start
}
if full {
length = float64(int64(len(array)) - start)
}
end := int64(0)
if length < 0 {
end = int64(len(array)) + int64(length)
} else {
end = start + int64(length)
}
s.push(array[start:end])
case "SORTED":
array := s.pop().([]interface{})
sort.Slice(array, func(i, j int) bool { return lt(array[i], array[j]) })
s.push(array)
case "SORTED_UNIQUE":
array := s.pop().([]interface{})
sort.Slice(array, func(i, j int) bool { return lt(array[i], array[j]) })
s.push(unique(array))
case "UNION":
array := s.pop().([]interface{})
for i := 1; i < len(ctx.AllExpression()); i++ {
array = append(array, s.pop().([]interface{})...)
}
sort.Slice(array, func(i, j int) bool { return lt(array[i], array[j]) })
s.push(array)
case "UNION_DISTINCT":
iset := New(s.pop().([]interface{})...)
for i := 1; i < len(ctx.AllExpression()); i++ {
iset = iset.Union(New(s.pop().([]interface{})...))
}
s.push(unique(iset.Values()))
case "UNIQUE":
s.push(unique(s.pop().([]interface{})))
case "UNSHIFT":
u := false
if len(ctx.AllExpression()) == 3 {
u = s.pop().(bool)
}
element := s.pop()
array := s.pop().([]interface{})
if u && contains(array, element) {
s.push(array)
} else {
s.push(append([]interface{}{element}, array...))
}
// Bit https://www.arangodb.com/docs/stable/aql/functions-bit.html
// case "BIT_AND":
// case "BIT_CONSTRUCT":
// case "BIT_DECONSTRUCT":
// case "BIT_FROM_STRING":
// case "BIT_NEGATE":
// case "BIT_OR":
// case "BIT_POPCOUNT":
// case "BIT_SHIFT_LEFT":
// case "BIT_SHIFT_RIGHT":
// case "BIT_TEST":
// case "BIT_TO_STRING":
// case "BIT_XOR":
// Date https://www.arangodb.com/docs/stable/aql/functions-date.html
// case "DATE_NOW":
// case "DATE_ISO8601":
// case "DATE_TIMESTAMP":
// case "IS_DATESTRING":
// case "DATE_DAYOFWEEK":
// case "DATE_YEAR":
// case "DATE_MONTH":
// case "DATE_DAY":
// case "DATE_HOUR":
// case "DATE_MINUTE":
// case "DATE_SECOND":
// case "DATE_MILLISECOND":
// case "DATE_DAYOFYEAR":
// case "DATE_ISOWEEK":
// case "DATE_LEAPYEAR":
// case "DATE_QUARTER":
// case "DATE_DAYS_IN_MONTH":
// case "DATE_TRUNC":
// case "DATE_ROUND":
// case "DATE_FORMAT":
// case "DATE_ADD":
// case "DATE_SUBTRACT":
// case "DATE_DIFF":
// case "DATE_COMPARE":
// Document https://www.arangodb.com/docs/stable/aql/functions-document.html
case "ATTRIBUTES":
if len(ctx.AllExpression()) == 3 {
s.pop() // always sort
}
removeInternal := false
if len(ctx.AllExpression()) >= 2 {
removeInternal = s.pop().(bool)
}
var keys []interface{}
for k := range s.pop().(map[string]interface{}) {
isInternalKey := strings.HasPrefix(k, "_")
if !removeInternal || !isInternalKey {
keys = append(keys, k)
}
}
sort.Slice(keys, func(i, j int) bool { return lt(keys[i], keys[j]) })
s.push(keys)
// case "COUNT":
case "HAS":
right, left := s.pop(), s.pop()
_, ok := left.(map[string]interface{})[right.(string)]
s.push(ok)
// case "KEEP":
// case "LENGTH":
// case "MATCHES":
case "MERGE":
var docs []map[string]interface{}
if len(ctx.AllExpression()) == 1 {
for _, doc := range s.pop().([]interface{}) {
docs = append([]map[string]interface{}{doc.(map[string]interface{})}, docs...)
}
} else {
for i := 0; i < len(ctx.AllExpression()); i++ {
docs = append(docs, s.pop().(map[string]interface{}))
}
}
doc := docs[len(docs)-1]
for i := len(docs) - 2; i >= 0; i-- {
for k, v := range docs[i] {
doc[k] = v
}
}
s.push(doc)
case "MERGE_RECURSIVE":
var doc map[string]interface{}
for i := 0; i < len(ctx.AllExpression()); i++ {
err := mergo.Merge(&doc, s.pop().(map[string]interface{}))
if err != nil {
panic(err)
}
}
s.push(doc)
// case "PARSE_IDENTIFIER":
// case "TRANSLATE":
// case "UNSET":
// case "UNSET_RECURSIVE":
case "VALUES":
removeInternal := false
if len(ctx.AllExpression()) == 2 {
removeInternal = s.pop().(bool)
}
var values []interface{}
for k, v := range s.pop().(map[string]interface{}) {
isInternalKey := strings.HasPrefix(k, "_")
if !removeInternal || !isInternalKey {
values = append(values, v)
}
}
sort.Slice(values, func(i, j int) bool { return lt(values[i], values[j]) })
s.push(values)
// case "ZIP":
// Numeric https://www.arangodb.com/docs/stable/aql/functions-numeric.html
case "ABS":
s.push(math.Abs(s.pop().(float64)))
case "ACOS":
v := s.pop().(float64)
asin := math.Acos(v)
if v > 1 || v < -1 {
s.push(nil)
} else {
s.push(asin)
}
case "ASIN":
v := s.pop().(float64)
asin := math.Asin(v)
if v > 1 || v < -1 {
s.push(nil)
} else {
s.push(asin)
}
case "ATAN":
s.push(math.Atan(s.pop().(float64)))
case "ATAN2":
s.push(math.Atan2(s.pop().(float64), s.pop().(float64)))
case "AVERAGE", "AVG":
count := 0
sum := float64(0)
array := s.pop().([]interface{})
for _, element := range array {
if element != nil {
count += 1
sum += toNumber(element)
}
}
if count == 0 {
s.push(nil)
} else {
s.push(sum / float64(count))
}
case "CEIL":
s.push(math.Ceil(s.pop().(float64)))
case "COS":
s.push(math.Cos(s.pop().(float64)))
case "DEGREES":
s.push(s.pop().(float64) * 180 / math.Pi)
case "EXP":
s.push(math.Exp(s.pop().(float64)))
case "EXP2":
s.push(math.Exp2(s.pop().(float64)))
case "FLOOR":
s.push(math.Floor(s.pop().(float64)))
case "LOG":
l := math.Log(s.pop().(float64))
if l <= 0 {
s.push(nil)
} else {
s.push(l)
}
case "LOG2":
l := math.Log2(s.pop().(float64))
if l <= 0 {
s.push(nil)
} else {
s.push(l)
}
case "LOG10":
l := math.Log10(s.pop().(float64))
if l <= 0 {
s.push(nil)
} else {
s.push(l)
}
case "MAX":
var set bool
var max float64
array := s.pop().([]interface{})
for _, element := range array {
if element != nil {
if !set || toNumber(element) > max {
max = toNumber(element)
set = true
}
}
}
if set {
s.push(max)
} else {
s.push(nil)
}
case "MEDIAN":
array := s.pop().([]interface{})
var numbers []float64
for _, element := range array {
if f, ok := element.(float64); ok {
numbers = append(numbers, f)
}
}
sort.Float64s(numbers) // sort the numbers
middlePos := len(numbers) / 2
switch {
case len(numbers) == 0:
s.push(nil)
case len(numbers)%2 == 1:
s.push(numbers[middlePos])
default:
s.push((numbers[middlePos-1] + numbers[middlePos]) / 2)
}
case "MIN":
var set bool
var min float64
array := s.pop().([]interface{})
for _, element := range array {
if element != nil {
if !set || toNumber(element) < min {
min = toNumber(element)
set = true
}
}
}
if set {
s.push(min)
} else {
s.push(nil)
}
// case "PERCENTILE":
case "PI":
s.push(math.Pi)
case "POW":
right, left := s.pop(), s.pop()
s.push(math.Pow(left.(float64), right.(float64)))
case "PRODUCT":
product := float64(1)
array := s.pop().([]interface{})
for _, element := range array {
if element != nil {
product *= toNumber(element)
}
}
s.push(product)
case "RADIANS":
s.push(s.pop().(float64) * math.Pi / 180)
case "RAND":
s.push(rand.Float64())
case "RANGE":
var array []interface{}
var start, end, step float64
if len(ctx.AllExpression()) == 2 {
right, left := s.pop(), s.pop()
start = math.Trunc(left.(float64))
end = math.Trunc(right.(float64))
step = 1
} else {
middle, right, left := s.pop(), s.pop(), s.pop()
start = left.(float64)
end = right.(float64)
step = middle.(float64)
}
for i := start; i <= end; i += step {
array = append(array, i)
}
s.push(array)
case "ROUND":
x := s.pop().(float64)
t := math.Trunc(x)
if math.Abs(x-t) == 0.5 {
s.push(x + 0.5)
} else {
s.push(math.Round(x))
}
case "SIN":
s.push(math.Sin(s.pop().(float64)))
case "SQRT":
s.push(math.Sqrt(s.pop().(float64)))
// case "STDDEV_POPULATION":
// case "STDDEV_SAMPLE":
// case "STDDEV":
case "SUM":
sum := float64(0)
array := s.pop().([]interface{})
for _, element := range array {
sum += toNumber(element)
}
s.push(sum)
case "TAN":
s.push(math.Tan(s.pop().(float64)))
// case "VARIANCE_POPULATION", "VARIANCE":
// case "VARIANCE_SAMPLE":
// String https://www.arangodb.com/docs/stable/aql/functions-string.html
// case "CHAR_LENGTH":
// case "CONCAT":
// case "CONCAT_SEPARATOR":
// case "CONTAINS":
// case "CRC32":
// case "ENCODE_URI_COMPONENT":
// case "FIND_FIRST":
// case "FIND_LAST":
// case "FNV64":
// case "IPV4_FROM_NUMBER":
// case "IPV4_TO_NUMBER":
// case "IS_IPV4":
// case "JSON_PARSE":
// case "JSON_STRINGIFY":
// case "LEFT":
// case "LENGTH":
// case "LEVENSHTEIN_DISTANCE":
// case "LIKE":
case "LOWER":
s.push(strings.ToLower(s.pop().(string)))
// case "LTRIM":
// case "MD5":
// case "NGRAM_POSITIONAL_SIMILARITY":
// case "NGRAM_SIMILARITY":
// case "RANDOM_TOKEN":
// case "REGEX_MATCHES":
// case "REGEX_SPLIT":
// case "REGEX_TEST":
// case "REGEX_REPLACE":
// case "REVERSE":
// case "RIGHT":
// case "RTRIM":
// case "SHA1":
// case "SHA512":
// case "SOUNDEX":
// case "SPLIT":
// case "STARTS_WITH":
// case "SUBSTITUTE":
// case "SUBSTRING":
// case "TOKENS":
// case "TO_BASE64":
// case "TO_HEX":
// case "TRIM":
case "UPPER":
s.push(strings.ToUpper(s.pop().(string)))
// case "UUID":
// Type cast https://www.arangodb.com/docs/stable/aql/functions-type-cast.html
case "TO_BOOL":
s.push(toBool(s.pop()))
case "TO_NUMBER":
s.push(toNumber(s.pop()))
// case "TO_STRING":
// case "TO_ARRAY":
// case "TO_LIST":
// case "IS_NULL":
// case "IS_BOOL":
// case "IS_NUMBER":
// case "IS_STRING":
// case "IS_ARRAY":
// case "IS_LIST":
// case "IS_OBJECT":
// case "IS_DOCUMENT":
// case "IS_DATESTRING":
// case "IS_IPV4":
// case "IS_KEY":
// case "TYPENAME":
}
}
func unique(array []interface{}) []interface{} {
seen := map[interface{}]bool{}
var filtered []interface{}
for _, e := range array {
_, ok := seen[e]
if !ok {
seen[e] = true
filtered = append(filtered, e)
}
}
return filtered
}
func contains(values []interface{}, e interface{}) bool {
for _, v := range values {
if e == v {
return true
}
}
return false
}
func stringSliceContains(values []string, e string) bool {
for _, v := range values {
if e == v {
return true
}
}
return false
}
var functionNames = []string{
"APPEND", "COUNT_DISTINCT", "COUNT_UNIQUE", "FIRST", "FLATTEN", "INTERLEAVE", "INTERSECTION", "JACCARD", "LAST",
"COUNT", "LENGTH", "MINUS", "NTH", "OUTERSECTION", "POP", "POSITION", "CONTAINS_ARRAY", "PUSH", "REMOVE_NTH",
"REPLACE_NTH", "REMOVE_VALUE", "REMOVE_VALUES", "REVERSE", "SHIFT", "SLICE", "SORTED", "SORTED_UNIQUE", "UNION",
"UNION_DISTINCT", "UNIQUE", "UNSHIFT", "BIT_AND", "BIT_CONSTRUCT", "BIT_DECONSTRUCT", "BIT_FROM_STRING",
"BIT_NEGATE", "BIT_OR", "BIT_POPCOUNT", "BIT_SHIFT_LEFT", "BIT_SHIFT_RIGHT", "BIT_TEST", "BIT_TO_STRING",
"BIT_XOR", "DATE_NOW", "DATE_ISO8601", "DATE_TIMESTAMP", "IS_DATESTRING", "DATE_DAYOFWEEK", "DATE_YEAR",
"DATE_MONTH", "DATE_DAY", "DATE_HOUR", "DATE_MINUTE", "DATE_SECOND", "DATE_MILLISECOND", "DATE_DAYOFYEAR",
"DATE_ISOWEEK", "DATE_LEAPYEAR", "DATE_QUARTER", "DATE_DAYS_IN_MONTH", "DATE_TRUNC", "DATE_ROUND", "DATE_FORMAT",
"DATE_ADD", "DATE_SUBTRACT", "DATE_DIFF", "DATE_COMPARE", "ATTRIBUTES", "COUNT", "HAS", "KEEP", "LENGTH",
"MATCHES", "MERGE", "MERGE_RECURSIVE", "PARSE_IDENTIFIER", "TRANSLATE", "UNSET", "UNSET_RECURSIVE", "VALUES",
"ZIP", "ABS", "ACOS", "ASIN", "ATAN", "ATAN2", "AVERAGE", "AVG", "CEIL", "COS", "DEGREES", "EXP", "EXP2", "FLOOR",
"LOG", "LOG2", "LOG10", "MAX", "MEDIAN", "MIN", "PERCENTILE", "PI", "POW", "PRODUCT", "RADIANS", "RAND", "RANGE",
"ROUND", "SIN", "SQRT", "STDDEV_POPULATION", "STDDEV_SAMPLE", "STDDEV", "SUM", "TAN", "VARIANCE_POPULATION",
"VARIANCE", "VARIANCE_SAMPLE", "CHAR_LENGTH", "CONCAT", "CONCAT_SEPARATOR", "CONTAINS", "CRC32",
"ENCODE_URI_COMPONENT", "FIND_FIRST", "FIND_LAST", "FNV64", "IPV4_FROM_NUMBER", "IPV4_TO_NUMBER", "IS_IPV4",
"JSON_PARSE", "JSON_STRINGIFY", "LEFT", "LENGTH", "LEVENSHTEIN_DISTANCE", "LIKE", "LOWER", "LTRIM", "MD5",
"NGRAM_POSITIONAL_SIMILARITY", "NGRAM_SIMILARITY", "RANDOM_TOKEN", "REGEX_MATCHES", "REGEX_SPLIT", "REGEX_TEST",
"REGEX_REPLACE", "REVERSE", "RIGHT", "RTRIM", "SHA1", "SHA512", "SOUNDEX", "SPLIT", "STARTS_WITH", "SUBSTITUTE",
"SUBSTRING", "TOKENS", "TO_BASE64", "TO_HEX", "TRIM", "UPPER", "UUID", "TO_BOOL", "TO_NUMBER", "TO_STRING",
"TO_ARRAY", "TO_LIST", "IS_NULL", "IS_BOOL", "IS_NUMBER", "IS_STRING", "IS_ARRAY", "IS_LIST", "IS_OBJECT",
"IS_DOCUMENT", "IS_DATESTRING", "IS_IPV4", "IS_KEY", "TYPENAME"}

380
caql/function_test.go Normal file
View File

@@ -0,0 +1,380 @@
package caql
import (
"encoding/json"
"math"
"reflect"
"testing"
)
func TestFunctions(t *testing.T) {
tests := []struct {
name string
saql string
wantRebuild string
wantValue interface{}
wantParseErr bool
wantRebuildErr bool
wantEvalErr bool
values string
}{
// https://www.arangodb.com/docs/3.7/aql/functions-array.html
{name: "APPEND", saql: `APPEND([1, 2, 3], [5, 6, 9])`, wantRebuild: `APPEND([1, 2, 3], [5, 6, 9])`, wantValue: jsonParse(`[1, 2, 3, 5, 6, 9]`)},
{name: "APPEND", saql: `APPEND([1, 2, 3], [3, 4, 5, 2, 9], true)`, wantRebuild: `APPEND([1, 2, 3], [3, 4, 5, 2, 9], true)`, wantValue: jsonParse(`[1, 2, 3, 4, 5, 9]`)},
{name: "COUNT_DISTINCT", saql: `COUNT_DISTINCT([1, 2, 3])`, wantRebuild: `COUNT_DISTINCT([1, 2, 3])`, wantValue: 3},
{name: "COUNT_DISTINCT", saql: `COUNT_DISTINCT(["yes", "no", "yes", "sauron", "no", "yes"])`, wantRebuild: `COUNT_DISTINCT(["yes", "no", "yes", "sauron", "no", "yes"])`, wantValue: 3},
{name: "FIRST", saql: `FIRST([1, 2, 3])`, wantRebuild: `FIRST([1, 2, 3])`, wantValue: 1},
{name: "FIRST", saql: `FIRST([])`, wantRebuild: `FIRST([])`, wantValue: nil},
// {name: "FLATTEN", saql: `FLATTEN([1, 2, [3, 4], 5, [6, 7], [8, [9, 10]]])`, wantRebuild: `FLATTEN([1, 2, [3, 4], 5, [6, 7], [8, [9, 10]]])`, wantValue:},
// {name: "FLATTEN", saql: `FLATTEN([1, 2, [3, 4], 5, [6, 7], [8, [9, 10]]], 2)`, wantRebuild: `FLATTEN([1, 2, [3, 4], 5, [6, 7], [8, [9, 10]]], 2)`, wantValue:},
// {name: "INTERLEAVE", saql: `INTERLEAVE([1, 1, 1], [2, 2, 2], [3, 3, 3])`, wantRebuild: `INTERLEAVE([1, 1, 1], [2, 2, 2], [3, 3, 3])`, wantValue:},
// {name: "INTERLEAVE", saql: `INTERLEAVE([1], [2, 2], [3, 3, 3])`, wantRebuild: `INTERLEAVE([1], [2, 2], [3, 3, 3])`, wantValue:},
{name: "INTERSECTION", saql: `INTERSECTION([1,2,3,4,5], [2,3,4,5,6], [3,4,5,6,7])`, wantRebuild: `INTERSECTION([1, 2, 3, 4, 5], [2, 3, 4, 5, 6], [3, 4, 5, 6, 7])`, wantValue: jsonParse(`[3, 4, 5]`)},
{name: "INTERSECTION", saql: `INTERSECTION([2,4,6], [8,10,12], [14,16,18])`, wantRebuild: `INTERSECTION([2, 4, 6], [8, 10, 12], [14, 16, 18])`, wantValue: jsonParse(`[]`)},
// {name: "JACCARD", saql: `JACCARD([1,2,3,4], [3,4,5,6])`, wantRebuild: `JACCARD([1,2,3,4], [3,4,5,6])`, wantValue: 0.3333333333333333},
// {name: "JACCARD", saql: `JACCARD([1,1,2,2,2,3], [2,2,3,4])`, wantRebuild: `JACCARD([1,1,2,2,2,3], [2,2,3,4])`, wantValue: 0.5},
// {name: "JACCARD", saql: `JACCARD([1,2,3], [])`, wantRebuild: `JACCARD([1, 2, 3], [])`, wantValue: 0},
// {name: "JACCARD", saql: `JACCARD([], [])`, wantRebuild: `JACCARD([], [])`, wantValue: 1},
{name: "LAST", saql: `LAST([1,2,3,4,5])`, wantRebuild: `LAST([1, 2, 3, 4, 5])`, wantValue: 5},
{name: "LENGTH", saql: `LENGTH("🥑")`, wantRebuild: `LENGTH("🥑")`, wantValue: 1},
{name: "LENGTH", saql: `LENGTH(1234)`, wantRebuild: `LENGTH(1234)`, wantValue: 4},
{name: "LENGTH", saql: `LENGTH([1,2,3,4,5,6,7])`, wantRebuild: `LENGTH([1, 2, 3, 4, 5, 6, 7])`, wantValue: 7},
{name: "LENGTH", saql: `LENGTH(false)`, wantRebuild: `LENGTH(false)`, wantValue: 0},
{name: "LENGTH", saql: `LENGTH({a:1, b:2, c:3, d:4, e:{f:5,g:6}})`, wantRebuild: `LENGTH({a: 1, b: 2, c: 3, d: 4, e: {f: 5, g: 6}})`, wantValue: 5},
{name: "MINUS", saql: `MINUS([1,2,3,4], [3,4,5,6], [5,6,7,8])`, wantRebuild: `MINUS([1, 2, 3, 4], [3, 4, 5, 6], [5, 6, 7, 8])`, wantValue: jsonParse(`[1, 2]`)},
{name: "NTH", saql: `NTH(["foo", "bar", "baz"], 2)`, wantRebuild: `NTH(["foo", "bar", "baz"], 2)`, wantValue: "baz"},
{name: "NTH", saql: `NTH(["foo", "bar", "baz"], 3)`, wantRebuild: `NTH(["foo", "bar", "baz"], 3)`, wantValue: nil},
{name: "NTH", saql: `NTH(["foo", "bar", "baz"], -1)`, wantRebuild: `NTH(["foo", "bar", "baz"], -1)`, wantValue: nil},
// {name: "OUTERSECTION", saql: `OUTERSECTION([1, 2, 3], [2, 3, 4], [3, 4, 5])`, wantRebuild: `OUTERSECTION([1, 2, 3], [2, 3, 4], [3, 4, 5])`, wantValue: jsonParse(`[1, 5]`)},
{name: "POP", saql: `POP([1, 2, 3, 4])`, wantRebuild: `POP([1, 2, 3, 4])`, wantValue: jsonParse(`[1, 2, 3]`)},
{name: "POP", saql: `POP([1])`, wantRebuild: `POP([1])`, wantValue: jsonParse(`[]`)},
{name: "POSITION", saql: `POSITION([2,4,6,8], 4)`, wantRebuild: `POSITION([2, 4, 6, 8], 4)`, wantValue: true},
{name: "POSITION", saql: `POSITION([2,4,6,8], 4, true)`, wantRebuild: `POSITION([2, 4, 6, 8], 4, true)`, wantValue: 1},
{name: "PUSH", saql: `PUSH([1, 2, 3], 4)`, wantRebuild: `PUSH([1, 2, 3], 4)`, wantValue: jsonParse(`[1, 2, 3, 4]`)},
{name: "PUSH", saql: `PUSH([1, 2, 2, 3], 2, true)`, wantRebuild: `PUSH([1, 2, 2, 3], 2, true)`, wantValue: jsonParse(`[1, 2, 2, 3]`)},
{name: "REMOVE_NTH", saql: `REMOVE_NTH(["a", "b", "c", "d", "e"], 1)`, wantRebuild: `REMOVE_NTH(["a", "b", "c", "d", "e"], 1)`, wantValue: jsonParse(`["a", "c", "d", "e"]`)},
{name: "REMOVE_NTH", saql: `REMOVE_NTH(["a", "b", "c", "d", "e"], -2)`, wantRebuild: `REMOVE_NTH(["a", "b", "c", "d", "e"], -2)`, wantValue: jsonParse(`["a", "b", "c", "e"]`)},
{name: "REPLACE_NTH", saql: `REPLACE_NTH(["a", "b", "c"], 1 , "z")`, wantRebuild: `REPLACE_NTH(["a", "b", "c"], 1, "z")`, wantValue: jsonParse(`["a", "z", "c"]`)},
{name: "REPLACE_NTH", saql: `REPLACE_NTH(["a", "b", "c"], 3 , "z")`, wantRebuild: `REPLACE_NTH(["a", "b", "c"], 3, "z")`, wantValue: jsonParse(`["a", "b", "c", "z"]`)},
{name: "REPLACE_NTH", saql: `REPLACE_NTH(["a", "b", "c"], 6, "z", "y")`, wantRebuild: `REPLACE_NTH(["a", "b", "c"], 6, "z", "y")`, wantValue: jsonParse(`["a", "b", "c", "y", "y", "y", "z"]`)},
{name: "REPLACE_NTH", saql: `REPLACE_NTH(["a", "b", "c"], -1, "z")`, wantRebuild: `REPLACE_NTH(["a", "b", "c"], -1, "z")`, wantValue: jsonParse(`["a", "b", "z"]`)},
{name: "REPLACE_NTH", saql: `REPLACE_NTH(["a", "b", "c"], -9, "z")`, wantRebuild: `REPLACE_NTH(["a", "b", "c"], -9, "z")`, wantValue: jsonParse(`["z", "b", "c"]`)},
{name: "REMOVE_VALUE", saql: `REMOVE_VALUE(["a", "b", "b", "a", "c"], "a")`, wantRebuild: `REMOVE_VALUE(["a", "b", "b", "a", "c"], "a")`, wantValue: jsonParse(`["b", "b", "c"]`)},
{name: "REMOVE_VALUE", saql: `REMOVE_VALUE(["a", "b", "b", "a", "c"], "a", 1)`, wantRebuild: `REMOVE_VALUE(["a", "b", "b", "a", "c"], "a", 1)`, wantValue: jsonParse(`["b", "b", "a", "c"]`)},
{name: "REMOVE_VALUES", saql: `REMOVE_VALUES(["a", "a", "b", "c", "d", "e", "f"], ["a", "f", "d"])`, wantRebuild: `REMOVE_VALUES(["a", "a", "b", "c", "d", "e", "f"], ["a", "f", "d"])`, wantValue: jsonParse(`["b", "c", "e"]`)},
{name: "REVERSE", saql: `REVERSE ([2,4,6,8,10])`, wantRebuild: `REVERSE([2, 4, 6, 8, 10])`, wantValue: jsonParse(`[10, 8, 6, 4, 2]`)},
{name: "SHIFT", saql: `SHIFT([1, 2, 3, 4])`, wantRebuild: `SHIFT([1, 2, 3, 4])`, wantValue: jsonParse(`[2, 3, 4]`)},
{name: "SHIFT", saql: `SHIFT([1])`, wantRebuild: `SHIFT([1])`, wantValue: jsonParse(`[]`)},
{name: "SLICE", saql: `SLICE([1, 2, 3, 4, 5], 0, 1)`, wantRebuild: `SLICE([1, 2, 3, 4, 5], 0, 1)`, wantValue: jsonParse(`[1]`)},
{name: "SLICE", saql: `SLICE([1, 2, 3, 4, 5], 1, 2)`, wantRebuild: `SLICE([1, 2, 3, 4, 5], 1, 2)`, wantValue: jsonParse(`[2, 3]`)},
{name: "SLICE", saql: `SLICE([1, 2, 3, 4, 5], 3)`, wantRebuild: `SLICE([1, 2, 3, 4, 5], 3)`, wantValue: jsonParse(`[4, 5]`)},
{name: "SLICE", saql: `SLICE([1, 2, 3, 4, 5], 1, -1)`, wantRebuild: `SLICE([1, 2, 3, 4, 5], 1, -1)`, wantValue: jsonParse(`[2, 3, 4]`)},
{name: "SLICE", saql: `SLICE([1, 2, 3, 4, 5], 0, -2)`, wantRebuild: `SLICE([1, 2, 3, 4, 5], 0, -2)`, wantValue: jsonParse(`[1, 2, 3]`)},
{name: "SLICE", saql: `SLICE([1, 2, 3, 4, 5], -3, 2)`, wantRebuild: `SLICE([1, 2, 3, 4, 5], -3, 2)`, wantValue: jsonParse(`[3, 4]`)},
{name: "SORTED", saql: `SORTED([8,4,2,10,6])`, wantRebuild: `SORTED([8, 4, 2, 10, 6])`, wantValue: jsonParse(`[2, 4, 6, 8, 10]`)},
{name: "SORTED_UNIQUE", saql: `SORTED_UNIQUE([8,4,2,10,6,2,8,6,4])`, wantRebuild: `SORTED_UNIQUE([8, 4, 2, 10, 6, 2, 8, 6, 4])`, wantValue: jsonParse(`[2, 4, 6, 8, 10]`)},
{name: "UNION", saql: `UNION([1, 2, 3], [1, 2])`, wantRebuild: `UNION([1, 2, 3], [1, 2])`, wantValue: jsonParse(`[1, 1, 2, 2, 3]`)},
{name: "UNION_DISTINCT", saql: `UNION_DISTINCT([1, 2, 3], [1, 2])`, wantRebuild: `UNION_DISTINCT([1, 2, 3], [1, 2])`, wantValue: jsonParse(`[1, 2, 3]`)},
{name: "UNIQUE", saql: `UNIQUE([1,2,2,3,3,3,4,4,4,4,5,5,5,5,5])`, wantRebuild: `UNIQUE([1, 2, 2, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 5])`, wantValue: jsonParse(`[1, 2, 3, 4, 5]`)},
{name: "UNSHIFT", saql: `UNSHIFT([1, 2, 3], 4)`, wantRebuild: `UNSHIFT([1, 2, 3], 4)`, wantValue: jsonParse(`[4, 1, 2, 3]`)},
{name: "UNSHIFT", saql: `UNSHIFT([1, 2, 3], 2, true)`, wantRebuild: `UNSHIFT([1, 2, 3], 2, true)`, wantValue: jsonParse(`[1, 2, 3]`)},
// https://www.arangodb.com/docs/3.7/aql/functions-bit.html
// {name: "BIT_CONSTRUCT", saql: `BIT_CONSTRUCT([1, 2, 3])`, wantRebuild: `BIT_CONSTRUCT([1, 2, 3])`, wantValue: 14},
// {name: "BIT_CONSTRUCT", saql: `BIT_CONSTRUCT([0, 4, 8])`, wantRebuild: `BIT_CONSTRUCT([0, 4, 8])`, wantValue: 273},
// {name: "BIT_CONSTRUCT", saql: `BIT_CONSTRUCT([0, 1, 10, 31])`, wantRebuild: `BIT_CONSTRUCT([0, 1, 10, 31])`, wantValue: 2147484675},
// {name: "BIT_DECONSTRUCT", saql: `BIT_DECONSTRUCT(14)`, wantRebuild: `BIT_DECONSTRUCT(14) `, wantValue: []interface{}{1, 2, 3}},
// {name: "BIT_DECONSTRUCT", saql: `BIT_DECONSTRUCT(273)`, wantRebuild: `BIT_DECONSTRUCT(273)`, wantValue: []interface{}{0, 4, 8}},
// {name: "BIT_DECONSTRUCT", saql: `BIT_DECONSTRUCT(2147484675)`, wantRebuild: `BIT_DECONSTRUCT(2147484675)`, wantValue: []interface{}{0, 1, 10, 31}},
// {name: "BIT_FROM_STRING", saql: `BIT_FROM_STRING("0111")`, wantRebuild: `BIT_FROM_STRING("0111")`, wantValue: 7},
// {name: "BIT_FROM_STRING", saql: `BIT_FROM_STRING("000000000000010")`, wantRebuild: `BIT_FROM_STRING("000000000000010")`, wantValue: 2},
// {name: "BIT_FROM_STRING", saql: `BIT_FROM_STRING("11010111011101")`, wantRebuild: `BIT_FROM_STRING("11010111011101")`, wantValue: 13789},
// {name: "BIT_FROM_STRING", saql: `BIT_FROM_STRING("100000000000000000000")`, wantRebuild: `BIT_FROM_STRING("100000000000000000000")`, wantValue: 1048756},
// {name: "BIT_NEGATE", saql: `BIT_NEGATE(0, 8)`, wantRebuild: `BIT_NEGATE(0, 8)`, wantValue: 255},
// {name: "BIT_NEGATE", saql: `BIT_NEGATE(0, 10)`, wantRebuild: `BIT_NEGATE(0, 10)`, wantValue: 1023},
// {name: "BIT_NEGATE", saql: `BIT_NEGATE(3, 4)`, wantRebuild: `BIT_NEGATE(3, 4)`, wantValue: 12},
// {name: "BIT_NEGATE", saql: `BIT_NEGATE(446359921, 32)`, wantRebuild: `BIT_NEGATE(446359921, 32)`, wantValue: 3848607374},
// {name: "BIT_OR", saql: `BIT_OR([1, 4, 8, 16])`, wantRebuild: `BIT_OR([1, 4, 8, 16])`, wantValue: 29},
// {name: "BIT_OR", saql: `BIT_OR([3, 7, 63])`, wantRebuild: `BIT_OR([3, 7, 63])`, wantValue: 63},
// {name: "BIT_OR", saql: `BIT_OR([255, 127, null, 63])`, wantRebuild: `BIT_OR([255, 127, null, 63])`, wantValue: 255},
// {name: "BIT_OR", saql: `BIT_OR(255, 127)`, wantRebuild: `BIT_OR(255, 127)`, wantValue: 255},
// {name: "BIT_OR", saql: `BIT_OR("foo")`, wantRebuild: `BIT_OR("foo")`, wantValue: nil},
// {name: "BIT_POPCOUNT", saql: `BIT_POPCOUNT(0)`, wantRebuild: `BIT_POPCOUNT(0)`, wantValue: 0},
// {name: "BIT_POPCOUNT", saql: `BIT_POPCOUNT(255)`, wantRebuild: `BIT_POPCOUNT(255)`, wantValue: 8},
// {name: "BIT_POPCOUNT", saql: `BIT_POPCOUNT(69399252)`, wantRebuild: `BIT_POPCOUNT(69399252)`, wantValue: 12},
// {name: "BIT_POPCOUNT", saql: `BIT_POPCOUNT("foo")`, wantRebuild: `BIT_POPCOUNT("foo")`, wantValue: nil},
// {name: "BIT_SHIFT_LEFT", saql: `BIT_SHIFT_LEFT(0, 1, 8)`, wantRebuild: `BIT_SHIFT_LEFT(0, 1, 8)`, wantValue: 0},
// {name: "BIT_SHIFT_LEFT", saql: `BIT_SHIFT_LEFT(7, 1, 16)`, wantRebuild: `BIT_SHIFT_LEFT(7, 1, 16)`, wantValue: 14},
// {name: "BIT_SHIFT_LEFT", saql: `BIT_SHIFT_LEFT(2, 10, 16)`, wantRebuild: `BIT_SHIFT_LEFT(2, 10, 16)`, wantValue: 2048},
// {name: "BIT_SHIFT_LEFT", saql: `BIT_SHIFT_LEFT(878836, 16, 32)`, wantRebuild: `BIT_SHIFT_LEFT(878836, 16, 32)`, wantValue: 1760821248},
// {name: "BIT_SHIFT_RIGHT", saql: `BIT_SHIFT_RIGHT(0, 1, 8)`, wantRebuild: `BIT_SHIFT_RIGHT(0, 1, 8)`, wantValue: 0},
// {name: "BIT_SHIFT_RIGHT", saql: `BIT_SHIFT_RIGHT(33, 1, 16)`, wantRebuild: `BIT_SHIFT_RIGHT(33, 1, 16)`, wantValue: 16},
// {name: "BIT_SHIFT_RIGHT", saql: `BIT_SHIFT_RIGHT(65536, 13, 16)`, wantRebuild: `BIT_SHIFT_RIGHT(65536, 13, 16)`, wantValue: 8},
// {name: "BIT_SHIFT_RIGHT", saql: `BIT_SHIFT_RIGHT(878836, 4, 32)`, wantRebuild: `BIT_SHIFT_RIGHT(878836, 4, 32)`, wantValue: 54927},
// {name: "BIT_TEST", saql: `BIT_TEST(0, 3)`, wantRebuild: `BIT_TEST(0, 3)`, wantValue: false},
// {name: "BIT_TEST", saql: `BIT_TEST(255, 0)`, wantRebuild: `BIT_TEST(255, 0)`, wantValue: true},
// {name: "BIT_TEST", saql: `BIT_TEST(7, 2)`, wantRebuild: `BIT_TEST(7, 2)`, wantValue: true},
// {name: "BIT_TEST", saql: `BIT_TEST(255, 8)`, wantRebuild: `BIT_TEST(255, 8)`, wantValue: false},
// {name: "BIT_TO_STRING", saql: `BIT_TO_STRING(7, 4)`, wantRebuild: `BIT_TO_STRING(7, 4)`, wantValue: "0111"},
// {name: "BIT_TO_STRING", saql: `BIT_TO_STRING(255, 8)`, wantRebuild: `BIT_TO_STRING(255, 8)`, wantValue: "11111111"},
// {name: "BIT_TO_STRING", saql: `BIT_TO_STRING(60, 8)`, wantRebuild: `BIT_TO_STRING(60, 8)`, wantValue: "00011110"},
// {name: "BIT_TO_STRING", saql: `BIT_TO_STRING(1048576, 32)`, wantRebuild: `BIT_TO_STRING(1048576, 32)`, wantValue: "00000000000100000000000000000000"},
// {name: "BIT_XOR", saql: `BIT_XOR([1, 4, 8, 16])`, wantRebuild: `BIT_XOR([1, 4, 8, 16])`, wantValue: 29},
// {name: "BIT_XOR", saql: `BIT_XOR([3, 7, 63])`, wantRebuild: `BIT_XOR([3, 7, 63])`, wantValue: 59},
// {name: "BIT_XOR", saql: `BIT_XOR([255, 127, null, 63])`, wantRebuild: `BIT_XOR([255, 127, null, 63])`, wantValue: 191},
// {name: "BIT_XOR", saql: `BIT_XOR(255, 257)`, wantRebuild: `BIT_XOR(255, 257)`, wantValue: 510},
// {name: "BIT_XOR", saql: `BIT_XOR("foo")`, wantRebuild: `BIT_XOR("foo")`, wantValue: nil},
// https://www.arangodb.com/docs/3.7/aql/functions-date.html
// DATE_TIMESTAMP("2014-05-07T14:19:09.522")
// DATE_TIMESTAMP("2014-05-07T14:19:09.522Z")
// DATE_TIMESTAMP("2014-05-07 14:19:09.522")
// DATE_TIMESTAMP("2014-05-07 14:19:09.522Z")
// DATE_TIMESTAMP(2014, 5, 7, 14, 19, 9, 522)
// DATE_TIMESTAMP(1399472349522)
// DATE_ISO8601("2014-05-07T14:19:09.522Z")
// DATE_ISO8601("2014-05-07 14:19:09.522Z")
// DATE_ISO8601(2014, 5, 7, 14, 19, 9, 522)
// DATE_ISO8601(1399472349522)
// {name: "DATE_TIMESTAMP", saql: `DATE_TIMESTAMP(2016, 12, -1)`, wantRebuild: `DATE_TIMESTAMP(2016, 12, -1)`, wantValue: nil},
// {name: "DATE_TIMESTAMP", saql: `DATE_TIMESTAMP(2016, 2, 32)`, wantRebuild: `DATE_TIMESTAMP(2016, 2, 32)`, wantValue: 1456963200000},
// {name: "DATE_TIMESTAMP", saql: `DATE_TIMESTAMP(1970, 1, 1, 26)`, wantRebuild: `DATE_TIMESTAMP(1970, 1, 1, 26)`, wantValue: 93600000},
// {name: "DATE_TRUNC", saql: `DATE_TRUNC('2017-02-03', 'month')`, wantRebuild: `DATE_TRUNC('2017-02-03', 'month')`, wantValue: "2017-02-01T00:00:00.000Z"},
// {name: "DATE_TRUNC", saql: `DATE_TRUNC('2017-02-03 04:05:06', 'hours')`, wantRebuild: `DATE_TRUNC('2017-02-03 04:05:06', 'hours')`, wantValue: "2017-02-03 04:00:00.000Z"},
// {name: "DATE_ROUND", saql: `DATE_ROUND('2000-04-28T11:11:11.111Z', 1, 'day')`, wantRebuild: `DATE_ROUND('2000-04-28T11:11:11.111Z', 1, 'day')`, wantValue: "2000-04-28T00:00:00.000Z"},
// {name: "DATE_ROUND", saql: `DATE_ROUND('2000-04-10T11:39:29Z', 15, 'minutes')`, wantRebuild: `DATE_ROUND('2000-04-10T11:39:29Z', 15, 'minutes')`, wantValue: "2000-04-10T11:30:00.000Z"},
// {name: "DATE_FORMAT", saql: `DATE_FORMAT(DATE_NOW(), "%q/%yyyy")`, wantRebuild: `DATE_FORMAT(DATE_NOW(), "%q/%yyyy")`},
// {name: "DATE_FORMAT", saql: `DATE_FORMAT(DATE_NOW(), "%dd.%mm.%yyyy %hh:%ii:%ss,%fff")`, wantRebuild: `DATE_FORMAT(DATE_NOW(), "%dd.%mm.%yyyy %hh:%ii:%ss,%fff")`, wantValue: "18.09.2015 15:30:49,374"},
// {name: "DATE_FORMAT", saql: `DATE_FORMAT("1969", "Summer of '%yy")`, wantRebuild: `DATE_FORMAT("1969", "Summer of '%yy")`, wantValue: "Summer of '69"},
// {name: "DATE_FORMAT", saql: `DATE_FORMAT("2016", "%%l = %l")`, wantRebuild: `DATE_FORMAT("2016", "%%l = %l")`, wantValue: "%l = 1"},
// {name: "DATE_FORMAT", saql: `DATE_FORMAT("2016-03-01", "%xxx%")`, wantRebuild: `DATE_FORMAT("2016-03-01", "%xxx%")`, wantValue: "063, trailing % ignored"},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_NOW(), -1, "day")`, wantRebuild: `DATE_ADD(DATE_NOW(), -1, "day")`, wantValue: "yesterday; also see DATE_SUBTRACT()"},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_NOW(), 3, "months")`, wantRebuild: `DATE_ADD(DATE_NOW(), 3, "months")`, wantValue: "in three months"},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_ADD("2015-04-01", 5, "years"), 1, "month")`, wantRebuild: `DATE_ADD(DATE_ADD("2015-04-01", 5, "years"), 1, "month")`, wantValue: "May 1st 2020"},
// {name: "DATE_ADD", saql: `DATE_ADD("2015-04-01", 12*5 + 1, "months")`, wantRebuild: `DATE_ADD("2015-04-01", 12*5 + 1, "months")`, wantValue: "also May 1st 2020"},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_TIMESTAMP(DATE_YEAR(DATE_NOW()), 12, 24), -4, "years")`, wantRebuild: `DATE_ADD(DATE_TIMESTAMP(DATE_YEAR(DATE_NOW()), 12, 24), -4, "years")`, wantValue: "Christmas four years ago"},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_ADD("2016-02", "month", 1), -1, "day")`, wantRebuild: `DATE_ADD(DATE_ADD("2016-02", "month", 1), -1, "day")`, wantValue: "last day of February (29th, because 2016 is a leap year!)"},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_NOW(), "P1Y")`, wantRebuild: `DATE_ADD(DATE_NOW(), "P1Y")`},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_NOW(), "P3M2W")`, wantRebuild: `DATE_ADD(DATE_NOW(), "P3M2W")`},
// {name: "DATE_ADD", saql: `DATE_ADD(DATE_NOW(), "P5DT26H")`, wantRebuild: `DATE_ADD(DATE_NOW(), "P5DT26H")`},
// {name: "DATE_ADD", saql: `DATE_ADD("2000-01-01", "PT4H")`, wantRebuild: `DATE_ADD("2000-01-01", "PT4H")`},
// {name: "DATE_ADD", saql: `DATE_ADD("2000-01-01", "PT30M44.4S"`, wantRebuild: `DATE_ADD("2000-01-01", "PT30M44.4S"`},
// {name: "DATE_ADD", saql: `DATE_ADD("2000-01-01", "P1Y2M3W4DT5H6M7.89S"`, wantRebuild: `DATE_ADD("2000-01-01", "P1Y2M3W4DT5H6M7.89S"`},
// {name: "DATE_SUBTRACT", saql: `DATE_SUBTRACT(DATE_NOW(), 1, "day")`, wantRebuild: `DATE_SUBTRACT(DATE_NOW(), 1, "day")`},
// {name: "DATE_SUBTRACT", saql: `DATE_SUBTRACT(DATE_TIMESTAMP(DATE_YEAR(DATE_NOW()), 12, 24), 4, "years")`, wantRebuild: `DATE_SUBTRACT(DATE_TIMESTAMP(DATE_YEAR(DATE_NOW()), 12, 24), 4, "years")`},
// {name: "DATE_SUBTRACT", saql: `DATE_SUBTRACT(DATE_ADD("2016-02", "month", 1), 1, "day")`, wantRebuild: `DATE_SUBTRACT(DATE_ADD("2016-02", "month", 1), 1, "day")`},
// {name: "DATE_SUBTRACT", saql: `DATE_SUBTRACT(DATE_NOW(), "P4D")`, wantRebuild: `DATE_SUBTRACT(DATE_NOW(), "P4D")`},
// {name: "DATE_SUBTRACT", saql: `DATE_SUBTRACT(DATE_NOW(), "PT1H3M")`, wantRebuild: `DATE_SUBTRACT(DATE_NOW(), "PT1H3M")`},
// DATE_COMPARE("1985-04-04", DATE_NOW(), "months", "days")
// DATE_COMPARE("1984-02-29", DATE_NOW(), "months", "days")
// DATE_COMPARE("2001-01-01T15:30:45.678Z", "2001-01-01T08:08:08.008Z", "years", "days")
// https://www.arangodb.com/docs/3.7/aql/functions-document.html
{name: "ATTRIBUTES", saql: `ATTRIBUTES({"foo": "bar", "_key": "123", "_custom": "yes"})`, wantRebuild: `ATTRIBUTES({"foo": "bar", "_key": "123", "_custom": "yes"})`, wantValue: jsonParse(`["_custom", "_key", "foo"]`)},
{name: "ATTRIBUTES", saql: `ATTRIBUTES({"foo": "bar", "_key": "123", "_custom": "yes"}, true)`, wantRebuild: `ATTRIBUTES({"foo": "bar", "_key": "123", "_custom": "yes"}, true)`, wantValue: jsonParse(`["foo"]`)},
{name: "ATTRIBUTES", saql: `ATTRIBUTES({"foo": "bar", "_key": "123", "_custom": "yes"}, false, true)`, wantRebuild: `ATTRIBUTES({"foo": "bar", "_key": "123", "_custom": "yes"}, false, true)`, wantValue: jsonParse(`["_custom", "_key", "foo"]`)},
{name: "HAS", saql: `HAS({name: "Jane"}, "name")`, wantRebuild: `HAS({name: "Jane"}, "name")`, wantValue: true},
{name: "HAS", saql: `HAS({name: "Jane"}, "age")`, wantRebuild: `HAS({name: "Jane"}, "age")`, wantValue: false},
{name: "HAS", saql: `HAS({name: null}, "name")`, wantRebuild: `HAS({name: null}, "name")`, wantValue: true},
// KEEP(doc, "firstname", "name", "likes")
// KEEP(doc, ["firstname", "name", "likes"])
// MATCHES({name: "jane", age: 27, active: true}, {age: 27, active: true})
// MATCHES({"test": 1}, [{"test": 1, "foo": "bar"}, {"foo": 1}, {"test": 1}], true)
{name: "MERGE", saql: `MERGE({"user1": {"name": "Jane"}}, {"user2": {"name": "Tom"}})`, wantRebuild: `MERGE({"user1": {"name": "Jane"}}, {"user2": {"name": "Tom"}})`, wantValue: jsonParse(`{"user1": {"name": "Jane"}, "user2": {"name": "Tom"}}`)},
{name: "MERGE", saql: `MERGE({"users": {"name": "Jane"}}, {"users": {"name": "Tom"}})`, wantRebuild: `MERGE({"users": {"name": "Jane"}}, {"users": {"name": "Tom"}})`, wantValue: jsonParse(`{"users": {"name": "Tom"}}`)},
{name: "MERGE", saql: `MERGE([{foo: "bar"}, {quux: "quetzalcoatl", ruled: true}, {bar: "baz", foo: "done"}])`, wantRebuild: `MERGE([{foo: "bar"}, {quux: "quetzalcoatl", ruled: true}, {bar: "baz", foo: "done"}])`, wantValue: jsonParse(`{"foo": "done", "quux": "quetzalcoatl", "ruled": true, "bar": "baz"}`)},
{name: "MERGE_RECURSIVE", saql: `MERGE_RECURSIVE({"user-1": {"name": "Jane", "livesIn": {"city": "LA"}}}, {"user-1": {"age": 42, "livesIn": {"state": "CA"}}})`, wantRebuild: `MERGE_RECURSIVE({"user-1": {"name": "Jane", "livesIn": {"city": "LA"}}}, {"user-1": {"age": 42, "livesIn": {"state": "CA"}}})`, wantValue: jsonParse(`{"user-1": {"name": "Jane", "livesIn": {"city": "LA", "state": "CA"}, "age": 42}}`)},
// {name: "TRANSLATE", saql: `TRANSLATE("FR", {US: "United States", UK: "United Kingdom", FR: "France"})`, wantRebuild: `TRANSLATE("FR", {US: "United States", UK: "United Kingdom", FR: "France"})`, wantValue: "France"},
// {name: "TRANSLATE", saql: `TRANSLATE(42, {foo: "bar", bar: "baz"})`, wantRebuild: `TRANSLATE(42, {foo: "bar", bar: "baz"})`, wantValue: 42},
// {name: "TRANSLATE", saql: `TRANSLATE(42, {foo: "bar", bar: "baz"}, "not found!")`, wantRebuild: `TRANSLATE(42, {foo: "bar", bar: "baz"}, "not found!")`, wantValue: "not found!"},
// UNSET(doc, "_id", "_key", "foo", "bar")
// UNSET(doc, ["_id", "_key", "foo", "bar"])
// UNSET_RECURSIVE(doc, "_id", "_key", "foo", "bar")
// UNSET_RECURSIVE(doc, ["_id", "_key", "foo", "bar"])
{name: "VALUES", saql: `VALUES({"_key": "users/jane", "name": "Jane", "age": 35})`, wantRebuild: `VALUES({"_key": "users/jane", "name": "Jane", "age": 35})`, wantValue: jsonParse(`[35, "Jane", "users/jane"]`)},
{name: "VALUES", saql: `VALUES({"_key": "users/jane", "name": "Jane", "age": 35}, true)`, wantRebuild: `VALUES({"_key": "users/jane", "name": "Jane", "age": 35}, true)`, wantValue: jsonParse(`[35, "Jane"]`)},
// {name: "ZIP", saql: `ZIP(["name", "active", "hobbies"], ["some user", true, ["swimming", "riding"]])`, wantRebuild: `ZIP(["name", "active", "hobbies"], ["some user", true, ["swimming", "riding"]])`, wantValue: jsonParse(`{"name": "some user", "active": true, "hobbies": ["swimming", "riding"]}`)},
// https://www.arangodb.com/docs/3.7/aql/functions-numeric.html
{name: "ABS", saql: `ABS(-5)`, wantRebuild: `ABS(-5)`, wantValue: 5},
{name: "ABS", saql: `ABS(+5)`, wantRebuild: `ABS(5)`, wantValue: 5},
{name: "ABS", saql: `ABS(3.5)`, wantRebuild: `ABS(3.5)`, wantValue: 3.5},
{name: "ACOS", saql: `ACOS(-1)`, wantRebuild: `ACOS(-1)`, wantValue: 3.141592653589793},
{name: "ACOS", saql: `ACOS(0)`, wantRebuild: `ACOS(0)`, wantValue: 1.5707963267948966},
{name: "ACOS", saql: `ACOS(1)`, wantRebuild: `ACOS(1)`, wantValue: 0},
{name: "ACOS", saql: `ACOS(2)`, wantRebuild: `ACOS(2)`, wantValue: nil},
{name: "ASIN", saql: `ASIN(1)`, wantRebuild: `ASIN(1)`, wantValue: 1.5707963267948966},
{name: "ASIN", saql: `ASIN(0)`, wantRebuild: `ASIN(0)`, wantValue: 0},
{name: "ASIN", saql: `ASIN(-1)`, wantRebuild: `ASIN(-1)`, wantValue: -1.5707963267948966},
{name: "ASIN", saql: `ASIN(2)`, wantRebuild: `ASIN(2)`, wantValue: nil},
{name: "ATAN", saql: `ATAN(-1)`, wantRebuild: `ATAN(-1)`, wantValue: -0.7853981633974483},
{name: "ATAN", saql: `ATAN(0)`, wantRebuild: `ATAN(0)`, wantValue: 0},
{name: "ATAN", saql: `ATAN(10)`, wantRebuild: `ATAN(10)`, wantValue: 1.4711276743037347},
{name: "AVERAGE", saql: `AVERAGE([5, 2, 9, 2])`, wantRebuild: `AVERAGE([5, 2, 9, 2])`, wantValue: 4.5},
{name: "AVERAGE", saql: `AVERAGE([-3, -5, 2])`, wantRebuild: `AVERAGE([-3, -5, 2])`, wantValue: -2},
{name: "AVERAGE", saql: `AVERAGE([999, 80, 4, 4, 4, 3, 3, 3])`, wantRebuild: `AVERAGE([999, 80, 4, 4, 4, 3, 3, 3])`, wantValue: 137.5},
{name: "CEIL", saql: `CEIL(2.49)`, wantRebuild: `CEIL(2.49)`, wantValue: 3},
{name: "CEIL", saql: `CEIL(2.50)`, wantRebuild: `CEIL(2.50)`, wantValue: 3},
{name: "CEIL", saql: `CEIL(-2.50)`, wantRebuild: `CEIL(-2.50)`, wantValue: -2},
{name: "CEIL", saql: `CEIL(-2.51)`, wantRebuild: `CEIL(-2.51)`, wantValue: -2},
{name: "COS", saql: `COS(1)`, wantRebuild: `COS(1)`, wantValue: 0.5403023058681398},
{name: "COS", saql: `COS(0)`, wantRebuild: `COS(0)`, wantValue: 1},
{name: "COS", saql: `COS(-3.141592653589783)`, wantRebuild: `COS(-3.141592653589783)`, wantValue: -1},
{name: "COS", saql: `COS(RADIANS(45))`, wantRebuild: `COS(RADIANS(45))`, wantValue: 0.7071067811865476},
{name: "DEGREES", saql: `DEGREES(0.7853981633974483)`, wantRebuild: `DEGREES(0.7853981633974483)`, wantValue: 45},
{name: "DEGREES", saql: `DEGREES(0)`, wantRebuild: `DEGREES(0)`, wantValue: 0},
{name: "DEGREES", saql: `DEGREES(3.141592653589793)`, wantRebuild: `DEGREES(3.141592653589793)`, wantValue: 180},
{name: "EXP", saql: `EXP(1)`, wantRebuild: `EXP(1)`, wantValue: 2.718281828459045},
{name: "EXP", saql: `EXP(10)`, wantRebuild: `EXP(10)`, wantValue: 22026.46579480671},
{name: "EXP", saql: `EXP(0)`, wantRebuild: `EXP(0)`, wantValue: 1},
{name: "EXP2", saql: `EXP2(16)`, wantRebuild: `EXP2(16)`, wantValue: 65536},
{name: "EXP2", saql: `EXP2(1)`, wantRebuild: `EXP2(1)`, wantValue: 2},
{name: "EXP2", saql: `EXP2(0)`, wantRebuild: `EXP2(0)`, wantValue: 1},
{name: "FLOOR", saql: `FLOOR(2.49)`, wantRebuild: `FLOOR(2.49)`, wantValue: 2},
{name: "FLOOR", saql: `FLOOR(2.50)`, wantRebuild: `FLOOR(2.50)`, wantValue: 2},
{name: "FLOOR", saql: `FLOOR(-2.50)`, wantRebuild: `FLOOR(-2.50)`, wantValue: -3},
{name: "FLOOR", saql: `FLOOR(-2.51)`, wantRebuild: `FLOOR(-2.51)`, wantValue: -3},
{name: "LOG", saql: `LOG(2.718281828459045)`, wantRebuild: `LOG(2.718281828459045)`, wantValue: 1},
{name: "LOG", saql: `LOG(10)`, wantRebuild: `LOG(10)`, wantValue: 2.302585092994046},
{name: "LOG", saql: `LOG(0)`, wantRebuild: `LOG(0)`, wantValue: nil},
{name: "LOG2", saql: `LOG2(1024)`, wantRebuild: `LOG2(1024)`, wantValue: 10},
{name: "LOG2", saql: `LOG2(8)`, wantRebuild: `LOG2(8)`, wantValue: 3},
{name: "LOG2", saql: `LOG2(0)`, wantRebuild: `LOG2(0)`, wantValue: nil},
{name: "LOG10", saql: `LOG10(10000)`, wantRebuild: `LOG10(10000)`, wantValue: 4},
{name: "LOG10", saql: `LOG10(10)`, wantRebuild: `LOG10(10)`, wantValue: 1},
{name: "LOG10", saql: `LOG10(0)`, wantRebuild: `LOG10(0)`, wantValue: nil},
{name: "MAX", saql: `MAX([5, 9, -2, null, 1])`, wantRebuild: `MAX([5, 9, -2, null, 1])`, wantValue: 9},
{name: "MAX", saql: `MAX([null, null])`, wantRebuild: `MAX([null, null])`, wantValue: nil},
{name: "MEDIAN", saql: `MEDIAN([1, 2, 3])`, wantRebuild: `MEDIAN([1, 2, 3])`, wantValue: 2},
{name: "MEDIAN", saql: `MEDIAN([1, 2, 3, 4])`, wantRebuild: `MEDIAN([1, 2, 3, 4])`, wantValue: 2.5},
{name: "MEDIAN", saql: `MEDIAN([4, 2, 3, 1])`, wantRebuild: `MEDIAN([4, 2, 3, 1])`, wantValue: 2.5},
{name: "MEDIAN", saql: `MEDIAN([999, 80, 4, 4, 4, 3, 3, 3])`, wantRebuild: `MEDIAN([999, 80, 4, 4, 4, 3, 3, 3])`, wantValue: 4},
{name: "MIN", saql: `MIN([5, 9, -2, null, 1])`, wantRebuild: `MIN([5, 9, -2, null, 1])`, wantValue: -2},
{name: "MIN", saql: `MIN([null, null])`, wantRebuild: `MIN([null, null])`, wantValue: nil},
// {name: "PERCENTILE", saql: `PERCENTILE([1, 2, 3, 4], 50)`, wantRebuild: `PERCENTILE([1, 2, 3, 4], 50)`, wantValue: 2},
// {name: "PERCENTILE", saql: `PERCENTILE([1, 2, 3, 4], 50, "rank")`, wantRebuild: `PERCENTILE([1, 2, 3, 4], 50, "rank")`, wantValue: 2},
// {name: "PERCENTILE", saql: `PERCENTILE([1, 2, 3, 4], 50, "interpolation")`, wantRebuild: `PERCENTILE([1, 2, 3, 4], 50, "interpolation")`, wantValue: 2.5},
{name: "PI", saql: `PI()`, wantRebuild: `PI()`, wantValue: 3.141592653589793},
{name: "POW", saql: `POW(2, 4)`, wantRebuild: `POW(2, 4)`, wantValue: 16},
{name: "POW", saql: `POW(5, -1)`, wantRebuild: `POW(5, -1)`, wantValue: 0.2},
{name: "POW", saql: `POW(5, 0)`, wantRebuild: `POW(5, 0)`, wantValue: 1},
{name: "PRODUCT", saql: `PRODUCT([1, 2, 3, 4])`, wantRebuild: `PRODUCT([1, 2, 3, 4])`, wantValue: 24},
{name: "PRODUCT", saql: `PRODUCT([null, -5, 6])`, wantRebuild: `PRODUCT([null, -5, 6])`, wantValue: -30},
{name: "PRODUCT", saql: `PRODUCT([])`, wantRebuild: `PRODUCT([])`, wantValue: 1},
{name: "RADIANS", saql: `RADIANS(180)`, wantRebuild: `RADIANS(180)`, wantValue: 3.141592653589793},
{name: "RADIANS", saql: `RADIANS(90)`, wantRebuild: `RADIANS(90)`, wantValue: 1.5707963267948966},
{name: "RADIANS", saql: `RADIANS(0)`, wantRebuild: `RADIANS(0)`, wantValue: 0},
// {name: "RAND", saql: `RAND()`, wantRebuild: `RAND()`, wantValue: 0.3503170117504508},
// {name: "RAND", saql: `RAND()`, wantRebuild: `RAND()`, wantValue: 0.6138226173882478},
{name: "RANGE", saql: `RANGE(1, 4)`, wantRebuild: `RANGE(1, 4)`, wantValue: []interface{}{float64(1), float64(2), float64(3), float64(4)}},
{name: "RANGE", saql: `RANGE(1, 4, 2)`, wantRebuild: `RANGE(1, 4, 2)`, wantValue: []interface{}{float64(1), float64(3)}},
{name: "RANGE", saql: `RANGE(1, 4, 3)`, wantRebuild: `RANGE(1, 4, 3)`, wantValue: []interface{}{float64(1), float64(4)}},
{name: "RANGE", saql: `RANGE(1.5, 2.5)`, wantRebuild: `RANGE(1.5, 2.5)`, wantValue: []interface{}{float64(1), float64(2)}},
{name: "RANGE", saql: `RANGE(1.5, 2.5, 1)`, wantRebuild: `RANGE(1.5, 2.5, 1)`, wantValue: []interface{}{1.5, 2.5}},
{name: "RANGE", saql: `RANGE(1.5, 2.5, 0.5)`, wantRebuild: `RANGE(1.5, 2.5, 0.5)`, wantValue: []interface{}{1.5, 2.0, 2.5}},
{name: "RANGE", saql: `RANGE(-0.75, 1.1, 0.5)`, wantRebuild: `RANGE(-0.75, 1.1, 0.5)`, wantValue: []interface{}{-0.75, -0.25, 0.25, 0.75}},
{name: "ROUND", saql: `ROUND(2.49)`, wantRebuild: `ROUND(2.49)`, wantValue: 2},
{name: "ROUND", saql: `ROUND(2.50)`, wantRebuild: `ROUND(2.50)`, wantValue: 3},
{name: "ROUND", saql: `ROUND(-2.50)`, wantRebuild: `ROUND(-2.50)`, wantValue: -2},
{name: "ROUND", saql: `ROUND(-2.51)`, wantRebuild: `ROUND(-2.51)`, wantValue: -3},
{name: "SQRT", saql: `SQRT(9)`, wantRebuild: `SQRT(9)`, wantValue: 3},
{name: "SQRT", saql: `SQRT(2)`, wantRebuild: `SQRT(2)`, wantValue: 1.4142135623730951},
{name: "POW", saql: `POW(4096, 1/4)`, wantRebuild: `POW(4096, 1 / 4)`, wantValue: 8},
{name: "POW", saql: `POW(27, 1/3)`, wantRebuild: `POW(27, 1 / 3)`, wantValue: 3},
{name: "POW", saql: `POW(9, 1/2)`, wantRebuild: `POW(9, 1 / 2)`, wantValue: 3},
// {name: "STDDEV_POPULATION", saql: `STDDEV_POPULATION([1, 3, 6, 5, 2])`, wantRebuild: `STDDEV_POPULATION([1, 3, 6, 5, 2])`, wantValue: 1.854723699099141},
// {name: "STDDEV_SAMPLE", saql: `STDDEV_SAMPLE([1, 3, 6, 5, 2])`, wantRebuild: `STDDEV_SAMPLE([1, 3, 6, 5, 2])`, wantValue: 2.0736441353327724},
{name: "SUM", saql: `SUM([1, 2, 3, 4])`, wantRebuild: `SUM([1, 2, 3, 4])`, wantValue: 10},
{name: "SUM", saql: `SUM([null, -5, 6])`, wantRebuild: `SUM([null, -5, 6])`, wantValue: 1},
{name: "SUM", saql: `SUM([])`, wantRebuild: `SUM([])`, wantValue: 0},
{name: "TAN", saql: `TAN(10)`, wantRebuild: `TAN(10)`, wantValue: 0.6483608274590866},
{name: "TAN", saql: `TAN(5)`, wantRebuild: `TAN(5)`, wantValue: -3.380515006246586},
{name: "TAN", saql: `TAN(0)`, wantRebuild: `TAN(0)`, wantValue: 0},
// {name: "VARIANCE_POPULATION", saql: `VARIANCE_POPULATION([1, 3, 6, 5, 2])`, wantRebuild: `VARIANCE_POPULATION([1, 3, 6, 5, 2])`, wantValue: 3.4400000000000004},
// {name: "VARIANCE_SAMPLE", saql: `VARIANCE_SAMPLE([1, 3, 6, 5, 2])`, wantRebuild: `VARIANCE_SAMPLE([1, 3, 6, 5, 2])`, wantValue: 4.300000000000001},
// Errors
{name: "Function Error 1", saql: "UNKNOWN(value)", wantRebuild: "UNKNOWN(value)", wantRebuildErr: true, wantEvalErr: true, values: `{"value": true}`},
{name: "Function Error 2", saql: "ABS(value, value2)", wantRebuild: "ABS(value, value2)", wantEvalErr: true, values: `{"value": true, "value2": false}`},
{name: "Function Error 3", saql: `ABS("abs")`, wantRebuild: `ABS("abs")`, wantEvalErr: true},
}
for _, tt := range tests {
parser := &Parser{}
t.Run(tt.name, func(t *testing.T) {
expr, err := parser.Parse(tt.saql)
if (err != nil) != tt.wantParseErr {
t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantParseErr)
if expr != nil {
t.Error(expr.String())
}
return
}
if err != nil {
return
}
got, err := expr.String()
if (err != nil) != tt.wantRebuildErr {
t.Error(expr.String())
t.Errorf("String() error = %v, wantErr %v", err, tt.wantParseErr)
return
}
if err != nil {
return
}
if got != tt.wantRebuild {
t.Errorf("String() got = %v, want %v", got, tt.wantRebuild)
}
var myJson map[string]interface{}
if tt.values != "" {
err = json.Unmarshal([]byte(tt.values), &myJson)
if err != nil {
t.Fatal(err)
}
}
value, err := expr.Eval(myJson)
if (err != nil) != tt.wantEvalErr {
t.Error(expr.String())
t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantParseErr)
return
}
if err != nil {
return
}
wantValue := tt.wantValue
if i, ok := wantValue.(int); ok {
wantValue = float64(i)
}
valueFloat, ok := value.(float64)
wantValueFloat, ok2 := wantValue.(float64)
if ok && ok2 {
if math.Abs(valueFloat-wantValueFloat) > 0.0001 {
t.Error(expr.String())
t.Errorf("Eval() got = %T %#v, want %T %#v", value, value, wantValue, wantValue)
}
} else {
if !reflect.DeepEqual(value, wantValue) {
t.Error(expr.String())
t.Errorf("Eval() got = %T %#v, want %T %#v", value, value, wantValue, wantValue)
}
}
})
}
}
func jsonParse(s string) interface{} {
if s == "" {
return nil
}
var j interface{}
err := json.Unmarshal([]byte(s), &j)
if err != nil {
panic(s + err.Error())
}
return j
}

355
caql/interpreter.go Normal file
View File

@@ -0,0 +1,355 @@
package caql
import (
"fmt"
"strconv"
"strings"
"github.com/SecurityBrewery/catalyst/generated/caql/parser"
)
type aqlInterpreter struct {
*parser.BaseCAQLParserListener
values map[string]interface{}
stack []interface{}
errs []error
}
// push is a helper function for pushing new node to the listener Stack.
func (s *aqlInterpreter) push(i interface{}) {
s.stack = append(s.stack, i)
}
// pop is a helper function for poping a node from the listener Stack.
func (s *aqlInterpreter) pop() (n interface{}) {
// Check that we have nodes in the stack.
size := len(s.stack)
if size < 1 {
s.appendErrors(ErrStack)
return
}
// Pop the last value from the Stack.
n, s.stack = s.stack[size-1], s.stack[:size-1]
return
}
func (s *aqlInterpreter) binaryPop() (interface{}, interface{}) {
right, left := s.pop(), s.pop()
return left, right
}
// ExitExpression is called when production expression is exited.
func (s *aqlInterpreter) ExitExpression(ctx *parser.ExpressionContext) {
switch {
case ctx.Value_literal() != nil:
// pass
case ctx.Reference() != nil:
// pass
case ctx.Operator_unary() != nil:
// pass
case ctx.T_PLUS() != nil:
s.push(plus(s.binaryPop()))
case ctx.T_MINUS() != nil:
s.push(minus(s.binaryPop()))
case ctx.T_TIMES() != nil:
s.push(times(s.binaryPop()))
case ctx.T_DIV() != nil:
s.push(div(s.binaryPop()))
case ctx.T_MOD() != nil:
s.push(mod(s.binaryPop()))
case ctx.T_RANGE() != nil:
s.push(aqlrange(s.binaryPop()))
case ctx.T_LT() != nil && ctx.GetEq_op() == nil:
s.push(lt(s.binaryPop()))
case ctx.T_GT() != nil && ctx.GetEq_op() == nil:
s.push(gt(s.binaryPop()))
case ctx.T_LE() != nil && ctx.GetEq_op() == nil:
s.push(le(s.binaryPop()))
case ctx.T_GE() != nil && ctx.GetEq_op() == nil:
s.push(ge(s.binaryPop()))
case ctx.T_IN() != nil && ctx.GetEq_op() == nil:
s.push(maybeNot(ctx, in(s.binaryPop())))
case ctx.T_EQ() != nil && ctx.GetEq_op() == nil:
s.push(eq(s.binaryPop()))
case ctx.T_NE() != nil && ctx.GetEq_op() == nil:
s.push(ne(s.binaryPop()))
case ctx.T_ALL() != nil && ctx.GetEq_op() != nil:
right, left := s.pop(), s.pop()
s.push(all(left.([]interface{}), getOp(ctx.GetEq_op().GetTokenType()), right))
case ctx.T_ANY() != nil && ctx.GetEq_op() != nil:
right, left := s.pop(), s.pop()
s.push(any(left.([]interface{}), getOp(ctx.GetEq_op().GetTokenType()), right))
case ctx.T_NONE() != nil && ctx.GetEq_op() != nil:
right, left := s.pop(), s.pop()
s.push(none(left.([]interface{}), getOp(ctx.GetEq_op().GetTokenType()), right))
case ctx.T_ALL() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
right, left := s.pop(), s.pop()
s.push(all(left.([]interface{}), in, right))
case ctx.T_ANY() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
right, left := s.pop(), s.pop()
s.push(any(left.([]interface{}), in, right))
case ctx.T_NONE() != nil && ctx.T_NOT() != nil && ctx.T_IN() != nil:
right, left := s.pop(), s.pop()
s.push(none(left.([]interface{}), in, right))
case ctx.T_LIKE() != nil:
m, err := like(s.binaryPop())
s.appendErrors(err)
s.push(maybeNot(ctx, m))
case ctx.T_REGEX_MATCH() != nil:
m, err := regexMatch(s.binaryPop())
s.appendErrors(err)
s.push(maybeNot(ctx, m))
case ctx.T_REGEX_NON_MATCH() != nil:
m, err := regexNonMatch(s.binaryPop())
s.appendErrors(err)
s.push(maybeNot(ctx, m))
case ctx.T_AND() != nil:
s.push(and(s.binaryPop()))
case ctx.T_OR() != nil:
s.push(or(s.binaryPop()))
case ctx.T_QUESTION() != nil && len(ctx.AllExpression()) == 3:
right, middle, left := s.pop(), s.pop(), s.pop()
s.push(ternary(left, middle, right))
case ctx.T_QUESTION() != nil && len(ctx.AllExpression()) == 2:
right, left := s.pop(), s.pop()
s.push(ternary(left, nil, right))
default:
panic("unkown expression")
}
}
func (s *aqlInterpreter) appendErrors(err error) {
if err != nil {
s.errs = append(s.errs, err)
}
}
// ExitOperator_unary is called when production operator_unary is exited.
func (s *aqlInterpreter) ExitOperator_unary(ctx *parser.Operator_unaryContext) {
value := s.pop()
switch {
case ctx.T_PLUS() != nil:
s.push(value.(float64))
case ctx.T_MINUS() != nil:
s.push(-value.(float64))
case ctx.T_NOT() != nil:
s.push(!toBool(value))
default:
panic(fmt.Sprintf("unexpected operation: %s", ctx.GetText()))
}
}
// ExitReference is called when production reference is exited.
func (s *aqlInterpreter) ExitReference(ctx *parser.ReferenceContext) {
switch {
case ctx.DOT() != nil:
reference := s.pop()
s.push(reference.(map[string]interface{})[ctx.T_STRING().GetText()])
case ctx.T_STRING() != nil:
s.push(s.getVar(ctx.T_STRING().GetText()))
case ctx.Compound_value() != nil:
// pass
case ctx.Function_call() != nil:
// pass
case ctx.T_OPEN() != nil:
// pass
case ctx.T_ARRAY_OPEN() != nil:
key := s.pop()
reference := s.pop()
if f, ok := key.(float64); ok {
index := int(f)
if index < 0 {
index = len(reference.([]interface{})) + index
}
s.push(reference.([]interface{})[index])
return
}
s.push(reference.(map[string]interface{})[key.(string)])
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}
// ExitCompound_value is called when production compound_value is exited.
func (s *aqlInterpreter) ExitCompound_value(ctx *parser.Compound_valueContext) {
// pass
}
// ExitFunction_call is called when production function_call is exited.
func (s *aqlInterpreter) ExitFunction_call(ctx *parser.Function_callContext) {
s.function(ctx)
}
// ExitValue_literal is called when production value_literal is exited.
func (s *aqlInterpreter) ExitValue_literal(ctx *parser.Value_literalContext) {
switch {
case ctx.T_QUOTED_STRING() != nil:
st, err := unquote(ctx.GetText())
s.appendErrors(err)
s.push(st)
case ctx.T_INT() != nil:
t := ctx.GetText()
switch {
case strings.HasPrefix(strings.ToLower(t), "0b"):
i64, err := strconv.ParseInt(t[2:], 2, 64)
s.appendErrors(err)
s.push(float64(i64))
case strings.HasPrefix(strings.ToLower(t), "0x"):
i64, err := strconv.ParseInt(t[2:], 16, 64)
s.appendErrors(err)
s.push(float64(i64))
default:
i, err := strconv.Atoi(t)
s.appendErrors(err)
s.push(float64(i))
}
case ctx.T_FLOAT() != nil:
i, err := strconv.ParseFloat(ctx.GetText(), 64)
s.appendErrors(err)
s.push(i)
case ctx.T_NULL() != nil:
s.push(nil)
case ctx.T_TRUE() != nil:
s.push(true)
case ctx.T_FALSE() != nil:
s.push(false)
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}
// ExitArray is called when production array is exited.
func (s *aqlInterpreter) ExitArray(ctx *parser.ArrayContext) {
array := []interface{}{}
for range ctx.AllExpression() {
// prepend element
array = append([]interface{}{s.pop()}, array...)
}
s.push(array)
}
// ExitObject is called when production object is exited.
func (s *aqlInterpreter) ExitObject(ctx *parser.ObjectContext) {
object := map[string]interface{}{}
for range ctx.AllObject_element() {
key, value := s.pop(), s.pop()
object[key.(string)] = value
}
s.push(object)
}
// ExitObject_element is called when production object_element is exited.
func (s *aqlInterpreter) ExitObject_element(ctx *parser.Object_elementContext) {
switch {
case ctx.T_STRING() != nil:
s.push(ctx.GetText())
s.push(s.getVar(ctx.GetText()))
case ctx.Object_element_name() != nil, ctx.T_ARRAY_OPEN() != nil:
key, value := s.pop(), s.pop()
s.push(key)
s.push(value)
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}
// ExitObject_element_name is called when production object_element_name is exited.
func (s *aqlInterpreter) ExitObject_element_name(ctx *parser.Object_element_nameContext) {
switch {
case ctx.T_STRING() != nil:
s.push(ctx.T_STRING().GetText())
case ctx.T_QUOTED_STRING() != nil:
st, err := unquote(ctx.T_QUOTED_STRING().GetText())
if err != nil {
s.appendErrors(fmt.Errorf("%w: %s", err, ctx.GetText()))
}
s.push(st)
default:
panic(fmt.Sprintf("unexpected value: %s", ctx.GetText()))
}
}
func (s *aqlInterpreter) getVar(identifier string) interface{} {
v, ok := s.values[identifier]
if !ok {
s.appendErrors(ErrUndefined)
}
return v
}
func maybeNot(ctx *parser.ExpressionContext, m bool) bool {
if ctx.T_NOT() != nil {
return !m
}
return m
}
func getOp(tokenType int) func(left, right interface{}) bool {
switch tokenType {
case parser.CAQLLexerT_EQ:
return eq
case parser.CAQLLexerT_NE:
return ne
case parser.CAQLLexerT_LT:
return lt
case parser.CAQLLexerT_GT:
return gt
case parser.CAQLLexerT_LE:
return le
case parser.CAQLLexerT_GE:
return ge
case parser.CAQLLexerT_IN:
return in
default:
panic("unkown token type")
}
}
func all(slice []interface{}, op func(interface{}, interface{}) bool, expr interface{}) bool {
for _, e := range slice {
if !op(e, expr) {
return false
}
}
return true
}
func any(slice []interface{}, op func(interface{}, interface{}) bool, expr interface{}) bool {
for _, e := range slice {
if op(e, expr) {
return true
}
}
return false
}
func none(slice []interface{}, op func(interface{}, interface{}) bool, expr interface{}) bool {
for _, e := range slice {
if op(e, expr) {
return false
}
}
return true
}

497
caql/operations.go Normal file
View File

@@ -0,0 +1,497 @@
package caql
import (
"math"
"regexp"
"sort"
"strconv"
"strings"
)
// Logical operators https://www.arangodb.com/docs/3.7/aql/operators.html#logical-operators
func or(left, right interface{}) interface{} {
if toBool(left) {
return left
}
return right
}
func and(left, right interface{}) interface{} {
if !toBool(left) {
return left
}
return right
}
func toBool(i interface{}) bool {
switch v := i.(type) {
case nil:
return false
case bool:
return v
case int:
return v != 0
case float64:
return v != 0
case string:
return v != ""
case []interface{}:
return true
case map[string]interface{}:
return true
default:
panic("bool conversion failed")
}
}
// Arithmetic operators https://www.arangodb.com/docs/3.7/aql/operators.html#arithmetic-operators
func plus(left, right interface{}) float64 {
return toNumber(left) + toNumber(right)
}
func minus(left, right interface{}) float64 {
return toNumber(left) - toNumber(right)
}
func times(left, right interface{}) float64 {
return round(toNumber(left) * toNumber(right))
}
func round(r float64) float64 {
return math.Round(r*100000) / 100000
}
func div(left, right interface{}) float64 {
b := toNumber(right)
if b == 0 {
return 0
}
return round(toNumber(left) / b)
}
func mod(left, right interface{}) float64 {
return math.Mod(toNumber(left), toNumber(right))
}
func toNumber(i interface{}) float64 {
switch v := i.(type) {
case nil:
return 0
case bool:
if v {
return 1
}
return 0
case float64:
switch {
case math.IsNaN(v):
return 0
case math.IsInf(v, 0):
return 0
}
return v
case string:
f, err := strconv.ParseFloat(strings.TrimSpace(v), 64)
if err != nil {
return 0
}
return f
case []interface{}:
if len(v) == 0 {
return 0
}
if len(v) == 1 {
return toNumber(v[0])
}
return 0
case map[string]interface{}:
return 0
default:
panic("number conversion error")
}
}
// Logical operators https://www.arangodb.com/docs/3.7/aql/operators.html#logical-operators
// Order https://www.arangodb.com/docs/3.7/aql/fundamentals-type-value-order.html
func eq(left, right interface{}) bool {
leftV, rightV := typeValue(left), typeValue(right)
if leftV != rightV {
return false
}
switch l := left.(type) {
case nil:
return true
case bool, float64, string:
return left == right
case []interface{}:
ra := right.([]interface{})
max := len(l)
if len(ra) > max {
max = len(ra)
}
for i := 0; i < max; i++ {
var li interface{} = nil
var rai interface{} = nil
if len(l) > i {
li = l[i]
}
if len(ra) > i {
rai = ra[i]
}
if !eq(li, rai) {
return false
}
}
return true
case map[string]interface{}:
ro := right.(map[string]interface{})
for _, key := range keys(l, ro) {
var li interface{} = nil
var rai interface{} = nil
if lv, ok := l[key]; ok {
li = lv
}
if rv, ok := ro[key]; ok {
rai = rv
}
if !eq(li, rai) {
return false
}
}
return true
default:
panic("unknown type")
}
}
func ne(left, right interface{}) bool {
return !eq(left, right)
}
func lt(left, right interface{}) bool {
leftV, rightV := typeValue(left), typeValue(right)
if leftV != rightV {
return leftV < rightV
}
switch l := left.(type) {
case nil:
return false
case bool:
return toNumber(l) < toNumber(right)
case int:
return l < right.(int)
case float64:
return l < right.(float64)
case string:
return l < right.(string)
case []interface{}:
ra := right.([]interface{})
max := len(l)
if len(ra) > max {
max = len(ra)
}
for i := 0; i < max; i++ {
var li interface{} = nil
var rai interface{} = nil
if len(l) > i {
li = l[i]
}
if len(ra) > i {
rai = ra[i]
}
if !eq(li, rai) {
return lt(li, rai)
}
}
return false
case map[string]interface{}:
ro := right.(map[string]interface{})
for _, key := range keys(l, ro) {
var li interface{} = nil
var rai interface{} = nil
if lv, ok := l[key]; ok {
li = lv
}
if rv, ok := ro[key]; ok {
rai = rv
}
if !eq(li, rai) {
return lt(li, rai)
}
}
return false
default:
panic("unknown type")
}
}
func keys(l map[string]interface{}, ro map[string]interface{}) []string {
var keys []string
seen := map[string]bool{}
for _, a := range []map[string]interface{}{l, ro} {
for k := range a {
if _, ok := seen[k]; !ok {
seen[k] = true
keys = append(keys, k)
}
}
}
sort.Strings(keys)
return keys
}
func gt(left, right interface{}) bool {
leftV, rightV := typeValue(left), typeValue(right)
if leftV != rightV {
return leftV > rightV
}
switch l := left.(type) {
case nil:
return false
case bool:
return toNumber(l) > toNumber(right)
case int:
return l > right.(int)
case float64:
return l > right.(float64)
case string:
return l > right.(string)
case []interface{}:
ra := right.([]interface{})
max := len(l)
if len(ra) > max {
max = len(ra)
}
for i := 0; i < max; i++ {
var li interface{} = nil
var rai interface{} = nil
if len(l) > i {
li = l[i]
}
if len(ra) > i {
rai = ra[i]
}
if !eq(li, rai) {
return gt(li, rai)
}
}
return false
case map[string]interface{}:
ro := right.(map[string]interface{})
for _, key := range keys(l, ro) {
var li interface{} = nil
var rai interface{} = nil
if lv, ok := l[key]; ok {
li = lv
}
if rv, ok := ro[key]; ok {
rai = rv
}
if !eq(li, rai) {
return gt(li, rai)
}
}
return false
default:
panic("unknown type")
}
}
func le(left, right interface{}) bool {
leftV, rightV := typeValue(left), typeValue(right)
if leftV != rightV {
return leftV <= rightV
}
switch l := left.(type) {
case nil:
return false
case bool:
return toNumber(l) <= toNumber(right)
case int:
return l <= right.(int)
case float64:
return l <= right.(float64)
case string:
return l <= right.(string)
case []interface{}:
ra := right.([]interface{})
max := len(l)
if len(ra) > max {
max = len(ra)
}
for i := 0; i < max; i++ {
var li interface{} = nil
var rai interface{} = nil
if len(l) > i {
li = l[i]
}
if len(ra) > i {
rai = ra[i]
}
if !eq(li, rai) {
return le(li, rai)
}
}
return true
case map[string]interface{}:
ro := right.(map[string]interface{})
for _, key := range keys(l, ro) {
var li interface{} = nil
var rai interface{} = nil
if lv, ok := l[key]; ok {
li = lv
}
if rv, ok := ro[key]; ok {
rai = rv
}
if !eq(li, rai) {
return lt(li, rai)
}
}
return true
default:
panic("unknown type")
}
}
func ge(left, right interface{}) bool {
leftV, rightV := typeValue(left), typeValue(right)
if leftV != rightV {
return leftV >= rightV
}
switch l := left.(type) {
case nil:
return false
case bool:
return toNumber(l) >= toNumber(right)
case int:
return l >= right.(int)
case float64:
return l >= right.(float64)
case string:
return l >= right.(string)
case []interface{}:
ra := right.([]interface{})
max := len(l)
if len(ra) > max {
max = len(ra)
}
for i := 0; i < max; i++ {
var li interface{} = nil
var rai interface{} = nil
if len(l) > i {
li = l[i]
}
if len(ra) > i {
rai = ra[i]
}
if !eq(li, rai) {
return ge(li, rai)
}
}
return true
case map[string]interface{}:
ro := right.(map[string]interface{})
for _, key := range keys(l, ro) {
var li interface{} = nil
var rai interface{} = nil
if lv, ok := l[key]; ok {
li = lv
}
if rv, ok := ro[key]; ok {
rai = rv
}
if !eq(li, rai) {
return gt(li, rai)
}
}
return true
default:
panic("unknown type")
}
}
func in(left, right interface{}) bool {
a, ok := right.([]interface{})
if !ok {
return false
}
for _, v := range a {
if left == v {
return true
}
}
return false
}
func like(left, right interface{}) (bool, error) {
return match(right.(string), left.(string))
}
func regexMatch(left, right interface{}) (bool, error) {
return regexp.Match(right.(string), []byte(left.(string)))
}
func regexNonMatch(left, right interface{}) (bool, error) {
m, err := regexp.Match(right.(string), []byte(left.(string)))
return !m, err
}
func typeValue(v interface{}) int {
switch v.(type) {
case nil:
return 0
case bool:
return 1
case float64, int:
return 2
case string:
return 3
case []interface{}:
return 4
case map[string]interface{}:
return 5
default:
panic("unknown type")
}
}
// Ternary operator https://www.arangodb.com/docs/3.7/aql/operators.html#ternary-operator
func ternary(left, middle, right interface{}) interface{} {
if toBool(left) {
if middle != nil {
return middle
}
return left
}
return right
}
// Range operators https://www.arangodb.com/docs/3.7/aql/operators.html#range-operator
func aqlrange(left, right interface{}) []float64 {
var v []float64
for i := int(left.(float64)); i <= int(right.(float64)); i++ {
v = append(v, float64(i))
}
return v
}

120
caql/parser.go Normal file
View File

@@ -0,0 +1,120 @@
package caql
import (
"errors"
"fmt"
"strconv"
"github.com/antlr/antlr4/runtime/Go/antlr"
"github.com/SecurityBrewery/catalyst/generated/caql/parser"
)
type Parser struct {
Searcher Searcher
Prefix string
}
func (p *Parser) Parse(aql string) (t *Tree, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("%s", r)
}
}()
// Setup the input
inputStream := antlr.NewInputStream(aql)
errorListener := &errorListener{}
// Create the Lexer
lexer := parser.NewCAQLLexer(inputStream)
lexer.RemoveErrorListeners()
lexer.AddErrorListener(errorListener)
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
// Create the Parser
aqlParser := parser.NewCAQLParser(stream)
aqlParser.RemoveErrorListeners()
aqlParser.AddErrorListener(errorListener)
aqlParser.SetErrorHandler(antlr.NewBailErrorStrategy())
if errorListener.errs != nil {
err = errorListener.errs[0]
}
return &Tree{aqlParser: aqlParser, parseContext: aqlParser.Parse(), searcher: p.Searcher, prefix: p.Prefix}, err
}
type Tree struct {
parseContext parser.IParseContext
aqlParser *parser.CAQLParser
searcher Searcher
prefix string
}
func (t *Tree) Eval(values map[string]interface{}) (i interface{}, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("%s", r)
}
}()
interpreter := aqlInterpreter{values: values}
antlr.ParseTreeWalkerDefault.Walk(&interpreter, t.parseContext)
if interpreter.errs != nil {
return nil, interpreter.errs[0]
}
return interpreter.stack[0], nil
}
func (t *Tree) String() (s string, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("%s", r)
}
}()
builder := aqlBuilder{searcher: t.searcher, prefix: t.prefix}
antlr.ParseTreeWalkerDefault.Walk(&builder, t.parseContext)
return builder.stack[0], err
}
func (t *Tree) BleveString() (s string, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("%s", r)
}
}()
builder := bleveBuilder{}
antlr.ParseTreeWalkerDefault.Walk(&builder, t.parseContext)
if builder.err != nil {
return "", builder.err
}
return builder.stack[0], err
}
type errorListener struct {
*antlr.DefaultErrorListener
errs []error
}
func (el *errorListener) SyntaxError(recognizer antlr.Recognizer, offendingSymbol interface{}, line, column int, msg string, e antlr.RecognitionException) {
el.errs = append(el.errs, fmt.Errorf("line "+strconv.Itoa(line)+":"+strconv.Itoa(column)+" "+msg))
}
func (el *errorListener) ReportAmbiguity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, exact bool, ambigAlts *antlr.BitSet, configs antlr.ATNConfigSet) {
el.errs = append(el.errs, errors.New("ReportAmbiguity"))
}
func (el *errorListener) ReportAttemptingFullContext(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex int, conflictingAlts *antlr.BitSet, configs antlr.ATNConfigSet) {
el.errs = append(el.errs, errors.New("ReportAttemptingFullContext"))
}
func (el *errorListener) ReportContextSensitivity(recognizer antlr.Parser, dfa *antlr.DFA, startIndex, stopIndex, prediction int, configs antlr.ATNConfigSet) {
el.errs = append(el.errs, errors.New("ReportContextSensitivity"))
}

352
caql/rql_test.go Normal file
View File

@@ -0,0 +1,352 @@
package caql
import (
"encoding/json"
"reflect"
"testing"
)
type MockSearcher struct{}
func (m MockSearcher) Search(_ string) (ids []string, err error) {
return []string{"1", "2", "3"}, nil
}
func TestParseSAQLEval(t *testing.T) {
tests := []struct {
name string
saql string
wantRebuild string
wantValue interface{}
wantParseErr bool
wantRebuildErr bool
wantEvalErr bool
values string
}{
// Custom
{name: "Compare 1", saql: "1 <= 2", wantRebuild: "1 <= 2", wantValue: true},
{name: "Compare 2", saql: "1 >= 2", wantRebuild: "1 >= 2", wantValue: false},
{name: "Compare 3", saql: "1 == 2", wantRebuild: "1 == 2", wantValue: false},
{name: "Compare 4", saql: "1 > 2", wantRebuild: "1 > 2", wantValue: false},
{name: "Compare 5", saql: "1 < 2", wantRebuild: "1 < 2", wantValue: true},
{name: "Compare 6", saql: "1 != 2", wantRebuild: "1 != 2", wantValue: true},
{name: "SymbolRef 1", saql: "name", wantRebuild: "name", wantValue: false, values: `{"name": false}`},
{name: "SymbolRef 2", saql: "d.name", wantRebuild: "d.name", wantValue: false, values: `{"d": {"name": false}}`},
{name: "SymbolRef 3", saql: "name == false", wantRebuild: "name == false", wantValue: true, values: `{"name": false}`},
{name: "SymbolRef Error 1", saql: "name, title", wantParseErr: true},
{name: "SymbolRef Error 2", saql: "unknown", wantRebuild: "unknown", wantValue: false, wantEvalErr: true, values: `{}`},
{name: "Misc 1", saql: `active == true && age < 39`, wantRebuild: `active == true AND age < 39`, wantValue: true, values: `{"active": true, "age": 2}`},
{name: "Misc 2", saql: `(attr == 10) AND foo == 'bar' OR NOT baz`, wantRebuild: `(attr == 10) AND foo == "bar" OR NOT baz`, wantValue: false, values: `{"attr": 2, "foo": "bar", "baz": true}`},
{name: "Misc 3", saql: `attr == 10 AND (foo == 'bar' OR foo == 'baz')`, wantRebuild: `attr == 10 AND (foo == "bar" OR foo == "baz")`, wantValue: false, values: `{"attr": 2, "foo": "bar", "baz": true}`},
{name: "Misc 4", saql: `5 > 1 AND "a" != "b"`, wantRebuild: `5 > 1 AND "a" != "b"`, wantValue: true},
{name: "LIKE 1", saql: `"foo" LIKE "%f%"`, wantRebuild: `"foo" LIKE "%f%"`, wantValue: true},
{name: "LIKE 2", saql: `"foo" NOT LIKE "%f%"`, wantRebuild: `"foo" NOT LIKE "%f%"`, wantValue: false},
{name: "LIKE 3", saql: `NOT "foo" LIKE "%f%"`, wantRebuild: `NOT "foo" LIKE "%f%"`, wantValue: false},
{name: "Summand 1", saql: "1 + 2", wantRebuild: "1 + 2", wantValue: 3},
{name: "Summand 2", saql: "1 - 2", wantRebuild: "1 - 2", wantValue: -1},
{name: "Factor 1", saql: "1 * 2", wantRebuild: "1 * 2", wantValue: 2},
{name: "Factor 2", saql: "1 / 2", wantRebuild: "1 / 2", wantValue: 0.5},
{name: "Factor 3", saql: "1.0 / 2.0", wantRebuild: "1.0 / 2.0", wantValue: 0.5},
{name: "Factor 4", saql: "1 % 2", wantRebuild: "1 % 2", wantValue: 1},
{name: "Term 1", saql: "(1 + 2) * 2", wantRebuild: "(1 + 2) * 2", wantValue: 6},
{name: "Term 2", saql: "2 * (1 + 2)", wantRebuild: "2 * (1 + 2)", wantValue: 6},
// https://www.arangodb.com/docs/3.7/aql/fundamentals-data-types.html
{name: "Null 1", saql: `null`, wantRebuild: "null"},
{name: "Bool 1", saql: `true`, wantRebuild: "true", wantValue: true},
{name: "Bool 2", saql: `false`, wantRebuild: "false", wantValue: false},
{name: "Numeric 1", saql: "1", wantRebuild: "1", wantValue: 1},
{name: "Numeric 2", saql: "+1", wantRebuild: "1", wantValue: 1},
{name: "Numeric 3", saql: "42", wantRebuild: "42", wantValue: 42},
{name: "Numeric 4", saql: "-1", wantRebuild: "-1", wantValue: -1},
{name: "Numeric 5", saql: "-42", wantRebuild: "-42", wantValue: -42},
{name: "Numeric 6", saql: "1.23", wantRebuild: "1.23", wantValue: 1.23},
{name: "Numeric 7", saql: "-99.99", wantRebuild: "-99.99", wantValue: -99.99},
{name: "Numeric 8", saql: "0.5", wantRebuild: "0.5", wantValue: 0.5},
{name: "Numeric 9", saql: ".5", wantRebuild: ".5", wantValue: 0.5},
{name: "Numeric 10", saql: "-4.87e103", wantRebuild: "-4.87e103", wantValue: -4.87e+103},
{name: "Numeric 11", saql: "0b10", wantRebuild: "0b10", wantValue: 2},
{name: "Numeric 12", saql: "0x10", wantRebuild: "0x10", wantValue: 16},
{name: "Numeric Error 1", saql: "1.", wantParseErr: true},
{name: "Numeric Error 2", saql: "01.23", wantParseErr: true},
{name: "Numeric Error 3", saql: "00.23", wantParseErr: true},
{name: "Numeric Error 4", saql: "00", wantParseErr: true},
// {name: "String 1", saql: `"yikes!"`, wantRebuild: `"yikes!"`, wantValue: "yikes!"},
// {name: "String 2", saql: `"don't know"`, wantRebuild: `"don't know"`, wantValue: "don't know"},
// {name: "String 3", saql: `"this is a \"quoted\" word"`, wantRebuild: `"this is a \"quoted\" word"`, wantValue: "this is a \"quoted\" word"},
// {name: "String 4", saql: `"this is a longer string."`, wantRebuild: `"this is a longer string."`, wantValue: "this is a longer string."},
// {name: "String 5", saql: `"the path separator on Windows is \\"`, wantRebuild: `"the path separator on Windows is \\"`, wantValue: "the path separator on Windows is \\"},
// {name: "String 6", saql: `'yikes!'`, wantRebuild: `"yikes!"`, wantValue: "yikes!"},
// {name: "String 7", saql: `'don\'t know'`, wantRebuild: `"don't know"`, wantValue: "don't know"},
// {name: "String 8", saql: `'this is a "quoted" word'`, wantRebuild: `"this is a \"quoted\" word"`, wantValue: "this is a \"quoted\" word"},
// {name: "String 9", saql: `'this is a longer string.'`, wantRebuild: `"this is a longer string."`, wantValue: "this is a longer string."},
// {name: "String 10", saql: `'the path separator on Windows is \\'`, wantRebuild: `"the path separator on Windows is \\"`, wantValue: `the path separator on Windows is \`},
{name: "Array 1", saql: "[]", wantRebuild: "[]", wantValue: []interface{}{}},
{name: "Array 2", saql: `[true]`, wantRebuild: `[true]`, wantValue: []interface{}{true}},
{name: "Array 3", saql: `[1, 2, 3]`, wantRebuild: `[1, 2, 3]`, wantValue: []interface{}{float64(1), float64(2), float64(3)}},
{
name: "Array 4", saql: `[-99, "yikes!", [false, ["no"], []], 1]`, wantRebuild: `[-99, "yikes!", [false, ["no"], []], 1]`,
wantValue: []interface{}{-99.0, "yikes!", []interface{}{false, []interface{}{"no"}, []interface{}{}}, float64(1)},
},
{name: "Array 5", saql: `[["fox", "marshal"]]`, wantRebuild: `[["fox", "marshal"]]`, wantValue: []interface{}{[]interface{}{"fox", "marshal"}}},
{name: "Array 6", saql: `[1, 2, 3,]`, wantRebuild: `[1, 2, 3]`, wantValue: []interface{}{float64(1), float64(2), float64(3)}},
{name: "Array Error 1", saql: "(1,2,3)", wantParseErr: true},
{name: "Array Access 1", saql: "u.friends[0]", wantRebuild: "u.friends[0]", wantValue: 7, values: `{"u": {"friends": [7,8,9]}}`},
{name: "Array Access 2", saql: "u.friends[2]", wantRebuild: "u.friends[2]", wantValue: 9, values: `{"u": {"friends": [7,8,9]}}`},
{name: "Array Access 3", saql: "u.friends[-1]", wantRebuild: "u.friends[-1]", wantValue: 9, values: `{"u": {"friends": [7,8,9]}}`},
{name: "Array Access 4", saql: "u.friends[-2]", wantRebuild: "u.friends[-2]", wantValue: 8, values: `{"u": {"friends": [7,8,9]}}`},
{name: "Object 1", saql: "{}", wantRebuild: "{}", wantValue: map[string]interface{}{}},
{name: "Object 2", saql: `{a: 1}`, wantRebuild: "{a: 1}", wantValue: map[string]interface{}{"a": float64(1)}},
{name: "Object 3", saql: `{'a': 1}`, wantRebuild: `{'a': 1}`, wantValue: map[string]interface{}{"a": float64(1)}},
{name: "Object 4", saql: `{"a": 1}`, wantRebuild: `{"a": 1}`, wantValue: map[string]interface{}{"a": float64(1)}},
{name: "Object 5", saql: `{'return': 1}`, wantRebuild: `{'return': 1}`, wantValue: map[string]interface{}{"return": float64(1)}},
{name: "Object 6", saql: `{"return": 1}`, wantRebuild: `{"return": 1}`, wantValue: map[string]interface{}{"return": float64(1)}},
{name: "Object 9", saql: `{a: 1,}`, wantRebuild: "{a: 1}", wantValue: map[string]interface{}{"a": float64(1)}},
{name: "Object 10", saql: `{"a": 1,}`, wantRebuild: `{"a": 1}`, wantValue: map[string]interface{}{"a": float64(1)}},
// {"Object 8", "{`return`: 1}", `{"return": 1}`, true},
// {"Object 7", "{´return´: 1}", `{"return": 1}`, true},
{name: "Object Error 1: return is a keyword", saql: `{like: 1}`, wantParseErr: true},
{name: "Object Access 1", saql: "u.address.city.name", wantRebuild: "u.address.city.name", wantValue: "Munich", values: `{"u": {"address": {"city": {"name": "Munich"}}}}`},
{name: "Object Access 2", saql: "u.friends[0].name.first", wantRebuild: "u.friends[0].name.first", wantValue: "Kevin", values: `{"u": {"friends": [{"name": {"first": "Kevin"}}]}}`},
{name: "Object Access 3", saql: `u["address"]["city"]["name"]`, wantRebuild: `u["address"]["city"]["name"]`, wantValue: "Munich", values: `{"u": {"address": {"city": {"name": "Munich"}}}}`},
{name: "Object Access 4", saql: `u["friends"][0]["name"]["first"]`, wantRebuild: `u["friends"][0]["name"]["first"]`, wantValue: "Kevin", values: `{"u": {"friends": [{"name": {"first": "Kevin"}}]}}`},
{name: "Object Access 5", saql: "u._key", wantRebuild: "u._key", wantValue: false, values: `{"u": {"_key": false}}`},
// This query language does not support binds
// https://www.arangodb.com/docs/3.7/aql/fundamentals-bind-parameters.html
// {name: "Bind 1", saql: "u.id == @id && u.name == @name", wantRebuild: `u.id == @id AND u.name == @name`, wantValue: true},
// {name: "Bind 2", saql: "u.id == CONCAT('prefix', @id, 'suffix') && u.name == @name", wantRebuild: `u.id == CONCAT('prefix', @id, 'suffix') AND u.name == @name`, wantValue: false},
// {name: "Bind 3", saql: "doc.@attr.@subattr", wantRebuild: `doc.@attr.@subattr`, wantValue: true, values: `{"doc": {"@attr": {"@subattr": true}}}`},
// {name: "Bind 4", saql: "doc[@attr][@subattr]", wantRebuild: `doc[@attr][@subattr]`, wantValue: true, values: `{"doc": {"@attr": {"@subattr": true}}}`},
// https://www.arangodb.com/docs/3.7/aql/fundamentals-type-value-order.html
{name: "Compare 7", saql: `null < false`, wantRebuild: `null < false`, wantValue: true},
{name: "Compare 8", saql: `null < true`, wantRebuild: `null < true`, wantValue: true},
{name: "Compare 9", saql: `null < 1`, wantRebuild: `null < 1`, wantValue: true},
{name: "Compare 10", saql: `null < ''`, wantRebuild: `null < ""`, wantValue: true},
{name: "Compare 11", saql: `null < ' '`, wantRebuild: `null < " "`, wantValue: true},
{name: "Compare 12", saql: `null < '3'`, wantRebuild: `null < "3"`, wantValue: true},
{name: "Compare 13", saql: `null < 'abc'`, wantRebuild: `null < "abc"`, wantValue: true},
{name: "Compare 14", saql: `null < []`, wantRebuild: `null < []`, wantValue: true},
{name: "Compare 15", saql: `null < {}`, wantRebuild: `null < {}`, wantValue: true},
{name: "Compare 16", saql: `false < true`, wantRebuild: `false < true`, wantValue: true},
{name: "Compare 17", saql: `false < 5`, wantRebuild: `false < 5`, wantValue: true},
{name: "Compare 18", saql: `false < ''`, wantRebuild: `false < ""`, wantValue: true},
{name: "Compare 19", saql: `false < ' '`, wantRebuild: `false < " "`, wantValue: true},
{name: "Compare 20", saql: `false < '7'`, wantRebuild: `false < "7"`, wantValue: true},
{name: "Compare 21", saql: `false < 'abc'`, wantRebuild: `false < "abc"`, wantValue: true},
{name: "Compare 22", saql: `false < []`, wantRebuild: `false < []`, wantValue: true},
{name: "Compare 23", saql: `false < {}`, wantRebuild: `false < {}`, wantValue: true},
{name: "Compare 24", saql: `true < 9`, wantRebuild: `true < 9`, wantValue: true},
{name: "Compare 25", saql: `true < ''`, wantRebuild: `true < ""`, wantValue: true},
{name: "Compare 26", saql: `true < ' '`, wantRebuild: `true < " "`, wantValue: true},
{name: "Compare 27", saql: `true < '11'`, wantRebuild: `true < "11"`, wantValue: true},
{name: "Compare 28", saql: `true < 'abc'`, wantRebuild: `true < "abc"`, wantValue: true},
{name: "Compare 29", saql: `true < []`, wantRebuild: `true < []`, wantValue: true},
{name: "Compare 30", saql: `true < {}`, wantRebuild: `true < {}`, wantValue: true},
{name: "Compare 31", saql: `13 < ''`, wantRebuild: `13 < ""`, wantValue: true},
{name: "Compare 32", saql: `15 < ' '`, wantRebuild: `15 < " "`, wantValue: true},
{name: "Compare 33", saql: `17 < '18'`, wantRebuild: `17 < "18"`, wantValue: true},
{name: "Compare 34", saql: `21 < 'abc'`, wantRebuild: `21 < "abc"`, wantValue: true},
{name: "Compare 35", saql: `23 < []`, wantRebuild: `23 < []`, wantValue: true},
{name: "Compare 36", saql: `25 < {}`, wantRebuild: `25 < {}`, wantValue: true},
{name: "Compare 37", saql: `'' < ' '`, wantRebuild: `"" < " "`, wantValue: true},
{name: "Compare 38", saql: `'' < '27'`, wantRebuild: `"" < "27"`, wantValue: true},
{name: "Compare 39", saql: `'' < 'abc'`, wantRebuild: `"" < "abc"`, wantValue: true},
{name: "Compare 40", saql: `'' < []`, wantRebuild: `"" < []`, wantValue: true},
{name: "Compare 41", saql: `'' < {}`, wantRebuild: `"" < {}`, wantValue: true},
{name: "Compare 42", saql: `[] < {}`, wantRebuild: `[] < {}`, wantValue: true},
{name: "Compare 43", saql: `[] < [29]`, wantRebuild: `[] < [29]`, wantValue: true},
{name: "Compare 44", saql: `[1] < [2]`, wantRebuild: `[1] < [2]`, wantValue: true},
{name: "Compare 45", saql: `[1, 2] < [2]`, wantRebuild: `[1, 2] < [2]`, wantValue: true},
{name: "Compare 46", saql: `[99, 99] < [100]`, wantRebuild: `[99, 99] < [100]`, wantValue: true},
{name: "Compare 47", saql: `[false] < [true]`, wantRebuild: `[false] < [true]`, wantValue: true},
{name: "Compare 48", saql: `[false, 1] < [false, '']`, wantRebuild: `[false, 1] < [false, ""]`, wantValue: true},
{name: "Compare 49", saql: `{} < {"a": 1}`, wantRebuild: `{} < {"a": 1}`, wantValue: true},
{name: "Compare 50", saql: `{} == {"a": null}`, wantRebuild: `{} == {"a": null}`, wantValue: true},
{name: "Compare 51", saql: `{"a": 1} < {"a": 2}`, wantRebuild: `{"a": 1} < {"a": 2}`, wantValue: true},
{name: "Compare 52", saql: `{"b": 1} < {"a": 0}`, wantRebuild: `{"b": 1} < {"a": 0}`, wantValue: true},
{name: "Compare 53", saql: `{"a": {"c": true}} < {"a": {"c": 0}}`, wantRebuild: `{"a": {"c": true}} < {"a": {"c": 0}}`, wantValue: true},
{name: "Compare 54", saql: `{"a": {"c": true, "a": 0}} < {"a": {"c": false, "a": 1}}`, wantRebuild: `{"a": {"c": true, "a": 0}} < {"a": {"c": false, "a": 1}}`, wantValue: true},
{name: "Compare 55", saql: `{"a": 1, "b": 2} == {"b": 2, "a": 1}`, wantRebuild: `{"a": 1, "b": 2} == {"b": 2, "a": 1}`, wantValue: true},
// https://www.arangodb.com/docs/3.7/aql/operators.html
{name: "Compare 56", saql: `0 == null`, wantRebuild: `0 == null`, wantValue: false},
{name: "Compare 57", saql: `1 > 0`, wantRebuild: `1 > 0`, wantValue: true},
{name: "Compare 58", saql: `true != null`, wantRebuild: `true != null`, wantValue: true},
{name: "Compare 59", saql: `45 <= "yikes!"`, wantRebuild: `45 <= "yikes!"`, wantValue: true},
{name: "Compare 60", saql: `65 != "65"`, wantRebuild: `65 != "65"`, wantValue: true},
{name: "Compare 61", saql: `65 == 65`, wantRebuild: `65 == 65`, wantValue: true},
{name: "Compare 62", saql: `1.23 > 1.32`, wantRebuild: `1.23 > 1.32`, wantValue: false},
{name: "Compare 63", saql: `1.5 IN [2, 3, 1.5]`, wantRebuild: `1.5 IN [2, 3, 1.5]`, wantValue: true},
{name: "Compare 64", saql: `"foo" IN null`, wantRebuild: `"foo" IN null`, wantValue: false},
{name: "Compare 65", saql: `42 NOT IN [17, 40, 50]`, wantRebuild: `42 NOT IN [17, 40, 50]`, wantValue: true},
{name: "Compare 66", saql: `"abc" == "abc"`, wantRebuild: `"abc" == "abc"`, wantValue: true},
{name: "Compare 67", saql: `"abc" == "ABC"`, wantRebuild: `"abc" == "ABC"`, wantValue: false},
{name: "Compare 68", saql: `"foo" LIKE "f%"`, wantRebuild: `"foo" LIKE "f%"`, wantValue: true},
{name: "Compare 69", saql: `"foo" NOT LIKE "f%"`, wantRebuild: `"foo" NOT LIKE "f%"`, wantValue: false},
{name: "Compare 70", saql: `"foo" =~ "^f[o].$"`, wantRebuild: `"foo" =~ "^f[o].$"`, wantValue: true},
{name: "Compare 71", saql: `"foo" !~ "[a-z]+bar$"`, wantRebuild: `"foo" !~ "[a-z]+bar$"`, wantValue: true},
{name: "Compare 72", saql: `"abc" LIKE "a%"`, wantRebuild: `"abc" LIKE "a%"`, wantValue: true},
{name: "Compare 73", saql: `"abc" LIKE "_bc"`, wantRebuild: `"abc" LIKE "_bc"`, wantValue: true},
{name: "Compare 74", saql: `"a_b_foo" LIKE "a\\_b\\_foo"`, wantRebuild: `"a_b_foo" LIKE "a\\_b\\_foo"`, wantValue: true},
// https://www.arangodb.com/docs/3.7/aql/operators.html#array-comparison-operators
{name: "Compare Array 1", saql: `[1, 2, 3] ALL IN [2, 3, 4]`, wantRebuild: `[1, 2, 3] ALL IN [2, 3, 4]`, wantValue: false},
{name: "Compare Array 2", saql: `[1, 2, 3] ALL IN [1, 2, 3]`, wantRebuild: `[1, 2, 3] ALL IN [1, 2, 3]`, wantValue: true},
{name: "Compare Array 3", saql: `[1, 2, 3] NONE IN [3]`, wantRebuild: `[1, 2, 3] NONE IN [3]`, wantValue: false},
{name: "Compare Array 4", saql: `[1, 2, 3] NONE IN [23, 42]`, wantRebuild: `[1, 2, 3] NONE IN [23, 42]`, wantValue: true},
{name: "Compare Array 5", saql: `[1, 2, 3] ANY IN [4, 5, 6]`, wantRebuild: `[1, 2, 3] ANY IN [4, 5, 6]`, wantValue: false},
{name: "Compare Array 6", saql: `[1, 2, 3] ANY IN [1, 42]`, wantRebuild: `[1, 2, 3] ANY IN [1, 42]`, wantValue: true},
{name: "Compare Array 7", saql: `[1, 2, 3] ANY == 2`, wantRebuild: `[1, 2, 3] ANY == 2`, wantValue: true},
{name: "Compare Array 8", saql: `[1, 2, 3] ANY == 4`, wantRebuild: `[1, 2, 3] ANY == 4`, wantValue: false},
{name: "Compare Array 9", saql: `[1, 2, 3] ANY > 0`, wantRebuild: `[1, 2, 3] ANY > 0`, wantValue: true},
{name: "Compare Array 10", saql: `[1, 2, 3] ANY <= 1`, wantRebuild: `[1, 2, 3] ANY <= 1`, wantValue: true},
{name: "Compare Array 11", saql: `[1, 2, 3] NONE < 99`, wantRebuild: `[1, 2, 3] NONE < 99`, wantValue: false},
{name: "Compare Array 12", saql: `[1, 2, 3] NONE > 10`, wantRebuild: `[1, 2, 3] NONE > 10`, wantValue: true},
{name: "Compare Array 13", saql: `[1, 2, 3] ALL > 2`, wantRebuild: `[1, 2, 3] ALL > 2`, wantValue: false},
{name: "Compare Array 14", saql: `[1, 2, 3] ALL > 0`, wantRebuild: `[1, 2, 3] ALL > 0`, wantValue: true},
{name: "Compare Array 15", saql: `[1, 2, 3] ALL >= 3`, wantRebuild: `[1, 2, 3] ALL >= 3`, wantValue: false},
{name: "Compare Array 16", saql: `["foo", "bar"] ALL != "moo"`, wantRebuild: `["foo", "bar"] ALL != "moo"`, wantValue: true},
{name: "Compare Array 17", saql: `["foo", "bar"] NONE == "bar"`, wantRebuild: `["foo", "bar"] NONE == "bar"`, wantValue: false},
{name: "Compare Array 18", saql: `["foo", "bar"] ANY == "foo"`, wantRebuild: `["foo", "bar"] ANY == "foo"`, wantValue: true},
// https://www.arangodb.com/docs/3.7/aql/operators.html#logical-operators
{name: "Logical 1", saql: "active == true OR age < 39", wantRebuild: "active == true OR age < 39", wantValue: true, values: `{"active": true, "age": 4}`},
{name: "Logical 2", saql: "active == true || age < 39", wantRebuild: "active == true OR age < 39", wantValue: true, values: `{"active": true, "age": 4}`},
{name: "Logical 3", saql: "active == true AND age < 39", wantRebuild: "active == true AND age < 39", wantValue: true, values: `{"active": true, "age": 4}`},
{name: "Logical 4", saql: "active == true && age < 39", wantRebuild: "active == true AND age < 39", wantValue: true, values: `{"active": true, "age": 4}`},
{name: "Logical 5", saql: "!active", wantRebuild: "NOT active", wantValue: false, values: `{"active": true}`},
{name: "Logical 6", saql: "NOT active", wantRebuild: "NOT active", wantValue: false, values: `{"active": true}`},
{name: "Logical 7", saql: "not active", wantRebuild: "NOT active", wantValue: false, values: `{"active": true}`},
{name: "Logical 8", saql: "NOT NOT active", wantRebuild: "NOT NOT active", wantValue: true, values: `{"active": true}`},
{name: "Logical 9", saql: `u.age > 15 && u.address.city != ""`, wantRebuild: `u.age > 15 AND u.address.city != ""`, wantValue: false, values: `{"u": {"age": 2, "address": {"city": "Munich"}}}`},
{name: "Logical 10", saql: `true || false`, wantRebuild: `true OR false`, wantValue: true},
{name: "Logical 11", saql: `NOT u.isInvalid`, wantRebuild: `NOT u.isInvalid`, wantValue: false, values: `{"u": {"isInvalid": true}}`},
{name: "Logical 12", saql: `1 || ! 0`, wantRebuild: `1 OR NOT 0`, wantValue: 1},
{name: "Logical 13", saql: `25 > 1 && 42 != 7`, wantRebuild: `25 > 1 AND 42 != 7`, wantValue: true},
{name: "Logical 14", saql: `22 IN [23, 42] || 23 NOT IN [22, 7]`, wantRebuild: `22 IN [23, 42] OR 23 NOT IN [22, 7]`, wantValue: true},
{name: "Logical 15", saql: `25 != 25`, wantRebuild: `25 != 25`, wantValue: false},
{name: "Logical 16", saql: `1 || 7`, wantRebuild: `1 OR 7`, wantValue: 1},
// {name: "Logical 17", saql: `null || "foo"`, wantRebuild: `null OR "foo"`, wantValue: "foo"},
{name: "Logical 17", saql: `null || "foo"`, wantRebuild: `null OR d._key IN ["1","2","3"]`, wantValue: "foo", values: `{"d": {"_key": "1"}}`}, // eval != rebuild
{name: "Logical 18", saql: `null && true`, wantRebuild: `null AND true`, wantValue: nil},
{name: "Logical 19", saql: `true && 23`, wantRebuild: `true AND 23`, wantValue: 23},
{name: "Logical 20", saql: "true == (6 < 8)", wantRebuild: "true == (6 < 8)", wantValue: true},
{name: "Logical 21", saql: "true == 6 < 8", wantRebuild: "true == 6 < 8", wantValue: true}, // does not work in go
// https://www.arangodb.com/docs/3.7/aql/operators.html#arithmetic-operators
{name: "Arithmetic 1", saql: `1 + 1`, wantRebuild: `1 + 1`, wantValue: 2},
{name: "Arithmetic 2", saql: `33 - 99`, wantRebuild: `33 - 99`, wantValue: -66},
{name: "Arithmetic 3", saql: `12.4 * 4.5`, wantRebuild: `12.4 * 4.5`, wantValue: 55.8},
{name: "Arithmetic 4", saql: `13.0 / 0.1`, wantRebuild: `13.0 / 0.1`, wantValue: 130.0},
{name: "Arithmetic 5", saql: `23 % 7`, wantRebuild: `23 % 7`, wantValue: 2},
{name: "Arithmetic 6", saql: `-15`, wantRebuild: `-15`, wantValue: -15},
{name: "Arithmetic 7", saql: `+9.99`, wantRebuild: `9.99`, wantValue: 9.99},
{name: "Arithmetic 8", saql: `1 + "a"`, wantRebuild: `1 + "a"`, wantValue: 1},
{name: "Arithmetic 9", saql: `1 + "99"`, wantRebuild: `1 + "99"`, wantValue: 100},
{name: "Arithmetic 10", saql: `1 + null`, wantRebuild: `1 + null`, wantValue: 1},
{name: "Arithmetic 11", saql: `null + 1`, wantRebuild: `null + 1`, wantValue: 1},
{name: "Arithmetic 12", saql: `3 + []`, wantRebuild: `3 + []`, wantValue: 3},
{name: "Arithmetic 13", saql: `24 + [2]`, wantRebuild: `24 + [2]`, wantValue: 26},
{name: "Arithmetic 14", saql: `24 + [2, 4]`, wantRebuild: `24 + [2, 4]`, wantValue: 24},
{name: "Arithmetic 15", saql: `25 - null`, wantRebuild: `25 - null`, wantValue: 25},
{name: "Arithmetic 16", saql: `17 - true`, wantRebuild: `17 - true`, wantValue: 16},
{name: "Arithmetic 17", saql: `23 * {}`, wantRebuild: `23 * {}`, wantValue: 0},
{name: "Arithmetic 18", saql: `5 * [7]`, wantRebuild: `5 * [7]`, wantValue: 35},
{name: "Arithmetic 19", saql: `24 / "12"`, wantRebuild: `24 / "12"`, wantValue: 2},
{name: "Arithmetic Error 1: Divison by zero", saql: `1 / 0`, wantRebuild: `1 / 0`, wantValue: 0},
// https://www.arangodb.com/docs/3.7/aql/operators.html#ternary-operator
{name: "Ternary 1", saql: `u.age > 15 || u.active == true ? u.userId : null`, wantRebuild: `u.age > 15 OR u.active == true ? u.userId : null`, wantValue: 45, values: `{"u": {"active": true, "age": 2, "userId": 45}}`},
{name: "Ternary 2", saql: `u.value ? : 'value is null, 0 or not present'`, wantRebuild: `u.value ? : "value is null, 0 or not present"`, wantValue: "value is null, 0 or not present", values: `{"u": {"value": 0}}`},
// https://www.arangodb.com/docs/3.7/aql/operators.html#range-operator
{name: "Range 1", saql: `2010..2013`, wantRebuild: `2010..2013`, wantValue: []float64{2010, 2011, 2012, 2013}},
// {"Array operators 1", `u.friends[*].name`, `u.friends[*].name`, false},
// Security
{name: "Security 1", saql: `doc.value == 1 || true REMOVE doc IN collection //`, wantParseErr: true},
{name: "Security 2", saql: `doc.value == 1 || true INSERT {foo: "bar"} IN collection //`, wantParseErr: true},
// https://www.arangodb.com/docs/3.7/aql/operators.html#operator-precedence
{name: "Precendence", saql: `2 > 15 && "a" != ""`, wantRebuild: `2 > 15 AND "a" != ""`, wantValue: false},
}
for _, tt := range tests {
parser := &Parser{
Searcher: &MockSearcher{},
}
t.Run(tt.name, func(t *testing.T) {
expr, err := parser.Parse(tt.saql)
if (err != nil) != tt.wantParseErr {
t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantParseErr)
if expr != nil {
t.Error(expr.String())
}
return
}
if err != nil {
return
}
got, err := expr.String()
if (err != nil) != tt.wantRebuildErr {
t.Error(expr.String())
t.Errorf("String() error = %v, wantErr %v", err, tt.wantParseErr)
return
}
if err != nil {
return
}
if got != tt.wantRebuild {
t.Errorf("String() got = %v, want %v", got, tt.wantRebuild)
}
var myJson map[string]interface{}
if tt.values != "" {
err = json.Unmarshal([]byte(tt.values), &myJson)
if err != nil {
t.Fatal(err)
}
}
value, err := expr.Eval(myJson)
if (err != nil) != tt.wantEvalErr {
t.Error(expr.String())
t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantParseErr)
return
}
if err != nil {
return
}
wantValue := tt.wantValue
if i, ok := wantValue.(int); ok {
wantValue = float64(i)
}
if !reflect.DeepEqual(value, wantValue) {
t.Error(expr.String())
t.Errorf("Eval() got = %T %#v, want %T %#v", value, value, wantValue, wantValue)
}
})
}
}

154
caql/set.go Normal file
View File

@@ -0,0 +1,154 @@
// Adapted from https://github.com/badgerodon/collections under the MIT License
// Original License:
//
// Copyright (c) 2012 Caleb Doxsey
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package caql
import "sort"
type (
Set struct {
hash map[interface{}]nothing
}
nothing struct{}
)
// Create a new set
func New(initial ...interface{}) *Set {
s := &Set{make(map[interface{}]nothing)}
for _, v := range initial {
s.Insert(v)
}
return s
}
// Find the difference between two sets
func (s *Set) Difference(set *Set) *Set {
n := make(map[interface{}]nothing)
for k := range s.hash {
if _, exists := set.hash[k]; !exists {
n[k] = nothing{}
}
}
return &Set{n}
}
// Call f for each item in the set
func (s *Set) Do(f func(interface{})) {
for k := range s.hash {
f(k)
}
}
// Test to see whether or not the element is in the set
func (s *Set) Has(element interface{}) bool {
_, exists := s.hash[element]
return exists
}
// Add an element to the set
func (s *Set) Insert(element interface{}) {
s.hash[element] = nothing{}
}
// Find the intersection of two sets
func (s *Set) Intersection(set *Set) *Set {
n := make(map[interface{}]nothing)
for k := range s.hash {
if _, exists := set.hash[k]; exists {
n[k] = nothing{}
}
}
return &Set{n}
}
// Return the number of items in the set
func (s *Set) Len() int {
return len(s.hash)
}
// Test whether or not this set is a proper subset of "set"
func (s *Set) ProperSubsetOf(set *Set) bool {
return s.SubsetOf(set) && s.Len() < set.Len()
}
// Remove an element from the set
func (s *Set) Remove(element interface{}) {
delete(s.hash, element)
}
func (s *Set) Minus(set *Set) *Set {
n := make(map[interface{}]nothing)
for k := range s.hash {
n[k] = nothing{}
}
for _, v := range set.Values() {
delete(n, v)
}
return &Set{n}
}
// Test whether or not this set is a subset of "set"
func (s *Set) SubsetOf(set *Set) bool {
if s.Len() > set.Len() {
return false
}
for k := range s.hash {
if _, exists := set.hash[k]; !exists {
return false
}
}
return true
}
// Find the union of two sets
func (s *Set) Union(set *Set) *Set {
n := make(map[interface{}]nothing)
for k := range s.hash {
n[k] = nothing{}
}
for k := range set.hash {
n[k] = nothing{}
}
return &Set{n}
}
func (s *Set) Values() []interface{} {
values := []interface{}{}
for k := range s.hash {
values = append(values, k)
}
sort.Slice(values, func(i, j int) bool { return lt(values[i], values[j]) })
return values
}

96
caql/set_test.go Normal file
View File

@@ -0,0 +1,96 @@
// Adapted from https://github.com/badgerodon/collections under the MIT License
// Original License:
//
// Copyright (c) 2012 Caleb Doxsey
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package caql
import (
"testing"
)
func Test(t *testing.T) {
s := New()
s.Insert(5)
if s.Len() != 1 {
t.Errorf("Length should be 1")
}
if !s.Has(5) {
t.Errorf("Membership test failed")
}
s.Remove(5)
if s.Len() != 0 {
t.Errorf("Length should be 0")
}
if s.Has(5) {
t.Errorf("The set should be empty")
}
// Difference
s1 := New(1, 2, 3, 4, 5, 6)
s2 := New(4, 5, 6)
s3 := s1.Difference(s2)
if s3.Len() != 3 {
t.Errorf("Length should be 3")
}
if !(s3.Has(1) && s3.Has(2) && s3.Has(3)) {
t.Errorf("Set should only contain 1, 2, 3")
}
// Intersection
s3 = s1.Intersection(s2)
if s3.Len() != 3 {
t.Errorf("Length should be 3 after intersection")
}
if !(s3.Has(4) && s3.Has(5) && s3.Has(6)) {
t.Errorf("Set should contain 4, 5, 6")
}
// Union
s4 := New(7, 8, 9)
s3 = s2.Union(s4)
if s3.Len() != 6 {
t.Errorf("Length should be 6 after union")
}
if !(s3.Has(7)) {
t.Errorf("Set should contain 4, 5, 6, 7, 8, 9")
}
// Subset
if !s1.SubsetOf(s1) {
t.Errorf("set should be a subset of itself")
}
// Proper Subset
if s1.ProperSubsetOf(s1) {
t.Errorf("set should not be a subset of itself")
}
}

79
caql/unquote.go Normal file
View File

@@ -0,0 +1,79 @@
// Adapted from https://github.com/golang/go
// Original License:
//
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the https://go.dev/LICENSE file.
package caql
import (
"strconv"
"strings"
"unicode/utf8"
)
// unquote interprets s as a single-quoted, double-quoted,
// or backquoted string literal, returning the string value
// that s quotes.
func unquote(s string) (string, error) {
n := len(s)
if n < 2 {
return "", strconv.ErrSyntax
}
quote := s[0]
if quote != s[n-1] {
return "", strconv.ErrSyntax
}
s = s[1 : n-1]
if quote == '`' {
if strings.ContainsRune(s, '`') {
return "", strconv.ErrSyntax
}
if strings.ContainsRune(s, '\r') {
// -1 because we know there is at least one \r to remove.
buf := make([]byte, 0, len(s)-1)
for i := 0; i < len(s); i++ {
if s[i] != '\r' {
buf = append(buf, s[i])
}
}
return string(buf), nil
}
return s, nil
}
if quote != '"' && quote != '\'' {
return "", strconv.ErrSyntax
}
if strings.ContainsRune(s, '\n') {
return "", strconv.ErrSyntax
}
// Is it trivial? Avoid allocation.
if !strings.ContainsRune(s, '\\') && !strings.ContainsRune(s, rune(quote)) {
switch quote {
case '"', '\'':
if utf8.ValidString(s) {
return s, nil
}
}
}
var runeTmp [utf8.UTFMax]byte
buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations.
for len(s) > 0 {
c, multibyte, ss, err := strconv.UnquoteChar(s, quote)
if err != nil {
return "", err
}
s = ss
if c < utf8.RuneSelf || !multibyte {
buf = append(buf, byte(c))
} else {
n := utf8.EncodeRune(runeTmp[:], c)
buf = append(buf, runeTmp[:n]...)
}
}
return string(buf), nil
}

125
caql/unquote_test.go Normal file
View File

@@ -0,0 +1,125 @@
// Adapted from https://github.com/golang/go
// Original License:
//
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the https://go.dev/LICENSE file.
package caql
import (
"strconv"
"testing"
)
type quoteTest struct {
in string
out string
ascii string
graphic string
}
var quotetests = []quoteTest{
{in: "\a\b\f\r\n\t\v", out: `"\a\b\f\r\n\t\v"`, ascii: `"\a\b\f\r\n\t\v"`, graphic: `"\a\b\f\r\n\t\v"`},
{"\\", `"\\"`, `"\\"`, `"\\"`},
{"abc\xffdef", `"abc\xffdef"`, `"abc\xffdef"`, `"abc\xffdef"`},
{"\u263a", `"☺"`, `"\u263a"`, `"☺"`},
{"\U0010ffff", `"\U0010ffff"`, `"\U0010ffff"`, `"\U0010ffff"`},
{"\x04", `"\x04"`, `"\x04"`, `"\x04"`},
// Some non-printable but graphic runes. Final column is double-quoted.
{"!\u00a0!\u2000!\u3000!", `"!\u00a0!\u2000!\u3000!"`, `"!\u00a0!\u2000!\u3000!"`, "\"!\u00a0!\u2000!\u3000!\""},
}
type unQuoteTest struct {
in string
out string
}
var unquotetests = []unQuoteTest{
{`""`, ""},
{`"a"`, "a"},
{`"abc"`, "abc"},
{`"☺"`, "☺"},
{`"hello world"`, "hello world"},
{`"\xFF"`, "\xFF"},
{`"\377"`, "\377"},
{`"\u1234"`, "\u1234"},
{`"\U00010111"`, "\U00010111"},
{`"\U0001011111"`, "\U0001011111"},
{`"\a\b\f\n\r\t\v\\\""`, "\a\b\f\n\r\t\v\\\""},
{`"'"`, "'"},
{`'a'`, "a"},
{`'☹'`, "☹"},
{`'\a'`, "\a"},
{`'\x10'`, "\x10"},
{`'\377'`, "\377"},
{`'\u1234'`, "\u1234"},
{`'\U00010111'`, "\U00010111"},
{`'\t'`, "\t"},
{`' '`, " "},
{`'\''`, "'"},
{`'"'`, "\""},
{"``", ``},
{"`a`", `a`},
{"`abc`", `abc`},
{"`☺`", `☺`},
{"`hello world`", `hello world`},
{"`\\xFF`", `\xFF`},
{"`\\377`", `\377`},
{"`\\`", `\`},
{"`\n`", "\n"},
{"` `", ` `},
{"` `", ` `},
{"`a\rb`", "ab"},
}
var misquoted = []string{
``,
`"`,
`"a`,
`"'`,
`b"`,
`"\"`,
`"\9"`,
`"\19"`,
`"\129"`,
`'\'`,
`'\9'`,
`'\19'`,
`'\129'`,
// `'ab'`,
`"\x1!"`,
`"\U12345678"`,
`"\z"`,
"`",
"`xxx",
"`\"",
`"\'"`,
`'\"'`,
"\"\n\"",
"\"\\n\n\"",
"'\n'",
}
func TestUnquote(t *testing.T) {
for _, tt := range unquotetests {
if out, err := unquote(tt.in); err != nil || out != tt.out {
t.Errorf("unquote(%#q) = %q, %v want %q, nil", tt.in, out, err, tt.out)
}
}
// run the quote tests too, backward
for _, tt := range quotetests {
if in, err := unquote(tt.out); in != tt.in {
t.Errorf("unquote(%#q) = %q, %v, want %q, nil", tt.out, in, err, tt.in)
}
}
for _, s := range misquoted {
if out, err := unquote(s); out != "" || err != strconv.ErrSyntax {
t.Errorf("unquote(%#q) = %q, %v want %q, %v", s, out, err, "", strconv.ErrSyntax)
}
}
}

155
caql/wildcard.go Normal file
View File

@@ -0,0 +1,155 @@
// Adapted from https://github.com/golang/go
// Original License:
//
// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the https://go.dev/LICENSE file.
package caql
import (
"errors"
"strings"
"unicode/utf8"
)
// ErrBadPattern indicates a pattern was malformed.
var ErrBadPattern = errors.New("syntax error in pattern")
// match reports whether name matches the shell pattern.
// The pattern syntax is:
//
// pattern:
// { term }
// term:
// '%' matches any sequence of non-/ characters
// '_' matches any single non-/ character
// c matches character c (c != '%', '_', '\\')
// '\\' c matches character c
//
// match requires pattern to match all of name, not just a substring.
// The only possible returned error is ErrBadPattern, when pattern
// is malformed.
//
func match(pattern, name string) (matched bool, err error) {
Pattern:
for len(pattern) > 0 {
var star bool
var chunk string
star, chunk, pattern = scanChunk(pattern)
if star && chunk == "" {
// Trailing * matches rest of string unless it has a /.
return !strings.ContainsRune(name, '/'), nil
}
// Look for match at current position.
t, ok, err := matchChunk(chunk, name)
// if we're the last chunk, make sure we've exhausted the name
// otherwise we'll give a false result even if we could still match
// using the star
if ok && (len(t) == 0 || len(pattern) > 0) {
name = t
continue
}
if err != nil {
return false, err
}
if star {
// Look for match skipping i+1 bytes.
// Cannot skip /.
for i := 0; i < len(name) && name[i] != '/'; i++ {
t, ok, err := matchChunk(chunk, name[i+1:])
if ok {
// if we're the last chunk, make sure we exhausted the name
if len(pattern) == 0 && len(t) > 0 {
continue
}
name = t
continue Pattern
}
if err != nil {
return false, err
}
}
}
// Before returning false with no error,
// check that the remainder of the pattern is syntactically valid.
for len(pattern) > 0 {
_, chunk, pattern = scanChunk(pattern)
if _, _, err := matchChunk(chunk, ""); err != nil {
return false, err
}
}
return false, nil
}
return len(name) == 0, nil
}
// scanChunk gets the next segment of pattern, which is a non-star string
// possibly preceded by a star.
func scanChunk(pattern string) (star bool, chunk, rest string) {
for len(pattern) > 0 && pattern[0] == '%' {
pattern = pattern[1:]
star = true
}
var i int
Scan:
for i = 0; i < len(pattern); i++ {
switch pattern[i] {
case '\\':
// error check handled in matchChunk: bad pattern.
if i+1 < len(pattern) {
i++
}
case '%':
break Scan
}
}
return star, pattern[0:i], pattern[i:]
}
// matchChunk checks whether chunk matches the beginning of s.
// If so, it returns the remainder of s (after the match).
// Chunk is all single-character operators: literals, char classes, and ?.
func matchChunk(chunk, s string) (rest string, ok bool, err error) {
// failed records whether the match has failed.
// After the match fails, the loop continues on processing chunk,
// checking that the pattern is well-formed but no longer reading s.
failed := false
for len(chunk) > 0 {
if !failed && len(s) == 0 {
failed = true
}
switch chunk[0] {
case '_':
if !failed {
if s[0] == '/' {
failed = true
}
_, n := utf8.DecodeRuneInString(s)
s = s[n:]
}
chunk = chunk[1:]
case '\\':
chunk = chunk[1:]
if len(chunk) == 0 {
return "", false, ErrBadPattern
}
fallthrough
default:
if !failed {
if chunk[0] != s[0] {
failed = true
}
s = s[1:]
}
chunk = chunk[1:]
}
}
if failed {
return "", false, nil
}
return s, true, nil
}

50
caql/wildcard_test.go Normal file
View File

@@ -0,0 +1,50 @@
// Adapted from https://github.com/golang/go
// Original License:
//
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the https://go.dev/LICENSE file.
package caql
import "testing"
type MatchTest struct {
pattern, s string
match bool
err error
}
var matchTests = []MatchTest{
{"abc", "abc", true, nil},
{"%", "abc", true, nil},
{"%c", "abc", true, nil},
{"a%", "a", true, nil},
{"a%", "abc", true, nil},
{"a%", "ab/c", false, nil},
{"a%/b", "abc/b", true, nil},
{"a%/b", "a/c/b", false, nil},
{"a%b%c%d%e%/f", "axbxcxdxe/f", true, nil},
{"a%b%c%d%e%/f", "axbxcxdxexxx/f", true, nil},
{"a%b%c%d%e%/f", "axbxcxdxe/xxx/f", false, nil},
{"a%b%c%d%e%/f", "axbxcxdxexxx/fff", false, nil},
{"a%b_c%x", "abxbbxdbxebxczzx", true, nil},
{"a%b_c%x", "abxbbxdbxebxczzy", false, nil},
{"a\\%b", "a%b", true, nil},
{"a\\%b", "ab", false, nil},
{"a_b", "a☺b", true, nil},
{"a___b", "a☺b", false, nil},
{"a_b", "a/b", false, nil},
{"a%b", "a/b", false, nil},
{"\\", "a", false, ErrBadPattern},
{"%x", "xxx", true, nil},
}
func TestMatch(t *testing.T) {
for _, tt := range matchTests {
ok, err := match(tt.pattern, tt.s)
if ok != tt.match || err != tt.err {
t.Errorf("match(%#q, %#q) = %v, %v want %v, %v", tt.pattern, tt.s, ok, err, tt.match, tt.err)
}
}
}