diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java index 3f336544a99b..55abf02e39ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -76,7 +76,7 @@ protected boolean shouldConvert(Expression exp) { String fieldName = ((OutputExpression)exp).getFieldName(); if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) { throw new VerificationException(exp, "Attempting to assign conflicting values to field '" + - fieldName + "'."); + fieldName + "'"); } outputs.add(fieldName); prevNames.add(fieldName); @@ -111,24 +111,24 @@ public DataType getInputType(Expression exp, String fieldName) { } @Override - public void tryOutputType(Expression exp, String fieldName, DataType valueType) { + public void tryOutputType(Expression expression, String fieldName, DataType valueType) { String fieldDesc; DataType fieldType; - if (exp instanceof AttributeExpression) { + if (expression instanceof AttributeExpression) { Attribute attribute = schema.getAttribute(fieldName); if (attribute == null) { - throw new VerificationException(exp, "Attribute '" + fieldName + "' not found."); + throw new VerificationException(expression, "Attribute '" + fieldName + "' not found."); } fieldDesc = "attribute"; fieldType = attribute.getDataType(); - } else if (exp instanceof IndexExpression) { + } else if (expression instanceof IndexExpression) { SDField field = schema.getConcreteField(fieldName); if (field == null) { - throw new VerificationException(exp, "Index field '" + fieldName + "' not found."); + throw new VerificationException(expression, "Index field '" + fieldName + "' not found."); } fieldDesc = "index field"; fieldType = field.getDataType(); - } else if (exp instanceof SummaryExpression) { + } else if (expression instanceof SummaryExpression) { SummaryField field = schema.getSummaryField(fieldName); if (field == null) { // Use document field if summary field is not found @@ -137,7 +137,7 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType) fieldDesc = "document field"; fieldType = sdField.getDataType(); } else { - throw new VerificationException(exp, "Summary field '" + fieldName + "' not found."); + throw new VerificationException(expression, "Summary field '" + fieldName + "' not found."); } } else { fieldDesc = "summary field"; @@ -148,8 +148,8 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType) } if ( ! fieldType.isAssignableFrom(valueType) && ! fieldType.isAssignableFrom(createCompatType(valueType))) { - throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc + - " '" + fieldName + "' which is " + fieldType.getName() + "."); + throw new VerificationException(expression, "Can not assign " + valueType.getName() + " to " + fieldDesc + + " '" + fieldName + "' which is " + fieldType.getName() + "."); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java index d5af996bd59b..ebaa0ef37b0e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -57,7 +57,7 @@ void requireThatOutputConflictThrows() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " + - "to assign conflicting values to field 'bar'.", + "to assign conflicting values to field 'bar'", Exceptions.toMessageString(e)); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java index 4053834784f8..898f3f5b23d7 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -36,7 +36,7 @@ void testAttributeChanged() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " + - "Attempting to assign conflicting values to field 'foo'.", + "Attempting to assign conflicting values to field 'foo'", Exceptions.toMessageString(e)); } } @@ -80,7 +80,7 @@ void testIndexChanged() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " + - "Attempting to assign conflicting values to field 'foo'.", + "Attempting to assign conflicting values to field 'foo'", Exceptions.toMessageString(e)); } } @@ -124,7 +124,7 @@ void testSummaryChanged() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " + - "to assign conflicting values to field 'foo'.", + "to assign conflicting values to field 'foo'", Exceptions.toMessageString(e)); } } @@ -186,7 +186,7 @@ void requireThatMultilineOutputConflictThrows() throws ParseException { } catch (IllegalArgumentException e) { assertEquals("For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " + - "Attempting to assign conflicting values to field 'cox'.", + "Attempting to assign conflicting values to field 'cox'", Exceptions.toMessageString(e)); } } diff --git a/document/abi-spec.json b/document/abi-spec.json index 89c0e4e27196..ba2313dcf0b1 100644 --- a/document/abi-spec.json +++ b/document/abi-spec.json @@ -200,6 +200,7 @@ "public int getCode()", "public com.yahoo.document.FieldPath buildFieldPath(java.lang.String)", "public com.yahoo.document.PrimitiveDataType getPrimitiveType()", + "public com.yahoo.document.DataType getNestedType()", "public void visitMembers(com.yahoo.vespa.objects.ObjectVisitor)", "public int compareTo(com.yahoo.document.DataType)", "public boolean isMultivalue()", diff --git a/document/src/main/java/com/yahoo/document/ArrayDataType.java b/document/src/main/java/com/yahoo/document/ArrayDataType.java index b14e6cd8a424..05b578f74457 100644 --- a/document/src/main/java/com/yahoo/document/ArrayDataType.java +++ b/document/src/main/java/com/yahoo/document/ArrayDataType.java @@ -24,10 +24,12 @@ public ArrayDataType(DataType nestedType, int code) { super("Array<"+nestedType.getName()+">", code, nestedType); } + @Override public ArrayDataType clone() { return (ArrayDataType) super.clone(); } + @Override public Array createFieldValue() { return new Array(this); } diff --git a/document/src/main/java/com/yahoo/document/CollectionDataType.java b/document/src/main/java/com/yahoo/document/CollectionDataType.java index d580e27d4371..98be8d4709f5 100644 --- a/document/src/main/java/com/yahoo/document/CollectionDataType.java +++ b/document/src/main/java/com/yahoo/document/CollectionDataType.java @@ -33,9 +33,8 @@ public CollectionDataType clone() { return type; } - public DataType getNestedType() { - return nestedType; - } + @Override + public DataType getNestedType() { return nestedType; } @Override protected FieldValue createByReflection(Object arg) { return null; } diff --git a/document/src/main/java/com/yahoo/document/DataType.java b/document/src/main/java/com/yahoo/document/DataType.java index 64bd462fb34d..7a49938334fa 100644 --- a/document/src/main/java/com/yahoo/document/DataType.java +++ b/document/src/main/java/com/yahoo/document/DataType.java @@ -291,9 +291,10 @@ public FieldPath buildFieldPath(String fieldPathString) { * * @return primitive data type, or null */ - public PrimitiveDataType getPrimitiveType() { - return null; - } + public PrimitiveDataType getPrimitiveType() { return null; } + + /** Returns the nested type of this if it is a collection type, null otherwise. */ + public DataType getNestedType() { return null; } @Override public void visitMembers(ObjectVisitor visitor) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AnyDataType.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AnyDataType.java new file mode 100644 index 000000000000..3b401ae40851 --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AnyDataType.java @@ -0,0 +1,30 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.datatypes.FieldValue; + +/** + * A DataType representing "any". This is (so far) only needed during type resolution of indexing pipelines + * so it is placed here. + * + * @author bratseth + */ +class AnyDataType extends DataType { + + static final AnyDataType instance = new AnyDataType(); + + private AnyDataType() { + super("any", DataType.lastPredefinedDataTypeId() + 1); + } + + @Override + public FieldValue createFieldValue() { throw new UnsupportedOperationException(); } + + @Override + public Class getValueClass() { throw new UnsupportedOperationException(); } + + @Override + public boolean isValueCompatible(FieldValue value) { return true; } + +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java index 5a12ce76d2b6..1a5bda47b8be 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java @@ -10,6 +10,7 @@ import java.math.BigDecimal; import java.math.MathContext; +import java.util.Objects; /** * @author Simon Thoresen Hult @@ -48,59 +49,45 @@ public String toString() { public ArithmeticExpression(Expression left, Operator op, Expression right) { super(requiredInputType(left, right)); - left.getClass(); // throws NullPointerException - op.getClass(); - right.getClass(); - this.left = left; - this.op = op; - this.right = right; + this.left = Objects.requireNonNull(left); + this.op = Objects.requireNonNull(op); + this.right = Objects.requireNonNull(right); } + public Expression getLeftHandSide() { return left; } + + public Operator getOperator() { return op; } + + public Expression getRightHandSide() { return right; } + @Override public ArithmeticExpression convertChildren(ExpressionConverter converter) { // TODO: branch()? return new ArithmeticExpression(converter.convert(left), op, converter.convert(right)); } - public Expression getLeftHandSide() { - return left; - } - - public Operator getOperator() { - return op; - } - - public Expression getRightHandSide() { - return right; - } - @Override - protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); - context.setValue(evaluate(context.setValue(input).execute(left).getValue(), - context.setValue(input).execute(right).getValue())); + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); + context.setCurrentType(evaluate(context.setCurrentType(input).verify(left).getCurrentType(), + context.setCurrentType(input).verify(right).getCurrentType())); } @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - context.setValueType(evaluate(context.setValueType(input).execute(left).getValueType(), - context.setValueType(input).execute(right).getValueType())); + protected void doExecute(ExecutionContext context) { + FieldValue input = context.getCurrentValue(); + context.setCurrentValue(evaluate(context.setCurrentValue(input).execute(left).getCurrentValue(), + context.setCurrentValue(input).execute(right).getCurrentValue())); } private static DataType requiredInputType(Expression lhs, Expression rhs) { DataType lhsType = lhs.requiredInputType(); DataType rhsType = rhs.requiredInputType(); - if (lhsType == null) { - return rhsType; - } - if (rhsType == null) { - return lhsType; - } - if (!lhsType.equals(rhsType)) { + if (lhsType == null) return rhsType; + if (rhsType == null) return lhsType; + if (!lhsType.equals(rhsType)) throw new VerificationException(ArithmeticExpression.class, "Operands require conflicting input types, " + lhsType.getName() + " vs " + rhsType.getName()); - } return lhsType; } @@ -115,20 +102,12 @@ public String toString() { } @Override - public boolean equals(Object obj) { - if (!(obj instanceof ArithmeticExpression)) { - return false; - } - ArithmeticExpression exp = (ArithmeticExpression)obj; - if (!left.equals(exp.left)) { - return false; - } - if (!op.equals(exp.op)) { - return false; - } - if (!right.equals(exp.right)) { - return false; - } + public boolean equals(Object object) { + if (!(object instanceof ArithmeticExpression expression)) return false; + + if (!left.equals(expression.left)) return false; + if (!op.equals(expression.op)) return false; + if (!right.equals(expression.right)) return false; return true; } @@ -138,39 +117,28 @@ public int hashCode() { } private DataType evaluate(DataType lhs, DataType rhs) { - if (lhs == null || rhs == null) { + if (lhs == null || rhs == null) throw new VerificationException(this, "Attempting to perform arithmetic on a null value"); - } - if (!(lhs instanceof NumericDataType) || - !(rhs instanceof NumericDataType)) - { + if (!(lhs instanceof NumericDataType) || !(rhs instanceof NumericDataType)) throw new VerificationException(this, "Attempting to perform unsupported arithmetic: [" + lhs.getName() + "] " + op + " [" + rhs.getName() + "]"); - } - if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE || - rhs == DataType.FLOAT || rhs == DataType.DOUBLE) - { - if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE) { + + if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE || rhs == DataType.FLOAT || rhs == DataType.DOUBLE) { + if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE) return DataType.DOUBLE; - } return DataType.FLOAT; } - if (lhs == DataType.LONG || rhs == DataType.LONG) { + if (lhs == DataType.LONG || rhs == DataType.LONG) return DataType.LONG; - } return DataType.INT; } private FieldValue evaluate(FieldValue lhs, FieldValue rhs) { - if (lhs == null || rhs == null) { - return null; - } - if (!(lhs instanceof NumericFieldValue) || - !(rhs instanceof NumericFieldValue)) - { + if (lhs == null || rhs == null) return null; + if (!(lhs instanceof NumericFieldValue) || !(rhs instanceof NumericFieldValue)) throw new IllegalArgumentException("Unsupported operation: [" + lhs.getDataType().getName() + "] " + op + " [" + rhs.getDataType().getName() + "]"); - } + BigDecimal lhsVal = asBigDecimal((NumericFieldValue)lhs); BigDecimal rhsVal = asBigDecimal((NumericFieldValue)rhs); return switch (op) { @@ -184,31 +152,27 @@ private FieldValue evaluate(FieldValue lhs, FieldValue rhs) { private FieldValue createFieldValue(FieldValue lhs, FieldValue rhs, BigDecimal val) { if (lhs instanceof FloatFieldValue || lhs instanceof DoubleFieldValue || - rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue) - { - if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue) { + rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue) { + if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue) return new DoubleFieldValue(val.doubleValue()); - } return new FloatFieldValue(val.floatValue()); } - if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue) { + if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue) return new LongFieldValue(val.longValue()); - } return new IntegerFieldValue(val.intValue()); } public static BigDecimal asBigDecimal(NumericFieldValue value) { - if (value instanceof ByteFieldValue) { + if (value instanceof ByteFieldValue) return BigDecimal.valueOf(((ByteFieldValue)value).getByte()); - } else if (value instanceof DoubleFieldValue) { + else if (value instanceof DoubleFieldValue) return BigDecimal.valueOf(((DoubleFieldValue)value).getDouble()); - } else if (value instanceof FloatFieldValue) { + else if (value instanceof FloatFieldValue) return BigDecimal.valueOf(((FloatFieldValue)value).getFloat()); - } else if (value instanceof IntegerFieldValue) { + else if (value instanceof IntegerFieldValue) return BigDecimal.valueOf(((IntegerFieldValue)value).getInteger()); - } else if (value instanceof LongFieldValue) { + else if (value instanceof LongFieldValue) return BigDecimal.valueOf(((LongFieldValue)value).getLong()); - } throw new IllegalArgumentException("Unsupported numeric field value type '" + value.getClass().getName() + "'"); } @@ -218,4 +182,5 @@ public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) left.select(predicate, operation); right.select(predicate, operation); } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java index fee96513786c..64544a4dafb6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/AttributeExpression.java @@ -14,4 +14,5 @@ public AttributeExpression(String fieldName) { public boolean equals(Object obj) { return super.equals(obj) && obj instanceof AttributeExpression; } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java index ea9e778ddfbe..2ad8a2bac573 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeExpression.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.LongFieldValue; import java.util.Base64; @@ -15,11 +16,28 @@ public Base64DecodeExpression() { super(DataType.STRING); } + @Override + public DataType setInputType(DataType inputType, VerificationContext context) { + super.setInputType(inputType, DataType.STRING, context); + return DataType.LONG; + } + + @Override + public DataType setOutputType(DataType outputType, VerificationContext context) { + super.setOutputType(outputType, DataType.LONG, context); + return DataType.STRING; + } + + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - String input = String.valueOf(context.getValue()); - if (input.isEmpty()) { - context.setValue(new LongFieldValue(Long.MIN_VALUE)); + String input = String.valueOf(context.getCurrentValue()); + if (input.isEmpty()) { // TODO: Or if context.getCurrentValue() is null or if context.getCurrentValue().getDataType() != DataType.STRING + context.setCurrentValue(new LongFieldValue(Long.MIN_VALUE)); return; } if (input.length() > 12) { @@ -33,23 +51,14 @@ protected void doExecute(ExecutionContext context) { for (int i = decoded.length; --i >= 0;) { output = (output << 8) + (((int)decoded[i]) & 0xff); } - context.setValue(new LongFieldValue(output)); - } - - @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + context.setCurrentValue(new LongFieldValue(output)); } @Override - public DataType createdOutputType() { - return DataType.LONG; - } + public DataType createdOutputType() { return DataType.LONG; } @Override - public String toString() { - return "base64decode"; - } + public String toString() { return "base64decode"; } @Override public boolean equals(Object obj) { @@ -57,7 +66,6 @@ public boolean equals(Object obj) { } @Override - public int hashCode() { - return getClass().hashCode(); - } + public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java index 5cdaf0ab5db1..e430d5ce8fb5 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeExpression.java @@ -15,38 +15,45 @@ public final class Base64EncodeExpression extends Expression { public Base64EncodeExpression() { super(DataType.LONG); } + + @Override + public DataType setInputType(DataType inputType, VerificationContext context) { + super.setInputType(inputType, DataType.LONG, context); + return DataType.STRING; + } + + @Override + public DataType setOutputType(DataType outputType, VerificationContext context) { + super.setOutputType(outputType, DataType.STRING, context); + return DataType.LONG; + } + + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - long input = ((LongFieldValue) context.getValue()).getLong(); + long input = ((LongFieldValue) context.getCurrentValue()).getLong(); byte[] output = new byte[8]; for (int i = 0; i < output.length; ++i) { output[i] = (byte)(input & 0xffL); input >>>= 8; } String encoded = Base64.getEncoder().encodeToString(output); - context.setValue(new StringFieldValue(encoded)); - } - - @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + context.setCurrentValue(new StringFieldValue(encoded)); } @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override - public String toString() { - return "base64encode"; - } + public String toString() { return "base64encode"; } @Override public boolean equals(Object obj) { - if (!(obj instanceof Base64EncodeExpression)) { - return false; - } + if (!(obj instanceof Base64EncodeExpression)) return false; return true; } @@ -54,4 +61,5 @@ public boolean equals(Object obj) { public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BinarizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BinarizeExpression.java index 491860cd788e..8c28f53be8ed 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BinarizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BinarizeExpression.java @@ -32,19 +32,29 @@ public BinarizeExpression(double threshold) { } @Override - protected void doExecute(ExecutionContext context) { - Optional tensor = ((TensorFieldValue)context.getValue()).getTensor(); - if (tensor.isEmpty()) return; - context.setValue(new TensorFieldValue(tensor.get().map(v -> v > threshold ? 1 : 0))); + public DataType setInputType(DataType inputType, VerificationContext context) { + return super.setInputType(inputType, TensorDataType.any(), context); + } + + @Override + public DataType setOutputType(DataType outputType, VerificationContext context) { + return super.setOutputType(outputType, TensorDataType.any(), context); } @Override protected void doVerify(VerificationContext context) { - type = context.getValueType(); + type = context.getCurrentType(); if (! (type instanceof TensorDataType)) throw new IllegalArgumentException("The 'binarize' function requires a tensor, but got " + type); } + @Override + protected void doExecute(ExecutionContext context) { + Optional tensor = ((TensorFieldValue)context.getCurrentValue()).getTensor(); + if (tensor.isEmpty()) return; + context.setCurrentValue(new TensorFieldValue(tensor.get().map(v -> v > threshold ? 1 : 0))); + } + @Override public DataType createdOutputType() { return type; } @@ -57,6 +67,9 @@ public String toString() { public int hashCode() { return Objects.hash(threshold, toString().hashCode()); } @Override - public boolean equals(Object o) { return o instanceof BinarizeExpression; } + public boolean equals(Object o) { + if ( ! (o instanceof BinarizeExpression other)) return false; + return this.threshold == other.threshold; + } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BusyWaitExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BusyWaitExpression.java index 8ae77aad1b1e..61fa94402032 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BusyWaitExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/BusyWaitExpression.java @@ -6,22 +6,19 @@ /** * Utility expression that will busy-wait the amount of time given in the numeric field. - * Non-numeric fields will be ignored + * Non-numeric fields will be ignored. + * * @author baldersheim */ public final class BusyWaitExpression extends Expression { + public BusyWaitExpression() { super(UnresolvedDataType.INSTANCE); } - private static double nihlakanta(int i) { - long a = 2 + i * 4L; - return (24 * (a+2))/(double)(a*(a+1)*(a+2)*(a+3)); - } - @Override protected void doExecute(ExecutionContext context) { - FieldValue value = context.getValue(); + FieldValue value = context.getCurrentValue(); if (value instanceof NumericFieldValue num) { double napSecs = num.getNumber().doubleValue(); long doom = System.nanoTime() + (long)(1_000_000_000.0 * napSecs); @@ -35,7 +32,11 @@ protected void doExecute(ExecutionContext context) { } } - @Override protected void doVerify(VerificationContext context) { } + private static double nihlakanta(int i) { + long a = 2 + i * 4L; + return (24 * (a+2))/(double)(a*(a+1)*(a+2)*(a+3)); + } + @Override public DataType createdOutputType() { return null; } @Override public String toString() { return "busy_wait"; } @Override public boolean equals(Object obj) { return obj instanceof BusyWaitExpression; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java index c12239a99c3d..be6d0f7c473c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -18,12 +18,12 @@ */ public final class CatExpression extends ExpressionList { - public CatExpression(Expression... lst) { - this(List.of(lst)); + public CatExpression(Expression... expressions) { + this(List.of(expressions)); } - public CatExpression(Collection lst) { - super(lst, resolveInputType(lst)); + public CatExpression(Collection expressions) { + super(expressions, resolveInputType(expressions)); } @Override @@ -31,42 +31,42 @@ public CatExpression convertChildren(ExpressionConverter converter) { return new CatExpression(convertChildList(converter)); } + @Override + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); + List types = new LinkedList<>(); + for (Expression exp : this) { + DataType val = context.setCurrentType(input).verify(exp).getCurrentType(); + types.add(val); + if (val == null) { + throw new VerificationException(this, "Attempting to concatenate a null value (" + exp + ")"); + } + } + context.setCurrentType(resolveOutputType(types)); + } + @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); + FieldValue input = context.getCurrentValue(); DataType inputType = input != null ? input.getDataType() : null; - VerificationContext ver = new VerificationContext(context); - context.fillVariableTypes(ver); + VerificationContext verificationContext = new VerificationContext(context.getFieldValue()); + context.fillVariableTypes(verificationContext); List values = new LinkedList<>(); List types = new LinkedList<>(); for (Expression exp : this) { - FieldValue val = context.setValue(input).execute(exp).getValue(); + FieldValue val = context.setCurrentValue(input).execute(exp).getCurrentValue(); values.add(val); DataType type; if (val != null) { type = val.getDataType(); } else { - type = ver.setValueType(inputType).execute(this).getValueType(); + type = verificationContext.setCurrentType(inputType).verify(this).getCurrentType(); } types.add(type); } DataType type = resolveOutputType(types); - context.setValue(type == DataType.STRING ? asString(values) : asCollection(type, values)); - } - - @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - List types = new LinkedList<>(); - for (Expression exp : this) { - DataType val = context.setValueType(input).execute(exp).getValueType(); - types.add(val); - if (val == null) { - throw new VerificationException(this, "Attempting to concatenate a null value (" + exp + ")"); - } - } - context.setValueType(resolveOutputType(types)); + context.setCurrentValue(type == DataType.STRING ? asString(values) : asCollection(type, values)); } private static DataType resolveInputType(Collection list) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java index 7a556bd90d05..02443ced8112 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java @@ -36,24 +36,24 @@ public ChoiceExpression convertChildren(ExpressionConverter converter) { return new ChoiceExpression(asList().stream().map(choice -> converter.branch().convert(choice)).toList()); } + @Override + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); + context.setCurrentType(input); + for (Expression exp : this) + context.setCurrentType(input).verify(exp); + } + @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); + FieldValue input = context.getCurrentValue(); for (Expression expression : this) { - context.setValue(input).execute(expression); - if (context.getValue() != null) + context.setCurrentValue(input).execute(expression); + if (context.getCurrentValue() != null) break; // value found } } - @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - context.setValueType(input); - for (Expression exp : this) - context.setValueType(input).execute(exp); - } - private static DataType resolveInputType(Collection list) { DataType previousInput = null; DataType previousOutput = null; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java index d7823b8d6a5e..48b69525df52 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateExpression.java @@ -13,24 +13,20 @@ public ClearStateExpression() { } @Override - protected void doExecute(ExecutionContext context) { + protected void doVerify(VerificationContext context) { context.clear(); } @Override - protected void doVerify(VerificationContext context) { + protected void doExecute(ExecutionContext context) { context.clear(); } @Override - public DataType createdOutputType() { - return null; - } + public DataType createdOutputType() { return null; } @Override - public String toString() { - return "clear_state"; - } + public String toString() { return "clear_state"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java index fbc47e59ffae..df1ec9e39b44 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java @@ -17,12 +17,10 @@ protected CompositeExpression(DataType inputType) { } protected static String toScriptBlock(Expression exp) { - if (exp instanceof ScriptExpression) { + if (exp instanceof ScriptExpression) return exp.toString(); - } - if (exp instanceof StatementExpression) { + if (exp instanceof StatementExpression) return new ScriptExpression((StatementExpression)exp).toString(); - } return new ScriptExpression(new StatementExpression(exp)).toString(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ConstantExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ConstantExpression.java index 76ff93295983..986556a343f4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ConstantExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ConstantExpression.java @@ -21,18 +21,31 @@ public ConstantExpression(FieldValue value) { this.value = Objects.requireNonNull(value); } - public FieldValue getValue() { - return value; + public FieldValue getValue() { return value; } + + @Override + public DataType setInputType(DataType inputType, VerificationContext context) { + super.setInputType(inputType, context); + return value.getDataType(); } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(value); + public DataType setOutputType(DataType outputType, VerificationContext context) { + // TODO: + //if (outputType != value.getDataType()) + // throw new IllegalArgumentException(this + " produces a " + value.getDataType() + ", but a " + + // outputType + " is required"); + return super.setOutputType(outputType, context); } @Override protected void doVerify(VerificationContext context) { - context.setValueType(value.getDataType()); + context.setCurrentType(value.getDataType()); + } + + @Override + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(value); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java index 4001e2a6fd04..173f086d9231 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EchoExpression.java @@ -21,34 +21,22 @@ public EchoExpression(PrintStream out) { this.out = out; } - public PrintStream getOutputStream() { - return out; - } + public PrintStream getOutputStream() { return out; } @Override protected void doExecute(ExecutionContext context) { - out.println(context.getValue()); - } - - @Override - protected void doVerify(VerificationContext context) { - // empty + out.println(context.getCurrentValue()); } @Override - public DataType createdOutputType() { - return null; - } + public DataType createdOutputType() { return null; } @Override - public String toString() { - return "echo"; - } + public String toString() { return "echo"; } @Override public boolean equals(Object obj) { if (!(obj instanceof EchoExpression rhs)) return false; - return out == rhs.out; } @@ -56,4 +44,5 @@ public boolean equals(Object obj) { public int hashCode() { return getClass().hashCode() + out.hashCode(); } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java index 8a51ce2c5b41..1fbad0298827 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/EmbedExpression.java @@ -64,38 +64,82 @@ else if ( ! embedders.containsKey(embedderId)) { } } + @Override + public DataType setInputType(DataType type, VerificationContext context) { + super.setInputType(type, context); + if ( ! (type == DataType.STRING) + && ! (type instanceof ArrayDataType array && array.getNestedType() == DataType.STRING)) + throw new IllegalArgumentException("embed request either a string or array input type, but got " + type); + return null; // embed cannot determine the output type from the input + } + + @Override + public DataType setOutputType(DataType type, VerificationContext context) { + super.setOutputType(type, TensorDataType.any(), context); + return getInputType(context); // the input (string vs. array of string) cannot be determined from the output + } + @Override public void setStatementOutput(DocumentType documentType, Field field) { targetType = toTargetTensor(field.getDataType()); destination = documentType.getName() + "." + field.getName(); } + @Override + protected void doVerify(VerificationContext context) { + targetType = toTargetTensor(getOutputType(context)); + if ( ! validTarget(targetType)) + throw new VerificationException(this, "The embedding target field must either be a dense 1d tensor, a mapped 1d tensor, a mapped 2d tensor, " + + "an array of dense 1d tensors, or a mixed 2d or 3d tensor"); + if (targetType.rank() == 2 && targetType.mappedSubtype().rank() == 2) { + if (embedderArguments.size() != 1) + throw new VerificationException(this, "When the embedding target field is a 2d mapped tensor " + + "the name of the tensor dimension that corresponds to the input array elements must " + + "be given as a second argument to embed, e.g: ... | embed splade paragraph | ..."); + if ( ! targetType.mappedSubtype().dimensionNames().contains(embedderArguments.get(0))) { + throw new VerificationException(this, "The dimension '" + embedderArguments.get(0) + "' given to embed " + + "is not a sparse dimension of the target type " + targetType); + + } + } + if (targetType.rank() == 3) { + if (embedderArguments.size() != 1) + throw new VerificationException(this, "When the embedding target field is a 3d tensor " + + "the name of the tensor dimension that corresponds to the input array elements must " + + "be given as a second argument to embed, e.g: ... | embed colbert paragraph | ..."); + if ( ! targetType.mappedSubtype().dimensionNames().contains(embedderArguments.get(0))) + throw new VerificationException(this, "The dimension '" + embedderArguments.get(0) + "' given to embed " + + "is not a sparse dimension of the target type " + targetType); + } + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - if (context.getValue() == null) return; + if (context.getCurrentValue() == null) return; Tensor output; - if (context.getValue().getDataType() == DataType.STRING) { + if (context.getCurrentValue().getDataType() == DataType.STRING) { output = embedSingleValue(context); } - else if (context.getValue().getDataType() instanceof ArrayDataType && - ((ArrayDataType)context.getValue().getDataType()).getNestedType() == DataType.STRING) { + else if (context.getCurrentValue().getDataType() instanceof ArrayDataType arrayType + && arrayType.getNestedType() == DataType.STRING) { output = embedArrayValue(context); } else { throw new IllegalArgumentException("Embedding can only be done on string or string array fields, not " + - context.getValue().getDataType()); + context.getCurrentValue().getDataType()); } - context.setValue(new TensorFieldValue(output)); + context.setCurrentValue(new TensorFieldValue(output)); } private Tensor embedSingleValue(ExecutionContext context) { - StringFieldValue input = (StringFieldValue)context.getValue(); + StringFieldValue input = (StringFieldValue)context.getCurrentValue(); return embed(input.getString(), targetType, context); } @SuppressWarnings("unchecked") private Tensor embedArrayValue(ExecutionContext context) { - var input = (Array)context.getValue(); + var input = (Array)context.getCurrentValue(); var builder = Tensor.Builder.of(targetType); if (targetType.rank() == 2) if (targetType.indexedSubtype().rank() == 1) @@ -177,47 +221,13 @@ private Tensor embed(String input, TensorType targetType, ExecutionContext conte targetType); } - @Override - protected void doVerify(VerificationContext context) { - String outputField = context.getOutputField(); - if (outputField == null) - throw new VerificationException(this, "No output field in this statement: " + - "Don't know what tensor type to embed into"); - targetType = toTargetTensor(context.getInputType(this, outputField)); - if ( ! validTarget(targetType)) - throw new VerificationException(this, "The embedding target field must either be a dense 1d tensor, a mapped 1d tensor, a mapped 2d tensor, " + - "an array of dense 1d tensors, or a mixed 2d or 3d tensor"); - if (targetType.rank() == 2 && targetType.mappedSubtype().rank() == 2) { - if (embedderArguments.size() != 1) - throw new VerificationException(this, "When the embedding target field is a 2d mapped tensor " + - "the name of the tensor dimension that corresponds to the input array elements must " + - "be given as a second argument to embed, e.g: ... | embed splade paragraph | ..."); - if ( ! targetType.mappedSubtype().dimensionNames().contains(embedderArguments.get(0))) { - throw new VerificationException(this, "The dimension '" + embedderArguments.get(0) + "' given to embed " + - "is not a sparse dimension of the target type " + targetType); - - } - } - if (targetType.rank() == 3) { - if (embedderArguments.size() != 1) - throw new VerificationException(this, "When the embedding target field is a 3d tensor " + - "the name of the tensor dimension that corresponds to the input array elements must " + - "be given as a second argument to embed, e.g: ... | embed colbert paragraph | ..."); - if ( ! targetType.mappedSubtype().dimensionNames().contains(embedderArguments.get(0))) - throw new VerificationException(this, "The dimension '" + embedderArguments.get(0) + "' given to embed " + - "is not a sparse dimension of the target type " + targetType); - } - - context.setValueType(createdOutputType()); - } - @Override public DataType createdOutputType() { return new TensorDataType(targetType); } private static TensorType toTargetTensor(DataType dataType) { - if (dataType instanceof ArrayDataType) return toTargetTensor(((ArrayDataType) dataType).getNestedType()); + if (dataType instanceof ArrayDataType) return toTargetTensor(dataType.getNestedType()); if ( ! ( dataType instanceof TensorDataType)) throw new IllegalArgumentException("Expected a tensor data type but got " + dataType); return ((TensorDataType)dataType).getTensorType(); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java index 7481363b7372..259a89e98032 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExactExpression.java @@ -23,7 +23,7 @@ */ public final class ExactExpression extends Expression { - private int maxTokenLength; + private final int maxTokenLength; private ExactExpression(OptionalInt maxTokenLength) { super(DataType.STRING); @@ -40,11 +40,11 @@ public ExactExpression(int maxTokenLength) { @Override protected void doExecute(ExecutionContext context) { - StringFieldValue input = (StringFieldValue) context.getValue(); + StringFieldValue input = (StringFieldValue) context.getCurrentValue(); if (input.getString().isEmpty()) return; StringFieldValue output = input.clone(); - context.setValue(output); + context.setCurrentValue(output); String prev = output.getString(); String next = toLowerCase(prev); @@ -74,24 +74,16 @@ protected void doExecute(ExecutionContext context) { } @Override - protected void doVerify(VerificationContext context) { - // empty - } - - @Override - public DataType createdOutputType() { - return null; - } + public DataType createdOutputType() { return null; } @Override - public String toString() - { - StringBuilder ret = new StringBuilder(); - ret.append("exact"); + public String toString() { + StringBuilder s = new StringBuilder(); + s.append("exact"); if (maxTokenLength != AnnotatorConfig.getDefaultMaxTokenLength()) { - ret.append(" max-token-length:" + maxTokenLength); + s.append(" max-token-length:").append(maxTokenLength); } - return ret.toString(); + return s.toString(); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java index dc7c11394f90..ea659eb622ea 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContext.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.collections.LazyMap; -import com.yahoo.document.DataType; import com.yahoo.document.FieldPath; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.language.Language; @@ -16,11 +15,11 @@ /** * @author Simon Thoresen Hult */ -public class ExecutionContext implements FieldTypeAdapter, FieldValueAdapter { +public class ExecutionContext { private final Map variables = new HashMap<>(); - private final FieldValueAdapter adapter; - private FieldValue value; + private final FieldValueAdapter fieldValue; + private FieldValue currentValue; private Language language; private final Map cache = LazyMap.newHashMap(); @@ -28,8 +27,8 @@ public ExecutionContext() { this(null); } - public ExecutionContext(FieldValueAdapter adapter) { - this.adapter = adapter; + public ExecutionContext(FieldValueAdapter fieldValue) { + this.fieldValue = fieldValue; this.language = Language.UNKNOWN; } @@ -44,46 +43,23 @@ public ExecutionContext execute(Expression expression) { * or a partial execution of only the statements accessing the available data. */ public boolean isComplete() { - return adapter != null && adapter.isComplete(); + return fieldValue != null && fieldValue.isComplete(); } - @Override - public DataType getInputType(Expression exp, String fieldName) { - return adapter.getInputType(exp, fieldName); + public FieldValue getFieldValue(String fieldName) { + return fieldValue.getInputValue(fieldName); } - @Override - public FieldValue getInputValue(String fieldName) { - if (adapter == null) { - throw new IllegalStateException("Can not get field '" + fieldName + "' because adapter is null"); - } - return adapter.getInputValue(fieldName); - } - - @Override - public FieldValue getInputValue(FieldPath fieldPath) { - if (adapter == null) { - throw new IllegalStateException("Can not get field '" + fieldPath + "' because adapter is null"); - } - return adapter.getInputValue(fieldPath); + public FieldValue getFieldValue(FieldPath fieldPath) { + return fieldValue.getInputValue(fieldPath); } - @Override - public void tryOutputType(Expression exp, String fieldName, DataType valueType) { - adapter.tryOutputType(exp, fieldName, valueType); - } - - @Override - public ExecutionContext setOutputValue(Expression exp, String fieldName, FieldValue fieldValue) { - if (adapter == null) - throw new IllegalStateException("Can not set field '" + fieldName + "' because adapter is null"); - adapter.setOutputValue(exp, fieldName, fieldValue); + public ExecutionContext setFieldValue(String fieldName, FieldValue fieldValue, Expression expression) { + this.fieldValue.setOutputValue(expression, fieldName, fieldValue); return this; } - public FieldValueAdapter getAdapter() { - return adapter; - } + public FieldValueAdapter getFieldValue() { return fieldValue; } public FieldValue getVariable(String name) { return variables.get(name); @@ -94,6 +70,33 @@ public ExecutionContext setVariable(String name, FieldValue value) { return this; } + public FieldValue getCurrentValue() { return currentValue; } + + public ExecutionContext setCurrentValue(FieldValue value) { + this.currentValue = value; + return this; + } + + /** Returns a cached value, or null if not present. */ + public Object getCachedValue(Object key) { + return cache.get(key); + } + + public void putCachedValue(String key, Object value) { + cache.put(key, value); + } + + /** Returns a mutable reference to the cache of this. */ + public Map getCache() { + return cache; + } + + void fillVariableTypes(VerificationContext vctx) { + for (var entry : variables.entrySet()) { + vctx.setVariable(entry.getKey(), entry.getValue().getDataType()); + } + } + public Language getLanguage() { return language; } public ExecutionContext setLanguage(Language language) { @@ -105,7 +108,7 @@ public Language resolveLanguage(Linguistics linguistics) { if (language != Language.UNKNOWN) return language; if (linguistics == null) return Language.ENGLISH; - Detection detection = linguistics.getDetector().detect(String.valueOf(value), null); + Detection detection = linguistics.getDetector().detect(String.valueOf(currentValue), null); if (detection == null) return Language.ENGLISH; Language detected = detection.getLanguage(); @@ -113,38 +116,11 @@ public Language resolveLanguage(Linguistics linguistics) { return detected; } - public FieldValue getValue() { return value; } - - public ExecutionContext setValue(FieldValue value) { - this.value = value; - return this; - } - - public void putCachedValue(String key, Object value) { - cache.put(key, value); - } - - /** Returns a cached value, or null if not present. */ - public Object getCachedValue(Object key) { - return cache.get(key); - } - - /** Returns a mutable reference to the cache of this. */ - public Map getCache() { - return cache; - } - /** Clears all state in this except the cache. */ public ExecutionContext clear() { variables.clear(); - value = null; + currentValue = null; return this; } - void fillVariableTypes(VerificationContext vctx) { - for (var entry : variables.entrySet()) { - vctx.setVariable(entry.getKey(), entry.getValue().getDataType()); - } - } - } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionValueExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionValueExpression.java index 118ab14ad7bb..363f08ddb07e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionValueExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionValueExpression.java @@ -30,17 +30,10 @@ protected void doExecute(ExecutionContext context) { } @Override - protected void doVerify(VerificationContext context) {} + public DataType createdOutputType() { return UnresolvedDataType.INSTANCE; } @Override - public DataType createdOutputType() { - return UnresolvedDataType.INSTANCE; - } - - @Override - public String toString() { - return "_"; - } + public String toString() { return "_"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index 06d13557316b..49808d93d489 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentUpdate; import com.yahoo.document.Field; +import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; @@ -22,17 +23,21 @@ */ public abstract class Expression extends Selectable { - private final DataType inputType; + private final DataType requiredInputType; + + // Input and output types resolved during verification + private DataType inputType; + private DataType outputType; /** * Creates an expression * - * @param inputType the type of the input this expression can work with. - * UnresolvedDataType.INSTANCE if it works with any type, - * and null if it does not consume any input. + * @param requiredInputType the type of the input this expression can work with. + * UnresolvedDataType.INSTANCE if it works with any type, + * and null if it does not consume any input. */ - protected Expression(DataType inputType) { - this.inputType = inputType; + protected Expression(DataType requiredInputType) { + this.requiredInputType = requiredInputType; } /** @@ -44,78 +49,85 @@ protected Expression(DataType inputType) { /** Sets the document type and field the statement this expression is part of will write to */ public void setStatementOutput(DocumentType documentType, Field field) {} - public final FieldValue execute(FieldValue val) { - return execute(new ExecutionContext().setValue(val)); - } + public final DataType requiredInputType() { return requiredInputType; } - public final Document execute(AdapterFactory factory, Document doc) { - return execute(factory.newDocumentAdapter(doc)); - } + public DataType getInputType(VerificationContext context) { return inputType; } - public final Document execute(DocumentAdapter adapter) { - execute((FieldValueAdapter)adapter); - return adapter.getFullOutput(); + /** + * Sets the input type of this and returns the resulting output type, or null if it cannot be + * uniquely determined. + * This default implementation returns the same type, which is appropriate for all statements + * that do not change the type. + * + * @param inputType the type to set as the input type of this, or null if it cannot be determined + * @param requiredType the type the input type must be assignable to + * @param context the context of this + * @throws IllegalArgumentException if inputType isn't assignable to requiredType + */ + protected final DataType setInputType(DataType inputType, DataType requiredType, VerificationContext context) { + // TODO: Activate type chedcking + // if ( ! (inputType instanceof TensorDataType)) + // throw new IllegalArgumentException(this + " requires a " + requiredType + ", but gets " + inputType); + this.inputType = inputType; + return inputType; } - public static DocumentUpdate execute(Expression expression, AdapterFactory factory, DocumentUpdate update) { - DocumentUpdate result = null; - for (UpdateAdapter adapter : factory.newUpdateAdapterList(update)) { - DocumentUpdate output = adapter.getExpression(expression).execute(adapter); - if (output == null) { - // ignore - } else if (result != null) { - result.addAll(output); - } else { - result = output; - } - } - if (result != null) { - result.setCreateIfNonExistent(update.getCreateIfNonExistent()); - } - return result; + /** + * Sets the input type of this and returns the resulting output type, or null if it cannot be + * uniquely determined. + * Subtypes may implement this by calling setInputType(inputType, requiredType, VerificationContext context). + */ + public DataType setInputType(DataType inputType, VerificationContext context) { + this.inputType = inputType; + return inputType; } - public final DocumentUpdate execute(UpdateAdapter adapter) { - execute((FieldValueAdapter)adapter); - return adapter.getOutput(); - } + /** + * Returns the output type this is must produce (since it is part of a statement expression), + * or null if this is not set or there is no output produced at the end of the statement. + */ + public DataType getOutputType(VerificationContext context) { return outputType; } - public final FieldValue execute(FieldValueAdapter adapter) { - return execute(new ExecutionContext(adapter)); + /** + * Sets the output type of this and returns the resulting input type, or null if it cannot be + * uniquely determined. + * This implementation returns the same type, which is appropriate for all statements + * that do not change the type. + * + * @param outputType the type to set as the output type of this, or null if it cannot be determined + * @param requiredType the type the output type must be assignable to + * @param context the context of this + * @throws IllegalArgumentException if outputType isn't assignable to requiredType + */ + protected final DataType setOutputType(DataType outputType, DataType requiredType, VerificationContext context) { + // TODO: Activate type checking + // if (outputType != null && ! requiredType.isAssignableFrom(outputType)) + // throw new IllegalArgumentException(this + " produces a " + outputType + " but " + requiredType + " is required"); + this.outputType = outputType; + return outputType; } - public final FieldValue execute(ExecutionContext context) { - DataType inputType = requiredInputType(); - if (inputType != null) { - FieldValue input = context.getValue(); - if (input == null) { - return null; - } - if (!inputType.isValueCompatible(input)) { - throw new IllegalArgumentException("Expression '" + this + "' expected " + inputType.getName() + - " input, got " + input.getDataType().getName()); - } - } - doExecute(context); - DataType outputType = createdOutputType(); - if (outputType != null) { - FieldValue output = context.getValue(); - if (output != null && !outputType.isValueCompatible(output)) { - throw new IllegalStateException("Expression '" + this + "' expected " + outputType.getName() + - " output, got " + output.getDataType().getName()); - } - } - return context.getValue(); + /** + * Sets the output type of this and returns the resulting input type, or null if it cannot be + * uniquely determined. + * Subtypes implement this by calling setOutputType(outputType, requiredType, VerificationContext context). + */ + public DataType setOutputType(DataType outputType, VerificationContext context) { + this.outputType = outputType; + return outputType; } - protected abstract void doExecute(ExecutionContext context); + public abstract DataType createdOutputType(); + + /** Implementations that don't change the type should implement this to do verification. */ + protected void doVerify(VerificationContext context) {} public final DataType verify() { return verify(new VerificationContext()); } public final DataType verify(DataType val) { - return verify(new VerificationContext().setValueType(val)); + return verify(new VerificationContext().setCurrentType(val)); } public final Document verify(Document doc) { @@ -160,23 +172,23 @@ public final DataType verify(FieldTypeAdapter adapter) { } public final DataType verify(VerificationContext context) { - if (inputType != null) { - DataType input = context.getValueType(); + if (requiredInputType != null) { + DataType input = context.getCurrentType(); if (input == null) { - throw new VerificationException(this, "Expected " + inputType.getName() + " input, but no input is specified"); + throw new VerificationException(this, "Expected " + requiredInputType.getName() + " input, but no input is specified"); } if (input.getPrimitiveType() == UnresolvedDataType.INSTANCE) { throw new VerificationException(this, "Failed to resolve input type"); } - if (!inputType.isAssignableFrom(input)) { - throw new VerificationException(this, "Expected " + inputType.getName() + " input, got " + + if (!requiredInputType.isAssignableFrom(input)) { + throw new VerificationException(this, "Expected " + requiredInputType.getName() + " input, got " + input.getName()); } } doVerify(context); DataType outputType = createdOutputType(); if (outputType != null) { - DataType output = context.getValueType(); + DataType output = context.getCurrentType(); if (output == null) { throw new VerificationException(this, "Expected " + outputType.getName() + " output, but no output is specified"); } @@ -187,14 +199,73 @@ public final DataType verify(VerificationContext context) { throw new VerificationException(this, "Expected " + outputType.getName() + " output, got " + output.getName()); } } - return context.getValueType(); + return context.getCurrentType(); + } + + public final FieldValue execute(FieldValue val) { + return execute(new ExecutionContext().setCurrentValue(val)); } - protected abstract void doVerify(VerificationContext context); + public final Document execute(AdapterFactory factory, Document doc) { + return execute(factory.newDocumentAdapter(doc)); + } - public final DataType requiredInputType() { return inputType; } + public final Document execute(DocumentAdapter adapter) { + execute((FieldValueAdapter)adapter); + return adapter.getFullOutput(); + } - public abstract DataType createdOutputType(); + public static DocumentUpdate execute(Expression expression, AdapterFactory factory, DocumentUpdate update) { + DocumentUpdate result = null; + for (UpdateAdapter adapter : factory.newUpdateAdapterList(update)) { + DocumentUpdate output = adapter.getExpression(expression).execute(adapter); + if (output == null) { + // ignore + } else if (result != null) { + result.addAll(output); + } else { + result = output; + } + } + if (result != null) { + result.setCreateIfNonExistent(update.getCreateIfNonExistent()); + } + return result; + } + + public final DocumentUpdate execute(UpdateAdapter adapter) { + execute((FieldValueAdapter)adapter); + return adapter.getOutput(); + } + + public final FieldValue execute(FieldValueAdapter adapter) { + return execute(new ExecutionContext(adapter)); + } + + public final FieldValue execute(ExecutionContext context) { + DataType inputType = requiredInputType(); + if (inputType != null) { + FieldValue input = context.getCurrentValue(); + if (input == null) return null; + + if (!inputType.isValueCompatible(input)) { + throw new IllegalArgumentException("Expression '" + this + "' expected " + inputType.getName() + + " input, got " + input.getDataType().getName()); + } + } + doExecute(context); + DataType outputType = createdOutputType(); + if (outputType != null) { + FieldValue output = context.getCurrentValue(); + if (output != null && !outputType.isValueCompatible(output)) { + throw new IllegalStateException("Expression '" + this + "' expected " + outputType.getName() + + " output, got " + output.getDataType().getName()); + } + } + return context.getCurrentValue(); + } + + protected abstract void doExecute(ExecutionContext context); /** Creates an expression with simple lingustics for testing */ public static Expression fromString(String expression) throws ParseException { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index ef9e6f41a6db..a31e09e59e1e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -8,9 +8,9 @@ import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Objects; @@ -19,7 +19,7 @@ */ public abstract class ExpressionList extends CompositeExpression implements Iterable { - private final List expressions = new LinkedList(); + private final List expressions = new ArrayList(); protected ExpressionList(Iterable expressions, DataType inputType) { super(inputType); @@ -28,6 +28,8 @@ protected ExpressionList(Iterable expressions, DataType inputType) } } + public List expressions() { return expressions; } + protected List convertChildList(ExpressionConverter converter) { return asList().stream().map(converter::convert).filter(Objects::nonNull).toList(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java index 9633c8dcfb93..6d1a72b33cce 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldTypeAdapter.java @@ -8,8 +8,8 @@ */ public interface FieldTypeAdapter { - DataType getInputType(Expression exp, String fieldName); + DataType getInputType(Expression expression, String fieldName); - void tryOutputType(Expression exp, String fieldName, DataType valueType); + void tryOutputType(Expression expression, String fieldName, DataType valueType); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java index 91622c4b85a4..685d7b12c823 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FieldValueAdapter.java @@ -12,7 +12,7 @@ public interface FieldValueAdapter extends FieldTypeAdapter { FieldValue getInputValue(String fieldName); FieldValue getInputValue(FieldPath fieldPath); - FieldValueAdapter setOutputValue(Expression exp, String fieldName, FieldValue fieldValue); + FieldValueAdapter setOutputValue(Expression expression, String fieldName, FieldValue fieldValue); /** Returns true if this has values for all possibly existing inputs, or represents a partial set of values. */ boolean isComplete(); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java index 557c91d4e533..6b44e4ef3cc3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/FlattenExpression.java @@ -17,37 +17,42 @@ import java.util.Map; /** + * Deprecated. + * * @author Simon Thoresen Hult */ +// TODO: Remove on Vespa 9 public final class FlattenExpression extends Expression { public FlattenExpression() { super(DataType.STRING); } + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - StringFieldValue input = (StringFieldValue) context.getValue(); + StringFieldValue input = (StringFieldValue) context.getCurrentValue(); SpanTree tree = input.getSpanTree(SpanTrees.LINGUISTICS); Map> map = new HashMap<>(); for (Annotation anno : tree) { SpanNode span = anno.getSpanNode(); - if (span == null) { - continue; - } - if (anno.getType() != AnnotationTypes.TERM) { - continue; - } + if (span == null) continue; + if (anno.getType() != AnnotationTypes.TERM) continue; + FieldValue val = anno.getFieldValue(); - String str; + String s; if (val instanceof StringFieldValue) { - str = ((StringFieldValue)val).getString(); + s = ((StringFieldValue)val).getString(); } else { - str = input.getString().substring(span.getFrom(), span.getTo()); + s = input.getString().substring(span.getFrom(), span.getTo()); } Integer pos = span.getTo(); List entry = map.computeIfAbsent(pos, k -> new LinkedList<>()); - entry.add(str); + entry.add(s); } String inputVal = String.valueOf(input); StringBuilder output = new StringBuilder(); @@ -61,23 +66,14 @@ protected void doExecute(ExecutionContext context) { output.append(inputVal.charAt(i)); } } - context.setValue(new StringFieldValue(output.toString())); + context.setCurrentValue(new StringFieldValue(output.toString())); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); - } - - @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override - public String toString() { - return "flatten"; - } + public String toString() { return "flatten"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index 256ae68ba656..32b8c3314ef9 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -18,75 +18,69 @@ */ public final class ForEachExpression extends CompositeExpression { - private final Expression exp; + private final Expression expression; - public ForEachExpression(Expression exp) { + public ForEachExpression(Expression expression) { super(UnresolvedDataType.INSTANCE); - this.exp = Objects.requireNonNull(exp); + this.expression = Objects.requireNonNull(expression); } - public Expression getInnerExpression() { - return exp; - } + public Expression getInnerExpression() { return expression; } @Override public ForEachExpression convertChildren(ExpressionConverter converter) { - Expression converted = converter.convert(exp); + Expression converted = converter.convert(expression); return converted != null ? new ForEachExpression(converted) : null; } @Override public void setStatementOutput(DocumentType documentType, Field field) { - exp.setStatementOutput(documentType, field); + expression.setStatementOutput(documentType, field); } @Override - protected void doExecute(final ExecutionContext context) { - FieldValue input = context.getValue(); - if (input instanceof Array || input instanceof WeightedSet) { - FieldValue next = new MyConverter(context, exp).convert(input); - if (next == null) { - VerificationContext vctx = new VerificationContext(context); - context.fillVariableTypes(vctx); - vctx.setValueType(input.getDataType()).execute(this); - next = vctx.getValueType().createFieldValue(); - } - context.setValue(next); - } else if (input instanceof Struct) { - context.setValue(new MyConverter(context, exp).convert(input)); - } else { - throw new IllegalArgumentException("Expected Array, Struct or WeightedSet input, got " + - input.getDataType().getName()); - } + public DataType setInputType(DataType inputType, VerificationContext context) { + if ( ! inputType.isMultivalue()) + throw new IllegalArgumentException("for_each requires a multivalue type, but gets " + inputType); + expression.setInputType(inputType.getNestedType(), context); + return super.setInputType(inputType, context); + } + + @Override + public DataType setOutputType(DataType outputType, VerificationContext context) { + if ( ! outputType.isMultivalue()) + throw new IllegalArgumentException("for_each produces a multivalue type, but " + outputType + " is required"); + expression.setOutputType(outputType.getNestedType(), context); + return super.setOutputType(outputType, context); } @Override protected void doVerify(VerificationContext context) { - DataType valueType = context.getValueType(); + DataType valueType = context.getCurrentType(); if (valueType instanceof ArrayDataType || valueType instanceof WeightedSetDataType) { // Set type for block evaluation - context.setValueType(((CollectionDataType)valueType).getNestedType()); + context.setCurrentType(((CollectionDataType)valueType).getNestedType()); - // Evaluate block, which sets value>Type to the output of the block - context.execute(exp); + // Evaluate block, which sets valueType to the output of the block + context.verify(expression); // Value type outside block becomes the collection type having the block output type as argument if (valueType instanceof ArrayDataType) { - context.setValueType(DataType.getArray(context.getValueType())); + context.setCurrentType(DataType.getArray(context.getCurrentType())); } else { WeightedSetDataType wset = (WeightedSetDataType)valueType; - context.setValueType(DataType.getWeightedSet(context.getValueType(), wset.createIfNonExistent(), wset.removeIfZero())); + context.setCurrentType(DataType.getWeightedSet(context.getCurrentType(), wset.createIfNonExistent(), wset.removeIfZero())); } } else if (valueType instanceof StructDataType) { for (Field field : ((StructDataType)valueType).getFields()) { DataType fieldType = field.getDataType(); - DataType structValueType = context.setValueType(fieldType).execute(exp).getValueType(); + DataType structValueType = context.setCurrentType(fieldType).verify(expression).getCurrentType(); if (!fieldType.isAssignableFrom(structValueType)) throw new VerificationException(this, "Expected " + fieldType.getName() + " output, got " + structValueType.getName()); } - context.setValueType(valueType); + context.setCurrentType(valueType); } else { throw new VerificationException(this, "Expected Array, Struct or WeightedSet input, got " + @@ -94,9 +88,29 @@ else if (valueType instanceof StructDataType) { } } + @Override + protected void doExecute(ExecutionContext context) { + FieldValue input = context.getCurrentValue(); + if (input instanceof Array || input instanceof WeightedSet) { + FieldValue next = new MyConverter(context, expression).convert(input); + if (next == null) { + VerificationContext verificationContext = new VerificationContext(context.getFieldValue()); + context.fillVariableTypes(verificationContext); + verificationContext.setCurrentType(input.getDataType()).verify(this); + next = verificationContext.getCurrentType().createFieldValue(); + } + context.setCurrentValue(next); + } else if (input instanceof Struct) { + context.setCurrentValue(new MyConverter(context, expression).convert(input)); + } else { + throw new IllegalArgumentException("Expected Array, Struct or WeightedSet input, got " + + input.getDataType().getName()); + } + } + @Override public DataType createdOutputType() { - if (exp.createdOutputType() == null) { + if (expression.createdOutputType() == null) { return null; } return UnresolvedDataType.INSTANCE; @@ -104,19 +118,19 @@ public DataType createdOutputType() { @Override public String toString() { - return "for_each { " + exp + " }"; + return "for_each { " + expression + " }"; } @Override public boolean equals(Object obj) { if (!(obj instanceof ForEachExpression rhs)) return false; - if (!exp.equals(rhs.exp)) return false; + if (!expression.equals(rhs.expression)) return false; return true; } @Override public int hashCode() { - return getClass().hashCode() + exp.hashCode(); + return getClass().hashCode() + expression.hashCode(); } private static final class MyConverter extends FieldValueConverter { @@ -137,14 +151,14 @@ protected boolean shouldConvert(FieldValue value) { @Override protected FieldValue doConvert(FieldValue value) { - context.setValue(value).execute(expression); - return context.getValue(); + context.setCurrentValue(value).execute(expression); + return context.getCurrentValue(); } } @Override public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { - select(exp, predicate, operation); + select(expression, predicate, operation); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java index bb9a1a034960..4ebbc8bdad25 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetFieldExpression.java @@ -19,27 +19,23 @@ public GetFieldExpression(String fieldName) { this.fieldName = fieldName; } - public String getFieldName() { - return fieldName; + public String getFieldName() { return fieldName; } + + @Override + public DataType setInputType(DataType inputType, VerificationContext context) { + super.setInputType(inputType, context); + return context.getFieldType(fieldName, this); } @Override - protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); - if (!(input instanceof StructuredFieldValue struct)) { - throw new IllegalArgumentException("Expected structured input, got " + input.getDataType().getName()); - } - Field field = struct.getField(fieldName); - if (field == null) { - throw new IllegalArgumentException("Field '" + fieldName + "' not found in struct type '" + - struct.getDataType().getName() + "'"); - } - context.setValue(struct.getFieldValue(field)); + public DataType setOutputType(DataType outputType, VerificationContext context) { + super.setOutputType(context.getFieldType(fieldName, this), outputType, context); + return AnyDataType.instance; } @Override protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); + DataType input = context.getCurrentType(); if (!(input instanceof StructuredDataType)) { throw new VerificationException(this, "Expected structured input, got " + input.getName()); } @@ -48,7 +44,20 @@ protected void doVerify(VerificationContext context) { throw new VerificationException(this, "Field '" + fieldName + "' not found in struct type '" + input.getName() + "'"); } - context.setValueType(field.getDataType()); + context.setCurrentType(field.getDataType()); + } + + @Override + protected void doExecute(ExecutionContext context) { + FieldValue input = context.getCurrentValue(); + if (!(input instanceof StructuredFieldValue struct)) + throw new IllegalArgumentException("Expected structured input, got " + input.getDataType().getName()); + + Field field = struct.getField(fieldName); + if (field == null) + throw new IllegalArgumentException("Field '" + fieldName + "' not found in struct type '" + + struct.getDataType().getName() + "'"); + context.setCurrentValue(struct.getFieldValue(field)); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java index 8b2604bfec5c..1f5e1c4f5b9d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarExpression.java @@ -8,29 +8,39 @@ */ public final class GetVarExpression extends Expression { - private final String varName; + private final String variableName; - public GetVarExpression(String varName) { + public GetVarExpression(String variableName) { super(null); - this.varName = varName; + this.variableName = variableName; } - public String getVariableName() { - return varName; + public String getVariableName() { return variableName; } + + @Override + public DataType setInputType(DataType inputType, VerificationContext context) { + super.setInputType(inputType, context); + return context.getVariable(variableName); } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(context.getVariable(varName)); + public DataType setOutputType(DataType outputType, VerificationContext context) { + super.setOutputType(context.getVariable(variableName), outputType, context); + return AnyDataType.instance; } @Override protected void doVerify(VerificationContext context) { - DataType input = context.getVariable(varName); + DataType input = context.getVariable(variableName); if (input == null) { - throw new VerificationException(this, "Variable '" + varName + "' not found"); + throw new VerificationException(this, "Variable '" + variableName + "' not found"); } - context.setValueType(input); + context.setCurrentType(input); + } + + @Override + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(context.getVariable(variableName)); } @Override @@ -40,19 +50,19 @@ public DataType createdOutputType() { @Override public String toString() { - return "get_var " + varName; + return "get_var " + variableName; } @Override public boolean equals(Object obj) { if (!(obj instanceof GetVarExpression rhs)) return false; - if (!varName.equals(rhs.varName)) return false; + if (!variableName.equals(rhs.variableName)) return false; return true; } @Override public int hashCode() { - return getClass().hashCode() + varName.hashCode(); + return getClass().hashCode() + variableName.hashCode(); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index 92fc852a61fe..58037e3927d5 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -15,68 +15,78 @@ */ public final class GuardExpression extends CompositeExpression { - private final Expression exp; + private final Expression expression; private final boolean shouldExecute; - public GuardExpression(Expression exp) { - super(exp.requiredInputType()); - this.exp = exp; - shouldExecute = shouldExecute(exp); + public GuardExpression(Expression expression) { + super(expression.requiredInputType()); + this.expression = expression; + shouldExecute = shouldExecute(expression); } - public Expression getInnerExpression() { - return exp; - } + public Expression getInnerExpression() { return expression; } @Override public GuardExpression convertChildren(ExpressionConverter converter) { - return new GuardExpression(converter.convert(exp)); + return new GuardExpression(converter.convert(expression)); } @Override - public void setStatementOutput(DocumentType documentType, Field field) { - exp.setStatementOutput(documentType, field); + public DataType setInputType(DataType inputType, VerificationContext context) { + super.setInputType(inputType, context); + return expression.setInputType(inputType, context); + } + + @Override + public DataType setOutputType(DataType outputType, VerificationContext context) { + super.setOutputType(outputType, context); + return expression.setOutputType(outputType, context); + } + + @Override + protected void doVerify(VerificationContext context) { + expression.verify(context); } @Override protected void doExecute(ExecutionContext context) { - if (!shouldExecute && context.getAdapter() instanceof UpdateAdapter) { - context.setValue(null); + if (!shouldExecute && context.getFieldValue() instanceof UpdateAdapter) { + context.setCurrentValue(null); } else { - exp.execute(context); + expression.execute(context); } } @Override - protected void doVerify(VerificationContext context) { - exp.verify(context); + public void setStatementOutput(DocumentType documentType, Field field) { + expression.setStatementOutput(documentType, field); } @Override public DataType createdOutputType() { - return exp.createdOutputType(); + return expression.createdOutputType(); } @Override public String toString() { - return "guard " + toScriptBlock(exp); + return "guard " + toScriptBlock(expression); } @Override public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { - select(exp, predicate, operation); + select(expression, predicate, operation); } @Override public boolean equals(Object obj) { if (!(obj instanceof GuardExpression rhs)) return false; - if (!exp.equals(rhs.exp)) return false; + if (!expression.equals(rhs.expression)) return false; return true; } @Override public int hashCode() { - return getClass().hashCode() + exp.hashCode(); + return getClass().hashCode() + expression.hashCode(); } private static boolean shouldExecute(Expression exp) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java index eb3e316ba816..bdfcd6155803 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HashExpression.java @@ -39,13 +39,27 @@ public void setStatementOutput(DocumentType documentType, Field field) { targetType = field.getDataType().getPrimitiveType(); } + @Override + protected void doVerify(VerificationContext context) { + String outputField = context.getOutputField(); + if (outputField == null) + throw new VerificationException(this, "No output field in this statement: " + + "Don't know what value to hash to"); + DataType outputFieldType = context.getFieldType(this); + if ( ! canStoreHash(outputFieldType)) + throw new VerificationException(this, "The type of the output field " + outputField + + " is not int or long but " + outputFieldType); + targetType = outputFieldType.getPrimitiveType(); + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - StringFieldValue input = (StringFieldValue) context.getValue(); + StringFieldValue input = (StringFieldValue) context.getCurrentValue(); if (targetType.equals(DataType.INT)) - context.setValue(new IntegerFieldValue(hashToInt(input.getString()))); + context.setCurrentValue(new IntegerFieldValue(hashToInt(input.getString()))); else if (targetType.equals(DataType.LONG)) - context.setValue(new LongFieldValue(hashToLong(input.getString()))); + context.setCurrentValue(new LongFieldValue(hashToLong(input.getString()))); else throw new IllegalStateException(); // won't happen } @@ -58,20 +72,6 @@ private long hashToLong(String value) { return hasher.hashString(value, StandardCharsets.UTF_8).asLong(); } - @Override - protected void doVerify(VerificationContext context) { - String outputField = context.getOutputField(); - if (outputField == null) - throw new VerificationException(this, "No output field in this statement: " + - "Don't know what value to hash to"); - DataType outputFieldType = context.getInputType(this, outputField); - if ( ! canStoreHash(outputFieldType)) - throw new VerificationException(this, "The type of the output field " + outputField + - " is not int or long but " + outputFieldType); - targetType = outputFieldType.getPrimitiveType(); - context.setValueType(createdOutputType()); - } - private boolean canStoreHash(DataType type) { if (type.equals(DataType.INT)) return true; if (type.equals(DataType.LONG)) return true; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java index a344a1d867ed..a7d314ff45a4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeExpression.java @@ -17,11 +17,16 @@ public HexDecodeExpression() { super(DataType.STRING); } + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - String input = String.valueOf(context.getValue()); + String input = String.valueOf(context.getCurrentValue()); if (input.isEmpty()) { - context.setValue(new LongFieldValue(Long.MIN_VALUE)); + context.setCurrentValue(new LongFieldValue(Long.MIN_VALUE)); return; } BigInteger output; @@ -36,23 +41,14 @@ protected void doExecute(ExecutionContext context) { if (output.compareTo(BigInteger.ZERO) > 0 && output.bitLength() == 64) { output = output.subtract(ULONG_MAX); // flip to negative } - context.setValue(new LongFieldValue(output.longValue())); - } - - @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + context.setCurrentValue(new LongFieldValue(output.longValue())); } @Override - public DataType createdOutputType() { - return DataType.LONG; - } + public DataType createdOutputType() { return DataType.LONG; } @Override - public String toString() { - return "hexdecode"; - } + public String toString() { return "hexdecode"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java index 249ddf03fc22..c32b08e12484 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeExpression.java @@ -15,31 +15,25 @@ public HexEncodeExpression() { } @Override - protected void doExecute(ExecutionContext context) { - long input = ((LongFieldValue) context.getValue()).getLong(); - context.setValue(new StringFieldValue(Long.toHexString(input))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + long input = ((LongFieldValue) context.getCurrentValue()).getLong(); + context.setCurrentValue(new StringFieldValue(Long.toHexString(input))); } @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override - public String toString() { - return "hexencode"; - } + public String toString() { return "hexencode"; } @Override public boolean equals(Object obj) { - if (!(obj instanceof HexEncodeExpression)) { - return false; - } + if (!(obj instanceof HexEncodeExpression)) return false; return true; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java index 17c4e7a49846..550355c9d242 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameExpression.java @@ -15,24 +15,20 @@ public HostNameExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new StringFieldValue(normalizeHostName(getDefaults().vespaHostname()))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new StringFieldValue(normalizeHostName(getDefaults().vespaHostname()))); } @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override - public String toString() { - return "hostname"; - } + public String toString() { return "hostname"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index 7362cbcbdf72..eeaee3ff1355 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -84,20 +84,32 @@ public void setStatementOutput(DocumentType documentType, Field field) { public Expression getIfFalseExpression() { return ifFalse; } + @Override + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); + context.setCurrentType(input).verify(left); + context.setCurrentType(input).verify(right); + var trueValue = context.setCurrentType(input).verify(ifTrue); + var falseValue = context.setCurrentType(input).verify(ifFalse); + var valueType = trueValue.getCurrentType().isAssignableFrom(falseValue.getCurrentType()) ? + trueValue.getCurrentType() : falseValue.getCurrentType(); + context.setCurrentType(valueType); + } + @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); - FieldValue leftValue = context.setValue(input).execute(left).getValue(); + FieldValue input = context.getCurrentValue(); + FieldValue leftValue = context.setCurrentValue(input).execute(left).getCurrentValue(); if (leftValue == null) { - context.setValue(null); + context.setCurrentValue(null); return; } - FieldValue rightValue = context.setValue(input).execute(right).getValue(); + FieldValue rightValue = context.setCurrentValue(input).execute(right).getCurrentValue(); if (rightValue == null) { - context.setValue(null); + context.setCurrentValue(null); return; } - context.setValue(input); + context.setCurrentValue(input); if (isTrue(leftValue, comparator, rightValue)) { ifTrue.execute(context); } else if (ifFalse != null) { @@ -105,18 +117,6 @@ protected void doExecute(ExecutionContext context) { } } - @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - context.setValueType(input).execute(left); - context.setValueType(input).execute(right); - var trueValue = context.setValueType(input).execute(ifTrue); - var falseValue = context.setValueType(input).execute(ifFalse); - var valueType = trueValue.getValueType().isAssignableFrom(falseValue.getValueType()) ? - trueValue.getValueType() : falseValue.getValueType(); - context.setValueType(valueType); - } - @Override public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) { select(left, predicate, operation); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java index d49ca4ae6798..48cf6ac237c3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java @@ -9,7 +9,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Objects; /** * @author Simon Thoresen Hult @@ -26,29 +25,30 @@ public InputExpression(String fieldName) { this.fieldName = fieldName; } - public String getFieldName() { - return fieldName; + public String getFieldName() { return fieldName; } + + @Override + protected void doVerify(VerificationContext context) { + DataType val = context.getFieldType(fieldName, this); + if (val == null) + throw new VerificationException(this, "Field '" + fieldName + "' not found"); + context.setCurrentType(val); } @Override protected void doExecute(ExecutionContext context) { if (fieldPath != null) - context.setValue(context.getInputValue(fieldPath)); + context.setCurrentValue(context.getFieldValue(fieldPath)); else - context.setValue(context.getInputValue(fieldName)); + context.setCurrentValue(context.getFieldValue(fieldName)); } @Override - protected void doVerify(VerificationContext context) { - DataType val = context.getInputType(this, fieldName); - if (val == null) - throw new VerificationException(this, "Field '" + fieldName + "' not found"); - context.setValueType(val); - } + public DataType createdOutputType() { return UnresolvedDataType.INSTANCE; } @Override - public DataType createdOutputType() { - return UnresolvedDataType.INSTANCE; + public DataType getOutputType(VerificationContext context) { + return context.getFieldType(fieldName, this); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java index 07c8dbeaa6b6..d4640086a311 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/JoinExpression.java @@ -22,17 +22,23 @@ public JoinExpression(String delimiter) { this.delimiter = delimiter; } - public String getDelimiter() { - return delimiter; + public String getDelimiter() { return delimiter; } + + @Override + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); + if (!(input instanceof ArrayDataType)) { + throw new VerificationException(this, "Expected Array input, got " + input.getName()); + } + context.setCurrentType(createdOutputType()); } @SuppressWarnings({ "unchecked" }) @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); - if (!(input instanceof Array)) { + FieldValue input = context.getCurrentValue(); + if (!(input instanceof Array)) throw new IllegalArgumentException("Expected Array input, got " + input.getDataType().getName()); - } StringBuilder output = new StringBuilder(); for (Iterator it = ((Array)input).fieldValueIterator(); it.hasNext(); ) { output.append(it.next()); @@ -40,22 +46,11 @@ protected void doExecute(ExecutionContext context) { output.append(delimiter); } } - context.setValue(new StringFieldValue(output.toString())); + context.setCurrentValue(new StringFieldValue(output.toString())); } @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - if (!(input instanceof ArrayDataType)) { - throw new VerificationException(this, "Expected Array input, got " + input.getName()); - } - context.setValueType(createdOutputType()); - } - - @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override public String toString() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpression.java index f4288e035f14..12fa802d59b1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpression.java @@ -19,19 +19,17 @@ public LiteralBoolExpression(boolean value) { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new BoolFieldValue(value)); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new BoolFieldValue(value)); } @Override - public DataType createdOutputType() { - return DataType.BOOL; - } + public DataType createdOutputType() { return DataType.BOOL; } @Override public String toString() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java index 82accf6406d0..8cf304043bed 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseExpression.java @@ -16,24 +16,20 @@ public LowerCaseExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new StringFieldValue(toLowerCase(String.valueOf(context.getValue())))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new StringFieldValue(toLowerCase(String.valueOf(context.getCurrentValue())))); } @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override - public String toString() { - return "lowercase"; - } + public String toString() { return "lowercase"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java index ad148f89d84c..e10e4f13294e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolver.java @@ -51,6 +51,7 @@ private static class Item { this.op = Objects.requireNonNull(op); this.exp = exp; } + } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java index fdfadf654007..8954951ce462 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java @@ -39,23 +39,24 @@ public NGramExpression(Linguistics linguistics, int gramSize) { this.gramSize = gramSize; } - public Linguistics getLinguistics() { - return linguistics; - } + public Linguistics getLinguistics() { return linguistics; } + + public int getGramSize() { return gramSize; } - public int getGramSize() { - return gramSize; + @Override + protected void doVerify(VerificationContext context) { + // empty } @Override protected void doExecute(ExecutionContext context) { - StringFieldValue input = (StringFieldValue) context.getValue(); + StringFieldValue input = (StringFieldValue) context.getCurrentValue(); if (input.getSpanTree(SpanTrees.LINGUISTICS) != null) { // This expression is already executed for this input instance return; } StringFieldValue output = input.clone(); - context.setValue(output); + context.setCurrentValue(output); SpanList spanList = output.setSpanTree(new SpanTree(SpanTrees.LINGUISTICS)).spanList(); int lastPosition = 0; @@ -86,11 +87,6 @@ private Span typedSpan(int from, int length, TokenType tokenType, SpanList spanL return (Span)spanList.span(from, length).annotate(AnnotationTypes.TOKEN_TYPE, tokenType.getValue()); } - @Override - protected void doVerify(VerificationContext context) { - // empty - } - @Override public DataType createdOutputType() { return null; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java index 0e6b77624cbb..d269fa1fd4a9 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeExpression.java @@ -22,33 +22,24 @@ public NormalizeExpression(Linguistics linguistics) { this.linguistics = linguistics; } - public Linguistics getLinguistics() { - return linguistics; - } + public Linguistics getLinguistics() { return linguistics; } - private static String escape(String str) { - StringBuilder buf = new StringBuilder(); - for (char c : str.toCharArray()) { - if (c >= ' ') { - buf.append(c); - } else { - buf.append(String.format("U+%04X", (int)c)); - } - } - return buf.toString(); + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override protected void doExecute(ExecutionContext context) { Transformer transformer = linguistics.getTransformer(); - var orig = String.valueOf(context.getValue()); + var orig = String.valueOf(context.getCurrentValue()); if (orig.isEmpty()) { return; // must be a no-op for all linguistics/language combinations } var lang = context.resolveLanguage(linguistics); var transformed = transformer.accentDrop(orig, lang); try { - context.setValue(new StringFieldValue(transformed)); + context.setCurrentValue(new StringFieldValue(transformed)); return; } catch (IllegalArgumentException ex) { String msg = ("bad normalize, \n" + @@ -57,13 +48,20 @@ protected void doExecute(ExecutionContext context) { "transformed: >>> " + escape(transformed) + " <<<"); logger.log(Level.SEVERE, msg); } - context.setValue(new StringFieldValue(transformer.accentDrop(String.valueOf(context.getValue()), - context.resolveLanguage(linguistics)))); + context.setCurrentValue(new StringFieldValue(transformer.accentDrop(String.valueOf(context.getCurrentValue()), + context.resolveLanguage(linguistics)))); } - @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + private static String escape(String str) { + StringBuilder buf = new StringBuilder(); + for (char c : str.toCharArray()) { + if (c >= ' ') { + buf.append(c); + } else { + buf.append(String.format("U+%04X", (int)c)); + } + } + return buf.toString(); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java index 039f123a6e7c..b00b38454628 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NowExpression.java @@ -20,29 +20,23 @@ public NowExpression(Timer timer) { this.timer = timer; } - public Timer getTimer() { - return timer; - } + public Timer getTimer() { return timer; } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new LongFieldValue(timer.currentTimeSeconds())); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new LongFieldValue(timer.currentTimeSeconds())); } @Override - public DataType createdOutputType() { - return DataType.LONG; - } + public DataType createdOutputType() { return DataType.LONG; } @Override - public String toString() { - return "now"; - } + public String toString() { return "now"; } @Override public boolean equals(Object obj) { @@ -70,6 +64,7 @@ private static class SystemTimer implements Timer { public long currentTimeSeconds() { return System.currentTimeMillis() / 1000; } + } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java index 91d88d929907..56531b3aae94 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateExpression.java @@ -29,9 +29,17 @@ public OptimizePredicateExpression() { this.optimizer = optimizer; } + @Override + protected void doVerify(VerificationContext context) { + checkVariable(context, "arity", DataType.INT, true); + checkVariable(context, "lower_bound", DataType.LONG, false); + checkVariable(context, "upper_bound", DataType.LONG, false); + context.setCurrentType(DataType.PREDICATE); + } + @Override protected void doExecute(ExecutionContext context) { - PredicateFieldValue predicate = ((PredicateFieldValue) context.getValue()).clone(); + PredicateFieldValue predicate = ((PredicateFieldValue) context.getCurrentValue()).clone(); IntegerFieldValue arity = (IntegerFieldValue) context.getVariable("arity"); LongFieldValue lower_bound = (LongFieldValue) context.getVariable("lower_bound"); LongFieldValue upper_bound = (LongFieldValue) context.getVariable("upper_bound"); @@ -39,37 +47,24 @@ protected void doExecute(ExecutionContext context) { Long upper = upper_bound != null? upper_bound.getLong() : null; PredicateOptions options = new PredicateOptions(arity.getInteger(), lower, upper); predicate.setPredicate(optimizer.process(predicate.getPredicate(), options)); - context.setValue(predicate); - } - - @Override - protected void doVerify(VerificationContext context) { - checkVariable(context, "arity", DataType.INT, true); - checkVariable(context, "lower_bound", DataType.LONG, false); - checkVariable(context, "upper_bound", DataType.LONG, false); - context.setValueType(DataType.PREDICATE); + context.setCurrentValue(predicate); } private void checkVariable(VerificationContext ctx, String var, DataType type, boolean required) { DataType input = ctx.getVariable(var); if (input == null) { - if (required) { + if (required) throw new VerificationException(this, "Variable '" + var + "' must be set"); - } } else if (input != type) { throw new VerificationException(this, "Variable '" + var + "' must have type " + type.getName()); } } @Override - public DataType createdOutputType() { - return null; - } + public DataType createdOutputType() { return null; } @Override - public String toString() { - return "optimize_predicate"; - } + public String toString() { return "optimize_predicate"; } @Override public int hashCode() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java index 4320d25a0823..a050eee7413f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/OutputExpression.java @@ -22,23 +22,24 @@ public OutputExpression(String image, String fieldName) { this.fieldName = fieldName; } - public String getFieldName() { - return fieldName; + public String getFieldName() { return fieldName; } + + @Override + protected void doVerify(VerificationContext context) { + context.tryOutputType(fieldName, context.getCurrentType(), this); } @Override protected void doExecute(ExecutionContext context) { - context.setOutputValue(this, fieldName, context.getValue()); + context.setFieldValue(fieldName, context.getCurrentValue(), this); } @Override - protected void doVerify(VerificationContext context) { - context.tryOutputType(this, fieldName, context.getValueType()); - } + public DataType createdOutputType() { return null; } @Override - public DataType createdOutputType() { - return null; + public DataType getInputType(VerificationContext context) { + return context.getFieldType(fieldName, this); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PackBitsExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PackBitsExpression.java new file mode 100644 index 000000000000..078639a8071b --- /dev/null +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/PackBitsExpression.java @@ -0,0 +1,95 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.indexinglanguage.expressions; + +import com.yahoo.document.DataType; +import com.yahoo.document.TensorDataType; +import com.yahoo.document.datatypes.TensorFieldValue; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; +import com.yahoo.tensor.Tensors; + +import java.util.Optional; + +/** + * Converts any tensor containing only ones and zeroes into one where each consecutive 8 values in the + * dense dimension are packed into a single byte. As a consequence the output type of this is a tensor + * where the dense dimension is 1/8th as large. + * + * @author bratseth + */ +public class PackBitsExpression extends Expression { + + private TensorType outputTensorType; + + /** Creates a pack_bits expression. */ + public PackBitsExpression() { + super(TensorDataType.any()); + } + + @Override + public DataType setInputType(DataType inputType, VerificationContext context) { + super.setInputType(inputType, context); + if ( ! validType(inputType)) + throw new IllegalArgumentException("pack_bits requires a tensor with one dense dimension, " + + "but got " + inputType); + outputTensorType = outputType(((TensorDataType)inputType).getTensorType()); + return new TensorDataType(outputTensorType); + } + + @Override + public DataType setOutputType(DataType outputType, VerificationContext context) { + super.setOutputType(outputType, context); + if ( ! validType(outputType)) + throw new IllegalArgumentException("pack_bits produces a tensor with one dense dimension, " + + "but need " + outputType); + outputTensorType = ((TensorDataType)outputType).getTensorType(); + return new TensorDataType(inputType(outputTensorType)); + } + + /** Returns whether this is a valid input or output from this. */ + private boolean validType(DataType type) { + if ( ! (type instanceof TensorDataType tensorType)) return false; + if ( tensorType.getTensorType().indexedSubtype().dimensions().size() != 1) return false; + return true; + } + + @Override + protected void doVerify(VerificationContext context) {} + + @Override + protected void doExecute(ExecutionContext context) { + Optional tensor = ((TensorFieldValue)context.getCurrentValue()).getTensor(); + if (tensor.isEmpty()) return; + Tensor packed = Tensors.packBits(tensor.get()); + context.setCurrentValue(new TensorFieldValue(packed)); + } + + @Override + public DataType createdOutputType() { return new TensorDataType(outputTensorType); } + + @Override + public String toString() { return "pack_bits"; } + + @Override + public int hashCode() { return toString().hashCode(); } + + @Override + public boolean equals(Object o) { return o instanceof PackBitsExpression; } + + /** Returns the type this requires when producing the given output type. */ + private TensorType inputType(TensorType givenType) { + var builder = new TensorType.Builder(TensorType.Value.INT8); // Any larger value type is also permissible + for (var d : givenType.dimensions()) + builder.dimension(d.size().isPresent() ? d.withSize(d.size().get() * 8) : d); + return builder.build(); + } + + /** Returns the type this produces from the given input type. */ + private TensorType outputType(TensorType givenType) { + var builder = new TensorType.Builder(TensorType.Value.INT8); + for (var d : givenType.dimensions()) + builder.dimension(d.size().isPresent() ? d.withSize((int) Math.ceil(d.size().get() / 8.0)) : d); + return builder.build(); + } + +} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java index 2f8917100a67..ade7cc72d149 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -20,9 +20,7 @@ public ParenthesisExpression(Expression innerExp) { this.innerExp = innerExp; } - public Expression getInnerExpression() { - return innerExp; - } + public Expression getInnerExpression() { return innerExp; } @Override public ParenthesisExpression convertChildren(ExpressionConverter converter) { @@ -35,13 +33,13 @@ public void setStatementOutput(DocumentType documentType, Field field) { } @Override - protected void doExecute(ExecutionContext context) { - innerExp.execute(context); + protected void doVerify(VerificationContext context) { + innerExp.verify(context); } @Override - protected void doVerify(VerificationContext context) { - innerExp.verify(context); + protected void doExecute(ExecutionContext context) { + innerExp.execute(context); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java index 2972d3ee3ccd..31a44172a180 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/RandomExpression.java @@ -23,26 +23,22 @@ public RandomExpression(Integer max) { this.max = max; } - public Integer getMaxValue() { - return max; - } + public Integer getMaxValue() { return max; } @Override - protected void doExecute(ExecutionContext context) { - int max; - max = Objects.requireNonNullElseGet(this.max, () -> Integer.parseInt(String.valueOf(context.getValue()))); - context.setValue(new IntegerFieldValue(ThreadLocalRandom.current().nextInt(max))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + int max; + max = Objects.requireNonNullElseGet(this.max, () -> Integer.parseInt(String.valueOf(context.getCurrentValue()))); + context.setCurrentValue(new IntegerFieldValue(ThreadLocalRandom.current().nextInt(max))); } @Override - public DataType createdOutputType() { - return DataType.INT; - } + public DataType createdOutputType() { return DataType.INT; } @Override public String toString() { @@ -60,4 +56,5 @@ public boolean equals(Object obj) { public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index 810ff261f2d3..bf241cf61784 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -43,33 +43,33 @@ public ScriptExpression convertChildren(ExpressionConverter converter) { .toList()); } + @Override + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); + for (Expression exp : this) + context.setCurrentType(input).verify(exp); + } + @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); + FieldValue input = context.getCurrentValue(); for (StatementExpression statement : this) { if (context.isComplete() || (statement.getInputFields().isEmpty() || containsAtLeastOneInputFrom(statement.getInputFields(), context))) { - context.setValue(input); + context.setCurrentValue(input); context.execute(statement); } } - context.setValue(input); + context.setCurrentValue(input); } private boolean containsAtLeastOneInputFrom(List inputFields, ExecutionContext context) { for (String inputField : inputFields) - if (context.getInputValue(inputField) != null) + if (context.getFieldValue(inputField) != null) return true; return false; } - @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - for (Expression exp : this) - context.setValueType(input).execute(exp); - } - private static DataType resolveInputType(Collection list) { DataType prev = null; for (Expression exp : list) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index 9c92dd452c32..d465d0aed66f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; -import java.util.Collection; import java.util.Collections; import java.util.List; @@ -46,29 +45,29 @@ public void setStatementOutput(DocumentType documentType, Field field) { } @Override - protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); for (Pair entry : cases) { - FieldValue val = context.getInputValue(entry.getFirst()); - if (val != null) { - context.setValue(val).execute(entry.getSecond()); - break; + DataType val = context.getFieldType(entry.getFirst(), this); + if (val == null) { + throw new VerificationException(this, "Field '" + entry.getFirst() + "' not found"); } + context.setCurrentType(val).verify(entry.getSecond()); } - context.setValue(input); + context.setCurrentType(input); } @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); + protected void doExecute(ExecutionContext context) { + FieldValue input = context.getCurrentValue(); for (Pair entry : cases) { - DataType val = context.getInputType(this, entry.getFirst()); - if (val == null) { - throw new VerificationException(this, "Field '" + entry.getFirst() + "' not found"); + FieldValue val = context.getFieldValue(entry.getFirst()); + if (val != null) { + context.setCurrentValue(val).execute(entry.getSecond()); + break; } - context.setValueType(val).execute(entry.getSecond()); } - context.setValueType(input); + context.setCurrentValue(input); } @Override @@ -79,9 +78,7 @@ public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) } @Override - public DataType createdOutputType() { - return null; - } + public DataType createdOutputType() { return null; } public List> getCases() { return Collections.unmodifiableList(cases); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java index da5338af6a6f..8dbbc0c974d6 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageExpression.java @@ -14,25 +14,24 @@ public final class SetLanguageExpression extends Expression { public SetLanguageExpression() { super(DataType.STRING); } - @Override - protected void doExecute(ExecutionContext context) { - context.setLanguage(Language.fromLanguageTag(String.valueOf(context.getValue()))); - } @Override protected void doVerify(VerificationContext context) { // empty } + @Override + protected void doExecute(ExecutionContext context) { + context.setLanguage(Language.fromLanguageTag(String.valueOf(context.getCurrentValue()))); + } + @Override public DataType createdOutputType() { return null; } @Override - public String toString() { - return "set_language"; - } + public String toString() { return "set_language"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java index 4f30e0b066f0..b4c92fc695b1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarExpression.java @@ -15,18 +15,11 @@ public SetVarExpression(String varName) { this.varName = varName; } - public String getVariableName() { - return varName; - } - - @Override - protected void doExecute(ExecutionContext context) { - context.setVariable(varName, context.getValue()); - } + public String getVariableName() { return varName; } @Override protected void doVerify(VerificationContext context) { - DataType next = context.getValueType(); + DataType next = context.getCurrentType(); DataType prev = context.getVariable(varName); if (prev != null && !prev.equals(next)) { throw new VerificationException(this, "Attempting to assign conflicting types to variable '" + varName + @@ -36,10 +29,13 @@ protected void doVerify(VerificationContext context) { } @Override - public DataType createdOutputType() { - return null; + protected void doExecute(ExecutionContext context) { + context.setVariable(varName, context.getCurrentValue()); } + @Override + public DataType createdOutputType() { return null; } + @Override public String toString() { return "set_var " + varName; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SleepExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SleepExpression.java index f7216fc1c975..cc43506539a1 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SleepExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SleepExpression.java @@ -6,17 +6,22 @@ /** * Utility expression that will sleep the amount of time given in the numeric field. - * Non-numeric fields will be ignored + * Non-numeric fields will be ignored. + * * @author baldersheim */ public final class SleepExpression extends Expression { + public SleepExpression() { super(UnresolvedDataType.INSTANCE); } + @Override + protected void doVerify(VerificationContext context) { } + @Override protected void doExecute(ExecutionContext context) { - FieldValue value = context.getValue(); + FieldValue value = context.getCurrentValue(); if (value instanceof NumericFieldValue num) { double napSecs = num.getNumber().doubleValue(); long nanos = (long)(napSecs*1_000_000_000.0); @@ -28,7 +33,6 @@ protected void doExecute(ExecutionContext context) { } } - @Override protected void doVerify(VerificationContext context) { } @Override public DataType createdOutputType() { return null; } @Override public String toString() { return "sleep"; } @Override public boolean equals(Object obj) { return obj instanceof SleepExpression; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java index f3a2b7ab4aee..b59273fc5ae2 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SplitExpression.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage.expressions; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.StringFieldValue; @@ -20,13 +21,32 @@ public SplitExpression(String splitString) { this.splitPattern = Pattern.compile(splitString); } - public Pattern getSplitPattern() { - return splitPattern; + public Pattern getSplitPattern() { return splitPattern; } + + @Override + public DataType setInputType(DataType input, VerificationContext context) { + super.setInputType(input, context); + if (input != DataType.STRING) + throw new IllegalArgumentException("split requires a string input, but got " + input); + return new ArrayDataType(DataType.STRING); + } + + @Override + public DataType setOutputType(DataType output, VerificationContext context) { + super.setOutputType(output, context); + if ( ! (output instanceof ArrayDataType) && output.getNestedType() == DataType.STRING) + throw new IllegalArgumentException("split produces a string array, but needs " + output); + return DataType.STRING; + } + + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override protected void doExecute(ExecutionContext context) { - String input = String.valueOf(context.getValue()); + String input = String.valueOf(context.getCurrentValue()); Array output = new Array<>(DataType.getArray(DataType.STRING)); if (!input.isEmpty()) { String[] splits = splitPattern.split(input); @@ -34,12 +54,7 @@ protected void doExecute(ExecutionContext context) { output.add(new StringFieldValue(split)); } } - context.setValue(output); - } - - @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + context.setCurrentValue(output); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index f9d8002100f5..e8ed41df7b9c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -11,14 +11,16 @@ import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import java.util.ArrayList; import java.util.Arrays; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; /** + * An indexing statement consisting of a list of indexing expressions, e.g "input foo | index | attribute". + * * @author Simon Thoresen Hult */ public final class StatementExpression extends ExpressionList { @@ -29,12 +31,12 @@ public final class StatementExpression extends ExpressionList { /** The name of the (last) output field this statement will write to, or null if none */ private String outputField; - public StatementExpression(Expression... lst) { - this(Arrays.asList(lst)); //TODO Can contain null - necessary ? + public StatementExpression(Expression... list) { + this(Arrays.asList(list)); // TODO: Can contain null - necessary ? } - public StatementExpression(Iterable lst) { - this(filterList(lst), null); + public StatementExpression(Iterable list) { + this(filterList(list), null); } private StatementExpression(Iterable list, Object unused) { @@ -53,6 +55,37 @@ public StatementExpression convertChildren(ExpressionConverter converter) { .toList()); } + @Override + protected void doVerify(VerificationContext context) { + if (expressions().isEmpty()) return; + + outputField = outputFieldName(); + if (outputField != null) + context.setOutputField(outputField); + + // Result input and output types: + // Some expressions can only determine their input from their output, and others only their output from + // their input. Therefore, we try resolving in both directions, which should always meet up to produce + // uniquely determined inputs and outputs of all expressions. + // forward: + int i = 0; + var inputType = getInputType(context); // A nested statement; input imposed from above + if (inputType == null) // otherwise the first expression will be an input deciding the type + inputType = expressions().get(i++).getOutputType(context); + while (i < expressions().size() && inputType != null) + inputType = expressions().get(i++).setInputType(inputType, context); + // reverse: + int j = expressions().size(); + var outputType = getOutputType(context); // A nested statement; output imposed from above + if (outputType == null) // otherwise the last expression will be an output deciding the type + outputType = expressions().get(--j).getInputType(context); + while (--j >= 0 && outputType != null) + outputType = expressions().get(j).setOutputType(outputType, context); + + for (Expression expression : expressions()) + context.verify(expression); + } + @Override protected void doExecute(ExecutionContext context) { for (Expression expression : this) { @@ -60,16 +93,12 @@ protected void doExecute(ExecutionContext context) { } } - @Override - protected void doVerify(VerificationContext context) { + private String outputFieldName() { for (Expression expression : this) { - if (expression instanceof OutputExpression) - outputField = ((OutputExpression)expression).getFieldName(); + if (expression instanceof OutputExpression output) + return output.getFieldName(); } - if (outputField != null) - context.setOutputField(outputField); - for (Expression expression : this) - context.execute(expression); + return null; } private static DataType resolveInputType(Iterable expressions) { @@ -115,16 +144,16 @@ public static StatementExpression newInstance(ScriptParserContext config) throws return ScriptParser.parseStatement(config); } - private static List filterList(Iterable lst) { - List ret = new LinkedList<>(); - for (Expression exp : lst) { - if (exp instanceof StatementExpression) { - ret.addAll(filterList((StatementExpression)exp)); - } else if (exp != null) { - ret.add(exp); + private static List filterList(Iterable list) { + List filtered = new ArrayList<>(); + for (Expression expression : list) { + if (expression instanceof StatementExpression statement) { + filtered.addAll(filterList(statement)); + } else if (expression != null) { + filtered.add(expression); } } - return ret; + return filtered; } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java index ea9ed4098953..00765e0b263c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringExpression.java @@ -22,24 +22,20 @@ public SubstringExpression(int from, int to) { this.to = to; } - public int getFrom() { - return from; - } + public int getFrom() { return from; } - public int getTo() { - return to; - } + public int getTo() { return to; } @Override - protected void doExecute(ExecutionContext context) { - String input = String.valueOf(context.getValue()); - String substring = Text.substringByCodepoints(input, from, to); - context.setValue(new StringFieldValue(substring)); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + String input = String.valueOf(context.getCurrentValue()); + String substring = Text.substringByCodepoints(input, from, to); + context.setCurrentValue(new StringFieldValue(substring)); } @Override diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index 34849096b366..120f08847f5e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -33,6 +33,16 @@ public SwitchExpression(Map cases, Expression this.cases.putAll(cases); } + public boolean isEmpty() { + return defaultExp == null && cases.isEmpty(); + } + + public Map getCases() { + return Collections.unmodifiableMap(cases); + } + + public Expression getDefaultExpression() { return defaultExp; } + @Override public SwitchExpression convertChildren(ExpressionConverter converter) { var convertedCases = new LinkedHashMap(); @@ -44,18 +54,6 @@ public SwitchExpression convertChildren(ExpressionConverter converter) { return new SwitchExpression(convertedCases, converter.branch().convert(defaultExp)); } - public boolean isEmpty() { - return defaultExp == null && cases.isEmpty(); - } - - public Map getCases() { - return Collections.unmodifiableMap(cases); - } - - public Expression getDefaultExpression() { - return defaultExp; - } - @Override public void setStatementOutput(DocumentType documentType, Field field) { defaultExp.setStatementOutput(documentType, field); @@ -63,9 +61,26 @@ public void setStatementOutput(DocumentType documentType, Field field) { expression.setStatementOutput(documentType, field); } + @Override + protected void doVerify(VerificationContext context) { + DataType input = context.getCurrentType(); + if (input == null) { + throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, but no input is specified"); + } + if (input != DataType.STRING) { + throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got " + + input.getName()); + } + for (Expression exp : cases.values()) { + context.setCurrentType(input).verify(exp); + } + context.setCurrentType(input).verify(defaultExp); + context.setCurrentType(input); + } + @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); + FieldValue input = context.getCurrentValue(); Expression exp = null; if (input != null) { if (!(input instanceof StringFieldValue)) { @@ -80,7 +95,7 @@ protected void doExecute(ExecutionContext context) { if (exp != null) { exp.execute(context); } - context.setValue(input); + context.setCurrentValue(input); } @Override @@ -92,26 +107,7 @@ public void selectMembers(ObjectPredicate predicate, ObjectOperation operation) } @Override - protected void doVerify(VerificationContext context) { - DataType input = context.getValueType(); - if (input == null) { - throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, but no input is specified"); - } - if (input != DataType.STRING) { - throw new VerificationException(this, "Expected " + DataType.STRING.getName() + " input, got " + - input.getName()); - } - for (Expression exp : cases.values()) { - context.setValueType(input).execute(exp); - } - context.setValueType(input).execute(defaultExp); - context.setValueType(input); - } - - @Override - public DataType createdOutputType() { - return null; - } + public DataType createdOutputType() { return null; } @Override public String toString() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java index c74e408e05be..b3e5e5c23f82 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ThisExpression.java @@ -13,24 +13,20 @@ public ThisExpression() { } @Override - protected void doExecute(ExecutionContext context) { + protected void doVerify(VerificationContext context) { // empty } @Override - protected void doVerify(VerificationContext context) { + protected void doExecute(ExecutionContext context) { // empty } @Override - public DataType createdOutputType() { - return UnresolvedDataType.INSTANCE; - } + public DataType createdOutputType() { return UnresolvedDataType.INSTANCE; } @Override - public String toString() { - return "this"; - } + public String toString() { return "this"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java index 779add656c01..303458a19ff7 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayExpression.java @@ -15,22 +15,22 @@ public ToArrayExpression() { super(UnresolvedDataType.INSTANCE); } + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(DataType.getArray(context.getCurrentType())); + } + @SuppressWarnings({ "unchecked", "rawtypes" }) @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); + FieldValue input = context.getCurrentValue(); DataType inputType = input.getDataType(); ArrayDataType outputType = DataType.getArray(inputType); Array output = outputType.createFieldValue(); output.add(input); - context.setValue(output); - } - - @Override - protected void doVerify(VerificationContext context) { - context.setValueType(DataType.getArray(context.getValueType())); + context.setCurrentValue(output); } @Override @@ -39,9 +39,7 @@ public DataType createdOutputType() { } @Override - public String toString() { - return "to_array"; - } + public String toString() { return "to_array"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolExpression.java index fa9d93efd031..3e0523995322 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolExpression.java @@ -16,9 +16,14 @@ public ToBoolExpression() { super(UnresolvedDataType.INSTANCE); } + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - context.setValue(new BoolFieldValue(toBooleanValue(context.getValue()))); + context.setCurrentValue(new BoolFieldValue(toBooleanValue(context.getCurrentValue()))); } private boolean toBooleanValue(FieldValue value) { @@ -30,19 +35,10 @@ private boolean toBooleanValue(FieldValue value) { } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); - } - - @Override - public DataType createdOutputType() { - return DataType.BOOL; - } + public DataType createdOutputType() { return DataType.BOOL; } @Override - public String toString() { - return "to_bool"; - } + public String toString() { return "to_bool"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java index 260138364681..1606a9a70f62 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteExpression.java @@ -14,24 +14,20 @@ public ToByteExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new ByteFieldValue(Byte.valueOf(String.valueOf(context.getValue())))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new ByteFieldValue(Byte.valueOf(String.valueOf(context.getCurrentValue())))); } @Override - public DataType createdOutputType() { - return DataType.BYTE; - } + public DataType createdOutputType() { return DataType.BYTE; } @Override - public String toString() { - return "to_byte"; - } + public String toString() { return "to_byte"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java index 7487c001e5bc..4f0e75c3f770 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleExpression.java @@ -14,24 +14,20 @@ public ToDoubleExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new DoubleFieldValue(Double.valueOf(String.valueOf(context.getValue())))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new DoubleFieldValue(Double.valueOf(String.valueOf(context.getCurrentValue())))); } @Override - public DataType createdOutputType() { - return DataType.DOUBLE; - } + public DataType createdOutputType() { return DataType.DOUBLE; } @Override - public String toString() { - return "to_double"; - } + public String toString() { return "to_double"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpression.java index ac3735188f4a..37032852b92e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpression.java @@ -10,33 +10,29 @@ * * @author bergum */ - public class ToEpochSecondExpression extends Expression { + public ToEpochSecondExpression() { super(DataType.STRING); //only accept string input } @Override - protected void doExecute(ExecutionContext context) { - String inputString = String.valueOf(context.getValue()); - long epochTime = Instant.parse(inputString).getEpochSecond(); - context.setValue(new LongFieldValue(epochTime)); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + String inputString = String.valueOf(context.getCurrentValue()); + long epochTime = Instant.parse(inputString).getEpochSecond(); + context.setCurrentValue(new LongFieldValue(epochTime)); } @Override - public DataType createdOutputType() { - return DataType.LONG; - } + public DataType createdOutputType() { return DataType.LONG; } @Override - public String toString() { - return "to_epoch_second"; - } + public String toString() { return "to_epoch_second"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java index d61a04350934..91b51e8b7e34 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatExpression.java @@ -14,24 +14,20 @@ public ToFloatExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new FloatFieldValue(Float.valueOf(String.valueOf(context.getValue())))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new FloatFieldValue(Float.valueOf(String.valueOf(context.getCurrentValue())))); } @Override - public DataType createdOutputType() { - return DataType.FLOAT; - } + public DataType createdOutputType() { return DataType.FLOAT; } @Override - public String toString() { - return "to_float"; - } + public String toString() { return "to_float"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java index 1efa0542e8c0..cbf1ff949ce8 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerExpression.java @@ -14,24 +14,20 @@ public ToIntegerExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new IntegerFieldValue(Integer.valueOf(String.valueOf(context.getValue())))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new IntegerFieldValue(Integer.valueOf(String.valueOf(context.getCurrentValue())))); } @Override - public DataType createdOutputType() { - return DataType.INT; - } + public DataType createdOutputType() { return DataType.INT; } @Override - public String toString() { - return "to_int"; - } + public String toString() { return "to_int"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java index d7ffee3cdf6b..dc601a9e2dd3 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongExpression.java @@ -14,24 +14,20 @@ public ToLongExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new LongFieldValue(Long.valueOf(String.valueOf(context.getValue())))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new LongFieldValue(Long.valueOf(String.valueOf(context.getCurrentValue())))); } @Override - public DataType createdOutputType() { - return DataType.LONG; - } + public DataType createdOutputType() { return DataType.LONG; } @Override - public String toString() { - return "to_long"; - } + public String toString() { return "to_long"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java index 7e4b61ceda32..0d4bafaf8ed5 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionExpression.java @@ -14,24 +14,20 @@ public ToPositionExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(PositionDataType.fromString(String.valueOf(context.getValue()))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(PositionDataType.fromString(String.valueOf(context.getCurrentValue()))); } @Override - public DataType createdOutputType() { - return PositionDataType.INSTANCE; - } + public DataType createdOutputType() { return PositionDataType.INSTANCE; } @Override - public String toString() { - return "to_pos"; - } + public String toString() { return "to_pos"; } @Override public boolean equals(Object obj) { @@ -42,5 +38,6 @@ public boolean equals(Object obj) { public int hashCode() { return getClass().hashCode(); } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java index 11e13477d62a..d1209e59710b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringExpression.java @@ -14,24 +14,20 @@ public ToStringExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new StringFieldValue(String.valueOf(context.getValue()))); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new StringFieldValue(String.valueOf(context.getCurrentValue()))); } @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override - public String toString() { - return "to_string"; - } + public String toString() { return "to_string"; } @Override public boolean equals(Object obj) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java index 1154bf39c05d..6ed50759e253 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetExpression.java @@ -20,36 +20,30 @@ public ToWsetExpression(boolean createIfNonExistent, boolean removeIfZero) { this.removeIfZero = removeIfZero; } - public boolean getCreateIfNonExistent() { - return createIfNonExistent; - } + public boolean getCreateIfNonExistent() { return createIfNonExistent; } + + public boolean getRemoveIfZero() { return removeIfZero; } - public boolean getRemoveIfZero() { - return removeIfZero; + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(DataType.getWeightedSet(context.getCurrentType(), createIfNonExistent, removeIfZero)); } @SuppressWarnings({ "unchecked", "rawtypes" }) @Override protected void doExecute(ExecutionContext context) { - FieldValue input = context.getValue(); + FieldValue input = context.getCurrentValue(); DataType inputType = input.getDataType(); WeightedSetDataType outputType = DataType.getWeightedSet(inputType, createIfNonExistent, removeIfZero); WeightedSet output = outputType.createFieldValue(); output.add(input); - context.setValue(output); - } - - @Override - protected void doVerify(VerificationContext context) { - context.setValueType(DataType.getWeightedSet(context.getValueType(), createIfNonExistent, removeIfZero)); + context.setCurrentValue(output); } @Override - public DataType createdOutputType() { - return UnresolvedDataType.INSTANCE; - } + public DataType createdOutputType() { return UnresolvedDataType.INSTANCE; } @Override public String toString() { @@ -60,24 +54,16 @@ public String toString() { @Override public boolean equals(Object obj) { - if (!(obj instanceof ToWsetExpression)) { - return false; - } - ToWsetExpression rhs = (ToWsetExpression)obj; - if (createIfNonExistent != rhs.createIfNonExistent) { - return false; - } - if (removeIfZero != rhs.removeIfZero) { - return false; - } + if (!(obj instanceof ToWsetExpression rhs)) return false; + + if (createIfNonExistent != rhs.createIfNonExistent) return false; + if (removeIfZero != rhs.removeIfZero) return false; return true; } @Override public int hashCode() { - return getClass().hashCode() + - createIfNonExistent.hashCode() + - removeIfZero.hashCode(); + return getClass().hashCode() + createIfNonExistent.hashCode() + removeIfZero.hashCode(); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java index a3c404e50c39..64d6c0a5a199 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeExpression.java @@ -23,19 +23,20 @@ public TokenizeExpression(Linguistics linguistics, AnnotatorConfig config) { this.config = config; } - public Linguistics getLinguistics() { - return linguistics; - } + public Linguistics getLinguistics() { return linguistics; } + + public AnnotatorConfig getConfig() { return config; } - public AnnotatorConfig getConfig() { - return config; + @Override + protected void doVerify(VerificationContext context) { + // empty } @Override protected void doExecute(ExecutionContext context) { - StringFieldValue input = (StringFieldValue)context.getValue(); + StringFieldValue input = (StringFieldValue)context.getCurrentValue(); StringFieldValue output = input.clone(); - context.setValue(output); + context.setCurrentValue(output); AnnotatorConfig cfg = new AnnotatorConfig(config); Language lang = context.resolveLanguage(linguistics); @@ -47,14 +48,7 @@ protected void doExecute(ExecutionContext context) { } @Override - protected void doVerify(VerificationContext context) { - // empty - } - - @Override - public DataType createdOutputType() { - return null; - } + public DataType createdOutputType() { return null; } @Override public String toString() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java index 3722791aa33d..78f89fabe460 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/TrimExpression.java @@ -14,30 +14,25 @@ public TrimExpression() { } @Override - protected void doExecute(ExecutionContext context) { - context.setValue(new StringFieldValue(String.valueOf(context.getValue()).trim())); + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); + protected void doExecute(ExecutionContext context) { + context.setCurrentValue(new StringFieldValue(String.valueOf(context.getCurrentValue()).trim())); } @Override - public DataType createdOutputType() { - return DataType.STRING; - } + public DataType createdOutputType() { return DataType.STRING; } @Override - public String toString() { - return "trim"; - } + public String toString() { return "trim"; } @Override public boolean equals(Object obj) { - if (!(obj instanceof TrimExpression)) { - return false; - } + if (!(obj instanceof TrimExpression)) return false; + return true; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java index f973f6a2fb82..401b3b0978c0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/UnresolvedFieldValue.java @@ -22,9 +22,7 @@ private static class Factory extends PrimitiveDataType.Factory { public static PrimitiveDataType.Factory getFactory() { return new Factory(); } @Override - public DataType getDataType() { - return UnresolvedDataType.INSTANCE; - } + public DataType getDataType() { return UnresolvedDataType.INSTANCE; } @Override @Deprecated diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java index a2926fb90398..ae49130f1ce7 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java @@ -9,69 +9,69 @@ /** * @author Simon Thoresen Hult */ -public class VerificationContext implements FieldTypeAdapter, Cloneable { +public class VerificationContext { private final Map variables = new HashMap<>(); - private final FieldTypeAdapter adapter; - private DataType value; + private final FieldTypeAdapter fieldType; + private DataType currentType; private String outputField; public VerificationContext() { - this.adapter = null; + this(null); } - public VerificationContext(FieldTypeAdapter adapter) { - this.adapter = adapter; + public VerificationContext(FieldTypeAdapter field) { + this.fieldType = field; } - public VerificationContext execute(Expression exp) { - if (exp != null) { - exp.verify(this); - } + public VerificationContext verify(Expression expression) { + if (expression != null) + expression.verify(this); return this; } - @Override - public DataType getInputType(Expression exp, String fieldName) { - return adapter.getInputType(exp, fieldName); + /** Returns the type of the field processed by this. */ + public DataType getFieldType(Expression expression) { + return fieldType.getInputType(expression, getOutputField()); } - @Override - public void tryOutputType(Expression exp, String fieldName, DataType valueType) { - adapter.tryOutputType(exp, fieldName, valueType); + /** Returns the type of the given field. */ + public DataType getFieldType(String fieldName, Expression expression) { + return fieldType.getInputType(expression, fieldName); } - public DataType getVariable(String name) { - return variables.get(name); + public void tryOutputType(String fieldName, DataType valueType, Expression expression) { + fieldType.tryOutputType(expression, fieldName, valueType); } - public VerificationContext setVariable(String name, DataType value) { - variables.put(name, value); + /** Returns the current value type */ + public DataType getCurrentType() { return currentType; } + + /** Returns the current value type */ + public VerificationContext setCurrentType(DataType value) { + this.currentType = value; return this; } - public DataType getValueType() { - return value; - } + public DataType getVariable(String name) { return variables.get(name); } - /** Sets the output value type */ - public VerificationContext setValueType(DataType value) { - this.value = value; + public VerificationContext setVariable(String name, DataType value) { + variables.put(name, value); return this; } - /** Sets the name of the (last) output field of the statement this is executed as a part of */ - public void setOutputField(String outputField) { this.outputField = outputField; } - /** * Returns the name of the (last) output field of the statement this is executed as a part of, * or null if none or not yet verified */ public String getOutputField() { return outputField; } + /** Sets the name of the (last) output field of the statement this is executed as a part of */ + public void setOutputField(String outputField) { this.outputField = outputField; } + public VerificationContext clear() { variables.clear(); - value = null; + currentType = null; return this; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java index b2462b738210..cfc7273d0901 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationException.java @@ -20,19 +20,15 @@ public VerificationException(Expression exp, String msg) { } } - public VerificationException(Class exp, String msg) { super(msg); this.type = exp; this.exp = exp.getName(); } - public String getExpression() { - return exp; - } - public Class getExpressionType() { - return type; - } + public String getExpression() { return exp; } + + public Class getExpressionType() { return type; } @Override public String toString() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java index 4ad1fc79790c..9d6826ba2535 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ZCurveExpression.java @@ -17,15 +17,20 @@ public ZCurveExpression() { super(PositionDataType.INSTANCE); } + @Override + protected void doVerify(VerificationContext context) { + context.setCurrentType(createdOutputType()); + } + @Override protected void doExecute(ExecutionContext context) { - Struct input = ((Struct) context.getValue()); + Struct input = ((Struct) context.getCurrentValue()); Integer x = getFieldValue(input, PositionDataType.FIELD_X); Integer y = getFieldValue(input, PositionDataType.FIELD_Y); if (x != null && y != null) { - context.setValue(new LongFieldValue(ZCurve.encode(x, y))); + context.setCurrentValue(new LongFieldValue(ZCurve.encode(x, y))); } else { - context.setValue(DataType.LONG.createFieldValue()); + context.setCurrentValue(DataType.LONG.createFieldValue()); } } @@ -35,25 +40,14 @@ private static Integer getFieldValue(Struct struct, String fieldName) { } @Override - protected void doVerify(VerificationContext context) { - context.setValueType(createdOutputType()); - } + public DataType createdOutputType() { return DataType.LONG; } @Override - public DataType createdOutputType() { - return DataType.LONG; - } - - @Override - public String toString() { - return "zcurve"; - } + public String toString() { return "zcurve"; } @Override public boolean equals(Object obj) { - if (!(obj instanceof ZCurveExpression)) { - return false; - } + if (!(obj instanceof ZCurveExpression)) return false; return true; } diff --git a/indexinglanguage/src/main/javacc/IndexingParser.jj b/indexinglanguage/src/main/javacc/IndexingParser.jj index 09aa3829ab91..fbf460ae6f6c 100644 --- a/indexinglanguage/src/main/javacc/IndexingParser.jj +++ b/indexinglanguage/src/main/javacc/IndexingParser.jj @@ -149,6 +149,7 @@ TOKEN : | | | + | | | | @@ -179,6 +180,7 @@ TOKEN : | | | + | | | | @@ -208,7 +210,6 @@ TOKEN : | | | - | | } @@ -326,6 +327,7 @@ Expression value() : val = normalizeExp() | val = nowExp() | val = optimizePredicateExp() | + val = packBitsExp() | val = passthroughExp() | val = randomExp() | val = script() | @@ -570,6 +572,12 @@ Expression optimizePredicateExp() : { } { return new OptimizePredicateExpression(); } } +Expression packBitsExp() : { } +{ + ( ) + { return new PackBitsExpression(); } +} + Expression passthroughExp() : { String val = defaultFieldName; @@ -846,6 +854,7 @@ String identifier() : | | | + | | | | diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTestCase.java index 0b0b00498860..b46232f0f2e1 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTestCase.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.ExecutionContext; import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; import com.yahoo.vespa.indexinglanguage.expressions.VerificationException; +import org.junit.Ignore; import org.junit.Test; import java.util.Map; @@ -59,6 +60,12 @@ public void testEmbedAndBinarize() { tester.testStatement("input myText | embed | binarize 3.0 | attribute 'myTensor'", "input text", "[0, 0, 0, 1]"); } + @Test + public void testEmbedBinarizeAndPack_bits() { + var tester = new EmbeddingScriptTester(Map.of("emb1", new EmbeddingScriptTester.MockIndexedEmbedder("myDocument.myTensor", -111))); + tester.testStatement("input myText | embed | binarize | pack_bits | attribute 'myTensor'", "input text", "tensor(x[2])", "[58, 192]"); + } + @SuppressWarnings("unchecked") @Test public void testArrayEmbed() { @@ -84,7 +91,7 @@ public void testArrayEmbed() { assertEquals(new ArrayDataType(new TensorDataType(tensorType)), expression.verify(verificationContext)); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(array); + context.setCurrentValue(array); expression.execute(context); assertTrue(adapter.values.containsKey("myTensorArray")); var tensorArray = (Array)adapter.values.get("myTensorArray"); @@ -121,7 +128,7 @@ public void testArrayEmbedWithConcatenation() { assertEquals(new TensorDataType(tensorType), expression.verify(verificationContext)); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(array); + context.setCurrentValue(array); expression.execute(context); assertTrue(adapter.values.containsKey("mySparseTensor")); var sparseTensor = (TensorFieldValue)adapter.values.get("mySparseTensor"); @@ -154,7 +161,7 @@ public void testArrayEmbedTo2dMixedTensor() { assertEquals(new TensorDataType(tensorType), expression.verify(verificationContext)); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(array); + context.setCurrentValue(array); expression.execute(context); assertTrue(adapter.values.containsKey("mySparseTensor")); var sparseTensor = (TensorFieldValue)adapter.values.get("mySparseTensor"); @@ -185,7 +192,7 @@ public void testArrayEmbedTo3dMixedTensor() { assertEquals(new TensorDataType(tensorType), expression.verify(new VerificationContext(adapter))); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(array); + context.setCurrentValue(array); expression.execute(context); assertTrue(adapter.values.containsKey("mySparseTensor")); var sparseTensor = (TensorFieldValue)adapter.values.get("mySparseTensor"); @@ -289,7 +296,7 @@ public void testEmbedToSparseTensor() { assertEquals(new TensorDataType(tensorType), expression.verify(verificationContext)); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(text); + context.setCurrentValue(text); expression.execute(context); assertTrue(adapter.values.containsKey("mySparseTensor")); var sparseTensor = (TensorFieldValue)adapter.values.get("mySparseTensor"); @@ -345,7 +352,7 @@ public void testArrayEmbedTo2MappedTensor() { assertEquals(new TensorDataType(tensorType), expression.verify(new VerificationContext(adapter))); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(array); + context.setCurrentValue(array); expression.execute(context); assertTrue(adapter.values.containsKey("my2DSparseTensor")); var sparse2DTensor = (TensorFieldValue)adapter.values.get("my2DSparseTensor"); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTester.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTester.java index b75834f1a163..c4a53c1af683 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTester.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/EmbeddingScriptTester.java @@ -32,8 +32,12 @@ public EmbeddingScriptTester(Map embedders) { } public void testStatement(String expressionString, String input, String expected) { + testStatement(expressionString, input, "tensor(d[4])", expected); + } + + public void testStatement(String expressionString, String input, String targetTensorType, String expected) { var expression = expressionFrom(expressionString); - TensorType tensorType = TensorType.fromSpec("tensor(d[4])"); + TensorType tensorType = TensorType.fromSpec(targetTensorType); SimpleTestAdapter adapter = new SimpleTestAdapter(); adapter.createField(new Field("myText", DataType.STRING)); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java index c4bd69663df7..2a145ae79e3f 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ScriptTestCase.java @@ -6,23 +6,14 @@ import com.yahoo.document.Document; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; -import com.yahoo.document.TensorDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.language.process.Embedder; -import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; import com.yahoo.vespa.indexinglanguage.expressions.*; import com.yahoo.vespa.indexinglanguage.parser.ParseException; import org.junit.Test; -import java.util.List; -import java.util.Map; - import static org.junit.Assert.*; /** @@ -118,7 +109,7 @@ public void testIntHash() throws ParseException { assertEquals(DataType.INT, expression.verify(verificationContext)); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(new StringFieldValue("input text")); + context.setCurrentValue(new StringFieldValue("input text")); expression.execute(context); assertTrue(adapter.values.containsKey("myInt")); assertEquals(-1425622096, adapter.values.get("myInt").getWrappedValue()); @@ -144,7 +135,7 @@ public void testIntArrayHash() throws ParseException { assertEquals(new ArrayDataType(DataType.INT), expression.verify(verificationContext)); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(array); + context.setCurrentValue(array); expression.execute(context); assertTrue(adapter.values.containsKey("myIntArray")); var intArray = (Array)adapter.values.get("myIntArray"); @@ -168,7 +159,7 @@ public void testLongHash() throws ParseException { assertEquals(DataType.LONG, expression.verify(verificationContext)); ExecutionContext context = new ExecutionContext(adapter); - context.setValue(new StringFieldValue("input text")); + context.setCurrentValue(new StringFieldValue("input text")); expression.execute(context); assertTrue(adapter.values.containsKey("myLong")); assertEquals(7678158186624760752L, adapter.values.get("myLong").getWrappedValue()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java index 07c6301339bb..458e1337eef0 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/SimpleTestAdapter.java @@ -17,8 +17,8 @@ */ public class SimpleTestAdapter implements FieldValueAdapter { - final Map types = new HashMap<>(); - final Map values = new HashMap<>(); + public final Map types = new HashMap<>(); + public final Map values = new HashMap<>(); public SimpleTestAdapter(Field... fields) { for (Field field : fields) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java index f7dafebba945..bee0add22c07 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticTestCase.java @@ -170,7 +170,7 @@ private void assertType(DataType lhs, Operator op, DataType rhs, DataType expect private static FieldValue evaluate(Expression lhs, Operator op, Expression rhs) { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new ArithmeticExpression(lhs, op, rhs).execute(ctx); - return ctx.getValue(); + return ctx.getCurrentValue(); } private static ArithmeticExpression newArithmetic(long lhs, Operator op, long rhs) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java index 08a0d8353a70..2d4f7afc6dc2 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64DecodeTestCase.java @@ -28,10 +28,10 @@ public void requireThatHashCodeAndEqualsAreImplemented() { @Test public void requireThatInputIsDecoded() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("zcIoHQ")); + ctx.setCurrentValue(new StringFieldValue("zcIoHQ")); new Base64DecodeExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof LongFieldValue); assertEquals(489210573L, ((LongFieldValue)val).getLong()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java index 20dfd762057f..2f73c07d3c8e 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/Base64EncodeTestCase.java @@ -28,10 +28,10 @@ public void requireThatHashCodeAndEqualsAreImplemented() { @Test public void requireThatInputIsEncoded() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new LongFieldValue(489210573L)); + ctx.setCurrentValue(new LongFieldValue(489210573L)); new Base64EncodeExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("zcIoHQAAAAA=", ((StringFieldValue)val).getString()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java index 17388f656567..cfaeafa5f6c5 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/CatTestCase.java @@ -234,16 +234,16 @@ private static FieldValue evaluate(FieldValue valA, FieldValue valB) { private static FieldValue evaluate(DataType typeA, FieldValue valA, DataType typeB, FieldValue valB) { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("a", typeA), new Field("b", typeB))); - ctx.setOutputValue(null, "a", valA); - ctx.setOutputValue(null, "b", valB); + ctx.setFieldValue("a", valA, null); + ctx.setFieldValue("b", valB, null); new CatExpression(new InputExpression("a"), new InputExpression("b")).execute(ctx); - return ctx.getValue(); + return ctx.getCurrentValue(); } private static DataType evaluate(DataType typeA, DataType typeB) { SimpleTestAdapter adapter = new SimpleTestAdapter(new Field("a", typeA), new Field("b", typeB)); VerificationContext ctx = new VerificationContext(adapter); new CatExpression(new InputExpression("a"), new InputExpression("b")).verify(ctx); - return ctx.getValueType(); + return ctx.getCurrentType(); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java index f36ef96270cf..05f3bfb9b117 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java @@ -2,11 +2,9 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; -import com.yahoo.document.Document; import com.yahoo.document.Field; import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.StringFieldValue; -import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; @@ -31,7 +29,7 @@ public void testChoiceExecution() { adapter.setValue("foo", new StringFieldValue("foo1")); ExecutionContext context = new ExecutionContext(adapter); choice.execute(context); - assertEquals("foo1", context.getValue().getWrappedValue()); + assertEquals("foo1", context.getCurrentValue().getWrappedValue()); } { // bar only @@ -39,7 +37,7 @@ public void testChoiceExecution() { adapter.setValue("bar", new StringFieldValue("bar1")); ExecutionContext context = new ExecutionContext(adapter); choice.execute(context); - assertEquals("bar1", context.getValue().getWrappedValue()); + assertEquals("bar1", context.getCurrentValue().getWrappedValue()); } { // both foo and bar @@ -49,7 +47,7 @@ public void testChoiceExecution() { choice.verify(adapter); ExecutionContext context = new ExecutionContext(adapter); choice.execute(context); - assertEquals("foo1", context.getValue().getWrappedValue()); + assertEquals("foo1", context.getCurrentValue().getWrappedValue()); } } @@ -63,7 +61,7 @@ public void testChoiceWithConstant() throws ParseException { adapter.setValue("timestamp", new LongFieldValue(34)); ExecutionContext context = new ExecutionContext(adapter); choice.execute(context); - assertEquals(34L, context.getValue().getWrappedValue()); + assertEquals(34L, context.getCurrentValue().getWrappedValue()); } { // fallback to default @@ -71,7 +69,7 @@ public void testChoiceWithConstant() throws ParseException { choice.verify(adapter); ExecutionContext context = new ExecutionContext(adapter); choice.execute(context); - assertEquals(99999999L, context.getValue().getWrappedValue()); + assertEquals(99999999L, context.getCurrentValue().getWrappedValue()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java index c9c24c0ecf03..cb480a36bf4d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ClearStateTestCase.java @@ -38,7 +38,7 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatVerificationContextIsCleared() { MyVerification ctx = new MyVerification(); - ctx.execute(new ClearStateExpression()); + ctx.verify(new ClearStateExpression()); assertTrue(ctx.cleared); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java index 6a7e88be51a0..9bce89b62330 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/EchoTestCase.java @@ -42,7 +42,7 @@ public void requireThatValueIsEchoed() { ByteArrayOutputStream out = new ByteArrayOutputStream(); ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("69")); + ctx.setCurrentValue(new StringFieldValue("69")); new EchoExpression(new PrintStream(out)).execute(ctx); assertEquals("69" + System.getProperty("line.separator"), out.toString()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java index b338c45f7a40..bff8b81d4fd1 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExactTestCase.java @@ -29,55 +29,55 @@ public void requireThatHashCodeAndEqualsAreImplemented() { @Test public void requireThatValueIsNotChanged() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("FOO")); + ctx.setCurrentValue(new StringFieldValue("FOO")); new ExactExpression().execute(ctx); - assertEquals("FOO", String.valueOf(ctx.getValue())); + assertEquals("FOO", String.valueOf(ctx.getCurrentValue())); } @Test public void requireThatValueIsAnnotated() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("FOO")); + ctx.setCurrentValue(new StringFieldValue("FOO")); new ExactExpression().execute(ctx); - assertAnnotation(0, 3, new StringFieldValue("foo"), (StringFieldValue)ctx.getValue()); + assertAnnotation(0, 3, new StringFieldValue("foo"), (StringFieldValue)ctx.getCurrentValue()); } @Test public void requireThatThereIsNoSegmentation() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("FOO BAR")); + ctx.setCurrentValue(new StringFieldValue("FOO BAR")); new ExactExpression().execute(ctx); - assertAnnotation(0, 7, new StringFieldValue("foo bar"), (StringFieldValue)ctx.getValue()); + assertAnnotation(0, 7, new StringFieldValue("foo bar"), (StringFieldValue)ctx.getCurrentValue()); } @Test public void requireThatRedundantAnnotationValueIsIgnored() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("foo")); + ctx.setCurrentValue(new StringFieldValue("foo")); new ExactExpression().execute(ctx); - assertAnnotation(0, 3, null, (StringFieldValue)ctx.getValue()); + assertAnnotation(0, 3, null, (StringFieldValue)ctx.getCurrentValue()); } @Test public void requireThatLongStringsAreNotAnnotated() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("foo")); + ctx.setCurrentValue(new StringFieldValue("foo")); new ExactExpression(2).execute(ctx); - assertNull(((StringFieldValue)ctx.getValue()).getSpanTree(SpanTrees.LINGUISTICS)); + assertNull(((StringFieldValue)ctx.getCurrentValue()).getSpanTree(SpanTrees.LINGUISTICS)); } @Test public void requireThatEmptyStringsAreNotAnnotated() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("")); + ctx.setCurrentValue(new StringFieldValue("")); new ExactExpression().execute(ctx); - assertNull(((StringFieldValue)ctx.getValue()).getSpanTree(SpanTrees.LINGUISTICS)); + assertNull(((StringFieldValue)ctx.getCurrentValue()).getSpanTree(SpanTrees.LINGUISTICS)); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java index 6d14b34bfb84..be048ba67fcc 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExecutionContextTestCase.java @@ -4,7 +4,6 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.language.Language; -import com.yahoo.language.Linguistics; import com.yahoo.language.simple.SimpleLinguistics; import org.junit.Test; @@ -19,8 +18,8 @@ public class ExecutionContextTestCase { public void requireThatValueCanBeSet() { ExecutionContext ctx = new ExecutionContext(); FieldValue val = new StringFieldValue("foo"); - ctx.setValue(val); - assertSame(val, ctx.getValue()); + ctx.setCurrentValue(val); + assertSame(val, ctx.getCurrentValue()); } @Test @@ -52,9 +51,9 @@ public void requireThatNullLanguageThrowsException() { @Test public void requireThatClearRemovesValue() { ExecutionContext ctx = new ExecutionContext(); - ctx.setValue(new StringFieldValue("foo")); + ctx.setCurrentValue(new StringFieldValue("foo")); ctx.clear(); - assertNull(ctx.getValue()); + assertNull(ctx.getCurrentValue()); } @Test @@ -88,9 +87,9 @@ public void requireThatResolveLanguageDefaultsToEnglishWithoutValue() { @Test public void requireThatLanguageCanBeResolved() { ExecutionContext ctx = new ExecutionContext(); - ctx.setValue(new StringFieldValue("\u3072\u3089\u304c\u306a")); + ctx.setCurrentValue(new StringFieldValue("\u3072\u3089\u304c\u306a")); assertEquals(Language.JAPANESE, ctx.resolveLanguage(new SimpleLinguistics())); - ctx.setValue(new StringFieldValue("\ud55c\uae00\uacfc")); + ctx.setCurrentValue(new StringFieldValue("\ud55c\uae00\uacfc")); assertEquals(Language.KOREAN, ctx.resolveLanguage(new SimpleLinguistics())); } @@ -98,9 +97,9 @@ public void requireThatLanguageCanBeResolved() { public void requireThatExplicitLanguagePreventsDetection() { ExecutionContext ctx = new ExecutionContext(); ctx.setLanguage(Language.ARABIC); - ctx.setValue(new StringFieldValue("\u3072\u3089\u304c\u306a")); + ctx.setCurrentValue(new StringFieldValue("\u3072\u3089\u304c\u306a")); assertEquals(Language.ARABIC, ctx.resolveLanguage(new SimpleLinguistics())); - ctx.setValue(new StringFieldValue("\ud55c\uae00\uacfc")); + ctx.setCurrentValue(new StringFieldValue("\ud55c\uae00\uacfc")); assertEquals(Language.ARABIC, ctx.resolveLanguage(new SimpleLinguistics())); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java index 05c4ae33cfce..755e47d64086 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionAssert.java @@ -18,18 +18,18 @@ public static void assertVerifyCtx(VerificationContext ctx, Expression exp, Data } public static void assertVerify(DataType valueBefore, Expression exp, DataType expectedValueAfter) { - assertVerifyCtx(new VerificationContext().setValueType(valueBefore), exp, expectedValueAfter); + assertVerifyCtx(new VerificationContext().setCurrentType(valueBefore), exp, expectedValueAfter); } public static void assertVerifyThrows(DataType valueBefore, Expression exp, String expectedException) { - assertVerifyCtxThrows(new VerificationContext().setValueType(valueBefore), exp, expectedException); + assertVerifyCtxThrows(new VerificationContext().setCurrentType(valueBefore), exp, expectedException); } interface CreateExpression { Expression create(); } public static void assertVerifyThrows(DataType valueBefore, CreateExpression createExp, String expectedException) { - assertVerifyCtxThrows(new VerificationContext().setValueType(valueBefore), createExp, expectedException); + assertVerifyCtxThrows(new VerificationContext().setCurrentType(valueBefore), createExp, expectedException); } public static void assertVerifyCtxThrows(VerificationContext ctx, CreateExpression createExp, String expectedException) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java index 4d77721a3f6e..07adc4190af7 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachTestCase.java @@ -75,7 +75,7 @@ public void requireThatEachTokenIsExecutedSeparately() { Array arr = new Array<>(DataType.getArray(DataType.STRING)); arr.add(new StringFieldValue("6")); arr.add(new StringFieldValue("9")); - ctx.setValue(arr); + ctx.setCurrentValue(arr); MyCollector exp = new MyCollector(); new ForEachExpression(exp).execute(ctx); @@ -103,10 +103,10 @@ public void requireThatArrayCanBeConverted() { Array before = new Array<>(DataType.getArray(DataType.STRING)); before.add(new StringFieldValue("6")); before.add(new StringFieldValue("9")); - ctx.setValue(before); + ctx.setCurrentValue(before); new ForEachExpression(new ToIntegerExpression()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof Array); Array after = (Array)val; @@ -118,11 +118,11 @@ public void requireThatArrayCanBeConverted() { @Test public void requireThatEmptyArrayCanBeConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new Array(DataType.getArray(DataType.STRING))); + ctx.setCurrentValue(new Array(DataType.getArray(DataType.STRING))); new ForEachExpression(new ToIntegerExpression()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof Array); assertEquals(DataType.INT, ((Array)val).getDataType().getNestedType()); assertTrue(((Array)val).isEmpty()); @@ -143,12 +143,12 @@ public void requireThatArrayWithNullCanBeConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); Array arr = new Array<>(DataType.getArray(DataType.STRING)); arr.add(new StringFieldValue("foo")); - ctx.setValue(arr); + ctx.setCurrentValue(arr); new ForEachExpression(SimpleExpression.newConversion(DataType.STRING, DataType.INT) .setExecuteValue(null)).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof Array); assertEquals(DataType.INT, ((Array)val).getDataType().getNestedType()); assertTrue(((Array)val).isEmpty()); @@ -160,10 +160,10 @@ public void requireThatWsetCanBeConverted() { WeightedSet before = new WeightedSet<>(DataType.getWeightedSet(DataType.STRING)); before.put(new StringFieldValue("6"), 9); before.put(new StringFieldValue("9"), 6); - ctx.setValue(before); + ctx.setCurrentValue(before); new ForEachExpression(new ToIntegerExpression()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof WeightedSet); WeightedSet after = (WeightedSet)val; @@ -175,11 +175,11 @@ public void requireThatWsetCanBeConverted() { @Test public void requireThatEmptyWsetCanBeConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new WeightedSet(DataType.getWeightedSet(DataType.STRING))); + ctx.setCurrentValue(new WeightedSet(DataType.getWeightedSet(DataType.STRING))); new ForEachExpression(new ToIntegerExpression()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof WeightedSet); assertEquals(DataType.INT, ((WeightedSet)val).getDataType().getNestedType()); assertTrue(((WeightedSet)val).isEmpty()); @@ -193,11 +193,11 @@ public void requireThatStructContentCanBeConverted() { struct.setFieldValue("my_str", new StringFieldValue(" foo ")); ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(struct); + ctx.setCurrentValue(struct); new ForEachExpression(new TrimExpression()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof Struct); assertEquals(type, val.getDataType()); assertEquals(new StringFieldValue("foo"), ((Struct)val).getFieldValue("my_str")); @@ -209,7 +209,7 @@ public void requireThatIncompatibleStructFieldsFailToValidate() { type.addField(new Field("my_int", DataType.INT)); VerificationContext ctx = new VerificationContext(new SimpleTestAdapter()); - ctx.setValueType(type); + ctx.setCurrentType(type); try { new ForEachExpression(new ToArrayExpression()).verify(ctx); @@ -227,7 +227,7 @@ public void requireThatIncompatibleStructFieldsFailToExecute() { struct.setFieldValue("my_int", new IntegerFieldValue(69)); ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(struct); + ctx.setCurrentValue(struct); try { new ForEachExpression(new ToArrayExpression()).execute(ctx); @@ -247,7 +247,7 @@ private static class MyCollector extends Expression { } @Override protected void doExecute(ExecutionContext context) { - lst.add(context.getValue()); + lst.add(context.getCurrentValue()); } @Override diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java index 768f312b27c8..20fdf562ff1c 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/GetVarTestCase.java @@ -51,7 +51,7 @@ public void requireThatSymbolIsRead() { ctx.setVariable("in", new IntegerFieldValue(69)); new GetVarExpression("in").execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof IntegerFieldValue); assertEquals(69, ((IntegerFieldValue)val).getInteger()); } @@ -59,12 +59,12 @@ public void requireThatSymbolIsRead() { @Test public void requireThatGetVarCanBeUsedToImplementSum() throws ParseException { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setOutputValue(null, "in", new StringFieldValue("0;1;2;3;4;5;6;7;8;9")); + ctx.setFieldValue("in", new StringFieldValue("0;1;2;3;4;5;6;7;8;9"), null); ScriptExpression.fromString("{ 0 | set_var tmp; " + " input in | split ';' | for_each { to_int + get_var tmp | set_var tmp };" + " get_var tmp | attribute out; }").execute(ctx); - FieldValue val = ctx.getInputValue("out"); + FieldValue val = ctx.getFieldValue("out"); assertTrue(val instanceof IntegerFieldValue); assertEquals(45, ((IntegerFieldValue)val).getInteger()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java index eed3185d7d83..9cb8977afd7b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexDecodeTestCase.java @@ -36,10 +36,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireInputIsDecoded() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("1d28c2cd")); + ctx.setCurrentValue(new StringFieldValue("1d28c2cd")); new HexDecodeExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof LongFieldValue); assertEquals(489210573L, ((LongFieldValue)val).getLong()); } @@ -47,10 +47,10 @@ public void requireInputIsDecoded() { @Test public void requireThatLargeInputIsDecoded() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("ff7a3c87fd74abff")); + ctx.setCurrentValue(new StringFieldValue("ff7a3c87fd74abff")); new HexDecodeExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof LongFieldValue); assertEquals(-37651092108694529L, ((LongFieldValue)val).getLong()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java index fb9e4c9b3793..670172319101 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HexEncodeTestCase.java @@ -36,10 +36,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatInputIsEncoded() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new LongFieldValue(489210573L)); + ctx.setCurrentValue(new LongFieldValue(489210573L)); new HexEncodeExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("1d28c2cd", ((StringFieldValue)val).getString()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java index d331adffd55f..82d8d6e5393a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/HostNameTestCase.java @@ -38,7 +38,7 @@ public void requireThatHostnameIsSet() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new HostNameExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals(HostNameExpression.normalizeHostName(getDefaults().vespaHostname()), ((StringFieldValue)val).getString()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java index c45b16beadb7..932652e65717 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenTestCase.java @@ -146,7 +146,7 @@ public void requireThatElseExpIsOptional() { Comparator.GT, new ConstantExpression(new IntegerFieldValue(9)), new ConstantExpression(new StringFieldValue("69"))); - FieldValue val = ctx.setValue(new IntegerFieldValue(96)).execute(exp).getValue(); + FieldValue val = ctx.setCurrentValue(new IntegerFieldValue(96)).execute(exp).getCurrentValue(); assertTrue(val instanceof IntegerFieldValue); assertEquals(96, ((IntegerFieldValue)val).getInteger()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java index c3739be747f6..4c2fd545b6c6 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/InputTestCase.java @@ -47,10 +47,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatFieldIsRead() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("in", DataType.STRING))); - ctx.setOutputValue(null, "in", new StringFieldValue("69")); + ctx.setFieldValue("in", new StringFieldValue("69"), null); new InputExpression("in").execute(ctx); - assertEquals(new StringFieldValue("69"), ctx.getValue()); + assertEquals(new StringFieldValue("69"), ctx.getCurrentValue()); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java index fd28a3aebef8..7c997ecebf26 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/JoinTestCase.java @@ -46,10 +46,10 @@ public void requireThatValueIsJoined() { Array arr = new Array<>(DataType.getArray(DataType.STRING)); arr.add(new StringFieldValue("6")); arr.add(new StringFieldValue("9")); - ctx.setValue(arr); + ctx.setCurrentValue(arr); new JoinExpression(";").execute(ctx); - assertEquals(new StringFieldValue("6;9"), ctx.getValue()); + assertEquals(new StringFieldValue("6;9"), ctx.getCurrentValue()); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpressionTestCase.java index 514f79d88f22..ab45136f4d71 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LiteralBoolExpressionTestCase.java @@ -46,7 +46,7 @@ public void testToString() { public void requireThatTrueBecomesTrue() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); context.execute(new LiteralBoolExpression(true)); - FieldValue value = context.getValue(); + FieldValue value = context.getCurrentValue(); assertTrue(value instanceof BoolFieldValue); assertTrue(((BoolFieldValue)value).getBoolean()); } @@ -55,7 +55,7 @@ public void requireThatTrueBecomesTrue() { public void requireThatFalseBecomesFalse() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); context.execute(new LiteralBoolExpression(false)); - FieldValue value = context.getValue(); + FieldValue value = context.getCurrentValue(); assertTrue(value instanceof BoolFieldValue); assertFalse(((BoolFieldValue)value).getBoolean()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java index 89f782ed5258..49085ec7da18 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/LowerCaseTestCase.java @@ -35,10 +35,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatStringIsLowerCased() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("FOO")); + ctx.setCurrentValue(new StringFieldValue("FOO")); new LowerCaseExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("foo", ((StringFieldValue)val).getString()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java index ebb39257a84f..88c8b1f8bd36 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/MathResolverTestCase.java @@ -97,7 +97,7 @@ private static Expression newInteger(int val) { } private static int evaluate(Expression exp) { - FieldValue val = new ExecutionContext(new SimpleTestAdapter()).execute(exp).getValue(); + FieldValue val = new ExecutionContext(new SimpleTestAdapter()).execute(exp).getCurrentValue(); assertTrue(val instanceof IntegerFieldValue); return ((IntegerFieldValue)val).getInteger(); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java index b4e266ab3ebb..a3ce6a204b24 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java @@ -53,10 +53,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void testNGrams() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); - context.setValue(new StringFieldValue("en gul Bille sang... ")); + context.setCurrentValue(new StringFieldValue("en gul Bille sang... ")); new NGramExpression(new SimpleLinguistics(), 3).execute(context); - StringFieldValue value = (StringFieldValue)context.getValue(); + StringFieldValue value = (StringFieldValue)context.getCurrentValue(); assertEquals("Grams are pure annotations - field value is unchanged", "en gul Bille sang... ", value.getString()); SpanTree gramTree = value.getSpanTree(SpanTrees.LINGUISTICS); @@ -80,15 +80,15 @@ public void testNGrams() { @Test public void requireThatExecuteCanBeCalledMultipleTimes() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); - context.setValue(new StringFieldValue("some random text string")); + context.setCurrentValue(new StringFieldValue("some random text string")); NGramExpression expression = new NGramExpression(new SimpleLinguistics(), 3); expression.execute(context); - SpanTree firstTree = ((StringFieldValue)context.getValue()).getSpanTree(SpanTrees.LINGUISTICS); + SpanTree firstTree = ((StringFieldValue)context.getCurrentValue()).getSpanTree(SpanTrees.LINGUISTICS); assertNotNull(firstTree); expression.execute(context); - SpanTree secondTree = ((StringFieldValue)context.getValue()).getSpanTree(SpanTrees.LINGUISTICS); + SpanTree secondTree = ((StringFieldValue)context.getCurrentValue()).getSpanTree(SpanTrees.LINGUISTICS); // The span tree instance should be the same. assertEquals(firstTree, secondTree); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java index 441b9f57b39c..2c8e2c029e2a 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NormalizeTestCase.java @@ -51,10 +51,10 @@ public void requireThatExpressionCanBeVerified() { public void requireThatInputIsNormalized() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); ctx.setLanguage(Language.ENGLISH); - ctx.setValue(new StringFieldValue("b\u00e9yonc\u00e8")); + ctx.setCurrentValue(new StringFieldValue("b\u00e9yonc\u00e8")); new NormalizeExpression(new SimpleLinguistics()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("beyonce", ((StringFieldValue)val).getString()); } @@ -93,11 +93,11 @@ public Transformer getTransformer() { public void requireThatBadNormalizeRetries() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); ctx.setLanguage(Language.ENGLISH); - ctx.setValue(new StringFieldValue("bad norm")); + ctx.setCurrentValue(new StringFieldValue("bad norm")); var linguistics = new MyMockLinguistics(); assertTrue(getFirst(linguistics.getTransformer())); new NormalizeExpression(linguistics).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("bad/norm", ((StringFieldValue)val).getString()); assertFalse(getFirst(linguistics.getTransformer())); @@ -108,11 +108,11 @@ public void requireThatEmptyIsNop() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); ctx.setLanguage(Language.ENGLISH); var orig = new StringFieldValue(""); - ctx.setValue(orig); + ctx.setCurrentValue(orig); var linguistics = new MyMockLinguistics(); assertTrue(getFirst(linguistics.getTransformer())); new NormalizeExpression(linguistics).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val == orig); assertTrue(getFirst(linguistics.getTransformer())); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java index 45ac36d9100a..4e9d51744b14 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NowTestCase.java @@ -45,7 +45,7 @@ public void requireThatCurrentTimeIsSet() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new NowExpression(new MyTimer()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof LongFieldValue); assertEquals(69L, ((LongFieldValue)val).getLong()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java index 73a906ad3385..750e3114b624 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OptimizePredicateTestCase.java @@ -26,7 +26,7 @@ public void requireThatOptimizerIsCalledWithCloneOfInput() { final Predicate predicateB = Mockito.mock(Predicate.class); final PredicateFieldValue input = new PredicateFieldValue(predicateA); ExecutionContext ctx = new ExecutionContext() - .setValue(input) + .setCurrentValue(input) .setVariable("arity", new IntegerFieldValue(10)); FieldValue output = new OptimizePredicateExpression( (predicate, options) -> { @@ -44,7 +44,7 @@ public void requireThatPredicateOptionsAreSet() { final Predicate predicate = Mockito.mock(Predicate.class); final PredicateFieldValue input = new PredicateFieldValue(predicate); ExecutionContext ctx = new ExecutionContext() - .setValue(input) + .setCurrentValue(input) .setVariable("arity", new IntegerFieldValue(10)); new OptimizePredicateExpression((predicate1, options) -> { assertEquals(10, options.getArity()); @@ -80,7 +80,7 @@ public void requireThatExpressionCanBeVerified() { assertVerifyThrows(DataType.INT, exp, "Expected predicate input, got int"); assertVerifyThrows(DataType.PREDICATE, exp, "Variable 'arity' must be set"); - VerificationContext context = new VerificationContext().setValueType(DataType.PREDICATE); + VerificationContext context = new VerificationContext().setCurrentType(DataType.PREDICATE); context.setVariable("arity", DataType.STRING); assertVerifyCtxThrows(context, exp, "Variable 'arity' must have type int"); context.setVariable("arity", DataType.INT); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java index 2f41ffcfc6fc..23cbc0ab9760 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/OutputAssert.java @@ -16,10 +16,10 @@ class OutputAssert { public static void assertExecute(OutputExpression exp) { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field(exp.getFieldName(), DataType.STRING))); - ctx.setValue(new StringFieldValue("69")); + ctx.setCurrentValue(new StringFieldValue("69")); ctx.execute(exp); - FieldValue out = ctx.getInputValue(exp.getFieldName()); + FieldValue out = ctx.getFieldValue(exp.getFieldName()); assertTrue(out instanceof StringFieldValue); assertEquals("69", ((StringFieldValue)out).getString()); } @@ -32,14 +32,14 @@ public static void assertVerify(OutputExpression exp) { } public static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) { - assertEquals(value, new VerificationContext(adapter).setValueType(value).execute(exp).getValueType()); + assertEquals(value, new VerificationContext(adapter).setCurrentType(value).verify(exp).getCurrentType()); } public static void assertVerifyThrows(FieldTypeAdapter adapter, DataType value, Expression exp, String expectedException) { try { - new VerificationContext(adapter).setValueType(value).execute(exp); + new VerificationContext(adapter).setCurrentType(value).verify(exp); fail(); } catch (VerificationException e) { assertEquals(expectedException, e.getMessage()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java index 9742dcadb3a7..54aaba282742 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisTestCase.java @@ -44,10 +44,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatNestedExpressionIsRun() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter(new Field("in", DataType.STRING))); - ctx.setOutputValue(null, "in", new StringFieldValue("69")); + ctx.setFieldValue("in", new StringFieldValue("69"), null); new ParenthesisExpression(new InputExpression("in")).execute(ctx); - assertTrue(ctx.getValue() instanceof StringFieldValue); - assertEquals("69", ((StringFieldValue)ctx.getValue()).getString()); + assertTrue(ctx.getCurrentValue() instanceof StringFieldValue); + assertEquals("69", ((StringFieldValue)ctx.getCurrentValue()).getString()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java index 20c2521d94e7..ada30f7d5ae7 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/RandomTestCase.java @@ -45,7 +45,7 @@ public void requireThatRandomValueIsSet() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new RandomExpression(69).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof IntegerFieldValue); assertTrue(((IntegerFieldValue)val).getInteger() < 69); } @@ -54,10 +54,10 @@ public void requireThatRandomValueIsSet() { @Test public void requireThatInputValueIsParsedAsMaxIfNoneIsConfigured() { for (int i = 0; i < 666; ++i) { - ExecutionContext ctx = new ExecutionContext().setValue(new IntegerFieldValue(69)); + ExecutionContext ctx = new ExecutionContext().setCurrentValue(new IntegerFieldValue(69)); new RandomExpression().execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof IntegerFieldValue); assertTrue(((IntegerFieldValue)val).getInteger() < 69); } @@ -72,7 +72,7 @@ public void requireThatIllegalInputThrowsNumberFormatException() { } try { - new RandomExpression().execute(new ExecutionContext().setValue(new StringFieldValue("foo"))); + new RandomExpression().execute(new ExecutionContext().setCurrentValue(new StringFieldValue("foo"))); fail(); } catch (NumberFormatException e) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java index c6fb7c1db8c4..a6fdf9db90e8 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptTestCase.java @@ -1,8 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage.expressions; +import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.Field; +import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; @@ -143,6 +145,24 @@ public void requireThatVariablesReplaceOthersOutsideScript() { assertEquals(new IntegerFieldValue(9), adapter.getInputValue("out")); } + @Test + // input debug_src | lowercase | summary | index | split ";" | for_each { + public void testSplitAndForEach() { + var adapter = new SimpleTestAdapter(); + adapter.createField(new Field("myString", DataType.STRING)); + adapter.createField(new Field("myArray", new ArrayDataType(DataType.STRING))); + adapter.setValue("myString", new StringFieldValue("my;test;values")); + var statement = + newStatement(new InputExpression("myString"), + new LowerCaseExpression(), + new SplitExpression(";"), + new ForEachExpression(new StatementExpression(new SubstringExpression(0, 1))), + new AttributeExpression("myArray")); + statement.verify(adapter); + statement.execute(adapter); + assertEquals("[m, t, v]", adapter.values.get("myArray").toString()); + } + private static ScriptExpression newScript(StatementExpression... args) { return new ScriptExpression(args); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java index 23392c001102..963a84b6990b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputTestCase.java @@ -73,7 +73,7 @@ public void requireThatSelectedExpressionIsRun() { } private static void assertVerify(FieldTypeAdapter adapter, DataType value, Expression exp) { - assertEquals(value, exp.verify(new VerificationContext(adapter).setValueType(value))); + assertEquals(value, exp.verify(new VerificationContext(adapter).setCurrentType(value))); } private static void assertVerifyThrows(FieldTypeAdapter adapter, Expression exp, String expectedException) { @@ -98,7 +98,7 @@ private static void assertSelect(List inputField, List available ExecutionContext ctx = new ExecutionContext(adapter); for (String fieldName : availableFields) { adapter.createField(new Field(fieldName, DataType.STRING)); - ctx.setOutputValue(null, fieldName, new StringFieldValue(fieldName)); + ctx.setFieldValue(fieldName, new StringFieldValue(fieldName), null); } List> cases = new LinkedList<>(); for (String fieldName : inputField) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java index 2efa67c54a44..fae3b0c25d4e 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetLanguageTestCase.java @@ -36,7 +36,7 @@ public void requireThatExpressionCanBeVerified() { @Test public void testsettingEnglish() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("en")); + ctx.setCurrentValue(new StringFieldValue("en")); new SetLanguageExpression().execute(ctx); assertEquals(Language.ENGLISH, ctx.getLanguage()); } @@ -44,7 +44,7 @@ public void testsettingEnglish() { @Test public void testSettingUnknown() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("unknown")); + ctx.setCurrentValue(new StringFieldValue("unknown")); new SetLanguageExpression().execute(ctx); assertEquals(Language.UNKNOWN, ctx.getLanguage()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java index b7a9e795888f..7ef79efa62cf 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetValueTestCase.java @@ -55,7 +55,7 @@ public void requireThatNullValueThrowsException() { public void requireThatValueIsSet() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new ConstantExpression(new StringFieldValue("69")).execute(ctx); - assertEquals(new StringFieldValue("69"), ctx.getValue()); + assertEquals(new StringFieldValue("69"), ctx.getCurrentValue()); } @Test diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java index 28c9ba65dad3..4364865ba24e 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SetVarTestCase.java @@ -39,7 +39,7 @@ public void requireThatExpressionCanBeVerified() { assertVerifyThrows(null, exp, "Expected any input, but no input is specified"); try { - new VerificationContext().setVariable("foo", DataType.INT).setValueType(DataType.STRING).execute(exp); + new VerificationContext().setVariable("foo", DataType.INT).setCurrentType(DataType.STRING).verify(exp); fail(); } catch (VerificationException e) { assertEquals("Attempting to assign conflicting types to variable 'foo', int vs string", e.getMessage()); @@ -49,7 +49,7 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatSymbolIsWritten() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new IntegerFieldValue(69)); + ctx.setCurrentValue(new IntegerFieldValue(69)); new SetVarExpression("out").execute(ctx); FieldValue val = ctx.getVariable("out"); @@ -60,11 +60,11 @@ public void requireThatSymbolIsWritten() { @Test public void requireThatVariableTypeCanNotChange() { VerificationContext ctx = new VerificationContext(new SimpleTestAdapter()); - ctx.setValueType(DataType.INT); + ctx.setCurrentType(DataType.INT); new SetVarExpression("out").verify(ctx); try { - ctx.setValueType(DataType.STRING); + ctx.setCurrentType(DataType.STRING); new SetVarExpression("out").verify(ctx); fail(); } catch (VerificationException e) { diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java index db9aa1898f53..e505b4e51931 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SimpleExpression.java @@ -42,14 +42,14 @@ public SimpleExpression setCreatedOutput(DataType createdOutput) { @Override protected void doExecute(ExecutionContext context) { if (hasExecuteValue) { - context.setValue(executeValue); + context.setCurrentValue(executeValue); } } @Override protected void doVerify(VerificationContext context) { if (hasVerifyValue) { - context.setValueType(verifyValue); + context.setCurrentType(verifyValue); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java index 76bafec93890..e7df02f9be25 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SplitTestCase.java @@ -44,10 +44,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsSplit() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("6;9")); + ctx.setCurrentValue(new StringFieldValue("6;9")); new SplitExpression(";").execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val.getDataType().equals(DataType.getArray(DataType.STRING))); assertTrue(val instanceof Array); @@ -61,16 +61,16 @@ public void requireThatNullInputProducesNullOutput() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); new SplitExpression(";").execute(ctx); - assertNull(ctx.getValue()); + assertNull(ctx.getCurrentValue()); } @Test public void requireThatEmptyInputProducesEmptyArray() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("")); + ctx.setCurrentValue(new StringFieldValue("")); new SplitExpression(";").execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val.getDataType().equals(DataType.getArray(DataType.STRING))); assertTrue(val instanceof Array); assertEquals(0, ((Array)val).size()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java index 3612e3bd9fca..f134191c4a3d 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/StatementTestCase.java @@ -108,7 +108,7 @@ public void requireThatStatementIsExecuted() { StatementExpression statement = newStatement(new ConstantExpression(new IntegerFieldValue(69))); newStatement(statement).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof IntegerFieldValue); assertEquals(69, ((IntegerFieldValue)val).getInteger()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java index aeb1947e97ca..f05e3117ba70 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SubstringTestCase.java @@ -67,10 +67,10 @@ private static void assertIndexOutOfBounds(int from, int to) { @Test public void requireThatStringIsSliced() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("666999")); + ctx.setCurrentValue(new StringFieldValue("666999")); new SubstringExpression(2, 4).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("69", ((StringFieldValue)val).getString()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java index 376677b6f57a..dddc8762b553 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchTestCase.java @@ -126,6 +126,6 @@ public void requireThatDefaultExpressionIsExecuted() { } private static void assertEvaluate(FieldValue input, Expression exp, FieldValue expectedOutVar) { - assertEquals(expectedOutVar, new ExecutionContext().setValue(input).execute(exp).getVariable("out")); + assertEquals(expectedOutVar, new ExecutionContext().setCurrentValue(input).execute(exp).getVariable("out")); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java index a04dfc82a595..21c399b8c0cf 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ThisTestCase.java @@ -35,9 +35,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsPreserved() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("69")); + ctx.setCurrentValue(new StringFieldValue("69")); new ThisExpression().execute(ctx); - assertEquals(new StringFieldValue("69"), ctx.getValue()); + assertEquals(new StringFieldValue("69"), ctx.getCurrentValue()); } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java index 6f024e3547d5..d1ac4152ee36 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToArrayTestCase.java @@ -39,9 +39,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("69")).execute(new ToArrayExpression()); + ctx.setCurrentValue(new StringFieldValue("69")).execute(new ToArrayExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertEquals(Array.class, val.getClass()); Array arr = (Array)val; diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java index 1452dc447b2a..6ac0a27c11be 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToBoolTestCase.java @@ -5,7 +5,6 @@ import com.yahoo.document.datatypes.BoolFieldValue; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.IntegerFieldValue; -import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; import org.junit.Test; @@ -40,8 +39,8 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatNonEmptyStringBecomesTrue() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); - context.setValue(new StringFieldValue("false")).execute(new ToBoolExpression()); - FieldValue value = context.getValue(); + context.setCurrentValue(new StringFieldValue("false")).execute(new ToBoolExpression()); + FieldValue value = context.getCurrentValue(); assertTrue(value instanceof BoolFieldValue); assertTrue(((BoolFieldValue)value).getBoolean()); } @@ -49,8 +48,8 @@ public void requireThatNonEmptyStringBecomesTrue() { @Test public void requireThatEmptyStringBecomesFalse() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); - context.setValue(new StringFieldValue("")).execute(new ToBoolExpression()); - FieldValue value = context.getValue(); + context.setCurrentValue(new StringFieldValue("")).execute(new ToBoolExpression()); + FieldValue value = context.getCurrentValue(); assertTrue(value instanceof BoolFieldValue); assertFalse(((BoolFieldValue)value).getBoolean()); } @@ -58,8 +57,8 @@ public void requireThatEmptyStringBecomesFalse() { @Test public void requireThatNonZeroBecomesTrue() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); - context.setValue(new IntegerFieldValue(37)).execute(new ToBoolExpression()); - FieldValue value = context.getValue(); + context.setCurrentValue(new IntegerFieldValue(37)).execute(new ToBoolExpression()); + FieldValue value = context.getCurrentValue(); assertTrue(value instanceof BoolFieldValue); assertTrue(((BoolFieldValue)value).getBoolean()); } @@ -67,8 +66,8 @@ public void requireThatNonZeroBecomesTrue() { @Test public void requireThatZeroBecomesFalse() { ExecutionContext context = new ExecutionContext(new SimpleTestAdapter()); - context.setValue(new IntegerFieldValue(0)).execute(new ToBoolExpression()); - FieldValue value = context.getValue(); + context.setCurrentValue(new IntegerFieldValue(0)).execute(new ToBoolExpression()); + FieldValue value = context.getCurrentValue(); assertTrue(value instanceof BoolFieldValue); assertFalse(((BoolFieldValue)value).getBoolean()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java index f536c37d12e3..43f53c45e09c 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToByteTestCase.java @@ -36,9 +36,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("69")).execute(new ToByteExpression()); + ctx.setCurrentValue(new StringFieldValue("69")).execute(new ToByteExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof ByteFieldValue); assertEquals(69, ((ByteFieldValue)val).getByte()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java index 3a3576f8524a..60d3f83c77a1 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToDoubleTestCase.java @@ -36,9 +36,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("6.9")).execute(new ToDoubleExpression()); + ctx.setCurrentValue(new StringFieldValue("6.9")).execute(new ToDoubleExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof DoubleFieldValue); assertEquals(6.9, ((DoubleFieldValue)val).getDouble(), 1e-6); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java index f07f9304ba95..0a98ef845996 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToEpochSecondExpressionTestCase.java @@ -34,8 +34,8 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConvertedWithMs() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("2023-12-24T17:00:43.000Z")).execute(new ToEpochSecondExpression()); - FieldValue val = ctx.getValue(); + ctx.setCurrentValue(new StringFieldValue("2023-12-24T17:00:43.000Z")).execute(new ToEpochSecondExpression()); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof LongFieldValue); assertEquals(1703437243L, ((LongFieldValue)val).getLong()); } @@ -43,8 +43,8 @@ public void requireThatValueIsConvertedWithMs() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("2023-12-24T17:00:43Z")).execute(new ToEpochSecondExpression()); - FieldValue val = ctx.getValue(); + ctx.setCurrentValue(new StringFieldValue("2023-12-24T17:00:43Z")).execute(new ToEpochSecondExpression()); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof LongFieldValue); assertEquals(1703437243L, ((LongFieldValue)val).getLong()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java index bba48158e928..32677aea11f5 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToFloatTestCase.java @@ -36,9 +36,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("6.9f")).execute(new ToFloatExpression()); + ctx.setCurrentValue(new StringFieldValue("6.9f")).execute(new ToFloatExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof FloatFieldValue); assertEquals(6.9f, ((FloatFieldValue)val).getFloat(), 1e-6); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java index d7f3f9093630..6812848e0100 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToIntegerTestCase.java @@ -36,9 +36,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("69")).execute(new ToIntegerExpression()); + ctx.setCurrentValue(new StringFieldValue("69")).execute(new ToIntegerExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof IntegerFieldValue); assertEquals(69, ((IntegerFieldValue)val).getInteger()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java index 03de90e70dc0..d88d1c6691d7 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToLongTestCase.java @@ -36,9 +36,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("69")).execute(new ToLongExpression()); + ctx.setCurrentValue(new StringFieldValue("69")).execute(new ToLongExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof LongFieldValue); assertEquals(69L, ((LongFieldValue)val).getLong()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java index 03cb6aa37c49..bd7fe2fe78bc 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToPositionTestCase.java @@ -38,9 +38,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatPositionIsParsed() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("6;9")).execute(new ToPositionExpression()); + ctx.setCurrentValue(new StringFieldValue("6;9")).execute(new ToPositionExpression()); - FieldValue out = ctx.getValue(); + FieldValue out = ctx.getCurrentValue(); assertTrue(out instanceof StructuredFieldValue); assertEquals(PositionDataType.INSTANCE, out.getDataType()); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java index 5e410d079f67..70ef2302e099 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToStringTestCase.java @@ -36,9 +36,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsConverted() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new IntegerFieldValue(69)).execute(new ToStringExpression()); + ctx.setCurrentValue(new IntegerFieldValue(69)).execute(new ToStringExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("69", ((StringFieldValue)val).getString()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java index 7df0667cc2db..0c86fc462209 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ToWsetTestCase.java @@ -65,9 +65,9 @@ private static void assertVerify(boolean createIfNonExistent, boolean removeIfZe private static void assertConvert(boolean createIfNonExistent, boolean removeIfZero) { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("69")).execute(new ToWsetExpression(createIfNonExistent, removeIfZero)); + ctx.setCurrentValue(new StringFieldValue("69")).execute(new ToWsetExpression(createIfNonExistent, removeIfZero)); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertEquals(WeightedSet.class, val.getClass()); WeightedSet wset = (WeightedSet)val; diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java index 7ed3ab410a32..9c0b17dd1220 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TokenizeTestCase.java @@ -55,10 +55,10 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatValueIsAnnotated() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("foo")); + ctx.setCurrentValue(new StringFieldValue("foo")); new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig()).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertNotNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS)); } @@ -66,10 +66,10 @@ public void requireThatValueIsAnnotated() { @Test public void requireThatLongWordIsDropped() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue("foo")); + ctx.setCurrentValue(new StringFieldValue("foo")); new TokenizeExpression(new SimpleLinguistics(), new AnnotatorConfig().setMaxTokenLength(2)).execute(ctx); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertNull(((StringFieldValue)val).getSpanTree(SpanTrees.LINGUISTICS)); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java index 8584bae87084..c2f9d4b126f4 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/TrimTestCase.java @@ -35,9 +35,9 @@ public void requireThatExpressionCanBeVerified() { @Test public void requireThatStringIsTrimmed() { ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter()); - ctx.setValue(new StringFieldValue(" 69 ")).execute(new TrimExpression()); + ctx.setCurrentValue(new StringFieldValue(" 69 ")).execute(new TrimExpression()); - FieldValue val = ctx.getValue(); + FieldValue val = ctx.getCurrentValue(); assertTrue(val instanceof StringFieldValue); assertEquals("69", ((StringFieldValue)val).getString()); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java index 98d74dc2a6a4..a24ba1343bc0 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContextTestCase.java @@ -16,8 +16,8 @@ public class VerificationContextTestCase { public void requireThatValueCanBeSet() { VerificationContext ctx = new VerificationContext(); DataType val = DataType.STRING; - ctx.setValueType(val); - assertSame(val, ctx.getValueType()); + ctx.setCurrentType(val); + assertSame(val, ctx.getCurrentType()); } @Test @@ -31,9 +31,9 @@ public void requireThatVariablesCanBeSet() { @Test public void requireThatClearRemovesValue() { VerificationContext ctx = new VerificationContext(); - ctx.setValueType(DataType.STRING); + ctx.setCurrentType(DataType.STRING); ctx.clear(); - assertNull(ctx.getValueType()); + assertNull(ctx.getCurrentType()); } @Test diff --git a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc index a398b9d9199f..9237ff5cd143 100644 --- a/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc +++ b/integration/schema-language-server/language-server/src/main/ccc/indexinglanguage/IndexingParser.ccc @@ -205,6 +205,7 @@ TOKEN : | | | + | | | | @@ -355,6 +356,7 @@ Expression value() : val = normalizeExp() | val = nowExp() | val = optimizePredicateExp() | + val = packBitsExp() | val = passthroughExp() | val = randomExp() | val = script() | @@ -612,6 +614,12 @@ Expression optimizePredicateExp() : { } { return new OptimizePredicateExpression(); } ; +Expression packBitsExp() : { } + + ( ) + { return new PackBitsExpression(); } +; + Expression passthroughExp() : { String val = defaultFieldName; diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index e400a336ad00..b5308884f40d 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1390,6 +1390,7 @@ "public abstract java.util.Optional size()", "public abstract com.yahoo.tensor.TensorType$Dimension$Type type()", "public abstract com.yahoo.tensor.TensorType$Dimension withName(java.lang.String)", + "public com.yahoo.tensor.TensorType$Dimension withSize(long)", "public boolean isIndexed()", "public boolean isMapped()", "public abstract java.lang.String toString()", @@ -1531,7 +1532,8 @@ ], "methods" : [ "public void ()", - "public static varargs com.yahoo.tensor.Tensor toSparse(com.yahoo.tensor.Tensor, java.lang.String[])" + "public static varargs com.yahoo.tensor.Tensor toSparse(com.yahoo.tensor.Tensor, java.lang.String[])", + "public static com.yahoo.tensor.Tensor packBits(com.yahoo.tensor.Tensor)" ], "fields" : [ ] }, diff --git a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java index fc0473c635a4..5b800aa58eeb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/IndexedTensor.java @@ -97,6 +97,7 @@ public double get(DirectIndexedAddress address) { return get(address.getDirectIndex()); } public DirectIndexedAddress directAddress() { return DirectIndexedAddress.of(dimensionSizes); } + /** * Returns the value at the given indexes as a float * diff --git a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java index 65c6677e7e36..386f24bfe623 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/MixedTensor.java @@ -344,7 +344,7 @@ public Tensor.Builder block(TensorAddress sparsePart, double[] values) { @Override public MixedTensor build() { - //TODO This can be solved more efficiently with a single map. + // TODO: This can be solved more efficiently with a single map. Set> entrySet = denseSubspaceMap.entrySet(); for (Map.Entry entry : entrySet) { TensorAddress sparsePart = entry.getKey(); @@ -356,7 +356,7 @@ public MixedTensor build() { } public static BoundBuilder of(TensorType type) { - //TODO Wire in expected map size to avoid expensive resize + // TODO: Wire in expected map size to avoid expensive resize return new BoundBuilder(type, INITIAL_HASH_CAPACITY); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index d225d6f46412..15c20e1a883f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -96,7 +96,7 @@ default int sizeAsInt() { /** * Returns the cell of this in some undefined order. - * A cell instances is only valid until next() is called. + * A cell instance is only valid until next() is called. * Call detach() on the cell to obtain a long-lived instance. */ Iterator cellIterator(); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index 0421f6a6a9da..c96a8decd161 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -111,15 +111,15 @@ public TensorAddress partialCopy(int[] indexMap) { return TensorAddressAny.ofUnsafe(labels); } - /** Creates a complete address by taking the mapped dimmensions from this and the indexed from the indexedPart */ - public TensorAddress fullAddressOf(List dimensions, int[] densePart) { + /** Creates a complete address by taking the mapped dimensions of this and adding the indexed from the indexedPart */ + public TensorAddress fullAddressOf(List dimensions, int[] indexedPart) { long[] labels = new long[dimensions.size()]; int mappedIndex = 0; int indexedIndex = 0; for (int i = 0; i < labels.length; i++) { TensorType.Dimension d = dimensions.get(i); if (d.isIndexed()) { - labels[i] = densePart[indexedIndex]; + labels[i] = indexedPart[indexedIndex]; indexedIndex++; } else { labels[i] = numericLabel(mappedIndex); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index a90dab6d185e..37b8d8d60018 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -373,6 +373,11 @@ private Dimension(String name) { /** Returns a copy of this with the name set to the given name */ public abstract Dimension withName(String name); + /** Returns a copy of this with the size set to the given value */ + public Dimension withSize(long size) { + return IndexedBoundDimension.indexed(name, size); + } + /** Returns true if this is an indexed bound or unbound type */ public boolean isIndexed() { return type() == Type.indexedBound || type() == Type.indexedUnbound; } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensors.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensors.java index 92cba5b9bd37..34743e4d0655 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensors.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensors.java @@ -46,4 +46,60 @@ public static Tensor toSparse(Tensor tensor, String ... dimensions) { return builder.build(); } + /** + * Converts any tensor containing only ones and zeroes into one where each consecutive 8 values in the + * dense dimension are packed into a single byte. As a consequence the output type of this is a tensor + * where the dense dimension is 1/8th as large. + * + * @throws IllegalArgumentException if the tensor has the wrong type or contains any other value than 0 or 1 + */ + public static Tensor packBits(Tensor tensor) { + if (tensor.type().indexedSubtype().dimensions().size() != 1) + throw new IllegalArgumentException("packBits requires a tensor with one dense dimensions, but got " + tensor.type()); + + // Create the packed type + var typeBuilder = new TensorType.Builder(TensorType.Value.INT8); + for (var d : tensor.type().dimensions()) + typeBuilder.dimension(d.size().isPresent() ? d.withSize((int) Math.ceil(d.size().get() / 8.0)) : d); + var packedType = typeBuilder.build(); + + // Pack it + Tensor.Builder builder = Tensor.Builder.of(packedType); + if (tensor instanceof IndexedTensor indexed) { + for (long i = 0; i < indexed.size(); ) { + long packedIndex = i / 8; + int packedValue = 0; + for (int j = 0; j < 8 && i < indexed.size(); j++) + packedValue = packInto(packedValue, indexed.get(i), j, i++); + builder.cell(packedValue, packedIndex); + } + } + else if (tensor instanceof MixedTensor mixed) { + for (var denseSubspace : mixed.getInternalDenseSubspaces()) { + for (int i = 0; i < denseSubspace.cells.length; ) { + var packedAddress = denseSubspace.sparseAddress.fullAddressOf(mixed.type().dimensions(), new int[]{i / 8}); + int packedValue = 0; + for (int j = 0; j < 8 && i < denseSubspace.cells.length; j++) + packedValue = packInto(packedValue, denseSubspace.cells[i], j, i++); + builder.cell(packedAddress, packedValue); + } + } + } + else { + throw new IllegalArgumentException("The argument is neither of type IndexedTensor or MixedTensor, but " + + tensor.getClass()); + } + return builder.build(); + } + + private static int packInto(int packedValue, double value, int bitPosition, long sourcePosition) { + if (value == 0.0) + return packedValue; + else if (value == 1.0) + return packedValue | ( 1 << ( 7 - bitPosition )); + else + throw new IllegalArgumentException("The tensor to be packed can only contain 0 or 1 values, " + + "but has " + value + " at position " + sourcePosition); + } + } diff --git a/vespajlib/src/test/java/com/yahoo/tensor/TensorsTestCase.java b/vespajlib/src/test/java/com/yahoo/tensor/TensorsTestCase.java index 84d337deeb70..6d146ca9c9fa 100644 --- a/vespajlib/src/test/java/com/yahoo/tensor/TensorsTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/tensor/TensorsTestCase.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; /** * @author bratseth @@ -24,9 +25,43 @@ void testToSparse() { } } + @Test + void testPackBits() { + assertPacked("tensor(x[2]):[129,14]", "tensor(x[16]):[1,0,0,0,0,0,0,1, 0,0,0,0,1,1,1,0]"); + assertPacked("tensor(x[2]):[129,14]", "tensor(x[15]):[1,0,0,0,0,0,0,1, 0,0,0,0,1,1,1]"); + assertPacked("tensor(x[1]):[128]", "tensor(x[1]):[1]"); + assertPacked("tensor(key{},x[2]):{a:[129,14], b:[12, 7]}", + "tensor(key{},x[16]):{a:[1,0,0,0,0,0,0,1, 0,0,0,0,1,1,1,0]," + + " b:[0,0,0,0,1,1,0,0, 0,0,0,0,0,1,1,1]}"); + assertPacked("tensor(key{},x[1]):{a:[160],b:[32]}", + "tensor(key{},x[3]):{a:[1,0,1],b:[0,0,1]}"); + assertPacked("tensor(key{},x[1]):{a:[128]}", "tensor(key{}, x[1]):{a:[1]}"); + + try { + Tensors.packBits(Tensor.from("tensor(x[1],y[1]):[1]")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("packBits requires a tensor with one dense dimensions, but got tensor(x[1],y[1])", + e.getMessage()); + } + try { + Tensors.packBits(Tensor.from("tensor(x[3]):[0, 1, 2]")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("The tensor to be packed can only contain 0 or 1 values, but has 2.0 at position 2", + e.getMessage()); + } + } + void assertConvertedToSparse(String inputType, String outputType, String tensorValue, String ... dimensions) { var tensor = Tensor.from(inputType + ":" + tensorValue); assertEquals(outputType + ":" + tensorValue, Tensors.toSparse(tensor, dimensions).toString(true, false)); } + void assertPacked(String expected, String input) { + assertEquals(Tensor.from(expected), Tensors.packBits(Tensor.from(input))); + } + }