Skip to content

Commit

Permalink
Merge pull request #32641 from vespa-engine/bratseth/pack_bits-take-2
Browse files Browse the repository at this point in the history
Bratseth/pack bits take 2
  • Loading branch information
bratseth authored Oct 23, 2024
2 parents ee135ff + 53ce47a commit cac7a8c
Show file tree
Hide file tree
Showing 145 changed files with 1,572 additions and 1,299 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ protected boolean shouldConvert(Expression exp) {
String fieldName = ((OutputExpression)exp).getFieldName();
if (outputs.contains(fieldName) && !prevNames.contains(fieldName)) {
throw new VerificationException(exp, "Attempting to assign conflicting values to field '" +
fieldName + "'.");
fieldName + "'");
}
outputs.add(fieldName);
prevNames.add(fieldName);
Expand Down Expand Up @@ -111,24 +111,24 @@ public DataType getInputType(Expression exp, String fieldName) {
}

@Override
public void tryOutputType(Expression exp, String fieldName, DataType valueType) {
public void tryOutputType(Expression expression, String fieldName, DataType valueType) {
String fieldDesc;
DataType fieldType;
if (exp instanceof AttributeExpression) {
if (expression instanceof AttributeExpression) {
Attribute attribute = schema.getAttribute(fieldName);
if (attribute == null) {
throw new VerificationException(exp, "Attribute '" + fieldName + "' not found.");
throw new VerificationException(expression, "Attribute '" + fieldName + "' not found.");
}
fieldDesc = "attribute";
fieldType = attribute.getDataType();
} else if (exp instanceof IndexExpression) {
} else if (expression instanceof IndexExpression) {
SDField field = schema.getConcreteField(fieldName);
if (field == null) {
throw new VerificationException(exp, "Index field '" + fieldName + "' not found.");
throw new VerificationException(expression, "Index field '" + fieldName + "' not found.");
}
fieldDesc = "index field";
fieldType = field.getDataType();
} else if (exp instanceof SummaryExpression) {
} else if (expression instanceof SummaryExpression) {
SummaryField field = schema.getSummaryField(fieldName);
if (field == null) {
// Use document field if summary field is not found
Expand All @@ -137,7 +137,7 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType)
fieldDesc = "document field";
fieldType = sdField.getDataType();
} else {
throw new VerificationException(exp, "Summary field '" + fieldName + "' not found.");
throw new VerificationException(expression, "Summary field '" + fieldName + "' not found.");
}
} else {
fieldDesc = "summary field";
Expand All @@ -148,8 +148,8 @@ public void tryOutputType(Expression exp, String fieldName, DataType valueType)
}
if ( ! fieldType.isAssignableFrom(valueType) &&
! fieldType.isAssignableFrom(createCompatType(valueType))) {
throw new VerificationException(exp, "Can not assign " + valueType.getName() + " to " + fieldDesc +
" '" + fieldName + "' which is " + fieldType.getName() + ".");
throw new VerificationException(expression, "Can not assign " + valueType.getName() + " to " + fieldDesc +
" '" + fieldName + "' which is " + fieldType.getName() + ".");
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ void requireThatOutputConflictThrows() throws ParseException {
}
catch (IllegalArgumentException e) {
assertEquals("For schema 'indexing_output_confict', field 'bar': For expression 'index bar': Attempting " +
"to assign conflicting values to field 'bar'.",
"to assign conflicting values to field 'bar'",
Exceptions.toMessageString(e));
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ void testAttributeChanged() throws ParseException {
}
catch (IllegalArgumentException e) {
assertEquals("For schema 'indexing_attribute_changed', field 'foo': For expression 'attribute foo': " +
"Attempting to assign conflicting values to field 'foo'.",
"Attempting to assign conflicting values to field 'foo'",
Exceptions.toMessageString(e));
}
}
Expand Down Expand Up @@ -80,7 +80,7 @@ void testIndexChanged() throws ParseException {
}
catch (IllegalArgumentException e) {
assertEquals("For schema 'indexing_index_changed', field 'foo': For expression 'index foo': " +
"Attempting to assign conflicting values to field 'foo'.",
"Attempting to assign conflicting values to field 'foo'",
Exceptions.toMessageString(e));
}
}
Expand Down Expand Up @@ -124,7 +124,7 @@ void testSummaryChanged() throws ParseException {
}
catch (IllegalArgumentException e) {
assertEquals("For schema 'indexing_summary_fail', field 'foo': For expression 'summary foo': Attempting " +
"to assign conflicting values to field 'foo'.",
"to assign conflicting values to field 'foo'",
Exceptions.toMessageString(e));
}
}
Expand Down Expand Up @@ -186,7 +186,7 @@ void requireThatMultilineOutputConflictThrows() throws ParseException {
}
catch (IllegalArgumentException e) {
assertEquals("For schema 'indexing_multiline_output_confict', field 'cox': For expression 'index cox': " +
"Attempting to assign conflicting values to field 'cox'.",
"Attempting to assign conflicting values to field 'cox'",
Exceptions.toMessageString(e));
}
}
Expand Down
1 change: 1 addition & 0 deletions document/abi-spec.json
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@
"public int getCode()",
"public com.yahoo.document.FieldPath buildFieldPath(java.lang.String)",
"public com.yahoo.document.PrimitiveDataType getPrimitiveType()",
"public com.yahoo.document.DataType getNestedType()",
"public void visitMembers(com.yahoo.vespa.objects.ObjectVisitor)",
"public int compareTo(com.yahoo.document.DataType)",
"public boolean isMultivalue()",
Expand Down
2 changes: 2 additions & 0 deletions document/src/main/java/com/yahoo/document/ArrayDataType.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@ public ArrayDataType(DataType nestedType, int code) {
super("Array<"+nestedType.getName()+">", code, nestedType);
}

@Override
public ArrayDataType clone() {
return (ArrayDataType) super.clone();
}

@Override
public Array createFieldValue() {
return new Array(this);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,8 @@ public CollectionDataType clone() {
return type;
}

public DataType getNestedType() {
return nestedType;
}
@Override
public DataType getNestedType() { return nestedType; }

@Override
protected FieldValue createByReflection(Object arg) { return null; }
Expand Down
7 changes: 4 additions & 3 deletions document/src/main/java/com/yahoo/document/DataType.java
Original file line number Diff line number Diff line change
Expand Up @@ -291,9 +291,10 @@ public FieldPath buildFieldPath(String fieldPathString) {
*
* @return primitive data type, or null
*/
public PrimitiveDataType getPrimitiveType() {
return null;
}
public PrimitiveDataType getPrimitiveType() { return null; }

/** Returns the nested type of this if it is a collection type, null otherwise. */
public DataType getNestedType() { return null; }

@Override
public void visitMembers(ObjectVisitor visitor) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.indexinglanguage.expressions;

import com.yahoo.document.DataType;
import com.yahoo.document.datatypes.FieldValue;

/**
* A DataType representing "any". This is (so far) only needed during type resolution of indexing pipelines
* so it is placed here.
*
* @author bratseth
*/
class AnyDataType extends DataType {

static final AnyDataType instance = new AnyDataType();

private AnyDataType() {
super("any", DataType.lastPredefinedDataTypeId() + 1);
}

@Override
public FieldValue createFieldValue() { throw new UnsupportedOperationException(); }

@Override
public Class<?> getValueClass() { throw new UnsupportedOperationException(); }

@Override
public boolean isValueCompatible(FieldValue value) { return true; }

}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import java.math.BigDecimal;
import java.math.MathContext;
import java.util.Objects;

/**
* @author Simon Thoresen Hult
Expand Down Expand Up @@ -48,59 +49,45 @@ public String toString() {

public ArithmeticExpression(Expression left, Operator op, Expression right) {
super(requiredInputType(left, right));
left.getClass(); // throws NullPointerException
op.getClass();
right.getClass();
this.left = left;
this.op = op;
this.right = right;
this.left = Objects.requireNonNull(left);
this.op = Objects.requireNonNull(op);
this.right = Objects.requireNonNull(right);
}

public Expression getLeftHandSide() { return left; }

public Operator getOperator() { return op; }

public Expression getRightHandSide() { return right; }

@Override
public ArithmeticExpression convertChildren(ExpressionConverter converter) {
// TODO: branch()?
return new ArithmeticExpression(converter.convert(left), op, converter.convert(right));
}

public Expression getLeftHandSide() {
return left;
}

public Operator getOperator() {
return op;
}

public Expression getRightHandSide() {
return right;
}

@Override
protected void doExecute(ExecutionContext context) {
FieldValue input = context.getValue();
context.setValue(evaluate(context.setValue(input).execute(left).getValue(),
context.setValue(input).execute(right).getValue()));
protected void doVerify(VerificationContext context) {
DataType input = context.getCurrentType();
context.setCurrentType(evaluate(context.setCurrentType(input).verify(left).getCurrentType(),
context.setCurrentType(input).verify(right).getCurrentType()));
}

@Override
protected void doVerify(VerificationContext context) {
DataType input = context.getValueType();
context.setValueType(evaluate(context.setValueType(input).execute(left).getValueType(),
context.setValueType(input).execute(right).getValueType()));
protected void doExecute(ExecutionContext context) {
FieldValue input = context.getCurrentValue();
context.setCurrentValue(evaluate(context.setCurrentValue(input).execute(left).getCurrentValue(),
context.setCurrentValue(input).execute(right).getCurrentValue()));
}

private static DataType requiredInputType(Expression lhs, Expression rhs) {
DataType lhsType = lhs.requiredInputType();
DataType rhsType = rhs.requiredInputType();
if (lhsType == null) {
return rhsType;
}
if (rhsType == null) {
return lhsType;
}
if (!lhsType.equals(rhsType)) {
if (lhsType == null) return rhsType;
if (rhsType == null) return lhsType;
if (!lhsType.equals(rhsType))
throw new VerificationException(ArithmeticExpression.class, "Operands require conflicting input types, " +
lhsType.getName() + " vs " + rhsType.getName());
}
return lhsType;
}

Expand All @@ -115,20 +102,12 @@ public String toString() {
}

@Override
public boolean equals(Object obj) {
if (!(obj instanceof ArithmeticExpression)) {
return false;
}
ArithmeticExpression exp = (ArithmeticExpression)obj;
if (!left.equals(exp.left)) {
return false;
}
if (!op.equals(exp.op)) {
return false;
}
if (!right.equals(exp.right)) {
return false;
}
public boolean equals(Object object) {
if (!(object instanceof ArithmeticExpression expression)) return false;

if (!left.equals(expression.left)) return false;
if (!op.equals(expression.op)) return false;
if (!right.equals(expression.right)) return false;
return true;
}

Expand All @@ -138,39 +117,28 @@ public int hashCode() {
}

private DataType evaluate(DataType lhs, DataType rhs) {
if (lhs == null || rhs == null) {
if (lhs == null || rhs == null)
throw new VerificationException(this, "Attempting to perform arithmetic on a null value");
}
if (!(lhs instanceof NumericDataType) ||
!(rhs instanceof NumericDataType))
{
if (!(lhs instanceof NumericDataType) || !(rhs instanceof NumericDataType))
throw new VerificationException(this, "Attempting to perform unsupported arithmetic: [" +
lhs.getName() + "] " + op + " [" + rhs.getName() + "]");
}
if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE ||
rhs == DataType.FLOAT || rhs == DataType.DOUBLE)
{
if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE) {

if (lhs == DataType.FLOAT || lhs == DataType.DOUBLE || rhs == DataType.FLOAT || rhs == DataType.DOUBLE) {
if (lhs == DataType.DOUBLE || rhs == DataType.DOUBLE)
return DataType.DOUBLE;
}
return DataType.FLOAT;
}
if (lhs == DataType.LONG || rhs == DataType.LONG) {
if (lhs == DataType.LONG || rhs == DataType.LONG)
return DataType.LONG;
}
return DataType.INT;
}

private FieldValue evaluate(FieldValue lhs, FieldValue rhs) {
if (lhs == null || rhs == null) {
return null;
}
if (!(lhs instanceof NumericFieldValue) ||
!(rhs instanceof NumericFieldValue))
{
if (lhs == null || rhs == null) return null;
if (!(lhs instanceof NumericFieldValue) || !(rhs instanceof NumericFieldValue))
throw new IllegalArgumentException("Unsupported operation: [" + lhs.getDataType().getName() + "] " +
op + " [" + rhs.getDataType().getName() + "]");
}

BigDecimal lhsVal = asBigDecimal((NumericFieldValue)lhs);
BigDecimal rhsVal = asBigDecimal((NumericFieldValue)rhs);
return switch (op) {
Expand All @@ -184,31 +152,27 @@ private FieldValue evaluate(FieldValue lhs, FieldValue rhs) {

private FieldValue createFieldValue(FieldValue lhs, FieldValue rhs, BigDecimal val) {
if (lhs instanceof FloatFieldValue || lhs instanceof DoubleFieldValue ||
rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue)
{
if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue) {
rhs instanceof FloatFieldValue || rhs instanceof DoubleFieldValue) {
if (lhs instanceof DoubleFieldValue || rhs instanceof DoubleFieldValue)
return new DoubleFieldValue(val.doubleValue());
}
return new FloatFieldValue(val.floatValue());
}
if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue) {
if (lhs instanceof LongFieldValue || rhs instanceof LongFieldValue)
return new LongFieldValue(val.longValue());
}
return new IntegerFieldValue(val.intValue());
}

public static BigDecimal asBigDecimal(NumericFieldValue value) {
if (value instanceof ByteFieldValue) {
if (value instanceof ByteFieldValue)
return BigDecimal.valueOf(((ByteFieldValue)value).getByte());
} else if (value instanceof DoubleFieldValue) {
else if (value instanceof DoubleFieldValue)
return BigDecimal.valueOf(((DoubleFieldValue)value).getDouble());
} else if (value instanceof FloatFieldValue) {
else if (value instanceof FloatFieldValue)
return BigDecimal.valueOf(((FloatFieldValue)value).getFloat());
} else if (value instanceof IntegerFieldValue) {
else if (value instanceof IntegerFieldValue)
return BigDecimal.valueOf(((IntegerFieldValue)value).getInteger());
} else if (value instanceof LongFieldValue) {
else if (value instanceof LongFieldValue)
return BigDecimal.valueOf(((LongFieldValue)value).getLong());
}
throw new IllegalArgumentException("Unsupported numeric field value type '" +
value.getClass().getName() + "'");
}
Expand All @@ -218,4 +182,5 @@ public void selectMembers(ObjectPredicate predicate, ObjectOperation operation)
left.select(predicate, operation);
right.select(predicate, operation);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,5 @@ public AttributeExpression(String fieldName) {
public boolean equals(Object obj) {
return super.equals(obj) && obj instanceof AttributeExpression;
}

}
Loading

0 comments on commit cac7a8c

Please sign in to comment.