Skip to content

Commit

Permalink
fix grammer for limit by
Browse files Browse the repository at this point in the history
  • Loading branch information
@timgl
timgl committed Oct 25, 2024
1 parent 98f87a1 commit 828687a
Show file tree
Hide file tree
Showing 17 changed files with 3,332 additions and 3,075 deletions.
3,056 changes: 1,589 additions & 1,467 deletions hogql_parser/HogQLParser.cpp
View file

Large diffs are not rendered by default.

70 changes: 52 additions & 18 deletions hogql_parser/HogQLParser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,22 +54,23 @@ class HogQLParser : public antlr4::Parser {
RuleSelectStmt = 24, RuleWithClause = 25, RuleTopClause = 26, RuleFromClause = 27,
RuleArrayJoinClause = 28, RuleWindowClause = 29, RulePrewhereClause = 30,
RuleWhereClause = 31, RuleGroupByClause = 32, RuleHavingClause = 33,
RuleOrderByClause = 34, RuleProjectionOrderByClause = 35, RuleLimitAndOffsetClause = 36,
RuleOffsetOnlyClause = 37, RuleSettingsClause = 38, RuleJoinExpr = 39,
RuleJoinOp = 40, RuleJoinOpCross = 41, RuleJoinConstraintClause = 42,
RuleSampleClause = 43, RuleOrderExprList = 44, RuleOrderExpr = 45, RuleRatioExpr = 46,
RuleSettingExprList = 47, RuleSettingExpr = 48, RuleWindowExpr = 49,
RuleWinPartitionByClause = 50, RuleWinOrderByClause = 51, RuleWinFrameClause = 52,
RuleWinFrameExtend = 53, RuleWinFrameBound = 54, RuleExpr = 55, RuleColumnTypeExpr = 56,
RuleColumnExprList = 57, RuleColumnExpr = 58, RuleColumnLambdaExpr = 59,
RuleHogqlxTagElement = 60, RuleHogqlxTagAttribute = 61, RuleWithExprList = 62,
RuleWithExpr = 63, RuleColumnIdentifier = 64, RuleNestedIdentifier = 65,
RuleTableExpr = 66, RuleTableFunctionExpr = 67, RuleTableIdentifier = 68,
RuleTableArgList = 69, RuleDatabaseIdentifier = 70, RuleFloatingLiteral = 71,
RuleNumberLiteral = 72, RuleLiteral = 73, RuleInterval = 74, RuleKeyword = 75,
RuleKeywordForAlias = 76, RuleAlias = 77, RuleIdentifier = 78, RuleEnumValue = 79,
RulePlaceholder = 80, RuleString = 81, RuleTemplateString = 82, RuleStringContents = 83,
RuleFullTemplateString = 84, RuleStringContentsFull = 85
RuleOrderByClause = 34, RuleProjectionOrderByClause = 35, RuleLimitByClause = 36,
RuleLimitAndOffsetClause = 37, RuleOffsetOnlyClause = 38, RuleSettingsClause = 39,
RuleJoinExpr = 40, RuleJoinOp = 41, RuleJoinOpCross = 42, RuleJoinConstraintClause = 43,
RuleSampleClause = 44, RuleLimitExpr = 45, RuleOrderExprList = 46, RuleOrderExpr = 47,
RuleRatioExpr = 48, RuleSettingExprList = 49, RuleSettingExpr = 50,
RuleWindowExpr = 51, RuleWinPartitionByClause = 52, RuleWinOrderByClause = 53,
RuleWinFrameClause = 54, RuleWinFrameExtend = 55, RuleWinFrameBound = 56,
RuleExpr = 57, RuleColumnTypeExpr = 58, RuleColumnExprList = 59, RuleColumnExpr = 60,
RuleColumnLambdaExpr = 61, RuleHogqlxTagElement = 62, RuleHogqlxTagAttribute = 63,
RuleWithExprList = 64, RuleWithExpr = 65, RuleColumnIdentifier = 66,
RuleNestedIdentifier = 67, RuleTableExpr = 68, RuleTableFunctionExpr = 69,
RuleTableIdentifier = 70, RuleTableArgList = 71, RuleDatabaseIdentifier = 72,
RuleFloatingLiteral = 73, RuleNumberLiteral = 74, RuleLiteral = 75,
RuleInterval = 76, RuleKeyword = 77, RuleKeywordForAlias = 78, RuleAlias = 79,
RuleIdentifier = 80, RuleEnumValue = 81, RulePlaceholder = 82, RuleString = 83,
RuleTemplateString = 84, RuleStringContents = 85, RuleFullTemplateString = 86,
RuleStringContentsFull = 87
};

explicit HogQLParser(antlr4::TokenStream *input);
Expand Down Expand Up @@ -125,6 +126,7 @@ class HogQLParser : public antlr4::Parser {
class HavingClauseContext;
class OrderByClauseContext;
class ProjectionOrderByClauseContext;
class LimitByClauseContext;
class LimitAndOffsetClauseContext;
class OffsetOnlyClauseContext;
class SettingsClauseContext;
Expand All @@ -133,6 +135,7 @@ class HogQLParser : public antlr4::Parser {
class JoinOpCrossContext;
class JoinConstraintClauseContext;
class SampleClauseContext;
class LimitExprContext;
class OrderExprListContext;
class OrderExprContext;
class RatioExprContext;
Expand Down Expand Up @@ -619,6 +622,7 @@ class HogQLParser : public antlr4::Parser {
HavingClauseContext *havingClause();
WindowClauseContext *windowClause();
OrderByClauseContext *orderByClause();
LimitByClauseContext *limitByClause();
LimitAndOffsetClauseContext *limitAndOffsetClause();
OffsetOnlyClauseContext *offsetOnlyClause();
SettingsClauseContext *settingsClause();
Expand Down Expand Up @@ -812,6 +816,22 @@ class HogQLParser : public antlr4::Parser {

ProjectionOrderByClauseContext* projectionOrderByClause();

class LimitByClauseContext : public antlr4::ParserRuleContext {
public:
LimitByClauseContext(antlr4::ParserRuleContext *parent, size_t invokingState);
virtual size_t getRuleIndex() const override;
antlr4::tree::TerminalNode *LIMIT();
LimitExprContext *limitExpr();
antlr4::tree::TerminalNode *BY();
ColumnExprListContext *columnExprList();


virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;

};

LimitByClauseContext* limitByClause();

class LimitAndOffsetClauseContext : public antlr4::ParserRuleContext {
public:
LimitAndOffsetClauseContext(antlr4::ParserRuleContext *parent, size_t invokingState);
Expand All @@ -820,8 +840,6 @@ class HogQLParser : public antlr4::Parser {
std::vector<ColumnExprContext *> columnExpr();
ColumnExprContext* columnExpr(size_t i);
antlr4::tree::TerminalNode *COMMA();
antlr4::tree::TerminalNode *BY();
ColumnExprListContext *columnExprList();
antlr4::tree::TerminalNode *WITH();
antlr4::tree::TerminalNode *TIES();
antlr4::tree::TerminalNode *OFFSET();
Expand Down Expand Up @@ -1025,6 +1043,22 @@ class HogQLParser : public antlr4::Parser {

SampleClauseContext* sampleClause();

class LimitExprContext : public antlr4::ParserRuleContext {
public:
LimitExprContext(antlr4::ParserRuleContext *parent, size_t invokingState);
virtual size_t getRuleIndex() const override;
std::vector<ColumnExprContext *> columnExpr();
ColumnExprContext* columnExpr(size_t i);
antlr4::tree::TerminalNode *COMMA();
antlr4::tree::TerminalNode *OFFSET();


virtual std::any accept(antlr4::tree::ParseTreeVisitor *visitor) override;

};

LimitExprContext* limitExpr();

class OrderExprListContext : public antlr4::ParserRuleContext {
public:
OrderExprListContext(antlr4::ParserRuleContext *parent, size_t invokingState);
Expand Down
4 changes: 3 additions & 1 deletion hogql_parser/HogQLParser.interp
View file

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions hogql_parser/HogQLParserBaseVisitor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,10 @@ class HogQLParserBaseVisitor : public HogQLParserVisitor {
return visitChildren(ctx);
}

virtual std::any visitLimitByClause(HogQLParser::LimitByClauseContext *ctx) override {
return visitChildren(ctx);
}

virtual std::any visitLimitAndOffsetClause(HogQLParser::LimitAndOffsetClauseContext *ctx) override {
return visitChildren(ctx);
}
Expand Down Expand Up @@ -211,6 +215,10 @@ class HogQLParserBaseVisitor : public HogQLParserVisitor {
return visitChildren(ctx);
}

virtual std::any visitLimitExpr(HogQLParser::LimitExprContext *ctx) override {
return visitChildren(ctx);
}

virtual std::any visitOrderExprList(HogQLParser::OrderExprListContext *ctx) override {
return visitChildren(ctx);
}
Expand Down
4 changes: 4 additions & 0 deletions hogql_parser/HogQLParserVisitor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@ class HogQLParserVisitor : public antlr4::tree::AbstractParseTreeVisitor {

virtual std::any visitProjectionOrderByClause(HogQLParser::ProjectionOrderByClauseContext *context) = 0;

virtual std::any visitLimitByClause(HogQLParser::LimitByClauseContext *context) = 0;

virtual std::any visitLimitAndOffsetClause(HogQLParser::LimitAndOffsetClauseContext *context) = 0;

virtual std::any visitOffsetOnlyClause(HogQLParser::OffsetOnlyClauseContext *context) = 0;
Expand All @@ -117,6 +119,8 @@ class HogQLParserVisitor : public antlr4::tree::AbstractParseTreeVisitor {

virtual std::any visitSampleClause(HogQLParser::SampleClauseContext *context) = 0;

virtual std::any visitLimitExpr(HogQLParser::LimitExprContext *context) = 0;

virtual std::any visitOrderExprList(HogQLParser::OrderExprListContext *context) = 0;

virtual std::any visitOrderExpr(HogQLParser::OrderExprContext *context) = 0;
Expand Down
2 changes: 1 addition & 1 deletion posthog/hogql/ast.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -798,7 +798,7 @@ class WindowFunction(Expr):

@dataclass(kw_only=True)
class LimitByExpr(Expr):
offset_value: int
offset_value: Expr
exprs: list[Expr]


Expand Down
2 changes: 1 addition & 1 deletion posthog/hogql/database/schema/persons.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def select_from_persons_table(
inner_select_where.append(extracted_where)

if node.limit:
inner_select.limit_by = ast.LimitByExpr(offset_value=1, exprs=[ast.Field(chain=["id"])])
inner_select.limit_by = ast.LimitByExpr(offset_value=ast.Constant(value=1), exprs=[ast.Field(chain=["id"])])
inner_select.limit = clone_expr(node.limit, clear_locations=True, clear_types=True)
inner_select.order_by = (
[
Expand Down
30 changes: 0 additions & 30 deletions posthog/hogql/database/schema/test/__snapshots__/test_persons.ambr
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,33 +127,3 @@
max_bytes_before_external_group_by=0
'''
# ---
# name: TestPersonOptimization.test_simple_filter
'''
SELECT persons.id AS id,
persons.properties AS properties
FROM
(SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$some_prop'), ''), 'null'), '^"|"$', ''), person.version) AS `properties___$some_prop`,
argMax(person.properties, person.version) AS properties,
person.id AS id
FROM person
WHERE and(equals(person.team_id, 2), in(id,
(SELECT where_optimization.id AS id
FROM person AS where_optimization
WHERE and(equals(where_optimization.team_id, 2), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(where_optimization.properties, '$some_prop'), ''), 'null'), '^"|"$', ''), 'something'), 0), notIn(where_optimization.id,
(SELECT limit_delete_optimization.id AS id
FROM person AS limit_delete_optimization
WHERE and(equals(limit_delete_optimization.team_id, 2), equals(limit_delete_optimization.is_deleted, 1)))))
LIMIT 1 BY where_optimization.id
LIMIT 100)))
GROUP BY person.id
HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0))) AS persons
WHERE ifNull(equals(persons.`properties___$some_prop`, 'something'), 0)
LIMIT 100 SETTINGS readonly=2,
max_execution_time=60,
allow_experimental_object_type=1,
format_csv_allow_double_quotes=0,
max_ast_elements=4000000,
max_expanded_ast_elements=4000000,
max_bytes_before_external_group_by=0
'''
# ---
6 changes: 4 additions & 2 deletions posthog/hogql/grammar/HogQLParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ selectStmt:
havingClause?
windowClause?
orderByClause?
limitByClause?
(limitAndOffsetClause | offsetOnlyClause)?
settingsClause?
;
Expand All @@ -83,10 +84,10 @@ groupByClause: GROUP BY ((CUBE | ROLLUP) LPAREN columnExprList RPAREN | columnEx
havingClause: HAVING columnExpr;
orderByClause: ORDER BY orderExprList;
projectionOrderByClause: ORDER BY columnExprList;
limitByClause: LIMIT limitExpr BY columnExprList;
limitAndOffsetClause
: LIMIT columnExpr (COMMA columnExpr)? ((WITH TIES) | BY columnExprList)? // compact OFFSET-optional form
: LIMIT columnExpr (COMMA columnExpr)? (WITH TIES)? // compact OFFSET-optional form
| LIMIT columnExpr (WITH TIES)? OFFSET columnExpr // verbose OFFSET-included form with WITH TIES
| LIMIT columnExpr OFFSET columnExpr (BY columnExprList)? // verbose OFFSET-included form with BY
;
offsetOnlyClause: OFFSET columnExpr;
settingsClause: SETTINGS settingExprList;
Expand Down Expand Up @@ -115,6 +116,7 @@ joinConstraintClause
;

sampleClause: SAMPLE ratioExpr (OFFSET ratioExpr)?;
limitExpr: columnExpr ((COMMA | OFFSET) columnExpr)?;
orderExprList: orderExpr (COMMA orderExpr)*;
orderExpr: columnExpr (ASCENDING | DESCENDING | DESC)? (NULLS (FIRST | LAST))? (COLLATE STRING_LITERAL)?;
ratioExpr: placeholder | numberLiteral (SLASH numberLiteral)?;
Expand Down
4 changes: 3 additions & 1 deletion posthog/hogql/grammar/HogQLParser.interp
View file

Large diffs are not rendered by default.

Loading

0 comments on commit 828687a

Please sign in to comment.