Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

3 changes/fixes around how tuples are output by the replayer. #513

Merged
merged 1 commit into from
Feb 20, 2024
Merged

3 changes/fixes around how tuples are output by the replayer. #513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions TrafficCapture/dockerSolution/src/main/docker/migrationConsole/humanReadableLogs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

logger = logging.getLogger(__name__)

LOG_JSON_TUPLE_FIELD = "message"
BASE64_ENCODED_TUPLE_PATHS = ["sourceRequest.body", "targetRequest.body", "sourceResponse.body", "targetResponse.body"]
# TODO: I'm not positive about the capitalization of the Content-Encoding and Content-Type headers.
# This version worked on my test cases, but not guaranteed to work in all cases.
Expand Down Expand Up @@ -149,9 +148,7 @@ def parse_body_value(raw_value: str, content_encoding: Optional[str],


def parse_tuple(line: str, line_no: int) -> dict:
item = json.loads(line)
message = item[LOG_JSON_TUPLE_FIELD]
tuple = json.loads(message)
tuple = json.loads(line)
try:
is_bulk_path = BULK_URI_PATH in get_element(URI_PATH, tuple, raise_on_error=True)
except DictionaryPathException as e:
Expand Down
10 changes: 3 additions & 7 deletions ...ficReplayer/src/main/java/org/opensearch/migrations/replay/ParsedHttpMessagesAsDicts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,13 +107,9 @@ private static Stream<ByteBuf> byteToByteBufStream(List<byte[]> incoming) {
}

private static byte[] getBytesFromByteBuf(ByteBuf buf) {
if (buf.hasArray()) {
return buf.array();
} else {
var bytes = new byte[buf.readableBytes()];
buf.getBytes(buf.readerIndex(), bytes);
return bytes;
}
var bytes = new byte[buf.readableBytes()];
buf.getBytes(buf.readerIndex(), bytes);
return bytes;
}

private static Map<String, Object> fillMap(LinkedHashMap<String, Object> map,
Expand Down
7 changes: 2 additions & 5 deletions TrafficCapture/trafficReplayer/src/main/resources/log4j2.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,8 @@ appender.OUTPUT_TUPLES.type = RollingFile
appender.OUTPUT_TUPLES.name = OUTPUT_TUPLES
appender.OUTPUT_TUPLES.fileName = ${tupleDir}/tuples.log
appender.OUTPUT_TUPLES.filePattern = ${tupleDir}/tuples-%d{yyyy-MM-dd-HH:mm}{UTC}.log
appender.OUTPUT_TUPLES.layout.type = JsonLayout
appender.OUTPUT_TUPLES.layout.properties = false
appender.OUTPUT_TUPLES.layout.complete = false
appender.OUTPUT_TUPLES.layout.eventEol = true
appender.OUTPUT_TUPLES.layout.compact = true
appender.OUTPUT_TUPLES.layout.type = PatternLayout
appender.OUTPUT_TUPLES.layout.pattern = %m%n
appender.OUTPUT_TUPLES.policies.type = Policies
appender.OUTPUT_TUPLES.policies.size.type = SizeBasedTriggeringPolicy
appender.OUTPUT_TUPLES.policies.size.size = 10 MB
Expand Down
12 changes: 6 additions & 6 deletions ...ficReplayer/src/test/java/org/opensearch/migrations/replay/ResultsToLogsConsumerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ public void testOutputterForGet() throws IOException {
" \"Status-Code\": 200,\n" +
" \"Reason-Phrase\": \"OK\",\n" +
" \"response_time_ms\": 0,\n" +
" \"body\": \"SFRUUC8xLjEgMjAwIE9LDQpDb250ZW50LXRyYW5zZmVyLWVuY29kaW5nOiBjaHVua2VkDQpEYXRlOiBUaHUsIDA4IEp1biAyMDIzIDIzOjA2OjIzIEdNVA0KVHJhbnNmZXItZW5jb2Rpbmc6IGNodW5rZWQNCkNvbnRlbnQtdHlwZTogdGV4dC9wbGFpbg0KRnVudGltZTogY2hlY2tJdCENCg0KMWUNCkkgc2hvdWxkIGJlIGRlY3J5cHRlZCB0ZXN0ZXIhCg0KMA0KDQo=\",\n" +
" \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" +
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was hoping there was funny message encoded ◡̈

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was hoping there was funny message encoded ◡̈

" \"Content-transfer-encoding\": \"chunked\",\n" +
" \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" +
" \"Content-type\": \"text/plain\",\n" +
Expand All @@ -180,7 +180,7 @@ public void testOutputterForGet() throws IOException {
" \"Status-Code\": 200,\n" +
" \"Reason-Phrase\": \"OK\",\n" +
" \"response_time_ms\": 267,\n" +
" \"body\": \"SFRUUC8xLjEgMjAwIE9LDQpDb250ZW50LXRyYW5zZmVyLWVuY29kaW5nOiBjaHVua2VkDQpEYXRlOiBUaHUsIDA4IEp1biAyMDIzIDIzOjA2OjIzIEdNVA0KVHJhbnNmZXItZW5jb2Rpbmc6IGNodW5rZWQNCkNvbnRlbnQtdHlwZTogdGV4dC9wbGFpbg0KRnVudGltZTogY2hlY2tJdCENCg0KMWUNCkkgc2hvdWxkIGJlIGRlY3J5cHRlZCB0ZXN0ZXIhCg0KMA0KDQo=\",\n" +
" \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" +
" \"Content-transfer-encoding\": \"chunked\",\n" +
" \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" +
" \"Content-type\": \"text/plain\",\n" +
Expand All @@ -201,7 +201,7 @@ public void testOutputterForPost() throws IOException {
" \"Request-URI\": \"/test\",\n" +
" \"Method\": \"POST\",\n" +
" \"HTTP-Version\": \"HTTP/1.1\",\n" +
" \"body\": \"UE9TVCAvdGVzdCBIVFRQLzEuMQpIb3N0OiBmb28uZXhhbXBsZQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24KQ29udGVudC1MZW5ndGg6IDYxNgoKewogICJzZXR0aW5ncyI6IHsKICAgICJpbmRleCI6IHsKICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LAogICAgICAibnVtYmVyX29mX3JlcGxpY2FzIjogMwogICAgfSwKICAgICJhbmFseXNpcyI6IHsKICAgICAgImFuYWx5emVyIjogewogICAgICAgICJuYW1lQW5hbHl6ZXIiOiB7CiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLAogICAgICAgICAgInRva2VuaXplciI6ICJrZXl3b3JkIiwKICAgICAgICAgICJmaWx0ZXIiOiAidXBwZXJjYXNlIgogICAgICAgIH0KICAgICAgfQogICAgfQogIH0sCiAgIm1hcHBpbmdzIjogewogICAgImVtcGxveWVlIjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYWdlIjogewogICAgICAgICAgInR5cGUiOiAibG9uZyIKICAgICAgICB9LAogICAgICAgICJsZXZlbCI6IHsKICAgICAgICAgICJ0eXBlIjogImxvbmciCiAgICAgICAgfSwKICAgICAgICAidGl0bGUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IgogICAgICAgIH0sCiAgICAgICAgIm5hbWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IiwKICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiCiAgICAgICAgfQogICAgICB9CiAgICB9CiAgfQp9Cg==\",\n" +
" \"body\": \"ewogICJzZXR0aW5ncyI6IHsKICAgICJpbmRleCI6IHsKICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LAogICAgICAibnVtYmVyX29mX3JlcGxpY2FzIjogMwogICAgfSwKICAgICJhbmFseXNpcyI6IHsKICAgICAgImFuYWx5emVyIjogewogICAgICAgICJuYW1lQW5hbHl6ZXIiOiB7CiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLAogICAgICAgICAgInRva2VuaXplciI6ICJrZXl3b3JkIiwKICAgICAgICAgICJmaWx0ZXIiOiAidXBwZXJjYXNlIgogICAgICAgIH0KICAgICAgfQogICAgfQogIH0sCiAgIm1hcHBpbmdzIjogewogICAgImVtcGxveWVlIjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYWdlIjogewogICAgICAgICAgInR5cGUiOiAibG9uZyIKICAgICAgICB9LAogICAgICAgICJsZXZlbCI6IHsKICAgICAgICAgICJ0eXBlIjogImxvbmciCiAgICAgICAgfSwKICAgICAgICAidGl0bGUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IgogICAgICAgIH0sCiAgICAgICAgIm5hbWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IiwKICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiCiAgICAgICAgfQogICAgICB9CiAgICB9CiAgfQp9Cg==\",\n" +
" \"Host\": \"foo.example\",\n" +
" \"Content-Type\": \"application/json\",\n" +
" \"Content-Length\": \"616\"\n" +
Expand All @@ -213,7 +213,7 @@ public void testOutputterForPost() throws IOException {
" \"Status-Code\": 200,\n" +
" \"Reason-Phrase\": \"OK\",\n" +
" \"response_time_ms\": 0,\n" +
" \"body\": \"SFRUUC8xLjEgMjAwIE9LDQpDb250ZW50LXRyYW5zZmVyLWVuY29kaW5nOiBjaHVua2VkDQpEYXRlOiBUaHUsIDA4IEp1biAyMDIzIDIzOjA2OjIzIEdNVA0KVHJhbnNmZXItZW5jb2Rpbmc6IGNodW5rZWQNCkNvbnRlbnQtdHlwZTogdGV4dC9wbGFpbg0KRnVudGltZTogY2hlY2tJdCENCg0KMWUNCkkgc2hvdWxkIGJlIGRlY3J5cHRlZCB0ZXN0ZXIhCg0KMA0KDQo=\",\n" +
" \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" +
" \"Content-transfer-encoding\": \"chunked\",\n" +
" \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" +
" \"Content-type\": \"text/plain\",\n" +
Expand All @@ -224,7 +224,7 @@ public void testOutputterForPost() throws IOException {
" \"Request-URI\": \"/test\",\n" +
" \"Method\": \"POST\",\n" +
" \"HTTP-Version\": \"HTTP/1.1\",\n" +
" \"body\": \"UE9TVCAvdGVzdCBIVFRQLzEuMQpIb3N0OiBmb28uZXhhbXBsZQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24KQ29udGVudC1MZW5ndGg6IDYxNgoKewogICJzZXR0aW5ncyI6IHsKICAgICJpbmRleCI6IHsKICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LAogICAgICAibnVtYmVyX29mX3JlcGxpY2FzIjogMwogICAgfSwKICAgICJhbmFseXNpcyI6IHsKICAgICAgImFuYWx5emVyIjogewogICAgICAgICJuYW1lQW5hbHl6ZXIiOiB7CiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLAogICAgICAgICAgInRva2VuaXplciI6ICJrZXl3b3JkIiwKICAgICAgICAgICJmaWx0ZXIiOiAidXBwZXJjYXNlIgogICAgICAgIH0KICAgICAgfQogICAgfQogIH0sCiAgIm1hcHBpbmdzIjogewogICAgImVtcGxveWVlIjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYWdlIjogewogICAgICAgICAgInR5cGUiOiAibG9uZyIKICAgICAgICB9LAogICAgICAgICJsZXZlbCI6IHsKICAgICAgICAgICJ0eXBlIjogImxvbmciCiAgICAgICAgfSwKICAgICAgICAidGl0bGUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IgogICAgICAgIH0sCiAgICAgICAgIm5hbWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IiwKICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiCiAgICAgICAgfQogICAgICB9CiAgICB9CiAgfQp9Cg==\",\n" +
" \"body\": \"ewogICJzZXR0aW5ncyI6IHsKICAgICJpbmRleCI6IHsKICAgICAgIm51bWJlcl9vZl9zaGFyZHMiOiA3LAogICAgICAibnVtYmVyX29mX3JlcGxpY2FzIjogMwogICAgfSwKICAgICJhbmFseXNpcyI6IHsKICAgICAgImFuYWx5emVyIjogewogICAgICAgICJuYW1lQW5hbHl6ZXIiOiB7CiAgICAgICAgICAidHlwZSI6ICJjdXN0b20iLAogICAgICAgICAgInRva2VuaXplciI6ICJrZXl3b3JkIiwKICAgICAgICAgICJmaWx0ZXIiOiAidXBwZXJjYXNlIgogICAgICAgIH0KICAgICAgfQogICAgfQogIH0sCiAgIm1hcHBpbmdzIjogewogICAgImVtcGxveWVlIjogewogICAgICAicHJvcGVydGllcyI6IHsKICAgICAgICAiYWdlIjogewogICAgICAgICAgInR5cGUiOiAibG9uZyIKICAgICAgICB9LAogICAgICAgICJsZXZlbCI6IHsKICAgICAgICAgICJ0eXBlIjogImxvbmciCiAgICAgICAgfSwKICAgICAgICAidGl0bGUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IgogICAgICAgIH0sCiAgICAgICAgIm5hbWUiOiB7CiAgICAgICAgICAidHlwZSI6ICJ0ZXh0IiwKICAgICAgICAgICJhbmFseXplciI6ICJuYW1lQW5hbHl6ZXIiCiAgICAgICAgfQogICAgICB9CiAgICB9CiAgfQp9Cg==\",\n" +
" \"Host\": \"foo.example\",\n" +
" \"Content-Type\": \"application/json\",\n" +
" \"Content-Length\": \"616\"\n" +
Expand All @@ -236,7 +236,7 @@ public void testOutputterForPost() throws IOException {
" \"Status-Code\": 200,\n" +
" \"Reason-Phrase\": \"OK\",\n" +
" \"response_time_ms\": 267,\n" +
" \"body\": \"SFRUUC8xLjEgMjAwIE9LDQpDb250ZW50LXRyYW5zZmVyLWVuY29kaW5nOiBjaHVua2VkDQpEYXRlOiBUaHUsIDA4IEp1biAyMDIzIDIzOjA2OjIzIEdNVA0KVHJhbnNmZXItZW5jb2Rpbmc6IGNodW5rZWQNCkNvbnRlbnQtdHlwZTogdGV4dC9wbGFpbg0KRnVudGltZTogY2hlY2tJdCENCg0KMWUNCkkgc2hvdWxkIGJlIGRlY3J5cHRlZCB0ZXN0ZXIhCg0KMA0KDQo=\",\n" +
" \"body\": \"SSBzaG91bGQgYmUgZGVjcnlwdGVkIHRlc3RlciEK\",\n" +
" \"Content-transfer-encoding\": \"chunked\",\n" +
" \"Date\": \"Thu, 08 Jun 2023 23:06:23 GMT\",\n" +
" \"Content-type\": \"text/plain\",\n" +
Expand Down