mirror of https://github.com/status-im/msgstat.git
Update stats with regards review
This commit is contained in:
Alexander Ewetumo
parent
62845300d0
commit
af19d77525
207
README.md
207
README.md
|
|
@ -1,13 +1,13 @@
|
|||
Msgstat
|
||||
----------
|
||||
Msgstat provides a CLI tool which analyses status-go log files and generates corresponding output of information
|
||||
related to messages delivered over whisper (either rpc or p2p).
|
||||
Msgstat provides a CLI tool which analyses status-go log files and generates corresponding aggregated data
|
||||
related to messages delivery over whisper (either rpc or p2p) protocols.
|
||||
|
||||
## Usage
|
||||
|
||||
Msgstat provides the processing functionality for the processing of all log files which would be used to produced aggregated collection of message delivery facts.
|
||||
Msgstat spits out structures that provide commulation of all related facts about a given message based on desired output format supported.
|
||||
|
||||
It by defaults expects processing data to be delivered either through sending given logs through `stdin` or through the use of the `file` flag.
|
||||
*It by defaults expects processed data to be delivered either through pipes through `stdin` or through the use of the `file` flag.*
|
||||
|
||||
- Processing log lines through stdin
|
||||
|
||||
|
|
@ -15,61 +15,214 @@ It by defaults expects processing data to be delivered either through sending gi
|
|||
> cat ./logs/status-im/status-go/11.21.2017.log | msgstat
|
||||
```
|
||||
|
||||
- Processing log lines through stdin with `yaml` output format
|
||||
|
||||
```bash
|
||||
> cat ./logs/status-im/status-go/11.21.2017.log | msgstat -format=yaml
|
||||
```
|
||||
|
||||
- Processing log lines from a file
|
||||
|
||||
```bash
|
||||
> msgstat read -file=./logs/status-im/status-go/11.21.2017.log
|
||||
> msgstat -file=./logs/status-im/status-go/11.21.2017.log
|
||||
```
|
||||
|
||||
Where all information is delivered through the `stdout`, except when the use of the `-out` flag is used.
|
||||
All processed data is written through `stdout`, except when the use of the `-out` flag is used to point to a output file to contain
|
||||
created data.
|
||||
|
||||
### Flags
|
||||
|
||||
#### `-file=./status-log`
|
||||
This flag is used to indicate that all a file should be used to retrieve all log lines to be processed line by line for the generation of processed log facts.
|
||||
|
||||
This flag is used to indicate that file should be used to retrieve all log lines to be processed.
|
||||
|
||||
#### `-out=./processed-log.json`
|
||||
This flag is used to indicate that all processed data should be stored into provided file.
|
||||
|
||||
#### `-format=json|yaml|csv`
|
||||
This flag is used to indicate that all processed data should be stored into provided path.
|
||||
|
||||
This flag is used to dictate the encoding format which should be used for generation of processed resource which would
|
||||
then be processed by external tools as needed.
|
||||
#### `-format=json|yaml|toml`
|
||||
|
||||
This flag is used to dictate the encoding format which should be used for aggregated data.
|
||||
|
||||
|
||||
### Log Format
|
||||
### Log Input Format
|
||||
|
||||
Msgstat expects that all log contain log lines seperated by new lines, and it targets logs which mark this format:
|
||||
Msgstat expects that all log contain log lines separated by new lines, and targets log lines such has below:
|
||||
|
||||
```
|
||||
INFO [10-31|14:57:13] Message delivery notification geth=StatusIM state=eyJ0eXBlIjoiSW5jb21pbmdNZXNzYWdlIiwicHJvdG9jb2wiOiJSUEMiLCJzdGF0dXMiOiJDYWNoZWQiLCJlbnZlbG9wZSI6IkJCcHNCQTFiZ0RXNXUyQ0lWL3hyRlcvaThFUG5mZGxwdmxCcnNrbnUwN3ZZVXBhNXRnT3NVWUdhbmljYmY1MjdpOEg5UmRTWERtWUhOSHlyakVSOS9pQkgrcGxWS3FmME9mMHlRMlRKSHB2ZklrUU9VV1g0dzdENE1mTmZXYVUycER3djJQUG43VjVjaDRJeE9Oajh4YnFwMjhMQnErMzhmczBxaG9CYzVLWTczcnZ0T1NsNm12dFRWcGg2RENsUEVQOE1zU2pvaDBYakdXL3lYTnpBeWxUODNnZzdtdFRrY1l3V0ZhSHk5OEV5V1lVTFg5WHBTeUJFVVVVQ25GWUVoRDV1ZzBaUkZyY2ltN29QTng2RWZVZUhuc3lXLysvZFFRb1FMcXI4K1lvRFJaUVZSTVVDSUZwbWFBRWVjQUkvVkFSMGhsTkdrV2pGY0JkNzVjZXdIZzJvWWtuKzgrQlA3bTJsL0FKdk1COFJMVW5tZlpBN1dlOUJyUnM5alFKZGJiSmpCcGRaYVNqS0ZoVCtTYnhvVi9oWllobmFFVDQyZzZJajh3d1hGRHNNck54MlpJUEcxMEZhdm02Y0JlNWgybnh2ME55ZHVmaS92eFZaaTRuRU9SZGpuQ0RRZlhab0Rab1EwSXpvS1N5VyIsInRpbWUiOjE1MDk0NTgyMzMsImVudmVsb3BlX2hhc2giOiIweDU1YmMyMmJlNGIxYTQxOTRhZmQ4OTNlZTYzZDA1ZjU4MmIwYzAxMTk3OGJkMzkxM2FlYTdlYmI4Mjc4ZmMwZGIiLCJzb3VyY2UiOnsic3ltS2V5SUQiOiIiLCJwdWJLZXkiOiIweCIsInNpZyI6IiIsInR0bCI6MCwidG9waWMiOiIweDAwMDAwMDAwIiwicGF5bG9hZCI6IjB4IiwicGFkZGluZyI6IjB4IiwicG93VGltZSI6MCwicG93VGFyZ2V0IjowLCJ0YXJnZXRQZWVyIjoiIn19
|
||||
```
|
||||
|
||||
Msgstat will extract any log lines with matching `Mesage Delivery Notification` and extract the `state` field and value which contains the needed delivery facts.
|
||||
Msgstat will extract any log lines with matching `Message Delivery Notification` header, extracting the `state` field and it's value which contains base64 encoding delivery data.
|
||||
|
||||
|
||||
### Report Format
|
||||
### Output Format
|
||||
|
||||
Msgstat currently outputs parsed message into the following format:
|
||||
Msgstat currently supports the following formats:
|
||||
|
||||
- `TOML`
|
||||
|
||||
```toml
|
||||
envelope = "0x156789a7892cfa5b5a45ca9f3187799d2a0293034175093ab8726ecbd8cbc6c6"
|
||||
protocol = "RPC"
|
||||
from_device = "0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3"
|
||||
to_device = ""
|
||||
payload = ""
|
||||
direction = "OutgoingMessage"
|
||||
sent_time = 1970-01-01T00:00:00Z
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:28Z
|
||||
status = "Pending"
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:32Z
|
||||
status = "Pending"
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:33Z
|
||||
status = "Pending"
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:34Z
|
||||
status = "Pending"
|
||||
|
||||
[request]
|
||||
SymKeyID = "d29dd5c6470d556c20e69b0f01b269b00c74e363cae442628171256a6df5ce38"
|
||||
PublicKey = []
|
||||
Sig = ""
|
||||
TTL = 20
|
||||
Topic = "0x0f1a4771"
|
||||
Payload = [116, 101, 115, 116, 32, 109, 101, 115, 115, 97, 103, 101, 32, 52, 32, 40, 34, 34, 32, 45, 62, 32, 34, 34, 44, 32, 97, 110, 111, 110, 32, 98, 114, 111, 97, 100, 99, 97, 115, 116, 41]
|
||||
Padding = []
|
||||
PowTime = 20
|
||||
PowTarget = 0.01
|
||||
TargetPeer = ""
|
||||
```
|
||||
|
||||
- `YAML`
|
||||
|
||||
```yaml
|
||||
envelope: 0x156789a7892cfa5b5a45ca9f3187799d2a0293034175093ab8726ecbd8cbc6c6
|
||||
protocol: RPC
|
||||
from_device: 0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3
|
||||
to_device: ""
|
||||
payload: ""
|
||||
direction: OutgoingMessage
|
||||
sent_time: 1970-01-01T01:00:00+01:00
|
||||
timeline:
|
||||
- when: 2017-11-01T15:35:28.139031412+01:00
|
||||
status: Pending
|
||||
- when: 2017-11-01T15:35:32.226407285+01:00
|
||||
status: Pending
|
||||
- when: 2017-11-01T15:35:33.357611609+01:00
|
||||
status: Pending
|
||||
- when: 2017-11-01T15:35:34.470666086+01:00
|
||||
status: Pending
|
||||
request:
|
||||
symkeyid: d29dd5c6470d556c20e69b0f01b269b00c74e363cae442628171256a6df5ce38
|
||||
publickey: []
|
||||
sig: ""
|
||||
ttl: 20
|
||||
topic: "0x0f1a4771"
|
||||
payload:
|
||||
- 116
|
||||
- 101
|
||||
- 115
|
||||
- 116
|
||||
- 32
|
||||
- 109
|
||||
- 101
|
||||
- 115
|
||||
- 115
|
||||
- 97
|
||||
- 103
|
||||
- 101
|
||||
- 32
|
||||
- 52
|
||||
- 32
|
||||
- 40
|
||||
- 34
|
||||
- 34
|
||||
- 32
|
||||
- 45
|
||||
- 62
|
||||
- 32
|
||||
- 34
|
||||
- 34
|
||||
- 44
|
||||
- 32
|
||||
- 97
|
||||
- 110
|
||||
- 111
|
||||
- 110
|
||||
- 32
|
||||
- 98
|
||||
- 114
|
||||
- 111
|
||||
- 97
|
||||
- 100
|
||||
- 99
|
||||
- 97
|
||||
- 115
|
||||
- 116
|
||||
- 41
|
||||
padding: []
|
||||
powtime: 20
|
||||
powtarget: 0.01
|
||||
targetpeer: ""
|
||||
```
|
||||
|
||||
- `JSON`
|
||||
|
||||
```json
|
||||
{
|
||||
"envelope": "0xa1182bd04fd5d60717b3ae0d19569a65b00340f2fb1ba31b99873eb4b34a66f0",
|
||||
"protocol": "RPC",
|
||||
"envelope": "#4T0NEM0Qrcm1uVVhBVmhYYW9HS1AwQ01xWmhwQmJGSDBnOHZpWnN2",
|
||||
"from_device": "4T0NEM0Qrcm1uVVhBVmhYYW9HS1A",
|
||||
"to_device": "4T0NEM0Qrcm1uVVhBVmhYYW9HS1A",
|
||||
"time": 2010-11-12,
|
||||
"timelines": [
|
||||
"from_device": "",
|
||||
"to_device": "",
|
||||
"payload": "",
|
||||
"direction": "OutgoingMessage",
|
||||
"sent_time": "2017-11-01T15:35:34+01:00",
|
||||
"timeline": [
|
||||
{
|
||||
"time_stamp": "2010-11-12 10:20:100T34343",
|
||||
"status": "pending"
|
||||
"when": "2017-11-01T15:35:34.470932867+01:00",
|
||||
"status": "Sent"
|
||||
},
|
||||
{
|
||||
"time_stamp": "2010-11-12 3:20:100T34343",
|
||||
"status": "rejected",
|
||||
"reason": "Failed to locate peer"
|
||||
"when": "2017-11-01T15:35:34.47104423+01:00",
|
||||
"status": "Queued"
|
||||
},
|
||||
{
|
||||
"when": "2017-11-01T15:35:34.480469935+01:00",
|
||||
"status": "Rejected",
|
||||
"reason": "processing message: does not match"
|
||||
},
|
||||
{
|
||||
"when": "2017-11-01T15:35:34.471033731+01:00",
|
||||
"status": "Cached"
|
||||
},
|
||||
{
|
||||
"when": "2017-11-01T15:35:34.48042816+01:00",
|
||||
"status": "Processing"
|
||||
},
|
||||
{
|
||||
"when": "2017-11-01T15:35:34.480454857+01:00",
|
||||
"status": "Rejected",
|
||||
"reason": "Envelope failed to be opened"
|
||||
}
|
||||
],
|
||||
"payload": "Thunder crash",
|
||||
"request": {
|
||||
"symKeyID": "d29dd5c6470d556c20e69b0f01b269b00c74e363cae442628171256a6df5ce38",
|
||||
"pubKey": "0x",
|
||||
"sig": "",
|
||||
"ttl": 20,
|
||||
"topic": "0x0f1a4771",
|
||||
"payload": "0x74657374206d657373616765203420282222202d3e2022222c20616e6f6e2062726f61646361737429",
|
||||
"padding": "0x",
|
||||
"powTime": 20,
|
||||
"powTarget": 0.01,
|
||||
"targetPeer": ""
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{"envelope":"0x156789a7892cfa5b5a45ca9f3187799d2a0293034175093ab8726ecbd8cbc6c6","protocol":"RPC","from_device":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","to_device":"","payload":"","direction":"OutgoingMessage","sent":"1970-01-01T01:00:00+01:00","timeline":[{"when":"2017-11-01T15:35:28.139031412+01:00","status":"Pending"},{"when":"2017-11-01T15:35:32.226407285+01:00","status":"Pending"},{"when":"2017-11-01T15:35:33.357611609+01:00","status":"Pending"},{"when":"2017-11-01T15:35:34.470666086+01:00","status":"Pending"}],"request":{"symKeyID":"d29dd5c6470d556c20e69b0f01b269b00c74e363cae442628171256a6df5ce38","pubKey":"0x","sig":"","ttl":20,"topic":"0x0f1a4771","payload":"0x74657374206d657373616765203420282222202d3e2022222c20616e6f6e2062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0x5a1a014659f3f5c58663131c13dfd0a516ed76bccd49977ac276b53e993df9f3","protocol":"RPC","from_device":"","to_device":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","payload":"","direction":"IncomingMessage","sent":"2017-11-01T15:35:28+01:00","timeline":[{"when":"2017-11-01T15:35:28.139825712+01:00","status":"Queued"},{"when":"2017-11-01T15:35:28.139818943+01:00","status":"Cached"},{"when":"2017-11-01T15:35:28.139775017+01:00","status":"Sent"},{"when":"2017-11-01T15:35:28.139858406+01:00","status":"Processing"},{"when":"2017-11-01T15:35:28.140230953+01:00","status":"Delivered"}],"request":{"symKeyID":"","pubKey":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","sig":"","ttl":20,"topic":"0x74f4dc66","payload":"0x74657374206d657373616765203520282222202d3e204b312c20656e6372797074656420616e6f6e2062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0xbe91585960ff848dd8b80b0dce60b0a65968737c88eb56efd52719f662c48fc2","protocol":"RPC","from_device":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","to_device":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","payload":"","direction":"OutgoingMessage","sent":"2017-11-01T15:35:32+01:00","timeline":[{"when":"2017-11-01T15:35:32.235979768+01:00","status":"Sent"},{"when":"2017-11-01T15:35:32.236015721+01:00","status":"Cached"},{"when":"2017-11-01T15:35:32.236022035+01:00","status":"Queued"},{"when":"2017-11-01T15:35:32.236199728+01:00","status":"Processing"},{"when":"2017-11-01T15:35:32.236958622+01:00","status":"Delivered"}],"request":{"symKeyID":"","pubKey":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","sig":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","ttl":20,"topic":"0x466a89bd","payload":"0x74657374206d657373616765203120284b31202d3e204b322c207369676e65642b656e637279707465642c2066726f6d20757329","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0x2c3b9fb2914c757b8889f9ba6c1a4f588fb930393e83f7b0af683a581db92cd0","protocol":"RPC","from_device":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","to_device":"","payload":"","direction":"IncomingMessage","sent":"2017-11-01T15:35:33+01:00","timeline":[{"when":"2017-11-01T15:35:33.365935611+01:00","status":"Cached"},{"when":"2017-11-01T15:35:33.365944059+01:00","status":"Queued"},{"when":"2017-11-01T15:35:33.365899397+01:00","status":"Sent"},{"when":"2017-11-01T15:35:33.366048829+01:00","status":"Processing"},{"when":"2017-11-01T15:35:33.366052298+01:00","status":"Rejected","reason":"processing message: does not match"},{"when":"2017-11-01T15:35:33.366524827+01:00","status":"Delivered"}],"request":{"symKeyID":"77791bc0096005426e75f1e56cb28c8fe50e3b8b9bcd8d3361958fff3ab803d7","pubKey":"0x","sig":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","ttl":20,"topic":"0x1c623ba1","payload":"0x74657374206d657373616765203320284b31202d3e2022222c207369676e65642062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0xa1182bd04fd5d60717b3ae0d19569a65b00340f2fb1ba31b99873eb4b34a66f0","protocol":"RPC","from_device":"","to_device":"","payload":"","direction":"OutgoingMessage","sent":"2017-11-01T15:35:34+01:00","timeline":[{"when":"2017-11-01T15:35:34.470932867+01:00","status":"Sent"},{"when":"2017-11-01T15:35:34.47104423+01:00","status":"Queued"},{"when":"2017-11-01T15:35:34.480469935+01:00","status":"Rejected","reason":"processing message: does not match"},{"when":"2017-11-01T15:35:34.471033731+01:00","status":"Cached"},{"when":"2017-11-01T15:35:34.48042816+01:00","status":"Processing"},{"when":"2017-11-01T15:35:34.480454857+01:00","status":"Rejected","reason":"Envelope failed to be opened"}],"request":{"symKeyID":"d29dd5c6470d556c20e69b0f01b269b00c74e363cae442628171256a6df5ce38","pubKey":"0x","sig":"","ttl":20,"topic":"0x0f1a4771","payload":"0x74657374206d657373616765203420282222202d3e2022222c20616e6f6e2062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0x156789a7892cfa5b5a45ca9f3187799d2a0293034175093ab8726ecbd8cbc6c6","protocol":"RPC","from_device":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","to_device":"","payload":"","direction":"OutgoingMessage","sent_time":"1970-01-01T01:00:00+01:00","timeline":[{"when":"2017-11-01T15:35:28.139031412+01:00","status":"Pending"},{"when":"2017-11-01T15:35:32.226407285+01:00","status":"Pending"},{"when":"2017-11-01T15:35:33.357611609+01:00","status":"Pending"},{"when":"2017-11-01T15:35:34.470666086+01:00","status":"Pending"}],"request":{"symKeyID":"d29dd5c6470d556c20e69b0f01b269b00c74e363cae442628171256a6df5ce38","pubKey":"0x","sig":"","ttl":20,"topic":"0x0f1a4771","payload":"0x74657374206d657373616765203420282222202d3e2022222c20616e6f6e2062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0x5a1a014659f3f5c58663131c13dfd0a516ed76bccd49977ac276b53e993df9f3","protocol":"RPC","from_device":"","to_device":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","payload":"","direction":"IncomingMessage","sent_time":"2017-11-01T15:35:28+01:00","timeline":[{"when":"2017-11-01T15:35:28.139825712+01:00","status":"Queued"},{"when":"2017-11-01T15:35:28.139818943+01:00","status":"Cached"},{"when":"2017-11-01T15:35:28.139775017+01:00","status":"Sent"},{"when":"2017-11-01T15:35:28.139858406+01:00","status":"Processing"},{"when":"2017-11-01T15:35:28.140230953+01:00","status":"Delivered"}],"request":{"symKeyID":"","pubKey":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","sig":"","ttl":20,"topic":"0x74f4dc66","payload":"0x74657374206d657373616765203520282222202d3e204b312c20656e6372797074656420616e6f6e2062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0xbe91585960ff848dd8b80b0dce60b0a65968737c88eb56efd52719f662c48fc2","protocol":"RPC","from_device":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","to_device":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","payload":"","direction":"OutgoingMessage","sent_time":"2017-11-01T15:35:32+01:00","timeline":[{"when":"2017-11-01T15:35:32.235979768+01:00","status":"Sent"},{"when":"2017-11-01T15:35:32.236015721+01:00","status":"Cached"},{"when":"2017-11-01T15:35:32.236022035+01:00","status":"Queued"},{"when":"2017-11-01T15:35:32.236199728+01:00","status":"Processing"},{"when":"2017-11-01T15:35:32.236958622+01:00","status":"Delivered"}],"request":{"symKeyID":"","pubKey":"0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2","sig":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","ttl":20,"topic":"0x466a89bd","payload":"0x74657374206d657373616765203120284b31202d3e204b322c207369676e65642b656e637279707465642c2066726f6d20757329","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0x2c3b9fb2914c757b8889f9ba6c1a4f588fb930393e83f7b0af683a581db92cd0","protocol":"RPC","from_device":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","to_device":"","payload":"","direction":"IncomingMessage","sent_time":"2017-11-01T15:35:33+01:00","timeline":[{"when":"2017-11-01T15:35:33.365935611+01:00","status":"Cached"},{"when":"2017-11-01T15:35:33.365944059+01:00","status":"Queued"},{"when":"2017-11-01T15:35:33.365899397+01:00","status":"Sent"},{"when":"2017-11-01T15:35:33.366048829+01:00","status":"Processing"},{"when":"2017-11-01T15:35:33.366052298+01:00","status":"Rejected","reason":"processing message: does not match"},{"when":"2017-11-01T15:35:33.366524827+01:00","status":"Delivered"}],"request":{"symKeyID":"77791bc0096005426e75f1e56cb28c8fe50e3b8b9bcd8d3361958fff3ab803d7","pubKey":"0x","sig":"0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafadcbf3dce3716253fbc391277fa5a086b60b283daf61fb5b1f26895f456c2f31ae3","ttl":20,"topic":"0x1c623ba1","payload":"0x74657374206d657373616765203320284b31202d3e2022222c207369676e65642062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
{"envelope":"0xa1182bd04fd5d60717b3ae0d19569a65b00340f2fb1ba31b99873eb4b34a66f0","protocol":"RPC","from_device":"","to_device":"","payload":"","direction":"OutgoingMessage","sent_time":"2017-11-01T15:35:34+01:00","timeline":[{"when":"2017-11-01T15:35:34.470932867+01:00","status":"Sent"},{"when":"2017-11-01T15:35:34.47104423+01:00","status":"Queued"},{"when":"2017-11-01T15:35:34.480469935+01:00","status":"Rejected","reason":"processing message: does not match"},{"when":"2017-11-01T15:35:34.471033731+01:00","status":"Cached"},{"when":"2017-11-01T15:35:34.48042816+01:00","status":"Processing"},{"when":"2017-11-01T15:35:34.480454857+01:00","status":"Rejected","reason":"Envelope failed to be opened"}],"request":{"symKeyID":"d29dd5c6470d556c20e69b0f01b269b00c74e363cae442628171256a6df5ce38","pubKey":"0x","sig":"","ttl":20,"topic":"0x0f1a4771","payload":"0x74657374206d657373616765203420282222202d3e2022222c20616e6f6e2062726f61646361737429","padding":"0x","powTime":20,"powTarget":0.01,"targetPeer":""}}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from_device = "0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeaf
|
|||
to_device = ""
|
||||
payload = ""
|
||||
direction = "OutgoingMessage"
|
||||
sent = 1970-01-01T00:00:00Z
|
||||
sent_time = 1970-01-01T00:00:00Z
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:28Z
|
||||
|
|
@ -39,7 +39,7 @@ from_device = ""
|
|||
to_device = "0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2"
|
||||
payload = ""
|
||||
direction = "IncomingMessage"
|
||||
sent = 2017-11-01T14:35:28Z
|
||||
sent_time = 2017-11-01T14:35:28Z
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:28Z
|
||||
|
|
@ -78,7 +78,7 @@ from_device = "0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeaf
|
|||
to_device = "0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2"
|
||||
payload = ""
|
||||
direction = "OutgoingMessage"
|
||||
sent = 2017-11-01T14:35:32Z
|
||||
sent_time = 2017-11-01T14:35:32Z
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:32Z
|
||||
|
|
@ -117,7 +117,7 @@ from_device = "0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeaf
|
|||
to_device = ""
|
||||
payload = ""
|
||||
direction = "IncomingMessage"
|
||||
sent = 2017-11-01T14:35:33Z
|
||||
sent_time = 2017-11-01T14:35:33Z
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:33Z
|
||||
|
|
@ -161,7 +161,7 @@ from_device = ""
|
|||
to_device = ""
|
||||
payload = ""
|
||||
direction = "OutgoingMessage"
|
||||
sent = 2017-11-01T14:35:34Z
|
||||
sent_time = 2017-11-01T14:35:34Z
|
||||
|
||||
[[timeline]]
|
||||
when = 2017-11-01T14:35:34Z
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from_device: 0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafad
|
|||
to_device: ""
|
||||
payload: ""
|
||||
direction: OutgoingMessage
|
||||
sent: 1970-01-01T01:00:00+01:00
|
||||
sent_time: 1970-01-01T01:00:00+01:00
|
||||
timeline:
|
||||
- when: 2017-11-01T15:35:28.139031412+01:00
|
||||
status: Pending
|
||||
|
|
@ -72,7 +72,7 @@ from_device: ""
|
|||
to_device: 0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2
|
||||
payload: ""
|
||||
direction: IncomingMessage
|
||||
sent: 2017-11-01T15:35:28+01:00
|
||||
sent_time: 2017-11-01T15:35:28+01:00
|
||||
timeline:
|
||||
- when: 2017-11-01T15:35:28.139825712+01:00
|
||||
status: Queued
|
||||
|
|
@ -217,7 +217,7 @@ from_device: 0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafad
|
|||
to_device: 0x0490161b00f2c47542d28c2e8908e77159b1720dccceb6393d7c001850122efc3b1709bcea490fd8f5634ba1a145aa0722d86b9330b0e39a8d493cb981fd459da2
|
||||
payload: ""
|
||||
direction: OutgoingMessage
|
||||
sent: 2017-11-01T15:35:32+01:00
|
||||
sent_time: 2017-11-01T15:35:32+01:00
|
||||
timeline:
|
||||
- when: 2017-11-01T15:35:32.235979768+01:00
|
||||
status: Sent
|
||||
|
|
@ -363,7 +363,7 @@ from_device: 0x04eedbaafd6adf4a9233a13e7b1c3c14461fffeba2e9054b8d456ce5f6ebeafad
|
|||
to_device: ""
|
||||
payload: ""
|
||||
direction: IncomingMessage
|
||||
sent: 2017-11-01T15:35:33+01:00
|
||||
sent_time: 2017-11-01T15:35:33+01:00
|
||||
timeline:
|
||||
- when: 2017-11-01T15:35:33.365935611+01:00
|
||||
status: Cached
|
||||
|
|
@ -438,7 +438,7 @@ from_device: ""
|
|||
to_device: ""
|
||||
payload: ""
|
||||
direction: OutgoingMessage
|
||||
sent: 2017-11-01T15:35:34+01:00
|
||||
sent_time: 2017-11-01T15:35:34+01:00
|
||||
timeline:
|
||||
- when: 2017-11-01T15:35:34.470932867+01:00
|
||||
status: Sent
|
||||
|
|
|
|||
28
main.go
28
main.go
|
|
@ -4,7 +4,6 @@ import (
|
|||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
|
@ -25,15 +24,6 @@ var (
|
|||
version = flag.Bool("v", false, "Print version")
|
||||
)
|
||||
|
||||
type wopCloser struct {
|
||||
io.Writer
|
||||
}
|
||||
|
||||
// Close does nothing.
|
||||
func (wopCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Usage = printUsage
|
||||
flag.Parse()
|
||||
|
|
@ -49,7 +39,8 @@ func main() {
|
|||
return
|
||||
}
|
||||
|
||||
var wc io.WriteCloser
|
||||
rc := io.ReadCloser(os.Stdin)
|
||||
wc := io.WriteCloser(os.Stdout)
|
||||
|
||||
if *targetFile != "" {
|
||||
targetOutputFile, err := os.Create(*targetFile)
|
||||
|
|
@ -59,12 +50,8 @@ func main() {
|
|||
}
|
||||
|
||||
wc = targetOutputFile
|
||||
} else {
|
||||
wc = wopCloser{Writer: os.Stdout}
|
||||
}
|
||||
|
||||
var rc io.ReadCloser
|
||||
|
||||
if *sourceFile != "" {
|
||||
targetInputFile, err := os.OpenFile(*sourceFile, os.O_RDONLY, 0700)
|
||||
if err != nil {
|
||||
|
|
@ -73,8 +60,6 @@ func main() {
|
|||
}
|
||||
|
||||
rc = targetInputFile
|
||||
} else {
|
||||
rc = ioutil.NopCloser(os.Stdin)
|
||||
}
|
||||
|
||||
if err := stats.ReadAggregates(rc, wc, *format); err != nil {
|
||||
|
|
@ -89,6 +74,7 @@ func main() {
|
|||
}
|
||||
}
|
||||
|
||||
// printVersion prints corresponding build version with associated build stamp and git commit if provided.
|
||||
func printVersion() {
|
||||
var vers []string
|
||||
vers = append(vers, versionStamp)
|
||||
|
|
@ -104,6 +90,7 @@ func printVersion() {
|
|||
fmt.Fprint(os.Stdout, strings.Join(vers, " "))
|
||||
}
|
||||
|
||||
// printUsage prints out usage message for CLI tool.
|
||||
func printUsage() {
|
||||
fmt.Fprintf(os.Stdout, `Usage: msgstat [options]
|
||||
Msgstat processes status-go lines to generate useful message delivery facts.
|
||||
|
|
@ -131,12 +118,9 @@ FLAGS:
|
|||
func hasIncomingData() bool {
|
||||
stat, err := os.Stdin.Stat()
|
||||
if err != nil {
|
||||
log.Printf("Error: unable to retrieve `stat` for os.Stdin: %+q", err)
|
||||
return false
|
||||
}
|
||||
|
||||
if stat.Size() == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
return stat.Size() > 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
|
@ -25,23 +26,35 @@ var (
|
|||
|
||||
// Timeline contains timeline and status string for an aggregated message.
|
||||
type Timeline struct {
|
||||
When time.Time `json:"when" toml:"when" yaml:"when" csv:"when"`
|
||||
Status string `json:"status" toml:"status" yaml:"status" csv:"status"`
|
||||
Error string `json:"reason,omitempty" toml:"reason,omitempty" yaml:"error,omitempty" csv:"error,omitempty"`
|
||||
// When defines the time when the notification was delivered.
|
||||
When time.Time `json:"when" toml:"when" yaml:"when"`
|
||||
// Status defines the current status of the element.
|
||||
Status string `json:"status" toml:"status" yaml:"status"`
|
||||
// Error defines the possible reason the message was rejected.
|
||||
Error string `json:"reason,omitempty" toml:"reason,omitempty" yaml:"error,omitempty"`
|
||||
}
|
||||
|
||||
// AggregatedMessage contains aggregated facts for a given message envelope and it's total
|
||||
// transition.
|
||||
type AggregatedMessage struct {
|
||||
Envelope string `json:"envelope" toml:"envelope" yaml:"envelope" csv:"envelope"`
|
||||
Protocol string `json:"protocol" toml:"protocol" yaml:"protocol" csv:"protocol"`
|
||||
FromDevice string `json:"from_device" toml:"from_device" yaml:"from_device" csv:"from_device"`
|
||||
ToDevice string `json:"to_device" toml:"to_device" yaml:"to_device" csv:"to_device"`
|
||||
Payload string `json:"payload" toml:"payload" yaml:"payload" csv:"payload"`
|
||||
Direction string `json:"direction" toml:"direction" yaml:"direction" csv:"direction"`
|
||||
Sent time.Time `json:"sent" toml:"sent" yaml:"sent" csv:"sent"`
|
||||
Timelines []Timeline `json:"timeline" toml:"timeline" yaml:"timeline" csv:"timeline"`
|
||||
Request whisper.NewMessage `json:"request" toml:"request" yaml:"request" csv:"request"`
|
||||
// Envelope holds the hash corresponding to the sent message.
|
||||
Envelope string `json:"envelope" toml:"envelope" yaml:"envelope"`
|
||||
// Protocol defines the means of transmission which is either P2P or RPC.
|
||||
Protocol string `json:"protocol" toml:"protocol" yaml:"protocol"`
|
||||
// FromDevice defines the public key hash which sent the message.
|
||||
FromDevice string `json:"from_device" toml:"from_device" yaml:"from_device"`
|
||||
// ToDevice defines the public key hash which will receive the message.
|
||||
ToDevice string `json:"to_device" toml:"to_device" yaml:"to_device"`
|
||||
// Payload defines the data attached to the message.
|
||||
Payload string `json:"payload" toml:"payload" yaml:"payload"`
|
||||
// Direction defines the route which a message comes through initially.
|
||||
Direction string `json:"direction" toml:"direction" yaml:"direction"`
|
||||
// Sent defines the time which the message was sent through the network.
|
||||
Sent time.Time `json:"sent_time" toml:"sent_time" yaml:"sent_time"`
|
||||
// Timeline defines the series of delivery changes that the message when through, this can be multiples in case of resending.
|
||||
Timeline []Timeline `json:"timeline" toml:"timeline" yaml:"timeline"`
|
||||
// Request defines the original whisper.NewMessage for this message, if found.
|
||||
Request whisper.NewMessage `json:"request" toml:"request" yaml:"request"`
|
||||
}
|
||||
|
||||
// ReadAggregates processes incoming data from the reader and writes appropriate
|
||||
|
|
@ -52,6 +65,37 @@ func ReadAggregates(r io.ReadCloser, w io.WriteCloser, format string) error {
|
|||
|
||||
format = strings.ToLower(format)
|
||||
|
||||
aggregates, err := parseLogReader(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, aggr := range aggregates {
|
||||
switch format {
|
||||
case "toml":
|
||||
if err := toml.NewEncoder(w).Encode(aggr); err != nil {
|
||||
return err
|
||||
}
|
||||
case "json":
|
||||
if err := json.NewEncoder(w).Encode(aggr); err != nil {
|
||||
return err
|
||||
}
|
||||
case "yaml":
|
||||
ymlData, err := yaml.Marshal(aggr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.Write(ymlData)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseLogReader parses out all log messages and returns associated AggregatedMessages
|
||||
// or returns error if encountered.
|
||||
func parseLogReader(r io.ReadCloser) ([]AggregatedMessage, error) {
|
||||
var order []string
|
||||
aggregates := make(map[string]AggregatedMessage)
|
||||
|
||||
|
|
@ -65,31 +109,25 @@ func ReadAggregates(r io.ReadCloser, w io.WriteCloser, format string) error {
|
|||
break
|
||||
}
|
||||
|
||||
return err
|
||||
log.Printf("Error occured while reading from Reader: %+q", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !messageHeader.MatchString(string(line)) {
|
||||
logLine := string(line)
|
||||
if !messageHeader.MatchString(logLine) {
|
||||
continue
|
||||
}
|
||||
|
||||
logLine := messageHeader.ReplaceAllLiteralString(string(line), "")
|
||||
logLine = strings.TrimSpace(logLine)
|
||||
logLine = strings.TrimPrefix(logLine, "geth=StatusIM")
|
||||
logLine = strings.TrimSpace(logLine)
|
||||
logLine = strings.TrimPrefix(logLine, "state=")
|
||||
|
||||
if qlogLine, derr := strconv.Unquote(logLine); derr == nil {
|
||||
logLine = qlogLine
|
||||
}
|
||||
|
||||
dataLog, err := base64.StdEncoding.DecodeString(logLine)
|
||||
dataLog, err := decodeLogLine(logLine)
|
||||
if err != nil {
|
||||
return err
|
||||
log.Printf("Error occured while parsing logline: %+q, %+q", err, logLine)
|
||||
continue
|
||||
}
|
||||
|
||||
var message common.MessageState
|
||||
if err := json.Unmarshal([]byte(dataLog), &message); err != nil {
|
||||
return err
|
||||
log.Printf("Error occured while unmarshaling logline to common.Message: %+q, %+q", err, dataLog)
|
||||
continue
|
||||
}
|
||||
|
||||
msgAggr, ok := aggregates[message.Hash]
|
||||
|
|
@ -132,7 +170,7 @@ func ReadAggregates(r io.ReadCloser, w io.WriteCloser, format string) error {
|
|||
msgAggr.Request = message.Source
|
||||
}
|
||||
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Pending",
|
||||
When: message.Received,
|
||||
})
|
||||
|
|
@ -141,22 +179,22 @@ func ReadAggregates(r io.ReadCloser, w io.WriteCloser, format string) error {
|
|||
msgAggr.Request = message.Source
|
||||
}
|
||||
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Sent",
|
||||
When: message.Received,
|
||||
})
|
||||
case "Resent":
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Resent",
|
||||
When: message.Received,
|
||||
})
|
||||
case "Queued":
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Queued",
|
||||
When: message.Received,
|
||||
})
|
||||
case "Cached":
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Cached",
|
||||
When: message.Received,
|
||||
})
|
||||
|
|
@ -169,18 +207,18 @@ func ReadAggregates(r io.ReadCloser, w io.WriteCloser, format string) error {
|
|||
msgAggr.FromDevice = message.FromDevice
|
||||
}
|
||||
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Delivered",
|
||||
When: message.Received,
|
||||
})
|
||||
case "Rejected":
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Rejected",
|
||||
When: message.Received,
|
||||
Error: message.RejectionError,
|
||||
})
|
||||
case "Processing":
|
||||
msgAggr.Timelines = append(msgAggr.Timelines, Timeline{
|
||||
msgAggr.Timeline = append(msgAggr.Timeline, Timeline{
|
||||
Status: "Processing",
|
||||
When: message.Received,
|
||||
})
|
||||
|
|
@ -189,27 +227,31 @@ func ReadAggregates(r io.ReadCloser, w io.WriteCloser, format string) error {
|
|||
aggregates[message.Hash] = msgAggr
|
||||
}
|
||||
|
||||
var messages []AggregatedMessage
|
||||
for _, hash := range order {
|
||||
aggr := aggregates[hash]
|
||||
|
||||
switch format {
|
||||
case "toml":
|
||||
if err := toml.NewEncoder(w).Encode(aggr); err != nil {
|
||||
return err
|
||||
}
|
||||
case "json":
|
||||
if err := json.NewEncoder(w).Encode(aggr); err != nil {
|
||||
return err
|
||||
}
|
||||
case "yaml":
|
||||
ymlData, err := yaml.Marshal(aggr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.Write(ymlData)
|
||||
}
|
||||
messages = append(messages, aggregates[hash])
|
||||
}
|
||||
|
||||
return nil
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// decodeLogLine returns a decoded base64 status message which it strips off
|
||||
// un-needed fields and uses the `state` field for it's decoding.
|
||||
func decodeLogLine(line string) (string, error) {
|
||||
logLine := messageHeader.ReplaceAllLiteralString(line, "")
|
||||
logLine = strings.TrimSpace(logLine)
|
||||
logLine = strings.TrimPrefix(logLine, "geth=StatusIM")
|
||||
logLine = strings.TrimSpace(logLine)
|
||||
logLine = strings.TrimPrefix(logLine, "state=")
|
||||
|
||||
if qlogLine, derr := strconv.Unquote(logLine); derr == nil {
|
||||
logLine = qlogLine
|
||||
}
|
||||
|
||||
decoded, err := base64.StdEncoding.DecodeString(logLine)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(decoded), nil
|
||||
}
|
||||
Loading…
Reference in New Issue