add script for easier setup of mailserver

Signed-off-by: Jakub Sokołowski <jakub@status.im>
This commit is contained in:
Jakub Sokołowski 2019-09-16 12:38:33 +03:00 committed by Jakub
parent a2f106e4c5
commit 294399916e
21 changed files with 393 additions and 31 deletions

1
.gitignore vendored
View File

@ -68,5 +68,4 @@ Session.vim
/cmd/*/.ethereum/
# created for running container
_assets/compose/mailserver/config.json
_assets/compose/bootnode/keys

26
MAILSERVER.md Normal file
View File

@ -0,0 +1,26 @@
# Description
This document describes the two easiest ways to start a Status Mailserver:
* [Docker Compose](https://docs.docker.com/compose/) - More self-contained and portable
* [Systemd Service](https://www.freedesktop.org/wiki/Software/systemd/) - More local and configurable
## Docker Compose
The simplest way is to just use:
```
make run-mailserver-docker
```
This will generate the necessary config, compose and then start the container.
For more details read the [README](_assets/compose/mailserver/README.md).
## Systemd Service
The other way is to run the `mailserver` under `systemd`:
```
make run-mailserver-systemd
```
This will generate the necessary config, define and then start the service.
For more details read the [README](_assets/systemd/README.md).

View File

@ -278,7 +278,7 @@ ci: lint canary-test test-unit test-e2e ##@tests Run all linters and tests at on
ci-race: lint canary-test test-unit test-e2e-race ##@tests Run all linters and tests at once + race
clean: ##@other Cleanup
rm -fr build/bin/*
rm -fr build/bin/* mailserver-config.json
deep-clean: clean
rm -Rdf .ethereumtest/StatusChain
@ -298,8 +298,11 @@ update-fleet-config: ##@other Update fleets configuration from fleets.status.im
@go generate ./static
@echo "Done"
run-bootnode: ##@Easy way to run a bootnode locally with Docker Compose
run-bootnode-docker: ##@Easy way to run a bootnode locally with Docker Compose
cd _assets/compose/bootnode/ && $(MAKE)
run-mailserver: ##@Easy way to run a mailserver locally with Docker Compose
cd _assets/compose/mailserver/ && $(MAKE)
run-mailserver-systemd: ##@Easy way to run a mailserver locally with systemd
@_assets/systemd/start.sh
run-mailserver-docker: ##@Easy way to run a mailserver locally with Docker Compose
@cd _assets/compose/mailserver/ && $(MAKE)

View File

@ -9,7 +9,7 @@
- [How to Contribute](CONTRIBUTING.md)
- [How to Release](RELEASING.md)
- [How to run a Bootnode](_assets/compose/bootnode)
- [How to run a Mailserver](_assets/compose/mailserver)
- [How to run a Mailserver](MAILSERVER.md)
- [How to configure status-go](/config)
# License

View File

@ -1,4 +1,4 @@
GIT_ROOT = $(shell git rev-parse --show-toplevel)
export GIT_ROOT = $(shell git rev-parse --show-toplevel)
RED := $(shell tput -Txterm setaf 1)
GREEN := $(shell tput -Txterm setaf 2)
@ -45,12 +45,10 @@ logs:
docker-compose logs -f -t --tail=100
enode:
@curl -s -XPOST http://localhost:$(RPC_PORT)/ \
-H 'Content-type: application/json' \
-d '{"jsonrpc":"2.0","method":"admin_nodeInfo","params":[],"id":1}' \
| jq -r '.result.enode' \
| grep -oP '\Kenode://[^?]+'
@$(GIT_ROOT)/_assets/scripts/get_enode.sh
config:
@$(GIT_ROOT)/_assets/scripts/gen_config.sh
info:
@echo "$(GREEN)Your mailserver is listening on:$(RESET) $(BOLD)$(PUBLIC_IP):$(LISTEN_PORT)$(RESET)"
@ -58,22 +56,9 @@ info:
@echo "$(GREEN)Your enode address is:$(RESET)"
show:
@docker ps --filter='name=$(CONTAINER_NAME)' --format="table {{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Ports}}"
config: ##@ Generate config for mailserver with current public IP.
@cat $(GIT_ROOT)/config/cli/fleet-$(FLEET_NAME).json \
| jq '.AdvertiseAddr = "$(PUBLIC_IP)"' \
| jq '.HTTPEnabled = true' \
| jq '.HTTPHost = "0.0.0.0"' \
| jq '.HTTPPort= $(RPC_PORT)' \
| jq '.APIModules = "$(API_MODULES)"' \
| jq '.RegisterTopics = ["$(REGISTER_TOPIC)"]' \
| jq '.WhisperConfig.Enabled = true' \
| jq '.WhisperConfig.EnableMailServer = true' \
| jq '.WhisperConfig.LightClient = false' \
| jq '.WhisperConfig.MailServerPassword = "$(MAIL_PASSWORD)"' \
> config.json
@docker ps \
--filter='name=$(CONTAINER_NAME)' \
--format="table {{.ID}}\t{{.Names}}\t{{.Status}}\t{{.Ports}}"
clean:
rm -f config.json
docker-compose rm -s -f

View File

@ -18,7 +18,7 @@ To simply start a container run `make`, other commands include:
* `make stop` - Stops the container.
* `make show` - Shows you current status of the container.
* `make logs` - Shows you logs of the container.
* `make config` - Creates `config.json` with your Public IP.
* `make config` - Creates `${DATA_PATH}/config.json` with your Public IP.
* `make enode` - Shows `enode://` address of the container.
# Settings
@ -35,6 +35,8 @@ All settings are passed through environment variables:
* `MAIL_PASSWORD` - Basic HTTP auth password for mailserver. (Default: `status-offline-inbox`)
* `LOG_LEVEL` - Set level of log messages to show. (`ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`)
The generated configuration file end up under `${DATA_PATH}/config.json`.
# Known Issues
If the discovery of your Public IP does not work please simply export the `PUBLIC_IP` env variable.
@ -50,7 +52,7 @@ bee56564926d status-go-mailserver Up 6 minutes 8080/tcp, 127.0.0
Your mailserver is listening on: 1.2.3.4:443
Make sure that address and port are available from the internet!
Your enode address is:
enode://dccd2f3c1df42c23af6672df28f287893ab70a5d45668637576a759b6db10b83e83fc02598f36c80ac094fbf8621419153cfe539f56d278ab099da21800f880c@127.0.0.1:30303
enode://dccd2f3c1df42c23af6672df28f287893ab70a5d45668637576a759b6db10b83e83fc02598f36c80ac094fbf8621419153cfe539f56d278ab099da21800f880c@1.2.3.4:30303
```
# F.A.Q.

View File

@ -11,5 +11,5 @@ services:
- '0.0.0.0:${LISTEN_PORT}:30303/tcp'
- '0.0.0.0:${LISTEN_PORT}:30303/udp'
volumes:
- '${PWD}/config.json:/config.json'
- '${DATA_PATH}/config.json:/config.json'
- '${DATA_PATH}:/data'

41
_assets/scripts/gen_config.sh Executable file
View File

@ -0,0 +1,41 @@
#!/usr/bin/env bash
GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
# Settings & defaults
RPC_PORT="${RPC_PORT:-8545}"
API_MODULES="${API_MODULES:-eth,net,web3,admin}"
FLEET_NAME="${FLEET_NAME:-eth.beta}"
REGISTER_TOPIC="${REGISTER_TOPIC:-whispermail}"
MAIL_PASSWORD="${MAIL_PASSWORD:-status-offline-inbox}"
DATA_PATH="${DATA_PATH:-/var/tmp/status-go-mail}"
CONFIG_PATH="${CONFIG_PATH:-${DATA_PATH}/config.json}"
if [[ -e "${CONFIG_PATH}" ]]; then
echo "Config already exits. Remove it to generate a new one."
exit 0
fi
# Necessary to make mailserver available publicly
export PUBLIC_IP=$(curl -s https://ipecho.net/plain)
# Assemble the filter for changing the config JSON
JQ_FILTER_ARRAY=(
".AdvertiseAddr = \"${PUBLIC_IP}\""
".HTTPEnabled = true"
".HTTPHost = \"0.0.0.0\""
".HTTPPort= ${RPC_PORT}"
".APIModules = \"${API_MODULES}\""
".RegisterTopics = [\"${REGISTER_TOPIC}\"]"
".WhisperConfig.Enabled = true"
".WhisperConfig.EnableMailServer = true"
".WhisperConfig.LightClient = false"
".WhisperConfig.MailServerPassword = \"${MAIL_PASSWORD}\""
)
JQ_FILTER=$(printf " | %s" "${JQ_FILTER_ARRAY[@]}")
echo "Generating config at: ${CONFIG_PATH}"
cat "${GIT_ROOT}/config/cli/fleet-${FLEET_NAME}.json" \
| jq "${JQ_FILTER:3}" > "${CONFIG_PATH}"

27
_assets/scripts/get_enode.sh Executable file
View File

@ -0,0 +1,27 @@
#!/usr/bin/env bash
RPC_ADDR="${RPC_ADDR:-localhost}"
RPC_PORT="${RPC_PORT:-8545}"
# might be provided by parent
if [[ -z "${PUBLIC_IP}" ]]; then
PUBLIC_IP=$(curl -s https://ipecho.net/plain)
fi
# query local
RESP_JSON=$(
curl -s -XPOST http://${RPC_ADDR}:${RPC_PORT}/ \
-H 'Content-type: application/json' \
-d '{"jsonrpc":"2.0","method":"admin_nodeInfo","params":[],"id":1}'
)
if [[ "$?" -ne 0 ]]; then
echo "RPC port not up, unable to query enode address!" 1>&2
exit 1
fi
# extract enode from JSON response
ENODE_RAW=$(echo "${RESP_JSON}" | jq -r '.result.enode')
# drop arguments at the end of enode address
ENODE_CLEAN=$(echo "${ENODE_RAW}" | grep -oP '\Kenode://[^?]+')
# replace localhost with public IP
echo "${ENODE_CLEAN}" | sed s/127.0.0.1/${PUBLIC_IP}/

52
_assets/systemd/README.md Normal file
View File

@ -0,0 +1,52 @@
# Status Mailserver
This folder contains setup for running your own Status Mailserver.
It uses [Systemd](https://www.freedesktop.org/wiki/Software/systemd/) for managing the Status Mailserver service.
The steps it takes are:
* Builds statusd
* Generates `statusd` config
* Generates `systemd` service
* Starts the service
# Usage
To simply configure and start the service run `./start.sh`.
In order to manage the new `statusd` service you use `systemctl` command:
* `systemctl --user start statusd` - Start the service
* `systemctl --user stop statusd` - Stop the service
* `systemctl --user status statusd` - Check service status
* `systemctl --user disable statusd` - Disable the service
* `journalctl --user-unit statusd` - Read the service logs
If you want to remove the service you can just remove its definition:
```
systemctl --user stop statusd
rm ~/.config/systemd/user/statusd.service
systemctl --user daemon-reload
```
# Settings
All settings are passed through environment variables:
* `SERVICE_NAME` - Name of the `systemd` service to be created. (Default: `statusd`)
* `PUBLIC_IP` - Your IP visible from the internet and advertised by the Mailserver.
* `LISTEN_PORT` - Mailserver TCP & UDP port, by default it's `30303` but you might want to use `443`.
* `RPC_PORT` - Control port making it possible to use the [JSON-RPC API](https://github.com/ethereum/wiki/wiki/JSON-RPC).
* `API_MODULES` - API modules to be made available via the `RPC_PORT`.
* `DATA_PATH` - Location of Mailserver storage and keys. (Default: `/var/tmp/status-go-mail`)
* `REGISTER_TOPIC` - Mynamic mailserver discovery topic. (Default: `whispermail`)
* `MAIL_PASSWORD` - Basic HTTP auth password for mailserver. (Default: `status-offline-inbox`)
* `LOG_LEVEL` - Set level of log messages to show. (`ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`)
The generated configuration file end up under `${DATA_PATH}/config.json`.
# Known Issues
* `No journal files were opened due to insufficient permissions.` from `systemctl`
- To see logs of a user systemd service you need to be a member of `systemd-journal` group.
- Use: `bash usermod -a -G systemd-journal ${USER}`

24
_assets/systemd/gen_service.sh Executable file
View File

@ -0,0 +1,24 @@
#!/usr/bin/env bash
GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
mkdir -p "${HOME}/.config/systemd/user"
cat >"${HOME}/.config/systemd/user/${SERVICE_NAME}.service" << EOF
[Unit]
Description=Status.im Mailserver Service
[Service]
Type=notify
Restart=on-failure
WatchdogSec=60s
WorkingDirectory=${DATA_PATH}
ExecStart=${GIT_ROOT}/build/bin/statusd \\
-log "${LOG_LEVEL}" \\
-log-without-color \\
-dir "${DATA_PATH}" \\
-c "./config.json"
[Install]
WantedBy=default.target
EOF

68
_assets/systemd/start.sh Executable file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env bash
RED=$(tput -Txterm setaf 1)
GRN=$(tput -Txterm setaf 2)
YLW=$(tput -Txterm setaf 3)
RST=$(tput -Txterm sgr0)
BLD=$(tput bold)
GIT_ROOT=$(cd "${BASH_SOURCE%/*}" && git rev-parse --show-toplevel)
# Settings & defaults
export SERVICE_NAME="${SERVICE_NAME:-statusd}"
export LOG_LEVEL="${LOG_LEVEL:-INFO}"
export LISTEN_PORT="${LISTEN_PORT:-30303}"
export DATA_PATH="${DATA_PATH:-/var/tmp/status-go-mail}"
# Necessary to make mailserver available publicly
export PUBLIC_IP=$(curl -s https://ipecho.net/plain)
function show_info() {
systemctl --user status --no-pager ${SERVICE_NAME}
echo
# just nice to show at the end
ENODE=$("${GIT_ROOT}/_assets/scripts/get_enode.sh")
echo "* ${GRN}Your mailserver is listening on:${RST} ${BLD}${PUBLIC_IP}:${LISTEN_PORT}${RST}"
echo "* ${YLW}Make sure that IP and TCP port are available from the internet!${RST}"
echo -e "${GRN}Your enode address is:${RST}\n${ENODE}"
exit 0
}
if ! [[ -x "$(command -v systemctl)" ]]; then
echo "${RED}Your system does not have systemd!${RST}"
exit 1
fi
# if the service is already up just show some info
if systemctl --user is-active --quiet ${SERVICE_NAME}; then
echo "${YLW}Service already started!${RST}"
show_info
fi
# if the service has failed just show the status
if systemctl --user is-failed --quiet ${SERVICE_NAME}; then
echo "${RED}Service has failed!${RST}"
systemctl --user status --no-pager ${SERVICE_NAME}
exit 1
fi
# Build the statusd binary
# TODO possibly download it in the future
if [[ ! -x "${GIT_ROOT}/build/bin/statusd" ]]; then
echo "* ${BLD}Building mailserver binary...${RST}"
cd "${GIT_ROOT}" && make statusgo
fi
echo "* ${BLD}Generating '${SERVICE_NAME}' config...${RST}"
"${GIT_ROOT}/_assets/scripts/gen_config.sh"
echo "* ${BLD}Generating '${SERVICE_NAME}' service...${RST}"
"${GIT_ROOT}/_assets/systemd/gen_service.sh"
echo "* ${BLD}Enabling '${SERVICE_NAME}' service...${RST}"
systemctl --user enable ${SERVICE_NAME}
echo "* ${BLD}Starting '${SERVICE_NAME}' service...${RST}"
systemctl --user restart ${SERVICE_NAME}
show_info

View File

@ -15,6 +15,7 @@ import (
"github.com/ethereum/go-ethereum/log"
gethmetrics "github.com/ethereum/go-ethereum/metrics"
"github.com/okzk/sdnotify"
"github.com/status-im/status-go/api"
"github.com/status-im/status-go/logutils"
nodemetrics "github.com/status-im/status-go/metrics/node"
@ -134,6 +135,16 @@ func main() {
return
}
err = sdnotify.Ready()
if err == sdnotify.ErrSdNotifyNoSocket {
logger.Debug("sd_notify socket not available")
} else if err != nil {
logger.Warn("sd_notify READY call failed", "error", err)
} else {
// systemd aliveness notifications, affects only Linux
go startSystemDWatchdog()
}
// handle interrupt signals
interruptCh := haltOnInterruptSignal(backend.StatusNode())
@ -168,6 +179,9 @@ func main() {
if gethNode != nil {
// wait till node has been stopped
gethNode.Wait()
if err := sdnotify.Stopping(); err != nil {
logger.Warn("sd_notify STOPPING call failed", "error", err)
}
}
}
@ -189,6 +203,15 @@ func setupLogging(config *params.NodeConfig) {
}
}
// loop for notifying systemd about process being alive
func startSystemDWatchdog() {
for range time.Tick(30 * time.Second) {
if err := sdnotify.Watchdog(); err != nil {
logger.Warn("sd_notify WATCHDOG call failed", "error", err)
}
}
}
// startCollectingStats collects various stats about the node and other protocols like Whisper.
func startCollectingNodeMetrics(interruptCh <-chan struct{}, statusNode *node.StatusNode) {
logger.Info("Starting collecting node metrics")

1
go.mod
View File

@ -25,6 +25,7 @@ require (
github.com/mutecomm/go-sqlcipher v0.0.0-20190227152316-55dbde17881f
github.com/pborman/uuid v1.2.0
github.com/prometheus/tsdb v0.10.0 // indirect
github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
github.com/russolsen/transit v0.0.0-20180705123435-0794b4c4505a
github.com/status-im/doubleratchet v2.0.0+incompatible
github.com/status-im/keycard-go v0.0.0-20190424133014-d95853db0f48 // indirect

2
go.sum
View File

@ -457,6 +457,8 @@ github.com/naoina/toml v0.0.0-20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G
github.com/nsf/termbox-go v0.0.0-20170211012700-3540b76b9c77/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170128050532-febf2d34b54a h1:m6hB6GkmZ/suOSKZM7yx3Yt+7iZ9HNfzacCykJqgXA8=
github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd h1:+iAPaTbi1gZpcpDwe/BW1fx7Xoesv69hLNGPheoyhBs=
github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd/go.mod h1:4soZNh0zW0LtYGdQ416i0jO0EIqMGcbtaspRS4BDvRQ=
github.com/olekukonko/tablewriter v0.0.0-20170128050532-febf2d34b54a/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

21
vendor/github.com/okzk/sdnotify/LICENSE generated vendored Normal file
View File

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 okzk
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

15
vendor/github.com/okzk/sdnotify/README.md generated vendored Normal file
View File

@ -0,0 +1,15 @@
# sdnotify
sd_notify utility for golang.
## Installation
go get github.com/okzk/sdnotify
## Example
see [sample/main.go](sample/main.go)
## License
MIT

9
vendor/github.com/okzk/sdnotify/notify.go generated vendored Normal file
View File

@ -0,0 +1,9 @@
// +build !linux
package sdnotify
// SdNotify sends a specified string to the systemd notification socket.
func SdNotify(state string) error {
// do nothing
return nil
}

23
vendor/github.com/okzk/sdnotify/notify_linux.go generated vendored Normal file
View File

@ -0,0 +1,23 @@
package sdnotify
import (
"net"
"os"
)
// SdNotify sends a specified string to the systemd notification socket.
func SdNotify(state string) error {
name := os.Getenv("NOTIFY_SOCKET")
if name == "" {
return ErrSdNotifyNoSocket
}
conn, err := net.DialUnix("unixgram", nil, &net.UnixAddr{Name: name, Net: "unixgram"})
if err != nil {
return err
}
defer conn.Close()
_, err = conn.Write([]byte(state))
return err
}

39
vendor/github.com/okzk/sdnotify/util.go generated vendored Normal file
View File

@ -0,0 +1,39 @@
package sdnotify
import (
"errors"
"fmt"
)
// ErrSdNotifyNoSocket is the error returned when the NOTIFY_SOCKET does not exist.
var ErrSdNotifyNoSocket = errors.New("No socket")
// Ready sends READY=1 to the systemd notify socket.
func Ready() error {
return SdNotify("READY=1")
}
// Stopping sends STOPPING=1 to the systemd notify socket.
func Stopping() error {
return SdNotify("STOPPING=1")
}
// Reloading sends RELOADING=1 to the systemd notify socket.
func Reloading() error {
return SdNotify("RELOADING=1")
}
// Errno sends ERRNO=? to the systemd notify socket.
func Errno(errno int) error {
return SdNotify(fmt.Sprintf("ERRNO=%d", errno))
}
// Status sends STATUS=? to the systemd notify socket.
func Status(status string) error {
return SdNotify("STATUS=" + status)
}
// Watchdog sends WATCHDOG=1 to the systemd notify socket.
func Watchdog() error {
return SdNotify("WATCHDOG=1")
}

2
vendor/modules.txt vendored
View File

@ -307,6 +307,8 @@ github.com/multiformats/go-multistream
github.com/mutecomm/go-sqlcipher
# github.com/olekukonko/tablewriter v0.0.0-20170128050532-febf2d34b54a
github.com/olekukonko/tablewriter
# github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
github.com/okzk/sdnotify
# github.com/opentracing/opentracing-go v1.0.2
github.com/opentracing/opentracing-go
github.com/opentracing/opentracing-go/ext