status-im
/
consul
mirror of https://github.com/status-im/consul.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactor uiserver to separate package, cleaner Reloading

This commit is contained in:
Paul Banks 2020-09-23 12:37:33 +01:00
parent f6d55e1d25
commit e4db845246
No known key found for this signature in database
GPG Key ID: C25A851A849B8221
20 changed files with 777 additions and 325 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
GNUmakefile
View File

@ -306,7 +306,7 @@ lint:
# also run as part of the release build script when it verifies that there are no # also run as part of the release build script when it verifies that there are no
# changes to the UI assets that aren't checked in. # changes to the UI assets that aren't checked in.
static-assets: static-assets:
@go-bindata-assetfs -modtime 1 -pkg agent -prefix pkg -o $(ASSETFS_PATH) ./pkg/web_ui/... @go-bindata-assetfs -modtime 1 -pkg uiserver -prefix pkg -o $(ASSETFS_PATH) ./pkg/web_ui/...
@go fmt $(ASSETFS_PATH) @go fmt $(ASSETFS_PATH)

45
agent/agent.go
View File

@ -255,6 +255,23 @@ type Agent struct {
// fail, the agent will be shutdown. // fail, the agent will be shutdown.
apiServers *apiServers apiServers *apiServers
// httpHandlers provides direct access to (one of) the HTTPHandlers started by
// this agent. This is used in tests to test HTTP endpoints without overhead
// of TCP connections etc.
//
// TODO: this is a temporary re-introduction after we removed a list of
// HTTPServers in favour of apiServers abstraction. Now that HTTPHandlers is
// stateful and has config reloading though it's not OK to just use a
// different instance of handlers in tests to the ones that the agent is wired
// up to since then config reloads won't actually affect the handlers under
// test while plumbing the external handlers in the TestAgent through bypasses
// testing that the agent itself is actually reloading the state correctly.
// Once we move `apiServers` to be a passed-in dependency for NewAgent, we
// should be able to remove this and have the Test Agent create the
// HTTPHandlers and pass them in removing the need to pull them back out
// again.
httpHandlers *HTTPHandlers
// wgServers is the wait group for all HTTP and DNS servers // wgServers is the wait group for all HTTP and DNS servers
// TODO: remove once dnsServers are handled by apiServers // TODO: remove once dnsServers are handled by apiServers
wgServers sync.WaitGroup wgServers sync.WaitGroup
@ -290,6 +307,11 @@ type Agent struct {
// IP. // IP.
httpConnLimiter connlimit.Limiter httpConnLimiter connlimit.Limiter
// configReloaders are subcomponents that need to be notified on a reload so
// they can update their internal state.
configReloaders []ConfigReloader
// enterpriseAgent embeds fields that we only access in consul-enterprise builds
enterpriseAgent enterpriseAgent
} }
@ -333,9 +355,6 @@ func New(bd BaseDeps) (*Agent, error) {
cache: bd.Cache, cache: bd.Cache,
} }
// Initialize the UI Config
a.uiConfig.Store(a.config.UIConfig)
a.serviceManager = NewServiceManager(&a) a.serviceManager = NewServiceManager(&a)
// TODO: do this somewhere else, maybe move to newBaseDeps // TODO: do this somewhere else, maybe move to newBaseDeps
@ -737,6 +756,8 @@ func (a *Agent) listenHTTP() ([]apiServer, error) {
agent: a, agent: a,
denylist: NewDenylist(a.config.HTTPBlockEndpoints), denylist: NewDenylist(a.config.HTTPBlockEndpoints),
} }
a.configReloaders = append(a.configReloaders, srv.ReloadConfig)
a.httpHandlers = srv
httpServer := &http.Server{ httpServer := &http.Server{
Addr: l.Addr().String(), Addr: l.Addr().String(),
TLSConfig: tlscfg, TLSConfig: tlscfg,
@ -3573,8 +3594,11 @@ func (a *Agent) reloadConfigInternal(newCfg *config.RuntimeConfig) error {
a.State.SetDiscardCheckOutput(newCfg.DiscardCheckOutput) a.State.SetDiscardCheckOutput(newCfg.DiscardCheckOutput)
// Reload metrics config for _, r := range a.configReloaders {
a.uiConfig.Store(newCfg.UIConfig) if err := r(newCfg); err != nil {
return err
}
}
return nil return nil
} }
@ -3828,14 +3852,3 @@ func defaultIfEmpty(val, defaultVal string) string {
} }
return defaultVal return defaultVal
} }
// getUIConfig is the canonical way to read the value of the UIConfig at
// runtime. It is thread safe and returns the most recent configuration which
// may have changed since the agent started due to config reload.
func (a *Agent) getUIConfig() config.UIConfig {
if cfg, ok := a.uiConfig.Load().(config.UIConfig); ok {
return cfg
}
// Shouldn't happen but be defensive
return config.UIConfig{}
}

30
agent/agent_test.go
View File

@ -3503,36 +3503,6 @@ func TestAgent_ReloadConfigTLSConfigFailure(t *testing.T) {
require.Len(t, tlsConf.RootCAs.Subjects(), 1) require.Len(t, tlsConf.RootCAs.Subjects(), 1)
} }
func TestAgent_ReloadConfigUIConfig(t *testing.T) {
t.Parallel()
dataDir := testutil.TempDir(t, "agent") // we manage the data dir
hcl := `
data_dir = "` + dataDir + `"
ui_config {
enabled = true // note that this is _not_ reloadable
metrics_provider = "foo"
}
`
a := NewTestAgent(t, hcl)
defer a.Shutdown()
uiCfg := a.getUIConfig()
require.Equal(t, "foo", uiCfg.MetricsProvider)
hcl = `
data_dir = "` + dataDir + `"
ui_config {
enabled = true
metrics_provider = "bar"
}
`
c := TestConfig(testutil.Logger(t), config.FileSource{Name: t.Name(), Format: "hcl", Data: hcl})
require.NoError(t, a.reloadConfigInternal(c))
uiCfg = a.getUIConfig()
require.Equal(t, "bar", uiCfg.MetricsProvider)
}
func TestAgent_consulConfig_AutoEncryptAllowTLS(t *testing.T) { func TestAgent_consulConfig_AutoEncryptAllowTLS(t *testing.T) {
t.Parallel() t.Parallel()
dataDir := testutil.TempDir(t, "agent") // we manage the data dir dataDir := testutil.TempDir(t, "agent") // we manage the data dir

258
agent/http.go
View File

@ -1,16 +1,13 @@
package agent package agent
import ( import (
"bytes"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"net/http/pprof" "net/http/pprof"
"net/url" "net/url"
"os"
"reflect" "reflect"
"regexp" "regexp"
"strconv" "strconv"
@ -21,8 +18,10 @@ import (
"github.com/armon/go-metrics" "github.com/armon/go-metrics"
"github.com/hashicorp/consul/acl" "github.com/hashicorp/consul/acl"
"github.com/hashicorp/consul/agent/cache" "github.com/hashicorp/consul/agent/cache"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/agent/consul" "github.com/hashicorp/consul/agent/consul"
"github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/structs"
"github.com/hashicorp/consul/agent/uiserver"
"github.com/hashicorp/consul/api" "github.com/hashicorp/consul/api"
"github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/lib"
"github.com/hashicorp/consul/logging" "github.com/hashicorp/consul/logging"
@ -78,135 +77,12 @@ func (e ForbiddenError) Error() string {
return "Access is restricted" return "Access is restricted"
} }
// HTTPHandlers provides http.Handler functions for the HTTP APi. // HTTPHandlers provides an HTTP api for an agent.
type HTTPHandlers struct { type HTTPHandlers struct {
agent *Agent agent *Agent
denylist *Denylist denylist *Denylist
} configReloaders []ConfigReloader
h http.Handler
// bufferedFile implements os.File and allows us to modify a file from disk by
// writing out the new version into a buffer and then serving file reads from
// that. It assumes you are modifying a real file and presents the actual file's
// info when queried.
type bufferedFile struct {
templated *bytes.Reader
info os.FileInfo
}
func newBufferedFile(buf *bytes.Buffer, raw http.File) *bufferedFile {
info, _ := raw.Stat()
return &bufferedFile{
templated: bytes.NewReader(buf.Bytes()),
info: info,
}
}
func (t *bufferedFile) Read(p []byte) (n int, err error) {
return t.templated.Read(p)
}
func (t *bufferedFile) Seek(offset int64, whence int) (int64, error) {
return t.templated.Seek(offset, whence)
}
func (t *bufferedFile) Close() error {
return nil
}
func (t *bufferedFile) Readdir(count int) ([]os.FileInfo, error) {
return nil, errors.New("not a directory")
}
func (t *bufferedFile) Stat() (os.FileInfo, error) {
return t, nil
}
func (t *bufferedFile) Name() string {
return t.info.Name()
}
func (t *bufferedFile) Size() int64 {
return int64(t.templated.Len())
}
func (t *bufferedFile) Mode() os.FileMode {
return t.info.Mode()
}
func (t *bufferedFile) ModTime() time.Time {
return t.info.ModTime()
}
func (t *bufferedFile) IsDir() bool {
return false
}
func (t *bufferedFile) Sys() interface{} {
return nil
}
type redirectFS struct {
fs http.FileSystem
}
func (fs *redirectFS) Open(name string) (http.File, error) {
file, err := fs.fs.Open(name)
if err != nil {
file, err = fs.fs.Open("/index.html")
}
return file, err
}
type settingsInjectedIndexFS struct {
fs http.FileSystem
UISettings map[string]interface{}
}
func (fs *settingsInjectedIndexFS) Open(name string) (http.File, error) {
file, err := fs.fs.Open(name)
if err != nil || name != "/index.html" {
return file, err
}
content, err := ioutil.ReadAll(file)
if err != nil {
return nil, fmt.Errorf("failed reading index.html: %s", err)
}
file.Seek(0, 0)
// Replace the placeholder in the meta ENV with the actual UI config settings.
// Ember passes the ENV with URL encoded JSON in a meta tag. We are replacing
// a key and value that is the encoded version of
// `"CONSUL_UI_SETTINGS_PLACEHOLDER":"__CONSUL_UI_SETTINGS_GO_HERE__"`
// with a URL-encoded JSON blob representing the actual config.
// First built an escaped, JSON blob from the settings passed.
bs, err := json.Marshal(fs.UISettings)
if err != nil {
return nil, fmt.Errorf("failed marshalling UI settings JSON: %s", err)
}
// We want to remove the first and last chars which will be the { and } since
// we are injecting these variabled into the middle of an existing object.
bs = bytes.Trim(bs, "{}")
// We use PathEscape because we don't want spaces to be turned into "+" like
// QueryEscape does.
escaped := url.PathEscape(string(bs))
content = bytes.Replace(content,
[]byte("%22CONSUL_UI_SETTINGS_PLACEHOLDER%22%3A%22__CONSUL_UI_SETTINGS_GO_HERE__%22"),
[]byte(escaped), 1)
// We also need to inject the content path. This used to be a go template
// hence the syntax but for now simple string replacement is fine esp. since
// all the other templated stuff above can't easily be done that was as we are
// replacing an entire placeholder element in an encoded JSON blob with
// multiple encoded JSON elements.
if path, ok := fs.UISettings["CONSUL_CONTENT_PATH"].(string); ok {
content = bytes.Replace(content, []byte("{{.ContentPath}}"), []byte(path), -1)
}
return newBufferedFile(bytes.NewBuffer(content), file), nil
} }
// endpoint is a Consul-specific HTTP handler that takes the usual arguments in // endpoint is a Consul-specific HTTP handler that takes the usual arguments in
@ -249,8 +125,45 @@ func (w *wrappedMux) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
w.handler.ServeHTTP(resp, req) w.handler.ServeHTTP(resp, req)
} }
// handler is used to attach our handlers to the mux // ReloadConfig updates any internal state when the config is changed at
// runtime.
func (s *HTTPHandlers) ReloadConfig(newCfg *config.RuntimeConfig) error {
for _, r := range s.configReloaders {
if err := r(newCfg); err != nil {
return err
}
}
return nil
}
// handler is used to initialize the Handler. In agent code we only ever call
// this once during agent initialization so it was always intended as a single
// pass init method. However many test rely on it as a cheaper way to get a
// handler to call ServeHTTP against and end up calling it multiple times on a
// single agent instance. Until this method had to manage state that might be
// affected by a reload or otherwise vary over time that was not problematic
// although it was wasteful to redo all this setup work multiple times in one
// test.
//
// Now uiserver and possibly other components need to handle reloadable state
// having test randomly clobber the state with the original config again for
// each call gets confusing fast. So handler will memoize it's response - it's
// allowed to call it multiple times on the same agent, but it will only do the
// work the first time and return the same handler on subsequent calls.
//
// The `enableDebug` argument used in the first call will be effective and a
// later change will not do anything. The same goes for the initial config. For
// example if config is reloaded with UI enabled but it was not originally, the
// http.Handler returned will still have it disabled.
//
// The first call must not be concurrent with any other call. Subsequent calls
// may be concurrent with HTTP requests since no state is modified.
func (s *HTTPHandlers) handler(enableDebug bool) http.Handler { func (s *HTTPHandlers) handler(enableDebug bool) http.Handler {
// Memoize multiple calls.
if s.h != nil {
return s.h
}
mux := http.NewServeMux() mux := http.NewServeMux()
// handleFuncMetrics takes the given pattern and handler and wraps to produce // handleFuncMetrics takes the given pattern and handler and wraps to produce
@ -347,38 +260,27 @@ func (s *HTTPHandlers) handler(enableDebug bool) http.Handler {
handlePProf("/debug/pprof/trace", pprof.Trace) handlePProf("/debug/pprof/trace", pprof.Trace)
if s.IsUIEnabled() { if s.IsUIEnabled() {
// Note that we _don't_ support reloading ui_config.{enabled,content_dir} // Note that we _don't_ support reloading ui_config.{enabled, content_dir,
// since this only runs at initial startup. // content_path} since this only runs at initial startup.
var uifs http.FileSystem uiHandler := uiserver.NewHandler(s.agent.config, s.agent.logger.Named(logging.HTTP))
// Use the custom UI dir if provided. s.configReloaders = append(s.configReloaders, uiHandler.ReloadConfig)
uiConfig := s.agent.getUIConfig()
if uiConfig.Dir != "" {
uifs = http.Dir(uiConfig.Dir)
} else {
fs := assetFS()
uifs = fs
}
uifs = &redirectFS{fs: &settingsInjectedIndexFS{ // Wrap it to add the headers specified by the http_config.response_headers
fs: uifs, // user config
UISettings: s.GetUIENVFromConfig(), uiHandlerWithHeaders := serveHandlerWithHeaders(
}} uiHandler,
// create a http handler using the ui file system
// and the headers specified by the http_config.response_headers user config
uifsWithHeaders := serveHandlerWithHeaders(
http.FileServer(uifs),
s.agent.config.HTTPResponseHeaders, s.agent.config.HTTPResponseHeaders,
) )
mux.Handle( mux.Handle(
"/robots.txt", "/robots.txt",
uifsWithHeaders, uiHandlerWithHeaders,
) )
mux.Handle( mux.Handle(
uiConfig.ContentPath, s.agent.config.UIConfig.ContentPath,
http.StripPrefix( http.StripPrefix(
uiConfig.ContentPath, s.agent.config.UIConfig.ContentPath,
uifsWithHeaders, uiHandlerWithHeaders,
), ),
) )
} }
@ -391,37 +293,11 @@ func (s *HTTPHandlers) handler(enableDebug bool) http.Handler {
h = mux h = mux
} }
h = s.enterpriseHandler(h) h = s.enterpriseHandler(h)
return &wrappedMux{ s.h = &wrappedMux{
mux: mux, mux: mux,
handler: h, handler: h,
} }
} return s.h
func (s *HTTPHandlers) GetUIENVFromConfig() map[string]interface{} {
uiCfg := s.agent.getUIConfig()
vars := map[string]interface{}{
"CONSUL_CONTENT_PATH": uiCfg.ContentPath,
"CONSUL_ACLS_ENABLED": s.agent.config.ACLsEnabled,
"CONSUL_METRICS_PROVIDER": uiCfg.MetricsProvider,
// We explicitly MUST NOT pass the metrics_proxy object since it might
// contain add_headers with secrets that the UI shouldn't know e.g. API
// tokens for the backend. The provider should either require the proxy to
// be configured and then use that or hit the backend directly from the
// browser.
"CONSUL_METRICS_PROXY_ENABLED": uiCfg.MetricsProxy.BaseURL != "",
"CONSUL_DASHBOARD_URL_TEMPLATES": uiCfg.DashboardURLTemplates,
}
// Only set this if there is some actual JSON or we'll cause a JSON
// marshalling error later during serving which ends up being silent.
if uiCfg.MetricsProviderOptionsJSON != "" {
vars["CONSUL_METRICS_PROVIDER_OPTIONS"] = json.RawMessage(uiCfg.MetricsProviderOptionsJSON)
}
s.addEnterpriseUIENVVars(vars)
return vars
} }
// nodeName returns the node name of the agent // nodeName returns the node name of the agent
@ -659,11 +535,17 @@ func (s *HTTPHandlers) marshalJSON(req *http.Request, obj interface{}) ([]byte,
// Returns true if the UI is enabled. // Returns true if the UI is enabled.
func (s *HTTPHandlers) IsUIEnabled() bool { func (s *HTTPHandlers) IsUIEnabled() bool {
return s.agent.config.UIDir != "" || s.agent.config.EnableUI // Note that we _don't_ support reloading ui_config.{enabled,content_dir}
// since this only runs at initial startup.
return s.agent.config.UIConfig.Dir != "" || s.agent.config.UIConfig.Enabled
} }
// Renders a simple index page // Renders a simple index page
func (s *HTTPHandlers) Index(resp http.ResponseWriter, req *http.Request) { func (s *HTTPHandlers) Index(resp http.ResponseWriter, req *http.Request) {
// Send special headers too since this endpoint isn't wrapped with something
// that sends them.
setHeaders(resp, s.agent.config.HTTPResponseHeaders)
// Check if this is a non-index path // Check if this is a non-index path
if req.URL.Path != "/" { if req.URL.Path != "/" {
resp.WriteHeader(http.StatusNotFound) resp.WriteHeader(http.StatusNotFound)
@ -678,8 +560,12 @@ func (s *HTTPHandlers) Index(resp http.ResponseWriter, req *http.Request) {
} }
// Redirect to the UI endpoint // Redirect to the UI endpoint
uiCfg := s.agent.getUIConfig() http.Redirect(
http.Redirect(resp, req, uiCfg.ContentPath, http.StatusMovedPermanently) // 301 resp,
req,
s.agent.config.UIConfig.ContentPath,
http.StatusMovedPermanently,
) // 301
} }
func decodeBody(body io.Reader, out interface{}) error { func decodeBody(body io.Reader, out interface{}) error {

2
agent/http_oss.go
View File

@ -55,8 +55,6 @@ func (s *HTTPHandlers) rewordUnknownEnterpriseFieldError(err error) error {
return err return err
} }
func (s *HTTPHandlers) addEnterpriseUIENVVars(_ map[string]interface{}) {}
func parseACLAuthMethodEnterpriseMeta(req *http.Request, _ *structs.ACLAuthMethodEnterpriseMeta) error { func parseACLAuthMethodEnterpriseMeta(req *http.Request, _ *structs.ACLAuthMethodEnterpriseMeta) error {
if methodNS := req.URL.Query().Get("authmethod-ns"); methodNS != "" { if methodNS := req.URL.Query().Get("authmethod-ns"); methodNS != "" {
return BadRequestError{Reason: "Invalid query parameter: \"authmethod-ns\" - Namespaces are a Consul Enterprise feature"} return BadRequestError{Reason: "Invalid query parameter: \"authmethod-ns\" - Namespaces are a Consul Enterprise feature"}

129
agent/http_test.go
View File

@ -20,6 +20,7 @@ import (
"time" "time"
"github.com/NYTimes/gziphandler" "github.com/NYTimes/gziphandler"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/structs"
tokenStore "github.com/hashicorp/consul/agent/token" tokenStore "github.com/hashicorp/consul/agent/token"
"github.com/hashicorp/consul/api" "github.com/hashicorp/consul/api"
@ -417,62 +418,56 @@ func TestHTTPAPI_TranslateAddrHeader(t *testing.T) {
func TestHTTPAPIResponseHeaders(t *testing.T) { func TestHTTPAPIResponseHeaders(t *testing.T) {
t.Parallel() t.Parallel()
a := NewTestAgent(t, ` a := NewTestAgent(t, `
ui_config {
# Explicitly disable UI so we can ensure the index replacement gets headers too.
enabled = false
}
http_config { http_config {
response_headers = { response_headers = {
"Access-Control-Allow-Origin" = "*" "Access-Control-Allow-Origin" = "*"
"X-XSS-Protection" = "1; mode=block" "X-XSS-Protection" = "1; mode=block"
}
}
`)
defer a.Shutdown()
resp := httptest.NewRecorder()
handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) {
return nil, nil
}
req, _ := http.NewRequest("GET", "/v1/agent/self", nil)
a.srv.wrap(handler, []string{"GET"})(resp, req)
origin := resp.Header().Get("Access-Control-Allow-Origin")
if origin != "*" {
t.Fatalf("bad Access-Control-Allow-Origin: expected %q, got %q", "*", origin)
}
xss := resp.Header().Get("X-XSS-Protection")
if xss != "1; mode=block" {
t.Fatalf("bad X-XSS-Protection header: expected %q, got %q", "1; mode=block", xss)
}
}
func TestUIResponseHeaders(t *testing.T) {
t.Parallel()
a := NewTestAgent(t, `
http_config {
response_headers = {
"Access-Control-Allow-Origin" = "*"
"X-Frame-Options" = "SAMEORIGIN" "X-Frame-Options" = "SAMEORIGIN"
} }
} }
`) `)
defer a.Shutdown() defer a.Shutdown()
requireHasHeadersSet(t, a, "/v1/agent/self")
// Check the Index page that just renders a simple message with UI disabled
// also gets the right headers.
requireHasHeadersSet(t, a, "/")
}
func requireHasHeadersSet(t *testing.T, a *TestAgent, path string) {
t.Helper()
resp := httptest.NewRecorder() resp := httptest.NewRecorder()
handler := func(resp http.ResponseWriter, req *http.Request) (interface{}, error) { req, _ := http.NewRequest("GET", path, nil)
return nil, nil a.srv.handler(true).ServeHTTP(resp, req)
}
req, _ := http.NewRequest("GET", "/ui", nil) hdrs := resp.Header()
a.srv.wrap(handler, []string{"GET"})(resp, req) require.Equal(t, "*", hdrs.Get("Access-Control-Allow-Origin"),
"Access-Control-Allow-Origin header value incorrect")
origin := resp.Header().Get("Access-Control-Allow-Origin") require.Equal(t, "1; mode=block", hdrs.Get("X-XSS-Protection"),
if origin != "*" { "X-XSS-Protection header value incorrect")
t.Fatalf("bad Access-Control-Allow-Origin: expected %q, got %q", "*", origin) }
}
frameOptions := resp.Header().Get("X-Frame-Options") func TestUIResponseHeaders(t *testing.T) {
if frameOptions != "SAMEORIGIN" { t.Parallel()
t.Fatalf("bad X-XSS-Protection header: expected %q, got %q", "SAMEORIGIN", frameOptions) a := NewTestAgent(t, `
} http_config {
response_headers = {
"Access-Control-Allow-Origin" = "*"
"X-XSS-Protection" = "1; mode=block"
"X-Frame-Options" = "SAMEORIGIN"
}
}
`)
defer a.Shutdown()
requireHasHeadersSet(t, a, "/ui")
} }
func TestAcceptEncodingGzip(t *testing.T) { func TestAcceptEncodingGzip(t *testing.T) {
@ -1210,34 +1205,36 @@ func TestEnableWebUI(t *testing.T) {
require.Equal(t, http.StatusOK, resp.Code) require.Equal(t, http.StatusOK, resp.Code)
// Validate that it actually sent the index page we expect since an error // Validate that it actually sent the index page we expect since an error
// during serving the special intercepted index.html in // during serving the special intercepted index.html can result in an empty
// settingsInjectedIndexFS.Open will actually result in http.FileServer just // response but a 200 status.
// serving a plain directory listing instead which still passes the above HTTP
// status assertion. This comment is part of our index.html template
require.Contains(t, resp.Body.String(), `<!-- CONSUL_VERSION:`) require.Contains(t, resp.Body.String(), `<!-- CONSUL_VERSION:`)
}
func TestEnableWebUIWithMetricsOptions(t *testing.T) { // Verify that we injected the variables we expected. The rest of injection
t.Parallel() // behavior is tested in the uiserver package, this just ensures it's plumbed
a := NewTestAgent(t, ` // in correctly.
ui_config { require.NotContains(t, resp.Body.String(), `__RUNTIME_BOOL`)
enabled = true
metrics_provider_options_json = "{\"foo\": 1}"
}
`)
defer a.Shutdown()
req, _ := http.NewRequest("GET", "/ui/", nil) // Reload the config with changed metrics provider options and verify that
resp := httptest.NewRecorder() // they are present in the output.
a.srv.handler(true).ServeHTTP(resp, req) newHCL := `
require.Equal(t, http.StatusOK, resp.Code) data_dir = "` + a.DataDir + `"
ui_config {
enabled = true
metrics_provider = "valid-but-unlikely-metrics-provider-name"
}
`
c := TestConfig(testutil.Logger(t), config.FileSource{Name: t.Name(), Format: "hcl", Data: newHCL})
require.NoError(t, a.reloadConfigInternal(c))
// Validate that it actually sent the index page we expect since an error // Now index requests should contain that metrics provider name.
// during serving the special intercepted index.html in {
// settingsInjectedIndexFS.Open will actually result in http.FileServer just req, _ := http.NewRequest("GET", "/ui/", nil)
// serving a plain directory listing instead which still passes the above HTTP resp := httptest.NewRecorder()
// status assertion. This comment is part of our index.html template a.srv.handler(true).ServeHTTP(resp, req)
require.Contains(t, resp.Body.String(), `<!-- CONSUL_VERSION:`) require.Equal(t, http.StatusOK, resp.Code)
require.Contains(t, resp.Body.String(), `<!-- CONSUL_VERSION:`)
require.Contains(t, resp.Body.String(), `valid-but-unlikely-metrics-provider-name`)
}
} }
func TestAllowedNets(t *testing.T) { func TestAllowedNets(t *testing.T) {

7
agent/reload.go Normal file
View File

@ -0,0 +1,7 @@
package agent
import "github.com/hashicorp/consul/agent/config"
// ConfigReloader is a function type which may be implemented to support reloading
// of configuration.
type ConfigReloader func(rtConfig *config.RuntimeConfig) error

2
agent/testagent.go
View File

@ -213,7 +213,7 @@ func (a *TestAgent) Start(t *testing.T) (err error) {
// Start the anti-entropy syncer // Start the anti-entropy syncer
a.Agent.StartSync() a.Agent.StartSync()
a.srv = &HTTPHandlers{agent: agent, denylist: NewDenylist(a.config.HTTPBlockEndpoints)} a.srv = a.Agent.httpHandlers
if err := a.waitForUp(); err != nil { if err := a.waitForUp(); err != nil {
a.Shutdown() a.Shutdown()

26
agent/bindata_assetfs.go → agent/uiserver/bindata_assetfs.go
View File

File diff suppressed because one or more lines are too long

21
agent/uiserver/buf_index_fs.go Normal file
View File

@ -0,0 +1,21 @@
package uiserver
import (
"net/http"
"os"
)
// bufIndexFS is an implementation of http.FS that intercepts requests for
// the index.html file and returns a pre-rendered file from memory.
type bufIndexFS struct {
fs http.FileSystem
indexRendered []byte
indexInfo os.FileInfo
}
func (fs *bufIndexFS) Open(name string) (http.File, error) {
if name == "/index.html" {
return newBufferedFile(fs.indexRendered, fs.indexInfo), nil
}
return fs.fs.Open(name)
}

67
agent/uiserver/buffered_file.go Normal file
View File

@ -0,0 +1,67 @@
package uiserver
import (
"bytes"
"errors"
"os"
"time"
)
// bufferedFile implements os.File and allows us to modify a file from disk by
// writing out the new version into a buffer and then serving file reads from
// that.
type bufferedFile struct {
buf *bytes.Reader
info os.FileInfo
}
func newBufferedFile(buf []byte, info os.FileInfo) *bufferedFile {
return &bufferedFile{
buf: bytes.NewReader(buf),
info: info,
}
}
func (t *bufferedFile) Read(p []byte) (n int, err error) {
return t.buf.Read(p)
}
func (t *bufferedFile) Seek(offset int64, whence int) (int64, error) {
return t.buf.Seek(offset, whence)
}
func (t *bufferedFile) Close() error {
return nil
}
func (t *bufferedFile) Readdir(count int) ([]os.FileInfo, error) {
return nil, errors.New("not a directory")
}
func (t *bufferedFile) Stat() (os.FileInfo, error) {
return t, nil
}
func (t *bufferedFile) Name() string {
return t.info.Name()
}
func (t *bufferedFile) Size() int64 {
return int64(t.buf.Len())
}
func (t *bufferedFile) Mode() os.FileMode {
return t.info.Mode()
}
func (t *bufferedFile) ModTime() time.Time {
return t.info.ModTime()
}
func (t *bufferedFile) IsDir() bool {
return false
}
func (t *bufferedFile) Sys() interface{} {
return nil
}

21
agent/uiserver/redirect_fs.go Normal file
View File

@ -0,0 +1,21 @@
package uiserver
import "net/http"
// redirectFS is an http.FS that serves the index.html file for any path that is
// not found on the underlying FS.
//
// TODO: it seems better to actually 404 bad paths or at least redirect them
// rather than pretend index.html is everywhere but this is behavior changing
// so I don't want to take it on as part of this refactor.
type redirectFS struct {
fs http.FileSystem
}
func (fs *redirectFS) Open(name string) (http.File, error) {
file, err := fs.fs.Open(name)
if err != nil {
file, err = fs.fs.Open("/index.html")
}
return file, err
}

52
agent/uiserver/ui_template_data.go Normal file
View File

@ -0,0 +1,52 @@
package uiserver
import (
"encoding/json"
"fmt"
"net/url"
"github.com/hashicorp/consul/agent/config"
)
// uiTemplateDataFromConfig returns the set of variables that should be injected
// into the UI's Env based on the given runtime UI config.
func uiTemplateDataFromConfig(cfg *config.RuntimeConfig) (map[string]interface{}, error) {
uiCfg := map[string]interface{}{
"metrics_provider": cfg.UIConfig.MetricsProvider,
// We explicitly MUST NOT pass the metrics_proxy object since it might
// contain add_headers with secrets that the UI shouldn't know e.g. API
// tokens for the backend. The provider should either require the proxy to
// be configured and then use that or hit the backend directly from the
// browser.
"metrics_proxy_enabled": cfg.UIConfig.MetricsProxy.BaseURL != "",
"dashboard_url_templates": cfg.UIConfig.DashboardURLTemplates,
}
// Only set this if there is some actual JSON or we'll cause a JSON
// marshalling error later during serving which ends up being silent.
if cfg.UIConfig.MetricsProviderOptionsJSON != "" {
uiCfg["metrics_provider_options"] = json.RawMessage(cfg.UIConfig.MetricsProviderOptionsJSON)
}
d := map[string]interface{}{
"ContentPath": cfg.UIConfig.ContentPath,
"ACLsEnabled": cfg.ACLsEnabled,
}
err := uiTemplateDataFromConfigEnterprise(cfg, d, uiCfg)
// Render uiCfg down to JSON ready to inject into the template
bs, err := json.Marshal(uiCfg)
if err != nil {
return nil, fmt.Errorf("failed marshalling UI Env JSON: %s", err)
}
// Need to also URLEncode it as it is passed through a META tag value. Path
// variant is correct to avoid converting spaces to "+". Note we don't just
// use html/template because it strips comments and uses a different encoding
// for this param than Ember which is OK but just one more weird thing to
// account for in the source...
d["UIConfigJSON"] = url.PathEscape(string(bs))
return d, err
}

9
agent/uiserver/ui_template_data_oss.go Normal file
View File

@ -0,0 +1,9 @@
// +build !consulent
package uiserver
import "github.com/hashicorp/consul/agent/config"
func uiTemplateDataFromConfigEnterprise(_ *config.RuntimeConfig, _ map[string]interface{}, _ map[string]interface{}) error {
return nil
}

176
agent/uiserver/uiserver.go Normal file
View File

@ -0,0 +1,176 @@
package uiserver
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"os"
"regexp"
"strings"
"sync/atomic"
"text/template"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/logging"
"github.com/hashicorp/go-hclog"
)
// Handler is the http.Handler that serves the Consul UI. It may serve from the
// compiled-in AssetFS or from and external dir. It provides a few important
// transformations on the index.html file and includes a proxy for metrics
// backends.
type Handler struct {
// state is a reloadableState struct accessed through an atomic value to make
// it safe to reload at run time. Each call to ServeHTTP will see the latest
// version of the state without internal locking needed.
state atomic.Value
logger hclog.Logger
}
// reloadableState encapsulates all the state that might be modified during
// ReloadConfig.
type reloadableState struct {
cfg *config.UIConfig
srv http.Handler
err error
}
// NewHandler returns a Handler that can be used to serve UI http requests. It
// accepts a full agent config since properties like ACLs being enabled affect
// the UI so we need more than just UIConfig parts.
func NewHandler(agentCfg *config.RuntimeConfig, logger hclog.Logger) *Handler {
h := &Handler{
logger: logger.Named(logging.UIServer),
}
// Don't return the error since this is likely the result of a
// misconfiguration and reloading config could fix it. Instead we'll capture
// it and return an error for all calls to ServeHTTP so the misconfiguration
// is visible. Sadly we can't log effectively
if err := h.ReloadConfig(agentCfg); err != nil {
h.state.Store(reloadableState{
err: err,
})
}
return h
}
// ServeHTTP implements http.Handler and serves UI HTTP requests
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// TODO: special case for compiled metrics assets in later PR
s := h.getState()
if s == nil {
panic("nil state")
}
if s.err != nil {
http.Error(w, "UI server is misconfigured.", http.StatusInternalServerError)
h.logger.Error("Failed to configure UI server: %s", s.err)
return
}
s.srv.ServeHTTP(w, r)
}
// ReloadConfig is called by the agent when the configuration is reloaded and
// updates the UIConfig values the handler uses to serve requests.
func (h *Handler) ReloadConfig(newCfg *config.RuntimeConfig) error {
newState := reloadableState{
cfg: &newCfg.UIConfig,
}
var fs http.FileSystem
if newCfg.UIConfig.Dir == "" {
// Serve from assetFS
fs = assetFS()
} else {
fs = http.Dir(newCfg.UIConfig.Dir)
}
// Render a new index.html with the new config values ready to serve.
buf, info, err := renderIndex(newCfg, fs)
if err != nil {
return err
}
// Create a new fs that serves the rendered index file or falls back to the
// underlying FS.
fs = &bufIndexFS{
fs: fs,
indexRendered: buf,
indexInfo: info,
}
// Wrap the buffering FS our redirect FS. This needs to happen later so that
// redirected requests for /index.html get served the rendered version not the
// original.
fs = &redirectFS{fs: fs}
newState.srv = http.FileServer(fs)
// Store the new state
h.state.Store(newState)
return nil
}
// getState is a helper to access the atomic internal state
func (h *Handler) getState() *reloadableState {
if cfg, ok := h.state.Load().(reloadableState); ok {
return &cfg
}
return nil
}
func renderIndex(cfg *config.RuntimeConfig, fs http.FileSystem) ([]byte, os.FileInfo, error) {
// Open the original index.html
f, err := fs.Open("/index.html")
if err != nil {
return nil, nil, err
}
defer f.Close()
content, err := ioutil.ReadAll(f)
if err != nil {
return nil, nil, fmt.Errorf("failed reading index.html: %s", err)
}
info, err := f.Stat()
if err != nil {
return nil, nil, fmt.Errorf("failed reading metadata for index.html: %s", err)
}
// Create template data from the current config.
tplData, err := uiTemplateDataFromConfig(cfg)
if err != nil {
return nil, nil, fmt.Errorf("failed loading UI config for template: %s", err)
}
// Sadly we can't perform all the replacements we need with Go template
// because some of them end up being rendered into an escaped json encoded
// meta tag by Ember build which messes up the Go template tags. After a few
// iterations of grossness, this seemed like the least bad for now. note we
// have to match the encoded double quotes around the JSON string value that
// is there as a placeholder so the end result is an actual JSON bool not a
// string containing "false" etc.
re := regexp.MustCompile(`%22__RUNTIME_BOOL_[A-Za-z0-9-_]+__%22`)
content = []byte(re.ReplaceAllStringFunc(string(content), func(str string) string {
// Trim the prefix and __ suffix
varName := strings.TrimSuffix(strings.TrimPrefix(str, "%22__RUNTIME_BOOL_"), "__%22")
if v, ok := tplData[varName].(bool); ok && v {
return "true"
}
return "false"
}))
tpl, err := template.New("index").Parse(string(content))
if err != nil {
return nil, nil, fmt.Errorf("failed parsing index.html template: %s", err)
}
var buf bytes.Buffer
err = tpl.Execute(&buf, tplData)
if err != nil {
return nil, nil, fmt.Errorf("failed to render index.html: %s", err)
}
return buf.Bytes(), info, nil
}

230
agent/uiserver/uiserver_test.go Normal file
View File

@ -0,0 +1,230 @@
package uiserver
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"regexp"
"testing"
"github.com/hashicorp/consul/agent/config"
"github.com/hashicorp/consul/sdk/testutil"
"github.com/stretchr/testify/require"
)
func TestUIServer(t *testing.T) {
cases := []struct {
name string
cfg *config.RuntimeConfig
path string
wantStatus int
wantContains []string
wantNotContains []string
wantEnv map[string]interface{}
wantUICfgJSON string
}{
{
name: "basic UI serving",
cfg: basicUIEnabledConfig(),
path: "/", // Note /index.html redirects to /
wantStatus: http.StatusOK,
wantContains: []string{"<!-- CONSUL_VERSION:"},
wantEnv: map[string]interface{}{
"CONSUL_ACLS_ENABLED": false,
},
},
{
// TODO: is this really what we want? It's what we've always done but
// seems a bit odd to not do an actual 301 but instead serve the
// index.html from every path... It also breaks the UI probably.
name: "unknown paths to serve index",
cfg: basicUIEnabledConfig(),
path: "/foo-bar-bazz-qux",
wantStatus: http.StatusOK,
wantContains: []string{"<!-- CONSUL_VERSION:"},
},
{
name: "injecting metrics vars",
cfg: basicUIEnabledConfig(
withMetricsProvider("foo"),
withMetricsProviderOptions(`{"bar":1}`),
),
path: "/",
wantStatus: http.StatusOK,
wantContains: []string{
"<!-- CONSUL_VERSION:",
},
wantEnv: map[string]interface{}{
"CONSUL_ACLS_ENABLED": false,
},
wantUICfgJSON: `{
"metrics_provider": "foo",
"metrics_provider_options": {
"bar":1
},
"metrics_proxy_enabled": false,
"dashboard_url_templates": null
}`,
},
{
name: "acls enabled",
cfg: basicUIEnabledConfig(withACLs()),
path: "/",
wantStatus: http.StatusOK,
wantContains: []string{"<!-- CONSUL_VERSION:"},
wantEnv: map[string]interface{}{
"CONSUL_ACLS_ENABLED": true,
},
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
h := NewHandler(tc.cfg, testutil.Logger(t))
req := httptest.NewRequest("GET", tc.path, nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
require.Equal(t, tc.wantStatus, rec.Code)
for _, want := range tc.wantContains {
require.Contains(t, rec.Body.String(), want)
}
for _, wantNot := range tc.wantNotContains {
require.NotContains(t, rec.Body.String(), wantNot)
}
env := extractEnv(t, rec.Body.String())
for k, v := range tc.wantEnv {
require.Equal(t, v, env[k])
}
if tc.wantUICfgJSON != "" {
require.JSONEq(t, tc.wantUICfgJSON, extractUIConfig(t, rec.Body.String()))
}
})
}
}
func extractMetaJSON(t *testing.T, name, content string) string {
t.Helper()
// Find and extract the env meta tag. Why yes I _am_ using regexp to parse
// HTML thanks for asking. In this case it's HTML with a very limited format
// so I don't feel too bad but maybe I should.
// https://stackoverflow.com/questions/1732348/regex-match-open-tags-except-xhtml-self-contained-tags/1732454#1732454
re := regexp.MustCompile(`<meta name="` + name + `+" content="([^"]*)"`)
matches := re.FindStringSubmatch(content)
require.Len(t, matches, 2, "didn't find the %s meta tag", name)
// Unescape the JSON
jsonStr, err := url.PathUnescape(matches[1])
require.NoError(t, err)
return jsonStr
}
func extractEnv(t *testing.T, content string) map[string]interface{} {
t.Helper()
js := extractMetaJSON(t, "consul-ui/config/environment", content)
var env map[string]interface{}
err := json.Unmarshal([]byte(js), &env)
require.NoError(t, err)
return env
}
func extractUIConfig(t *testing.T, content string) string {
t.Helper()
return extractMetaJSON(t, "consul-ui/ui_config", content)
}
type cfgFunc func(cfg *config.RuntimeConfig)
func basicUIEnabledConfig(opts ...cfgFunc) *config.RuntimeConfig {
cfg := &config.RuntimeConfig{
UIConfig: config.UIConfig{
Enabled: true,
},
}
for _, f := range opts {
f(cfg)
}
return cfg
}
func withACLs() cfgFunc {
return func(cfg *config.RuntimeConfig) {
cfg.ACLDatacenter = "dc1"
cfg.ACLDefaultPolicy = "deny"
cfg.ACLsEnabled = true
}
}
func withMetricsProvider(name string) cfgFunc {
return func(cfg *config.RuntimeConfig) {
cfg.UIConfig.MetricsProvider = name
}
}
func withMetricsProviderOptions(jsonStr string) cfgFunc {
return func(cfg *config.RuntimeConfig) {
cfg.UIConfig.MetricsProviderOptionsJSON = jsonStr
}
}
// TestMultipleIndexRequests validates that the buffered file mechanism works
// beyond the first request. The initial implementation did not as it shared an
// bytes.Reader between callers.
func TestMultipleIndexRequests(t *testing.T) {
h := NewHandler(basicUIEnabledConfig(), testutil.Logger(t))
for i := 0; i < 3; i++ {
req := httptest.NewRequest("GET", "/", nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code)
require.Contains(t, rec.Body.String(), "<!-- CONSUL_VERSION:",
"request %d didn't return expected content", i+1)
}
}
func TestReload(t *testing.T) {
h := NewHandler(basicUIEnabledConfig(), testutil.Logger(t))
{
req := httptest.NewRequest("GET", "/", nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code)
require.Contains(t, rec.Body.String(), "<!-- CONSUL_VERSION:")
require.NotContains(t, rec.Body.String(), "exotic-metrics-provider-name")
}
// Reload the config with the changed metrics provider name
newCfg := basicUIEnabledConfig(
withMetricsProvider("exotic-metrics-provider-name"),
)
h.ReloadConfig(newCfg)
// Now we should see the new provider name in the output of index
{
req := httptest.NewRequest("GET", "/", nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
require.Equal(t, http.StatusOK, rec.Code)
require.Contains(t, rec.Body.String(), "<!-- CONSUL_VERSION:")
require.Contains(t, rec.Body.String(), "exotic-metrics-provider-name")
}
}

2
build-support/functions/20-build.sh
View File

@ -165,7 +165,7 @@ function build_assetfs {
( (
tar -c pkg/web_ui GNUmakefile | docker cp - ${container_id}:/consul && tar -c pkg/web_ui GNUmakefile | docker cp - ${container_id}:/consul &&
status "Running build in container" && docker start -i ${container_id} && status "Running build in container" && docker start -i ${container_id} &&
status "Copying back artifacts" && docker cp ${container_id}:/consul/bindata_assetfs.go ${sdir}/agent/bindata_assetfs.go status "Copying back artifacts" && docker cp ${container_id}:/consul/bindata_assetfs.go ${sdir}/agent/uiserver/bindata_assetfs.go
) )
ret=$? ret=$?
docker rm ${container_id} > /dev/null docker rm ${container_id} > /dev/null

1
logging/names.go
View File

@ -52,6 +52,7 @@ const (
TLSUtil string = "tlsutil" TLSUtil string = "tlsutil"
Transaction string = "txn" Transaction string = "txn"
UsageMetrics string = "usage_metrics" UsageMetrics string = "usage_metrics"
UIServer string = "ui_server"
WAN string = "wan" WAN string = "wan"
Watch string = "watch" Watch string = "watch"
Vault string = "vault" Vault string = "vault"

20
ui-v2/config/environment.js
View File

@ -120,16 +120,18 @@ module.exports = function(environment, $ = process.env) {
}); });
break; break;
case environment === 'production': case environment === 'production':
// Make sure all templated variables check for existence first
// before outputting them, this means they all should be conditionals
ENV = Object.assign({}, ENV, { ENV = Object.assign({}, ENV, {
// This ENV var is a special placeholder that Consul will replace // These values are placeholders that are replaced when Consul renders
// entirely with multiple vars from the runtime config for example // the index.html based on runtime config. They can't use Go template
// CONSUL_ACLs_ENABLED and CONSUL_NSPACES_ENABLED. The actual key here // syntax since this object ends up JSON and URLencoded in an HTML meta
// won't really exist in the actual ember ENV when it's being served // tag which obscured the Go template tag syntax.
// through Consul. See settingsInjectedIndexFS.Open in Go code for the //
// details. // __RUNTIME_BOOL_Xxxx__ will be replaced with either "true" or "false"
CONSUL_UI_SETTINGS_PLACEHOLDER: "__CONSUL_UI_SETTINGS_GO_HERE__", // depending on whether the named variable is true or valse in the data
// returned from `uiTemplateDataFromConfig`.
CONSUL_ACLS_ENABLED: '__RUNTIME_BOOL_ACLsEnabled__',
CONSUL_SSO_ENABLED: '__RUNTIME_BOOL_SSOEnabled__',
CONSUL_NSPACES_ENABLED: '__RUNTIME_BOOL_NSpacesEnabled__',
}); });
break; break;
} }

2
ui-v2/lib/startup/templates/head.html.js
View File

@ -1,5 +1,7 @@
module.exports = ({ appName, environment, rootURL, config }) => ` module.exports = ({ appName, environment, rootURL, config }) => `
<!-- CONSUL_VERSION: ${config.CONSUL_VERSION} --> <!-- CONSUL_VERSION: ${config.CONSUL_VERSION} -->
<meta name="consul-ui/ui_config" content="{{ .UIConfigJSON }}" />
<link rel="icon" type="image/png" href="${rootURL}assets/favicon-32x32.png" sizes="32x32"> <link rel="icon" type="image/png" href="${rootURL}assets/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="${rootURL}assets/favicon-16x16.png" sizes="16x16"> <link rel="icon" type="image/png" href="${rootURL}assets/favicon-16x16.png" sizes="16x16">
<link integrity="" rel="stylesheet" href="${rootURL}assets/vendor.css"> <link integrity="" rel="stylesheet" href="${rootURL}assets/vendor.css">