peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
package peering
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"sort"
|
|
|
|
|
|
|
|
"github.com/hashicorp/go-hclog"
|
|
|
|
|
proxycfg: server-local config entry data sources
This is the OSS portion of enterprise PR 2056.
This commit provides server-local implementations of the proxycfg.ConfigEntry
and proxycfg.ConfigEntryList interfaces, that source data from streaming events.
It makes use of the LocalMaterializer type introduced for peering replication,
adding the necessary support for authorization.
It also adds support for "wildcard" subscriptions (within a topic) to the event
publisher, as this is needed to fetch service-resolvers for all services when
configuring mesh gateways.
Currently, events will be emitted for just the ingress-gateway, service-resolver,
and mesh config entry types, as these are the only entries required by proxycfg
— the events will be emitted on topics named IngressGateway, ServiceResolver,
and MeshConfig topics respectively.
Though these events will only be consumed "locally" for now, they can also be
consumed via the gRPC endpoint (confirmed using grpcurl) so using them from
client agents should be a case of swapping the LocalMaterializer for an
RPCMaterializer.
2022-07-01 15:09:47 +00:00
|
|
|
"github.com/hashicorp/consul/acl"
|
|
|
|
"github.com/hashicorp/consul/acl/resolver"
|
peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
"github.com/hashicorp/consul/agent/cache"
|
|
|
|
"github.com/hashicorp/consul/agent/consul/stream"
|
|
|
|
"github.com/hashicorp/consul/agent/structs"
|
|
|
|
"github.com/hashicorp/consul/agent/submatview"
|
|
|
|
"github.com/hashicorp/consul/proto/pbservice"
|
|
|
|
"github.com/hashicorp/consul/proto/pbsubscribe"
|
|
|
|
)
|
|
|
|
|
|
|
|
type Subscriber interface {
|
|
|
|
Subscribe(req *stream.SubscribeRequest) (*stream.Subscription, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
type exportedServiceRequest struct {
|
|
|
|
logger hclog.Logger
|
|
|
|
req structs.ServiceSpecificRequest
|
|
|
|
sub Subscriber
|
|
|
|
}
|
|
|
|
|
2022-05-19 19:21:44 +00:00
|
|
|
func newExportedStandardServiceRequest(logger hclog.Logger, svc structs.ServiceName, sub Subscriber) *exportedServiceRequest {
|
peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
req := structs.ServiceSpecificRequest{
|
|
|
|
ServiceName: svc.Name,
|
2022-05-19 19:21:44 +00:00
|
|
|
Connect: false,
|
peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
EnterpriseMeta: svc.EnterpriseMeta,
|
|
|
|
}
|
|
|
|
return &exportedServiceRequest{
|
|
|
|
logger: logger,
|
|
|
|
req: req,
|
|
|
|
sub: sub,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// CacheInfo implements submatview.Request
|
|
|
|
func (e *exportedServiceRequest) CacheInfo() cache.RequestInfo {
|
|
|
|
return e.req.CacheInfo()
|
|
|
|
}
|
|
|
|
|
2022-05-25 15:13:23 +00:00
|
|
|
func (e *exportedServiceRequest) getTopic() pbsubscribe.Topic {
|
2022-05-19 19:21:44 +00:00
|
|
|
if e.req.Connect {
|
2022-05-25 15:13:23 +00:00
|
|
|
return pbsubscribe.Topic_ServiceHealthConnect
|
2022-05-19 19:21:44 +00:00
|
|
|
}
|
2022-05-25 15:13:23 +00:00
|
|
|
// Using the Topic_ServiceHealth will ignore proxies unless the ServiceName is a proxy name.
|
|
|
|
return pbsubscribe.Topic_ServiceHealth
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewMaterializer implements submatview.Request
|
|
|
|
func (e *exportedServiceRequest) NewMaterializer() (submatview.Materializer, error) {
|
|
|
|
// TODO(peering): reinstate this
|
|
|
|
// if e.req.Connect {
|
|
|
|
// return nil, fmt.Errorf("connect views are not supported")
|
|
|
|
// }
|
peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
reqFn := func(index uint64) *pbsubscribe.SubscribeRequest {
|
2022-05-19 19:21:44 +00:00
|
|
|
return &pbsubscribe.SubscribeRequest{
|
proxycfg: server-local config entry data sources
This is the OSS portion of enterprise PR 2056.
This commit provides server-local implementations of the proxycfg.ConfigEntry
and proxycfg.ConfigEntryList interfaces, that source data from streaming events.
It makes use of the LocalMaterializer type introduced for peering replication,
adding the necessary support for authorization.
It also adds support for "wildcard" subscriptions (within a topic) to the event
publisher, as this is needed to fetch service-resolvers for all services when
configuring mesh gateways.
Currently, events will be emitted for just the ingress-gateway, service-resolver,
and mesh config entry types, as these are the only entries required by proxycfg
— the events will be emitted on topics named IngressGateway, ServiceResolver,
and MeshConfig topics respectively.
Though these events will only be consumed "locally" for now, they can also be
consumed via the gRPC endpoint (confirmed using grpcurl) so using them from
client agents should be a case of swapping the LocalMaterializer for an
RPCMaterializer.
2022-07-01 15:09:47 +00:00
|
|
|
Topic: e.getTopic(),
|
|
|
|
Subject: &pbsubscribe.SubscribeRequest_NamedSubject{
|
|
|
|
NamedSubject: &pbsubscribe.NamedSubject{
|
|
|
|
Key: e.req.ServiceName,
|
|
|
|
Namespace: e.req.EnterpriseMeta.NamespaceOrEmpty(),
|
|
|
|
Partition: e.req.EnterpriseMeta.PartitionOrEmpty(),
|
|
|
|
},
|
|
|
|
},
|
peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
Token: e.req.Token,
|
|
|
|
Datacenter: e.req.Datacenter,
|
|
|
|
Index: index,
|
|
|
|
}
|
|
|
|
}
|
proxycfg: server-local config entry data sources
This is the OSS portion of enterprise PR 2056.
This commit provides server-local implementations of the proxycfg.ConfigEntry
and proxycfg.ConfigEntryList interfaces, that source data from streaming events.
It makes use of the LocalMaterializer type introduced for peering replication,
adding the necessary support for authorization.
It also adds support for "wildcard" subscriptions (within a topic) to the event
publisher, as this is needed to fetch service-resolvers for all services when
configuring mesh gateways.
Currently, events will be emitted for just the ingress-gateway, service-resolver,
and mesh config entry types, as these are the only entries required by proxycfg
— the events will be emitted on topics named IngressGateway, ServiceResolver,
and MeshConfig topics respectively.
Though these events will only be consumed "locally" for now, they can also be
consumed via the gRPC endpoint (confirmed using grpcurl) so using them from
client agents should be a case of swapping the LocalMaterializer for an
RPCMaterializer.
2022-07-01 15:09:47 +00:00
|
|
|
deps := submatview.LocalMaterializerDeps{
|
|
|
|
Backend: e.sub,
|
|
|
|
ACLResolver: DANGER_NO_AUTH{},
|
|
|
|
Deps: submatview.Deps{
|
|
|
|
View: newExportedServicesView(),
|
|
|
|
Logger: e.logger,
|
|
|
|
Request: reqFn,
|
|
|
|
},
|
peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
}
|
proxycfg: server-local config entry data sources
This is the OSS portion of enterprise PR 2056.
This commit provides server-local implementations of the proxycfg.ConfigEntry
and proxycfg.ConfigEntryList interfaces, that source data from streaming events.
It makes use of the LocalMaterializer type introduced for peering replication,
adding the necessary support for authorization.
It also adds support for "wildcard" subscriptions (within a topic) to the event
publisher, as this is needed to fetch service-resolvers for all services when
configuring mesh gateways.
Currently, events will be emitted for just the ingress-gateway, service-resolver,
and mesh config entry types, as these are the only entries required by proxycfg
— the events will be emitted on topics named IngressGateway, ServiceResolver,
and MeshConfig topics respectively.
Though these events will only be consumed "locally" for now, they can also be
consumed via the gRPC endpoint (confirmed using grpcurl) so using them from
client agents should be a case of swapping the LocalMaterializer for an
RPCMaterializer.
2022-07-01 15:09:47 +00:00
|
|
|
return submatview.NewLocalMaterializer(deps), nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// DANGER_NO_AUTH implements submatview.ACLResolver to short-circuit authorization
|
|
|
|
// in cases where it is handled somewhere else (e.g. in an RPC handler).
|
|
|
|
type DANGER_NO_AUTH struct{}
|
|
|
|
|
|
|
|
func (DANGER_NO_AUTH) ResolveTokenAndDefaultMeta(string, *acl.EnterpriseMeta, *acl.AuthorizerContext) (resolver.Result, error) {
|
|
|
|
return resolver.Result{Authorizer: acl.ManageAll()}, nil
|
peering: initial sync (#12842)
- Add endpoints related to peering: read, list, generate token, initiate peering
- Update node/service/check table indexing to account for peers
- Foundational changes for pushing service updates to a peer
- Plumb peer name through Health.ServiceNodes path
see: ENT-1765, ENT-1280, ENT-1283, ENT-1283, ENT-1756, ENT-1739, ENT-1750, ENT-1679,
ENT-1709, ENT-1704, ENT-1690, ENT-1689, ENT-1702, ENT-1701, ENT-1683, ENT-1663,
ENT-1650, ENT-1678, ENT-1628, ENT-1658, ENT-1640, ENT-1637, ENT-1597, ENT-1634,
ENT-1613, ENT-1616, ENT-1617, ENT-1591, ENT-1588, ENT-1596, ENT-1572, ENT-1555
Co-authored-by: R.B. Boyer <rb@hashicorp.com>
Co-authored-by: freddygv <freddy@hashicorp.com>
Co-authored-by: Chris S. Kim <ckim@hashicorp.com>
Co-authored-by: Evan Culver <eculver@hashicorp.com>
Co-authored-by: Nitya Dhanushkodi <nitya@hashicorp.com>
2022-04-21 22:34:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// Type implements submatview.Request
|
|
|
|
func (e *exportedServiceRequest) Type() string {
|
|
|
|
return "leader.peering.stream.exportedServiceRequest"
|
|
|
|
}
|
|
|
|
|
|
|
|
// exportedServicesView implements submatview.View for storing the view state
|
|
|
|
// of an exported service's health result. We store it as a map to make updates and
|
|
|
|
// deletions a little easier but we could just store a result type
|
|
|
|
// (IndexedCheckServiceNodes) and update it in place for each event - that
|
|
|
|
// involves re-sorting each time etc. though.
|
|
|
|
//
|
|
|
|
// Unlike rpcclient.healthView, there is no need for a filter because for exported services
|
|
|
|
// we export all instances unconditionally.
|
|
|
|
type exportedServicesView struct {
|
|
|
|
state map[string]*pbservice.CheckServiceNode
|
|
|
|
}
|
|
|
|
|
|
|
|
func newExportedServicesView() *exportedServicesView {
|
|
|
|
return &exportedServicesView{
|
|
|
|
state: make(map[string]*pbservice.CheckServiceNode),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Reset implements submatview.View
|
|
|
|
func (s *exportedServicesView) Reset() {
|
|
|
|
s.state = make(map[string]*pbservice.CheckServiceNode)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Update implements submatview.View
|
|
|
|
func (s *exportedServicesView) Update(events []*pbsubscribe.Event) error {
|
|
|
|
for _, event := range events {
|
|
|
|
serviceHealth := event.GetServiceHealth()
|
|
|
|
if serviceHealth == nil {
|
|
|
|
return fmt.Errorf("unexpected event type for service health view: %T",
|
|
|
|
event.GetPayload())
|
|
|
|
}
|
|
|
|
|
|
|
|
id := serviceHealth.CheckServiceNode.UniqueID()
|
|
|
|
switch serviceHealth.Op {
|
|
|
|
case pbsubscribe.CatalogOp_Register:
|
|
|
|
s.state[id] = serviceHealth.CheckServiceNode
|
|
|
|
|
|
|
|
case pbsubscribe.CatalogOp_Deregister:
|
|
|
|
delete(s.state, id)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// Result returns the CheckServiceNodes stored by this view.
|
|
|
|
// Result implements submatview.View
|
|
|
|
func (s *exportedServicesView) Result(index uint64) interface{} {
|
|
|
|
result := pbservice.IndexedCheckServiceNodes{
|
|
|
|
Nodes: make([]*pbservice.CheckServiceNode, 0, len(s.state)),
|
|
|
|
Index: index,
|
|
|
|
}
|
|
|
|
for _, node := range s.state {
|
|
|
|
result.Nodes = append(result.Nodes, node)
|
|
|
|
}
|
|
|
|
sortCheckServiceNodes(&result)
|
|
|
|
|
|
|
|
return &result
|
|
|
|
}
|
|
|
|
|
|
|
|
// sortCheckServiceNodes stable sorts the results to match memdb semantics.
|
|
|
|
func sortCheckServiceNodes(n *pbservice.IndexedCheckServiceNodes) {
|
|
|
|
sort.SliceStable(n.Nodes, func(i, j int) bool {
|
|
|
|
return n.Nodes[i].UniqueID() < n.Nodes[j].UniqueID()
|
|
|
|
})
|
|
|
|
}
|