diff --git a/agent/connect/ca/provider_consul_test.go b/agent/connect/ca/provider_consul_test.go index 13ac312564..adf627333f 100644 --- a/agent/connect/ca/provider_consul_test.go +++ b/agent/connect/ca/provider_consul_test.go @@ -35,7 +35,7 @@ func (c *consulCAMockDelegate) ApplyCARequest(req *structs.CARequest) (interface return true, nil case structs.CAOpDeleteProviderState: - if err := c.state.CADeleteProviderState(req.ProviderState.ID); err != nil { + if err := c.state.CADeleteProviderState(idx+1, req.ProviderState.ID); err != nil { return nil, err } diff --git a/agent/consul/fsm/commands_oss.go b/agent/consul/fsm/commands_oss.go index e0272bf805..793bc83a02 100644 --- a/agent/consul/fsm/commands_oss.go +++ b/agent/consul/fsm/commands_oss.go @@ -205,7 +205,7 @@ func (c *FSM) applyTombstoneOperation(buf []byte, index uint64) interface{} { []metrics.Label{{Name: "op", Value: string(req.Op)}}) switch req.Op { case structs.TombstoneReap: - return c.state.ReapTombstones(req.ReapIndex) + return c.state.ReapTombstones(index, req.ReapIndex) default: c.logger.Warn("Invalid Tombstone operation", "operation", req.Op) return fmt.Errorf("Invalid Tombstone operation '%s'", req.Op) @@ -339,7 +339,7 @@ func (c *FSM) applyConnectCAOperation(buf []byte, index uint64) interface{} { return act case structs.CAOpDeleteProviderState: - if err := c.state.CADeleteProviderState(req.ProviderState.ID); err != nil { + if err := c.state.CADeleteProviderState(index, req.ProviderState.ID); err != nil { return err } @@ -359,7 +359,7 @@ func (c *FSM) applyConnectCAOperation(buf []byte, index uint64) interface{} { } return act case structs.CAOpIncrementProviderSerialNumber: - sn, err := c.state.CAIncrementProviderSerialNumber() + sn, err := c.state.CAIncrementProviderSerialNumber(index) if err != nil { return err } @@ -381,7 +381,9 @@ func (c *FSM) applyConnectCALeafOperation(buf []byte, index uint64) interface{} []metrics.Label{{Name: "op", Value: string(req.Op)}}) switch req.Op { case structs.CALeafOpIncrementIndex: - if err := c.state.CALeafSetIndex(index); err != nil { + // Use current index as the new value as well as the value to write at. + // TODO(banks) do we even use this op any more? + if err := c.state.CALeafSetIndex(index, index); err != nil { return err } return index diff --git a/agent/consul/fsm/fsm.go b/agent/consul/fsm/fsm.go index 7b2b7b7b01..85abe19287 100644 --- a/agent/consul/fsm/fsm.go +++ b/agent/consul/fsm/fsm.go @@ -202,7 +202,9 @@ func (c *FSM) Restore(old io.ReadCloser) error { return fmt.Errorf("Unrecognized msg type %d", msg) } } - restore.Commit() + if err := restore.Commit(); err != nil { + return err + } // External code might be calling State(), so we need to synchronize // here to make sure we swap in the new state store atomically. diff --git a/agent/consul/state/acl.go b/agent/consul/state/acl.go index 87ce9a5e86..c1b6e6aa0f 100644 --- a/agent/consul/state/acl.go +++ b/agent/consul/state/acl.go @@ -288,7 +288,7 @@ func (s *Restore) ACLAuthMethod(method *structs.ACLAuthMethod) error { // ACLBootstrap is used to perform a one-time ACL bootstrap operation on a // cluster to get the first management token. func (s *Store) ACLBootstrap(idx, resetIndex uint64, token *structs.ACLToken, legacy bool) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // We must have initialized before this will ever be possible. @@ -310,16 +310,15 @@ func (s *Store) ACLBootstrap(idx, resetIndex uint64, token *structs.ACLToken, le if err := tx.Insert("index", &IndexEntry{"acl-token-bootstrap", idx}); err != nil { return fmt.Errorf("failed to mark ACL bootstrapping as complete: %v", err) } - tx.Commit() - return nil + return tx.Commit() } // CanBootstrapACLToken checks if bootstrapping is possible and returns the reset index func (s *Store) CanBootstrapACLToken() (bool, uint64, error) { - txn := s.db.Txn(false) + tx := s.db.Txn(false) // Lookup the bootstrap sentinel - out, err := txn.First("index", "id", "acl-token-bootstrap") + out, err := tx.First("index", "id", "acl-token-bootstrap") if err != nil { return false, 0, err } @@ -340,7 +339,7 @@ func (s *Store) CanBootstrapACLToken() (bool, uint64, error) { // to update the name. Unlike the older functions to operate specifically on role or policy links // this function does not itself handle the case where the id cannot be found. Instead the // getName function should handle that and return an error if necessary -func (s *Store) resolveACLLinks(tx *memdb.Txn, links []agentpb.ACLLink, getName func(*memdb.Txn, string) (string, error)) (int, error) { +func (s *Store) resolveACLLinks(tx *txn, links []agentpb.ACLLink, getName func(*txn, string) (string, error)) (int, error) { var numValid int for linkIndex, link := range links { if link.ID != "" { @@ -366,7 +365,7 @@ func (s *Store) resolveACLLinks(tx *memdb.Txn, links []agentpb.ACLLink, getName // associated with the ID of the link. Ideally this will be a no-op if the names are already correct // however if a linked resource was renamed it might be stale. This function will treat the incoming // links with copy-on-write semantics and its output will indicate whether any modifications were made. -func (s *Store) fixupACLLinks(tx *memdb.Txn, original []agentpb.ACLLink, getName func(*memdb.Txn, string) (string, error)) ([]agentpb.ACLLink, bool, error) { +func (s *Store) fixupACLLinks(tx *txn, original []agentpb.ACLLink, getName func(*txn, string) (string, error)) ([]agentpb.ACLLink, bool, error) { owned := false links := original @@ -406,7 +405,7 @@ func (s *Store) fixupACLLinks(tx *memdb.Txn, original []agentpb.ACLLink, getName return links, owned, nil } -func (s *Store) resolveTokenPolicyLinks(tx *memdb.Txn, token *structs.ACLToken, allowMissing bool) (int, error) { +func (s *Store) resolveTokenPolicyLinks(tx *txn, token *structs.ACLToken, allowMissing bool) (int, error) { var numValid int for linkIndex, link := range token.Policies { if link.ID != "" { @@ -434,7 +433,7 @@ func (s *Store) resolveTokenPolicyLinks(tx *memdb.Txn, token *structs.ACLToken, // stale when a linked policy was deleted or renamed. This will correct them and generate a newly allocated // token only when fixes are needed. If the policy links are still accurate then we just return the original // token. -func (s *Store) fixupTokenPolicyLinks(tx *memdb.Txn, original *structs.ACLToken) (*structs.ACLToken, error) { +func (s *Store) fixupTokenPolicyLinks(tx *txn, original *structs.ACLToken) (*structs.ACLToken, error) { owned := false token := original @@ -480,7 +479,7 @@ func (s *Store) fixupTokenPolicyLinks(tx *memdb.Txn, original *structs.ACLToken) return token, nil } -func (s *Store) resolveTokenRoleLinks(tx *memdb.Txn, token *structs.ACLToken, allowMissing bool) (int, error) { +func (s *Store) resolveTokenRoleLinks(tx *txn, token *structs.ACLToken, allowMissing bool) (int, error) { var numValid int for linkIndex, link := range token.Roles { if link.ID != "" { @@ -508,7 +507,7 @@ func (s *Store) resolveTokenRoleLinks(tx *memdb.Txn, token *structs.ACLToken, al // stale when a linked role was deleted or renamed. This will correct them and generate a newly allocated // token only when fixes are needed. If the role links are still accurate then we just return the original // token. -func (s *Store) fixupTokenRoleLinks(tx *memdb.Txn, original *structs.ACLToken) (*structs.ACLToken, error) { +func (s *Store) fixupTokenRoleLinks(tx *txn, original *structs.ACLToken) (*structs.ACLToken, error) { owned := false token := original @@ -554,7 +553,7 @@ func (s *Store) fixupTokenRoleLinks(tx *memdb.Txn, original *structs.ACLToken) ( return token, nil } -func (s *Store) resolveRolePolicyLinks(tx *memdb.Txn, role *structs.ACLRole, allowMissing bool) error { +func (s *Store) resolveRolePolicyLinks(tx *txn, role *structs.ACLRole, allowMissing bool) error { for linkIndex, link := range role.Policies { if link.ID != "" { policy, err := s.getPolicyWithTxn(tx, nil, link.ID, s.aclPolicyGetByID, &role.EnterpriseMeta) @@ -580,7 +579,7 @@ func (s *Store) resolveRolePolicyLinks(tx *memdb.Txn, role *structs.ACLRole, all // stale when a linked policy was deleted or renamed. This will correct them and generate a newly allocated // role only when fixes are needed. If the policy links are still accurate then we just return the original // role. -func (s *Store) fixupRolePolicyLinks(tx *memdb.Txn, original *structs.ACLRole) (*structs.ACLRole, error) { +func (s *Store) fixupRolePolicyLinks(tx *txn, original *structs.ACLRole) (*structs.ACLRole, error) { owned := false role := original @@ -628,7 +627,7 @@ func (s *Store) fixupRolePolicyLinks(tx *memdb.Txn, original *structs.ACLRole) ( // ACLTokenSet is used to insert an ACL rule into the state store. func (s *Store) ACLTokenSet(idx uint64, token *structs.ACLToken, legacy bool) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call set on the ACL @@ -636,12 +635,11 @@ func (s *Store) ACLTokenSet(idx uint64, token *structs.ACLToken, legacy bool) er return err } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) ACLTokenBatchSet(idx uint64, tokens structs.ACLTokens, cas, allowMissingPolicyAndRoleIDs, prohibitUnprivileged bool) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, token := range tokens { @@ -650,13 +648,12 @@ func (s *Store) ACLTokenBatchSet(idx uint64, tokens structs.ACLTokens, cas, allo } } - tx.Commit() - return nil + return tx.Commit() } // aclTokenSetTxn is the inner method used to insert an ACL token with the // proper indexes into the state store. -func (s *Store) aclTokenSetTxn(tx *memdb.Txn, idx uint64, token *structs.ACLToken, cas, allowMissingPolicyAndRoleIDs, prohibitUnprivileged, legacy bool) error { +func (s *Store) aclTokenSetTxn(tx *txn, idx uint64, token *structs.ACLToken, cas, allowMissingPolicyAndRoleIDs, prohibitUnprivileged, legacy bool) error { // Check that the ID is set if token.SecretID == "" { return ErrMissingACLTokenSecret @@ -826,7 +823,7 @@ func (s *Store) ACLTokenBatchGet(ws memdb.WatchSet, accessors []string) (uint64, return idx, tokens, nil } -func (s *Store) aclTokenGetTxn(tx *memdb.Txn, ws memdb.WatchSet, value, index string, entMeta *structs.EnterpriseMeta) (*structs.ACLToken, error) { +func (s *Store) aclTokenGetTxn(tx *txn, ws memdb.WatchSet, value, index string, entMeta *structs.EnterpriseMeta) (*structs.ACLToken, error) { watchCh, rawToken, err := s.aclTokenGetFromIndex(tx, value, index, entMeta) if err != nil { return nil, fmt.Errorf("failed acl token lookup: %v", err) @@ -1021,7 +1018,7 @@ func (s *Store) ACLTokenDeleteByAccessor(idx uint64, accessor string, entMeta *s } func (s *Store) ACLTokenBatchDelete(idx uint64, tokenIDs []string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, tokenID := range tokenIDs { @@ -1030,23 +1027,21 @@ func (s *Store) ACLTokenBatchDelete(idx uint64, tokenIDs []string) error { } } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) aclTokenDelete(idx uint64, value, index string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclTokenDeleteTxn(tx, idx, value, index, entMeta); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclTokenDeleteTxn(tx *memdb.Txn, idx uint64, value, index string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) aclTokenDeleteTxn(tx *txn, idx uint64, value, index string, entMeta *structs.EnterpriseMeta) error { // Look up the existing token _, token, err := s.aclTokenGetFromIndex(tx, value, index, entMeta) if err != nil { @@ -1064,7 +1059,7 @@ func (s *Store) aclTokenDeleteTxn(tx *memdb.Txn, idx uint64, value, index string return s.aclTokenDeleteWithToken(tx, token.(*structs.ACLToken), idx) } -func (s *Store) aclTokenDeleteAllForAuthMethodTxn(tx *memdb.Txn, idx uint64, methodName string, methodMeta *structs.EnterpriseMeta) error { +func (s *Store) aclTokenDeleteAllForAuthMethodTxn(tx *txn, idx uint64, methodName string, methodMeta *structs.EnterpriseMeta) error { // collect all the tokens linked with the given auth method. iter, err := s.aclTokenListByAuthMethod(tx, methodName, methodMeta, structs.WildcardEnterpriseMeta()) if err != nil { @@ -1090,7 +1085,7 @@ func (s *Store) aclTokenDeleteAllForAuthMethodTxn(tx *memdb.Txn, idx uint64, met } func (s *Store) ACLPolicyBatchSet(idx uint64, policies structs.ACLPolicies) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, policy := range policies { @@ -1099,23 +1094,21 @@ func (s *Store) ACLPolicyBatchSet(idx uint64, policies structs.ACLPolicies) erro } } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) ACLPolicySet(idx uint64, policy *structs.ACLPolicy) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclPolicySetTxn(tx, idx, policy); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclPolicySetTxn(tx *memdb.Txn, idx uint64, policy *structs.ACLPolicy) error { +func (s *Store) aclPolicySetTxn(tx *txn, idx uint64, policy *structs.ACLPolicy) error { // Check that the ID is set if policy.ID == "" { return ErrMissingACLPolicyID @@ -1209,9 +1202,9 @@ func (s *Store) ACLPolicyBatchGet(ws memdb.WatchSet, ids []string) (uint64, stru return idx, policies, nil } -type aclPolicyGetFn func(*memdb.Txn, string, *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) +type aclPolicyGetFn func(*txn, string, *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) -func (s *Store) getPolicyWithTxn(tx *memdb.Txn, ws memdb.WatchSet, value string, fn aclPolicyGetFn, entMeta *structs.EnterpriseMeta) (*structs.ACLPolicy, error) { +func (s *Store) getPolicyWithTxn(tx *txn, ws memdb.WatchSet, value string, fn aclPolicyGetFn, entMeta *structs.EnterpriseMeta) (*structs.ACLPolicy, error) { watchCh, policy, err := fn(tx, value, entMeta) if err != nil { return nil, fmt.Errorf("failed acl policy lookup: %v", err) @@ -1269,7 +1262,7 @@ func (s *Store) ACLPolicyDeleteByName(idx uint64, name string, entMeta *structs. } func (s *Store) ACLPolicyBatchDelete(idx uint64, policyIDs []string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, policyID := range policyIDs { @@ -1277,23 +1270,21 @@ func (s *Store) ACLPolicyBatchDelete(idx uint64, policyIDs []string) error { return err } } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) aclPolicyDelete(idx uint64, value string, fn aclPolicyGetFn, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclPolicyDeleteTxn(tx, idx, value, fn, entMeta); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclPolicyDeleteTxn(tx *memdb.Txn, idx uint64, value string, fn aclPolicyGetFn, entMeta *structs.EnterpriseMeta) error { +func (s *Store) aclPolicyDeleteTxn(tx *txn, idx uint64, value string, fn aclPolicyGetFn, entMeta *structs.EnterpriseMeta) error { // Look up the existing token _, rawPolicy, err := fn(tx, value, entMeta) if err != nil { @@ -1314,7 +1305,7 @@ func (s *Store) aclPolicyDeleteTxn(tx *memdb.Txn, idx uint64, value string, fn a } func (s *Store) ACLRoleBatchSet(idx uint64, roles structs.ACLRoles, allowMissingPolicyIDs bool) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, role := range roles { @@ -1323,23 +1314,21 @@ func (s *Store) ACLRoleBatchSet(idx uint64, roles structs.ACLRoles, allowMissing } } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) ACLRoleSet(idx uint64, role *structs.ACLRole) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclRoleSetTxn(tx, idx, role, false); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclRoleSetTxn(tx *memdb.Txn, idx uint64, role *structs.ACLRole, allowMissing bool) error { +func (s *Store) aclRoleSetTxn(tx *txn, idx uint64, role *structs.ACLRole, allowMissing bool) error { // Check that the ID is set if role.ID == "" { return ErrMissingACLRoleID @@ -1403,7 +1392,7 @@ func (s *Store) aclRoleSetTxn(tx *memdb.Txn, idx uint64, role *structs.ACLRole, return s.aclRoleInsert(tx, role) } -type aclRoleGetFn func(*memdb.Txn, string, *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) +type aclRoleGetFn func(*txn, string, *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) func (s *Store) ACLRoleGetByID(ws memdb.WatchSet, id string, entMeta *structs.EnterpriseMeta) (uint64, *structs.ACLRole, error) { return s.aclRoleGet(ws, id, s.aclRoleGetByID, entMeta) @@ -1434,7 +1423,7 @@ func (s *Store) ACLRoleBatchGet(ws memdb.WatchSet, ids []string) (uint64, struct return idx, roles, nil } -func (s *Store) getRoleWithTxn(tx *memdb.Txn, ws memdb.WatchSet, value string, fn aclRoleGetFn, entMeta *structs.EnterpriseMeta) (*structs.ACLRole, error) { +func (s *Store) getRoleWithTxn(tx *txn, ws memdb.WatchSet, value string, fn aclRoleGetFn, entMeta *structs.EnterpriseMeta) (*structs.ACLRole, error) { watchCh, rawRole, err := fn(tx, value, entMeta) if err != nil { return nil, fmt.Errorf("failed acl role lookup: %v", err) @@ -1510,7 +1499,7 @@ func (s *Store) ACLRoleDeleteByName(idx uint64, name string, entMeta *structs.En } func (s *Store) ACLRoleBatchDelete(idx uint64, roleIDs []string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, roleID := range roleIDs { @@ -1518,23 +1507,21 @@ func (s *Store) ACLRoleBatchDelete(idx uint64, roleIDs []string) error { return err } } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) aclRoleDelete(idx uint64, value string, fn aclRoleGetFn, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclRoleDeleteTxn(tx, idx, value, fn, entMeta); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclRoleDeleteTxn(tx *memdb.Txn, idx uint64, value string, fn aclRoleGetFn, entMeta *structs.EnterpriseMeta) error { +func (s *Store) aclRoleDeleteTxn(tx *txn, idx uint64, value string, fn aclRoleGetFn, entMeta *structs.EnterpriseMeta) error { // Look up the existing role _, rawRole, err := fn(tx, value, entMeta) if err != nil { @@ -1551,7 +1538,7 @@ func (s *Store) aclRoleDeleteTxn(tx *memdb.Txn, idx uint64, value string, fn acl } func (s *Store) ACLBindingRuleBatchSet(idx uint64, rules structs.ACLBindingRules) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, rule := range rules { @@ -1560,22 +1547,20 @@ func (s *Store) ACLBindingRuleBatchSet(idx uint64, rules structs.ACLBindingRules } } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) ACLBindingRuleSet(idx uint64, rule *structs.ACLBindingRule) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclBindingRuleSetTxn(tx, idx, rule); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclBindingRuleSetTxn(tx *memdb.Txn, idx uint64, rule *structs.ACLBindingRule) error { +func (s *Store) aclBindingRuleSetTxn(tx *txn, idx uint64, rule *structs.ACLBindingRule) error { // Check that the ID and AuthMethod are set if rule.ID == "" { return ErrMissingACLBindingRuleID @@ -1671,29 +1656,27 @@ func (s *Store) ACLBindingRuleDeleteByID(idx uint64, id string, entMeta *structs } func (s *Store) ACLBindingRuleBatchDelete(idx uint64, bindingRuleIDs []string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, bindingRuleID := range bindingRuleIDs { s.aclBindingRuleDeleteTxn(tx, idx, bindingRuleID, nil) } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) aclBindingRuleDelete(idx uint64, id string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclBindingRuleDeleteTxn(tx, idx, id, entMeta); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclBindingRuleDeleteTxn(tx *memdb.Txn, idx uint64, id string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) aclBindingRuleDeleteTxn(tx *txn, idx uint64, id string, entMeta *structs.EnterpriseMeta) error { // Look up the existing binding rule _, rawRule, err := s.aclBindingRuleGetByID(tx, id, entMeta) if err != nil { @@ -1712,7 +1695,7 @@ func (s *Store) aclBindingRuleDeleteTxn(tx *memdb.Txn, idx uint64, id string, en return nil } -func (s *Store) aclBindingRuleDeleteAllForAuthMethodTxn(tx *memdb.Txn, idx uint64, methodName string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) aclBindingRuleDeleteAllForAuthMethodTxn(tx *txn, idx uint64, methodName string, entMeta *structs.EnterpriseMeta) error { // collect them all iter, err := s.aclBindingRuleListByAuthMethod(tx, methodName, entMeta) if err != nil { @@ -1738,7 +1721,7 @@ func (s *Store) aclBindingRuleDeleteAllForAuthMethodTxn(tx *memdb.Txn, idx uint6 } func (s *Store) ACLAuthMethodBatchSet(idx uint64, methods structs.ACLAuthMethods) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, method := range methods { @@ -1748,23 +1731,21 @@ func (s *Store) ACLAuthMethodBatchSet(idx uint64, methods structs.ACLAuthMethods return err } } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) ACLAuthMethodSet(idx uint64, method *structs.ACLAuthMethod) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclAuthMethodSetTxn(tx, idx, method); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclAuthMethodSetTxn(tx *memdb.Txn, idx uint64, method *structs.ACLAuthMethod) error { +func (s *Store) aclAuthMethodSetTxn(tx *txn, idx uint64, method *structs.ACLAuthMethod) error { // Check that the Name and Type are set if method.Name == "" { return ErrMissingACLAuthMethodName @@ -1813,7 +1794,7 @@ func (s *Store) aclAuthMethodGet(ws memdb.WatchSet, name string, entMeta *struct return idx, method, nil } -func (s *Store) getAuthMethodWithTxn(tx *memdb.Txn, ws memdb.WatchSet, name string, entMeta *structs.EnterpriseMeta) (*structs.ACLAuthMethod, error) { +func (s *Store) getAuthMethodWithTxn(tx *txn, ws memdb.WatchSet, name string, entMeta *structs.EnterpriseMeta) (*structs.ACLAuthMethod, error) { watchCh, rawMethod, err := s.aclAuthMethodGetByName(tx, name, entMeta) if err != nil { return nil, fmt.Errorf("failed acl auth method lookup: %v", err) @@ -1854,7 +1835,7 @@ func (s *Store) ACLAuthMethodDeleteByName(idx uint64, name string, entMeta *stru } func (s *Store) ACLAuthMethodBatchDelete(idx uint64, names []string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, name := range names { @@ -1865,23 +1846,21 @@ func (s *Store) ACLAuthMethodBatchDelete(idx uint64, names []string, entMeta *st s.aclAuthMethodDeleteTxn(tx, idx, name, entMeta) } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) aclAuthMethodDelete(idx uint64, name string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.aclAuthMethodDeleteTxn(tx, idx, name, entMeta); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) aclAuthMethodDeleteTxn(tx *memdb.Txn, idx uint64, name string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) aclAuthMethodDeleteTxn(tx *txn, idx uint64, name string, entMeta *structs.EnterpriseMeta) error { // Look up the existing method _, rawMethod, err := s.aclAuthMethodGetByName(tx, name, entMeta) if err != nil { diff --git a/agent/consul/state/acl_oss.go b/agent/consul/state/acl_oss.go index 2970ea77ff..14e9e54904 100644 --- a/agent/consul/state/acl_oss.go +++ b/agent/consul/state/acl_oss.go @@ -206,7 +206,7 @@ func authMethodsTableSchema() *memdb.TableSchema { ///// ACL Policy Functions ///// /////////////////////////////////////////////////////////////////////////////// -func (s *Store) aclPolicyInsert(tx *memdb.Txn, policy *structs.ACLPolicy) error { +func (s *Store) aclPolicyInsert(tx *txn, policy *structs.ACLPolicy) error { if err := tx.Insert("acl-policies", policy); err != nil { return fmt.Errorf("failed inserting acl policy: %v", err) } @@ -218,19 +218,19 @@ func (s *Store) aclPolicyInsert(tx *memdb.Txn, policy *structs.ACLPolicy) error return nil } -func (s *Store) aclPolicyGetByID(tx *memdb.Txn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) aclPolicyGetByID(tx *txn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("acl-policies", "id", id) } -func (s *Store) aclPolicyGetByName(tx *memdb.Txn, name string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) aclPolicyGetByName(tx *txn, name string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("acl-policies", "name", name) } -func (s *Store) aclPolicyList(tx *memdb.Txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclPolicyList(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-policies", "id") } -func (s *Store) aclPolicyDeleteWithPolicy(tx *memdb.Txn, policy *structs.ACLPolicy, idx uint64) error { +func (s *Store) aclPolicyDeleteWithPolicy(tx *txn, policy *structs.ACLPolicy, idx uint64) error { // remove the policy if err := tx.Delete("acl-policies", policy); err != nil { return fmt.Errorf("failed deleting acl policy: %v", err) @@ -243,11 +243,11 @@ func (s *Store) aclPolicyDeleteWithPolicy(tx *memdb.Txn, policy *structs.ACLPoli return nil } -func (s *Store) aclPolicyMaxIndex(tx *memdb.Txn, _ *structs.ACLPolicy, _ *structs.EnterpriseMeta) uint64 { +func (s *Store) aclPolicyMaxIndex(tx *txn, _ *structs.ACLPolicy, _ *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "acl-policies") } -func (s *Store) aclPolicyUpsertValidateEnterprise(*memdb.Txn, *structs.ACLPolicy, *structs.ACLPolicy) error { +func (s *Store) aclPolicyUpsertValidateEnterprise(*txn, *structs.ACLPolicy, *structs.ACLPolicy) error { return nil } @@ -259,7 +259,7 @@ func (s *Store) ACLPolicyUpsertValidateEnterprise(*structs.ACLPolicy, *structs.A ///// ACL Token Functions ///// /////////////////////////////////////////////////////////////////////////////// -func (s *Store) aclTokenInsert(tx *memdb.Txn, token *structs.ACLToken) error { +func (s *Store) aclTokenInsert(tx *txn, token *structs.ACLToken) error { // insert the token into memdb if err := tx.Insert("acl-tokens", token); err != nil { return fmt.Errorf("failed inserting acl token: %v", err) @@ -273,35 +273,35 @@ func (s *Store) aclTokenInsert(tx *memdb.Txn, token *structs.ACLToken) error { return nil } -func (s *Store) aclTokenGetFromIndex(tx *memdb.Txn, id string, index string, entMeta *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) aclTokenGetFromIndex(tx *txn, id string, index string, entMeta *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("acl-tokens", index, id) } -func (s *Store) aclTokenListAll(tx *memdb.Txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclTokenListAll(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-tokens", "id") } -func (s *Store) aclTokenListLocal(tx *memdb.Txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclTokenListLocal(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-tokens", "local", true) } -func (s *Store) aclTokenListGlobal(tx *memdb.Txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclTokenListGlobal(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-tokens", "local", false) } -func (s *Store) aclTokenListByPolicy(tx *memdb.Txn, policy string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclTokenListByPolicy(tx *txn, policy string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-tokens", "policies", policy) } -func (s *Store) aclTokenListByRole(tx *memdb.Txn, role string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclTokenListByRole(tx *txn, role string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-tokens", "roles", role) } -func (s *Store) aclTokenListByAuthMethod(tx *memdb.Txn, authMethod string, _, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclTokenListByAuthMethod(tx *txn, authMethod string, _, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-tokens", "authmethod", authMethod) } -func (s *Store) aclTokenDeleteWithToken(tx *memdb.Txn, token *structs.ACLToken, idx uint64) error { +func (s *Store) aclTokenDeleteWithToken(tx *txn, token *structs.ACLToken, idx uint64) error { // remove the token if err := tx.Delete("acl-tokens", token); err != nil { return fmt.Errorf("failed deleting acl token: %v", err) @@ -314,11 +314,11 @@ func (s *Store) aclTokenDeleteWithToken(tx *memdb.Txn, token *structs.ACLToken, return nil } -func (s *Store) aclTokenMaxIndex(tx *memdb.Txn, _ *structs.ACLToken, entMeta *structs.EnterpriseMeta) uint64 { +func (s *Store) aclTokenMaxIndex(tx *txn, _ *structs.ACLToken, entMeta *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "acl-tokens") } -func (s *Store) aclTokenUpsertValidateEnterprise(tx *memdb.Txn, token *structs.ACLToken, existing *structs.ACLToken) error { +func (s *Store) aclTokenUpsertValidateEnterprise(tx *txn, token *structs.ACLToken, existing *structs.ACLToken) error { return nil } @@ -330,7 +330,7 @@ func (s *Store) ACLTokenUpsertValidateEnterprise(token *structs.ACLToken, existi ///// ACL Role Functions ///// /////////////////////////////////////////////////////////////////////////////// -func (s *Store) aclRoleInsert(tx *memdb.Txn, role *structs.ACLRole) error { +func (s *Store) aclRoleInsert(tx *txn, role *structs.ACLRole) error { // insert the role into memdb if err := tx.Insert("acl-roles", role); err != nil { return fmt.Errorf("failed inserting acl role: %v", err) @@ -343,23 +343,23 @@ func (s *Store) aclRoleInsert(tx *memdb.Txn, role *structs.ACLRole) error { return nil } -func (s *Store) aclRoleGetByID(tx *memdb.Txn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) aclRoleGetByID(tx *txn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("acl-roles", "id", id) } -func (s *Store) aclRoleGetByName(tx *memdb.Txn, name string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) aclRoleGetByName(tx *txn, name string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("acl-roles", "name", name) } -func (s *Store) aclRoleList(tx *memdb.Txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclRoleList(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-roles", "id") } -func (s *Store) aclRoleListByPolicy(tx *memdb.Txn, policy string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclRoleListByPolicy(tx *txn, policy string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-roles", "policies", policy) } -func (s *Store) aclRoleDeleteWithRole(tx *memdb.Txn, role *structs.ACLRole, idx uint64) error { +func (s *Store) aclRoleDeleteWithRole(tx *txn, role *structs.ACLRole, idx uint64) error { // remove the role if err := tx.Delete("acl-roles", role); err != nil { return fmt.Errorf("failed deleting acl role: %v", err) @@ -372,11 +372,11 @@ func (s *Store) aclRoleDeleteWithRole(tx *memdb.Txn, role *structs.ACLRole, idx return nil } -func (s *Store) aclRoleMaxIndex(tx *memdb.Txn, _ *structs.ACLRole, _ *structs.EnterpriseMeta) uint64 { +func (s *Store) aclRoleMaxIndex(tx *txn, _ *structs.ACLRole, _ *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "acl-roles") } -func (s *Store) aclRoleUpsertValidateEnterprise(tx *memdb.Txn, role *structs.ACLRole, existing *structs.ACLRole) error { +func (s *Store) aclRoleUpsertValidateEnterprise(tx *txn, role *structs.ACLRole, existing *structs.ACLRole) error { return nil } @@ -388,7 +388,7 @@ func (s *Store) ACLRoleUpsertValidateEnterprise(role *structs.ACLRole, existing ///// ACL Binding Rule Functions ///// /////////////////////////////////////////////////////////////////////////////// -func (s *Store) aclBindingRuleInsert(tx *memdb.Txn, rule *structs.ACLBindingRule) error { +func (s *Store) aclBindingRuleInsert(tx *txn, rule *structs.ACLBindingRule) error { // insert the role into memdb if err := tx.Insert("acl-binding-rules", rule); err != nil { return fmt.Errorf("failed inserting acl role: %v", err) @@ -402,19 +402,19 @@ func (s *Store) aclBindingRuleInsert(tx *memdb.Txn, rule *structs.ACLBindingRule return nil } -func (s *Store) aclBindingRuleGetByID(tx *memdb.Txn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) aclBindingRuleGetByID(tx *txn, id string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("acl-binding-rules", "id", id) } -func (s *Store) aclBindingRuleList(tx *memdb.Txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclBindingRuleList(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-binding-rules", "id") } -func (s *Store) aclBindingRuleListByAuthMethod(tx *memdb.Txn, method string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclBindingRuleListByAuthMethod(tx *txn, method string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-binding-rules", "authmethod", method) } -func (s *Store) aclBindingRuleDeleteWithRule(tx *memdb.Txn, rule *structs.ACLBindingRule, idx uint64) error { +func (s *Store) aclBindingRuleDeleteWithRule(tx *txn, rule *structs.ACLBindingRule, idx uint64) error { // remove the rule if err := tx.Delete("acl-binding-rules", rule); err != nil { return fmt.Errorf("failed deleting acl binding rule: %v", err) @@ -427,11 +427,11 @@ func (s *Store) aclBindingRuleDeleteWithRule(tx *memdb.Txn, rule *structs.ACLBin return nil } -func (s *Store) aclBindingRuleMaxIndex(tx *memdb.Txn, _ *structs.ACLBindingRule, entMeta *structs.EnterpriseMeta) uint64 { +func (s *Store) aclBindingRuleMaxIndex(tx *txn, _ *structs.ACLBindingRule, entMeta *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "acl-binding-rules") } -func (s *Store) aclBindingRuleUpsertValidateEnterprise(tx *memdb.Txn, rule *structs.ACLBindingRule, existing *structs.ACLBindingRule) error { +func (s *Store) aclBindingRuleUpsertValidateEnterprise(tx *txn, rule *structs.ACLBindingRule, existing *structs.ACLBindingRule) error { return nil } @@ -443,7 +443,7 @@ func (s *Store) ACLBindingRuleUpsertValidateEnterprise(rule *structs.ACLBindingR ///// ACL Auth Method Functions ///// /////////////////////////////////////////////////////////////////////////////// -func (s *Store) aclAuthMethodInsert(tx *memdb.Txn, method *structs.ACLAuthMethod) error { +func (s *Store) aclAuthMethodInsert(tx *txn, method *structs.ACLAuthMethod) error { // insert the role into memdb if err := tx.Insert("acl-auth-methods", method); err != nil { return fmt.Errorf("failed inserting acl role: %v", err) @@ -457,15 +457,15 @@ func (s *Store) aclAuthMethodInsert(tx *memdb.Txn, method *structs.ACLAuthMethod return nil } -func (s *Store) aclAuthMethodGetByName(tx *memdb.Txn, method string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) aclAuthMethodGetByName(tx *txn, method string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("acl-auth-methods", "id", method) } -func (s *Store) aclAuthMethodList(tx *memdb.Txn, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) aclAuthMethodList(tx *txn, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("acl-auth-methods", "id") } -func (s *Store) aclAuthMethodDeleteWithMethod(tx *memdb.Txn, method *structs.ACLAuthMethod, idx uint64) error { +func (s *Store) aclAuthMethodDeleteWithMethod(tx *txn, method *structs.ACLAuthMethod, idx uint64) error { // remove the method if err := tx.Delete("acl-auth-methods", method); err != nil { return fmt.Errorf("failed deleting acl auth method: %v", err) @@ -478,11 +478,11 @@ func (s *Store) aclAuthMethodDeleteWithMethod(tx *memdb.Txn, method *structs.ACL return nil } -func (s *Store) aclAuthMethodMaxIndex(tx *memdb.Txn, _ *structs.ACLAuthMethod, entMeta *structs.EnterpriseMeta) uint64 { +func (s *Store) aclAuthMethodMaxIndex(tx *txn, _ *structs.ACLAuthMethod, entMeta *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "acl-auth-methods") } -func (s *Store) aclAuthMethodUpsertValidateEnterprise(tx *memdb.Txn, method *structs.ACLAuthMethod, existing *structs.ACLAuthMethod) error { +func (s *Store) aclAuthMethodUpsertValidateEnterprise(tx *txn, method *structs.ACLAuthMethod, existing *structs.ACLAuthMethod) error { return nil } diff --git a/agent/consul/state/acl_test.go b/agent/consul/state/acl_test.go index e5b665fb87..843629d08e 100644 --- a/agent/consul/state/acl_test.go +++ b/agent/consul/state/acl_test.go @@ -4105,7 +4105,7 @@ func TestStateStore_resolveACLLinks(t *testing.T) { }, } - _, err := s.resolveACLLinks(tx, links, func(*memdb.Txn, string) (string, error) { + _, err := s.resolveACLLinks(tx, links, func(*txn, string) (string, error) { err := fmt.Errorf("Should not be attempting to resolve an empty id") require.Fail(t, err.Error()) return "", err @@ -4131,7 +4131,7 @@ func TestStateStore_resolveACLLinks(t *testing.T) { }, } - numValid, err := s.resolveACLLinks(tx, links, func(_ *memdb.Txn, linkID string) (string, error) { + numValid, err := s.resolveACLLinks(tx, links, func(_ *txn, linkID string) (string, error) { switch linkID { case "e81887b4-836b-4053-a1fa-7e8305902be9": return "foo", nil @@ -4161,7 +4161,7 @@ func TestStateStore_resolveACLLinks(t *testing.T) { }, } - numValid, err := s.resolveACLLinks(tx, links, func(_ *memdb.Txn, linkID string) (string, error) { + numValid, err := s.resolveACLLinks(tx, links, func(_ *txn, linkID string) (string, error) { require.Equal(t, "b985e082-25d3-45a9-9dd8-fd1a41b83b0d", linkID) return "", nil }) @@ -4201,7 +4201,7 @@ func TestStateStore_fixupACLLinks(t *testing.T) { tx := s.db.Txn(false) defer tx.Abort() - newLinks, cloned, err := s.fixupACLLinks(tx, links, func(_ *memdb.Txn, linkID string) (string, error) { + newLinks, cloned, err := s.fixupACLLinks(tx, links, func(_ *txn, linkID string) (string, error) { switch linkID { case "40b57f86-97ea-40e4-a99a-c399cc81f4dd": return "foo", nil @@ -4228,7 +4228,7 @@ func TestStateStore_fixupACLLinks(t *testing.T) { tx := s.db.Txn(false) defer tx.Abort() - newLinks, cloned, err := s.fixupACLLinks(tx, links, func(_ *memdb.Txn, linkID string) (string, error) { + newLinks, cloned, err := s.fixupACLLinks(tx, links, func(_ *txn, linkID string) (string, error) { switch linkID { case "40b57f86-97ea-40e4-a99a-c399cc81f4dd": return "foo", nil @@ -4260,7 +4260,7 @@ func TestStateStore_fixupACLLinks(t *testing.T) { tx := s.db.Txn(false) defer tx.Abort() - newLinks, cloned, err := s.fixupACLLinks(tx, links, func(_ *memdb.Txn, linkID string) (string, error) { + newLinks, cloned, err := s.fixupACLLinks(tx, links, func(_ *txn, linkID string) (string, error) { switch linkID { case "40b57f86-97ea-40e4-a99a-c399cc81f4dd": return "foo", nil @@ -4287,7 +4287,7 @@ func TestStateStore_fixupACLLinks(t *testing.T) { tx := s.db.Txn(false) defer tx.Abort() - _, _, err := s.fixupACLLinks(tx, links, func(*memdb.Txn, string) (string, error) { + _, _, err := s.fixupACLLinks(tx, links, func(*txn, string) (string, error) { return "", fmt.Errorf("Resolver Error") }) diff --git a/agent/consul/state/autopilot.go b/agent/consul/state/autopilot.go index 7c03dd7ad9..06ab5d3d24 100644 --- a/agent/consul/state/autopilot.go +++ b/agent/consul/state/autopilot.go @@ -74,22 +74,21 @@ func (s *Store) AutopilotConfig() (uint64, *autopilot.Config, error) { // AutopilotSetConfig is used to set the current Autopilot configuration. func (s *Store) AutopilotSetConfig(idx uint64, config *autopilot.Config) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.autopilotSetConfigTxn(idx, tx, config); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } // AutopilotCASConfig is used to try updating the Autopilot configuration with a // given Raft index. If the CAS index specified is not equal to the last observed index // for the config, then the call is a noop, func (s *Store) AutopilotCASConfig(idx, cidx uint64, config *autopilot.Config) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Check for an existing config @@ -110,11 +109,11 @@ func (s *Store) AutopilotCASConfig(idx, cidx uint64, config *autopilot.Config) ( return false, err } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } -func (s *Store) autopilotSetConfigTxn(idx uint64, tx *memdb.Txn, config *autopilot.Config) error { +func (s *Store) autopilotSetConfigTxn(idx uint64, tx *txn, config *autopilot.Config) error { // Check for an existing config existing, err := tx.First("autopilot-config", "id") if err != nil { diff --git a/agent/consul/state/catalog.go b/agent/consul/state/catalog.go index 575213d177..33127c39e2 100644 --- a/agent/consul/state/catalog.go +++ b/agent/consul/state/catalog.go @@ -10,7 +10,7 @@ import ( "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/types" - "github.com/hashicorp/go-memdb" + memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/go-uuid" ) @@ -226,18 +226,17 @@ func (s *Restore) Registration(idx uint64, req *structs.RegisterRequest) error { // registration is performed within a single transaction to avoid race // conditions on state updates. func (s *Store) EnsureRegistration(idx uint64, req *structs.RegisterRequest) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.ensureRegistrationTxn(tx, idx, req); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) ensureCheckIfNodeMatches(tx *memdb.Txn, idx uint64, node string, check *structs.HealthCheck) error { +func (s *Store) ensureCheckIfNodeMatches(tx *txn, idx uint64, node string, check *structs.HealthCheck) error { if check.Node != node { return fmt.Errorf("check node %q does not match node %q", check.Node, node) @@ -251,7 +250,7 @@ func (s *Store) ensureCheckIfNodeMatches(tx *memdb.Txn, idx uint64, node string, // ensureRegistrationTxn is used to make sure a node, service, and check // registration is performed within a single transaction to avoid race // conditions on state updates. -func (s *Store) ensureRegistrationTxn(tx *memdb.Txn, idx uint64, req *structs.RegisterRequest) error { +func (s *Store) ensureRegistrationTxn(tx *txn, idx uint64, req *structs.RegisterRequest) error { if _, err := s.validateRegisterRequestTxn(tx, req); err != nil { return err } @@ -316,7 +315,7 @@ func (s *Store) ensureRegistrationTxn(tx *memdb.Txn, idx uint64, req *structs.Re // EnsureNode is used to upsert node registration or modification. func (s *Store) EnsureNode(idx uint64, node *structs.Node) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call the node upsert @@ -324,13 +323,12 @@ func (s *Store) EnsureNode(idx uint64, node *structs.Node) error { return err } - tx.Commit() - return nil + return tx.Commit() } // ensureNoNodeWithSimilarNameTxn checks that no other node has conflict in its name // If allowClashWithoutID then, getting a conflict on another node without ID will be allowed -func (s *Store) ensureNoNodeWithSimilarNameTxn(tx *memdb.Txn, node *structs.Node, allowClashWithoutID bool) error { +func (s *Store) ensureNoNodeWithSimilarNameTxn(tx *txn, node *structs.Node, allowClashWithoutID bool) error { // Retrieve all of the nodes enodes, err := tx.Get("nodes", "id") if err != nil { @@ -366,7 +364,7 @@ func (s *Store) ensureNoNodeWithSimilarNameTxn(tx *memdb.Txn, node *structs.Node // ensureNodeCASTxn updates a node only if the existing index matches the given index. // Returns a bool indicating if a write happened and any error. -func (s *Store) ensureNodeCASTxn(tx *memdb.Txn, idx uint64, node *structs.Node) (bool, error) { +func (s *Store) ensureNodeCASTxn(tx *txn, idx uint64, node *structs.Node) (bool, error) { // Retrieve the existing entry. existing, err := getNodeTxn(tx, node.Node) if err != nil { @@ -396,7 +394,7 @@ func (s *Store) ensureNodeCASTxn(tx *memdb.Txn, idx uint64, node *structs.Node) // ensureNodeTxn is the inner function called to actually create a node // registration or modify an existing one in the state store. It allows // passing in a memdb transaction so it may be part of a larger txn. -func (s *Store) ensureNodeTxn(tx *memdb.Txn, idx uint64, node *structs.Node) error { +func (s *Store) ensureNodeTxn(tx *txn, idx uint64, node *structs.Node) error { // See if there's an existing node with this UUID, and make sure the // name is the same. var n *structs.Node @@ -494,7 +492,7 @@ func (s *Store) GetNode(id string) (uint64, *structs.Node, error) { return idx, node, nil } -func getNodeTxn(tx *memdb.Txn, nodeName string) (*structs.Node, error) { +func getNodeTxn(tx *txn, nodeName string) (*structs.Node, error) { node, err := tx.First("nodes", "id", nodeName) if err != nil { return nil, fmt.Errorf("node lookup failed: %s", err) @@ -505,7 +503,7 @@ func getNodeTxn(tx *memdb.Txn, nodeName string) (*structs.Node, error) { return nil, nil } -func getNodeIDTxn(tx *memdb.Txn, id types.NodeID) (*structs.Node, error) { +func getNodeIDTxn(tx *txn, id types.NodeID) (*structs.Node, error) { strnode := string(id) uuidValue, err := uuid.ParseUUID(strnode) if err != nil { @@ -591,7 +589,7 @@ func (s *Store) NodesByMeta(ws memdb.WatchSet, filters map[string]string) (uint6 // DeleteNode is used to delete a given node by its ID. func (s *Store) DeleteNode(idx uint64, nodeName string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call the node deletion. @@ -599,14 +597,13 @@ func (s *Store) DeleteNode(idx uint64, nodeName string) error { return err } - tx.Commit() - return nil + return tx.Commit() } // deleteNodeCASTxn is used to try doing a node delete operation with a given // raft index. If the CAS index specified is not equal to the last observed index for // the given check, then the call is a noop, otherwise a normal check delete is invoked. -func (s *Store) deleteNodeCASTxn(tx *memdb.Txn, idx, cidx uint64, nodeName string) (bool, error) { +func (s *Store) deleteNodeCASTxn(tx *txn, idx, cidx uint64, nodeName string) (bool, error) { // Look up the node. node, err := getNodeTxn(tx, nodeName) if err != nil { @@ -633,7 +630,7 @@ func (s *Store) deleteNodeCASTxn(tx *memdb.Txn, idx, cidx uint64, nodeName strin // deleteNodeTxn is the inner method used for removing a node from // the store within a given transaction. -func (s *Store) deleteNodeTxn(tx *memdb.Txn, idx uint64, nodeName string) error { +func (s *Store) deleteNodeTxn(tx *txn, idx uint64, nodeName string) error { // Look up the node. node, err := tx.First("nodes", "id", nodeName) if err != nil { @@ -725,7 +722,7 @@ func (s *Store) deleteNodeTxn(tx *memdb.Txn, idx uint64, nodeName string) error // EnsureService is called to upsert creation of a given NodeService. func (s *Store) EnsureService(idx uint64, node string, svc *structs.NodeService) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call the service registration upsert @@ -733,15 +730,14 @@ func (s *Store) EnsureService(idx uint64, node string, svc *structs.NodeService) return err } - tx.Commit() - return nil + return tx.Commit() } var errCASCompareFailed = errors.New("compare-and-set: comparison failed") // ensureServiceCASTxn updates a service only if the existing index matches the given index. // Returns an error if the write didn't happen and nil if write was successful. -func (s *Store) ensureServiceCASTxn(tx *memdb.Txn, idx uint64, node string, svc *structs.NodeService) error { +func (s *Store) ensureServiceCASTxn(tx *txn, idx uint64, node string, svc *structs.NodeService) error { // Retrieve the existing service. _, existing, err := firstWatchCompoundWithTxn(tx, "services", "id", &svc.EnterpriseMeta, node, svc.ID) if err != nil { @@ -766,7 +762,7 @@ func (s *Store) ensureServiceCASTxn(tx *memdb.Txn, idx uint64, node string, svc // ensureServiceTxn is used to upsert a service registration within an // existing memdb transaction. -func (s *Store) ensureServiceTxn(tx *memdb.Txn, idx uint64, node string, svc *structs.NodeService) error { +func (s *Store) ensureServiceTxn(tx *txn, idx uint64, node string, svc *structs.NodeService) error { // Check for existing service _, existing, err := firstWatchCompoundWithTxn(tx, "services", "id", &svc.EnterpriseMeta, node, svc.ID) if err != nil { @@ -863,7 +859,7 @@ func (s *Store) ServiceList(ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) return s.serviceListTxn(tx, ws, entMeta) } -func (s *Store) serviceListTxn(tx *memdb.Txn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.ServiceList, error) { +func (s *Store) serviceListTxn(tx *txn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.ServiceList, error) { idx := s.catalogServicesMaxIndex(tx, entMeta) services, err := s.catalogServiceList(tx, entMeta, true) @@ -967,7 +963,7 @@ func (s *Store) ServicesByNodeMeta(ws memdb.WatchSet, filters map[string]string, // * return when the last instance of a service is removed // * block until an instance for this service is available, or another // service is unregistered. -func (s *Store) maxIndexForService(tx *memdb.Txn, serviceName string, serviceExists, checks bool, entMeta *structs.EnterpriseMeta) uint64 { +func (s *Store) maxIndexForService(tx *txn, serviceName string, serviceExists, checks bool, entMeta *structs.EnterpriseMeta) uint64 { idx, _ := s.maxIndexAndWatchChForService(tx, serviceName, serviceExists, checks, entMeta) return idx } @@ -986,7 +982,7 @@ func (s *Store) maxIndexForService(tx *memdb.Txn, serviceName string, serviceExi // returned for the chan. This allows for blocking watchers to _only_ watch this // one chan in the common case, falling back to watching all touched MemDB // indexes in more complicated cases. -func (s *Store) maxIndexAndWatchChForService(tx *memdb.Txn, serviceName string, serviceExists, checks bool, entMeta *structs.EnterpriseMeta) (uint64, <-chan struct{}) { +func (s *Store) maxIndexAndWatchChForService(tx *txn, serviceName string, serviceExists, checks bool, entMeta *structs.EnterpriseMeta) (uint64, <-chan struct{}) { if !serviceExists { res, err := s.catalogServiceLastExtinctionIndex(tx, entMeta) if missingIdx, ok := res.(*IndexEntry); ok && err == nil { @@ -1003,7 +999,7 @@ func (s *Store) maxIndexAndWatchChForService(tx *memdb.Txn, serviceName string, } // Wrapper for maxIndexAndWatchChForService that operates on a list of ServiceNodes -func (s *Store) maxIndexAndWatchChsForServiceNodes(tx *memdb.Txn, +func (s *Store) maxIndexAndWatchChsForServiceNodes(tx *txn, nodes structs.ServiceNodes, watchChecks bool) (uint64, []<-chan struct{}) { var watchChans []<-chan struct{} @@ -1210,7 +1206,7 @@ func (s *Store) ServiceAddressNodes(ws memdb.WatchSet, address string, entMeta * // parseServiceNodes iterates over a services query and fills in the node details, // returning a ServiceNodes slice. -func (s *Store) parseServiceNodes(tx *memdb.Txn, ws memdb.WatchSet, services structs.ServiceNodes) (structs.ServiceNodes, error) { +func (s *Store) parseServiceNodes(tx *txn, ws memdb.WatchSet, services structs.ServiceNodes) (structs.ServiceNodes, error) { // We don't want to track an unlimited number of nodes, so we pull a // top-level watch to use as a fallback. allNodes, err := tx.Get("nodes", "id") @@ -1267,7 +1263,7 @@ func (s *Store) NodeService(nodeName string, serviceID string, entMeta *structs. return idx, service, nil } -func (s *Store) getNodeServiceTxn(tx *memdb.Txn, nodeName, serviceID string, entMeta *structs.EnterpriseMeta) (*structs.NodeService, error) { +func (s *Store) getNodeServiceTxn(tx *txn, nodeName, serviceID string, entMeta *structs.EnterpriseMeta) (*structs.NodeService, error) { // Query the service _, service, err := firstWatchCompoundWithTxn(tx, "services", "id", entMeta, nodeName, serviceID) if err != nil { @@ -1395,7 +1391,7 @@ func (s *Store) NodeServiceList(ws memdb.WatchSet, nodeNameOrID string, entMeta // DeleteService is used to delete a given service associated with a node. func (s *Store) DeleteService(idx uint64, nodeName, serviceID string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call the service deletion @@ -1403,14 +1399,13 @@ func (s *Store) DeleteService(idx uint64, nodeName, serviceID string, entMeta *s return err } - tx.Commit() - return nil + return tx.Commit() } // deleteServiceCASTxn is used to try doing a service delete operation with a given // raft index. If the CAS index specified is not equal to the last observed index for // the given service, then the call is a noop, otherwise a normal delete is invoked. -func (s *Store) deleteServiceCASTxn(tx *memdb.Txn, idx, cidx uint64, nodeName, serviceID string, entMeta *structs.EnterpriseMeta) (bool, error) { +func (s *Store) deleteServiceCASTxn(tx *txn, idx, cidx uint64, nodeName, serviceID string, entMeta *structs.EnterpriseMeta) (bool, error) { // Look up the service. service, err := s.getNodeServiceTxn(tx, nodeName, serviceID, entMeta) if err != nil { @@ -1437,7 +1432,7 @@ func (s *Store) deleteServiceCASTxn(tx *memdb.Txn, idx, cidx uint64, nodeName, s // deleteServiceTxn is the inner method called to remove a service // registration within an existing transaction. -func (s *Store) deleteServiceTxn(tx *memdb.Txn, idx uint64, nodeName, serviceID string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) deleteServiceTxn(tx *txn, idx uint64, nodeName, serviceID string, entMeta *structs.EnterpriseMeta) error { // Look up the service. _, service, err := firstWatchCompoundWithTxn(tx, "services", "id", entMeta, nodeName, serviceID) if err != nil { @@ -1533,7 +1528,7 @@ func (s *Store) deleteServiceTxn(tx *memdb.Txn, idx uint64, nodeName, serviceID // EnsureCheck is used to store a check registration in the db. func (s *Store) EnsureCheck(idx uint64, hc *structs.HealthCheck) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call the check registration @@ -1541,12 +1536,11 @@ func (s *Store) EnsureCheck(idx uint64, hc *structs.HealthCheck) error { return err } - tx.Commit() - return nil + return tx.Commit() } // updateAllServiceIndexesOfNode updates the Raft index of all the services associated with this node -func (s *Store) updateAllServiceIndexesOfNode(tx *memdb.Txn, idx uint64, nodeID string) error { +func (s *Store) updateAllServiceIndexesOfNode(tx *txn, idx uint64, nodeID string) error { services, err := tx.Get("services", "node", nodeID) if err != nil { return fmt.Errorf("failed updating services for node %s: %s", nodeID, err) @@ -1565,7 +1559,7 @@ func (s *Store) updateAllServiceIndexesOfNode(tx *memdb.Txn, idx uint64, nodeID // ensureCheckCASTxn updates a check only if the existing index matches the given index. // Returns a bool indicating if a write happened and any error. -func (s *Store) ensureCheckCASTxn(tx *memdb.Txn, idx uint64, hc *structs.HealthCheck) (bool, error) { +func (s *Store) ensureCheckCASTxn(tx *txn, idx uint64, hc *structs.HealthCheck) (bool, error) { // Retrieve the existing entry. _, existing, err := s.getNodeCheckTxn(tx, hc.Node, hc.CheckID, &hc.EnterpriseMeta) if err != nil { @@ -1592,10 +1586,10 @@ func (s *Store) ensureCheckCASTxn(tx *memdb.Txn, idx uint64, hc *structs.HealthC return true, nil } -// ensureCheckTransaction is used as the inner method to handle inserting +// ensureCheckTxn is used as the inner method to handle inserting // a health check into the state store. It ensures safety against inserting // checks with no matching node or service. -func (s *Store) ensureCheckTxn(tx *memdb.Txn, idx uint64, hc *structs.HealthCheck) error { +func (s *Store) ensureCheckTxn(tx *txn, idx uint64, hc *structs.HealthCheck) error { // Check if we have an existing health check _, existing, err := firstWatchCompoundWithTxn(tx, "checks", "id", &hc.EnterpriseMeta, hc.Node, string(hc.CheckID)) if err != nil { @@ -1680,18 +1674,10 @@ func (s *Store) ensureCheckTxn(tx *memdb.Txn, idx uint64, hc *structs.HealthChec } } } - if modified { - // We update the modify index, ONLY if something has changed, thus - // With constant output, no change is seen when watching a service - // With huge number of nodes where anti-entropy updates continuously - // the checks, but not the values within the check - hc.ModifyIndex = idx + if !modified { + return nil } - - // TODO (state store) TODO (catalog) - should we be reinserting at all. Similar - // code in ensureServiceTxn simply returns nil when the service being inserted - // already exists without modifications thereby avoiding the memdb insertions - // and also preventing some blocking queries from waking unnecessarily. + hc.ModifyIndex = idx return s.catalogInsertCheck(tx, hc, idx) } @@ -1706,7 +1692,7 @@ func (s *Store) NodeCheck(nodeName string, checkID types.CheckID, entMeta *struc // nodeCheckTxn is used as the inner method to handle reading a health check // from the state store. -func (s *Store) getNodeCheckTxn(tx *memdb.Txn, nodeName string, checkID types.CheckID, entMeta *structs.EnterpriseMeta) (uint64, *structs.HealthCheck, error) { +func (s *Store) getNodeCheckTxn(tx *txn, nodeName string, checkID types.CheckID, entMeta *structs.EnterpriseMeta) (uint64, *structs.HealthCheck, error) { // Get the table index. idx := s.catalogChecksMaxIndex(tx, entMeta) @@ -1822,7 +1808,7 @@ func (s *Store) ChecksInStateByNodeMeta(ws memdb.WatchSet, state string, filters return s.parseChecksByNodeMeta(tx, ws, idx, iter, filters) } -func (s *Store) checksInStateTxn(tx *memdb.Txn, ws memdb.WatchSet, state string, entMeta *structs.EnterpriseMeta) (uint64, memdb.ResultIterator, error) { +func (s *Store) checksInStateTxn(tx *txn, ws memdb.WatchSet, state string, entMeta *structs.EnterpriseMeta) (uint64, memdb.ResultIterator, error) { // Get the table index. idx := s.catalogChecksMaxIndex(tx, entMeta) @@ -1844,7 +1830,7 @@ func (s *Store) checksInStateTxn(tx *memdb.Txn, ws memdb.WatchSet, state string, // parseChecksByNodeMeta is a helper function used to deduplicate some // repetitive code for returning health checks filtered by node metadata fields. -func (s *Store) parseChecksByNodeMeta(tx *memdb.Txn, ws memdb.WatchSet, +func (s *Store) parseChecksByNodeMeta(tx *txn, ws memdb.WatchSet, idx uint64, iter memdb.ResultIterator, filters map[string]string) (uint64, structs.HealthChecks, error) { // We don't want to track an unlimited number of nodes, so we pull a @@ -1879,7 +1865,7 @@ func (s *Store) parseChecksByNodeMeta(tx *memdb.Txn, ws memdb.WatchSet, // DeleteCheck is used to delete a health check registration. func (s *Store) DeleteCheck(idx uint64, node string, checkID types.CheckID, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call the check deletion @@ -1887,14 +1873,13 @@ func (s *Store) DeleteCheck(idx uint64, node string, checkID types.CheckID, entM return err } - tx.Commit() - return nil + return tx.Commit() } // deleteCheckCASTxn is used to try doing a check delete operation with a given // raft index. If the CAS index specified is not equal to the last observed index for // the given check, then the call is a noop, otherwise a normal check delete is invoked. -func (s *Store) deleteCheckCASTxn(tx *memdb.Txn, idx, cidx uint64, node string, checkID types.CheckID, entMeta *structs.EnterpriseMeta) (bool, error) { +func (s *Store) deleteCheckCASTxn(tx *txn, idx, cidx uint64, node string, checkID types.CheckID, entMeta *structs.EnterpriseMeta) (bool, error) { // Try to retrieve the existing health check. _, hc, err := s.getNodeCheckTxn(tx, node, checkID, entMeta) if err != nil { @@ -1921,7 +1906,7 @@ func (s *Store) deleteCheckCASTxn(tx *memdb.Txn, idx, cidx uint64, node string, // deleteCheckTxn is the inner method used to call a health // check deletion within an existing transaction. -func (s *Store) deleteCheckTxn(tx *memdb.Txn, idx uint64, node string, checkID types.CheckID, entMeta *structs.EnterpriseMeta) error { +func (s *Store) deleteCheckTxn(tx *txn, idx uint64, node string, checkID types.CheckID, entMeta *structs.EnterpriseMeta) error { // Try to retrieve the existing health check. _, hc, err := firstWatchCompoundWithTxn(tx, "checks", "id", entMeta, node, string(checkID)) if err != nil { @@ -2038,7 +2023,7 @@ func (s *Store) checkServiceNodes(ws memdb.WatchSet, serviceName string, connect return s.checkServiceNodesTxn(tx, ws, serviceName, connect, entMeta) } -func (s *Store) checkServiceNodesTxn(tx *memdb.Txn, ws memdb.WatchSet, serviceName string, connect bool, entMeta *structs.EnterpriseMeta) (uint64, structs.CheckServiceNodes, error) { +func (s *Store) checkServiceNodesTxn(tx *txn, ws memdb.WatchSet, serviceName string, connect bool, entMeta *structs.EnterpriseMeta) (uint64, structs.CheckServiceNodes, error) { // Function for lookup index := "service" if connect { @@ -2228,7 +2213,7 @@ func (s *Store) GatewayServices(ws memdb.WatchSet, gateway string, entMeta *stru // and query for an associated node and a set of checks. This is the inner // method used to return a rich set of results from a more simple query. func (s *Store) parseCheckServiceNodes( - tx *memdb.Txn, ws memdb.WatchSet, idx uint64, + tx *txn, ws memdb.WatchSet, idx uint64, serviceName string, services structs.ServiceNodes, err error) (uint64, structs.CheckServiceNodes, error) { if err != nil { @@ -2353,7 +2338,7 @@ func (s *Store) ServiceDump(ws memdb.WatchSet, kind structs.ServiceKind, useKind } } -func (s *Store) serviceDumpAllTxn(tx *memdb.Txn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.CheckServiceNodes, error) { +func (s *Store) serviceDumpAllTxn(tx *txn, ws memdb.WatchSet, entMeta *structs.EnterpriseMeta) (uint64, structs.CheckServiceNodes, error) { // Get the table index idx := s.catalogMaxIndexWatch(tx, ws, entMeta, true) @@ -2371,7 +2356,7 @@ func (s *Store) serviceDumpAllTxn(tx *memdb.Txn, ws memdb.WatchSet, entMeta *str return s.parseCheckServiceNodes(tx, nil, idx, "", results, err) } -func (s *Store) serviceDumpKindTxn(tx *memdb.Txn, ws memdb.WatchSet, kind structs.ServiceKind, entMeta *structs.EnterpriseMeta) (uint64, structs.CheckServiceNodes, error) { +func (s *Store) serviceDumpKindTxn(tx *txn, ws memdb.WatchSet, kind structs.ServiceKind, entMeta *structs.EnterpriseMeta) (uint64, structs.CheckServiceNodes, error) { // unlike when we are dumping all services here we only need to watch the kind specific index entry for changing (or nodes, checks) // updating any services, nodes or checks will bump the appropriate service kind index so there is no need to watch any of the individual // entries @@ -2395,7 +2380,7 @@ func (s *Store) serviceDumpKindTxn(tx *memdb.Txn, ws memdb.WatchSet, kind struct // parseNodes takes an iterator over a set of nodes and returns a struct // containing the nodes along with all of their associated services // and/or health checks. -func (s *Store) parseNodes(tx *memdb.Txn, ws memdb.WatchSet, idx uint64, +func (s *Store) parseNodes(tx *txn, ws memdb.WatchSet, idx uint64, iter memdb.ResultIterator, entMeta *structs.EnterpriseMeta) (uint64, structs.NodeDump, error) { // We don't want to track an unlimited number of services, so we pull a @@ -2455,7 +2440,7 @@ func (s *Store) parseNodes(tx *memdb.Txn, ws memdb.WatchSet, idx uint64, } // checkSessionsTxn returns the IDs of all sessions associated with a health check -func checkSessionsTxn(tx *memdb.Txn, hc *structs.HealthCheck) ([]*sessionCheck, error) { +func checkSessionsTxn(tx *txn, hc *structs.HealthCheck) ([]*sessionCheck, error) { mappings, err := getCompoundWithTxn(tx, "session_checks", "node_check", &hc.EnterpriseMeta, hc.Node, string(hc.CheckID)) if err != nil { return nil, fmt.Errorf("failed session checks lookup: %s", err) @@ -2469,7 +2454,7 @@ func checkSessionsTxn(tx *memdb.Txn, hc *structs.HealthCheck) ([]*sessionCheck, } // updateGatewayServices associates services with gateways as specified in a gateway config entry -func (s *Store) updateGatewayServices(tx *memdb.Txn, idx uint64, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta) error { +func (s *Store) updateGatewayServices(tx *txn, idx uint64, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta) error { var ( noChange bool gatewayServices structs.GatewayServices @@ -2526,7 +2511,7 @@ func (s *Store) updateGatewayServices(tx *memdb.Txn, idx uint64, conf structs.Co // insertion into the memdb table, specific to ingress gateways. The boolean // returned indicates that there are no changes necessary to the memdb table. func (s *Store) ingressConfigGatewayServices( - tx *memdb.Txn, + tx *txn, gateway structs.ServiceName, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta, @@ -2571,7 +2556,7 @@ func (s *Store) ingressConfigGatewayServices( // boolean returned indicates that there are no changes necessary to the memdb // table. func (s *Store) terminatingConfigGatewayServices( - tx *memdb.Txn, + tx *txn, gateway structs.ServiceName, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta, @@ -2611,7 +2596,7 @@ func (s *Store) terminatingConfigGatewayServices( } // updateGatewayNamespace is used to target all services within a namespace -func (s *Store) updateGatewayNamespace(tx *memdb.Txn, idx uint64, service *structs.GatewayService, entMeta *structs.EnterpriseMeta) error { +func (s *Store) updateGatewayNamespace(tx *txn, idx uint64, service *structs.GatewayService, entMeta *structs.EnterpriseMeta) error { services, err := s.catalogServiceListByKind(tx, structs.ServiceKindTypical, entMeta) if err != nil { return fmt.Errorf("failed querying services: %s", err) @@ -2658,7 +2643,7 @@ func (s *Store) updateGatewayNamespace(tx *memdb.Txn, idx uint64, service *struc // updateGatewayService associates services with gateways after an eligible event // ie. Registering a service in a namespace targeted by a gateway -func (s *Store) updateGatewayService(tx *memdb.Txn, idx uint64, mapping *structs.GatewayService) error { +func (s *Store) updateGatewayService(tx *txn, idx uint64, mapping *structs.GatewayService) error { // Check if mapping already exists in table if it's already in the table // Avoid insert if nothing changed existing, err := tx.First(gatewayServicesTableName, "id", mapping.Gateway, mapping.Service, mapping.Port) @@ -2689,7 +2674,7 @@ func (s *Store) updateGatewayService(tx *memdb.Txn, idx uint64, mapping *structs // checkWildcardForGatewaysAndUpdate checks whether a service matches a // wildcard definition in gateway config entries and if so adds it the the // gateway-services table. -func (s *Store) checkGatewayWildcardsAndUpdate(tx *memdb.Txn, idx uint64, svc *structs.NodeService) error { +func (s *Store) checkGatewayWildcardsAndUpdate(tx *txn, idx uint64, svc *structs.NodeService) error { // Do not associate non-typical services with gateways or consul services if svc.Kind != structs.ServiceKindTypical || svc.Service == "consul" { return nil @@ -2718,18 +2703,18 @@ func (s *Store) checkGatewayWildcardsAndUpdate(tx *memdb.Txn, idx uint64, svc *s // serviceGateways returns all GatewayService entries with the given service name. This effectively looks up // all the gateways mapped to this service. -func (s *Store) serviceGateways(tx *memdb.Txn, name string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) serviceGateways(tx *txn, name string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get(gatewayServicesTableName, "service", structs.NewServiceName(name, entMeta)) } -func (s *Store) gatewayServices(tx *memdb.Txn, name string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) gatewayServices(tx *txn, name string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get(gatewayServicesTableName, "gateway", structs.NewServiceName(name, entMeta)) } // TODO(ingress): How to handle index rolling back when a config entry is // deleted that references a service? // We might need something like the service_last_extinction index? -func (s *Store) serviceGatewayNodes(tx *memdb.Txn, ws memdb.WatchSet, service string, kind structs.ServiceKind, entMeta *structs.EnterpriseMeta) (uint64, structs.ServiceNodes, error) { +func (s *Store) serviceGatewayNodes(tx *txn, ws memdb.WatchSet, service string, kind structs.ServiceKind, entMeta *structs.EnterpriseMeta) (uint64, structs.ServiceNodes, error) { // Look up gateway name associated with the service gws, err := s.serviceGateways(tx, service, entMeta) if err != nil { @@ -2781,7 +2766,7 @@ func (s *Store) serviceGatewayNodes(tx *memdb.Txn, ws memdb.WatchSet, service st // checkProtocolMatch filters out any GatewayService entries added from a wildcard with a protocol // that doesn't match the one configured in their discovery chain. func (s *Store) checkProtocolMatch( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, svc *structs.GatewayService, ) (uint64, bool, error) { diff --git a/agent/consul/state/catalog_oss.go b/agent/consul/state/catalog_oss.go index b6f20db65d..1c93546855 100644 --- a/agent/consul/state/catalog_oss.go +++ b/agent/consul/state/catalog_oss.go @@ -4,6 +4,7 @@ package state import ( "fmt" + "github.com/hashicorp/consul/agent/structs" memdb "github.com/hashicorp/go-memdb" ) @@ -167,7 +168,7 @@ func serviceKindIndexName(kind structs.ServiceKind, _ *structs.EnterpriseMeta) s } } -func (s *Store) catalogUpdateServicesIndexes(tx *memdb.Txn, idx uint64, _ *structs.EnterpriseMeta) error { +func (s *Store) catalogUpdateServicesIndexes(tx *txn, idx uint64, _ *structs.EnterpriseMeta) error { // overall services index if err := indexUpdateMaxTxn(tx, idx, "services"); err != nil { return fmt.Errorf("failed updating index: %s", err) @@ -176,7 +177,7 @@ func (s *Store) catalogUpdateServicesIndexes(tx *memdb.Txn, idx uint64, _ *struc return nil } -func (s *Store) catalogUpdateServiceKindIndexes(tx *memdb.Txn, kind structs.ServiceKind, idx uint64, _ *structs.EnterpriseMeta) error { +func (s *Store) catalogUpdateServiceKindIndexes(tx *txn, kind structs.ServiceKind, idx uint64, _ *structs.EnterpriseMeta) error { // service-kind index if err := indexUpdateMaxTxn(tx, idx, serviceKindIndexName(kind, nil)); err != nil { return fmt.Errorf("failed updating index: %s", err) @@ -185,7 +186,7 @@ func (s *Store) catalogUpdateServiceKindIndexes(tx *memdb.Txn, kind structs.Serv return nil } -func (s *Store) catalogUpdateServiceIndexes(tx *memdb.Txn, serviceName string, idx uint64, _ *structs.EnterpriseMeta) error { +func (s *Store) catalogUpdateServiceIndexes(tx *txn, serviceName string, idx uint64, _ *structs.EnterpriseMeta) error { // per-service index if err := indexUpdateMaxTxn(tx, idx, serviceIndexName(serviceName, nil)); err != nil { return fmt.Errorf("failed updating index: %s", err) @@ -194,14 +195,14 @@ func (s *Store) catalogUpdateServiceIndexes(tx *memdb.Txn, serviceName string, i return nil } -func (s *Store) catalogUpdateServiceExtinctionIndex(tx *memdb.Txn, idx uint64, _ *structs.EnterpriseMeta) error { +func (s *Store) catalogUpdateServiceExtinctionIndex(tx *txn, idx uint64, _ *structs.EnterpriseMeta) error { if err := tx.Insert("index", &IndexEntry{serviceLastExtinctionIndexName, idx}); err != nil { return fmt.Errorf("failed updating missing service extinction index: %s", err) } return nil } -func (s *Store) catalogInsertService(tx *memdb.Txn, svc *structs.ServiceNode) error { +func (s *Store) catalogInsertService(tx *txn, svc *structs.ServiceNode) error { // Insert the service and update the index if err := tx.Insert("services", svc); err != nil { return fmt.Errorf("failed inserting service: %s", err) @@ -222,53 +223,53 @@ func (s *Store) catalogInsertService(tx *memdb.Txn, svc *structs.ServiceNode) er return nil } -func (s *Store) catalogServicesMaxIndex(tx *memdb.Txn, _ *structs.EnterpriseMeta) uint64 { +func (s *Store) catalogServicesMaxIndex(tx *txn, _ *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "services") } -func (s *Store) catalogServiceMaxIndex(tx *memdb.Txn, serviceName string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { +func (s *Store) catalogServiceMaxIndex(tx *txn, serviceName string, _ *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch("index", "id", serviceIndexName(serviceName, nil)) } -func (s *Store) catalogServiceKindMaxIndex(tx *memdb.Txn, ws memdb.WatchSet, kind structs.ServiceKind, entMeta *structs.EnterpriseMeta) uint64 { +func (s *Store) catalogServiceKindMaxIndex(tx *txn, ws memdb.WatchSet, kind structs.ServiceKind, entMeta *structs.EnterpriseMeta) uint64 { return maxIndexWatchTxn(tx, ws, serviceKindIndexName(kind, nil)) } -func (s *Store) catalogServiceList(tx *memdb.Txn, _ *structs.EnterpriseMeta, _ bool) (memdb.ResultIterator, error) { +func (s *Store) catalogServiceList(tx *txn, _ *structs.EnterpriseMeta, _ bool) (memdb.ResultIterator, error) { return tx.Get("services", "id") } -func (s *Store) catalogServiceListByKind(tx *memdb.Txn, kind structs.ServiceKind, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogServiceListByKind(tx *txn, kind structs.ServiceKind, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("services", "kind", string(kind)) } -func (s *Store) catalogServiceListByNode(tx *memdb.Txn, node string, _ *structs.EnterpriseMeta, _ bool) (memdb.ResultIterator, error) { +func (s *Store) catalogServiceListByNode(tx *txn, node string, _ *structs.EnterpriseMeta, _ bool) (memdb.ResultIterator, error) { return tx.Get("services", "node", node) } -func (s *Store) catalogServiceNodeList(tx *memdb.Txn, name string, index string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogServiceNodeList(tx *txn, name string, index string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("services", index, name) } -func (s *Store) catalogServiceLastExtinctionIndex(tx *memdb.Txn, _ *structs.EnterpriseMeta) (interface{}, error) { +func (s *Store) catalogServiceLastExtinctionIndex(tx *txn, _ *structs.EnterpriseMeta) (interface{}, error) { return tx.First("index", "id", serviceLastExtinctionIndexName) } -func (s *Store) catalogMaxIndex(tx *memdb.Txn, _ *structs.EnterpriseMeta, checks bool) uint64 { +func (s *Store) catalogMaxIndex(tx *txn, _ *structs.EnterpriseMeta, checks bool) uint64 { if checks { return maxIndexTxn(tx, "nodes", "services", "checks") } return maxIndexTxn(tx, "nodes", "services") } -func (s *Store) catalogMaxIndexWatch(tx *memdb.Txn, ws memdb.WatchSet, _ *structs.EnterpriseMeta, checks bool) uint64 { +func (s *Store) catalogMaxIndexWatch(tx *txn, ws memdb.WatchSet, _ *structs.EnterpriseMeta, checks bool) uint64 { if checks { return maxIndexWatchTxn(tx, ws, "nodes", "services", "checks") } return maxIndexWatchTxn(tx, ws, "nodes", "services") } -func (s *Store) catalogUpdateCheckIndexes(tx *memdb.Txn, idx uint64, _ *structs.EnterpriseMeta) error { +func (s *Store) catalogUpdateCheckIndexes(tx *txn, idx uint64, _ *structs.EnterpriseMeta) error { // update the universal index entry if err := tx.Insert("index", &IndexEntry{"checks", idx}); err != nil { return fmt.Errorf("failed updating index: %s", err) @@ -276,36 +277,36 @@ func (s *Store) catalogUpdateCheckIndexes(tx *memdb.Txn, idx uint64, _ *structs. return nil } -func (s *Store) catalogChecksMaxIndex(tx *memdb.Txn, _ *structs.EnterpriseMeta) uint64 { +func (s *Store) catalogChecksMaxIndex(tx *txn, _ *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "checks") } -func (s *Store) catalogListChecksByNode(tx *memdb.Txn, node string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogListChecksByNode(tx *txn, node string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("checks", "node", node) } -func (s *Store) catalogListChecksByService(tx *memdb.Txn, service string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogListChecksByService(tx *txn, service string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("checks", "service", service) } -func (s *Store) catalogListChecksInState(tx *memdb.Txn, state string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogListChecksInState(tx *txn, state string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { // simpler than normal due to the use of the CompoundMultiIndex return tx.Get("checks", "status", state) } -func (s *Store) catalogListChecks(tx *memdb.Txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogListChecks(tx *txn, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("checks", "id") } -func (s *Store) catalogListNodeChecks(tx *memdb.Txn, node string) (memdb.ResultIterator, error) { +func (s *Store) catalogListNodeChecks(tx *txn, node string) (memdb.ResultIterator, error) { return tx.Get("checks", "node_service_check", node, false) } -func (s *Store) catalogListServiceChecks(tx *memdb.Txn, node string, service string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogListServiceChecks(tx *txn, node string, service string, _ *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("checks", "node_service", node, service) } -func (s *Store) catalogInsertCheck(tx *memdb.Txn, chk *structs.HealthCheck, idx uint64) error { +func (s *Store) catalogInsertCheck(tx *txn, chk *structs.HealthCheck, idx uint64) error { // Insert the check if err := tx.Insert("checks", chk); err != nil { return fmt.Errorf("failed inserting check: %s", err) @@ -318,11 +319,11 @@ func (s *Store) catalogInsertCheck(tx *memdb.Txn, chk *structs.HealthCheck, idx return nil } -func (s *Store) catalogChecksForNodeService(tx *memdb.Txn, node string, service string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func (s *Store) catalogChecksForNodeService(tx *txn, node string, service string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get("checks", "node_service", node, service) } -func (s *Store) validateRegisterRequestTxn(tx *memdb.Txn, args *structs.RegisterRequest) (*structs.EnterpriseMeta, error) { +func (s *Store) validateRegisterRequestTxn(tx *txn, args *structs.RegisterRequest) (*structs.EnterpriseMeta, error) { return nil, nil } diff --git a/agent/consul/state/catalog_test.go b/agent/consul/state/catalog_test.go index bc0c052947..7faf724714 100644 --- a/agent/consul/state/catalog_test.go +++ b/agent/consul/state/catalog_test.go @@ -105,7 +105,7 @@ func TestStateStore_ensureNoNodeWithSimilarNameTxn(t *testing.T) { if err := s.EnsureRegistration(2, req); err != nil { t.Fatalf("err: %s", err) } - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() defer tx.Abort() node := &structs.Node{ ID: makeRandomNodeID(t), @@ -2363,7 +2363,9 @@ func TestStateStore_EnsureCheck(t *testing.T) { if err := s.EnsureCheck(4, check); err != nil { t.Fatalf("err: %s", err) } - testCheckOutput(t, 4, 3, check.Output) + // Since there was no change to the check it won't actually have been updated + // so the ModifyIndex index should still be 3 + testCheckOutput(t, 3, 3, check.Output) // Do modify the heathcheck check = &structs.HealthCheck{ @@ -4380,10 +4382,10 @@ func TestStateStore_ensureServiceCASTxn(t *testing.T) { } // attempt to update with a 0 index - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() err := s.ensureServiceCASTxn(tx, 3, "node1", &ns) require.Equal(t, err, errCASCompareFailed) - tx.Commit() + require.NoError(t, tx.Commit()) // ensure no update happened tx = s.db.Txn(false) @@ -4391,14 +4393,14 @@ func TestStateStore_ensureServiceCASTxn(t *testing.T) { require.NoError(t, err) require.NotNil(t, nsRead) require.Equal(t, uint64(2), nsRead.ModifyIndex) - tx.Commit() + require.NoError(t, tx.Commit()) ns.ModifyIndex = 99 // attempt to update with a non-matching index - tx = s.db.Txn(true) + tx = s.db.WriteTxnRestore() err = s.ensureServiceCASTxn(tx, 4, "node1", &ns) require.Equal(t, err, errCASCompareFailed) - tx.Commit() + require.NoError(t, tx.Commit()) // ensure no update happened tx = s.db.Txn(false) @@ -4406,14 +4408,14 @@ func TestStateStore_ensureServiceCASTxn(t *testing.T) { require.NoError(t, err) require.NotNil(t, nsRead) require.Equal(t, uint64(2), nsRead.ModifyIndex) - tx.Commit() + require.NoError(t, tx.Commit()) ns.ModifyIndex = 2 // update with the matching modify index - tx = s.db.Txn(true) + tx = s.db.WriteTxnRestore() err = s.ensureServiceCASTxn(tx, 7, "node1", &ns) require.NoError(t, err) - tx.Commit() + require.NoError(t, tx.Commit()) // ensure the update happened tx = s.db.Txn(false) @@ -4421,7 +4423,7 @@ func TestStateStore_ensureServiceCASTxn(t *testing.T) { require.NoError(t, err) require.NotNil(t, nsRead) require.Equal(t, uint64(7), nsRead.ModifyIndex) - tx.Commit() + require.NoError(t, tx.Commit()) } func TestStateStore_GatewayServices_Terminating(t *testing.T) { diff --git a/agent/consul/state/config_entry.go b/agent/consul/state/config_entry.go index 1fbaea78be..7a053eaf5d 100644 --- a/agent/consul/state/config_entry.go +++ b/agent/consul/state/config_entry.go @@ -106,7 +106,7 @@ func (s *Store) ConfigEntry(ws memdb.WatchSet, kind, name string, entMeta *struc return s.configEntryTxn(tx, ws, kind, name, entMeta) } -func (s *Store) configEntryTxn(tx *memdb.Txn, ws memdb.WatchSet, kind, name string, entMeta *structs.EnterpriseMeta) (uint64, structs.ConfigEntry, error) { +func (s *Store) configEntryTxn(tx *txn, ws memdb.WatchSet, kind, name string, entMeta *structs.EnterpriseMeta) (uint64, structs.ConfigEntry, error) { // Get the index idx := maxIndexTxn(tx, configTableName) @@ -141,7 +141,7 @@ func (s *Store) ConfigEntriesByKind(ws memdb.WatchSet, kind string, entMeta *str return s.configEntriesByKindTxn(tx, ws, kind, entMeta) } -func (s *Store) configEntriesByKindTxn(tx *memdb.Txn, ws memdb.WatchSet, kind string, entMeta *structs.EnterpriseMeta) (uint64, []structs.ConfigEntry, error) { +func (s *Store) configEntriesByKindTxn(tx *txn, ws memdb.WatchSet, kind string, entMeta *structs.EnterpriseMeta) (uint64, []structs.ConfigEntry, error) { // Get the index idx := maxIndexTxn(tx, configTableName) @@ -167,19 +167,18 @@ func (s *Store) configEntriesByKindTxn(tx *memdb.Txn, ws memdb.WatchSet, kind st // EnsureConfigEntry is called to do an upsert of a given config entry. func (s *Store) EnsureConfigEntry(idx uint64, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.ensureConfigEntryTxn(tx, idx, conf, entMeta); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } // ensureConfigEntryTxn upserts a config entry inside of a transaction. -func (s *Store) ensureConfigEntryTxn(tx *memdb.Txn, idx uint64, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta) error { +func (s *Store) ensureConfigEntryTxn(tx *txn, idx uint64, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta) error { // Check for existing configuration. existing, err := s.firstConfigEntryWithTxn(tx, conf.GetKind(), conf.GetName(), entMeta) if err != nil { @@ -217,7 +216,7 @@ func (s *Store) ensureConfigEntryTxn(tx *memdb.Txn, idx uint64, conf structs.Con // EnsureConfigEntryCAS is called to do a check-and-set upsert of a given config entry. func (s *Store) EnsureConfigEntryCAS(idx, cidx uint64, conf structs.ConfigEntry, entMeta *structs.EnterpriseMeta) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Check for existing configuration. @@ -246,12 +245,12 @@ func (s *Store) EnsureConfigEntryCAS(idx, cidx uint64, conf structs.ConfigEntry, return false, err } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } func (s *Store) DeleteConfigEntry(idx uint64, kind, name string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Try to retrieve the existing config entry. @@ -294,11 +293,10 @@ func (s *Store) DeleteConfigEntry(idx uint64, kind, name string, entMeta *struct return fmt.Errorf("failed updating index: %s", err) } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) insertConfigEntryWithTxn(tx *memdb.Txn, idx uint64, conf structs.ConfigEntry) error { +func (s *Store) insertConfigEntryWithTxn(tx *txn, idx uint64, conf structs.ConfigEntry) error { if conf == nil { return fmt.Errorf("cannot insert nil config entry") } @@ -330,7 +328,7 @@ func (s *Store) insertConfigEntryWithTxn(tx *memdb.Txn, idx uint64, conf structs // May return *ConfigEntryGraphValidationError if there is a concern to surface // to the caller that they can correct. func (s *Store) validateProposedConfigEntryInGraph( - tx *memdb.Txn, + tx *txn, idx uint64, kind, name string, next structs.ConfigEntry, @@ -371,7 +369,7 @@ func (s *Store) validateProposedConfigEntryInGraph( } func (s *Store) checkGatewayClash( - tx *memdb.Txn, + tx *txn, name, selfKind, otherKind string, entMeta *structs.EnterpriseMeta, ) error { @@ -393,7 +391,7 @@ var serviceGraphKinds = []string{ } func (s *Store) validateProposedConfigEntryInServiceGraph( - tx *memdb.Txn, + tx *txn, idx uint64, kind, name string, next structs.ConfigEntry, @@ -449,7 +447,7 @@ func (s *Store) validateProposedConfigEntryInServiceGraph( } func (s *Store) testCompileDiscoveryChain( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, chainName string, overrides map[structs.ConfigEntryKindName]structs.ConfigEntry, @@ -512,7 +510,7 @@ func (s *Store) readDiscoveryChainConfigEntries( } func (s *Store) readDiscoveryChainConfigEntriesTxn( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, serviceName string, overrides map[structs.ConfigEntryKindName]structs.ConfigEntry, @@ -701,7 +699,7 @@ func anyKey(m map[structs.ServiceID]struct{}) (structs.ServiceID, bool) { // // If an override is returned the index returned will be 0. func (s *Store) getProxyConfigEntryTxn( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, name string, overrides map[structs.ConfigEntryKindName]structs.ConfigEntry, @@ -726,7 +724,7 @@ func (s *Store) getProxyConfigEntryTxn( // // If an override is returned the index returned will be 0. func (s *Store) getServiceConfigEntryTxn( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, serviceName string, overrides map[structs.ConfigEntryKindName]structs.ConfigEntry, @@ -751,7 +749,7 @@ func (s *Store) getServiceConfigEntryTxn( // // If an override is returned the index returned will be 0. func (s *Store) getRouterConfigEntryTxn( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, serviceName string, overrides map[structs.ConfigEntryKindName]structs.ConfigEntry, @@ -776,7 +774,7 @@ func (s *Store) getRouterConfigEntryTxn( // // If an override is returned the index returned will be 0. func (s *Store) getSplitterConfigEntryTxn( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, serviceName string, overrides map[structs.ConfigEntryKindName]structs.ConfigEntry, @@ -801,7 +799,7 @@ func (s *Store) getSplitterConfigEntryTxn( // // If an override is returned the index returned will be 0. func (s *Store) getResolverConfigEntryTxn( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, serviceName string, overrides map[structs.ConfigEntryKindName]structs.ConfigEntry, @@ -822,7 +820,7 @@ func (s *Store) getResolverConfigEntryTxn( } func (s *Store) configEntryWithOverridesTxn( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, kind string, name string, @@ -842,7 +840,7 @@ func (s *Store) configEntryWithOverridesTxn( } func (s *Store) validateProposedIngressProtocolsInServiceGraph( - tx *memdb.Txn, + tx *txn, next structs.ConfigEntry, entMeta *structs.EnterpriseMeta, ) error { @@ -886,7 +884,7 @@ func (s *Store) validateProposedIngressProtocolsInServiceGraph( // protocolForService returns the service graph protocol associated to the // provided service, checking all relevant config entries. func (s *Store) protocolForService( - tx *memdb.Txn, + tx *txn, ws memdb.WatchSet, svc structs.ServiceName, ) (uint64, string, error) { diff --git a/agent/consul/state/config_entry_oss.go b/agent/consul/state/config_entry_oss.go index cb9cd6a70e..534e6bc840 100644 --- a/agent/consul/state/config_entry_oss.go +++ b/agent/consul/state/config_entry_oss.go @@ -49,25 +49,25 @@ func configTableSchema() *memdb.TableSchema { } } -func (s *Store) firstConfigEntryWithTxn(tx *memdb.Txn, +func (s *Store) firstConfigEntryWithTxn(tx *txn, kind, name string, entMeta *structs.EnterpriseMeta) (interface{}, error) { return tx.First(configTableName, "id", kind, name) } -func (s *Store) firstWatchConfigEntryWithTxn(tx *memdb.Txn, +func (s *Store) firstWatchConfigEntryWithTxn(tx *txn, kind, name string, entMeta *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch(configTableName, "id", kind, name) } -func (s *Store) validateConfigEntryEnterprise(tx *memdb.Txn, conf structs.ConfigEntry) error { +func (s *Store) validateConfigEntryEnterprise(tx *txn, conf structs.ConfigEntry) error { return nil } -func getAllConfigEntriesWithTxn(tx *memdb.Txn, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { +func getAllConfigEntriesWithTxn(tx *txn, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get(configTableName, "id") } -func getConfigEntryKindsWithTxn(tx *memdb.Txn, +func getConfigEntryKindsWithTxn(tx *txn, kind string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get(configTableName, "kind", kind) } diff --git a/agent/consul/state/connect_ca.go b/agent/consul/state/connect_ca.go index cd7b387276..fb5dd2ffc8 100644 --- a/agent/consul/state/connect_ca.go +++ b/agent/consul/state/connect_ca.go @@ -116,7 +116,7 @@ func (s *Store) CAConfig(ws memdb.WatchSet) (uint64, *structs.CAConfiguration, e return s.caConfigTxn(tx, ws) } -func (s *Store) caConfigTxn(tx *memdb.Txn, ws memdb.WatchSet) (uint64, *structs.CAConfiguration, error) { +func (s *Store) caConfigTxn(tx *txn, ws memdb.WatchSet) (uint64, *structs.CAConfiguration, error) { // Get the CA config ch, c, err := tx.FirstWatch(caConfigTableName, "id") if err != nil { @@ -135,22 +135,21 @@ func (s *Store) caConfigTxn(tx *memdb.Txn, ws memdb.WatchSet) (uint64, *structs. // CASetConfig is used to set the current CA configuration. func (s *Store) CASetConfig(idx uint64, config *structs.CAConfiguration) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.caSetConfigTxn(idx, tx, config); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } // CACheckAndSetConfig is used to try updating the CA configuration with a // given Raft index. If the CAS index specified is not equal to the last observed index // for the config, then the call is a noop, func (s *Store) CACheckAndSetConfig(idx, cidx uint64, config *structs.CAConfiguration) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Check for an existing config @@ -171,11 +170,11 @@ func (s *Store) CACheckAndSetConfig(idx, cidx uint64, config *structs.CAConfigur return false, err } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } -func (s *Store) caSetConfigTxn(idx uint64, tx *memdb.Txn, config *structs.CAConfiguration) error { +func (s *Store) caSetConfigTxn(idx uint64, tx *txn, config *structs.CAConfiguration) error { // Check for an existing config prev, err := tx.First(caConfigTableName, "id") if err != nil { @@ -237,7 +236,7 @@ func (s *Store) CARoots(ws memdb.WatchSet) (uint64, structs.CARoots, error) { return s.caRootsTxn(tx, ws) } -func (s *Store) caRootsTxn(tx *memdb.Txn, ws memdb.WatchSet) (uint64, structs.CARoots, error) { +func (s *Store) caRootsTxn(tx *txn, ws memdb.WatchSet) (uint64, structs.CARoots, error) { // Get the index idx := maxIndexTxn(tx, caRootTableName) @@ -279,7 +278,7 @@ func (s *Store) CARootActive(ws memdb.WatchSet) (uint64, *structs.CARoot, error) // // The first boolean result returns whether the transaction succeeded or not. func (s *Store) CARootSetCAS(idx, cidx uint64, rs []*structs.CARoot) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // There must be exactly one active CA root. @@ -336,8 +335,8 @@ func (s *Store) CARootSetCAS(idx, cidx uint64, rs []*structs.CARoot) (bool, erro return false, fmt.Errorf("failed updating index: %s", err) } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } // CAProviderState is used to pull the built-in provider states from the snapshot. @@ -391,7 +390,7 @@ func (s *Store) CAProviderState(id string) (uint64, *structs.CAConsulProviderSta // CASetProviderState is used to set the current built-in CA provider state. func (s *Store) CASetProviderState(idx uint64, state *structs.CAConsulProviderState) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Check for an existing config @@ -417,20 +416,16 @@ func (s *Store) CASetProviderState(idx uint64, state *structs.CAConsulProviderSt return false, fmt.Errorf("failed updating index: %s", err) } - tx.Commit() - - return true, nil + err = tx.Commit() + return err == nil, err } // CADeleteProviderState is used to remove the built-in Consul CA provider // state for the given ID. -func (s *Store) CADeleteProviderState(id string) error { - tx := s.db.Txn(true) +func (s *Store) CADeleteProviderState(idx uint64, id string) error { + tx := s.db.WriteTxn(idx) defer tx.Abort() - // Get the index - idx := maxIndexTxn(tx, caBuiltinProviderTableName) - // Check for an existing config existing, err := tx.First(caBuiltinProviderTableName, "id", id) if err != nil { @@ -450,13 +445,11 @@ func (s *Store) CADeleteProviderState(id string) error { return fmt.Errorf("failed updating index: %s", err) } - tx.Commit() - - return nil + return tx.Commit() } -func (s *Store) CALeafSetIndex(index uint64) error { - tx := s.db.Txn(true) +func (s *Store) CALeafSetIndex(idx uint64, index uint64) error { + tx := s.db.WriteTxn(idx) defer tx.Abort() return indexUpdateMaxTxn(tx, index, caLeafIndexName) @@ -484,8 +477,8 @@ func (s *Store) CARootsAndConfig(ws memdb.WatchSet) (uint64, structs.CARoots, *s return idx, roots, config, nil } -func (s *Store) CAIncrementProviderSerialNumber() (uint64, error) { - tx := s.db.Txn(true) +func (s *Store) CAIncrementProviderSerialNumber(idx uint64) (uint64, error) { + tx := s.db.WriteTxn(idx) defer tx.Abort() existing, err := tx.First("index", "id", caBuiltinProviderSerialNumber) @@ -507,7 +500,6 @@ func (s *Store) CAIncrementProviderSerialNumber() (uint64, error) { return 0, fmt.Errorf("failed updating index: %s", err) } - tx.Commit() - - return next, nil + err = tx.Commit() + return next, err } diff --git a/agent/consul/state/connect_ca_test.go b/agent/consul/state/connect_ca_test.go index 9aba81abd6..df14b49de1 100644 --- a/agent/consul/state/connect_ca_test.go +++ b/agent/consul/state/connect_ca_test.go @@ -429,15 +429,15 @@ func TestStore_CABuiltinProvider(t *testing.T) { // Since we've already written to the builtin provider table the serial // numbers will initialize from the max index of the provider table. // That's why this first serial is 2 and not 1. - sn, err := s.CAIncrementProviderSerialNumber() + sn, err := s.CAIncrementProviderSerialNumber(10) assert.NoError(err) assert.Equal(uint64(2), sn) - sn, err = s.CAIncrementProviderSerialNumber() + sn, err = s.CAIncrementProviderSerialNumber(10) assert.NoError(err) assert.Equal(uint64(3), sn) - sn, err = s.CAIncrementProviderSerialNumber() + sn, err = s.CAIncrementProviderSerialNumber(10) assert.NoError(err) assert.Equal(uint64(4), sn) } diff --git a/agent/consul/state/coordinate.go b/agent/consul/state/coordinate.go index 95b27fb1c4..dc9958ab6c 100644 --- a/agent/consul/state/coordinate.go +++ b/agent/consul/state/coordinate.go @@ -130,7 +130,7 @@ func (s *Store) Coordinates(ws memdb.WatchSet) (uint64, structs.Coordinates, err // CoordinateBatchUpdate processes a batch of coordinate updates and applies // them in a single transaction. func (s *Store) CoordinateBatchUpdate(idx uint64, updates structs.Coordinates) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Upsert the coordinates. @@ -167,6 +167,5 @@ func (s *Store) CoordinateBatchUpdate(idx uint64, updates structs.Coordinates) e return fmt.Errorf("failed updating index: %s", err) } - tx.Commit() - return nil + return tx.Commit() } diff --git a/agent/consul/state/coordinate_test.go b/agent/consul/state/coordinate_test.go index 98ec794a8f..e4d27ccb20 100644 --- a/agent/consul/state/coordinate_test.go +++ b/agent/consul/state/coordinate_test.go @@ -270,11 +270,11 @@ func TestStateStore_Coordinate_Snapshot_Restore(t *testing.T) { Node: "node3", Coord: &coordinate.Coordinate{Height: math.NaN()}, } - tx := s.db.Txn(true) + tx := s.db.WriteTxn(5) if err := tx.Insert("coordinates", badUpdate); err != nil { t.Fatalf("err: %v", err) } - tx.Commit() + require.NoError(t, tx.Commit()) // Snapshot the coordinates. snap := s.Snapshot() diff --git a/agent/consul/state/federation_state.go b/agent/consul/state/federation_state.go index 1a96f78215..bd69c519a9 100644 --- a/agent/consul/state/federation_state.go +++ b/agent/consul/state/federation_state.go @@ -59,7 +59,7 @@ func (s *Restore) FederationState(g *structs.FederationState) error { } func (s *Store) FederationStateBatchSet(idx uint64, configs structs.FederationStates) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, config := range configs { @@ -68,25 +68,23 @@ func (s *Store) FederationStateBatchSet(idx uint64, configs structs.FederationSt } } - tx.Commit() - return nil + return tx.Commit() } // FederationStateSet is called to do an upsert of a given federation state. func (s *Store) FederationStateSet(idx uint64, config *structs.FederationState) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.federationStateSetTxn(tx, idx, config); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } // federationStateSetTxn upserts a federation state inside of a transaction. -func (s *Store) federationStateSetTxn(tx *memdb.Txn, idx uint64, config *structs.FederationState) error { +func (s *Store) federationStateSetTxn(tx *txn, idx uint64, config *structs.FederationState) error { if config.Datacenter == "" { return fmt.Errorf("missing datacenter on federation state") } @@ -136,7 +134,7 @@ func (s *Store) FederationStateGet(ws memdb.WatchSet, datacenter string) (uint64 return s.federationStateGetTxn(tx, ws, datacenter) } -func (s *Store) federationStateGetTxn(tx *memdb.Txn, ws memdb.WatchSet, datacenter string) (uint64, *structs.FederationState, error) { +func (s *Store) federationStateGetTxn(tx *txn, ws memdb.WatchSet, datacenter string) (uint64, *structs.FederationState, error) { // Get the index idx := maxIndexTxn(tx, federationStateTableName) @@ -166,7 +164,7 @@ func (s *Store) FederationStateList(ws memdb.WatchSet) (uint64, []*structs.Feder return s.federationStateListTxn(tx, ws) } -func (s *Store) federationStateListTxn(tx *memdb.Txn, ws memdb.WatchSet) (uint64, []*structs.FederationState, error) { +func (s *Store) federationStateListTxn(tx *txn, ws memdb.WatchSet) (uint64, []*structs.FederationState, error) { // Get the index idx := maxIndexTxn(tx, federationStateTableName) @@ -184,19 +182,18 @@ func (s *Store) federationStateListTxn(tx *memdb.Txn, ws memdb.WatchSet) (uint64 } func (s *Store) FederationStateDelete(idx uint64, datacenter string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.federationStateDeleteTxn(tx, idx, datacenter); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } func (s *Store) FederationStateBatchDelete(idx uint64, datacenters []string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() for _, datacenter := range datacenters { @@ -205,11 +202,10 @@ func (s *Store) FederationStateBatchDelete(idx uint64, datacenters []string) err } } - tx.Commit() - return nil + return tx.Commit() } -func (s *Store) federationStateDeleteTxn(tx *memdb.Txn, idx uint64, datacenter string) error { +func (s *Store) federationStateDeleteTxn(tx *txn, idx uint64, datacenter string) error { // Try to retrieve the existing federation state. existing, err := tx.First(federationStateTableName, "id", datacenter) if err != nil { diff --git a/agent/consul/state/graveyard.go b/agent/consul/state/graveyard.go index 96e0c9dbfa..bfae5f2c27 100644 --- a/agent/consul/state/graveyard.go +++ b/agent/consul/state/graveyard.go @@ -2,6 +2,7 @@ package state import ( "fmt" + "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/go-memdb" ) @@ -27,7 +28,7 @@ func NewGraveyard(gc *TombstoneGC) *Graveyard { } // InsertTxn adds a new tombstone. -func (g *Graveyard) InsertTxn(tx *memdb.Txn, key string, idx uint64, entMeta *structs.EnterpriseMeta) error { +func (g *Graveyard) InsertTxn(tx *txn, key string, idx uint64, entMeta *structs.EnterpriseMeta) error { stone := &Tombstone{ Key: key, Index: idx, @@ -50,7 +51,7 @@ func (g *Graveyard) InsertTxn(tx *memdb.Txn, key string, idx uint64, entMeta *st // GetMaxIndexTxn returns the highest index tombstone whose key matches the // given context, using a prefix match. -func (g *Graveyard) GetMaxIndexTxn(tx *memdb.Txn, prefix string, entMeta *structs.EnterpriseMeta) (uint64, error) { +func (g *Graveyard) GetMaxIndexTxn(tx *txn, prefix string, entMeta *structs.EnterpriseMeta) (uint64, error) { stones, err := getWithTxn(tx, "tombstones", "id_prefix", prefix, entMeta) if err != nil { return 0, fmt.Errorf("failed querying tombstones: %s", err) @@ -67,7 +68,7 @@ func (g *Graveyard) GetMaxIndexTxn(tx *memdb.Txn, prefix string, entMeta *struct } // DumpTxn returns all the tombstones. -func (g *Graveyard) DumpTxn(tx *memdb.Txn) (memdb.ResultIterator, error) { +func (g *Graveyard) DumpTxn(tx *txn) (memdb.ResultIterator, error) { iter, err := tx.Get("tombstones", "id") if err != nil { return nil, err @@ -78,7 +79,7 @@ func (g *Graveyard) DumpTxn(tx *memdb.Txn) (memdb.ResultIterator, error) { // RestoreTxn is used when restoring from a snapshot. For general inserts, use // InsertTxn. -func (g *Graveyard) RestoreTxn(tx *memdb.Txn, stone *Tombstone) error { +func (g *Graveyard) RestoreTxn(tx *txn, stone *Tombstone) error { if err := g.insertTombstoneWithTxn(tx, "tombstones", stone, true); err != nil { return fmt.Errorf("failed inserting tombstone: %s", err) } @@ -88,7 +89,7 @@ func (g *Graveyard) RestoreTxn(tx *memdb.Txn, stone *Tombstone) error { // ReapTxn cleans out all tombstones whose index values are less than or equal // to the given idx. This prevents unbounded storage growth of the tombstones. -func (g *Graveyard) ReapTxn(tx *memdb.Txn, idx uint64) error { +func (g *Graveyard) ReapTxn(tx *txn, idx uint64) error { // This does a full table scan since we currently can't index on a // numeric value. Since this is all in-memory and done infrequently // this pretty reasonable. diff --git a/agent/consul/state/graveyard_oss.go b/agent/consul/state/graveyard_oss.go index 5637706846..157f23e1d3 100644 --- a/agent/consul/state/graveyard_oss.go +++ b/agent/consul/state/graveyard_oss.go @@ -4,11 +4,9 @@ package state import ( "fmt" - - "github.com/hashicorp/go-memdb" ) -func (g *Graveyard) insertTombstoneWithTxn(tx *memdb.Txn, +func (g *Graveyard) insertTombstoneWithTxn(tx *txn, table string, stone *Tombstone, updateMax bool) error { if err := tx.Insert("tombstones", stone); err != nil { diff --git a/agent/consul/state/graveyard_test.go b/agent/consul/state/graveyard_test.go index 332beaa35d..7b7d533d29 100644 --- a/agent/consul/state/graveyard_test.go +++ b/agent/consul/state/graveyard_test.go @@ -3,6 +3,8 @@ package state import ( "testing" "time" + + "github.com/stretchr/testify/require" ) func TestGraveyard_Lifecycle(t *testing.T) { @@ -14,7 +16,7 @@ func TestGraveyard_Lifecycle(t *testing.T) { // Create some tombstones. func() { - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() defer tx.Abort() if err := g.InsertTxn(tx, "foo/in/the/house", 2, nil); err != nil { @@ -29,7 +31,7 @@ func TestGraveyard_Lifecycle(t *testing.T) { if err := g.InsertTxn(tx, "some/other/path", 9, nil); err != nil { t.Fatalf("err: %s", err) } - tx.Commit() + require.NoError(t, tx.Commit()) }() // Check some prefixes. @@ -62,13 +64,13 @@ func TestGraveyard_Lifecycle(t *testing.T) { // Reap some tombstones. func() { - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() defer tx.Abort() if err := g.ReapTxn(tx, 6); err != nil { t.Fatalf("err: %s", err) } - tx.Commit() + require.NoError(t, tx.Commit()) }() // Check prefixes to see that the reap took effect at the right index. @@ -121,7 +123,7 @@ func TestGraveyard_GC_Trigger(t *testing.T) { // GC. s := testStateStore(t) func() { - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() defer tx.Abort() if err := g.InsertTxn(tx, "foo/in/the/house", 2, nil); err != nil { @@ -136,13 +138,13 @@ func TestGraveyard_GC_Trigger(t *testing.T) { // Now commit. func() { - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() defer tx.Abort() if err := g.InsertTxn(tx, "foo/in/the/house", 2, nil); err != nil { t.Fatalf("err: %s", err) } - tx.Commit() + require.NoError(t, tx.Commit()) }() // Make sure the GC got hinted. @@ -170,7 +172,7 @@ func TestGraveyard_Snapshot_Restore(t *testing.T) { // Create some tombstones. func() { - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() defer tx.Abort() if err := g.InsertTxn(tx, "foo/in/the/house", 2, nil); err != nil { @@ -185,7 +187,7 @@ func TestGraveyard_Snapshot_Restore(t *testing.T) { if err := g.InsertTxn(tx, "some/other/path", 9, nil); err != nil { t.Fatalf("err: %s", err) } - tx.Commit() + require.NoError(t, tx.Commit()) }() // Verify the index was set correctly. @@ -232,7 +234,7 @@ func TestGraveyard_Snapshot_Restore(t *testing.T) { func() { s := testStateStore(t) func() { - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() defer tx.Abort() for _, stone := range dump { @@ -240,7 +242,7 @@ func TestGraveyard_Snapshot_Restore(t *testing.T) { t.Fatalf("err: %s", err) } } - tx.Commit() + require.NoError(t, tx.Commit()) }() // Verify that the restore works. diff --git a/agent/consul/state/intention.go b/agent/consul/state/intention.go index 61bb9bd0a1..b20977df71 100644 --- a/agent/consul/state/intention.go +++ b/agent/consul/state/intention.go @@ -157,20 +157,19 @@ func (s *Store) Intentions(ws memdb.WatchSet) (uint64, structs.Intentions, error // IntentionSet creates or updates an intention. func (s *Store) IntentionSet(idx uint64, ixn *structs.Intention) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.intentionSetTxn(tx, idx, ixn); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } // intentionSetTxn is the inner method used to insert an intention with // the proper indexes into the state store. -func (s *Store) intentionSetTxn(tx *memdb.Txn, idx uint64, ixn *structs.Intention) error { +func (s *Store) intentionSetTxn(tx *txn, idx uint64, ixn *structs.Intention) error { // ID is required if ixn.ID == "" { return ErrMissingIntentionID @@ -253,20 +252,19 @@ func (s *Store) IntentionGet(ws memdb.WatchSet, id string) (uint64, *structs.Int // IntentionDelete deletes the given intention by ID. func (s *Store) IntentionDelete(idx uint64, id string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.intentionDeleteTxn(tx, idx, id); err != nil { return fmt.Errorf("failed intention delete: %s", err) } - tx.Commit() - return nil + return tx.Commit() } // intentionDeleteTxn is the inner method used to delete a intention // with the proper indexes into the state store. -func (s *Store) intentionDeleteTxn(tx *memdb.Txn, idx uint64, queryID string) error { +func (s *Store) intentionDeleteTxn(tx *txn, idx uint64, queryID string) error { // Pull the query. wrapped, err := tx.First(intentionsTableName, "id", queryID) if err != nil { diff --git a/agent/consul/state/kvs.go b/agent/consul/state/kvs.go index ab412c133b..2090c42d8b 100644 --- a/agent/consul/state/kvs.go +++ b/agent/consul/state/kvs.go @@ -88,21 +88,20 @@ func (s *Restore) Tombstone(stone *Tombstone) error { // ReapTombstones is used to delete all the tombstones with an index // less than or equal to the given index. This is used to prevent // unbounded storage growth of the tombstones. -func (s *Store) ReapTombstones(index uint64) error { - tx := s.db.Txn(true) +func (s *Store) ReapTombstones(idx uint64, index uint64) error { + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.kvsGraveyard.ReapTxn(tx, index); err != nil { return fmt.Errorf("failed to reap kvs tombstones: %s", err) } - tx.Commit() - return nil + return tx.Commit() } // KVSSet is used to store a key/value pair. func (s *Store) KVSSet(idx uint64, entry *structs.DirEntry) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Perform the actual set. @@ -110,8 +109,7 @@ func (s *Store) KVSSet(idx uint64, entry *structs.DirEntry) error { return err } - tx.Commit() - return nil + return tx.Commit() } // kvsSetTxn is used to insert or update a key/value pair in the state @@ -119,7 +117,7 @@ func (s *Store) KVSSet(idx uint64, entry *structs.DirEntry) error { // If updateSession is true, then the incoming entry will set the new // session (should be validated before calling this). Otherwise, we will keep // whatever the existing session is. -func (s *Store) kvsSetTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntry, updateSession bool) error { +func (s *Store) kvsSetTxn(tx *txn, idx uint64, entry *structs.DirEntry, updateSession bool) error { // Retrieve an existing KV pair existingNode, err := firstWithTxn(tx, "kvs", "id", entry.Key, &entry.EnterpriseMeta) if err != nil { @@ -172,7 +170,7 @@ func (s *Store) KVSGet(ws memdb.WatchSet, key string, entMeta *structs.Enterpris // kvsGetTxn is the inner method that gets a KVS entry inside an existing // transaction. -func (s *Store) kvsGetTxn(tx *memdb.Txn, +func (s *Store) kvsGetTxn(tx *txn, ws memdb.WatchSet, key string, entMeta *structs.EnterpriseMeta) (uint64, *structs.DirEntry, error) { // Get the table index. @@ -205,7 +203,7 @@ func (s *Store) KVSList(ws memdb.WatchSet, // kvsListTxn is the inner method that gets a list of KVS entries matching a // prefix. -func (s *Store) kvsListTxn(tx *memdb.Txn, +func (s *Store) kvsListTxn(tx *txn, ws memdb.WatchSet, prefix string, entMeta *structs.EnterpriseMeta) (uint64, structs.DirEntries, error) { // Get the table indexes. @@ -241,7 +239,7 @@ func (s *Store) kvsListTxn(tx *memdb.Txn, // KVSDelete is used to perform a shallow delete on a single key in the // the state store. func (s *Store) KVSDelete(idx uint64, key string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Perform the actual delete @@ -249,13 +247,12 @@ func (s *Store) KVSDelete(idx uint64, key string, entMeta *structs.EnterpriseMet return err } - tx.Commit() - return nil + return tx.Commit() } // kvsDeleteTxn is the inner method used to perform the actual deletion // of a key/value pair within an existing transaction. -func (s *Store) kvsDeleteTxn(tx *memdb.Txn, idx uint64, key string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) kvsDeleteTxn(tx *txn, idx uint64, key string, entMeta *structs.EnterpriseMeta) error { // Look up the entry in the state store. entry, err := firstWithTxn(tx, "kvs", "id", key, entMeta) if err != nil { @@ -278,7 +275,7 @@ func (s *Store) kvsDeleteTxn(tx *memdb.Txn, idx uint64, key string, entMeta *str // observed index for the given key, then the call is a noop, otherwise // a normal KV delete is invoked. func (s *Store) KVSDeleteCAS(idx, cidx uint64, key string, entMeta *structs.EnterpriseMeta) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() set, err := s.kvsDeleteCASTxn(tx, idx, cidx, key, entMeta) @@ -286,13 +283,13 @@ func (s *Store) KVSDeleteCAS(idx, cidx uint64, key string, entMeta *structs.Ente return false, err } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } // kvsDeleteCASTxn is the inner method that does a CAS delete within an existing // transaction. -func (s *Store) kvsDeleteCASTxn(tx *memdb.Txn, idx, cidx uint64, key string, entMeta *structs.EnterpriseMeta) (bool, error) { +func (s *Store) kvsDeleteCASTxn(tx *txn, idx, cidx uint64, key string, entMeta *structs.EnterpriseMeta) (bool, error) { // Retrieve the existing kvs entry, if any exists. entry, err := firstWithTxn(tx, "kvs", "id", key, entMeta) if err != nil { @@ -319,7 +316,7 @@ func (s *Store) kvsDeleteCASTxn(tx *memdb.Txn, idx, cidx uint64, key string, ent // write the entry to the state store or bail. Returns a bool indicating // if a write happened and any error. func (s *Store) KVSSetCAS(idx uint64, entry *structs.DirEntry) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() set, err := s.kvsSetCASTxn(tx, idx, entry) @@ -327,13 +324,13 @@ func (s *Store) KVSSetCAS(idx uint64, entry *structs.DirEntry) (bool, error) { return false, err } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } // kvsSetCASTxn is the inner method used to do a CAS inside an existing // transaction. -func (s *Store) kvsSetCASTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntry) (bool, error) { +func (s *Store) kvsSetCASTxn(tx *txn, idx uint64, entry *structs.DirEntry) (bool, error) { // Retrieve the existing entry. existing, err := firstWithTxn(tx, "kvs", "id", entry.Key, &entry.EnterpriseMeta) if err != nil { @@ -364,15 +361,14 @@ func (s *Store) kvsSetCASTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntry) // in the state store. If any keys are modified, the last index is // set, otherwise this is a no-op. func (s *Store) KVSDeleteTree(idx uint64, prefix string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.kvsDeleteTreeTxn(tx, idx, prefix, entMeta); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } // KVSLockDelay returns the expiration time for any lock delay associated with @@ -384,7 +380,7 @@ func (s *Store) KVSLockDelay(key string, entMeta *structs.EnterpriseMeta) time.T // KVSLock is similar to KVSSet but only performs the set if the lock can be // acquired. func (s *Store) KVSLock(idx uint64, entry *structs.DirEntry) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() locked, err := s.kvsLockTxn(tx, idx, entry) @@ -392,13 +388,13 @@ func (s *Store) KVSLock(idx uint64, entry *structs.DirEntry) (bool, error) { return false, err } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } // kvsLockTxn is the inner method that does a lock inside an existing // transaction. -func (s *Store) kvsLockTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntry) (bool, error) { +func (s *Store) kvsLockTxn(tx *txn, idx uint64, entry *structs.DirEntry) (bool, error) { // Verify that a session is present. if entry.Session == "" { return false, fmt.Errorf("missing session") @@ -450,7 +446,7 @@ func (s *Store) kvsLockTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntry) ( // KVSUnlock is similar to KVSSet but only performs the set if the lock can be // unlocked (the key must already exist and be locked). func (s *Store) KVSUnlock(idx uint64, entry *structs.DirEntry) (bool, error) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() unlocked, err := s.kvsUnlockTxn(tx, idx, entry) @@ -458,13 +454,13 @@ func (s *Store) KVSUnlock(idx uint64, entry *structs.DirEntry) (bool, error) { return false, err } - tx.Commit() - return true, nil + err = tx.Commit() + return err == nil, err } // kvsUnlockTxn is the inner method that does an unlock inside an existing // transaction. -func (s *Store) kvsUnlockTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntry) (bool, error) { +func (s *Store) kvsUnlockTxn(tx *txn, idx uint64, entry *structs.DirEntry) (bool, error) { // Verify that a session is present. if entry.Session == "" { return false, fmt.Errorf("missing session") @@ -502,7 +498,7 @@ func (s *Store) kvsUnlockTxn(tx *memdb.Txn, idx uint64, entry *structs.DirEntry) // kvsCheckSessionTxn checks to see if the given session matches the current // entry for a key. -func (s *Store) kvsCheckSessionTxn(tx *memdb.Txn, +func (s *Store) kvsCheckSessionTxn(tx *txn, key string, session string, entMeta *structs.EnterpriseMeta) (*structs.DirEntry, error) { entry, err := firstWithTxn(tx, "kvs", "id", key, entMeta) @@ -523,7 +519,7 @@ func (s *Store) kvsCheckSessionTxn(tx *memdb.Txn, // kvsCheckIndexTxn checks to see if the given modify index matches the current // entry for a key. -func (s *Store) kvsCheckIndexTxn(tx *memdb.Txn, +func (s *Store) kvsCheckIndexTxn(tx *txn, key string, cidx uint64, entMeta *structs.EnterpriseMeta) (*structs.DirEntry, error) { entry, err := firstWithTxn(tx, "kvs", "id", key, entMeta) diff --git a/agent/consul/state/kvs_oss.go b/agent/consul/state/kvs_oss.go index 76dcb2ab56..c737401de8 100644 --- a/agent/consul/state/kvs_oss.go +++ b/agent/consul/state/kvs_oss.go @@ -16,7 +16,7 @@ func kvsIndexer() *memdb.StringFieldIndex { } } -func (s *Store) insertKVTxn(tx *memdb.Txn, entry *structs.DirEntry, updateMax bool) error { +func (s *Store) insertKVTxn(tx *txn, entry *structs.DirEntry, updateMax bool) error { if err := tx.Insert("kvs", entry); err != nil { return err } @@ -33,7 +33,7 @@ func (s *Store) insertKVTxn(tx *memdb.Txn, entry *structs.DirEntry, updateMax bo return nil } -func (s *Store) kvsListEntriesTxn(tx *memdb.Txn, ws memdb.WatchSet, prefix string, entMeta *structs.EnterpriseMeta) (uint64, structs.DirEntries, error) { +func (s *Store) kvsListEntriesTxn(tx *txn, ws memdb.WatchSet, prefix string, entMeta *structs.EnterpriseMeta) (uint64, structs.DirEntries, error) { var ents structs.DirEntries var lindex uint64 @@ -56,7 +56,7 @@ func (s *Store) kvsListEntriesTxn(tx *memdb.Txn, ws memdb.WatchSet, prefix strin // kvsDeleteTreeTxn is the inner method that does a recursive delete inside an // existing transaction. -func (s *Store) kvsDeleteTreeTxn(tx *memdb.Txn, idx uint64, prefix string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) kvsDeleteTreeTxn(tx *txn, idx uint64, prefix string, entMeta *structs.EnterpriseMeta) error { // For prefix deletes, only insert one tombstone and delete the entire subtree deleted, err := tx.DeletePrefix("kvs", "id_prefix", prefix) if err != nil { @@ -77,11 +77,11 @@ func (s *Store) kvsDeleteTreeTxn(tx *memdb.Txn, idx uint64, prefix string, entMe return nil } -func kvsMaxIndex(tx *memdb.Txn, entMeta *structs.EnterpriseMeta) uint64 { +func kvsMaxIndex(tx *txn, entMeta *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "kvs", "tombstones") } -func (s *Store) kvsDeleteWithEntry(tx *memdb.Txn, entry *structs.DirEntry, idx uint64) error { +func (s *Store) kvsDeleteWithEntry(tx *txn, entry *structs.DirEntry, idx uint64) error { // Delete the entry and update the index. if err := tx.Delete("kvs", entry); err != nil { return fmt.Errorf("failed deleting kvs entry: %s", err) diff --git a/agent/consul/state/kvs_test.go b/agent/consul/state/kvs_test.go index 5de0f72576..b9830b8a2c 100644 --- a/agent/consul/state/kvs_test.go +++ b/agent/consul/state/kvs_test.go @@ -41,7 +41,7 @@ func TestStateStore_ReapTombstones(t *testing.T) { } // Reap the tombstones <= 6. - if err := s.ReapTombstones(6); err != nil { + if err := s.ReapTombstones(8, 6); err != nil { t.Fatalf("err: %s", err) } @@ -55,7 +55,7 @@ func TestStateStore_ReapTombstones(t *testing.T) { } // Now reap them all. - if err := s.ReapTombstones(7); err != nil { + if err := s.ReapTombstones(9, 7); err != nil { t.Fatalf("err: %s", err) } @@ -466,7 +466,7 @@ func TestStateStore_KVSList(t *testing.T) { // Now reap the tombstones and make sure we get the latest index // since there are no matching keys. - if err := s.ReapTombstones(6); err != nil { + if err := s.ReapTombstones(8, 6); err != nil { t.Fatalf("err: %s", err) } idx, _, err = s.KVSList(nil, "foo/bar/baz", nil) @@ -536,7 +536,7 @@ func TestStateStore_KVSDelete(t *testing.T) { // Now reap the tombstone and watch the index revert to the remaining // foo/bar key's index. - if err := s.ReapTombstones(3); err != nil { + if err := s.ReapTombstones(4, 3); err != nil { t.Fatalf("err: %s", err) } idx, _, err = s.KVSList(nil, "foo", nil) @@ -549,7 +549,7 @@ func TestStateStore_KVSDelete(t *testing.T) { // Deleting a nonexistent key should be idempotent and not return an // error - if err := s.KVSDelete(4, "foo", nil); err != nil { + if err := s.KVSDelete(5, "foo", nil); err != nil { t.Fatalf("err: %s", err) } if idx := s.maxIndex("kvs"); idx != 3 { @@ -618,7 +618,7 @@ func TestStateStore_KVSDeleteCAS(t *testing.T) { // Now reap the tombstone and watch the index move up to the table // index since there are no matching keys. - if err := s.ReapTombstones(4); err != nil { + if err := s.ReapTombstones(6, 4); err != nil { t.Fatalf("err: %s", err) } idx, _, err = s.KVSList(nil, "bar", nil) @@ -631,7 +631,7 @@ func TestStateStore_KVSDeleteCAS(t *testing.T) { // A delete on a nonexistent key should be idempotent and not return an // error - ok, err = s.KVSDeleteCAS(6, 2, "bar", nil) + ok, err = s.KVSDeleteCAS(7, 2, "bar", nil) if !ok || err != nil { t.Fatalf("expected (true, nil), got: (%v, %#v)", ok, err) } @@ -901,7 +901,7 @@ func TestStateStore_KVSDeleteTree(t *testing.T) { // Now reap the tombstones and watch the index revert to the remaining // foo/zorp key's index. - if err := s.ReapTombstones(5); err != nil { + if err := s.ReapTombstones(6, 5); err != nil { t.Fatalf("err: %s", err) } idx, _, err = s.KVSList(nil, "foo", nil) @@ -958,7 +958,7 @@ func TestStateStore_Watches_PrefixDelete(t *testing.T) { } // Reap tombstone and verify list on the same key reverts its index value - if err := s.ReapTombstones(wantIndex); err != nil { + if err := s.ReapTombstones(8, wantIndex); err != nil { t.Fatalf("err: %s", err) } @@ -973,11 +973,11 @@ func TestStateStore_Watches_PrefixDelete(t *testing.T) { // Set a different key to bump the index. This shouldn't fire the // watch since there's a different prefix. - testSetKey(t, s, 8, "some/other/key", "", nil) + testSetKey(t, s, 9, "some/other/key", "", nil) // Now ask for the index for a node within the prefix that was deleted // We expect to get the max index in the tree - wantIndex = 8 + wantIndex = 9 ws = memdb.NewWatchSet() got, _, err = s.KVSList(ws, "foo/bar/baz", nil) if err != nil { @@ -1000,10 +1000,10 @@ func TestStateStore_Watches_PrefixDelete(t *testing.T) { } // Delete all the keys, special case where tombstones are not inserted - if err := s.KVSDeleteTree(9, "", nil); err != nil { + if err := s.KVSDeleteTree(10, "", nil); err != nil { t.Fatalf("unexpected err: %s", err) } - wantIndex = 9 + wantIndex = 10 got, _, err = s.KVSList(nil, "/foo/bar", nil) if err != nil { t.Fatalf("err: %s", err) @@ -1382,7 +1382,7 @@ func TestStateStore_Tombstone_Snapshot_Restore(t *testing.T) { defer snap.Close() // Alter the real state store. - if err := s.ReapTombstones(4); err != nil { + if err := s.ReapTombstones(5, 4); err != nil { t.Fatalf("err: %s", err) } idx, _, err := s.KVSList(nil, "foo/bar", nil) @@ -1433,7 +1433,7 @@ func TestStateStore_Tombstone_Snapshot_Restore(t *testing.T) { // Make sure it reaps correctly. We should still get a 4 for // the index here because it will be using the last index from // the tombstone table. - if err := s.ReapTombstones(4); err != nil { + if err := s.ReapTombstones(6, 4); err != nil { t.Fatalf("err: %s", err) } idx, _, err = s.KVSList(nil, "foo/bar", nil) diff --git a/agent/consul/state/memdb_wrapper.go b/agent/consul/state/memdb_wrapper.go new file mode 100644 index 0000000000..263c2c9923 --- /dev/null +++ b/agent/consul/state/memdb_wrapper.go @@ -0,0 +1,93 @@ +package state + +import ( + "github.com/hashicorp/go-memdb" +) + +// changeTrackerDB is a thin wrapper around memdb.DB which enables TrackChanges on +// all write transactions. When the transaction is committed the changes are +// sent to the eventPublisher which will create and emit change events. +type changeTrackerDB struct { + db *memdb.MemDB + // TODO(streaming): add publisher +} + +// Txn exists to maintain backwards compatibility with memdb.DB.Txn. Preexisting +// code may use it to create a read-only transaction, but it will panic if called +// with write=true. +// +// Deprecated: use either ReadTxn, or WriteTxn. +func (db *changeTrackerDB) Txn(write bool) *txn { + if write { + panic("don't use db.Txn(true), use db.WriteTxn(idx uin64)") + } + return db.ReadTxn() +} + +// ReadTxn returns a read-only transaction which behaves exactly the same as +// memdb.Txn +func (db *changeTrackerDB) ReadTxn() *txn { + return &txn{Txn: db.db.Txn(false)} +} + +// WriteTxn returns a wrapped memdb.Txn suitable for writes to the state store. +// It will track changes and publish events for the changes when Commit +// is called. +// +// The idx argument must be the index of the current Raft operation. Almost +// all mutations to state should happen as part of a raft apply so the index of +// the log being applied can be passed to WriteTxn. +// The exceptional cases are transactions that are executed on an empty +// memdb.DB as part of Restore, and those executed by tests where we insert +// data directly into the DB. These cases may use WriteTxnRestore. +func (db *changeTrackerDB) WriteTxn(idx uint64) *txn { + t := &txn{ + Txn: db.db.Txn(true), + Index: idx, + } + t.Txn.TrackChanges() + return t +} + +// WriteTxnRestore returns a wrapped RW transaction that does NOT have change +// tracking enabled. This should only be used in Restore where we need to +// replace the entire contents of the Store without a need to track the changes. +// WriteTxnRestore uses a zero index since the whole restore doesn't really occur +// at one index - the effect is to write many values that were previously +// written across many indexes. +func (db *changeTrackerDB) WriteTxnRestore() *txn { + t := &txn{ + Txn: db.db.Txn(true), + Index: 0, + } + return t +} + +// txn wraps a memdb.Txn to capture changes and send them to the EventPublisher. +// +// This can not be done with txn.Defer because the callback passed to Defer is +// invoked after commit completes, and because the callback can not return an +// error. Any errors from the callback would be lost, which would result in a +// missing change event, even though the state store had changed. +type txn struct { + // Index in raft where the write is occurring. The value is zero for a + // read-only transaction, and for a WriteTxnRestore transaction. + // Index is stored so that it may be passed along to any subscribers as part + // of a change event. + Index uint64 + *memdb.Txn +} + +// Commit first pushes changes to EventPublisher, then calls Commit on the +// underlying transaction. +// +// Note that this function, unlike memdb.Txn, returns an error which must be checked +// by the caller. A non-nil error indicates that a commit failed and was not +// applied. +func (tx *txn) Commit() error { + // changes may be empty if this is a read-only or WriteTxnRestore transaction. + // TODO(streaming): publish changes: changes := tx.Txn.Changes() + + tx.Txn.Commit() + return nil +} diff --git a/agent/consul/state/operations_oss.go b/agent/consul/state/operations_oss.go index 27ce1ced85..4c382694b9 100644 --- a/agent/consul/state/operations_oss.go +++ b/agent/consul/state/operations_oss.go @@ -7,30 +7,30 @@ import ( "github.com/hashicorp/go-memdb" ) -func firstWithTxn(tx *memdb.Txn, +func firstWithTxn(tx *txn, table, index, idxVal string, entMeta *structs.EnterpriseMeta) (interface{}, error) { return tx.First(table, index, idxVal) } -func firstWatchWithTxn(tx *memdb.Txn, +func firstWatchWithTxn(tx *txn, table, index, idxVal string, entMeta *structs.EnterpriseMeta) (<-chan struct{}, interface{}, error) { return tx.FirstWatch(table, index, idxVal) } -func firstWatchCompoundWithTxn(tx *memdb.Txn, +func firstWatchCompoundWithTxn(tx *txn, table, index string, _ *structs.EnterpriseMeta, idxVals ...interface{}) (<-chan struct{}, interface{}, error) { return tx.FirstWatch(table, index, idxVals...) } -func getWithTxn(tx *memdb.Txn, +func getWithTxn(tx *txn, table, index, idxVal string, entMeta *structs.EnterpriseMeta) (memdb.ResultIterator, error) { return tx.Get(table, index, idxVal) } -func getCompoundWithTxn(tx *memdb.Txn, table, index string, +func getCompoundWithTxn(tx *txn, table, index string, _ *structs.EnterpriseMeta, idxVals ...interface{}) (memdb.ResultIterator, error) { return tx.Get(table, index, idxVals...) diff --git a/agent/consul/state/prepared_query.go b/agent/consul/state/prepared_query.go index 89a8f83493..c639e5293d 100644 --- a/agent/consul/state/prepared_query.go +++ b/agent/consul/state/prepared_query.go @@ -130,20 +130,19 @@ func (s *Restore) PreparedQuery(query *structs.PreparedQuery) error { // PreparedQuerySet is used to create or update a prepared query. func (s *Store) PreparedQuerySet(idx uint64, query *structs.PreparedQuery) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.preparedQuerySetTxn(tx, idx, query); err != nil { return err } - tx.Commit() - return nil + return tx.Commit() } // preparedQuerySetTxn is the inner method used to insert a prepared query with // the proper indexes into the state store. -func (s *Store) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *structs.PreparedQuery) error { +func (s *Store) preparedQuerySetTxn(tx *txn, idx uint64, query *structs.PreparedQuery) error { // Check that the ID is set. if query.ID == "" { return ErrMissingQueryID @@ -247,20 +246,19 @@ func (s *Store) preparedQuerySetTxn(tx *memdb.Txn, idx uint64, query *structs.Pr // PreparedQueryDelete deletes the given query by ID. func (s *Store) PreparedQueryDelete(idx uint64, queryID string) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() if err := s.preparedQueryDeleteTxn(tx, idx, queryID); err != nil { return fmt.Errorf("failed prepared query delete: %s", err) } - tx.Commit() - return nil + return tx.Commit() } // preparedQueryDeleteTxn is the inner method used to delete a prepared query // with the proper indexes into the state store. -func (s *Store) preparedQueryDeleteTxn(tx *memdb.Txn, idx uint64, queryID string) error { +func (s *Store) preparedQueryDeleteTxn(tx *txn, idx uint64, queryID string) error { // Pull the query. wrapped, err := tx.First("prepared-queries", "id", queryID) if err != nil { diff --git a/agent/consul/state/session.go b/agent/consul/state/session.go index 8f32bd3944..6bab209b9c 100644 --- a/agent/consul/state/session.go +++ b/agent/consul/state/session.go @@ -155,7 +155,7 @@ func (s *Restore) Session(sess *structs.Session) error { // SessionCreate is used to register a new session in the state store. func (s *Store) SessionCreate(idx uint64, sess *structs.Session) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // This code is technically able to (incorrectly) update an existing @@ -170,14 +170,13 @@ func (s *Store) SessionCreate(idx uint64, sess *structs.Session) error { return err } - tx.Commit() - return nil + return tx.Commit() } // sessionCreateTxn is the inner method used for creating session entries in // an open transaction. Any health checks registered with the session will be // checked for failing status. Returns any error encountered. -func (s *Store) sessionCreateTxn(tx *memdb.Txn, idx uint64, sess *structs.Session) error { +func (s *Store) sessionCreateTxn(tx *txn, idx uint64, sess *structs.Session) error { // Check that we have a session ID if sess.ID == "" { return ErrMissingSessionID @@ -289,7 +288,7 @@ func (s *Store) NodeSessions(ws memdb.WatchSet, nodeID string, entMeta *structs. // implicitly invalidate the session and invoke the specified // session destroy behavior. func (s *Store) SessionDestroy(idx uint64, sessionID string, entMeta *structs.EnterpriseMeta) error { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() // Call the session deletion. @@ -297,13 +296,12 @@ func (s *Store) SessionDestroy(idx uint64, sessionID string, entMeta *structs.En return err } - tx.Commit() - return nil + return tx.Commit() } // deleteSessionTxn is the inner method, which is used to do the actual // session deletion and handle session invalidation, etc. -func (s *Store) deleteSessionTxn(tx *memdb.Txn, idx uint64, sessionID string, entMeta *structs.EnterpriseMeta) error { +func (s *Store) deleteSessionTxn(tx *txn, idx uint64, sessionID string, entMeta *structs.EnterpriseMeta) error { // Look up the session. sess, err := firstWithTxn(tx, "sessions", "id", sessionID, entMeta) if err != nil { diff --git a/agent/consul/state/session_oss.go b/agent/consul/state/session_oss.go index 1208f4d82b..a9f5ee50db 100644 --- a/agent/consul/state/session_oss.go +++ b/agent/consul/state/session_oss.go @@ -35,7 +35,7 @@ func nodeChecksIndexer() *memdb.CompoundIndex { } } -func (s *Store) sessionDeleteWithSession(tx *memdb.Txn, session *structs.Session, idx uint64) error { +func (s *Store) sessionDeleteWithSession(tx *txn, session *structs.Session, idx uint64) error { if err := tx.Delete("sessions", session); err != nil { return fmt.Errorf("failed deleting session: %s", err) } @@ -48,7 +48,7 @@ func (s *Store) sessionDeleteWithSession(tx *memdb.Txn, session *structs.Session return nil } -func (s *Store) insertSessionTxn(tx *memdb.Txn, session *structs.Session, idx uint64, updateMax bool) error { +func (s *Store) insertSessionTxn(tx *txn, session *structs.Session, idx uint64, updateMax bool) error { if err := tx.Insert("sessions", session); err != nil { return err } @@ -80,11 +80,11 @@ func (s *Store) insertSessionTxn(tx *memdb.Txn, session *structs.Session, idx ui return nil } -func (s *Store) allNodeSessionsTxn(tx *memdb.Txn, node string) (structs.Sessions, error) { +func (s *Store) allNodeSessionsTxn(tx *txn, node string) (structs.Sessions, error) { return s.nodeSessionsTxn(tx, nil, node, nil) } -func (s *Store) nodeSessionsTxn(tx *memdb.Txn, +func (s *Store) nodeSessionsTxn(tx *txn, ws memdb.WatchSet, node string, entMeta *structs.EnterpriseMeta) (structs.Sessions, error) { sessions, err := tx.Get("sessions", "node", node) @@ -100,11 +100,11 @@ func (s *Store) nodeSessionsTxn(tx *memdb.Txn, return result, nil } -func (s *Store) sessionMaxIndex(tx *memdb.Txn, entMeta *structs.EnterpriseMeta) uint64 { +func (s *Store) sessionMaxIndex(tx *txn, entMeta *structs.EnterpriseMeta) uint64 { return maxIndexTxn(tx, "sessions") } -func (s *Store) validateSessionChecksTxn(tx *memdb.Txn, session *structs.Session) error { +func (s *Store) validateSessionChecksTxn(tx *txn, session *structs.Session) error { // Go over the session checks and ensure they exist. for _, checkID := range session.CheckIDs() { check, err := tx.First("checks", "id", session.Node, string(checkID)) diff --git a/agent/consul/state/state_store.go b/agent/consul/state/state_store.go index c07fcf945d..2c057f4b1a 100644 --- a/agent/consul/state/state_store.go +++ b/agent/consul/state/state_store.go @@ -3,8 +3,9 @@ package state import ( "errors" "fmt" + "github.com/hashicorp/consul/agent/structs" - "github.com/hashicorp/go-memdb" + memdb "github.com/hashicorp/go-memdb" ) var ( @@ -97,7 +98,7 @@ const ( // from the Raft log through the FSM. type Store struct { schema *memdb.DBSchema - db *memdb.MemDB + db *changeTrackerDB // abandonCh is used to signal watchers that this state store has been // abandoned (usually during a restore). This is only ever closed. @@ -114,7 +115,7 @@ type Store struct { // works by starting a read transaction against the whole state store. type Snapshot struct { store *Store - tx *memdb.Txn + tx *txn lastIndex uint64 } @@ -122,7 +123,7 @@ type Snapshot struct { // data to a state store. type Restore struct { store *Store - tx *memdb.Txn + tx *txn } // IndexEntry keeps a record of the last index per-table. @@ -155,11 +156,13 @@ func NewStateStore(gc *TombstoneGC) (*Store, error) { // Create and return the state store. s := &Store{ schema: schema, - db: db, abandonCh: make(chan struct{}), kvsGraveyard: NewGraveyard(gc), lockDelay: NewDelay(), } + s.db = &changeTrackerDB{ + db: db, + } return s, nil } @@ -206,7 +209,7 @@ func (s *Snapshot) Close() { // the state store. It works by doing all the restores inside of a single // transaction. func (s *Store) Restore() *Restore { - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() return &Restore{s, tx} } @@ -218,8 +221,8 @@ func (s *Restore) Abort() { // Commit commits the changes made by a restore. This or Abort should always be // called. -func (s *Restore) Commit() { - s.tx.Commit() +func (s *Restore) Commit() error { + return s.tx.Commit() } // AbandonCh returns a channel you can wait on to know if the state store was @@ -244,11 +247,11 @@ func (s *Store) maxIndex(tables ...string) uint64 { // maxIndexTxn is a helper used to retrieve the highest known index // amongst a set of tables in the db. -func maxIndexTxn(tx *memdb.Txn, tables ...string) uint64 { +func maxIndexTxn(tx *txn, tables ...string) uint64 { return maxIndexWatchTxn(tx, nil, tables...) } -func maxIndexWatchTxn(tx *memdb.Txn, ws memdb.WatchSet, tables ...string) uint64 { +func maxIndexWatchTxn(tx *txn, ws memdb.WatchSet, tables ...string) uint64 { var lindex uint64 for _, table := range tables { ch, ti, err := tx.FirstWatch("index", "id", table) @@ -265,7 +268,7 @@ func maxIndexWatchTxn(tx *memdb.Txn, ws memdb.WatchSet, tables ...string) uint64 // indexUpdateMaxTxn is used when restoring entries and sets the table's index to // the given idx only if it's greater than the current index. -func indexUpdateMaxTxn(tx *memdb.Txn, idx uint64, table string) error { +func indexUpdateMaxTxn(tx *txn, idx uint64, table string) error { ti, err := tx.First("index", "id", table) if err != nil { return fmt.Errorf("failed to retrieve existing index: %s", err) diff --git a/agent/consul/state/state_store_test.go b/agent/consul/state/state_store_test.go index ccc067b630..2c418af1ca 100644 --- a/agent/consul/state/state_store_test.go +++ b/agent/consul/state/state_store_test.go @@ -296,11 +296,11 @@ func TestStateStore_indexUpdateMaxTxn(t *testing.T) { testRegisterNode(t, s, 0, "foo") testRegisterNode(t, s, 1, "bar") - tx := s.db.Txn(true) + tx := s.db.WriteTxnRestore() if err := indexUpdateMaxTxn(tx, 3, "nodes"); err != nil { t.Fatalf("err: %s", err) } - tx.Commit() + require.NoError(t, tx.Commit()) if max := s.maxIndex("nodes"); max != 3 { t.Fatalf("bad max: %d", max) diff --git a/agent/consul/state/txn.go b/agent/consul/state/txn.go index f8c02e25a9..e5a7dd5f77 100644 --- a/agent/consul/state/txn.go +++ b/agent/consul/state/txn.go @@ -5,11 +5,10 @@ import ( "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/api" - "github.com/hashicorp/go-memdb" ) // txnKVS handles all KV-related operations. -func (s *Store) txnKVS(tx *memdb.Txn, idx uint64, op *structs.TxnKVOp) (structs.TxnResults, error) { +func (s *Store) txnKVS(tx *txn, idx uint64, op *structs.TxnKVOp) (structs.TxnResults, error) { var entry *structs.DirEntry var err error @@ -111,7 +110,7 @@ func (s *Store) txnKVS(tx *memdb.Txn, idx uint64, op *structs.TxnKVOp) (structs. } // txnSession handles all Session-related operations. -func (s *Store) txnSession(tx *memdb.Txn, idx uint64, op *structs.TxnSessionOp) error { +func (s *Store) txnSession(tx *txn, idx uint64, op *structs.TxnSessionOp) error { var err error switch op.Verb { @@ -128,7 +127,7 @@ func (s *Store) txnSession(tx *memdb.Txn, idx uint64, op *structs.TxnSessionOp) } // txnIntention handles all Intention-related operations. -func (s *Store) txnIntention(tx *memdb.Txn, idx uint64, op *structs.TxnIntentionOp) error { +func (s *Store) txnIntention(tx *txn, idx uint64, op *structs.TxnIntentionOp) error { switch op.Op { case structs.IntentionOpCreate, structs.IntentionOpUpdate: return s.intentionSetTxn(tx, idx, op.Intention) @@ -140,7 +139,7 @@ func (s *Store) txnIntention(tx *memdb.Txn, idx uint64, op *structs.TxnIntention } // txnNode handles all Node-related operations. -func (s *Store) txnNode(tx *memdb.Txn, idx uint64, op *structs.TxnNodeOp) (structs.TxnResults, error) { +func (s *Store) txnNode(tx *txn, idx uint64, op *structs.TxnNodeOp) (structs.TxnResults, error) { var entry *structs.Node var err error @@ -209,7 +208,7 @@ func (s *Store) txnNode(tx *memdb.Txn, idx uint64, op *structs.TxnNodeOp) (struc } // txnService handles all Service-related operations. -func (s *Store) txnService(tx *memdb.Txn, idx uint64, op *structs.TxnServiceOp) (structs.TxnResults, error) { +func (s *Store) txnService(tx *txn, idx uint64, op *structs.TxnServiceOp) (structs.TxnResults, error) { switch op.Verb { case api.ServiceGet: entry, err := s.getNodeServiceTxn(tx, op.Node, op.Service.ID, &op.Service.EnterpriseMeta) @@ -271,7 +270,7 @@ func newTxnResultFromNodeServiceEntry(entry *structs.NodeService) structs.TxnRes } // txnCheck handles all Check-related operations. -func (s *Store) txnCheck(tx *memdb.Txn, idx uint64, op *structs.TxnCheckOp) (structs.TxnResults, error) { +func (s *Store) txnCheck(tx *txn, idx uint64, op *structs.TxnCheckOp) (structs.TxnResults, error) { var entry *structs.HealthCheck var err error @@ -333,7 +332,7 @@ func (s *Store) txnCheck(tx *memdb.Txn, idx uint64, op *structs.TxnCheckOp) (str } // txnDispatch runs the given operations inside the state store transaction. -func (s *Store) txnDispatch(tx *memdb.Txn, idx uint64, ops structs.TxnOps) (structs.TxnResults, structs.TxnErrors) { +func (s *Store) txnDispatch(tx *txn, idx uint64, ops structs.TxnOps) (structs.TxnResults, structs.TxnErrors) { results := make(structs.TxnResults, 0, len(ops)) errors := make(structs.TxnErrors, 0, len(ops)) for i, op := range ops { @@ -383,7 +382,7 @@ func (s *Store) txnDispatch(tx *memdb.Txn, idx uint64, ops structs.TxnOps) (stru // is done in a full write transaction on the state store, so reads and writes // are possible func (s *Store) TxnRW(idx uint64, ops structs.TxnOps) (structs.TxnResults, structs.TxnErrors) { - tx := s.db.Txn(true) + tx := s.db.WriteTxn(idx) defer tx.Abort() results, errors := s.txnDispatch(tx, idx, ops) @@ -391,7 +390,12 @@ func (s *Store) TxnRW(idx uint64, ops structs.TxnOps) (structs.TxnResults, struc return nil, errors } - tx.Commit() + err := tx.Commit() + if err != nil { + return nil, structs.TxnErrors{ + {What: err.Error(), OpIndex: 0}, + } + } return results, nil } diff --git a/go.mod b/go.mod index ece300471a..4be2d2524a 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/hashicorp/go-connlimit v0.2.0 github.com/hashicorp/go-discover v0.0.0-20200501174627-ad1e96bde088 github.com/hashicorp/go-hclog v0.12.0 - github.com/hashicorp/go-memdb v1.0.3 + github.com/hashicorp/go-memdb v1.1.0 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/go-multierror v1.1.0 github.com/hashicorp/go-raftchunking v0.6.1 @@ -46,7 +46,7 @@ require ( github.com/hashicorp/go-syslog v1.0.0 github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-version v1.2.0 - github.com/hashicorp/golang-lru v0.5.1 + github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/hcl v1.0.0 github.com/hashicorp/hil v0.0.0-20160711231837-1e86c6b523c5 github.com/hashicorp/memberlist v0.2.2 diff --git a/go.sum b/go.sum index 617a8ec299..b70fed59f4 100644 --- a/go.sum +++ b/go.sum @@ -212,8 +212,8 @@ github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.1.0 h1:vN9wG1D6KG6YHRTWr8512cxGOVgTMEfgEdSj/hr8MPc= github.com/hashicorp/go-immutable-radix v1.1.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-memdb v1.0.3 h1:iiqzNk8jKB6/sLRj623Ui/Vi1zf21LOUpgzGjTge6a8= -github.com/hashicorp/go-memdb v1.0.3/go.mod h1:LWQ8R70vPrS4OEY9k28D2z8/Zzyu34NVzeRibGAzHO0= +github.com/hashicorp/go-memdb v1.1.0 h1:ClvpUXpBA6UDs5+vc1h3wqe4UJU+rwum7CU219SeCbk= +github.com/hashicorp/go-memdb v1.1.0/go.mod h1:LWQ8R70vPrS4OEY9k28D2z8/Zzyu34NVzeRibGAzHO0= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -246,6 +246,8 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hil v0.0.0-20160711231837-1e86c6b523c5 h1:uk280DXEbQiCOZgCOI3elFSeNxf8YIZiNsbr2pQLYD0= diff --git a/vendor/github.com/hashicorp/go-memdb/README.md b/vendor/github.com/hashicorp/go-memdb/README.md index 350f08d7dd..f445a756d5 100644 --- a/vendor/github.com/hashicorp/go-memdb/README.md +++ b/vendor/github.com/hashicorp/go-memdb/README.md @@ -37,45 +37,57 @@ The full documentation is available on [Godoc](http://godoc.org/github.com/hashi Example ======= -Below is a simple example of usage +Below is a [simple example](https://play.golang.org/p/gCGE9FA4og1) of usage ```go // Create a sample struct type Person struct { - Email string - Name string - Age int + Email string + Name string + Age int } // Create the DB schema schema := &memdb.DBSchema{ - Tables: map[string]*memdb.TableSchema{ - "person": &memdb.TableSchema{ - Name: "person", - Indexes: map[string]*memdb.IndexSchema{ - "id": &memdb.IndexSchema{ - Name: "id", - Unique: true, - Indexer: &memdb.StringFieldIndex{Field: "Email"}, - }, - }, - }, - }, + Tables: map[string]*memdb.TableSchema{ + "person": &memdb.TableSchema{ + Name: "person", + Indexes: map[string]*memdb.IndexSchema{ + "id": &memdb.IndexSchema{ + Name: "id", + Unique: true, + Indexer: &memdb.StringFieldIndex{Field: "Email"}, + }, + "age": &memdb.IndexSchema{ + Name: "age", + Unique: false, + Indexer: &memdb.IntFieldIndex{Field: "Age"}, + }, + }, + }, + }, } // Create a new data base db, err := memdb.NewMemDB(schema) if err != nil { - panic(err) + panic(err) } // Create a write transaction txn := db.Txn(true) -// Insert a new person -p := &Person{"joe@aol.com", "Joe", 30} -if err := txn.Insert("person", p); err != nil { - panic(err) +// Insert some people +people := []*Person{ + &Person{"joe@aol.com", "Joe", 30}, + &Person{"lucy@aol.com", "Lucy", 35}, + &Person{"tariq@aol.com", "Tariq", 21}, + &Person{"dorothy@aol.com", "Dorothy", 53}, +} +for _, p := range people { + if err := txn.Insert("person", p); err != nil { + panic(err) + } } // Commit the transaction @@ -88,11 +100,47 @@ defer txn.Abort() // Lookup by email raw, err := txn.First("person", "id", "joe@aol.com") if err != nil { - panic(err) + panic(err) } // Say hi! -fmt.Printf("Hello %s!", raw.(*Person).Name) +fmt.Printf("Hello %s!\n", raw.(*Person).Name) +// List all the people +it, err := txn.Get("person", "id") +if err != nil { + panic(err) +} + +fmt.Println("All the people:") +for obj := it.Next(); obj != nil; obj = it.Next() { + p := obj.(*Person) + fmt.Printf(" %s\n", p.Name) +} + +// Range scan over people with ages between 25 and 35 inclusive +it, err = txn.LowerBound("person", "age", 25) +if err != nil { + panic(err) +} + +fmt.Println("People aged 25 - 35:") +for obj := it.Next(); obj != nil; obj = it.Next() { + p := obj.(*Person) + if p.Age > 35 { + break + } + fmt.Printf(" %s is aged %d\n", p.Name, p.Age) +} +// Output: +// Hello Joe! +// All the people: +// Dorothy +// Joe +// Lucy +// Tariq +// People aged 25 - 35: +// Joe is aged 30 +// Lucy is aged 35 ``` diff --git a/vendor/github.com/hashicorp/go-memdb/changes.go b/vendor/github.com/hashicorp/go-memdb/changes.go new file mode 100644 index 0000000000..35089f5ce7 --- /dev/null +++ b/vendor/github.com/hashicorp/go-memdb/changes.go @@ -0,0 +1,34 @@ +package memdb + +// Changes describes a set of mutations to memDB tables performed during a +// transaction. +type Changes []Change + +// Change describes a mutation to an object in a table. +type Change struct { + Table string + Before interface{} + After interface{} + + // primaryKey stores the raw key value from the primary index so that we can + // de-duplicate multiple updates of the same object in the same transaction + // but we don't expose this implementation detail to the consumer. + primaryKey []byte +} + +// Created returns true if the mutation describes a new object being inserted. +func (m *Change) Created() bool { + return m.Before == nil && m.After != nil +} + +// Updated returns true if the mutation describes an existing object being +// updated. +func (m *Change) Updated() bool { + return m.Before != nil && m.After != nil +} + +// Deleted returns true if the mutation describes an existing object being +// deleted. +func (m *Change) Deleted() bool { + return m.Before != nil && m.After == nil +} diff --git a/vendor/github.com/hashicorp/go-memdb/index.go b/vendor/github.com/hashicorp/go-memdb/index.go index e368319e54..604dff7e99 100644 --- a/vendor/github.com/hashicorp/go-memdb/index.go +++ b/vendor/github.com/hashicorp/go-memdb/index.go @@ -73,7 +73,7 @@ func (s *StringFieldIndex) FromObject(obj interface{}) (bool, []byte, error) { if isPtr && !fv.IsValid() { val := "" - return true, []byte(val), nil + return false, []byte(val), nil } val := fv.String() diff --git a/vendor/github.com/hashicorp/go-memdb/txn.go b/vendor/github.com/hashicorp/go-memdb/txn.go index 762a81b7c8..5325d9f1e9 100644 --- a/vendor/github.com/hashicorp/go-memdb/txn.go +++ b/vendor/github.com/hashicorp/go-memdb/txn.go @@ -33,9 +33,25 @@ type Txn struct { rootTxn *iradix.Txn after []func() + // changes is used to track the changes performed during the transaction. If + // it is nil at transaction start then changes are not tracked. + changes Changes + modified map[tableIndex]*iradix.Txn } +// TrackChanges enables change tracking for the transaction. If called at any +// point before commit, subsequent mutations will be recorded and can be +// retrieved using ChangeSet. Once this has been called on a transaction it +// can't be unset. As with other Txn methods it's not safe to call this from a +// different goroutine than the one making mutations or committing the +// transaction. +func (txn *Txn) TrackChanges() { + if txn.changes == nil { + txn.changes = make(Changes, 0, 1) + } +} + // readableIndex returns a transaction usable for reading the given // index in a table. If a write transaction is in progress, we may need // to use an existing modified txn. @@ -101,6 +117,7 @@ func (txn *Txn) Abort() { // Clear the txn txn.rootTxn = nil txn.modified = nil + txn.changes = nil // Release the writer lock since this is invalid txn.db.writer.Unlock() @@ -265,6 +282,14 @@ func (txn *Txn) Insert(table string, obj interface{}) error { indexTxn.Insert(val, obj) } } + if txn.changes != nil { + txn.changes = append(txn.changes, Change{ + Table: table, + Before: existing, // might be nil on a create + After: obj, + primaryKey: idVal, + }) + } return nil } @@ -332,6 +357,14 @@ func (txn *Txn) Delete(table string, obj interface{}) error { } } } + if txn.changes != nil { + txn.changes = append(txn.changes, Change{ + Table: table, + Before: existing, + After: nil, // Now nil indicates deletion + primaryKey: idVal, + }) + } return nil } @@ -376,6 +409,19 @@ func (txn *Txn) DeletePrefix(table string, prefix_index string, prefix string) ( if !ok { return false, fmt.Errorf("object missing primary index") } + if txn.changes != nil { + // Record the deletion + idTxn := txn.writableIndex(table, id) + existing, ok := idTxn.Get(idVal) + if ok { + txn.changes = append(txn.changes, Change{ + Table: table, + Before: existing, + After: nil, // Now nil indicates deletion + primaryKey: idVal, + }) + } + } // Remove the object from all the indexes except the given prefix index for name, indexSchema := range tableSchema.Indexes { if name == deletePrefixIndex { @@ -413,6 +459,7 @@ func (txn *Txn) DeletePrefix(table string, prefix_index string, prefix string) ( } } } + } if foundAny { indexTxn := txn.writableIndex(table, deletePrefixIndex) @@ -629,6 +676,82 @@ func (txn *Txn) LowerBound(table, index string, args ...interface{}) (ResultIter return iter, nil } +// objectID is a tuple of table name and the raw internal id byte slice +// converted to a string. It's only converted to a string to make it comparable +// so this struct can be used as a map index. +type objectID struct { + Table string + IndexVal string +} + +// mutInfo stores metadata about mutations to allow collapsing multiple +// mutations to the same object into one. +type mutInfo struct { + firstBefore interface{} + lastIdx int +} + +// Changes returns the set of object changes that have been made in the +// transaction so far. If change tracking is not enabled it wil always return +// nil. It can be called before or after Commit. If it is before Commit it will +// return all changes made so far which may not be the same as the final +// Changes. After abort it will always return nil. As with other Txn methods +// it's not safe to call this from a different goroutine than the one making +// mutations or committing the transaction. Mutations will appear in the order +// they were performed in the transaction but multiple operations to the same +// object will be collapsed so only the effective overall change to that object +// is present. If transaction operations are dependent (e.g. copy object X to Y +// then delete X) this might mean the set of mutations is incomplete to verify +// history, but it is complete in that the net effect is preserved (Y got a new +// value, X got removed). +func (txn *Txn) Changes() Changes { + if txn.changes == nil { + return nil + } + + // De-duplicate mutations by key so all take effect at the point of the last + // write but we keep the mutations in order. + dups := make(map[objectID]mutInfo) + for i, m := range txn.changes { + oid := objectID{ + Table: m.Table, + IndexVal: string(m.primaryKey), + } + // Store the latest mutation index for each key value + mi, ok := dups[oid] + if !ok { + // First entry for key, store the before value + mi.firstBefore = m.Before + } + mi.lastIdx = i + dups[oid] = mi + } + if len(dups) == len(txn.changes) { + // No duplicates found, fast path return it as is + return txn.changes + } + + // Need to remove the duplicates + cs := make(Changes, 0, len(dups)) + for i, m := range txn.changes { + oid := objectID{ + Table: m.Table, + IndexVal: string(m.primaryKey), + } + mi := dups[oid] + if mi.lastIdx == i { + // This was the latest value for this key copy it with the before value in + // case it's different. Note that m is not a pointer so we are not + // modifying the txn.changeSet here - it's already a copy. + m.Before = mi.firstBefore + cs = append(cs, m) + } + } + // Store the de-duped version in case this is called again + txn.changes = cs + return cs +} + func (txn *Txn) getIndexIterator(table, index string, args ...interface{}) (*iradix.Iterator, []byte, error) { // Get the index value to scan indexSchema, val, err := txn.getIndexValue(table, index, args...) diff --git a/vendor/github.com/hashicorp/golang-lru/go.mod b/vendor/github.com/hashicorp/golang-lru/go.mod index 824cb97e83..8ad8826b36 100644 --- a/vendor/github.com/hashicorp/golang-lru/go.mod +++ b/vendor/github.com/hashicorp/golang-lru/go.mod @@ -1 +1,3 @@ module github.com/hashicorp/golang-lru + +go 1.12 diff --git a/vendor/github.com/hashicorp/golang-lru/lru.go b/vendor/github.com/hashicorp/golang-lru/lru.go index 1cbe04b7d0..4e5e9d8fd0 100644 --- a/vendor/github.com/hashicorp/golang-lru/lru.go +++ b/vendor/github.com/hashicorp/golang-lru/lru.go @@ -37,7 +37,7 @@ func (c *Cache) Purge() { c.lock.Unlock() } -// Add adds a value to the cache. Returns true if an eviction occurred. +// Add adds a value to the cache. Returns true if an eviction occurred. func (c *Cache) Add(key, value interface{}) (evicted bool) { c.lock.Lock() evicted = c.lru.Add(key, value) @@ -71,8 +71,8 @@ func (c *Cache) Peek(key interface{}) (value interface{}, ok bool) { return value, ok } -// ContainsOrAdd checks if a key is in the cache without updating the -// recent-ness or deleting it for being stale, and if not, adds the value. +// ContainsOrAdd checks if a key is in the cache without updating the +// recent-ness or deleting it for being stale, and if not, adds the value. // Returns whether found and whether an eviction occurred. func (c *Cache) ContainsOrAdd(key, value interface{}) (ok, evicted bool) { c.lock.Lock() @@ -85,18 +85,52 @@ func (c *Cache) ContainsOrAdd(key, value interface{}) (ok, evicted bool) { return false, evicted } -// Remove removes the provided key from the cache. -func (c *Cache) Remove(key interface{}) { +// PeekOrAdd checks if a key is in the cache without updating the +// recent-ness or deleting it for being stale, and if not, adds the value. +// Returns whether found and whether an eviction occurred. +func (c *Cache) PeekOrAdd(key, value interface{}) (previous interface{}, ok, evicted bool) { c.lock.Lock() - c.lru.Remove(key) + defer c.lock.Unlock() + + previous, ok = c.lru.Peek(key) + if ok { + return previous, true, false + } + + evicted = c.lru.Add(key, value) + return nil, false, evicted +} + +// Remove removes the provided key from the cache. +func (c *Cache) Remove(key interface{}) (present bool) { + c.lock.Lock() + present = c.lru.Remove(key) c.lock.Unlock() + return +} + +// Resize changes the cache size. +func (c *Cache) Resize(size int) (evicted int) { + c.lock.Lock() + evicted = c.lru.Resize(size) + c.lock.Unlock() + return evicted } // RemoveOldest removes the oldest item from the cache. -func (c *Cache) RemoveOldest() { +func (c *Cache) RemoveOldest() (key interface{}, value interface{}, ok bool) { c.lock.Lock() - c.lru.RemoveOldest() + key, value, ok = c.lru.RemoveOldest() c.lock.Unlock() + return +} + +// GetOldest returns the oldest entry +func (c *Cache) GetOldest() (key interface{}, value interface{}, ok bool) { + c.lock.Lock() + key, value, ok = c.lru.GetOldest() + c.lock.Unlock() + return } // Keys returns a slice of the keys in the cache, from oldest to newest. diff --git a/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go b/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go index 5673773b22..a86c8539e0 100644 --- a/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go +++ b/vendor/github.com/hashicorp/golang-lru/simplelru/lru.go @@ -73,6 +73,9 @@ func (c *LRU) Add(key, value interface{}) (evicted bool) { func (c *LRU) Get(key interface{}) (value interface{}, ok bool) { if ent, ok := c.items[key]; ok { c.evictList.MoveToFront(ent) + if ent.Value.(*entry) == nil { + return nil, false + } return ent.Value.(*entry).value, true } return @@ -142,6 +145,19 @@ func (c *LRU) Len() int { return c.evictList.Len() } +// Resize changes the cache size. +func (c *LRU) Resize(size int) (evicted int) { + diff := c.Len() - size + if diff < 0 { + diff = 0 + } + for i := 0; i < diff; i++ { + c.removeOldest() + } + c.size = size + return diff +} + // removeOldest removes the oldest item from the cache. func (c *LRU) removeOldest() { ent := c.evictList.Back() diff --git a/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go b/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go index 74c7077440..92d70934d6 100644 --- a/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go +++ b/vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go @@ -10,7 +10,7 @@ type LRUCache interface { // updates the "recently used"-ness of the key. #value, isFound Get(key interface{}) (value interface{}, ok bool) - // Check if a key exsists in cache without updating the recent-ness. + // Checks if a key exists in cache without updating the recent-ness. Contains(key interface{}) (ok bool) // Returns key's value without updating the "recently used"-ness of the key. @@ -31,6 +31,9 @@ type LRUCache interface { // Returns the number of items in the cache. Len() int - // Clear all cache entries + // Clears all cache entries. Purge() + + // Resizes cache, returning number evicted + Resize(int) int } diff --git a/vendor/modules.txt b/vendor/modules.txt index bdab239b8c..0cb8ea4ab1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -208,7 +208,7 @@ github.com/hashicorp/go-discover/provider/vsphere github.com/hashicorp/go-hclog # github.com/hashicorp/go-immutable-radix v1.1.0 github.com/hashicorp/go-immutable-radix -# github.com/hashicorp/go-memdb v1.0.3 +# github.com/hashicorp/go-memdb v1.1.0 github.com/hashicorp/go-memdb # github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/go-msgpack/codec @@ -230,7 +230,7 @@ github.com/hashicorp/go-syslog github.com/hashicorp/go-uuid # github.com/hashicorp/go-version v1.2.0 github.com/hashicorp/go-version -# github.com/hashicorp/golang-lru v0.5.1 +# github.com/hashicorp/golang-lru v0.5.4 github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru # github.com/hashicorp/hcl v1.0.0