gered/vault-sidekick
Archived
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

- fixed up and clean some of the code

Browse source 
- added the scenerio, lease expired but asking to renew
- added the actual renew of lease code, rather than retrieve new lease for autditing purposes
This commit is contained in:
Rohith 2015-09-18 14:12:09 +01:00
parent 477c10e62f
commit be5612f524
6 changed files with 159 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Makefile
View file

@ -12,12 +12,14 @@ build:
mkdir -p build
go build -o build/${NAME}
clean:
rm -rf ./build 2>/dev/null
authors:
git log --format='%aN <%aE>' | sort -u > AUTHORS
test:
go get github.com/stretchr/testify
go get gopkg.in/yaml.v2
go get github.com/stretchr/testify/assert
go test -v
changelog: release

9
README.md
View file

@ -54,7 +54,7 @@ The above say's
- Apply the IAM policy, renew the policy when required and file the API tokens to .s3_creds in the /etc/secrets directory
- Read the template at /etc/templates/db.tmpl, produce the content from Vault and write to /etc/credentials file
**Output Format**
**Output Formatting**
The following output formats are supported: json, yaml, ini, txt
@ -83,3 +83,10 @@ In order to change the output format:
The default format is 'txt' which has the following behavour. If the number of keys in a resource is > 1, a file is created per key. Thus using the example
(build/vault-sidekick -cn=secret:password:fn=test) we would end up with files: test.this, test.nothing and test.demo
**Resource Options**
- **fn**: (filaname) by default all file are relative to the output directory specified and will have the name NAME.RESOURCE; the fn options allows you to switch names and paths to write the files
- **rn**: (renewal) allow you to set the renewal time on a resource, but default we take the lease time from the secret and use that, the rn feature allows use to override it
- **fmt**: (format) allows you to specify the output format of the resource / secret.
- **cn**: (comman name) is used in conjunction with the PKI resource. The common argument is passed as an argument when make a request to issue the certs.

4
main.go
View file

@ -54,9 +54,7 @@ func main() {
if err := rn.isValid(); err != nil {
showUsage("%s", err)
}
if err := vault.watch(rn, ch); err != nil {
showUsage("unable to add watch on resource: %s, %s", rn, err)
}
vault.watch(rn, ch)
}
// step: we simply wait for events i.e. secrets from vault and write them to the output directory

19
tests/ca-csr.json Normal file
View file

@ -0,0 +1,19 @@
{
"CN": "Test CA",
"CA": {
"expiry": "87600h"
},
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "GB",
"L": "London",
"O": "Test CA",
"OU": "Dev Environment",
"ST": "London"
}
]
}

50
tests/ca_bundle.pem Normal file
View file

@ -0,0 +1,50 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEAwpt4MLm+ChHfAmjy35cQ6jpBEKEGy+VLu79cLvCWo3t+VM8s
vWQTh7toZZ0MkDm3RNiefXui3Y/hCmzOHouBL+dPQv8Q+BMH7wihAMEB+qn5oTty
2eHi6srhm/clJx+HHLir2TAUT5n26OA6TUQdLhvtsSc7DquWOFnhX0k0xsNj3QwZ
q6OLvKSlLDoKvkc84L87Kzlf0MOpKa3/3pp2klEYLmviTi5xgVNLm9ViYzVkx+JR
7B/gqw8WWuJY9n1eC4m+C9AOxt6PpUKt2lpqT4olkLW9z40tWJAd/EaCg5S6PJcQ
qZkKb3Dhp8AKAtn4qAJwgiqgV7yPaMR26FOxGQIDAQABAoIBAQC6kCkNLUQOi0ts
GAieaUiCBA7UTkshtVSBTNam+Wawm3dk/qg5eHNwsC1JHOIqcepMSg7G5XwhRAnN
4LlJdxwGPI40ACrYaAY3FeKjwmSPVdxGwaM9VdwgkxbuWmR3uTXiRvgYYotWNJgM
cMnzwNTom0Wni6CGU+DTbPcuThQWZxTudt343ID0xkD9+PEwQrO8YoohgLuQoTzR
tHHc0HlXkH2ZMpEjU+lFcSLA/vIiph0PIQtYSkS/691GMeh7jjjtKkS/rVwSkVfI
q9lyBTu509YOjFDRqgvvnbLK5FoZtQW7zvdnc0VNj4enE3CyhaDGjIgWwP752+3S
BmLYknoxAoGBAMwgvJIKXEMRmDGohDfcoZ8sHe8nIBIiekXW0YaxJIziOq5eNomc
9c5IZQGfmnIuvqXaaKn+Y/J+AYPUWJ10EIiPOKw+BgRwMTh0RsNd49QBqBnKOtuK
UGy8cZFlRpQVoZAoXOzZW3hso5qlqJzO8soxps+EeCnfnnddC2CdBaRTAoGBAPQP
Yn/bUtDWLXb4BkbCNSgZffOc6vqV7vl4LfeqMxSRpp4PL1r7jeR1qlcD7rbDPMX6
1NX3ZHwgb30Z2M08ejaHd9X22k6pe917hLd24NRnu/DO1NqeUCQa7lhE9O0E+ZrD
YkUZ8pP6y8G1bV6K3LiCdl0o4jXRX1aGsa8c4adjAoGAKNPTY5JW6cM3IZeG+nVS
jjeQtSiqLXZf5mAVAE+l89e7zOxjFBskvuGT8kMt7PCUiS+qB3YuH248d1Wdc8Cn
HekneKvfIDwgXB5FmQXKb7j1GlNsekSr2VPHk0EiYLQC4IZyL505wlhYULIZi2OJ
BA/yQUdJkXZ8h3tAr044tqkCgYAQ0FqF2nNLJeY98vpjt4938sGlneLmXpv3Hdt0
24nnWd1zuDIX/4qX+a9BjWjNuIegUBaHoyKOFqH3qWcxRIBa71xHJlmF39FDwfWz
ugHlQDxHa8hoQ03cHuras+13wsb7bYiAoDgBD98nujsNr11jbMGAy4dCE+mQiXkG
SmQVZwKBgQCN8uq5rK/fgnG8EANBZ5zPX4jSmb6/cAEIU9GAxCqacGU88WWTnrhG
PGIE6Kr0Pf6dR1RFn1uU9k+5yT1ZQelxXqhPoPgUU39ML/JejamjjUX+O7zUyNUE
vwytWGU9z1G10zDP1NHOqhBY9V8eKPuWrPSMWgvmccry0Nvge5L+Tw==
-----END RSA PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIIDvjCCAqigAwIBAgIIGdHvCqvNv1IwCwYJKoZIhvcNAQELMG0xCzAJBgNVBAYT
AkdCMRAwDgYDVQQKEwdUZXN0IENBMRgwFgYDVQQLEw9EZXYgRW52aXJvbm1lbnQx
DzANBgNVBAcTBkxvbmRvbjEPMA0GA1UECBMGTG9uZG9uMRAwDgYDVQQDEwdUZXN0
IENBMB4XDTE1MDkxODEyMzAwMFoXDTI1MDkxNTEyMzAwMFowbTELMAkGA1UEBhMC
R0IxEDAOBgNVBAoTB1Rlc3QgQ0ExGDAWBgNVBAsTD0RldiBFbnZpcm9ubWVudDEP
MA0GA1UEBxMGTG9uZG9uMQ8wDQYDVQQIEwZMb25kb24xEDAOBgNVBAMTB1Rlc3Qg
Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDCm3gwub4KEd8CaPLf
lxDqOkEQoQbL5Uu7v1wu8Jaje35Uzyy9ZBOHu2hlnQyQObdE2J59e6Ldj+EKbM4e
i4Ev509C/xD4EwfvCKEAwQH6qfmhO3LZ4eLqyuGb9yUnH4ccuKvZMBRPmfbo4DpN
RB0uG+2xJzsOq5Y4WeFfSTTGw2PdDBmro4u8pKUsOgq+RzzgvzsrOV/Qw6kprf/e
mnaSURgua+JOLnGBU0ub1WJjNWTH4lHsH+CrDxZa4lj2fV4Lib4L0A7G3o+lQq3a
WmpPiiWQtb3PjS1YkB38RoKDlLo8lxCpmQpvcOGnwAoC2fioAnCCKqBXvI9oxHbo
U7EZAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIABjASBgNVHRMBAf8ECDAGAQH/AgEC
MB0GA1UdDgQWBBTN648BAc/3Ay412N+W/fsX4Hp+IjAfBgNVHSMEGDAWgBTN648B
Ac/3Ay412N+W/fsX4Hp+IjALBgkqhkiG9w0BAQsDggEBADXCD3BFEcqywDHEKC7r
81fNTTV+pDf+992NkNw//jeYWlof6DQymJhn8HHnj7bQj4P68tFsj1hQWoFbOeD6
jWpubhO5VezncWCMNo/tHVENbbzexAHNtI9/Vgbkkc+/BNgedeldMf4L5a1SvdfI
mTvkI3/HuCRRgF29dBamN1LE3jKzX0oIdrota6konsrOsJkfzjiRQM8z0n4TNDMG
JfqqW6I+EDhqAnIdRfLoe8xiTRghaONnCx+rPIfm7NM7flbcHE3J0jyKBt6aK+vV
6Zumbks1EnYkqlZUUo2nUz+3mBNmTQA5ua7w5vKXZOCcwfVkWiT7/ifZ1JsHJNP0
Jaw=
-----END CERTIFICATE-----

112
vault.go
View file

@ -53,17 +53,29 @@ type vaultEventsChannel chan vaultResourceEvent
// watchedResource ... is a resource which is being watched - i.e. when the item is coming up for renewal
// lets grab it and renew the lease
type watchedResource struct {
// the upstream listener to the event
listener vaultEventsChannel
// the resource itself
resource *vaultResource
// the last time the resource was retrieved
lastUpdated time.Time
// the duration until the next renewal
// the time which the lease expires
leaseExpireTime time.Time
// the duration until we next time to renew lease
renewalTime time.Duration
// the secret
secret *api.Secret
}
// updateSecret ... sets the secret for the watched resource and updates the various counters / timers
func (r *watchedResource) updateSecret(secret *api.Secret) {
r.secret = secret
r.lastUpdated = time.Now()
r.leaseExpireTime = r.lastUpdated.Add(time.Duration(secret.LeaseDuration))
glog.V(10).Infof("updating secret on resource: %s, leaseId: %s, lease: %s, expiration: %s",
r.resource, r.secret.LeaseID, r.secret.LeaseID, r.leaseExpireTime)
}
// notifyOnRenewal ... creates a trigger and notifies when a resource is up for renewal
func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) {
go func() {
@ -72,12 +84,12 @@ func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) {
// step: if the answer is no, we set the notification between 80-95% of the lease time of the secret
if r.renewalTime <= 0 {
glog.V(10).Infof("Calculating the renewal between 80-95 pcent of lease time: %d seconds", r.secret.LeaseDuration)
glog.V(10).Infof("calculating the renewal between 80-95 pcent of lease time: %d seconds", r.secret.LeaseDuration)
r.renewalTime = time.Duration(getRandomWithin(
int(float64(r.secret.LeaseDuration) * 0.8),
int(float64(r.secret.LeaseDuration) * 0.95))) * time.Second
}
glog.V(3).Infof("Setting a renewal notification on resource: %s, time: %s", r.resource, r.renewalTime)
glog.V(3).Infof("setting a renewal notification on resource: %s, time: %s", r.resource, r.renewalTime)
// step: wait for the duration
<- time.After(r.renewalTime)
// step: send the notification on the renewal channel
@ -90,7 +102,7 @@ func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) {
// token : the token to use when speaking to vault
func newVaultService(url, token string) (*vaultService, error) {
var err error
glog.Infof("Creating a new vault client: %s", url)
glog.Infof("creating a new vault client: %s", url)
// step: create the config for client
service := new(vaultService)
@ -132,12 +144,12 @@ func (r vaultService) vaultServiceProcessor() {
// - if we error attempting to retrieve the secret, we background and reschedule an attempt to add it
// - if ok, we grab the lease it and lease time, we setup a notification on renewal
case x := <-r.resourceCh:
glog.V(3).Infof("Adding a resource into the service processor, resource: %s", x.resource)
glog.V(3).Infof("adding a resource into the service processor, resource: %s", x.resource)
// step: retrieve the resource from vault
secret, err := r.get(x.resource)
err := r.get(x)
if err != nil {
glog.Errorf("Failed to retrieve the resource: %s from vault, error: %s", x.resource, err)
glog.Errorf("failed to retrieve the resource: %s from vault, error: %s", x.resource, err)
// reschedule the attempt for later
go func(x *watchedResource) {
<- time.After(time.Duration(getRandomWithin(2,10)) * time.Second)
@ -145,9 +157,6 @@ func (r vaultService) vaultServiceProcessor() {
}(x)
break
}
// step: update the item references
x.secret = secret
x.lastUpdated = time.Now()
// step: setup a timer for renewal
x.notifyOnRenewal(renewing)
@ -155,20 +164,28 @@ func (r vaultService) vaultServiceProcessor() {
// step: add to the list of resources
items = append(items, x)
r.upstream(x, secret)
// step: update the upstream consumers
r.upstream(x, x.secret)
// A watched resource is coming up for renewal
// - we attempt to grab the resource from vault
// - if we encounter an error, we reschedule the attempt for the future
// - if we're ok, we update the watchedResource and we send a notification of the change upstream
case x := <-renewing:
glog.V(3).Infof("Resource: %s coming up for renewal, attempting to renew now", x.resource)
glog.V(3).Infof("resource: %s, lease: %s coming up for renewal, attempting to renew now", x.resource, x.secret.LeaseID)
// step: we attempt to renew the lease on a resource and if not successfully we reschedule
// a renewal notification for the future
// - we also have to handle the scenario where the lease has expired
secret, err := r.get(x.resource)
// step: we need to check if the lease has expired?
if time.Now().Before(x.leaseExpireTime) {
glog.V(3).Infof("the lease on resource: %s has expired, we need to get a new lease", x.resource)
x.secret.Renewable = false
}
err := r.renew(x)
if err != nil {
glog.Errorf("Failed to retrieve the resounce: %s for renewal, error: %s", x.resource, err)
glog.Errorf("failed to renew the resounce: %s for renewal, error: %s", x.resource, err)
// reschedule the attempt for later
go func(x *watchedResource) {
<- time.After(time.Duration(getRandomWithin(3,20)) * time.Second)
@ -177,22 +194,18 @@ func (r vaultService) vaultServiceProcessor() {
break
}
// step: update the item references
x.secret = secret
x.lastUpdated = time.Now()
// step: setup a timer for renewal
x.notifyOnRenewal(renewing)
// step: update any listener upstream
r.upstream(x, secret)
r.upstream(x, x.secret)
// The statistics timer has gone off; we iterate the watched items and
case <-r.statCh.C:
glog.V(3).Infof("Stats: %d resources being watched", len(items))
glog.V(3).Infof("stats: %d resources being watched", len(items))
for _, item := range items {
glog.V(3).Infof("resourse: %s, lease id: %s, renewal in: %s seconds",
item.resource, item.secret.LeaseID, item.renewalTime)
glog.V(3).Infof("resourse: %s, lease id: %s, renewal in: %s seconds, expiration: %s",
item.resource, item.secret.LeaseID, item.renewalTime, item.leaseExpireTime)
}
}
}
@ -202,7 +215,7 @@ func (r vaultService) vaultServiceProcessor() {
func (r vaultService) upstream(item *watchedResource, s *api.Secret) {
// step: chunk this into a go-routine not to block us
go func() {
glog.V(6).Infof("Sending the event for resource: %s upstream to listener: %v", item.resource, item.listener)
glog.V(6).Infof("sending the event for resource: %s upstream to listener: %v", item.resource, item.listener)
item.listener <- vaultResourceEvent{
resource: item.resource,
secret: s.Data,
@ -210,40 +223,65 @@ func (r vaultService) upstream(item *watchedResource, s *api.Secret) {
}()
}
// renew ... attempts to renew the lease on a resource
// rn : the resource we wish to renew the lease on
func (r vaultService) renew(rn *watchedResource) error {
// step: can this secret be renewed - otherwise we can just grab a new lease
if !rn.secret.Renewable {
glog.V(4).Infof("the resource: %s is not renewable, retrieving a new lease instead", rn.resource)
return r.get(rn)
}
// step: extend the lease on a resource
glog.V(3).Infof("attempting to renew the lease: %s on resource: %s", rn.secret.LeaseID, rn.resource)
secret, err := r.client.Sys().Renew(rn.secret.LeaseID, 0)
if err != nil {
glog.V(4).Infof("unable to renew the lease on resource: %s", rn.resource)
return err
}
// step: update the secret
rn.updateSecret(secret)
return nil
}
// get ... retrieve a secret from the vault
func (r vaultService) get(rn *vaultResource) (*api.Secret, error) {
var err error
func (r vaultService) get(rn *watchedResource) (err error) {
var secret *api.Secret
glog.V(5).Infof("Attempting to retrieve the resource: %s from vault", rn)
switch rn.resource {
glog.V(5).Infof("attempting to retrieve the resource: %s from vault", rn.resource)
switch rn.resource.resource {
case "pki":
secret, err = r.client.Logical().Write(fmt.Sprintf("%s/issue/%s", rn.resource, rn.name),
secret, err = r.client.Logical().Write(fmt.Sprintf("%s/issue/%s", rn.resource.resource, rn.resource.name),
map[string]interface{}{
"common_name": rn.options[OptionCommonName],
"common_name": rn.resource.options[OptionCommonName],
})
case "aws":
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name))
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource.resource, rn.resource.name))
case "mysql":
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name))
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource.resource, rn.resource.name))
case "secret":
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/%s", rn.resource, rn.name))
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/%s", rn.resource.resource, rn.resource.name))
}
if secret == nil && err == nil {
return nil, fmt.Errorf("does not exist")
return fmt.Errorf("does not exist")
}
return secret, err
// step: update the watched resource
rn.updateSecret(secret)
return err
}
// watch ... add a watch on a resource and inform, renew which required and inform us when
// the resource is ready
func (r *vaultService) watch(rn *vaultResource, ch vaultEventsChannel) error {
glog.V(10).Infof("Adding the resource: %s, listener: %v to service processor", rn, ch)
func (r *vaultService) watch(rn *vaultResource, ch vaultEventsChannel) {
glog.V(10).Infof("adding the resource: %s, listener: %v to service processor", rn, ch)
r.resourceCh <- &watchedResource{
resource: rn,
listener: ch,
}
return nil
}