diff --git a/Makefile b/Makefile index 2907c17..05799b8 100644 --- a/Makefile +++ b/Makefile @@ -12,13 +12,15 @@ build: mkdir -p build go build -o build/${NAME} +clean: + rm -rf ./build 2>/dev/null + authors: git log --format='%aN <%aE>' | sort -u > AUTHORS test: - go get github.com/stretchr/testify - go get gopkg.in/yaml.v2 + go get github.com/stretchr/testify/assert go test -v changelog: release - git log $(shell git tag | tail -n1)..HEAD --no-merges --format=%B > changelog + git log $(shell git tag | tail -n1)..HEAD --no-merges --format=%B > changelog \ No newline at end of file diff --git a/README.md b/README.md index 3dc2f2f..72cfb35 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ The above say's - Apply the IAM policy, renew the policy when required and file the API tokens to .s3_creds in the /etc/secrets directory - Read the template at /etc/templates/db.tmpl, produce the content from Vault and write to /etc/credentials file -**Output Format** +**Output Formatting** The following output formats are supported: json, yaml, ini, txt @@ -82,4 +82,11 @@ In order to change the output format: ``` The default format is 'txt' which has the following behavour. If the number of keys in a resource is > 1, a file is created per key. Thus using the example -(build/vault-sidekick -cn=secret:password:fn=test) we would end up with files: test.this, test.nothing and test.demo \ No newline at end of file +(build/vault-sidekick -cn=secret:password:fn=test) we would end up with files: test.this, test.nothing and test.demo + +**Resource Options** + +- **fn**: (filaname) by default all file are relative to the output directory specified and will have the name NAME.RESOURCE; the fn options allows you to switch names and paths to write the files +- **rn**: (renewal) allow you to set the renewal time on a resource, but default we take the lease time from the secret and use that, the rn feature allows use to override it +- **fmt**: (format) allows you to specify the output format of the resource / secret. +- **cn**: (comman name) is used in conjunction with the PKI resource. The common argument is passed as an argument when make a request to issue the certs. \ No newline at end of file diff --git a/main.go b/main.go index 07104aa..83a528c 100644 --- a/main.go +++ b/main.go @@ -54,9 +54,7 @@ func main() { if err := rn.isValid(); err != nil { showUsage("%s", err) } - if err := vault.watch(rn, ch); err != nil { - showUsage("unable to add watch on resource: %s, %s", rn, err) - } + vault.watch(rn, ch) } // step: we simply wait for events i.e. secrets from vault and write them to the output directory diff --git a/tests/ca-csr.json b/tests/ca-csr.json new file mode 100644 index 0000000..3f076c4 --- /dev/null +++ b/tests/ca-csr.json @@ -0,0 +1,19 @@ +{ + "CN": "Test CA", + "CA": { + "expiry": "87600h" + }, + "key": { + "algo": "rsa", + "size": 2048 + }, + "names": [ + { + "C": "GB", + "L": "London", + "O": "Test CA", + "OU": "Dev Environment", + "ST": "London" + } + ] +} diff --git a/tests/ca_bundle.pem b/tests/ca_bundle.pem new file mode 100644 index 0000000..ac43b2c --- /dev/null +++ b/tests/ca_bundle.pem @@ -0,0 +1,50 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAwpt4MLm+ChHfAmjy35cQ6jpBEKEGy+VLu79cLvCWo3t+VM8s +vWQTh7toZZ0MkDm3RNiefXui3Y/hCmzOHouBL+dPQv8Q+BMH7wihAMEB+qn5oTty +2eHi6srhm/clJx+HHLir2TAUT5n26OA6TUQdLhvtsSc7DquWOFnhX0k0xsNj3QwZ +q6OLvKSlLDoKvkc84L87Kzlf0MOpKa3/3pp2klEYLmviTi5xgVNLm9ViYzVkx+JR +7B/gqw8WWuJY9n1eC4m+C9AOxt6PpUKt2lpqT4olkLW9z40tWJAd/EaCg5S6PJcQ +qZkKb3Dhp8AKAtn4qAJwgiqgV7yPaMR26FOxGQIDAQABAoIBAQC6kCkNLUQOi0ts +GAieaUiCBA7UTkshtVSBTNam+Wawm3dk/qg5eHNwsC1JHOIqcepMSg7G5XwhRAnN +4LlJdxwGPI40ACrYaAY3FeKjwmSPVdxGwaM9VdwgkxbuWmR3uTXiRvgYYotWNJgM +cMnzwNTom0Wni6CGU+DTbPcuThQWZxTudt343ID0xkD9+PEwQrO8YoohgLuQoTzR +tHHc0HlXkH2ZMpEjU+lFcSLA/vIiph0PIQtYSkS/691GMeh7jjjtKkS/rVwSkVfI +q9lyBTu509YOjFDRqgvvnbLK5FoZtQW7zvdnc0VNj4enE3CyhaDGjIgWwP752+3S +BmLYknoxAoGBAMwgvJIKXEMRmDGohDfcoZ8sHe8nIBIiekXW0YaxJIziOq5eNomc +9c5IZQGfmnIuvqXaaKn+Y/J+AYPUWJ10EIiPOKw+BgRwMTh0RsNd49QBqBnKOtuK +UGy8cZFlRpQVoZAoXOzZW3hso5qlqJzO8soxps+EeCnfnnddC2CdBaRTAoGBAPQP +Yn/bUtDWLXb4BkbCNSgZffOc6vqV7vl4LfeqMxSRpp4PL1r7jeR1qlcD7rbDPMX6 +1NX3ZHwgb30Z2M08ejaHd9X22k6pe917hLd24NRnu/DO1NqeUCQa7lhE9O0E+ZrD +YkUZ8pP6y8G1bV6K3LiCdl0o4jXRX1aGsa8c4adjAoGAKNPTY5JW6cM3IZeG+nVS +jjeQtSiqLXZf5mAVAE+l89e7zOxjFBskvuGT8kMt7PCUiS+qB3YuH248d1Wdc8Cn +HekneKvfIDwgXB5FmQXKb7j1GlNsekSr2VPHk0EiYLQC4IZyL505wlhYULIZi2OJ +BA/yQUdJkXZ8h3tAr044tqkCgYAQ0FqF2nNLJeY98vpjt4938sGlneLmXpv3Hdt0 +24nnWd1zuDIX/4qX+a9BjWjNuIegUBaHoyKOFqH3qWcxRIBa71xHJlmF39FDwfWz +ugHlQDxHa8hoQ03cHuras+13wsb7bYiAoDgBD98nujsNr11jbMGAy4dCE+mQiXkG +SmQVZwKBgQCN8uq5rK/fgnG8EANBZ5zPX4jSmb6/cAEIU9GAxCqacGU88WWTnrhG +PGIE6Kr0Pf6dR1RFn1uU9k+5yT1ZQelxXqhPoPgUU39ML/JejamjjUX+O7zUyNUE +vwytWGU9z1G10zDP1NHOqhBY9V8eKPuWrPSMWgvmccry0Nvge5L+Tw== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIDvjCCAqigAwIBAgIIGdHvCqvNv1IwCwYJKoZIhvcNAQELMG0xCzAJBgNVBAYT +AkdCMRAwDgYDVQQKEwdUZXN0IENBMRgwFgYDVQQLEw9EZXYgRW52aXJvbm1lbnQx +DzANBgNVBAcTBkxvbmRvbjEPMA0GA1UECBMGTG9uZG9uMRAwDgYDVQQDEwdUZXN0 +IENBMB4XDTE1MDkxODEyMzAwMFoXDTI1MDkxNTEyMzAwMFowbTELMAkGA1UEBhMC +R0IxEDAOBgNVBAoTB1Rlc3QgQ0ExGDAWBgNVBAsTD0RldiBFbnZpcm9ubWVudDEP +MA0GA1UEBxMGTG9uZG9uMQ8wDQYDVQQIEwZMb25kb24xEDAOBgNVBAMTB1Rlc3Qg +Q0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDCm3gwub4KEd8CaPLf +lxDqOkEQoQbL5Uu7v1wu8Jaje35Uzyy9ZBOHu2hlnQyQObdE2J59e6Ldj+EKbM4e +i4Ev509C/xD4EwfvCKEAwQH6qfmhO3LZ4eLqyuGb9yUnH4ccuKvZMBRPmfbo4DpN +RB0uG+2xJzsOq5Y4WeFfSTTGw2PdDBmro4u8pKUsOgq+RzzgvzsrOV/Qw6kprf/e +mnaSURgua+JOLnGBU0ub1WJjNWTH4lHsH+CrDxZa4lj2fV4Lib4L0A7G3o+lQq3a +WmpPiiWQtb3PjS1YkB38RoKDlLo8lxCpmQpvcOGnwAoC2fioAnCCKqBXvI9oxHbo +U7EZAgMBAAGjZjBkMA4GA1UdDwEB/wQEAwIABjASBgNVHRMBAf8ECDAGAQH/AgEC +MB0GA1UdDgQWBBTN648BAc/3Ay412N+W/fsX4Hp+IjAfBgNVHSMEGDAWgBTN648B +Ac/3Ay412N+W/fsX4Hp+IjALBgkqhkiG9w0BAQsDggEBADXCD3BFEcqywDHEKC7r +81fNTTV+pDf+992NkNw//jeYWlof6DQymJhn8HHnj7bQj4P68tFsj1hQWoFbOeD6 +jWpubhO5VezncWCMNo/tHVENbbzexAHNtI9/Vgbkkc+/BNgedeldMf4L5a1SvdfI +mTvkI3/HuCRRgF29dBamN1LE3jKzX0oIdrota6konsrOsJkfzjiRQM8z0n4TNDMG +JfqqW6I+EDhqAnIdRfLoe8xiTRghaONnCx+rPIfm7NM7flbcHE3J0jyKBt6aK+vV +6Zumbks1EnYkqlZUUo2nUz+3mBNmTQA5ua7w5vKXZOCcwfVkWiT7/ifZ1JsHJNP0 +Jaw= +-----END CERTIFICATE----- diff --git a/vault.go b/vault.go index 5a4314d..f4ae53e 100644 --- a/vault.go +++ b/vault.go @@ -53,17 +53,29 @@ type vaultEventsChannel chan vaultResourceEvent // watchedResource ... is a resource which is being watched - i.e. when the item is coming up for renewal // lets grab it and renew the lease type watchedResource struct { + // the upstream listener to the event listener vaultEventsChannel // the resource itself resource *vaultResource // the last time the resource was retrieved lastUpdated time.Time - // the duration until the next renewal + // the time which the lease expires + leaseExpireTime time.Time + // the duration until we next time to renew lease renewalTime time.Duration // the secret secret *api.Secret } +// updateSecret ... sets the secret for the watched resource and updates the various counters / timers +func (r *watchedResource) updateSecret(secret *api.Secret) { + r.secret = secret + r.lastUpdated = time.Now() + r.leaseExpireTime = r.lastUpdated.Add(time.Duration(secret.LeaseDuration)) + glog.V(10).Infof("updating secret on resource: %s, leaseId: %s, lease: %s, expiration: %s", + r.resource, r.secret.LeaseID, r.secret.LeaseID, r.leaseExpireTime) +} + // notifyOnRenewal ... creates a trigger and notifies when a resource is up for renewal func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) { go func() { @@ -72,12 +84,12 @@ func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) { // step: if the answer is no, we set the notification between 80-95% of the lease time of the secret if r.renewalTime <= 0 { - glog.V(10).Infof("Calculating the renewal between 80-95 pcent of lease time: %d seconds", r.secret.LeaseDuration) + glog.V(10).Infof("calculating the renewal between 80-95 pcent of lease time: %d seconds", r.secret.LeaseDuration) r.renewalTime = time.Duration(getRandomWithin( int(float64(r.secret.LeaseDuration) * 0.8), int(float64(r.secret.LeaseDuration) * 0.95))) * time.Second } - glog.V(3).Infof("Setting a renewal notification on resource: %s, time: %s", r.resource, r.renewalTime) + glog.V(3).Infof("setting a renewal notification on resource: %s, time: %s", r.resource, r.renewalTime) // step: wait for the duration <- time.After(r.renewalTime) // step: send the notification on the renewal channel @@ -90,7 +102,7 @@ func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) { // token : the token to use when speaking to vault func newVaultService(url, token string) (*vaultService, error) { var err error - glog.Infof("Creating a new vault client: %s", url) + glog.Infof("creating a new vault client: %s", url) // step: create the config for client service := new(vaultService) @@ -132,12 +144,12 @@ func (r vaultService) vaultServiceProcessor() { // - if we error attempting to retrieve the secret, we background and reschedule an attempt to add it // - if ok, we grab the lease it and lease time, we setup a notification on renewal case x := <-r.resourceCh: - glog.V(3).Infof("Adding a resource into the service processor, resource: %s", x.resource) + glog.V(3).Infof("adding a resource into the service processor, resource: %s", x.resource) // step: retrieve the resource from vault - secret, err := r.get(x.resource) + err := r.get(x) if err != nil { - glog.Errorf("Failed to retrieve the resource: %s from vault, error: %s", x.resource, err) + glog.Errorf("failed to retrieve the resource: %s from vault, error: %s", x.resource, err) // reschedule the attempt for later go func(x *watchedResource) { <- time.After(time.Duration(getRandomWithin(2,10)) * time.Second) @@ -145,9 +157,6 @@ func (r vaultService) vaultServiceProcessor() { }(x) break } - // step: update the item references - x.secret = secret - x.lastUpdated = time.Now() // step: setup a timer for renewal x.notifyOnRenewal(renewing) @@ -155,20 +164,28 @@ func (r vaultService) vaultServiceProcessor() { // step: add to the list of resources items = append(items, x) - r.upstream(x, secret) + // step: update the upstream consumers + r.upstream(x, x.secret) // A watched resource is coming up for renewal // - we attempt to grab the resource from vault // - if we encounter an error, we reschedule the attempt for the future // - if we're ok, we update the watchedResource and we send a notification of the change upstream case x := <-renewing: - glog.V(3).Infof("Resource: %s coming up for renewal, attempting to renew now", x.resource) + glog.V(3).Infof("resource: %s, lease: %s coming up for renewal, attempting to renew now", x.resource, x.secret.LeaseID) // step: we attempt to renew the lease on a resource and if not successfully we reschedule // a renewal notification for the future + // - we also have to handle the scenario where the lease has expired - secret, err := r.get(x.resource) + // step: we need to check if the lease has expired? + if time.Now().Before(x.leaseExpireTime) { + glog.V(3).Infof("the lease on resource: %s has expired, we need to get a new lease", x.resource) + x.secret.Renewable = false + } + + err := r.renew(x) if err != nil { - glog.Errorf("Failed to retrieve the resounce: %s for renewal, error: %s", x.resource, err) + glog.Errorf("failed to renew the resounce: %s for renewal, error: %s", x.resource, err) // reschedule the attempt for later go func(x *watchedResource) { <- time.After(time.Duration(getRandomWithin(3,20)) * time.Second) @@ -177,22 +194,18 @@ func (r vaultService) vaultServiceProcessor() { break } - // step: update the item references - x.secret = secret - x.lastUpdated = time.Now() - // step: setup a timer for renewal x.notifyOnRenewal(renewing) // step: update any listener upstream - r.upstream(x, secret) + r.upstream(x, x.secret) // The statistics timer has gone off; we iterate the watched items and case <-r.statCh.C: - glog.V(3).Infof("Stats: %d resources being watched", len(items)) + glog.V(3).Infof("stats: %d resources being watched", len(items)) for _, item := range items { - glog.V(3).Infof("resourse: %s, lease id: %s, renewal in: %s seconds", - item.resource, item.secret.LeaseID, item.renewalTime) + glog.V(3).Infof("resourse: %s, lease id: %s, renewal in: %s seconds, expiration: %s", + item.resource, item.secret.LeaseID, item.renewalTime, item.leaseExpireTime) } } } @@ -202,7 +215,7 @@ func (r vaultService) vaultServiceProcessor() { func (r vaultService) upstream(item *watchedResource, s *api.Secret) { // step: chunk this into a go-routine not to block us go func() { - glog.V(6).Infof("Sending the event for resource: %s upstream to listener: %v", item.resource, item.listener) + glog.V(6).Infof("sending the event for resource: %s upstream to listener: %v", item.resource, item.listener) item.listener <- vaultResourceEvent{ resource: item.resource, secret: s.Data, @@ -210,40 +223,65 @@ func (r vaultService) upstream(item *watchedResource, s *api.Secret) { }() } +// renew ... attempts to renew the lease on a resource +// rn : the resource we wish to renew the lease on +func (r vaultService) renew(rn *watchedResource) error { + + // step: can this secret be renewed - otherwise we can just grab a new lease + if !rn.secret.Renewable { + glog.V(4).Infof("the resource: %s is not renewable, retrieving a new lease instead", rn.resource) + return r.get(rn) + } + + // step: extend the lease on a resource + glog.V(3).Infof("attempting to renew the lease: %s on resource: %s", rn.secret.LeaseID, rn.resource) + secret, err := r.client.Sys().Renew(rn.secret.LeaseID, 0) + if err != nil { + glog.V(4).Infof("unable to renew the lease on resource: %s", rn.resource) + return err + } + + // step: update the secret + rn.updateSecret(secret) + + return nil +} + // get ... retrieve a secret from the vault -func (r vaultService) get(rn *vaultResource) (*api.Secret, error) { - var err error +func (r vaultService) get(rn *watchedResource) (err error) { var secret *api.Secret - glog.V(5).Infof("Attempting to retrieve the resource: %s from vault", rn) - switch rn.resource { + glog.V(5).Infof("attempting to retrieve the resource: %s from vault", rn.resource) + switch rn.resource.resource { case "pki": - secret, err = r.client.Logical().Write(fmt.Sprintf("%s/issue/%s", rn.resource, rn.name), + secret, err = r.client.Logical().Write(fmt.Sprintf("%s/issue/%s", rn.resource.resource, rn.resource.name), map[string]interface{}{ - "common_name": rn.options[OptionCommonName], + "common_name": rn.resource.options[OptionCommonName], }) case "aws": - secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name)) + secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource.resource, rn.resource.name)) case "mysql": - secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name)) + secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource.resource, rn.resource.name)) case "secret": - secret, err = r.client.Logical().Read(fmt.Sprintf("%s/%s", rn.resource, rn.name)) + secret, err = r.client.Logical().Read(fmt.Sprintf("%s/%s", rn.resource.resource, rn.resource.name)) } if secret == nil && err == nil { - return nil, fmt.Errorf("does not exist") + return fmt.Errorf("does not exist") } - return secret, err + // step: update the watched resource + rn.updateSecret(secret) + + return err } // watch ... add a watch on a resource and inform, renew which required and inform us when // the resource is ready -func (r *vaultService) watch(rn *vaultResource, ch vaultEventsChannel) error { - glog.V(10).Infof("Adding the resource: %s, listener: %v to service processor", rn, ch) +func (r *vaultService) watch(rn *vaultResource, ch vaultEventsChannel) { + glog.V(10).Infof("adding the resource: %s, listener: %v to service processor", rn, ch) r.resourceCh <- &watchedResource{ resource: rn, listener: ch, } - return nil }