diff --git a/.gitignore b/.gitignore index daf913b..1c0f3ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,9 @@ + +*.iml +*.swp +.idea/ +build/ + # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..064b12e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ + +language: go +go: + - 1.4 + - 1.5 +install: + - make test diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..aeeb078 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,4 @@ +FROM gliderlabs/alpine:latest +MAINTAINER Rohith + +ENTRYPOINT [ "/vault-sidekick" ] diff --git a/LICENCE b/LICENCE new file mode 100644 index 0000000..5e0fd33 --- /dev/null +++ b/LICENCE @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "{}" +replaced with your own identifying information. (Don't include +the brackets!) The text should be enclosed in the appropriate +comment syntax for the file format. We also recommend that a +file or class name and description of purpose be included on the +same "printed page" as the copyright notice for easier +identification within third-party archives. + +Copyright {yyyy} {name of copyright owner} + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2907c17 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ + +NAME=vault-sidekick +AUTHOR=gambol99 +HARDWARE=$(shell uname -m) +VERSION=$(shell awk '/const Version/ { print $$4 }' version.go | sed 's/"//g') + +.PHONY: test examples authors changelog build + +default: build + +build: + mkdir -p build + go build -o build/${NAME} + +authors: + git log --format='%aN <%aE>' | sort -u > AUTHORS + +test: + go get github.com/stretchr/testify + go get gopkg.in/yaml.v2 + go test -v + +changelog: release + git log $(shell git tag | tail -n1)..HEAD --no-merges --format=%B > changelog diff --git a/README.md b/README.md new file mode 100644 index 0000000..3dc2f2f --- /dev/null +++ b/README.md @@ -0,0 +1,85 @@ + +### **Vault Side Kick** +----- +**Summary:** +> Vault Sidekick is a add-on container which can be used as a generic entry-point for interacting with Hashicorp [Vault](https://vaultproject.io) service, retrieving secrets +(both static and dynamic) and PKI certs. The sidekick will take care of renewal's and extension of leases for you and renew the credentials in the specified format for you. + +**Usage:** + +```shell +[jest@starfury vault-sidekick]$ build/vault-sidekick -help +Usage of build/vault-sidekick: + -alsologtostderr=false: log to standard error as well as files + -cn=: a resource to retrieve and monitor from vault (e.g. pki:name:cert.name, secret:db_password, aws:s3_backup) + -log_backtrace_at=:0: when logging hits line file:N, emit a stack trace + -log_dir="": If non-empty, write log files in this directory + -logtostderr=false: log to standard error instead of files + -output="/etc/secrets": the full path to write the protected resources (VAULT_OUTPUT if available) + -renew=true: whether or not to renew secrets from vault + -stderrthreshold=0: logs at or above this threshold go to stderr + -token="": the token used to authenticate to teh vault service (VAULT_TOKEN if available) + -tokenfile="": the full path to file containing the vault token used for authentication (VAULT_TOKEN_FILE if available) + -v=0: log level for V logs + -vault="https://127.0.0.1:8200": the url the vault service is running behind (VAULT_ADDR if available) + -vmodule=: comma-separated list of pattern=N settings for file-filtered logging +``` + +**Example Usage** + +The below is taken from a [Kubernetes](https://github.com/kubernetes/kubernetes) pod specification; + +```YAML +spec: + containers: + - name: vault-side-kick + image: gambol99/vault-sidekick:latest + args: + - -output=/etc/secrets + - -rn=pki:example.com:cn=commons.example.com,exec=/usr/bin/nginx_restart.sh,ctr=.*nginx_server.* + - -rn=secret:db/prod/username:fn=.credentials + - -rn=secret:db/prod/password + - -rn=aws:s3_backsup:fn=.s3_creds + - -rb=template:database_credentials:tpl=/etc/templates/db.tmpl,fn=/etc/credentials + volumeMounts: + - name: secrets + mountPath: /etc/secrets +``` + +The above say's + + - Write all the secrets to the /etc/secrets directory + - Retrieve a dynamic certificate pair for me, with the common name: 'commons.example.com' and renew the cert when it expires automatically + - Retrieve the two static secrets /db/prod/{username,password} and write them to .credentials and password.secret respectively + - Apply the IAM policy, renew the policy when required and file the API tokens to .s3_creds in the /etc/secrets directory + - Read the template at /etc/templates/db.tmpl, produce the content from Vault and write to /etc/credentials file + +**Output Format** + +The following output formats are supported: json, yaml, ini, txt + +Using the following at the demo secrets + +```shell +[jest@starfury vault-sidekick]$ vault write secret/password this=is demo=value nothing=more +Success! Data written to: secret/password +[jest@starfury vault-sidekick]$ vault read secret/password +Key Value +lease_id secret/password/7908eceb-9bde-e7de-23da-96131505214a +lease_duration 2592000 +lease_renewable false +demo value +nothing more +this is +``` + +In order to change the output format: + +```shell +[jest@starfury vault-sidekick]$ build/vault-sidekick -cn=secret:password:fmt=ini -logtostderr=true -dry-run +[jest@starfury vault-sidekick]$ build/vault-sidekick -cn=secret:password:fmt=json -logtostderr=true -dry-run +[jest@starfury vault-sidekick]$ build/vault-sidekick -cn=secret:password:fmt=yaml -logtostderr=true -dry-run +``` + +The default format is 'txt' which has the following behavour. If the number of keys in a resource is > 1, a file is created per key. Thus using the example +(build/vault-sidekick -cn=secret:password:fn=test) we would end up with files: test.this, test.nothing and test.demo \ No newline at end of file diff --git a/config.go b/config.go new file mode 100644 index 0000000..a2b91c2 --- /dev/null +++ b/config.go @@ -0,0 +1,114 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "net/url" + "github.com/golang/glog" + "time" +) + +// config ... the command line configuration +type config struct { + // the url for th vault server + vaultURL string + // the token to connect to vault with + vaultToken string + // a file / path containing a token for vault + vaultTokenFile string + // the place to write the resources + secretsDirectory string + // whether or not to renew the leases on our resources + renewResources bool + // whether of not to remove the token post connection + deleteToken bool + // switch on dry run + dryRun bool + // the resource items to retrieve + resources *vaultResources + // the interval for producing statistics + statsInterval time.Duration +} + +var ( + options config +) + +func init() { + options.resources = new(vaultResources) + flag.StringVar(&options.vaultURL, "vault", getEnv("VAULT_ADDR", "https://127.0.0.1:8200"), "the url the vault service is running behind (VAULT_ADDR if available)") + flag.StringVar(&options.vaultToken, "token", "", "the token used to authenticate to teh vault service (VAULT_TOKEN if available)") + flag.StringVar(&options.vaultTokenFile, "tokenfile", getEnv("VAULT_TOKEN_FILE", ""), "the full path to file containing the vault token used for authentication (VAULT_TOKEN_FILE if available)") + flag.StringVar(&options.secretsDirectory, "output", getEnv("VAULT_OUTPUT", "/etc/secrets"), "the full path to write the protected resources (VAULT_OUTPUT if available)") + flag.BoolVar(&options.deleteToken, "delete-token", false, "once the we have connected to vault, delete the token file from disk") + flag.BoolVar(&options.renewResources, "renew", true, "whether or not to renew secrets from vault") + flag.BoolVar(&options.dryRun, "dry-run", false, "perform a dry run, printing the content to screen") + flag.DurationVar(&options.statsInterval, "stats", time.Duration(5) * time.Minute, "the interval to produce statistics on the accessed resources") + flag.Var(options.resources, "cn", "a resource to retrieve and monitor from vault (e.g. pki:name:cert.name, secret:db_password, aws:s3_backup)") +} + +// parseOptions ... validate the command line options and validates them +func parseOptions() error { + flag.Parse() + + return validateOptions(&options) +} + +// validateOptions ... parses and validates the command line options +func validateOptions(cfg *config) error { + // step: validate the vault url + url, err := url.Parse(cfg.vaultURL) + if err != nil { + return fmt.Errorf("invalid vault url: '%s' specified", cfg.vaultURL) + } + // step: ensure the protocol scheme + if url.Scheme != "https" { + glog.Warningf("protocol scheme: %s is not secure and should be https", url.Scheme) + } + + // step: check if the token is in the VAULT_TOKEN var + if cfg.vaultToken == "" { + cfg.vaultToken = getEnv("VAULT_TOKEN", "") + } + + // step: ensure we have a token + if cfg.vaultToken == "" && cfg.vaultTokenFile == "" { + return fmt.Errorf("you have not either a token or a token file to authenticate with") + } + + // step: read in the token if required + if cfg.vaultTokenFile != "" { + exists, err := fileExists(cfg.vaultTokenFile) + if !exists { + return fmt.Errorf("the token file: %s does not exists, please check", cfg.vaultTokenFile) + } + if err != nil { + return fmt.Errorf("unable to check for token file: %s, error: %s", cfg.vaultTokenFile, err) + } + // step: read in the token + content, err := ioutil.ReadFile(cfg.vaultTokenFile) + if err != nil { + return fmt.Errorf("unable to read in token from file: %s, error: %s", cfg.vaultTokenFile, err) + } + cfg.vaultToken = string(content) + } + + return nil +} diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000..12d6f77 --- /dev/null +++ b/config_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestValidateOptions(t *testing.T) { + cfg := config{ + vaultToken: "", + vaultURL: "https://127.0.0.1:8200", + } + assert.NotNil(t, validateOptions(&cfg)) + cfg.vaultToken = "dkskdkskdsjdkjs" + assert.Nil(t, validateOptions(&cfg)) + cfg.vaultURL = "https://127.0.0.1" + assert.Nil(t, validateOptions(&cfg)) +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..07104aa --- /dev/null +++ b/main.go @@ -0,0 +1,152 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/golang/glog" + "gopkg.in/yaml.v2" +) + +func main() { + // step: parse and validate the command line / environment options + if err := parseOptions(); err != nil { + showUsage("invalid options, %s", err) + } + + // step: create a client to vault + vault, err := newVaultService(options.vaultURL, options.vaultToken) + if err != nil { + glog.Errorf("failed to create a vault client, error: %s", err) + } + + // step: setup the termination signals + signalChannel := make(chan os.Signal) + signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + // step: create a channel to receive events upon and add our resources for renewal + ch := make(vaultEventsChannel, 10) + + for _, rn := range options.resources.items { + // step: valid the resource + if err := rn.isValid(); err != nil { + showUsage("%s", err) + } + if err := vault.watch(rn, ch); err != nil { + showUsage("unable to add watch on resource: %s, %s", rn, err) + } + } + + // step: we simply wait for events i.e. secrets from vault and write them to the output directory + for { + select { + case evt := <-ch: + // step: write the secret to the output directory + go processResource(evt.resource, evt.secret) + + case <-signalChannel: + glog.Infof("recieved a termination signal, shutting down the service") + os.Exit(0) + } + } + +} + +// processResource ... write the resource to file, converting into the selected format +func processResource(rn *vaultResource, data map[string]interface{}) error { + var content []byte + var err error + + // step: determine the resource path + resourcePath := rn.filename() + if !strings.HasPrefix(resourcePath, "/") { + resourcePath = fmt.Sprintf("%s/%s", options.secretsDirectory, resourcePath) + } + + // step: get the output format + contentFormat := rn.getFormat() + glog.V(3).Infof("saving resource: %s, format: %s", rn, contentFormat) + + switch contentFormat { + case "yaml": + // marshall the content to yaml + if content, err = yaml.Marshal(data); err != nil { + return err + } + case "ini": + var buf bytes.Buffer + for key, val := range data { + buf.WriteString(fmt.Sprintf("%s = %s\n", key, val)) + } + content = buf.Bytes() + case "txt": + keys := getKeys(data) + if len(keys) > 1 { + // step: for plain formats we need to iterate the keys and produce a file per key + for suffix, content := range data { + filename := fmt.Sprintf("%s.%s", resourcePath, suffix) + // step: write the file + if err := writeFile(filename, []byte(fmt.Sprintf("%s", content))); err != nil { + glog.Errorf("failed to write resource: %s, elemment: %s, filename: %s, error: %s", + rn, suffix, filename, err) + continue + } + } + return nil + } + + // step: we only have the one key, so will write plain + value, _ := data[keys[0]] + content = []byte(fmt.Sprintf("%s", value)) + case "json": + if content, err = json.MarshalIndent(data, "", " "); err != nil { + return err + } + } + + // step: write the content to file + if err := writeFile(resourcePath, content); err != nil { + glog.Errorf("failed to write the resource: %s to file: %s, error: %s", rn, resourcePath, err) + return err + } + + return nil +} + +// writeFile ... writes the content of a file +func writeFile(filename string, content []byte) error { + // step: are we dry running? + if options.dryRun { + glog.Infof("dry-run: filename: %s, content:", filename) + fmt.Printf("%s\n", string(content)) + return nil + } + + if err := ioutil.WriteFile(filename, content, 0440); err != nil { + return err + } + + return nil +} \ No newline at end of file diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..a34bf67 --- /dev/null +++ b/main_test.go @@ -0,0 +1,17 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main diff --git a/services/sidekick-rc.yaml b/services/sidekick-rc.yaml new file mode 100644 index 0000000..9407964 --- /dev/null +++ b/services/sidekick-rc.yaml @@ -0,0 +1,45 @@ +# +# Author: Rohith +# Date: 2015-05-20 23:36:02 +0100 (Wed, 20 May 2015) +# +# vim:ts=2:sw=2:et +# +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: sidekick-demo +spec: + replicas: 1 + selector: + name: sidekick-demo + template: + metadata: + labels: + name: sidekick-web + spec: + containers: + - name: vault-sidekick + image: gambol99/vault-sidekick:latest + env: + - name: VAULT_ADDR + value: https://VAULT_IP:8200 + - name: VAULT_TOKEN + value: TOKEN + args: + - -v=3 + - -output=/etc/secrets + - -rn=secret:/prod/db;fn=db.creds,fmt=yaml + volumeMounts: + - name: secrets + mountPath: /etc/secrets + - name: apache + image: fedora/apache + volumeMounts: + - name: secrets + mountPath: /etc/secrets + ports: + - containerPort: 80 + volumes: + - name: secrets + emptyDir: {} \ No newline at end of file diff --git a/tests/demo-content.tmpl b/tests/demo-content.tmpl new file mode 100644 index 0000000..6f4dd0a --- /dev/null +++ b/tests/demo-content.tmpl @@ -0,0 +1,2 @@ + +username: {{ getSecretValue "secret:username", "username", "rn=1h" }} diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..57e9935 --- /dev/null +++ b/utils.go @@ -0,0 +1,81 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( +"os" +"math/rand" +"time" + "flag" + "fmt" +) + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) +} + +// showUsage ... prints the command usage and exits +// message : an error message to display if exiting with an error +func showUsage(message string, args ... interface{}) { + flag.PrintDefaults() + if message != "" { + fmt.Printf("\n[error] " + message + "\n", args...) + os.Exit(1) + } + + os.Exit(0) +} + +// getKeys ... retrieve a list of keys from the map +func getKeys(data map[string]interface{}) []string { + var list []string + for key, _ := range data { + list = append(list, key) + } + return list +} + +// randomInt ... generate a random integer between min and max +// min : the smallest number we can accept +// max : the largest number we can accept +func getRandomWithin(min, max int) int { + return rand.Intn(max - min) + min +} + +// getEnv ... checks to see if an environment variable exists otherwise uses the default +// env : the name of the environment variable you are checking for +// value : the default value to return if the value is not there +func getEnv(env, value string) string { + if v := os.Getenv(env); v != "" { + return v + } + + return value +} + +// fileExists ... checks to see if a file exists +// filename : the full path to the file you are checking for +func fileExists(filename string) (bool, error) { + if _, err := os.Stat(filename); err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} diff --git a/vault.go b/vault.go new file mode 100644 index 0000000..5a4314d --- /dev/null +++ b/vault.go @@ -0,0 +1,249 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "time" + + "github.com/hashicorp/vault/api" + "github.com/golang/glog" + "fmt" +) + +// a channel to send resource +type resourceChannel chan *vaultResource + +// vaultService ... is the main interface into the vault API - placing into a structure +// allows one to easily mock it and two to simplify the interface for us +type vaultService struct { + // the vault client + client *api.Client + // the vault config + config *api.Config + // a channel to inform of a new resource to processor + resourceCh chan *watchedResource + // the statistics channel + statCh *time.Ticker +} + +type vaultResourceEvent struct { + // the resource this relates to + resource *vaultResource + // the secret associated + secret map[string]interface{} +} + +// a channel of events +type vaultEventsChannel chan vaultResourceEvent + +// watchedResource ... is a resource which is being watched - i.e. when the item is coming up for renewal +// lets grab it and renew the lease +type watchedResource struct { + listener vaultEventsChannel + // the resource itself + resource *vaultResource + // the last time the resource was retrieved + lastUpdated time.Time + // the duration until the next renewal + renewalTime time.Duration + // the secret + secret *api.Secret +} + +// notifyOnRenewal ... creates a trigger and notifies when a resource is up for renewal +func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) { + go func() { + // step: check if the resource has a pre-configured renewal time + r.renewalTime = r.resource.leaseTime() + + // step: if the answer is no, we set the notification between 80-95% of the lease time of the secret + if r.renewalTime <= 0 { + glog.V(10).Infof("Calculating the renewal between 80-95 pcent of lease time: %d seconds", r.secret.LeaseDuration) + r.renewalTime = time.Duration(getRandomWithin( + int(float64(r.secret.LeaseDuration) * 0.8), + int(float64(r.secret.LeaseDuration) * 0.95))) * time.Second + } + glog.V(3).Infof("Setting a renewal notification on resource: %s, time: %s", r.resource, r.renewalTime) + // step: wait for the duration + <- time.After(r.renewalTime) + // step: send the notification on the renewal channel + ch <- r + }() +} + +// newVaultService ... creates a new implementation to speak to vault and retrieve the resources +// url : the url of the vault service +// token : the token to use when speaking to vault +func newVaultService(url, token string) (*vaultService, error) { + var err error + glog.Infof("Creating a new vault client: %s", url) + + // step: create the config for client + service := new(vaultService) + service.config = api.DefaultConfig() + service.config.Address = url + + // step: create the service processor channels + service.resourceCh = make(chan *watchedResource, 20) + service.statCh = time.NewTicker(options.statsInterval) + + // step: create the actual client + service.client, err = api.NewClient(service.config) + if err != nil { + return nil, err + } + + // step: set the token for the client + service.client.SetToken(token) + + // step: start the service processor off + service.vaultServiceProcessor() + + return service, nil +} + +// vaultServiceProcessor ... is the background routine responsible for retrieving the resources, renewing when required and +// informing those who are watching the resource that something has changed +func (r vaultService) vaultServiceProcessor() { + go func() { + // a list of resource being watched + items := make([]*watchedResource, 0) + // the channel to receive renewal notifications on + renewing := make(chan *watchedResource, 5) + + for { + select { + // A new resource is being added to the service processor; + // - we retrieve the resource from vault + // - if we error attempting to retrieve the secret, we background and reschedule an attempt to add it + // - if ok, we grab the lease it and lease time, we setup a notification on renewal + case x := <-r.resourceCh: + glog.V(3).Infof("Adding a resource into the service processor, resource: %s", x.resource) + + // step: retrieve the resource from vault + secret, err := r.get(x.resource) + if err != nil { + glog.Errorf("Failed to retrieve the resource: %s from vault, error: %s", x.resource, err) + // reschedule the attempt for later + go func(x *watchedResource) { + <- time.After(time.Duration(getRandomWithin(2,10)) * time.Second) + r.resourceCh <- x + }(x) + break + } + // step: update the item references + x.secret = secret + x.lastUpdated = time.Now() + + // step: setup a timer for renewal + x.notifyOnRenewal(renewing) + + // step: add to the list of resources + items = append(items, x) + + r.upstream(x, secret) + + // A watched resource is coming up for renewal + // - we attempt to grab the resource from vault + // - if we encounter an error, we reschedule the attempt for the future + // - if we're ok, we update the watchedResource and we send a notification of the change upstream + case x := <-renewing: + glog.V(3).Infof("Resource: %s coming up for renewal, attempting to renew now", x.resource) + // step: we attempt to renew the lease on a resource and if not successfully we reschedule + // a renewal notification for the future + + secret, err := r.get(x.resource) + if err != nil { + glog.Errorf("Failed to retrieve the resounce: %s for renewal, error: %s", x.resource, err) + // reschedule the attempt for later + go func(x *watchedResource) { + <- time.After(time.Duration(getRandomWithin(3,20)) * time.Second) + renewing <- x + }(x) + break + } + + // step: update the item references + x.secret = secret + x.lastUpdated = time.Now() + + // step: setup a timer for renewal + x.notifyOnRenewal(renewing) + + // step: update any listener upstream + r.upstream(x, secret) + + // The statistics timer has gone off; we iterate the watched items and + case <-r.statCh.C: + glog.V(3).Infof("Stats: %d resources being watched", len(items)) + for _, item := range items { + glog.V(3).Infof("resourse: %s, lease id: %s, renewal in: %s seconds", + item.resource, item.secret.LeaseID, item.renewalTime) + } + } + } + }() +} + +func (r vaultService) upstream(item *watchedResource, s *api.Secret) { + // step: chunk this into a go-routine not to block us + go func() { + glog.V(6).Infof("Sending the event for resource: %s upstream to listener: %v", item.resource, item.listener) + item.listener <- vaultResourceEvent{ + resource: item.resource, + secret: s.Data, + } + }() +} + +// get ... retrieve a secret from the vault +func (r vaultService) get(rn *vaultResource) (*api.Secret, error) { + var err error + var secret *api.Secret + + glog.V(5).Infof("Attempting to retrieve the resource: %s from vault", rn) + switch rn.resource { + case "pki": + secret, err = r.client.Logical().Write(fmt.Sprintf("%s/issue/%s", rn.resource, rn.name), + map[string]interface{}{ + "common_name": rn.options[OptionCommonName], + }) + case "aws": + secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name)) + case "mysql": + secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name)) + case "secret": + secret, err = r.client.Logical().Read(fmt.Sprintf("%s/%s", rn.resource, rn.name)) + } + if secret == nil && err == nil { + return nil, fmt.Errorf("does not exist") + } + + return secret, err +} + +// watch ... add a watch on a resource and inform, renew which required and inform us when +// the resource is ready +func (r *vaultService) watch(rn *vaultResource, ch vaultEventsChannel) error { + glog.V(10).Infof("Adding the resource: %s, listener: %v to service processor", rn, ch) + r.resourceCh <- &watchedResource{ + resource: rn, + listener: ch, + } + + return nil +} diff --git a/vault_resource.go b/vault_resource.go new file mode 100644 index 0000000..7950244 --- /dev/null +++ b/vault_resource.go @@ -0,0 +1,161 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "regexp" + "time" +) + +const ( + // OptionFilename ... option to set the filename of the resource + OptionFilename = "fn" + // OptionsFormat ... option to set the output format (yaml, xml, json) + OptionFormat = "fmt" + // OptionsCommonName ... use by the PKI resource + OptionCommonName = "cn" + // OptionTemplatePath ... the full path to a template + OptionsTemplatePath = "tpl" + // OptionRenew ... a duration to renew the resource + OptionRenew = "rn" +) + +var ( + resourceFormatRegex = regexp.MustCompile("^(yaml|json|ini|txt)$") + + // a map of valid resource to retrieve from vault + validResources = map[string]bool{ + "pki": true, + "aws": true, + "secret": true, + "mysql": true, + "tpl": true, + } + +) + +func newVaultResource() *vaultResource { + return &vaultResource{ + options: make(map[string]string, 0), + } +} + +// resource ... the structure which defined a resource set from vault +type vaultResource struct { + // the namespace of the resource + resource string + // the name of the resource + name string + // additional options to the resource + options map[string]string +} + +// leaseTime ... get the renew time otherwise return 0 +func (r vaultResource) leaseTime() time.Duration { + if _, found := r.options[OptionRenew]; found { + duration, _ := time.ParseDuration(r.options[OptionRenew]) + return duration + } + + return time.Duration(0) +} + +// isValid ... checks to see if the resource is valid +func (r vaultResource) isValid() error { + // step: check the resource type + if _, found := validResources[r.resource]; !found { + return fmt.Errorf("unsupported resource type: %s", r.resource) + } + + // step: check the options + if err := r.isValidOptions(); err != nil { + return fmt.Errorf("invalid resource options: %s, %s", r.options, err) + } + + // step: check is have all the required options to this resource type + if err := r.isValidResource(); err != nil { + return fmt.Errorf("invalid resource: %s, %s", r, err) + } + + return nil +} + +// getFormat ... get the format of the resource +func (r vaultResource) getFormat() string { + if format, found := r.options[OptionFormat]; found { + return format + } + return "txt" +} + + +// isValidResource ... validate the resource meets the requirements +func (r vaultResource) isValidResource() error { + switch r.resource { + case "pki": + if _, found := r.options[OptionCommonName]; !found { + return fmt.Errorf("pki resource requires a common name specified") + } + case "tpl": + if _, found := r.options[OptionsTemplatePath]; !found { + return fmt.Errorf("template resource requires a template path option") + } + } + + return nil +} + +// isValidOptions ... iterates through the options and check they are ok +func (r vaultResource) isValidOptions() error { + // check the filename directive + for opt, val := range r.options { + switch opt { + case OptionFormat: + if matched := resourceFormatRegex.MatchString(r.options[OptionFormat]); !matched { + return fmt.Errorf("unsupported output format: %s", r.options[OptionFormat]) + } + case OptionRenew: + if _, err := time.ParseDuration(val); err != nil { + return fmt.Errorf("the renew option: %s is not value", val) + } + case OptionFilename: + case OptionCommonName: + case OptionsTemplatePath: + if exists, _ := fileExists(val); !exists { + return fmt.Errorf("the template file: %s does not exist", val) + } + } + } + + return nil +} + +// resourceFilename ... generates a resource filename by default the resource name and resource type, which +// can override by the OPTION_FILENAME option +func (r vaultResource) filename() string { + if path, found := r.options[OptionFilename]; found { + return path + } + + return fmt.Sprintf("%s.%s", r.name, r.resource) +} + +// String ... a string representation of the struct +func (r vaultResource) String() string { + return fmt.Sprintf("%s/%s", r.resource, r.name) +} diff --git a/vault_resource_test.go b/vault_resource_test.go new file mode 100644 index 0000000..e0ae7c6 --- /dev/null +++ b/vault_resource_test.go @@ -0,0 +1,50 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResourceFilename(t *testing.T) { + rn := vaultResource{ + name: "test_secret", + resource: "secret", + options: map[string]string{}, + } + assert.Equal(t, "test_secret.secret", rn.filename()) + rn.options[OptionFilename] = "credentials" + assert.Equal(t, "credentials", rn.filename()) +} + + +func TestIsValid(t *testing.T) { + resource := newVaultResource() + resource.name = "/test/name" + resource.resource = "secret" + + assert.Nil(t, resource.isValid()) + resource.resource = "nothing" + assert.NotNil(t, resource.isValid()) + resource.resource = "pki" + assert.NotNil(t, resource.isValid()) + resource.options[OptionCommonName] = "common.example.com" + assert.Nil(t, resource.isValid()) + +} diff --git a/vault_resources.go b/vault_resources.go new file mode 100644 index 0000000..897d2a7 --- /dev/null +++ b/vault_resources.go @@ -0,0 +1,77 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "regexp" + "strings" +) + +var ( + resourceRegex = regexp.MustCompile("^([\\w]+):([\\w\\\\/\\-_\\.]+):?(.*)") + resourceOptionsRegex = regexp.MustCompile("([\\w\\d]{2,3})=([\\w\\d\\/\\.\\-_]+)[,]?") +) + +// resources ... a collection of type resource +type vaultResources struct { + // an array of resource to retrieve + items []*vaultResource +} + +func (r vaultResources) size() int { + return len(r.items) +} + +// Set ... implementation for the parser +func (r *vaultResources) Set(value string) error { + rn := new(vaultResource) + + // step: extract the resource type and name + if matched := resourceRegex.MatchString(value); !matched { + return fmt.Errorf("invalid resource specification, should be TYPE:NAME:?(OPTION_NAME=VALUE,)") + } + + // step: extract the matches + matches := resourceRegex.FindAllStringSubmatch(value, -1) + rn.resource = matches[0][1] + rn.name = matches[0][2] + rn.options = make(map[string]string, 0) + + // step: do we have any options for the resource? + if len(matches[0]) == 4 { + opts := matches[0][3] + for len(opts) > 0 { + if matched := resourceOptionsRegex.MatchString(opts); !matched { + return fmt.Errorf("invalid resource options specified: '%s', please check usage", opts) + } + + matches := resourceOptionsRegex.FindAllStringSubmatch(opts, -1) + rn.options[matches[0][1]] = matches[0][2] + opts = strings.TrimPrefix(opts, matches[0][0]) + } + } + // step: append to the list of resources + r.items = append(r.items, rn) + + return nil +} + +// String ... returns a string representation of the struct +func (r vaultResources) String() string { + return "" +} diff --git a/vault_resources_test.go b/vault_resources_test.go new file mode 100644 index 0000000..c0e5b7a --- /dev/null +++ b/vault_resources_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + + +func TestSetResources(t *testing.T) { + var items vaultResources + + assert.Nil(t, items.Set("secret:test:fn=filename.test,fmt=yaml")) + assert.Nil(t, items.Set("secret:test:fn=filename.test,")) + assert.Nil(t, items.Set("secret:/db/prod/username")) + assert.Nil(t, items.Set("secret:/db/prod:fn=filename.test,fmt=yaml")) + assert.Nil(t, items.Set("secret:test:fn=filename.test,")) + assert.Nil(t, items.Set("pki:example-dot-com:cn=blah.example.com")) + assert.Nil(t, items.Set("pki:example-dot-com:cn=blah.example.com,fn=/etc/certs/ssl/blah.example.com")) + assert.Nil(t, items.Set("pki:example-dot-com:cn=blah.example.com,rn=10s")) + assert.NotNil(t, items.Set("secret:")) + assert.NotNil(t, items.Set("secret:test:fn=filename.test,fmt=")) + assert.NotNil(t, items.Set("secret::fn=filename.test,fmt=yaml")) + assert.NotNil(t, items.Set("secret:te1st:fn=filename.test,fmt=")) + assert.NotNil(t, items.Set("fn=filename.test,fmt=yaml")) +} + +func TestResourceSize(t *testing.T) { + var items vaultResources + items.Set("secret:test:fn=filename.test,fmt=yaml") + items.Set("secret:test:fn=fileame.test") + assert.Equal(t, 2, items.size()) +} + +func TestResources(t *testing.T) { + var items vaultResources + items.Set("secret:test:fn=filename.test,fmt=yaml") + items.Set("secret:test:fn=fileame.test") + + if passed := assert.Equal(t, len(items.items), 2); !passed { + t.FailNow() + } + + rn := items.items[0] + assert.Equal(t, "secret", rn.resource) + assert.Equal(t, "test", rn.name) + assert.Equal(t, 2, len(rn.options)) + assert.Equal(t, "filename.test", rn.options[OptionFilename]) + assert.Equal(t, "yaml", rn.options[OptionFormat]) + rn = items.items[1] + assert.Equal(t, "secret", rn.resource) + assert.Equal(t, "test", rn.name) + assert.Equal(t, 1, len(rn.options)) + assert.Equal(t, "fileame.test", rn.options[OptionFilename]) +} diff --git a/vault_test.go b/vault_test.go new file mode 100644 index 0000000..82a3d2c --- /dev/null +++ b/vault_test.go @@ -0,0 +1,18 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + diff --git a/version.go b/version.go new file mode 100644 index 0000000..337c284 --- /dev/null +++ b/version.go @@ -0,0 +1,22 @@ +/* +Copyright 2015 Home Office All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +const ( + Version = "0.0.1" + GitSha = "" +) \ No newline at end of file