gered/vault-sidekick
Archived
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

- adding the initial commit

Browse source
This commit is contained in:
Rohith 2015-09-18 10:14:15 +01:00
parent daeb716799
commit 477c10e62f
20 changed files with 1421 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored
View file

@ -1,3 +1,9 @@
*.iml
*.swp
.idea/
build/
# Compiled Object files, Static and Dynamic libs (Shared Objects) # Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o *.o
*.a *.a

7
.travis.yml Normal file
View file

@ -0,0 +1,7 @@
language: go
go:
- 1.4
- 1.5
install:
- make test

4
Dockerfile Normal file
View file

@ -0,0 +1,4 @@
FROM gliderlabs/alpine:latest
MAINTAINER Rohith <gambol99@gmail.com>
ENTRYPOINT [ "/vault-sidekick" ]

201
LICENCE Normal file
View file

@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

24
Makefile Normal file
View file

@ -0,0 +1,24 @@
NAME=vault-sidekick
AUTHOR=gambol99
HARDWARE=$(shell uname -m)
VERSION=$(shell awk '/const Version/ { print $$4 }' version.go | sed 's/"//g')
.PHONY: test examples authors changelog build
default: build
build:
mkdir -p build
go build -o build/${NAME}
authors:
git log --format='%aN <%aE>' | sort -u > AUTHORS
test:
go get github.com/stretchr/testify
go get gopkg.in/yaml.v2
go test -v
changelog: release
git log $(shell git tag | tail -n1)..HEAD --no-merges --format=%B > changelog

85
README.md Normal file
View file

@ -0,0 +1,85 @@
### **Vault Side Kick**
-----
**Summary:**
> Vault Sidekick is a add-on container which can be used as a generic entry-point for interacting with Hashicorp [Vault](https://vaultproject.io) service, retrieving secrets
(both static and dynamic) and PKI certs. The sidekick will take care of renewal's and extension of leases for you and renew the credentials in the specified format for you.
**Usage:**
```shell
[jest@starfury vault-sidekick]$ build/vault-sidekick -help
Usage of build/vault-sidekick:
-alsologtostderr=false: log to standard error as well as files
-cn=: a resource to retrieve and monitor from vault (e.g. pki:name:cert.name, secret:db_password, aws:s3_backup)
-log_backtrace_at=:0: when logging hits line file:N, emit a stack trace
-log_dir="": If non-empty, write log files in this directory
-logtostderr=false: log to standard error instead of files
-output="/etc/secrets": the full path to write the protected resources (VAULT_OUTPUT if available)
-renew=true: whether or not to renew secrets from vault
-stderrthreshold=0: logs at or above this threshold go to stderr
-token="": the token used to authenticate to teh vault service (VAULT_TOKEN if available)
-tokenfile="": the full path to file containing the vault token used for authentication (VAULT_TOKEN_FILE if available)
-v=0: log level for V logs
-vault="https://127.0.0.1:8200": the url the vault service is running behind (VAULT_ADDR if available)
-vmodule=: comma-separated list of pattern=N settings for file-filtered logging
```
**Example Usage**
The below is taken from a [Kubernetes](https://github.com/kubernetes/kubernetes) pod specification;
```YAML
spec:
containers:
- name: vault-side-kick
image: gambol99/vault-sidekick:latest
args:
- -output=/etc/secrets
- -rn=pki:example.com:cn=commons.example.com,exec=/usr/bin/nginx_restart.sh,ctr=.*nginx_server.*
- -rn=secret:db/prod/username:fn=.credentials
- -rn=secret:db/prod/password
- -rn=aws:s3_backsup:fn=.s3_creds
- -rb=template:database_credentials:tpl=/etc/templates/db.tmpl,fn=/etc/credentials
volumeMounts:
- name: secrets
mountPath: /etc/secrets
```
The above say's
- Write all the secrets to the /etc/secrets directory
- Retrieve a dynamic certificate pair for me, with the common name: 'commons.example.com' and renew the cert when it expires automatically
- Retrieve the two static secrets /db/prod/{username,password} and write them to .credentials and password.secret respectively
- Apply the IAM policy, renew the policy when required and file the API tokens to .s3_creds in the /etc/secrets directory
- Read the template at /etc/templates/db.tmpl, produce the content from Vault and write to /etc/credentials file
**Output Format**
The following output formats are supported: json, yaml, ini, txt
Using the following at the demo secrets
```shell
[jest@starfury vault-sidekick]$ vault write secret/password this=is demo=value nothing=more
Success! Data written to: secret/password
[jest@starfury vault-sidekick]$ vault read secret/password
Key Value
lease_id secret/password/7908eceb-9bde-e7de-23da-96131505214a
lease_duration 2592000
lease_renewable false
demo value
nothing more
this is
```
In order to change the output format:
```shell
[jest@starfury vault-sidekick]$ build/vault-sidekick -cn=secret:password:fmt=ini -logtostderr=true -dry-run
[jest@starfury vault-sidekick]$ build/vault-sidekick -cn=secret:password:fmt=json -logtostderr=true -dry-run
[jest@starfury vault-sidekick]$ build/vault-sidekick -cn=secret:password:fmt=yaml -logtostderr=true -dry-run
```
The default format is 'txt' which has the following behavour. If the number of keys in a resource is > 1, a file is created per key. Thus using the example
(build/vault-sidekick -cn=secret:password:fn=test) we would end up with files: test.this, test.nothing and test.demo

114
config.go Normal file
View file

@ -0,0 +1,114 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"flag"
"fmt"
"io/ioutil"
"net/url"
"github.com/golang/glog"
"time"
)
// config ... the command line configuration
type config struct {
// the url for th vault server
vaultURL string
// the token to connect to vault with
vaultToken string
// a file / path containing a token for vault
vaultTokenFile string
// the place to write the resources
secretsDirectory string
// whether or not to renew the leases on our resources
renewResources bool
// whether of not to remove the token post connection
deleteToken bool
// switch on dry run
dryRun bool
// the resource items to retrieve
resources *vaultResources
// the interval for producing statistics
statsInterval time.Duration
}
var (
options config
)
func init() {
options.resources = new(vaultResources)
flag.StringVar(&options.vaultURL, "vault", getEnv("VAULT_ADDR", "https://127.0.0.1:8200"), "the url the vault service is running behind (VAULT_ADDR if available)")
flag.StringVar(&options.vaultToken, "token", "", "the token used to authenticate to teh vault service (VAULT_TOKEN if available)")
flag.StringVar(&options.vaultTokenFile, "tokenfile", getEnv("VAULT_TOKEN_FILE", ""), "the full path to file containing the vault token used for authentication (VAULT_TOKEN_FILE if available)")
flag.StringVar(&options.secretsDirectory, "output", getEnv("VAULT_OUTPUT", "/etc/secrets"), "the full path to write the protected resources (VAULT_OUTPUT if available)")
flag.BoolVar(&options.deleteToken, "delete-token", false, "once the we have connected to vault, delete the token file from disk")
flag.BoolVar(&options.renewResources, "renew", true, "whether or not to renew secrets from vault")
flag.BoolVar(&options.dryRun, "dry-run", false, "perform a dry run, printing the content to screen")
flag.DurationVar(&options.statsInterval, "stats", time.Duration(5) * time.Minute, "the interval to produce statistics on the accessed resources")
flag.Var(options.resources, "cn", "a resource to retrieve and monitor from vault (e.g. pki:name:cert.name, secret:db_password, aws:s3_backup)")
}
// parseOptions ... validate the command line options and validates them
func parseOptions() error {
flag.Parse()
return validateOptions(&options)
}
// validateOptions ... parses and validates the command line options
func validateOptions(cfg *config) error {
// step: validate the vault url
url, err := url.Parse(cfg.vaultURL)
if err != nil {
return fmt.Errorf("invalid vault url: '%s' specified", cfg.vaultURL)
}
// step: ensure the protocol scheme
if url.Scheme != "https" {
glog.Warningf("protocol scheme: %s is not secure and should be https", url.Scheme)
}
// step: check if the token is in the VAULT_TOKEN var
if cfg.vaultToken == "" {
cfg.vaultToken = getEnv("VAULT_TOKEN", "")
}
// step: ensure we have a token
if cfg.vaultToken == "" && cfg.vaultTokenFile == "" {
return fmt.Errorf("you have not either a token or a token file to authenticate with")
}
// step: read in the token if required
if cfg.vaultTokenFile != "" {
exists, err := fileExists(cfg.vaultTokenFile)
if !exists {
return fmt.Errorf("the token file: %s does not exists, please check", cfg.vaultTokenFile)
}
if err != nil {
return fmt.Errorf("unable to check for token file: %s, error: %s", cfg.vaultTokenFile, err)
}
// step: read in the token
content, err := ioutil.ReadFile(cfg.vaultTokenFile)
if err != nil {
return fmt.Errorf("unable to read in token from file: %s, error: %s", cfg.vaultTokenFile, err)
}
cfg.vaultToken = string(content)
}
return nil
}

35
config_test.go Normal file
View file

@ -0,0 +1,35 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestValidateOptions(t *testing.T) {
cfg := config{
vaultToken: "",
vaultURL: "https://127.0.0.1:8200",
}
assert.NotNil(t, validateOptions(&cfg))
cfg.vaultToken = "dkskdkskdsjdkjs"
assert.Nil(t, validateOptions(&cfg))
cfg.vaultURL = "https://127.0.0.1"
assert.Nil(t, validateOptions(&cfg))
}

152
main.go Normal file
View file

@ -0,0 +1,152 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"os/signal"
"strings"
"syscall"
"github.com/golang/glog"
"gopkg.in/yaml.v2"
)
func main() {
// step: parse and validate the command line / environment options
if err := parseOptions(); err != nil {
showUsage("invalid options, %s", err)
}
// step: create a client to vault
vault, err := newVaultService(options.vaultURL, options.vaultToken)
if err != nil {
glog.Errorf("failed to create a vault client, error: %s", err)
}
// step: setup the termination signals
signalChannel := make(chan os.Signal)
signal.Notify(signalChannel, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
// step: create a channel to receive events upon and add our resources for renewal
ch := make(vaultEventsChannel, 10)
for _, rn := range options.resources.items {
// step: valid the resource
if err := rn.isValid(); err != nil {
showUsage("%s", err)
}
if err := vault.watch(rn, ch); err != nil {
showUsage("unable to add watch on resource: %s, %s", rn, err)
}
}
// step: we simply wait for events i.e. secrets from vault and write them to the output directory
for {
select {
case evt := <-ch:
// step: write the secret to the output directory
go processResource(evt.resource, evt.secret)
case <-signalChannel:
glog.Infof("recieved a termination signal, shutting down the service")
os.Exit(0)
}
}
}
// processResource ... write the resource to file, converting into the selected format
func processResource(rn *vaultResource, data map[string]interface{}) error {
var content []byte
var err error
// step: determine the resource path
resourcePath := rn.filename()
if !strings.HasPrefix(resourcePath, "/") {
resourcePath = fmt.Sprintf("%s/%s", options.secretsDirectory, resourcePath)
}
// step: get the output format
contentFormat := rn.getFormat()
glog.V(3).Infof("saving resource: %s, format: %s", rn, contentFormat)
switch contentFormat {
case "yaml":
// marshall the content to yaml
if content, err = yaml.Marshal(data); err != nil {
return err
}
case "ini":
var buf bytes.Buffer
for key, val := range data {
buf.WriteString(fmt.Sprintf("%s = %s\n", key, val))
}
content = buf.Bytes()
case "txt":
keys := getKeys(data)
if len(keys) > 1 {
// step: for plain formats we need to iterate the keys and produce a file per key
for suffix, content := range data {
filename := fmt.Sprintf("%s.%s", resourcePath, suffix)
// step: write the file
if err := writeFile(filename, []byte(fmt.Sprintf("%s", content))); err != nil {
glog.Errorf("failed to write resource: %s, elemment: %s, filename: %s, error: %s",
rn, suffix, filename, err)
continue
}
}
return nil
}
// step: we only have the one key, so will write plain
value, _ := data[keys[0]]
content = []byte(fmt.Sprintf("%s", value))
case "json":
if content, err = json.MarshalIndent(data, "", " "); err != nil {
return err
}
}
// step: write the content to file
if err := writeFile(resourcePath, content); err != nil {
glog.Errorf("failed to write the resource: %s to file: %s, error: %s", rn, resourcePath, err)
return err
}
return nil
}
// writeFile ... writes the content of a file
func writeFile(filename string, content []byte) error {
// step: are we dry running?
if options.dryRun {
glog.Infof("dry-run: filename: %s, content:", filename)
fmt.Printf("%s\n", string(content))
return nil
}
if err := ioutil.WriteFile(filename, content, 0440); err != nil {
return err
}
return nil
}

17
main_test.go Normal file
View file

@ -0,0 +1,17 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

45
services/sidekick-rc.yaml Normal file
View file

@ -0,0 +1,45 @@
#
# Author: Rohith
# Date: 2015-05-20 23:36:02 +0100 (Wed, 20 May 2015)
#
# vim:ts=2:sw=2:et
#
---
apiVersion: v1
kind: ReplicationController
metadata:
name: sidekick-demo
spec:
replicas: 1
selector:
name: sidekick-demo
template:
metadata:
labels:
name: sidekick-web
spec:
containers:
- name: vault-sidekick
image: gambol99/vault-sidekick:latest
env:
- name: VAULT_ADDR
value: https://VAULT_IP:8200
- name: VAULT_TOKEN
value: TOKEN
args:
- -v=3
- -output=/etc/secrets
- -rn=secret:/prod/db;fn=db.creds,fmt=yaml
volumeMounts:
- name: secrets
mountPath: /etc/secrets
- name: apache
image: fedora/apache
volumeMounts:
- name: secrets
mountPath: /etc/secrets
ports:
- containerPort: 80
volumes:
- name: secrets
emptyDir: {}

2
tests/demo-content.tmpl Normal file
View file

@ -0,0 +1,2 @@
username: {{ getSecretValue "secret:username", "username", "rn=1h" }}

81
utils.go Normal file
View file

@ -0,0 +1,81 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"os"
"math/rand"
"time"
"flag"
"fmt"
)
func init() {
rand.Seed(int64(time.Now().Nanosecond()))
}
// showUsage ... prints the command usage and exits
// message : an error message to display if exiting with an error
func showUsage(message string, args ... interface{}) {
flag.PrintDefaults()
if message != "" {
fmt.Printf("\n[error] " + message + "\n", args...)
os.Exit(1)
}
os.Exit(0)
}
// getKeys ... retrieve a list of keys from the map
func getKeys(data map[string]interface{}) []string {
var list []string
for key, _ := range data {
list = append(list, key)
}
return list
}
// randomInt ... generate a random integer between min and max
// min : the smallest number we can accept
// max : the largest number we can accept
func getRandomWithin(min, max int) int {
return rand.Intn(max - min) + min
}
// getEnv ... checks to see if an environment variable exists otherwise uses the default
// env : the name of the environment variable you are checking for
// value : the default value to return if the value is not there
func getEnv(env, value string) string {
if v := os.Getenv(env); v != "" {
return v
}
return value
}
// fileExists ... checks to see if a file exists
// filename : the full path to the file you are checking for
func fileExists(filename string) (bool, error) {
if _, err := os.Stat(filename); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}

249
vault.go Normal file
View file

@ -0,0 +1,249 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"time"
"github.com/hashicorp/vault/api"
"github.com/golang/glog"
"fmt"
)
// a channel to send resource
type resourceChannel chan *vaultResource
// vaultService ... is the main interface into the vault API - placing into a structure
// allows one to easily mock it and two to simplify the interface for us
type vaultService struct {
// the vault client
client *api.Client
// the vault config
config *api.Config
// a channel to inform of a new resource to processor
resourceCh chan *watchedResource
// the statistics channel
statCh *time.Ticker
}
type vaultResourceEvent struct {
// the resource this relates to
resource *vaultResource
// the secret associated
secret map[string]interface{}
}
// a channel of events
type vaultEventsChannel chan vaultResourceEvent
// watchedResource ... is a resource which is being watched - i.e. when the item is coming up for renewal
// lets grab it and renew the lease
type watchedResource struct {
listener vaultEventsChannel
// the resource itself
resource *vaultResource
// the last time the resource was retrieved
lastUpdated time.Time
// the duration until the next renewal
renewalTime time.Duration
// the secret
secret *api.Secret
}
// notifyOnRenewal ... creates a trigger and notifies when a resource is up for renewal
func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) {
go func() {
// step: check if the resource has a pre-configured renewal time
r.renewalTime = r.resource.leaseTime()
// step: if the answer is no, we set the notification between 80-95% of the lease time of the secret
if r.renewalTime <= 0 {
glog.V(10).Infof("Calculating the renewal between 80-95 pcent of lease time: %d seconds", r.secret.LeaseDuration)
r.renewalTime = time.Duration(getRandomWithin(
int(float64(r.secret.LeaseDuration) * 0.8),
int(float64(r.secret.LeaseDuration) * 0.95))) * time.Second
}
glog.V(3).Infof("Setting a renewal notification on resource: %s, time: %s", r.resource, r.renewalTime)
// step: wait for the duration
<- time.After(r.renewalTime)
// step: send the notification on the renewal channel
ch <- r
}()
}
// newVaultService ... creates a new implementation to speak to vault and retrieve the resources
// url : the url of the vault service
// token : the token to use when speaking to vault
func newVaultService(url, token string) (*vaultService, error) {
var err error
glog.Infof("Creating a new vault client: %s", url)
// step: create the config for client
service := new(vaultService)
service.config = api.DefaultConfig()
service.config.Address = url
// step: create the service processor channels
service.resourceCh = make(chan *watchedResource, 20)
service.statCh = time.NewTicker(options.statsInterval)
// step: create the actual client
service.client, err = api.NewClient(service.config)
if err != nil {
return nil, err
}
// step: set the token for the client
service.client.SetToken(token)
// step: start the service processor off
service.vaultServiceProcessor()
return service, nil
}
// vaultServiceProcessor ... is the background routine responsible for retrieving the resources, renewing when required and
// informing those who are watching the resource that something has changed
func (r vaultService) vaultServiceProcessor() {
go func() {
// a list of resource being watched
items := make([]*watchedResource, 0)
// the channel to receive renewal notifications on
renewing := make(chan *watchedResource, 5)
for {
select {
// A new resource is being added to the service processor;
// - we retrieve the resource from vault
// - if we error attempting to retrieve the secret, we background and reschedule an attempt to add it
// - if ok, we grab the lease it and lease time, we setup a notification on renewal
case x := <-r.resourceCh:
glog.V(3).Infof("Adding a resource into the service processor, resource: %s", x.resource)
// step: retrieve the resource from vault
secret, err := r.get(x.resource)
if err != nil {
glog.Errorf("Failed to retrieve the resource: %s from vault, error: %s", x.resource, err)
// reschedule the attempt for later
go func(x *watchedResource) {
<- time.After(time.Duration(getRandomWithin(2,10)) * time.Second)
r.resourceCh <- x
}(x)
break
}
// step: update the item references
x.secret = secret
x.lastUpdated = time.Now()
// step: setup a timer for renewal
x.notifyOnRenewal(renewing)
// step: add to the list of resources
items = append(items, x)
r.upstream(x, secret)
// A watched resource is coming up for renewal
// - we attempt to grab the resource from vault
// - if we encounter an error, we reschedule the attempt for the future
// - if we're ok, we update the watchedResource and we send a notification of the change upstream
case x := <-renewing:
glog.V(3).Infof("Resource: %s coming up for renewal, attempting to renew now", x.resource)
// step: we attempt to renew the lease on a resource and if not successfully we reschedule
// a renewal notification for the future
secret, err := r.get(x.resource)
if err != nil {
glog.Errorf("Failed to retrieve the resounce: %s for renewal, error: %s", x.resource, err)
// reschedule the attempt for later
go func(x *watchedResource) {
<- time.After(time.Duration(getRandomWithin(3,20)) * time.Second)
renewing <- x
}(x)
break
}
// step: update the item references
x.secret = secret
x.lastUpdated = time.Now()
// step: setup a timer for renewal
x.notifyOnRenewal(renewing)
// step: update any listener upstream
r.upstream(x, secret)
// The statistics timer has gone off; we iterate the watched items and
case <-r.statCh.C:
glog.V(3).Infof("Stats: %d resources being watched", len(items))
for _, item := range items {
glog.V(3).Infof("resourse: %s, lease id: %s, renewal in: %s seconds",
item.resource, item.secret.LeaseID, item.renewalTime)
}
}
}
}()
}
func (r vaultService) upstream(item *watchedResource, s *api.Secret) {
// step: chunk this into a go-routine not to block us
go func() {
glog.V(6).Infof("Sending the event for resource: %s upstream to listener: %v", item.resource, item.listener)
item.listener <- vaultResourceEvent{
resource: item.resource,
secret: s.Data,
}
}()
}
// get ... retrieve a secret from the vault
func (r vaultService) get(rn *vaultResource) (*api.Secret, error) {
var err error
var secret *api.Secret
glog.V(5).Infof("Attempting to retrieve the resource: %s from vault", rn)
switch rn.resource {
case "pki":
secret, err = r.client.Logical().Write(fmt.Sprintf("%s/issue/%s", rn.resource, rn.name),
map[string]interface{}{
"common_name": rn.options[OptionCommonName],
})
case "aws":
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name))
case "mysql":
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/creds/%s", rn.resource, rn.name))
case "secret":
secret, err = r.client.Logical().Read(fmt.Sprintf("%s/%s", rn.resource, rn.name))
}
if secret == nil && err == nil {
return nil, fmt.Errorf("does not exist")
}
return secret, err
}
// watch ... add a watch on a resource and inform, renew which required and inform us when
// the resource is ready
func (r *vaultService) watch(rn *vaultResource, ch vaultEventsChannel) error {
glog.V(10).Infof("Adding the resource: %s, listener: %v to service processor", rn, ch)
r.resourceCh <- &watchedResource{
resource: rn,
listener: ch,
}
return nil
}

161
vault_resource.go Normal file
View file

@ -0,0 +1,161 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"regexp"
"time"
)
const (
// OptionFilename ... option to set the filename of the resource
OptionFilename = "fn"
// OptionsFormat ... option to set the output format (yaml, xml, json)
OptionFormat = "fmt"
// OptionsCommonName ... use by the PKI resource
OptionCommonName = "cn"
// OptionTemplatePath ... the full path to a template
OptionsTemplatePath = "tpl"
// OptionRenew ... a duration to renew the resource
OptionRenew = "rn"
)
var (
resourceFormatRegex = regexp.MustCompile("^(yaml|json|ini|txt)$")
// a map of valid resource to retrieve from vault
validResources = map[string]bool{
"pki": true,
"aws": true,
"secret": true,
"mysql": true,
"tpl": true,
}
)
func newVaultResource() *vaultResource {
return &vaultResource{
options: make(map[string]string, 0),
}
}
// resource ... the structure which defined a resource set from vault
type vaultResource struct {
// the namespace of the resource
resource string
// the name of the resource
name string
// additional options to the resource
options map[string]string
}
// leaseTime ... get the renew time otherwise return 0
func (r vaultResource) leaseTime() time.Duration {
if _, found := r.options[OptionRenew]; found {
duration, _ := time.ParseDuration(r.options[OptionRenew])
return duration
}
return time.Duration(0)
}
// isValid ... checks to see if the resource is valid
func (r vaultResource) isValid() error {
// step: check the resource type
if _, found := validResources[r.resource]; !found {
return fmt.Errorf("unsupported resource type: %s", r.resource)
}
// step: check the options
if err := r.isValidOptions(); err != nil {
return fmt.Errorf("invalid resource options: %s, %s", r.options, err)
}
// step: check is have all the required options to this resource type
if err := r.isValidResource(); err != nil {
return fmt.Errorf("invalid resource: %s, %s", r, err)
}
return nil
}
// getFormat ... get the format of the resource
func (r vaultResource) getFormat() string {
if format, found := r.options[OptionFormat]; found {
return format
}
return "txt"
}
// isValidResource ... validate the resource meets the requirements
func (r vaultResource) isValidResource() error {
switch r.resource {
case "pki":
if _, found := r.options[OptionCommonName]; !found {
return fmt.Errorf("pki resource requires a common name specified")
}
case "tpl":
if _, found := r.options[OptionsTemplatePath]; !found {
return fmt.Errorf("template resource requires a template path option")
}
}
return nil
}
// isValidOptions ... iterates through the options and check they are ok
func (r vaultResource) isValidOptions() error {
// check the filename directive
for opt, val := range r.options {
switch opt {
case OptionFormat:
if matched := resourceFormatRegex.MatchString(r.options[OptionFormat]); !matched {
return fmt.Errorf("unsupported output format: %s", r.options[OptionFormat])
}
case OptionRenew:
if _, err := time.ParseDuration(val); err != nil {
return fmt.Errorf("the renew option: %s is not value", val)
}
case OptionFilename:
case OptionCommonName:
case OptionsTemplatePath:
if exists, _ := fileExists(val); !exists {
return fmt.Errorf("the template file: %s does not exist", val)
}
}
}
return nil
}
// resourceFilename ... generates a resource filename by default the resource name and resource type, which
// can override by the OPTION_FILENAME option
func (r vaultResource) filename() string {
if path, found := r.options[OptionFilename]; found {
return path
}
return fmt.Sprintf("%s.%s", r.name, r.resource)
}
// String ... a string representation of the struct
func (r vaultResource) String() string {
return fmt.Sprintf("%s/%s", r.resource, r.name)
}

50
vault_resource_test.go Normal file
View file

@ -0,0 +1,50 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestResourceFilename(t *testing.T) {
rn := vaultResource{
name: "test_secret",
resource: "secret",
options: map[string]string{},
}
assert.Equal(t, "test_secret.secret", rn.filename())
rn.options[OptionFilename] = "credentials"
assert.Equal(t, "credentials", rn.filename())
}
func TestIsValid(t *testing.T) {
resource := newVaultResource()
resource.name = "/test/name"
resource.resource = "secret"
assert.Nil(t, resource.isValid())
resource.resource = "nothing"
assert.NotNil(t, resource.isValid())
resource.resource = "pki"
assert.NotNil(t, resource.isValid())
resource.options[OptionCommonName] = "common.example.com"
assert.Nil(t, resource.isValid())
}

77
vault_resources.go Normal file
View file

@ -0,0 +1,77 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"fmt"
"regexp"
"strings"
)
var (
resourceRegex = regexp.MustCompile("^([\\w]+):([\\w\\\\/\\-_\\.]+):?(.*)")
resourceOptionsRegex = regexp.MustCompile("([\\w\\d]{2,3})=([\\w\\d\\/\\.\\-_]+)[,]?")
)
// resources ... a collection of type resource
type vaultResources struct {
// an array of resource to retrieve
items []*vaultResource
}
func (r vaultResources) size() int {
return len(r.items)
}
// Set ... implementation for the parser
func (r *vaultResources) Set(value string) error {
rn := new(vaultResource)
// step: extract the resource type and name
if matched := resourceRegex.MatchString(value); !matched {
return fmt.Errorf("invalid resource specification, should be TYPE:NAME:?(OPTION_NAME=VALUE,)")
}
// step: extract the matches
matches := resourceRegex.FindAllStringSubmatch(value, -1)
rn.resource = matches[0][1]
rn.name = matches[0][2]
rn.options = make(map[string]string, 0)
// step: do we have any options for the resource?
if len(matches[0]) == 4 {
opts := matches[0][3]
for len(opts) > 0 {
if matched := resourceOptionsRegex.MatchString(opts); !matched {
return fmt.Errorf("invalid resource options specified: '%s', please check usage", opts)
}
matches := resourceOptionsRegex.FindAllStringSubmatch(opts, -1)
rn.options[matches[0][1]] = matches[0][2]
opts = strings.TrimPrefix(opts, matches[0][0])
}
}
// step: append to the list of resources
r.items = append(r.items, rn)
return nil
}
// String ... returns a string representation of the struct
func (r vaultResources) String() string {
return ""
}

71
vault_resources_test.go Normal file
View file

@ -0,0 +1,71 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestSetResources(t *testing.T) {
var items vaultResources
assert.Nil(t, items.Set("secret:test:fn=filename.test,fmt=yaml"))
assert.Nil(t, items.Set("secret:test:fn=filename.test,"))
assert.Nil(t, items.Set("secret:/db/prod/username"))
assert.Nil(t, items.Set("secret:/db/prod:fn=filename.test,fmt=yaml"))
assert.Nil(t, items.Set("secret:test:fn=filename.test,"))
assert.Nil(t, items.Set("pki:example-dot-com:cn=blah.example.com"))
assert.Nil(t, items.Set("pki:example-dot-com:cn=blah.example.com,fn=/etc/certs/ssl/blah.example.com"))
assert.Nil(t, items.Set("pki:example-dot-com:cn=blah.example.com,rn=10s"))
assert.NotNil(t, items.Set("secret:"))
assert.NotNil(t, items.Set("secret:test:fn=filename.test,fmt="))
assert.NotNil(t, items.Set("secret::fn=filename.test,fmt=yaml"))
assert.NotNil(t, items.Set("secret:te1st:fn=filename.test,fmt="))
assert.NotNil(t, items.Set("fn=filename.test,fmt=yaml"))
}
func TestResourceSize(t *testing.T) {
var items vaultResources
items.Set("secret:test:fn=filename.test,fmt=yaml")
items.Set("secret:test:fn=fileame.test")
assert.Equal(t, 2, items.size())
}
func TestResources(t *testing.T) {
var items vaultResources
items.Set("secret:test:fn=filename.test,fmt=yaml")
items.Set("secret:test:fn=fileame.test")
if passed := assert.Equal(t, len(items.items), 2); !passed {
t.FailNow()
}
rn := items.items[0]
assert.Equal(t, "secret", rn.resource)
assert.Equal(t, "test", rn.name)
assert.Equal(t, 2, len(rn.options))
assert.Equal(t, "filename.test", rn.options[OptionFilename])
assert.Equal(t, "yaml", rn.options[OptionFormat])
rn = items.items[1]
assert.Equal(t, "secret", rn.resource)
assert.Equal(t, "test", rn.name)
assert.Equal(t, 1, len(rn.options))
assert.Equal(t, "fileame.test", rn.options[OptionFilename])
}

18
vault_test.go Normal file
View file

@ -0,0 +1,18 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main

22
version.go Normal file
View file

@ -0,0 +1,22 @@
/*
Copyright 2015 Home Office All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
const (
Version = "0.0.1"
GitSha = ""
)