write out cert ca_chain as a pem bundle for pki secrets

this is a useful format for mongo's PEMKeyFile option which expects
the provided key file to be the private key + cert concatenated
together

.gitignore

@@ -5,6 +5,7 @@
 .idea/
 bin/
 release/
+secrets/
 
 # Compiled Object files, Static and Dynamic libs (Shared Objects)
 *.o

.travis.yml

@@ -1,3 +1,5 @@
+services:
+- docker
 env:
   global:
   - secure: Vq4593WUn2Hng93OynvydesiWu7dAnhIFhX6KtmVwqyuWuk60D8bJAd9qiJYax6+jqsAVnouaD+0TtfLb+SFJ9+vFMqykP+iSVJyC6p3DCRqMmKN+KN20hmq5WSvoE9Rs1elNV770plZdDbAIJ36iFsSpJnrp6dA9vFs0G4E2ceLw0KTJDW7C/eFO1LFiWdgsIrq4DpSCGqjcAzrRKsAkajBmsLNO//MFlsUnnJX4YBQKzibSAyoHeTHPitkvZ6Qf11N7pFMqeX78Stqeat7Xks/C0RzCOZipIGX/yU5qTGg8yGS9FG1qDkNf2Vh4osZySRHy4QJfukTlrLBB1skPHfQpU6n3bNVkF2dhQv0M4TzkrFXB+DF5WyrvKhM7i/w6WvTmg6Zc66ut0qXbTUOSJeZpuHo7c195ly51GxmbNIDw9F2RrJbjRqJrrJ9EXGCBC1wbz4wIYXt5sGh2bl1HdZqGEYkzFfXsIrFc6e8mal/AEMwos3hfezoU9kyriM5sA4M4+yK4ERQYL3Kc4UsFg4oyZrbna6k4Pas/mdFYeM3jhQxOKWxCaSSoeKE/FrbKcMb2/4btQe12Ik9zUIca58VJ04KawEBoyF/+YTSCpQOapnoFmWJoCrXJZlIU56zdn4LXNjDL+pENnjDXhutQck3ZqErcewQQzSepIUtFQo=
@@ -5,30 +7,29 @@ env:
   - AUTHOR_EMAIL=gambol99@gmail.com
   - REGISTRY_USERNAME=ukhomeofficedigital+vault_sidekick
   - REGISTRY=quay.io
-services:
+
-- docker
 language: go
-go:
+go: 1.8.1
-- 1.7.1
+
-install:
+install: true
+script:
 - make test
-- if ([[ ${TRAVIS_BRANCH} == "master" ]] && [[ ${TRAVIS_PULL_REQUEST} != "true" ]]) || [[ -n ${TRAVIS_TAG} ]]; then
+- if ([[ ${TRAVIS_BRANCH} == "master" ]] && [[ ${TRAVIS_PULL_REQUEST} == "false" ]]) || [[ -n ${TRAVIS_TAG} ]]; then
+    GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.gitsha=${TRAVIS_TAG:-git+${TRAVIS_COMMIT}}" -o bin/vault-sidekick_linux_amd64;
+    GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.gitsha=${TRAVIS_TAG:-git+${TRAVIS_COMMIT}}" -o bin/vault-sidekick_darwin_amd64;
+    GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.gitsha=${TRAVIS_TAG:-git+${TRAVIS_COMMIT}}" -o bin/vault-sidekick_windows_amd64.exe;
     docker login -u ${REGISTRY_USERNAME} -p ${REGISTRY_TOKEN} -e ${AUTHOR_EMAIL} ${REGISTRY};
-    VERSION=latest make docker-release;
+    VERSION=${TRAVIS_TAG:-latest} make docker-release;
   fi
-before_deploy:
-- NAME=GOOS=linux GOARCH=amd64 godep go build -o bin/vault-sidekick-linux-amd64
-after_deploy:
-- docker login -u ${REGISTRY_USERNAME} -p ${REGISTRY_TOKEN} -e ${AUTHOR_EMAIL} ${REGISTRY}
-- VERSION=$TRAVIS_TAG make docker-release
 deploy:
   provider: releases
   skip_cleanup: true
   on:
-    go: 1.7.1
     repo: UKHomeOffice/vault-sidekick
     tags: true
   api_key:
     secure: "${GITHUB_TOKEN}"
   file:
-  - bin/vault-sidekick-linux-amd64
+    - bin/vault-sidekick_linux_amd64
+    - bin/vault-sidekick_darwin_amd64
+    - bin/vault-sidekick_windows_amd64.exe

CHANGELOG.md

@@ -1,4 +1,35 @@
 
+#### **Version v0.3.4**
+
+##### FEATURES
+
+ * Adding a jitter option to the resources
+
+#### **Version v0.3.3**
+
+##### FEATURES
+
+ * Loading vault url from kubernetes vault auth file, exit if vault url is not set
+
+#### **Version v0.3.2**
+
+##### FEATURES
+
+ * Added kubernetes-vault support
+ * Added onetime only mode via the one-shot option
+ * Added the 'retries' parameter to resources to allow optional maxRetries
+
+#### **Version v0.3.1**
+
+##### FEATURES
+
+ * Added a mode option to the resource specification enabling secrets to set the file permissions
+ * Fixed a bug in the renewal time, when a resource does not have a custom update and the lease time is 0s
+ * Cleaned up some of the vetting issues
+ * Change the travis build to use golang v1.8.1
+ * Added a version flag -version and passing the gitsha in the version
+ * Updated the kubernete deployment files
+
 #### **Version v0.1.0**
 
 ##### FEATURES

Dockerfile

@@ -1,9 +1,13 @@
-FROM alpine:3.4
+FROM alpine:3.5
 MAINTAINER Rohith <gambol99@gmail.com>
 
 RUN apk update && \
     apk add ca-certificates bash
 
+RUN adduser -D vault
+
 ADD bin/vault-sidekick /vault-sidekick
 
+USER vault
+
 ENTRYPOINT [ "/vault-sidekick" ]

Makefile

@@ -2,9 +2,11 @@
 NAME=vault-sidekick
 AUTHOR ?= ukhomeofficedigital
 REGISTRY ?= quay.io
-GOVERSION ?= 1.7.1
+GOVERSION ?= 1.8.1
 HARDWARE=$(shell uname -m)
-VERSION ?= $(shell awk '/Version =/ { print $$3 }' main.go | sed 's/"//g')
+VERSION ?= $(shell awk '/release =/ { print $$3 }' main.go | sed 's/"//g')
+GIT_SHA=$(shell git --no-pager describe --always --dirty)
+LFLAGS ?= -X main.gitsha=${GIT_SHA}
 VETARGS?=-asmdecl -atomic -bool -buildtags -copylocks -methods -nilfunc -printf -rangeloops -shift -structtags -unsafeptr
 
 .PHONY: test authors changelog build docker static release
@@ -14,12 +16,12 @@ default: build
 build: deps
 	@echo "--> Compiling the project"
 	mkdir -p bin
-	godep go build -o bin/${NAME}
+	godep go build -ldflags '-w ${LFLAGS}' -o bin/${NAME}
 
 static: deps
 	@echo "--> Compiling the static binary"
 	mkdir -p bin
-	CGO_ENABLED=0 GOOS=linux godep go build -a -tags netgo -ldflags '-w' -o bin/${NAME}
+	CGO_ENABLED=0 GOOS=linux godep go build -a -tags netgo -ldflags '-w ${LFLAGS}' -o bin/${NAME}
 
 docker-build:
 	@echo "--> Compiling the project"

README.md

@@ -12,30 +12,53 @@ Vault Sidekick is a add-on container which can be used as a generic entry-point
 **Usage:**
 
 ```shell
-[jest@starfury vault-sidekick]$ bin/vault-sidekick --help
+$ sudo docker run --rm quay.io/ukhomeofficedigital/vault-sidekick:v0.3.3 -help
-Usage of bin/vault-sidekick:
+Usage of /vault-sidekick:
-  -alsologtostderr=false: log to standard error as well as files
+  -alsologtostderr
-  -auth="": a configuration file in a json or yaml containing authentication arguments
+    	log to standard error as well as files
-  -cn=: a resource to retrieve and monitor from vault (e.g. pki:name:cert.name, secret:db_password, aws:s3_backup)
+  -auth string
-  -ca-cert="": a CA certificate to use in order to validate the vault service certificate
+    	a configuration file in json or yaml containing authentication arguments
-  -delete-token=false: once the we have connected to vault, delete the token file from disk
+  -ca-cert string
-  -dryrun=false: perform a dry run, printing the content to screen
+    	the path to the file container the CA used to verify the vault service
-  -log_backtrace_at=:0: when logging hits line file:N, emit a stack trace
+  -cn value
-  -log_dir="": If non-empty, write log files in this directory
+    	a resource to retrieve and monitor from vault
-  -logtostderr=false: log to standard error instead of files
+  -dryrun
-  -output="/etc/secrets": the full path to write the protected resources (VAULT_OUTPUT if available)
+    	perform a dry run, printing the content to screen
-  -stats=5m0s: the interval to produce statistics on the accessed resources
+  -exec-timeout duration
-  -stderrthreshold=0: logs at or above this threshold go to stderr
+    	the timeout applied to commands on the exec option (default 1m0s)
-  -tls-skip-verify=false: skip verifying the vault certificate
+  -format string
-  -token="": the token used to authenticate to teh vault service (VAULT_TOKEN if available)
+    	the auth file format (default "default")
-  -v=0: log level for V logs
+  -log_backtrace_at value
-  -vault="https://127.0.0.1:8200": the url the vault service is running behind (VAULT_ADDR if available)
+    	when logging hits line file:N, emit a stack trace
-  -vmodule=: comma-separated list of pattern=N settings for file-filtered logging
+  -log_dir string
+    	If non-empty, write log files in this directory
+  -logtostderr
+    	log to standard error instead of files
+  -one-shot
+    	retrieve resources from vault once and then exit
+  -output string
+    	the full path to write resources or VAULT_OUTPUT (default "/etc/secrets")
+  -stats duration
+    	the interval to produce statistics on the accessed resources (default 1h0m0s)
+  -stderrthreshold value
+    	logs at or above this threshold go to stderr
+  -tls-skip-verify
+    	whether to check and verify the vault service certificate
+  -v value
+    	log level for V logs
+  -vault string
+    	url the vault service or VAULT_ADDR (default "https://127.0.0.1:8200")
+  -version
+    	show the vault-sidekick version
+  -vmodule value
+    	comma-separated list of pattern=N settings for file-filtered logging
 ```
 
 **Building**
 
-There is a Makefile in the base repository, so assuming you have make and go: # make
+There is a Makefile in the base repository, so assuming you have make and go:
+
+`$ make`
 
 **Example Usage**
 
@@ -45,19 +68,19 @@ The below is taken from a [Kubernetes](https://github.com/kubernetes/kubernetes)
 spec:
       containers:
       - name: vault-side-kick
-        image: gambol99/vault-sidekick:latest
+        image: quay.io/ukhomeofficedigital/vault-sidekick:v0.3.3
         args:
           - -output=/etc/secrets
           - -cn=pki:project1/certs/example.com:common_name=commons.example.com,revoke=true,update=2h
           - -cn=secret:secret/db/prod/username:file=.credentials
-          - -cn=secret:secret/db/prod/password
+          - -cn=secret:secret/db/prod/password:retries=true
           - -cn=aws:aws/creds/s3_backup_policy:file=.s3_creds
         volumeMounts:
           - name: secrets
             mountPath: /etc/secrets
 ```
 
-The above say's
+The above equates to:
 
  - Write all the secrets to the /etc/secrets directory
  - Retrieve a dynamic certificate pair for me, with the common name: 'commons.example.com' and renew the cert when it expires automatically
@@ -67,7 +90,7 @@ The above say's
 
 **Authentication**
 
-A authentication file can be specified in either yaml of json format which contains a method field, indicating one of the authentication
+An authentication file can be specified in either yaml of json format which contains a method field, indicating one of the authentication
 methods provided by vault i.e. userpass, token, github etc and then followed by the required arguments for that plugin.
 
 If the required arguments for that plugin are not contained in the authentication file, fallbacks from environment variables are used.
@@ -136,6 +159,7 @@ bundle format is very similar in the sense it similar takes the private key and
 **Resource Options**
 
 - **file**: (filaname) by default all file are relative to the output directory specified and will have the name NAME.RESOURCE; the fn options allows you to switch names and paths to write the files
+- **mode**: (mode) overrides the default file permissions of the secret from 0664
 - **create**: (create) create the resource
 - **update**: (update) override the lease time of this resource and get/renew a secret on the specified duration e.g 1m, 2d, 5m10s
 - **renew**: (renewal) override the default behavour on this resource, renew the resource when coming close to expiration e.g true, TRUE
@@ -143,3 +167,5 @@ bundle format is very similar in the sense it similar takes the private key and
 - **revoke**: (revoke) revoke the old lease when you get retrieve a old one e.g. true, TRUE (default to allow the lease to expire and naturally revoke)
 - **fmt**: (format) allows you to specify the output format of the resource / secret, e.g json, yaml, ini, txt
 - **exec** (execute) execute's a command when resource is updated or changed
+- **retries**: (retries) the maximum number of times to retry retrieving a resource. If not set, resources will be retried indefinitely
+* **jitter**: (jitter) an optional maximum jitter duration. If specified, a random duration between 0 and `jitter` will be subtracted from the renewal time for the resource

auth_approle.go

@@ -28,8 +28,8 @@ type authAppRolePlugin struct {
 }
 
 type appRoleLogin struct {
-	RoleId   string `json:"role_id,omitempty"`
+	RoleID   string `json:"role_id,omitempty"`
-	SecretId string `json:"secret_id,omitempty"`
+	SecretID string `json:"secret_id,omitempty"`
 }
 
 // NewAppRolePlugin creates a new App Role plugin
@@ -40,21 +40,17 @@ func NewAppRolePlugin(client *api.Client) AuthInterface {
 }
 
 // Create a approle plugin with the secret id and role id provided in the file
-func (r authAppRolePlugin) Create(cfg map[string]string) (string, error) {
+func (r authAppRolePlugin) Create(cfg *vaultAuthOptions) (string, error) {
-	// step: extract the options
+	if cfg.RoleID == "" {
-	roleId, _ := cfg["role_id"]
+		cfg.RoleID = os.Getenv("VAULT_SIDEKICK_ROLE_ID")
-	secretId, _ := cfg["secret_id"]
-
-	if roleId == "" {
-		roleId = os.Getenv("VAULT_SIDEKICK_ROLE_ID")
 	}
-	if secretId == "" {
+	if cfg.SecretID == "" {
-		secretId = os.Getenv("VAULT_SIDEKICK_SECRET_ID")
+		cfg.SecretID = os.Getenv("VAULT_SIDEKICK_SECRET_ID")
 	}
 
 	// step: create the token request
 	request := r.client.NewRequest("POST", "/v1/auth/approle/login")
-	login := appRoleLogin{SecretId: secretId, RoleId: roleId}
+	login := appRoleLogin{SecretID: cfg.SecretID, RoleID: cfg.RoleID}
 	if err := request.SetJSONBody(login); err != nil {
 		return "", err
 	}

auth_kubernetes.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/hashicorp/vault/api"
+)
+
+type authKubernetesPlugin struct {
+	client *api.Client
+}
+
+type kubernetesLogin struct {
+	Role string `json:"role"`
+	JWT  string `json:"jwt"`
+}
+
+func NewKubernetesPlugin(client *api.Client) AuthInterface {
+	return &authKubernetesPlugin{
+		client: client,
+	}
+}
+
+func (p authKubernetesPlugin) Create(cfg *vaultAuthOptions) (string, error) {
+	if cfg.RoleID == "" {
+		cfg.RoleID = os.Getenv("VAULT_SIDEKICK_K8S_ROLE")
+	}
+	if cfg.FileName == "" {
+		cfg.FileName = os.Getenv("VAULT_SIDEKICK_K8S_TOKEN_FILE")
+		// default to the typical location for this
+		if cfg.FileName == "" {
+			cfg.FileName = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+		}
+	}
+
+	// read kubernetes serviceaccount token file (a jwt token)
+	tokenBytes, err := ioutil.ReadFile(cfg.FileName)
+	if err != nil {
+		return "", err
+	}
+
+	// create token request
+	request := p.client.NewRequest("POST", "/v1/auth/kubernetes/login")
+	body := kubernetesLogin{Role: cfg.RoleID, JWT: string(tokenBytes)}
+	err = request.SetJSONBody(body)
+	if err != nil {
+		return "", err
+	}
+
+	// execute api request
+	resp, err := p.client.RawRequest(request)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	// parse secret response
+	secret, err := api.ParseSecret(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return secret.Auth.ClientToken, nil
+}

auth_token.go

@@ -37,17 +37,16 @@ func NewUserTokenPlugin(client *api.Client) AuthInterface {
 }
 
 // Create retrieves the token from an environment variable or file
-func (r authTokenPlugin) Create(cfg map[string]string) (string, error) {
+func (r authTokenPlugin) Create(cfg *vaultAuthOptions) (string, error) {
-	filename, _ := cfg["filename"]
+	if cfg.FileName != "" {
-	if filename != "" {
+		content, err := readConfigFile(cfg.FileName, cfg.FileFormat)
-		content, err := readConfigFile(filename)
 		if err != nil {
 			return "", err
 		}
 		// check: ensure we have a token in the file
-		token, found := content["token"]
+		token := content.Token
-		if !found {
+		if token == "" {
-			return "", fmt.Errorf("the auth file: %s does not contain a token", filename)
+			return "", fmt.Errorf("the auth file: %s does not contain a token", cfg.FileName)
 		}
 
 		return token, nil

auth_userpass.go

@@ -41,21 +41,18 @@ func NewUserPassPlugin(client *api.Client) AuthInterface {
 }
 
 // Create a userpass plugin with the username and password provide in the file
-func (r authUserPassPlugin) Create(cfg map[string]string) (string, error) {
+func (r authUserPassPlugin) Create(cfg *vaultAuthOptions) (string, error) {
 	// step: extract the options
-	username, _ := cfg["username"]
+	if cfg.Username == "" {
-	password, _ := cfg["password"]
+		cfg.Username = os.Getenv("VAULT_SIDEKICK_USERNAME")
-
-	if username == "" {
-		username = os.Getenv("VAULT_SIDEKICK_USERNAME")
 	}
-	if password == "" {
+	if cfg.Password == "" {
-		password = os.Getenv("VAULT_SIDEKICK_PASSWORD")
+		cfg.Password = os.Getenv("VAULT_SIDEKICK_PASSWORD")
 	}
 
 	// step: create the token request
-	request := r.client.NewRequest("POST", fmt.Sprintf("/v1/auth/userpass/login/%s", username))
+	request := r.client.NewRequest("POST", fmt.Sprintf("/v1/auth/userpass/login/%s", cfg.Username))
-	if err := request.SetJSONBody(userPassLogin{Password: password}); err != nil {
+	if err := request.SetJSONBody(userPassLogin{Password: cfg.Password}); err != nil {
 		return "", err
 	}
 	// step: make the request

config.go

@@ -20,16 +20,34 @@ import (
 	"flag"
 	"fmt"
 	"net/url"
+	"os"
 	"time"
 )
 
+type vaultAuthOptions struct {
+	ClientToken   string
+	Token         string
+	LeaseDuration int
+	Renewable     bool
+	Method        string
+	VaultURL      string `json:"vaultAddr"`
+	RoleID        string `json:"role_id" yaml:"role_id"`
+	SecretID      string `json:"secret_id" yaml:"secret_id"`
+	FileName      string
+	FileFormat    string
+	Username      string
+	Password      string
+}
+
 type config struct {
 	// the url for th vault server
 	vaultURL string
 	// a file containing the authenticate options
 	vaultAuthFile string
+	// whether or not the auth file format is default
+	vaultAuthFileFormat string
 	// the authentication options
-	vaultAuthOptions map[string]string
+	vaultAuthOptions *vaultAuthOptions
 	// the vault ca file
 	vaultCaFile string
 	// the place to write the resources
@@ -37,13 +55,17 @@ type config struct {
 	// switch on dry run
 	dryRun bool
 	// skip tls verify
-	tlsVerify bool
+	skipTLSVerify bool
 	// the resource items to retrieve
 	resources *VaultResources
 	// the interval for producing statistics
 	statsInterval time.Duration
 	// the timeout for a exec command
 	execTimeout time.Duration
+	// version flag
+	showVersion bool
+	// one-shot mode
+	oneShot bool
 }
 
 var (
@@ -53,42 +75,60 @@ var (
 func init() {
 	// step: setup some defaults
 	options.resources = new(VaultResources)
-	options.vaultAuthOptions = map[string]string{VaultAuth: "token"}
+	options.vaultAuthOptions = &vaultAuthOptions{
+		Method: "token",
+	}
 
 	flag.StringVar(&options.vaultURL, "vault", getEnv("VAULT_ADDR", "https://127.0.0.1:8200"), "url the vault service or VAULT_ADDR")
-	flag.StringVar(&options.vaultAuthFile, "auth", "", "a configuration file in json or yaml containing authentication arguments")
+	flag.StringVar(&options.vaultAuthFile, "auth", getEnv("AUTH_FILE", ""), "a configuration file in json or yaml containing authentication arguments")
+	flag.StringVar(&options.vaultAuthFileFormat, "format", getEnv("AUTH_FORMAT", "default"), "the auth file format")
+	flag.StringVar(&options.vaultAuthOptions.Method, "method", getEnv("AUTH_METHOD", "token"), "the authentication method to use (use of an auth file will override this setting)")
 	flag.StringVar(&options.outputDir, "output", getEnv("VAULT_OUTPUT", "/etc/secrets"), "the full path to write resources or VAULT_OUTPUT")
 	flag.BoolVar(&options.dryRun, "dryrun", false, "perform a dry run, printing the content to screen")
-	flag.BoolVar(&options.tlsVerify, "tls-skip-verify", false, "whether to check and verify the vault service certificate")
+	flag.BoolVar(&options.skipTLSVerify, "tls-skip-verify", false, "whether to check and verify the vault service certificate")
 	flag.StringVar(&options.vaultCaFile, "ca-cert", "", "the path to the file container the CA used to verify the vault service")
 	flag.DurationVar(&options.statsInterval, "stats", time.Duration(1)*time.Hour, "the interval to produce statistics on the accessed resources")
 	flag.DurationVar(&options.execTimeout, "exec-timeout", time.Duration(60)*time.Second, "the timeout applied to commands on the exec option")
+	flag.BoolVar(&options.showVersion, "version", false, "show the vault-sidekick version")
 	flag.Var(options.resources, "cn", "a resource to retrieve and monitor from vault")
+	flag.BoolVar(&options.oneShot, "one-shot", false, "retrieve resources from vault once and then exit")
 }
 
 // parseOptions validate the command line options and validates them
 func parseOptions() error {
 	flag.Parse()
-
 	return validateOptions(&options)
 }
 
 // validateOptions parses and validates the command line options
 func validateOptions(cfg *config) (err error) {
-	// step: validate the vault url
-	if _, err = url.Parse(cfg.vaultURL); err != nil {
-		return fmt.Errorf("invalid vault url: '%s' specified", cfg.vaultURL)
-	}
-
 	// step: read in the token if required
+
 	if cfg.vaultAuthFile != "" {
 		if exists, _ := fileExists(cfg.vaultAuthFile); !exists {
 			return fmt.Errorf("the token file: %s does not exists, please check", cfg.vaultAuthFile)
 		}
-		options.vaultAuthOptions, err = readConfigFile(options.vaultAuthFile)
+
+		cfg.vaultAuthOptions, err = readConfigFile(cfg.vaultAuthFile, cfg.vaultAuthFileFormat)
 		if err != nil {
 			return fmt.Errorf("unable to read in authentication options from: %s, error: %s", cfg.vaultAuthFile, err)
 		}
+		if cfg.vaultAuthOptions.VaultURL != "" {
+			cfg.vaultURL = cfg.vaultAuthOptions.VaultURL
+		}
+	}
+
+	if cfg.vaultURL == "" {
+		cfg.vaultURL = os.Getenv("VAULT_ADDR")
+	}
+
+	if cfg.vaultURL == "" {
+		return fmt.Errorf("VAULT_ADDR is unset")
+	}
+
+	// step: validate the vault url
+	if _, err = url.Parse(cfg.vaultURL); err != nil {
+		return fmt.Errorf("invalid vault url: '%s' specified", cfg.vaultURL)
 	}
 
 	if cfg.vaultCaFile != "" {
@@ -97,7 +137,7 @@ func validateOptions(cfg *config) (err error) {
 		}
 	}
 
-	if cfg.tlsVerify == true && cfg.vaultCaFile != "" {
+	if cfg.skipTLSVerify == true && cfg.vaultCaFile != "" {
 		return fmt.Errorf("you are skipping the tls but supplying a CA, doesn't make sense")
 	}
 

config_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"os"
+	"testing"
+)
+
+func TestValidateOptionsWithoutVaultURL(t *testing.T) {
+	os.Setenv("VAULT_ADDR", "")
+
+	cfg := &config{}
+	err := validateOptions(cfg)
+
+	if err == nil {
+		t.Errorf("should have raised error: %v", err)
+	}
+
+}
+
+func TestValidateOptionsWithEnvFallback(t *testing.T) {
+	os.Setenv("VAULT_ADDR", "http://testurl:8080")
+
+	cfg := &config{}
+	err := validateOptions(cfg)
+
+	if err != nil {
+		t.Errorf("raised an error: %v", err)
+	}
+
+	actual := cfg.vaultURL
+	expected := "http://testurl:8080"
+
+	if actual != expected {
+		t.Errorf("Expected Vault URL to be %s got %s", expected, actual)
+	}
+
+}
+
+func TestValidateOptionsWithInvalidVaultURL(t *testing.T) {
+	cfg := &config{
+		vaultURL: "%invalid_url",
+	}
+	err := validateOptions(cfg)
+
+	if err == nil {
+		t.Errorf("should have raised error")
+	}
+}
+
+func TestValidateOptionsWithInvalidVaultURLFromAuthFile(t *testing.T) {
+	cfg := &config{
+		vaultAuthFile: "tests/invalid_kubernetes_vault_auth_file.json",
+	}
+	err := validateOptions(cfg)
+
+	if err == nil {
+		t.Errorf("should have raised error")
+	}
+}
+
+func TestValidateOptionsWithVaultURLFromAuthFile(t *testing.T) {
+	cfg := &config{
+		vaultAuthFile: "tests/kubernetes_vault_auth_file.json",
+	}
+	err := validateOptions(cfg)
+
+	if err != nil {
+		t.Errorf("raising an error %v", err)
+	}
+
+	actual := cfg.vaultURL
+	expected := "http://testurl:8080"
+
+	if actual != expected {
+		t.Errorf("Expected Vault URL to be %s got %s", expected, actual)
+	}
+}

formats.go

@@ -19,68 +19,94 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"strings"
 
 	"github.com/golang/glog"
 	"gopkg.in/yaml.v2"
 )
 
-func writeIniFile(filename string, data map[string]interface{}) error {
+func writeIniFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	var buf bytes.Buffer
 	for key, val := range data {
 		buf.WriteString(fmt.Sprintf("%s = %v\n", key, val))
 	}
 
-	return writeFile(filename, buf.Bytes())
+	return writeFile(filename, buf.Bytes(), mode)
 }
 
-func writeCSVFile(filename string, data map[string]interface{}) error {
+func writeCSVFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	var buf bytes.Buffer
 	for key, val := range data {
 		buf.WriteString(fmt.Sprintf("%s,%v\n", key, val))
 	}
 
-	return writeFile(filename, buf.Bytes())
+	return writeFile(filename, buf.Bytes(), mode)
 }
 
-func writeYAMLFile(filename string, data map[string]interface{}) error {
+func writeYAMLFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	// marshall the content to yaml
 	content, err := yaml.Marshal(data)
 	if err != nil {
 		return err
 	}
 
-	return writeFile(filename, content)
+	return writeFile(filename, content, mode)
 }
 
-func writeEnvFile(filename string, data map[string]interface{}) error {
+func writeEnvFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	var buf bytes.Buffer
 	for key, val := range data {
 		buf.WriteString(fmt.Sprintf("%s=%v\n", strings.ToUpper(key), val))
 	}
 
-	return writeFile(filename, buf.Bytes())
+	return writeFile(filename, buf.Bytes(), mode)
 }
 
-func writeCertificateFile(filename string, data map[string]interface{}) error {
+func getCombinedCaChain(dataValue interface{}) (string, error) {
+	arr, ok := dataValue.([]interface{})
+	if !ok {
+		return "", errors.New("ca_chain not of type array")
+	}
+	caChain := make([]string, len(arr))
+	for i := range arr {
+		caChain[i] = arr[i].(string)
+	}
+	return strings.Join(caChain, "\n"), nil
+}
+
+func writeCertificateFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	files := map[string]string{
 		"certificate": "crt",
 		"issuing_ca":  "ca",
+		"ca_chain":    "ca.pem",
 		"private_key": "key",
 	}
 	for key, suffix := range files {
-		filename := fmt.Sprintf("%s.%s", filename, suffix)
+		name := fmt.Sprintf("%s.%s", filename, suffix)
 		content, found := data[key]
 		if !found {
-			glog.Errorf("didn't find the certification option: %s in the resource: %s", key, filename)
+			glog.Errorf("didn't find the certification option: %s in the resource: %s", key, name)
 			continue
 		}
 
 		// step: write the file
-		if err := writeFile(filename, []byte(fmt.Sprintf("%s", content))); err != nil {
+		var contentToWrite string
-			glog.Errorf("failed to write resource: %s, elemment: %s, filename: %s, error: %s", filename, suffix, filename, err)
+		var err error
+		if key == "ca_chain" {
+			contentToWrite, err = getCombinedCaChain(data[key])
+			if err != nil {
+				glog.Errorf("failed to parse ca_chain: %s", err)
+			}
+		} else {
+			contentToWrite = fmt.Sprintf("%s", content)
+		}
+
+		if err := writeFile(name, []byte(contentToWrite), mode); err != nil {
+			glog.Errorf("failed to write resource: %s, element: %s, filename: %s, error: %s", filename, suffix, name, err)
 			continue
 		}
 	}
@@ -89,33 +115,44 @@ func writeCertificateFile(filename string, data map[string]interface{}) error {
 
 }
 
-func writeCertificateBundleFile(filename string, data map[string]interface{}) error {
+func writeCertificateBundleFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	bundleFile := fmt.Sprintf("%s-bundle.pem", filename)
 	keyFile := fmt.Sprintf("%s-key.pem", filename)
 	caFile := fmt.Sprintf("%s-ca.pem", filename)
+	caChainFile := fmt.Sprintf("%s-ca-chain.pem", filename)
 	certFile := fmt.Sprintf("%s.pem", filename)
 
 	bundle := fmt.Sprintf("%s\n\n%s", data["certificate"], data["issuing_ca"])
 	key := fmt.Sprintf("%s\n", data["private_key"])
 	ca := fmt.Sprintf("%s\n", data["issuing_ca"])
 	certificate := fmt.Sprintf("%s\n", data["certificate"])
+	caChain, err := getCombinedCaChain(data["ca_chain"])
+	if err != nil {
+		glog.Errorf("failed to parse ca_chain: %s", err)
+		return err
+	}
 
-	if err := writeFile(bundleFile, []byte(bundle)); err != nil {
+	if err := writeFile(bundleFile, []byte(bundle), mode); err != nil {
 		glog.Errorf("failed to write the bundled certificate file, error: %s", err)
 		return err
 	}
 
-	if err := writeFile(certFile, []byte(certificate)); err != nil {
+	if err := writeFile(certFile, []byte(certificate), mode); err != nil {
 		glog.Errorf("failed to write the certificate file, errro: %s", err)
 		return err
 	}
 
-	if err := writeFile(caFile, []byte(ca)); err != nil {
+	if err := writeFile(caFile, []byte(ca), mode); err != nil {
 		glog.Errorf("failed to write the ca file, errro: %s", err)
 		return err
 	}
 
-	if err := writeFile(keyFile, []byte(key)); err != nil {
+	if err := writeFile(caChainFile, []byte(caChain), mode); err != nil {
+		glog.Errorf("failed to write the ca_chain file, errro: %s", err)
+		return err
+	}
+
+	if err := writeFile(keyFile, []byte(key), mode); err != nil {
 		glog.Errorf("failed to write the key file, errro: %s", err)
 		return err
 	}
@@ -123,15 +160,60 @@ func writeCertificateBundleFile(filename string, data map[string]interface{}) er
 	return nil
 }
 
-func writeTxtFile(filename string, data map[string]interface{}) error {
+func writeKeyCertificateBundleFile(filename string, data map[string]interface{}, mode os.FileMode) error {
+	bundleFile := fmt.Sprintf("%s-bundle.pem", filename)
+	keyFile := fmt.Sprintf("%s-key.pem", filename)
+	caFile := fmt.Sprintf("%s-ca.pem", filename)
+	caChainFile := fmt.Sprintf("%s-ca-chain.pem", filename)
+	certFile := fmt.Sprintf("%s.pem", filename)
+
+	bundle := fmt.Sprintf("%s\n%s", data["private_key"], data["certificate"])
+	key := fmt.Sprintf("%s\n", data["private_key"])
+	ca := fmt.Sprintf("%s\n", data["issuing_ca"])
+	certificate := fmt.Sprintf("%s\n", data["certificate"])
+	caChain, err := getCombinedCaChain(data["ca_chain"])
+	if err != nil {
+		glog.Errorf("failed to parse ca_chain: %s", err)
+		return err
+	}
+
+	if err := writeFile(bundleFile, []byte(bundle), mode); err != nil {
+		glog.Errorf("failed to write the bundled certificate file, error: %s", err)
+		return err
+	}
+
+	if err := writeFile(certFile, []byte(certificate), mode); err != nil {
+		glog.Errorf("failed to write the certificate file, errro: %s", err)
+		return err
+	}
+
+	if err := writeFile(caFile, []byte(ca), mode); err != nil {
+		glog.Errorf("failed to write the ca file, errro: %s", err)
+		return err
+	}
+
+	if err := writeFile(caChainFile, []byte(caChain), mode); err != nil {
+		glog.Errorf("failed to write the ca_chain file, errro: %s", err)
+		return err
+	}
+
+	if err := writeFile(keyFile, []byte(key), mode); err != nil {
+		glog.Errorf("failed to write the key file, errro: %s", err)
+		return err
+	}
+
+	return nil
+}
+
+func writeTxtFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	keys := getKeys(data)
 	if len(keys) > 1 {
 		// step: for plain formats we need to iterate the keys and produce a file per key
 		for suffix, content := range data {
-			filename := fmt.Sprintf("%s.%s", filename, suffix)
+			name := fmt.Sprintf("%s.%s", filename, suffix)
-			if err := writeFile(filename, []byte(fmt.Sprintf("%v", content))); err != nil {
+			if err := writeFile(name, []byte(fmt.Sprintf("%v", content)), mode); err != nil {
 				glog.Errorf("failed to write resource: %s, elemment: %s, filename: %s, error: %s",
-					filename, suffix, filename, err)
+					filename, suffix, name, err)
 				continue
 			}
 		}
@@ -142,22 +224,20 @@ func writeTxtFile(filename string, data map[string]interface{}) error {
 	value, _ := data[keys[0]]
 	content := []byte(fmt.Sprintf("%s", value))
 
-	return writeFile(filename, content)
+	return writeFile(filename, content, mode)
 }
 
-func writeJSONFile(filename string, data map[string]interface{}) error {
+func writeJSONFile(filename string, data map[string]interface{}, mode os.FileMode) error {
 	content, err := json.MarshalIndent(data, "", "    ")
 	if err != nil {
 		return err
 	}
 
-	return writeFile(filename, content)
+	return writeFile(filename, content, mode)
 }
 
-// writeFile ... writes the content to a file .. dah
+// writeFile writes the file to stdout or an actual file
-//	filename		: the path to the file
+func writeFile(filename string, content []byte, mode os.FileMode) error {
-//	content			: the content to be written
-func writeFile(filename string, content []byte) error {
 	if options.dryRun {
 		glog.Infof("dry-run: filename: %s, content:", filename)
 		fmt.Printf("%s\n", string(content))
@@ -165,5 +245,5 @@ func writeFile(filename string, content []byte) error {
 	}
 	glog.V(3).Infof("saving the file: %s", filename)
 
-	return ioutil.WriteFile(filename, content, 0664)
+	return ioutil.WriteFile(filename, content, mode)
 }

generate.go

@@ -21,32 +21,31 @@ import (
 	"io"
 )
 
-var StdChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()-_=+,.?/:;{}[]`~")
+var stdChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()-_=+,.?/:;{}[]`~")
 
-func NewPassword(length int) string {
+func newPassword(length int) string {
-	return rand_char(length, StdChars)
+	return randString(length, stdChars)
 }
 
-func rand_char(length int, chars []byte) string {
+func randString(length int, chars []byte) string {
-	new_pword := make([]byte, length)
+	pass := make([]byte, length)
-	random_data := make([]byte, length+(length/4)) // storage for random bytes.
+	data := make([]byte, length+(length/4)) // storage for random bytes.
 	clen := byte(len(chars))
 	maxrb := byte(256 - (256 % len(chars)))
 	i := 0
 	for {
-		if _, err := io.ReadFull(rand.Reader, random_data); err != nil {
+		if _, err := io.ReadFull(rand.Reader, data); err != nil {
 			panic(err)
 		}
-		for _, c := range random_data {
+		for _, c := range data {
 			if c >= maxrb {
 				continue
 			}
-			new_pword[i] = chars[c%clen]
+			pass[i] = chars[c%clen]
 			i++
 			if i == length {
-				return string(new_pword)
+				return string(pass)
 			}
 		}
 	}
-	panic("unreachable")
 }

kube/deployment.yaml

@@ -0,0 +1,87 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: vault-sidekick
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        name: vault-sidekick
+      annotations:
+        build: https://github.com/UKHomeOffice/vault-sidekick
+    spec:
+      containers:
+      - name: sidekick
+        image: quay.io/ukhomeofficedigital/vault-sidekick:v0.3.1
+        imagePullPolicy: Always
+        resources:
+          limits:
+            cpu: 100m
+            memory: 50Mi
+        args:
+          - -cn=pki:services/${NAMESPACE}/pki/issue/default:fmt=bundle,common_name=demo.${NAMESPACE}.svc.cluster.local,file=platform,mode=0600
+          - -ca-cert=/ca/caroot.bundle
+          - -logtostderr=true
+          - -v=3
+        env:
+        - name: VAULT_ADDR
+          value: https://vault.vault.svc.cluster.local:8200
+        - name: VAULT_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: store-token
+              key: token
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: secrets
+          mountPath: /etc/secrets
+        - name: ca-bundle
+          mountPath: /ca
+      - name: nginx
+        image: quay.io/ukhomeofficedigital/nginx-proxy:v3.0.0
+        resources:
+          limits:
+            cpu: 400m
+            memory: 256Mi
+        ports:
+        - name: http
+          containerPort: 80
+        - name: https
+          containerPort: 443
+        env:
+        - name: LOAD_BALANCER_CIDR
+          value: 10.0.0.0/8
+        - name: PROXY_SERVICE_HOST
+          value: 127.0.0.1
+        - name: PROXY_SERVICE_PORT
+          value: "8080"
+        - name: SERVER_CERT
+          value: /etc/secrets/platform.pem
+        - name: SERVER_KEY
+          value: /etc/secrets/platform-key.pem
+        - name: SSL_CIPHERS
+          value: ECDHE-RSA-AES128-GCM-SHA256:AES256+EECDH:AES256+EDH:!aNULL
+        - name: ENABLE_UUID_PARAM
+          value: "FALSE"
+        - name: NAXSI_USE_DEFAULT_RULES
+          value: "FALSE"
+        - name: PORT_IN_HOST_HEADER
+          value: "FALSE"
+        - name: ERROR_REDIRECT_CODES
+          value: "599"
+        - name: ADD_NGINX_LOCATION_CFG
+          value: "add_header Strict-Transport-Security \"max-age=31536000; includeSubdomains\";"
+        volumeMounts:
+        - name: secrets
+          mountPath: /etc/secrets
+      volumes:
+      - name: secrets
+        emptyDir: {}
+      - name: ca-bundle
+        secret:
+          secretName: ca-bundle

kube/service.yml

@@ -1,4 +1,3 @@
----
 apiVersion: v1
 kind: Service
 metadata:

main.go

@@ -17,26 +17,36 @@ limitations under the License.
 package main
 
 import (
+	"fmt"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/golang/glog"
 )
 
-const (
+var (
-	Prog    = "vault-sidekick"
+	prog    = "vault-sidekick"
-	Version = "v0.2.1"
+	release = "v0.3.4"
+	gitsha  = ""
 )
 
 func main() {
+	version := fmt.Sprintf("%s (git+sha %s)", release, gitsha)
 	// step: parse and validate the command line / environment options
-
 	if err := parseOptions(); err != nil {
 		showUsage("invalid options, %s", err)
 	}
+	if options.showVersion {
+		fmt.Printf("%s %s\n", prog, version)
+		return
+	}
+	glog.Infof("starting the %s, %s", prog, version)
 
-	glog.Infof("starting the %s, version: %s", Prog, Version)
+	if options.oneShot {
+		glog.Infof("running in one-shot mode")
+	}
 
 	// step: create a client to vault
 	vault, err := NewVaultService(options.vaultURL)
@@ -59,15 +69,51 @@ func main() {
 		vault.Watch(rn)
 	}
 
+	toProcess := options.resources.items
+	toProcessLock := &sync.Mutex{}
+	failedResource := false
+	if options.oneShot && len(toProcess) == 0 {
+		glog.Infof("nothing to retrieve from vault. exiting...")
+		os.Exit(0)
+	}
 	// step: we simply wait for events i.e. secrets from vault and write them to the output directory
 	for {
 		select {
 		case evt := <-updates:
 			glog.V(10).Infof("recieved an update from the resource: %s", evt.Resource)
 			go func(r VaultEvent) {
+				toProcessLock.Lock()
+				defer toProcessLock.Unlock()
+				switch r.Type {
+				case EventTypeSuccess:
 					if err := processResource(evt.Resource, evt.Secret); err != nil {
 						glog.Errorf("failed to write out the update, error: %s", err)
 					}
+					if options.oneShot {
+						for i, r := range toProcess {
+							if evt.Resource == r {
+								toProcess = append(toProcess[:i], toProcess[i+1:]...)
+							}
+						}
+					}
+				case EventTypeFailure:
+					if evt.Resource.maxRetries > 0 && evt.Resource.maxRetries < evt.Resource.retries {
+						for i, r := range toProcess {
+							if evt.Resource == r {
+								toProcess = append(toProcess[:i], toProcess[i+1:]...)
+								failedResource = true
+							}
+						}
+					}
+				}
+				if len(toProcess) == 0 {
+					glog.Infof("no resources left to process. exiting...")
+					if failedResource {
+						os.Exit(1)
+					} else {
+						os.Exit(0)
+					}
+				}
 			}(evt)
 		case <-signalChannel:
 			glog.Infof("recieved a termination signal, shutting down the service")

services/demo-ns.yml

@@ -1,7 +0,0 @@
----
-kind: Namespace
-apiVersion: v1
-metadata:
-  name: demo
-  labels:
-    name: demo

services/demo-rc.yaml

@@ -1,58 +0,0 @@
----
-apiVersion: v1
-kind: ReplicationController
-metadata:
-  namespace: demo
-  name: vault-demo
-spec:
-  replicas: 1
-  selector:
-    name: vault-demo
-  template:
-    metadata:
-      labels:
-        name: vault-demo
-    spec:
-      containers:
-      - name: vault-sidekick
-        image: gambol99/vault-sidekick:0.0.1
-        imagePullPolicy: Always
-        args:
-          - -logtostderr=true
-          - -v=4
-          - -tls-skip-verify=true
-          - -auth=/etc/token/vault-token.yml
-          - -output=/etc/secrets
-          - -cn=secret:db:update=3h,revoke=true
-          - -cn=pki:example-dot-com:cn=demo.example.com,fmt=cert,file=demo.example.com
-          - -vault=https://vault.services.cluster.local:8200
-        volumeMounts:
-          - name: secrets
-            mountPath: /etc/secrets
-          - name: token
-            mountPath: /etc/token
-      - name: nginx-tls-sidekick
-        image: quay.io/ukhomeofficedigital/nginx-tls-sidekick
-        imagePullPolicy: Always
-        args:
-          - ./run.sh
-          - -p
-          - 443:127.0.0.1:80:demo.example.com
-        ports:
-          - containerPort: 443
-        volumeMounts:
-          - name: secrets
-            mountPath: /etc/secrets
-      - name: apache
-        image: fedora/apache
-        ports:
-        - containerPort: 80
-        volumeMounts:
-          - name: secrets
-            mountPath: /etc/secrets
-      volumes:
-        - name: secrets
-          emptyDir: {}
-        - name: token
-          secret:
-            secretName: vault-token

services/demo-secrets.yml

@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Secret
-metadata:
-  namespace: demo
-  name: vault-token
-data:
-  #
-  # vault auth-enable userpass
-  # vault write auth/userpass/users/demo password=SOME_PASSWORD policies=root
-  #
-  vault-token.yml: |
-    method: userpass
-    username: demo
-    password: SOME_PASSWORD

tests/approle_auth_file.json

@@ -0,0 +1 @@
+{"method": "approle", "role_id": "admin", "secret_id": "foobar"}

tests/approle_auth_file.yml

@@ -0,0 +1,3 @@
+method: approle
+role_id: admin
+secret_id: foobar

tests/auth_file.json

@@ -0,0 +1 @@
+{"method": "userpass", "username": "admin", "password": "foobar"}

tests/invalid_kubernetes_vault_auth_file.json

@@ -0,0 +1 @@
+{"renewable": true, "leaseDuration": 60, "vaultAddr": "%invalid_url", "token": "foobar"}

tests/kubernetes_vault_auth_file.json

@@ -0,0 +1 @@
+{"renewable": true, "leaseDuration": 60, "vaultAddr": "http://testurl:8080", "token": "foobar"}

tests/token_auth_file.json

@@ -0,0 +1 @@
+{"method": "token", "token": "foobar"}

tests/token_auth_file.yml

@@ -0,0 +1,2 @@
+method: token
+token: foobar

tests/userpass_auth_file.json

@@ -0,0 +1 @@
+{"method": "userpass", "username": "admin", "password": "foobar"}

tests/userpass_auth_file.yml

@@ -0,0 +1,3 @@
+method: userpass
+username: admin
+password: foobar

utils.go

@@ -27,10 +27,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/golang/glog"
-	"gopkg.in/yaml.v2"
 	"os/exec"
 	"path/filepath"
+
+	"github.com/golang/glog"
+	"gopkg.in/yaml.v2"
 )
 
 func init() {
@@ -69,7 +70,7 @@ func getKeys(data map[string]interface{}) []string {
 
 // readConfigFile read in a configuration file
 //	filename		: the path to the file
-func readConfigFile(filename string) (map[string]string, error) {
+func readConfigFile(filename, fileFormat string) (*vaultAuthOptions, error) {
 	// step: check the file exists
 	if exists, err := fileExists(filename); !exists {
 		return nil, fmt.Errorf("the file: %s does not exist", filename)
@@ -84,43 +85,49 @@ func readConfigFile(filename string) (map[string]string, error) {
 	case ".yml":
 		return readYAMLFile(filename)
 	default:
-		return readJSONFile(filename)
+		return readJSONFile(filename, fileFormat)
 	}
-	return nil, fmt.Errorf("unsupported config file format: %s", suffix)
 }
 
 // readJsonFile read in and unmarshall the data into a map
 //	filename	: the path to the file container the json data
-func readJSONFile(filename string) (map[string]string, error) {
+func readJSONFile(filename, format string) (*vaultAuthOptions, error) {
-	data := make(map[string]string, 0)
+	opts := &vaultAuthOptions{}
 
 	content, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return data, err
+		return nil, err
 	}
 	// unmarshall the data
-	err = json.Unmarshal(content, &data)
+	err = json.Unmarshal(content, &opts)
+	if err != nil && format == "default" {
+		return nil, err
+	}
 	if err != nil {
-		return data, err
+		return nil, err
+	}
+	if format == "kubernetes-vault" && opts.ClientToken != "" {
+		opts.Method = "token"
+		opts.Token = opts.ClientToken
 	}
 
-	return data, nil
+	return opts, nil
 }
 
 // readYAMLFile read in and unmarshall the data into a map
 //	filename	: the path to the file container the yaml data
-func readYAMLFile(filename string) (map[string]string, error) {
+func readYAMLFile(filename string) (*vaultAuthOptions, error) {
-	data := make(map[string]string, 0)
+	o := &vaultAuthOptions{}
 	content, err := ioutil.ReadFile(filename)
 	if err != nil {
-		return data, err
+		return nil, err
 	}
-	err = yaml.Unmarshal(content, data)
+	err = yaml.Unmarshal(content, o)
 	if err != nil {
-		return data, err
+		return nil, err
 	}
 
-	return data, nil
+	return o, nil
 }
 
 // getDurationWithin generate a random integer between min and max
@@ -169,21 +176,23 @@ func processResource(rn *VaultResource, data map[string]interface{}) (err error)
 	case "yaml":
 		fallthrough
 	case "yml":
-		err = writeYAMLFile(filename, data)
+		err = writeYAMLFile(filename, data, rn.fileMode)
 	case "json":
-		err = writeJSONFile(filename, data)
+		err = writeJSONFile(filename, data, rn.fileMode)
 	case "ini":
-		err = writeIniFile(filename, data)
+		err = writeIniFile(filename, data, rn.fileMode)
 	case "csv":
-		err = writeCSVFile(filename, data)
+		err = writeCSVFile(filename, data, rn.fileMode)
 	case "env":
-		err = writeEnvFile(filename, data)
+		err = writeEnvFile(filename, data, rn.fileMode)
 	case "cert":
-		err = writeCertificateFile(filename, data)
+		err = writeCertificateFile(filename, data, rn.fileMode)
 	case "txt":
-		err = writeTxtFile(filename, data)
+		err = writeTxtFile(filename, data, rn.fileMode)
 	case "bundle":
-		err = writeCertificateBundleFile(filename, data)
+		err = writeCertificateBundleFile(filename, data, rn.fileMode)
+	case "key-cert-bundle":
+		err = writeKeyCertificateBundleFile(filename, data, rn.fileMode)
 	default:
 		return fmt.Errorf("unknown output format: %s", rn.format)
 	}
@@ -198,7 +207,7 @@ func processResource(rn *VaultResource, data map[string]interface{}) (err error)
 		cmd := exec.Command(rn.execPath, filename)
 		cmd.Start()
 		timer := time.AfterFunc(options.execTimeout, func() {
-			if err := cmd.Process.Kill(); err != nil {
+			if err = cmd.Process.Kill(); err != nil {
 				glog.Errorf("failed to kill the command, pid: %d, error: %s", cmd.Process.Pid, err)
 			}
 		})

utils_test.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestReadConfigFileKubernetesVault(t *testing.T) {
+	o, err := readConfigFile("tests/kubernetes_vault_auth_file.json", "kubernetes-vault")
+	if err != nil {
+		t.Errorf("raising an error: %v", err)
+	}
+
+	tokenExpected := "foobar"
+
+	if o.Token != tokenExpected {
+		t.Errorf("Expected user %s got %s", tokenExpected, o.Token)
+	}
+}
+
+func TestReadConfigUserPassJSON(t *testing.T) {
+	o, err := readConfigFile("tests/userpass_auth_file.json", "default")
+	if err != nil {
+		t.Errorf("raising an error: %v", err)
+	}
+
+	userExpected := "admin"
+	passwordExpected := "foobar"
+
+	if o.Username != userExpected {
+		t.Errorf("Expected user %s got %s", userExpected, o.Username)
+	}
+
+	if o.Password != passwordExpected {
+		t.Errorf("Expected user %s got %s", passwordExpected, o.Password)
+	}
+}
+
+func TestReadConfigUserPassYAML(t *testing.T) {
+	o, err := readConfigFile("tests/userpass_auth_file.yml", "default")
+	if err != nil {
+		t.Errorf("raising an error: %v", err)
+	}
+
+	userExpected := "admin"
+	passwordExpected := "foobar"
+
+	if o.Username != userExpected {
+		t.Errorf("Expected user %s got %s", userExpected, o.Username)
+	}
+
+	if o.Password != passwordExpected {
+		t.Errorf("Expected user %s got %s", passwordExpected, o.Password)
+	}
+}
+
+func TestReadConfigAppRoleJSON(t *testing.T) {
+	o, err := readConfigFile("tests/approle_auth_file.json", "default")
+	if err != nil {
+		t.Errorf("raising an error: %v", err)
+	}
+
+	roleIDExpected := "admin"
+	secretIDExpected := "foobar"
+
+	if o.RoleID != roleIDExpected {
+		t.Errorf("Expected roleID %s got %s", roleIDExpected, o.RoleID)
+	}
+
+	if o.SecretID != secretIDExpected {
+		t.Errorf("Expected secretID %s got %s", secretIDExpected, o.SecretID)
+	}
+}
+
+func TestReadConfigAppRoleYAML(t *testing.T) {
+	o, err := readConfigFile("tests/approle_auth_file.yml", "default")
+	if err != nil {
+		t.Errorf("raising an error: %v", err)
+	}
+
+	roleIDExpected := "admin"
+	secretIDExpected := "foobar"
+
+	if o.RoleID != roleIDExpected {
+		t.Errorf("Expected roleID %s got %s", roleIDExpected, o.RoleID)
+	}
+
+	if o.SecretID != secretIDExpected {
+		t.Errorf("Expected secretID %s got %s", secretIDExpected, o.SecretID)
+	}
+}
+func TestReadConfigTokenJSON(t *testing.T) {
+	o, err := readConfigFile("tests/token_auth_file.json", "default")
+	if err != nil {
+		t.Errorf("raising an error: %v", err)
+	}
+
+	expected := "foobar"
+
+	if o.Token != expected {
+		t.Errorf("Expected token %s got %s", expected, o.Token)
+	}
+}
+
+func TestReadConfigTokenYAML(t *testing.T) {
+	o, err := readConfigFile("tests/token_auth_file.yml", "default")
+	if err != nil {
+		t.Errorf("raising an error: %v", err)
+	}
+
+	expected := "foobar"
+
+	if o.Token != expected {
+		t.Errorf("Expected token %s got %s", expected, o.Token)
+	}
+}

vault.go

@@ -31,15 +31,10 @@ import (
 	"github.com/hashicorp/vault/api"
 )
 
-const (
-	// VaultAuth the method to use when authenticating to vault
-	VaultAuth = "method"
-)
-
 // AuthInterface is the authentication interface
 type AuthInterface interface {
 	// Create and handle renewals of the token
-	Create(map[string]string) (string, error)
+	Create(*vaultAuthOptions) (string, error)
 }
 
 // VaultService is the main interface into the vault API - placing into a structure
@@ -64,8 +59,17 @@ type VaultEvent struct {
 	Resource *VaultResource
 	// the secret associated
 	Secret map[string]interface{}
+	// type of this event (success or failure)
+	Type EventType
 }
 
+type EventType int
+
+const (
+	EventTypeSuccess EventType = iota
+	EventTypeFailure EventType = iota
+)
+
 // NewVaultService creates a new implementation to speak to vault and retrieve the resources
 //	url			: the url of the vault service
 func NewVaultService(url string) (*VaultService, error) {
@@ -132,6 +136,12 @@ func (r *VaultService) vaultServiceProcessor() {
 			//  - if we error attempting to retrieve the secret, we background and reschedule an attempt to add it
 			//  - if ok, we grab the lease it and lease time, we setup a notification on renewal
 			case x := <-retrieveChannel:
+				// step: skip this resource if it's reached maxRetries
+				if x.resource.maxRetries > 0 && x.resource.retries > x.resource.maxRetries {
+					glog.V(4).Infof("skipping resource %s as it's failed %d/%d times", x.resource.retries, x.resource.maxRetries+1)
+					break
+				}
+
 				// step: save the current lease if we have one
 				leaseID := ""
 				if x.secret != nil && x.secret.LeaseID != "" {
@@ -144,10 +154,16 @@ func (r *VaultService) vaultServiceProcessor() {
 					glog.Errorf("failed to retrieve the resource: %s from vault, error: %s", x.resource, err)
 					// reschedule the attempt for later
 					r.scheduleIn(x, retrieveChannel, getDurationWithin(3, 10))
+					x.resource.retries++
+					r.upstream(VaultEvent{
+						Resource: x.resource,
+						Type:     EventTypeFailure,
+					})
 					break
 				}
 
 				glog.V(4).Infof("successfully retrieved resource: %s, leaseID: %s", x.resource, x.secret.LeaseID)
+				x.resource.retries = 0
 
 				// step: if we had a previous lease and the option is to revoke, lets throw into the revoke channel
 				if leaseID != "" && x.resource.revoked {
@@ -165,13 +181,22 @@ func (r *VaultService) vaultServiceProcessor() {
 				x.notifyOnRenewal(renewChannel)
 
 				// step: update the upstream consumers
-				r.upstream(x)
+				r.upstream(VaultEvent{
+					Resource: x.resource,
+					Secret:   x.secret.Data,
+					Type:     EventTypeSuccess,
+				})
 
 			// A watched resource is coming up for renewal
 			// 	- we attempt to renew the resource from vault
 			//	- if we encounter an error, we reschedule the attempt for the future
 			//	- if we're ok, we update the watchedResource and we send a notification of the change upstream
 			case x := <-renewChannel:
+				// step: skip this resource if it's reached maxRetries
+				if x.resource.maxRetries > 0 && x.resource.retries > x.resource.maxRetries {
+					glog.V(4).Infof("skipping resource %s as it's failed %d/%d times", x.resource.retries, x.resource.maxRetries+1)
+					break
+				}
 
 				glog.V(4).Infof("resource: %s, lease: %s up for renewal, renewable: %t, revoked: %t", x.resource,
 					x.secret.LeaseID, x.resource.renewable, x.resource.revoked)
@@ -196,11 +221,19 @@ func (r *VaultService) vaultServiceProcessor() {
 					// step: lets renew the resource
 					err := r.renew(x)
 					if err != nil {
-						glog.Errorf("failed to renew the resounce: %s for renewal, error: %s", x.resource, err)
+						glog.Errorf("failed to renew the resource: %s for renewal, error: %s", x.resource, err)
 						// reschedule the attempt for later
 						r.scheduleIn(x, renewChannel, getDurationWithin(3, 10))
+						x.resource.retries++
+						r.upstream(VaultEvent{
+							Resource: x.resource,
+							Type:     EventTypeFailure,
+						})
 						break
 					}
+
+					glog.V(4).Infof("successfully renewed resource: %s, leaseID: %s", x.resource, x.secret.LeaseID)
+					x.resource.retries = 0
 				}
 
 				// step: the option for this resource is not to renew the secret but regenerate a new secret
@@ -214,7 +247,11 @@ func (r *VaultService) vaultServiceProcessor() {
 				x.notifyOnRenewal(renewChannel)
 
 				// step: update any listener upstream
-				r.upstream(x)
+				r.upstream(VaultEvent{
+					Resource: x.resource,
+					Secret:   x.secret.Data,
+					Type:     EventTypeSuccess,
+				})
 
 			// We receive a lease ID along on the channel, just revoke the lease when you can
 			case x := <-revokeChannel:
@@ -260,14 +297,11 @@ func (r VaultService) scheduleIn(rn *watchedResource, ch chan *watchedResource,
 
 // upstream ... the resource has changed thus we notify the upstream listener
 //	item		: the item which has changed
-func (r VaultService) upstream(item *watchedResource) {
+func (r VaultService) upstream(item VaultEvent) {
 	// step: chunk this into a go-routine not to block us
 	for _, listener := range r.listeners {
 		go func(ch chan VaultEvent) {
-			ch <- VaultEvent{
+			ch <- item
-				Resource: item.resource,
-				Secret:   item.secret.Data,
-			}
 		}(listener)
 	}
 }
@@ -312,7 +346,8 @@ func (r VaultService) revoke(lease string) error {
 
 // get retrieves a secret from the vault
 //	rn			: the watched resource
-func (r VaultService) get(rn *watchedResource) (err error) {
+func (r VaultService) get(rn *watchedResource) error {
+	var err error
 	var secret *api.Secret
 	// step: not sure who to cast map[string]string to map[string]interface{} doesn't like it anyway i try and do it
 
@@ -320,7 +355,7 @@ func (r VaultService) get(rn *watchedResource) (err error) {
 	for k, v := range rn.resource.options {
 		params[k] = interface{}(v)
 	}
-	glog.V(10).Infof("get, resource: %s, path: %s, params: %v", rn.resource.resource, rn.resource.path, params)
+	glog.V(10).Infof("resource: %s, path: %s, params: %v", rn.resource.resource, rn.resource.path, params)
 
 	glog.V(5).Infof("attempting to retrieve the resource: %s from vault", rn.resource)
 	// step: perform a request to vault
@@ -332,7 +367,6 @@ func (r VaultService) get(rn *watchedResource) (err error) {
 		}
 		resp, err := r.client.RawRequest(request)
 		if err != nil {
-			fmt.Printf("FAILED HERE")
 			return err
 		}
 		// step: read the response
@@ -370,7 +404,7 @@ func (r VaultService) get(rn *watchedResource) (err error) {
 		// We must generate the secret if we have the create flag
 		if rn.resource.create && secret == nil && err == nil {
 			glog.V(3).Infof("Create param specified, creating resource: %s", rn.resource.path)
-			params["value"] = NewPassword(int(rn.resource.size))
+			params["value"] = newPassword(int(rn.resource.size))
 			secret, err = r.client.Logical().Write(fmt.Sprintf(rn.resource.path), params)
 			glog.V(3).Infof("Secret created: %s", rn.resource.path)
 			if err == nil {
@@ -425,14 +459,17 @@ func newVaultClient(opts *config) (*api.Client, error) {
 		return nil, err
 	}
 
-	plugin, _ := opts.vaultAuthOptions[VaultAuth]
+	plugin := opts.vaultAuthOptions.Method
 	switch plugin {
 	case "userpass":
 		token, err = NewUserPassPlugin(client).Create(opts.vaultAuthOptions)
+	case "kubernetes":
+		token, err = NewKubernetesPlugin(client).Create(opts.vaultAuthOptions)
 	case "approle":
 		token, err = NewAppRolePlugin(client).Create(opts.vaultAuthOptions)
 	case "token":
-		opts.vaultAuthOptions["filename"] = options.vaultAuthFile
+		opts.vaultAuthOptions.FileName = options.vaultAuthFile
+		opts.vaultAuthOptions.FileFormat = options.vaultAuthFileFormat
 		token, err = NewUserTokenPlugin(client).Create(opts.vaultAuthOptions)
 	default:
 		return nil, fmt.Errorf("unsupported authentication plugin: %s", plugin)
@@ -457,23 +494,22 @@ func buildHTTPTransport(opts *config) (*http.Transport, error) {
 			KeepAlive: 10 * time.Second,
 		}).Dial,
 		TLSHandshakeTimeout: 10 * time.Second,
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: opts.skipTLSVerify,
+		},
 	}
-	// step: are we skip the tls verify?
+	if opts.skipTLSVerify {
-	if options.tlsVerify {
+		glog.Warning("skipping TLS verification is not recommended")
-		transport.TLSClientConfig = &tls.Config{
-			InsecureSkipVerify: true,
-		}
 	}
 	// step: are we loading a CA file
 	if opts.vaultCaFile != "" {
-		// step: load the ca file
+		glog.V(3).Infof("loading the ca certificate: %s", opts.vaultCaFile)
 		caCert, err := ioutil.ReadFile(opts.vaultCaFile)
 		if err != nil {
 			return nil, fmt.Errorf("unable to read in the ca: %s, reason: %s", opts.vaultCaFile, err)
 		}
 		caCertPool := x509.NewCertPool()
 		caCertPool.AppendCertsFromPEM(caCert)
-		// step: add the ca to the root
 		transport.TLSClientConfig.RootCAs = caCertPool
 	}
 

vault_resource.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"regexp"
 	"time"
 )
@@ -43,12 +44,20 @@ const (
 	optionCreate = "create"
 	// optionSize sets the initial size of a password secret
 	optionSize = "size"
+	// optionsMode is the file permissions on the secret
+	optionMode = "mode"
+	// optionMaxRetries is the maximum number of retries that should be attempted
+	optionMaxRetries = "retries"
+	// optionMaxJitter is the maximum amount of jitter that should be applied
+	// to updates for this resource. If non-zero, a random value between 0 and
+	// maxJitter will be subtracted from the update period.
+	optionMaxJitter = "jitter"
 	// defaultSize sets the default size of a generic secret
 	defaultSize = 20
 )
 
 var (
-	resourceFormatRegex = regexp.MustCompile("^(yaml|yml|json|env|ini|txt|cert|bundle|csv)$")
+	resourceFormatRegex = regexp.MustCompile("^(yaml|yml|json|env|ini|txt|cert|bundle|key-cert-bundle|csv)$")
 
 	// a map of valid resource to retrieve from vault
 	validResources = map[string]bool{
@@ -67,10 +76,11 @@ var (
 
 func defaultVaultResource() *VaultResource {
 	return &VaultResource{
+		fileMode:  os.FileMode(0664),
 		format:    "yaml",
+		options:   make(map[string]string, 0),
 		renewable: false,
 		revoked:   false,
-		options:   make(map[string]string, 0),
 		size:      defaultSize,
 	}
 }
@@ -103,6 +113,17 @@ type VaultResource struct {
 	execPath string
 	// additional options to the resource
 	options map[string]string
+	// the file permissions on the resource
+	fileMode os.FileMode
+	// maxRetries is the maximum number of times this resource should be
+	// attempted to be retrieved from Vault before failing
+	maxRetries int
+	// retries is the number of times this resource has been retried since it
+	// last succeeded
+	retries int
+	// maxJitter is the maximum jitter duration to use for this resource when
+	// performing renewals
+	maxJitter time.Duration
 }
 
 // GetFilename generates a resource filename by default the resource name and resource type, which
@@ -152,5 +173,9 @@ func (r *VaultResource) isValidResource() error {
 
 // String returns a string representation of the struct
 func (r VaultResource) String() string {
-	return fmt.Sprintf("type: %s, path:%s", r.resource, r.path)
+	str := fmt.Sprintf("type: %s, path: %s", r.resource, r.path)
+	if r.maxRetries > 0 {
+		str = fmt.Sprintf("%s, attempts: %d/%d", str, r.retries, r.maxRetries+1)
+	}
+	return str
 }

vault_resources.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"strconv"
@@ -63,11 +64,23 @@ func (r *VaultResources) Set(value string) error {
 				return fmt.Errorf("invalid resource option: %s, must have a value", x)
 			}
 			// step: set the name and value
-			name := kp[0]
+			name := strings.TrimSpace(kp[0])
 			value := strings.Replace(kp[1], "|", ",", -1)
 
 			// step: extract the control options from the path resource parameters
 			switch name {
+			case optionMode:
+				if !strings.HasPrefix(value, "0") {
+					value = "0" + value
+				}
+				if len(value) != 4 {
+					return errors.New("the file permission invalid, should be octal 0444 or alike")
+				}
+				v, err := strconv.ParseUint(value, 0, 32)
+				if err != nil {
+					return errors.New("invalid file permissions on resource")
+				}
+				rn.fileMode = os.FileMode(v)
 			case optionFormat:
 				if matched := resourceFormatRegex.MatchString(value); !matched {
 					return fmt.Errorf("unsupported output format: %s", value)
@@ -118,6 +131,18 @@ func (r *VaultResources) Set(value string) error {
 				rn.filename = value
 			case optionTemplatePath:
 				rn.templateFile = value
+			case optionMaxRetries:
+				maxRetries, err := strconv.ParseInt(value, 10, 32)
+				if err != nil {
+					return fmt.Errorf("the retries option: %s is invalid, should be an integer", value)
+				}
+				rn.maxRetries = int(maxRetries)
+			case optionMaxJitter:
+				maxJitter, err := time.ParseDuration(value)
+				if err != nil {
+					return fmt.Errorf("the jitter option: %s is invalid, should be in duration format", value)
+				}
+				rn.maxJitter = maxJitter
 			default:
 				rn.options[name] = value
 			}

watched_resource.go

@@ -19,7 +19,6 @@ package main
 import (
 	"time"
 
-	"fmt"
 	"github.com/golang/glog"
 	"github.com/hashicorp/vault/api"
 )
@@ -51,8 +50,19 @@ func (r *watchedResource) notifyOnRenewal(ch chan *watchedResource) {
 		r.renewalTime = r.resource.update
 		// step: if the answer is no, we set the notification between 80-95% of the lease time of the secret
 		if r.renewalTime <= 0 {
+			// if there is no lease time, we canout set a renewal, just fade into the background
+			if r.secret.LeaseDuration <= 0 {
+				glog.Warningf("resource: %s has no lease duration, no custom update set, so item will not be updated", r.resource.path)
+				return
+			}
 			r.renewalTime = r.calculateRenewal()
-			fmt.Printf("seconds: %s", r.renewalTime)
+		}
+		if r.resource.maxJitter != 0 {
+			glog.V(4).Infof("using maxJitter (%s) to calculate renewal time", r.resource.maxJitter)
+			r.renewalTime = time.Duration(getDurationWithin(
+				int((r.renewalTime-r.resource.maxJitter)/time.Second),
+				int(r.renewalTime/time.Second),
+			))
 		}
 		glog.V(3).Infof("setting a renewal notification on resource: %s, time: %s", r.resource, r.renewalTime)
 		// step: wait for the duration