<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io

import (
	"errors"
	"io"
	"sync"
	"time"

	"github.com/tsuru/tsuru/log"
)

type keepAliveWriter struct {
	w         io.Writer
	interval  time.Duration
	ping      chan bool
	done      chan bool
	msg       []byte
	lastByte  byte
	withError bool
	writeLock sync.Mutex
}

func NewKeepAliveWriter(w io.Writer, interval time.Duration, msg string) *keepAliveWriter {
	writer := &keepAliveWriter{w: w, interval: interval, msg: append([]byte(msg), '\n')}
	writer.ping = make(chan bool)
	writer.done = make(chan bool)
	go writer.keepAlive()
	return writer
}

func (w *keepAliveWriter) writeInterval() {
	w.writeLock.Lock()
	defer w.writeLock.Unlock()
	msg := []byte{}
	if w.lastByte != '\n' {
		msg = []byte("\n")
	}
	msg = append(msg, w.msg...)
	numBytes, err := w.w.Write(msg)
	if err != nil {
		log.Debugf("Error writing keepalive, exiting loop: %s", err.Error())
		w.withError = true
		return
	}
	if numBytes != len(msg) {
		log.Debugf("Short write on keepalive, exiting loop.")
		w.withError = true
		return
	}
}

func (w *keepAliveWriter) keepAlive() {
	for {
		select {
		case <-w.ping:
		case <-w.done:
			return
		case <-time.After(w.interval):
			if w.writeInterval(); w.withError {
				return
			}
		}
	}
}

func (w *keepAliveWriter) Write(b []byte) (int, error) {
	if len(b) == 0 {
		return 0, nil
	}
	w.writeLock.Lock()
	defer w.writeLock.Unlock()
	if w.withError {
		return 0, errors.New("Error in previous write.")
	}
	if w.ping != nil {
		w.ping <- true
	}
	w.lastByte = b[len(b)-1]
	written, err := w.w.Write(b)
	if err != nil {
		if w.done != nil {
			close(w.done)
			w.done = nil
		}
		if w.ping != nil {
			close(w.ping)
			w.ping = nil
		}
	}
	return written, err
}
<commit_msg>io: fix for race conditions in keepalivewriter<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io

import (
	"io"
	"sync"
	"time"

	"github.com/tsuru/tsuru/log"
)

type keepAliveWriter struct {
	w         io.Writer
	interval  time.Duration
	ping      chan bool
	done      chan bool
	msg       []byte
	lastByte  byte
	running   bool
	writeLock sync.Mutex
}

func NewKeepAliveWriter(w io.Writer, interval time.Duration, msg string) *keepAliveWriter {
	writer := &keepAliveWriter{w: w, interval: interval, msg: append([]byte(msg), '\n')}
	writer.ping = make(chan bool)
	writer.done = make(chan bool)
	writer.running = true
	go writer.keepAlive()
	return writer
}

func (w *keepAliveWriter) writeInterval() {
	w.writeLock.Lock()
	defer w.writeLock.Unlock()
	msg := []byte{}
	if w.lastByte != '\n' {
		msg = []byte("\n")
	}
	msg = append(msg, w.msg...)
	numBytes, err := w.w.Write(msg)
	if err != nil {
		log.Debugf("Error writing keepalive, exiting loop: %s", err.Error())
		w.stop()
	} else if numBytes != len(msg) {
		log.Debugf("Short write on keepalive, exiting loop.")
		w.stop()
	}
}

func (w *keepAliveWriter) stop() {
	if !w.running {
		return
	}
	w.running = false
	close(w.done)
	close(w.ping)
}

func (w *keepAliveWriter) keepAlive() {
	for {
		select {
		case <-w.ping:
		case <-w.done:
			return
		case <-time.After(w.interval):
			w.writeInterval()
		}
	}
}

func (w *keepAliveWriter) Write(b []byte) (int, error) {
	if len(b) == 0 {
		return 0, nil
	}
	w.writeLock.Lock()
	defer w.writeLock.Unlock()
	if w.running {
		w.ping <- true
	}
	w.lastByte = b[len(b)-1]
	written, err := w.w.Write(b)
	if err != nil {
		w.stop()
	}
	return written, err
}
<|endoftext|>


<commit_before>package storage_test

import (
	"context"
	"testing"
	"time"

	"github.com/matrix-org/gomatrixserverlib"
	"github.com/stretchr/testify/assert"

	"github.com/matrix-org/dendrite/federationapi/storage"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/test"
	"github.com/matrix-org/dendrite/test/testrig"
)

func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
	b, baseClose := testrig.CreateBaseDendrite(t, dbType)
	connStr, dbClose := test.PrepareDBConnectionString(t, dbType)
	db, err := storage.NewDatabase(b, &config.DatabaseOptions{
		ConnectionString: config.DataSource(connStr),
	}, b.Caches, b.Cfg.Global.ServerName)
	if err != nil {
		t.Fatalf("NewDatabase returned %s", err)
	}
	return db, func() {
		dbClose()
		baseClose()
	}
}

func TestExpireEDUs(t *testing.T) {
	var expireEDUTypes = map[string]time.Duration{
		gomatrixserverlib.MReceipt: time.Millisecond,
	}

	ctx := context.Background()
	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
		db, close := mustCreateFederationDatabase(t, dbType)
		defer close()
		// insert some data
		for i := 0; i < 100; i++ {
			receipt, err := db.StoreJSON(ctx, "{}")
			assert.NoError(t, err)

			err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MReceipt, expireEDUTypes)
			assert.NoError(t, err)
		}
		// add data without expiry
		receipt, err := db.StoreJSON(ctx, "{}")
		assert.NoError(t, err)

		// m.read_marker gets the default expiry of 24h, so won't be deleted further down in this test
		err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, "m.read_marker", expireEDUTypes)
		assert.NoError(t, err)

		// Delete expired EDUs
		err = db.DeleteExpiredEDUs(ctx)
		assert.NoError(t, err)

		// verify the data is gone
		data, err := db.GetPendingEDUs(ctx, "localhost", 100)
		assert.NoError(t, err)
		assert.Equal(t, 1, len(data))

		// check that m.direct_to_device is never expired
		receipt, err = db.StoreJSON(ctx, "{}")
		assert.NoError(t, err)

		err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MDirectToDevice, expireEDUTypes)
		assert.NoError(t, err)

		err = db.DeleteExpiredEDUs(ctx)
		assert.NoError(t, err)

		// We should get two EDUs, the m.read_marker and the m.direct_to_device
		data, err = db.GetPendingEDUs(ctx, "localhost", 100)
		assert.NoError(t, err)
		assert.Equal(t, 2, len(data))
	})
}
<commit_msg>De-race `TestExpireEDUs` (#2654)<commit_after>package storage_test

import (
	"context"
	"testing"
	"time"

	"github.com/matrix-org/gomatrixserverlib"
	"github.com/stretchr/testify/assert"

	"github.com/matrix-org/dendrite/federationapi/storage"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/dendrite/test"
	"github.com/matrix-org/dendrite/test/testrig"
)

func mustCreateFederationDatabase(t *testing.T, dbType test.DBType) (storage.Database, func()) {
	b, baseClose := testrig.CreateBaseDendrite(t, dbType)
	connStr, dbClose := test.PrepareDBConnectionString(t, dbType)
	db, err := storage.NewDatabase(b, &config.DatabaseOptions{
		ConnectionString: config.DataSource(connStr),
	}, b.Caches, b.Cfg.Global.ServerName)
	if err != nil {
		t.Fatalf("NewDatabase returned %s", err)
	}
	return db, func() {
		dbClose()
		baseClose()
	}
}

func TestExpireEDUs(t *testing.T) {
	var expireEDUTypes = map[string]time.Duration{
		gomatrixserverlib.MReceipt: 0,
	}

	ctx := context.Background()
	test.WithAllDatabases(t, func(t *testing.T, dbType test.DBType) {
		db, close := mustCreateFederationDatabase(t, dbType)
		defer close()
		// insert some data
		for i := 0; i < 100; i++ {
			receipt, err := db.StoreJSON(ctx, "{}")
			assert.NoError(t, err)

			err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MReceipt, expireEDUTypes)
			assert.NoError(t, err)
		}
		// add data without expiry
		receipt, err := db.StoreJSON(ctx, "{}")
		assert.NoError(t, err)

		// m.read_marker gets the default expiry of 24h, so won't be deleted further down in this test
		err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, "m.read_marker", expireEDUTypes)
		assert.NoError(t, err)

		// Delete expired EDUs
		err = db.DeleteExpiredEDUs(ctx)
		assert.NoError(t, err)

		// verify the data is gone
		data, err := db.GetPendingEDUs(ctx, "localhost", 100)
		assert.NoError(t, err)
		assert.Equal(t, 1, len(data))

		// check that m.direct_to_device is never expired
		receipt, err = db.StoreJSON(ctx, "{}")
		assert.NoError(t, err)

		err = db.AssociateEDUWithDestination(ctx, "localhost", receipt, gomatrixserverlib.MDirectToDevice, expireEDUTypes)
		assert.NoError(t, err)

		err = db.DeleteExpiredEDUs(ctx)
		assert.NoError(t, err)

		// We should get two EDUs, the m.read_marker and the m.direct_to_device
		data, err = db.GetPendingEDUs(ctx, "localhost", 100)
		assert.NoError(t, err)
		assert.Equal(t, 2, len(data))
	})
}
<|endoftext|>


<commit_before>package provision

import (
    "bytes"
    "fmt"
	"io/ioutil"
	"path"
    "strings"
    "text/template"

    "github.com/docker/machine/libmachine/auth"
    "github.com/docker/machine/libmachine/cert"
    "github.com/docker/machine/libmachine/kubernetes"
    "github.com/docker/machine/libmachine/log"
)

func xferCert(p Provisioner, certPath string, targetPath string) error {
	certXferCmd := "printf '%%s' '%s' | sudo tee %s"

	certContents, err := ioutil.ReadFile(certPath)
	if err != nil {
		return err
	}

    /*
     * TODO: Until we start dynamically generating the configuration file,
     * these must have a known naming convention on the machine.
     */
     _, certFile := path.Split(certPath)
     certFile = strings.Split(certFile, "_")[1]

	if _, err := p.SSHCommand(fmt.Sprintf(certXferCmd, string(certContents), path.Join(targetPath, certFile))); err != nil {
		return err
	}

	return nil
}

func fixPermissions(p Provisioner, certPath string, targetPath string) error {
	_, certFile := path.Split(certPath)

    /*
     * TODO: Until we start dynamically generating the configuration file,
     * these must have a known naming convention on the machine.
     */
     certFile = strings.Split(certFile, "_")[1]

	_, err := p.SSHCommand(fmt.Sprintf("sudo chmod 0400 %s", path.Join(targetPath, certFile)))
	if  err != nil {
		return err
	}

	return nil
}

func configureKubernetes(p Provisioner, k8sOptions *kubernetes.KubernetesOptions, authOptions auth.AuthOptions) (error) {
    log.Info("Configuring kubernetes...")

    /* CAB: Test theory that we can force an update by pushing a new manifest */
    if _, err := p.SSHCommand("sudo /bin/sh /usr/local/etc/init.d/kubelet stop"); err != nil {
        log.Info("Errored while attempting to stop the kubelet: %s", err)
    }

    /* Generate and install certificates. Then kick off kubernetes */
	driver := p.GetDriver()
	machine := driver.GetMachineName()
	bits := 2048	// Based on the initial configuration
	ip, err := driver.GetIP()
	if err != nil {
		return fmt.Errorf("Error retrieving address: %s", err)
	}

	err = cert.GenerateCert(
		[]string{ip, "localhost"},
		k8sOptions.K8SAPICert,
		k8sOptions.K8SAPIKey,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		kubernetes.GenOrg(machine, "api"),
		bits)

	if err != nil {
		return fmt.Errorf("Error generating API cert: %s", err)
	}

	err = cert.GenerateCert(
		[]string{""},
		k8sOptions.K8SAdminCert,
		k8sOptions.K8SAdminKey,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		kubernetes.GenOrg(machine, "admin"),
		bits)

	if err != nil {
		return fmt.Errorf("Error generating Admin cert: %s", err)
	}

	err = cert.GenerateCert(
		[]string{},
		k8sOptions.K8SProxyCert,
		k8sOptions.K8SProxyKey,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		kubernetes.GenOrg(machine, "proxy"),
		bits)

	if err != nil {
		return fmt.Errorf("Error generating proxy cert: %s", err)
	}

	/* Copy certs into place */
	log.Info("Copying certs to the remote system...")

	/* CAB: This should probably be an option */
	targetDir := k8sOptions.K8SCertPath

    /* Kick off the kubernetes run */
    if _, err := p.SSHCommand(fmt.Sprintf("printf '%q,%s,%d' |sudo tee %s", k8sOptions.K8SToken, "kuser",0,path.Join(targetDir, "tokenfile.txt"))); err != nil {
        return err
    }

	if err := xferCert(p, k8sOptions.K8SAPIKey, targetDir + "/apiserver"); err != nil {
		return err
	}

	if err := fixPermissions(p, k8sOptions.K8SAPIKey, targetDir + "/apiserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SAPICert, targetDir + "/apiserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SProxyCert, targetDir + "/proxyserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SProxyKey, targetDir + "/proxyserver"); err != nil {
		return err
	}

	if err := fixPermissions(p, k8sOptions.K8SProxyKey, targetDir + "/proxyserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SAdminCert, targetDir + "/kubelet"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SAdminKey, targetDir + "/kubelet"); err != nil {
		return err
	}

	if err := fixPermissions(p, k8sOptions.K8SAdminKey, targetDir + "/kubelet"); err != nil {
		return err
	}

	/* Copy the CA cert to a known location */
	if _, err := p.SSHCommand(fmt.Sprintf("sudo cp /home/docker/.docker/ca.pem %s/ca.pem", targetDir)); err != nil {
		return err
	}

    /* Generate and copy a new YAML file to the target */
    configFile, err := Generatek8sManifest(machine, targetDir)
    if err != nil {
        return err
    }

    /* TOOD: The target manifest directory should be a parameter throughout here */
    if _, err := p.SSHCommand(fmt.Sprintf("printf '%%s' '%s' | sudo tee %s", configFile, "/etc/kubernetes/manifests/kubernetes.yaml")); err != nil {
        return err
    }

	/* Lastly, start the kubelet */
    if _, err := p.SSHCommand("sudo /bin/sh /usr/local/etc/init.d/kubelet start"); err != nil {
        return err
    }

    return nil
}

func Generatek8sManifest(name string, targetDir string) (string, error) {
    type ConfigDetails struct {
        ClusterName   string
        CertDir       string
    }

    details := ConfigDetails{name, targetDir}
    var result bytes.Buffer

    k8sConfigTmpl := `apiVersion: v1
kind: Pod
clusters:
  - cluster:
      certificate-authority: {{.CertDir}}/ca.pem
metadata:
  name: {{.ClusterName}}
spec:
  hostNetwork: true
  volumes:
    - name: "certs"
      hostPath:
        path: "{{.CertDir}}"
    - name: "policies"
      hostPath:
        path: "/etc/kubernetes/policies"
  containers:
    - name: "etcd"
      image: "b.gcr.io/kuar/etcd:2.1.1"
      args:
        - "--data-dir=/var/lib/etcd"
        - "--advertise-client-urls=http://127.0.0.1:2379"
        - "--listen-client-urls=http://127.0.0.1:2379"
        - "--listen-peer-urls=http://127.0.0.1:2380"
        - "--name=etcd"
    - name: "controller-manager"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      args:
        - "/hyperkube"
        - "controller-manager"
        - "--master=http://127.0.0.1:8080"
        - "--v=2"
    - name: "apiserver"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      volumeMounts:
        - name: "certs"
          mountPath: "{{.CertDir}}"
          readOnly: true
        - name: "policies"
          mountPath: "/etc/kubernetes/policies"
          readOnly: true
      args:
        - "/hyperkube"
        - "apiserver"
        - "--authorization-mode=AlwaysAllow"
        - "--client-ca-file=/var/run/kubernetes/ca.pem"
        - "--token-auth-file={{.CertDir}}/tokenfile.txt"
        - "--allow-privileged=true"
        - "--service-cluster-ip-range=10.0.20.0/24"
        - "--insecure-bind-address=0.0.0.0"
        - "--insecure-port=8080"
        - "--secure-port=6443"
        - "--etcd-servers=http://127.0.0.1:2379"
        - "--v=2"
    - name: "proxy"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      securityContext:
        privileged: true
      args:
        - "/hyperkube"
        - "proxy"
        - "--master=http://127.0.0.1:8080"
        - "--v=2"
    - name: "scheduler"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      args:
        - "/hyperkube"
        - "scheduler"
        - "--master=http://127.0.0.1:8080"
        - "--v=2"

`
    t, err := template.New("k8sConfig").Parse(k8sConfigTmpl)
    if err != nil {
        return "", err
    }

   err = t.Execute(&result, details)

    return result.String(), err
}<commit_msg>Added tls cert info to apiserver<commit_after>package provision

import (
    "bytes"
    "fmt"
	"io/ioutil"
	"path"
    "strings"
    "text/template"

    "github.com/docker/machine/libmachine/auth"
    "github.com/docker/machine/libmachine/cert"
    "github.com/docker/machine/libmachine/kubernetes"
    "github.com/docker/machine/libmachine/log"
)

func xferCert(p Provisioner, certPath string, targetPath string) error {
	certXferCmd := "printf '%%s' '%s' | sudo tee %s"

	certContents, err := ioutil.ReadFile(certPath)
	if err != nil {
		return err
	}

    /*
     * TODO: Until we start dynamically generating the configuration file,
     * these must have a known naming convention on the machine.
     */
     _, certFile := path.Split(certPath)
     certFile = strings.Split(certFile, "_")[1]

	if _, err := p.SSHCommand(fmt.Sprintf(certXferCmd, string(certContents), path.Join(targetPath, certFile))); err != nil {
		return err
	}

	return nil
}

func fixPermissions(p Provisioner, certPath string, targetPath string) error {
	_, certFile := path.Split(certPath)

    /*
     * TODO: Until we start dynamically generating the configuration file,
     * these must have a known naming convention on the machine.
     */
     certFile = strings.Split(certFile, "_")[1]

	_, err := p.SSHCommand(fmt.Sprintf("sudo chmod 0400 %s", path.Join(targetPath, certFile)))
	if  err != nil {
		return err
	}

	return nil
}

func configureKubernetes(p Provisioner, k8sOptions *kubernetes.KubernetesOptions, authOptions auth.AuthOptions) (error) {
    log.Info("Configuring kubernetes...")

    /* CAB: Test theory that we can force an update by pushing a new manifest */
    if _, err := p.SSHCommand("sudo /bin/sh /usr/local/etc/init.d/kubelet stop"); err != nil {
        log.Info("Errored while attempting to stop the kubelet: %s", err)
    }

    /* Generate and install certificates. Then kick off kubernetes */
	driver := p.GetDriver()
	machine := driver.GetMachineName()
	bits := 2048	// Based on the initial configuration
	ip, err := driver.GetIP()
	if err != nil {
		return fmt.Errorf("Error retrieving address: %s", err)
	}

	err = cert.GenerateCert(
		[]string{ip, "localhost"},
		k8sOptions.K8SAPICert,
		k8sOptions.K8SAPIKey,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		kubernetes.GenOrg(machine, "api"),
		bits)

	if err != nil {
		return fmt.Errorf("Error generating API cert: %s", err)
	}

	err = cert.GenerateCert(
		[]string{""},
		k8sOptions.K8SAdminCert,
		k8sOptions.K8SAdminKey,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		kubernetes.GenOrg(machine, "admin"),
		bits)

	if err != nil {
		return fmt.Errorf("Error generating Admin cert: %s", err)
	}

	err = cert.GenerateCert(
		[]string{},
		k8sOptions.K8SProxyCert,
		k8sOptions.K8SProxyKey,
		authOptions.CaCertPath,
		authOptions.CaPrivateKeyPath,
		kubernetes.GenOrg(machine, "proxy"),
		bits)

	if err != nil {
		return fmt.Errorf("Error generating proxy cert: %s", err)
	}

	/* Copy certs into place */
	log.Info("Copying certs to the remote system...")

	/* CAB: This should probably be an option */
	targetDir := k8sOptions.K8SCertPath

    /* Kick off the kubernetes run */
    if _, err := p.SSHCommand(fmt.Sprintf("printf '%q,%s,%d' |sudo tee %s", k8sOptions.K8SToken, "kuser",0,path.Join(targetDir, "tokenfile.txt"))); err != nil {
        return err
    }

	if err := xferCert(p, k8sOptions.K8SAPIKey, targetDir + "/apiserver"); err != nil {
		return err
	}

	if err := fixPermissions(p, k8sOptions.K8SAPIKey, targetDir + "/apiserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SAPICert, targetDir + "/apiserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SProxyCert, targetDir + "/proxyserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SProxyKey, targetDir + "/proxyserver"); err != nil {
		return err
	}

	if err := fixPermissions(p, k8sOptions.K8SProxyKey, targetDir + "/proxyserver"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SAdminCert, targetDir + "/kubelet"); err != nil {
		return err
	}

	if err := xferCert(p, k8sOptions.K8SAdminKey, targetDir + "/kubelet"); err != nil {
		return err
	}

	if err := fixPermissions(p, k8sOptions.K8SAdminKey, targetDir + "/kubelet"); err != nil {
		return err
	}

	/* Copy the CA cert to a known location */
	if _, err := p.SSHCommand(fmt.Sprintf("sudo cp /home/docker/.docker/ca.pem %s/ca.pem", targetDir)); err != nil {
		return err
	}

    /* Generate and copy a new YAML file to the target */
    configFile, err := Generatek8sManifest(machine, targetDir)
    if err != nil {
        return err
    }

    /* TOOD: The target manifest directory should be a parameter throughout here */
    if _, err := p.SSHCommand(fmt.Sprintf("printf '%%s' '%s' | sudo tee %s", configFile, "/etc/kubernetes/manifests/kubernetes.yaml")); err != nil {
        return err
    }

	/* Lastly, start the kubelet */
    if _, err := p.SSHCommand("sudo /bin/sh /usr/local/etc/init.d/kubelet start"); err != nil {
        return err
    }

    return nil
}

func Generatek8sManifest(name string, targetDir string) (string, error) {
    type ConfigDetails struct {
        ClusterName   string
        CertDir       string
    }

    details := ConfigDetails{name, targetDir}
    var result bytes.Buffer

    k8sConfigTmpl := `apiVersion: v1
kind: Pod
clusters:
  - cluster:
      certificate-authority: {{.CertDir}}/ca.pem
metadata:
  name: {{.ClusterName}}
spec:
  hostNetwork: true
  volumes:
    - name: "certs"
      hostPath:
        path: "{{.CertDir}}"
    - name: "policies"
      hostPath:
        path: "/etc/kubernetes/policies"
  containers:
    - name: "etcd"
      image: "b.gcr.io/kuar/etcd:2.1.1"
      args:
        - "--data-dir=/var/lib/etcd"
        - "--advertise-client-urls=http://127.0.0.1:2379"
        - "--listen-client-urls=http://127.0.0.1:2379"
        - "--listen-peer-urls=http://127.0.0.1:2380"
        - "--name=etcd"
    - name: "controller-manager"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      args:
        - "/hyperkube"
        - "controller-manager"
        - "--master=http://127.0.0.1:8080"
        - "--v=2"
    - name: "apiserver"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      volumeMounts:
        - name: "certs"
          mountPath: "{{.CertDir}}"
          readOnly: true
        - name: "policies"
          mountPath: "/etc/kubernetes/policies"
          readOnly: true
      args:
        - "/hyperkube"
        - "apiserver"
        - "--authorization-mode=AlwaysAllow"
        - "--client-ca-file=/var/run/kubernetes/ca.pem"
        - "--token-auth-file={{.CertDir}}/tokenfile.txt"
        - "--allow-privileged=true"
        - "--service-cluster-ip-range=10.0.20.0/24"
        - "--insecure-bind-address=0.0.0.0"
        - "--insecure-port=8080"
        - "--secure-port=6443"
        - "--etcd-servers=http://127.0.0.1:2379"
        - "--tls-cert-file={{.CertDir}}/apiserver/cert.pem"
        - "--tls-private-key-file={{.CertDir}}/apiserver/key.pem"
        - "--v=2"
    - name: "proxy"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      securityContext:
        privileged: true
      args:
        - "/hyperkube"
        - "proxy"
        - "--master=http://127.0.0.1:8080"
        - "--v=2"
    - name: "scheduler"
      image: "gcr.io/google_containers/hyperkube:v1.0.3"
      args:
        - "/hyperkube"
        - "scheduler"
        - "--master=http://127.0.0.1:8080"
        - "--v=2"

`
    t, err := template.New("k8sConfig").Parse(k8sConfigTmpl)
    if err != nil {
        return "", err
    }

   err = t.Execute(&result, details)

    return result.String(), err
}<|endoftext|>


<commit_before>package steam

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"sync"
	"time"

	"github.com/Philipp15b/go-steam/netutil"
)

// Allows to load initial server list from Steam Directory Web API.
// Call InitializeSteamDirectory() before Connect(), and Connect()
// will use loaded server list instead of hardcoded one
func InitializeSteamDirectory() error {
	return steamDirectoryCache.Initialize()
}

var steamDirectoryCache *steamDirectory = &steamDirectory{}

type steamDirectory struct {
	sync.RWMutex
	servers       []string
	isInitialized bool
}

// Get servers list from steam directory and save them for later use
func (sd *steamDirectory) Initialize() error {
	sd.Lock()
	defer sd.Unlock()
	client := new(http.Client)
	resp, err := client.Get(fmt.Sprintf("https://api.steampowered.com/ISteamDirectory/GetCMList/v1/?cellId=0"))
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	r := struct {
		Response struct {
			ServerList []string
			Result     uint32
			Message    string
		}
	}{}
	if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return err
	}
	if r.Response.Result != 1 {
		return fmt.Errorf("Failed to get steam directory, result: %v, message: %v\n", r.Response.Result, r.Response.Message)
	}
	if len(r.Response.ServerList) == 0 {
		return fmt.Errorf("Steam returned zero servers for steam directory request\n")
	}
	sd.servers = r.Response.ServerList
	sd.isInitialized = true
	return nil
}

func (sd *steamDirectory) GetRandomCM() *netutil.PortAddr {
	sd.RLock()
	defer sd.RUnlock()
	if !sd.isInitialized {
		panic("steam directory is not initialized")
	}
	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
	addr := netutil.ParsePortAddr(sd.servers[rng.Int31n(int32(len(sd.servers)))])
	return addr
}

func (sd *steamDirectory) IsInitialized() bool {
	sd.RLock()
	defer sd.RUnlock()
	isInitialized := sd.isInitialized
	return isInitialized
}
<commit_msg>Fix function descriptions<commit_after>package steam

import (
	"encoding/json"
	"fmt"
	"math/rand"
	"net/http"
	"sync"
	"time"

	"github.com/Philipp15b/go-steam/netutil"
)

// Load initial server list from Steam Directory Web API.
// Call InitializeSteamDirectory() before Connect() to use
// steam directory server list instead of static one.
func InitializeSteamDirectory() error {
	return steamDirectoryCache.Initialize()
}

var steamDirectoryCache *steamDirectory = &steamDirectory{}

type steamDirectory struct {
	sync.RWMutex
	servers       []string
	isInitialized bool
}

// Get server list from steam directory and save it for later
func (sd *steamDirectory) Initialize() error {
	sd.Lock()
	defer sd.Unlock()
	client := new(http.Client)
	resp, err := client.Get(fmt.Sprintf("https://api.steampowered.com/ISteamDirectory/GetCMList/v1/?cellId=0"))
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	r := struct {
		Response struct {
			ServerList []string
			Result     uint32
			Message    string
		}
	}{}
	if err = json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return err
	}
	if r.Response.Result != 1 {
		return fmt.Errorf("Failed to get steam directory, result: %v, message: %v\n", r.Response.Result, r.Response.Message)
	}
	if len(r.Response.ServerList) == 0 {
		return fmt.Errorf("Steam returned zero servers for steam directory request\n")
	}
	sd.servers = r.Response.ServerList
	sd.isInitialized = true
	return nil
}

func (sd *steamDirectory) GetRandomCM() *netutil.PortAddr {
	sd.RLock()
	defer sd.RUnlock()
	if !sd.isInitialized {
		panic("steam directory is not initialized")
	}
	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
	addr := netutil.ParsePortAddr(sd.servers[rng.Int31n(int32(len(sd.servers)))])
	return addr
}

func (sd *steamDirectory) IsInitialized() bool {
	sd.RLock()
	defer sd.RUnlock()
	isInitialized := sd.isInitialized
	return isInitialized
}
<|endoftext|>


<commit_before>package workers

import (
  "github.com/ryadzenine/dolphin/models"
  "github.com/ryadzenine/dolphin/models/np"
  "github.com/ryadzenine/dolphin/mpi"
)

func SimpleWorker(data_stream chan models.SLPoint, est *np.RevezEstimator,
  queue mpi.MessagesQueue, tau int, name string) {
  i := 1
  vc := make(map[string]int) // version control map
  for {
    select {
    case data := <-data_stream:
      if i == 1 {
        for _, v := range queue.Queues() {
          vc[v] = 0
        }
      }
      // ici on va faire des computations
      if i%tau == 0 {
        stat := queue.ReadStates(vc)
        // Block of code just to covert to the good types
        states := make([]models.State, 0, len(stat))
        for _, v := range stat {
          states = append(states, v.(models.State))
        }
        // We know append the knew versions
        for key, v := range stat {
          vc[key] = v.Version()
        }
        acc := make([]float64, len(est.Points))
        if len(states) != 0 {
          acc = models.States(states).ComputeAgregation()
        }
        est.ComputeDistributedStep(acc, data)
      } else {
        est.ComputeStep(data)
      }
      queue.Write(name, est.State())
      i = i + 1
    }
  }
}
<commit_msg>optimisation de la stratégie d'ecriture<commit_after>package workers

import (
  "github.com/ryadzenine/dolphin/models"
  "github.com/ryadzenine/dolphin/models/np"
  "github.com/ryadzenine/dolphin/mpi"
)

func SimpleWorker(data_stream chan models.SLPoint, est *np.RevezEstimator,
  queue mpi.MessagesQueue, tau int, name string) {
  i := 1
  vc := make(map[string]int) // version control map
  for {
    select {
    case data := <-data_stream:
      if i == 1 {
        for _, v := range queue.Queues() {
          vc[v] = 0
        }
      }
      // ici on va faire des computations
      if i%tau == 0 {
        stat := queue.ReadStates(vc)
        // Block of code just to covert to the good types
        states := make([]models.State, 0, len(stat))
        for _, v := range stat {
          states = append(states, v.(models.State))
        }
        // We know append the knew versions
        for key, v := range stat {
          vc[key] = v.Version()
        }
        acc := make([]float64, len(est.Points))
        if len(states) != 0 {
          acc = models.States(states).ComputeAgregation()
        }
        est.ComputeDistributedStep(acc, data)
      } else {
        est.ComputeStep(data)
      }
      if j := i % tau; tau-j < 3 {
        queue.Write(name, est.State())
      }
      i = i + 1
    }
  }
}
<|endoftext|>


<commit_before>package kasper

import (
	"log"
	"github.com/Shopify/sarama"
	"time"
)

type TopicProcessor struct {
	config              *TopicProcessorConfig
	containerId         int
	client              sarama.Client
	offsetManager       sarama.OffsetManager
	partitionProcessors []*partitionProcessor
	inputTopics         []string
	partitions          []int32
}

func partitionsOfTopics(topics []string, client sarama.Client) []int32 {
	partitionsSet := make(map[int32]struct{})
	for _, topic := range topics {
		partitions, err := client.Partitions(topic)
		if err != nil {
			log.Fatal(err)
		}
		for _, partition := range partitions {
			partitionsSet[partition] = struct{}{}
		}
	}
	i := 0
	partitions := make([]int32, len(partitionsSet))
	for partition := range partitionsSet {
		partitions[i] = partition
		i++
	}
	return partitions
}

func NewTopicProcessor(config *TopicProcessorConfig, makeProcessor func() MessageProcessor, containerId int) *TopicProcessor {
	// TODO: check all input topics are covered by a Serde
	// TODO: check all input partitions and make sure PartitionAssignment is valid
	// TODO: check containerId is within [0, ContainerCount)
	inputTopics := config.InputTopics
	brokerList := config.BrokerList
	client, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		log.Fatal(err)
	}
	partitions := config.partitionsForContainer(containerId)
	offsetManager, err := sarama.NewOffsetManagerFromClient(config.kafkaConsumerGroup(), client)
	if err != nil {
		log.Fatal(err)
	}
	partitionProcessors := make([]*partitionProcessor, len(partitions))
	topicProcessor := TopicProcessor{
		config,
		containerId,
		client,
		offsetManager,
		partitionProcessors,
		inputTopics,
		partitions,
	}
	for i, partition := range partitions {
		processor := makeProcessor()
		partitionProcessors[i] = newPartitionProcessor(&topicProcessor, processor, partition)
	}
	return &topicProcessor
}

func (tp *TopicProcessor) Run() {
	/* FIXME factor these out to sub-functions */
	consumerMessagesChan := make(chan *sarama.ConsumerMessage)
	for _, ch := range tp.consumerMessageChannels() {
		go func(c <-chan *sarama.ConsumerMessage) {
			for msg := range c {
				consumerMessagesChan <- msg
			}
		}(ch)
	}
	producerSuccessesChan := make(chan *sarama.ProducerMessage)
	for _, ch := range tp.producerSuccessesChannels() {
		go func(c <-chan *sarama.ProducerMessage) {
			for msg := range c {
				producerSuccessesChan <- msg
			}
		}(ch)
	}
	producerErrorsChan := make(chan *sarama.ProducerError)
	for _, ch := range tp.producerErrorsChannels() {
		go func(c <-chan *sarama.ProducerError) {
			for msg := range c {
				producerErrorsChan <- msg
			}
		}(ch)
	}

	/* TODO: call Stop() on this ticker when implementing proper shutdown */
	markOffsetTicker := time.NewTicker(tp.config.AutoMarkOffsetsInterval) /* TODO: handle AutoMarkOffsetsInterval <= 0 */
	for {
		select {
		case consumerMessage := <-consumerMessagesChan:
			pp := tp.partitionProcessors[consumerMessage.Partition]
			pp.processConsumerMessage(consumerMessage)
		case producerMessage := <-producerSuccessesChan:
			pp := tp.partitionProcessors[producerMessage.Partition]
			pp.processProducerMessageSuccess(producerMessage)
		case producerError := <-producerErrorsChan:
			log.Fatal(producerError) /* FIXME Handle this gracefully with a retry count / backoff period */
		case <-markOffsetTicker.C:
			for _, pp := range tp.partitionProcessors {
				pp.markOffsets()
			}
		}
	}
}

func (tp *TopicProcessor) consumerMessageChannels() []<-chan *sarama.ConsumerMessage {
	var chans []<-chan *sarama.ConsumerMessage
	for _, partitionProcessor := range tp.partitionProcessors {
		partitionChannels := partitionProcessor.consumerMessageChannels()
		for _, ch := range partitionChannels {
			chans = append(chans, ch)
		}
	}
	return chans
}

func (tp *TopicProcessor) producerSuccessesChannels() []<-chan *sarama.ProducerMessage {
	var chans []<-chan *sarama.ProducerMessage
	for _, partitionProcessor := range tp.partitionProcessors {
		ch := partitionProcessor.producer.Successes()
		chans = append(chans, ch)
	}
	return chans
}

func (tp *TopicProcessor) producerErrorsChannels() []<-chan *sarama.ProducerError {
	var chans []<-chan *sarama.ProducerError
	for _, partitionProcessor := range tp.partitionProcessors {
		ch := partitionProcessor.producer.Errors()
		chans = append(chans, ch)
	}
	return chans
}
<commit_msg>Update doc<commit_after>/*
kasper is a lightweight Kafka stream processing library.
 */
package kasper

import (
	"log"
	"github.com/Shopify/sarama"
	"time"
)

type TopicProcessor struct {
	config              *TopicProcessorConfig
	containerId         int
	client              sarama.Client
	offsetManager       sarama.OffsetManager
	partitionProcessors []*partitionProcessor
	inputTopics         []string
	partitions          []int32
}

func partitionsOfTopics(topics []string, client sarama.Client) []int32 {
	partitionsSet := make(map[int32]struct{})
	for _, topic := range topics {
		partitions, err := client.Partitions(topic)
		if err != nil {
			log.Fatal(err)
		}
		for _, partition := range partitions {
			partitionsSet[partition] = struct{}{}
		}
	}
	i := 0
	partitions := make([]int32, len(partitionsSet))
	for partition := range partitionsSet {
		partitions[i] = partition
		i++
	}
	return partitions
}

// NewTopicProcessor creates a new TopicProcessor with the given config.
// It requires a factory function that creates MessageProcessor instances and a container id.
// The container id must be a number between 0 and config.ContainerCount - 1.
func NewTopicProcessor(config *TopicProcessorConfig, makeProcessor func() MessageProcessor, containerId int) *TopicProcessor {
	// TODO: check all input topics are covered by a Serde
	// TODO: check all input partitions and make sure PartitionAssignment is valid
	// TODO: check containerId is within [0, ContainerCount)
	inputTopics := config.InputTopics
	brokerList := config.BrokerList
	client, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		log.Fatal(err)
	}
	partitions := config.partitionsForContainer(containerId)
	offsetManager, err := sarama.NewOffsetManagerFromClient(config.kafkaConsumerGroup(), client)
	if err != nil {
		log.Fatal(err)
	}
	partitionProcessors := make([]*partitionProcessor, len(partitions))
	topicProcessor := TopicProcessor{
		config,
		containerId,
		client,
		offsetManager,
		partitionProcessors,
		inputTopics,
		partitions,
	}
	for i, partition := range partitions {
		processor := makeProcessor()
		partitionProcessors[i] = newPartitionProcessor(&topicProcessor, processor, partition)
	}
	return &topicProcessor
}

func (tp *TopicProcessor) Run() {
	/* FIXME factor these out to sub-functions */
	consumerMessagesChan := make(chan *sarama.ConsumerMessage)
	for _, ch := range tp.consumerMessageChannels() {
		go func(c <-chan *sarama.ConsumerMessage) {
			for msg := range c {
				consumerMessagesChan <- msg
			}
		}(ch)
	}
	producerSuccessesChan := make(chan *sarama.ProducerMessage)
	for _, ch := range tp.producerSuccessesChannels() {
		go func(c <-chan *sarama.ProducerMessage) {
			for msg := range c {
				producerSuccessesChan <- msg
			}
		}(ch)
	}
	producerErrorsChan := make(chan *sarama.ProducerError)
	for _, ch := range tp.producerErrorsChannels() {
		go func(c <-chan *sarama.ProducerError) {
			for msg := range c {
				producerErrorsChan <- msg
			}
		}(ch)
	}

	/* TODO: call Stop() on this ticker when implementing proper shutdown */
	markOffsetTicker := time.NewTicker(tp.config.AutoMarkOffsetsInterval) /* TODO: handle AutoMarkOffsetsInterval <= 0 */
	for {
		select {
		case consumerMessage := <-consumerMessagesChan:
			pp := tp.partitionProcessors[consumerMessage.Partition]
			pp.processConsumerMessage(consumerMessage)
		case producerMessage := <-producerSuccessesChan:
			pp := tp.partitionProcessors[producerMessage.Partition]
			pp.processProducerMessageSuccess(producerMessage)
		case producerError := <-producerErrorsChan:
			log.Fatal(producerError) /* FIXME Handle this gracefully with a retry count / backoff period */
		case <-markOffsetTicker.C:
			for _, pp := range tp.partitionProcessors {
				pp.markOffsets()
			}
		}
	}
}

func (tp *TopicProcessor) consumerMessageChannels() []<-chan *sarama.ConsumerMessage {
	var chans []<-chan *sarama.ConsumerMessage
	for _, partitionProcessor := range tp.partitionProcessors {
		partitionChannels := partitionProcessor.consumerMessageChannels()
		for _, ch := range partitionChannels {
			chans = append(chans, ch)
		}
	}
	return chans
}

func (tp *TopicProcessor) producerSuccessesChannels() []<-chan *sarama.ProducerMessage {
	var chans []<-chan *sarama.ProducerMessage
	for _, partitionProcessor := range tp.partitionProcessors {
		ch := partitionProcessor.producer.Successes()
		chans = append(chans, ch)
	}
	return chans
}

func (tp *TopicProcessor) producerErrorsChannels() []<-chan *sarama.ProducerError {
	var chans []<-chan *sarama.ProducerError
	for _, partitionProcessor := range tp.partitionProcessors {
		ch := partitionProcessor.producer.Errors()
		chans = append(chans, ch)
	}
	return chans
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
)

func TestAccAWSStorageGatewayTapePool_basic(t *testing.T) {
	var TapePool storagegateway.PoolInfo
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_storagegateway_tape_pool.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSStorageGatewayTapePoolDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSStorageGatewayTapePoolBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					testAccMatchResourceAttrRegionalARN(resourceName, "arn", "storagegateway", regexp.MustCompile(`tapepool/pool-.+`)),
					resource.TestCheckResourceAttr(resourceName, "pool_name", rName),
					resource.TestCheckResourceAttr(resourceName, "storage_class", "GLACIER"),
					resource.TestCheckResourceAttr(resourceName, "retention_look_type", "NONE"),
					resource.TestCheckResourceAttr(resourceName, "retention_lock_time_in_days", "0"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSStorageGatewayTapePool_tags(t *testing.T) {
	var TapePool storagegateway.PoolInfo
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_storagegateway_tape_pool.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSStorageGatewayTapePoolDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSStorageGatewayTapePoolConfigTags1(rName, "key1", "value1"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSStorageGatewayTapePoolConfigTags2(rName, "key1", "value1updated", "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "2"),
					resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"),
					resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
				),
			},
			{
				Config: testAccAWSStorageGatewayTapePoolConfigTags1(rName, "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
				),
			},
		},
	})
}

func TestAccAWSStorageGatewayTapePool_disappears(t *testing.T) {
	var storedIscsiVolume storagegateway.PoolInfo
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_storagegateway_tape_pool.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSStorageGatewayTapePoolDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSStorageGatewayTapePoolBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &storedIscsiVolume),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsStorageGatewayTapePool(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSStorageGatewayTapePoolExists(resourceName string, TapePool *storagegateway.PoolInfo) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[resourceName]
		if !ok {
			return fmt.Errorf("Not found: %s", resourceName)
		}

		conn := testAccProvider.Meta().(*AWSClient).storagegatewayconn

		input := &storagegateway.ListTapePoolsInput{
			PoolARNs: []*string{aws.String(rs.Primary.ID)},
		}

		output, err := conn.ListTapePools(input)

		if err != nil {
			return fmt.Errorf("error reading Storage Gateway Tape Pool: %s", err)
		}

		if output == nil || len(output.PoolInfos) == 0 || output.PoolInfos[0] == nil || aws.StringValue(output.PoolInfos[0].PoolARN) != rs.Primary.ID {
			return fmt.Errorf("Storage Gateway Tape Pool %q not found", rs.Primary.ID)
		}

		*TapePool = *output.PoolInfos[0]

		return nil
	}
}

func testAccCheckAWSStorageGatewayTapePoolDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).storagegatewayconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_storagegateway_tape_pool" {
			continue
		}

		input := &storagegateway.ListTapePoolsInput{
			PoolARNs: []*string{aws.String(rs.Primary.ID)},
		}

		output, err := conn.ListTapePools(input)

		if err != nil {
			return err
		}

		if output == nil || len(output.PoolInfos) == 0 || output.PoolInfos[0] == nil || aws.StringValue(output.PoolInfos[0].PoolARN) != rs.Primary.ID {
			return fmt.Errorf("Storage Gateway Tape Pool %q not found", rs.Primary.ID)
		}
	}

	return nil
}

func testAccAWSStorageGatewayTapePoolBasicConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_storagegateway_tape_pool" "test" {
  pool_name     = %[1]q
  storage_class = "GLACIER"
}
`, rName)
}

func testAccAWSStorageGatewayTapePoolConfigTags1(rName, tagKey1, tagValue1 string) string {
	return fmt.Sprintf(`
resource "aws_storagegateway_tape_pool" "test" {
  pool_name     = %[1]q
  storage_class = "GLACIER"

  tags = {
    %[2]q = %[3]q
  }
}
`, rName, tagKey1, tagValue1)
}

func testAccAWSStorageGatewayTapePoolConfigTags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string {
	return fmt.Sprintf(`
resource "aws_storagegateway_tape_pool" "test" {
  pool_name     = %[1]q
  storage_class = "GLACIER"

  tags = {
    %[2]q = %[3]q
    %[4]q = %[5]q
  }
}
`, rName, tagKey1, tagValue1, tagKey2, tagValue2)
}
<commit_msg>add retention test<commit_after>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/storagegateway"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
)

func TestAccAWSStorageGatewayTapePool_basic(t *testing.T) {
	var TapePool storagegateway.PoolInfo
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_storagegateway_tape_pool.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSStorageGatewayTapePoolDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSStorageGatewayTapePoolBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					testAccMatchResourceAttrRegionalARN(resourceName, "arn", "storagegateway", regexp.MustCompile(`tapepool/pool-.+`)),
					resource.TestCheckResourceAttr(resourceName, "pool_name", rName),
					resource.TestCheckResourceAttr(resourceName, "storage_class", "GLACIER"),
					resource.TestCheckResourceAttr(resourceName, "retention_look_type", "NONE"),
					resource.TestCheckResourceAttr(resourceName, "retention_lock_time_in_days", "0"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSStorageGatewayTapePool_retention(t *testing.T) {
	var TapePool storagegateway.PoolInfo
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_storagegateway_tape_pool.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSStorageGatewayTapePoolDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSStorageGatewayTapePoolRetentionConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					testAccMatchResourceAttrRegionalARN(resourceName, "arn", "storagegateway", regexp.MustCompile(`tapepool/pool-.+`)),
					resource.TestCheckResourceAttr(resourceName, "pool_name", rName),
					resource.TestCheckResourceAttr(resourceName, "storage_class", "GLACIER"),
					resource.TestCheckResourceAttr(resourceName, "retention_look_type", "GOVERNANCE"),
					resource.TestCheckResourceAttr(resourceName, "retention_lock_time_in_days", "1"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func TestAccAWSStorageGatewayTapePool_tags(t *testing.T) {
	var TapePool storagegateway.PoolInfo
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_storagegateway_tape_pool.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSStorageGatewayTapePoolDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSStorageGatewayTapePoolConfigTags1(rName, "key1", "value1"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1"),
				),
			},
			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
			{
				Config: testAccAWSStorageGatewayTapePoolConfigTags2(rName, "key1", "value1updated", "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "2"),
					resource.TestCheckResourceAttr(resourceName, "tags.key1", "value1updated"),
					resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
				),
			},
			{
				Config: testAccAWSStorageGatewayTapePoolConfigTags1(rName, "key2", "value2"),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &TapePool),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "1"),
					resource.TestCheckResourceAttr(resourceName, "tags.key2", "value2"),
				),
			},
		},
	})
}

func TestAccAWSStorageGatewayTapePool_disappears(t *testing.T) {
	var storedIscsiVolume storagegateway.PoolInfo
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "aws_storagegateway_tape_pool.test"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAWSStorageGatewayTapePoolDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccAWSStorageGatewayTapePoolBasicConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAWSStorageGatewayTapePoolExists(resourceName, &storedIscsiVolume),
					testAccCheckResourceDisappears(testAccProvider, resourceAwsStorageGatewayTapePool(), resourceName),
				),
				ExpectNonEmptyPlan: true,
			},
		},
	})
}

func testAccCheckAWSStorageGatewayTapePoolExists(resourceName string, TapePool *storagegateway.PoolInfo) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[resourceName]
		if !ok {
			return fmt.Errorf("Not found: %s", resourceName)
		}

		conn := testAccProvider.Meta().(*AWSClient).storagegatewayconn

		input := &storagegateway.ListTapePoolsInput{
			PoolARNs: []*string{aws.String(rs.Primary.ID)},
		}

		output, err := conn.ListTapePools(input)

		if err != nil {
			return fmt.Errorf("error reading Storage Gateway Tape Pool: %s", err)
		}

		if output == nil || len(output.PoolInfos) == 0 || output.PoolInfos[0] == nil || aws.StringValue(output.PoolInfos[0].PoolARN) != rs.Primary.ID {
			return fmt.Errorf("Storage Gateway Tape Pool %q not found", rs.Primary.ID)
		}

		*TapePool = *output.PoolInfos[0]

		return nil
	}
}

func testAccCheckAWSStorageGatewayTapePoolDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).storagegatewayconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_storagegateway_tape_pool" {
			continue
		}

		input := &storagegateway.ListTapePoolsInput{
			PoolARNs: []*string{aws.String(rs.Primary.ID)},
		}

		output, err := conn.ListTapePools(input)

		if err != nil {
			return err
		}

		if output == nil || len(output.PoolInfos) == 0 || output.PoolInfos[0] == nil || aws.StringValue(output.PoolInfos[0].PoolARN) != rs.Primary.ID {
			return fmt.Errorf("Storage Gateway Tape Pool %q not found", rs.Primary.ID)
		}
	}

	return nil
}

func testAccAWSStorageGatewayTapePoolBasicConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_storagegateway_tape_pool" "test" {
  pool_name     = %[1]q
  storage_class = "GLACIER"
}
`, rName)
}

func testAccAWSStorageGatewayTapePoolRetentionConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_storagegateway_tape_pool" "test" {
  pool_name                   = %[1]q
  storage_class               = "GLACIER"
  retention_look_type         = "GOVERNANCE"
  retention_lock_time_in_days = 1
}
`, rName)
}

func testAccAWSStorageGatewayTapePoolConfigTags1(rName, tagKey1, tagValue1 string) string {
	return fmt.Sprintf(`
resource "aws_storagegateway_tape_pool" "test" {
  pool_name     = %[1]q
  storage_class = "GLACIER"

  tags = {
    %[2]q = %[3]q
  }
}
`, rName, tagKey1, tagValue1)
}

func testAccAWSStorageGatewayTapePoolConfigTags2(rName, tagKey1, tagValue1, tagKey2, tagValue2 string) string {
	return fmt.Sprintf(`
resource "aws_storagegateway_tape_pool" "test" {
  pool_name     = %[1]q
  storage_class = "GLACIER"

  tags = {
    %[2]q = %[3]q
    %[4]q = %[5]q
  }
}
`, rName, tagKey1, tagValue1, tagKey2, tagValue2)
}
<|endoftext|>


<commit_before>package git

import (
	"path"
	"strings"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
	"gopkg.in/src-d/go-git.v4/plumbing/format/index"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/storage"

	"gopkg.in/src-d/go-billy.v4"
)

// Commit stores the current contents of the index in a new commit along with
// a log message from the user describing the changes.
func (w *Worktree) Commit(msg string, opts *CommitOptions) (plumbing.Hash, error) {
	if err := opts.Validate(w.r); err != nil {
		return plumbing.ZeroHash, err
	}

	if opts.All {
		if err := w.autoAddModifiedAndDeleted(); err != nil {
			return plumbing.ZeroHash, err
		}
	}

	idx, err := w.r.Storer.Index()
	if err != nil {
		return plumbing.ZeroHash, err
	}

	h := &buildTreeHelper{
		fs: w.Filesystem,
		s:  w.r.Storer,
	}

	tree, err := h.BuildTree(idx)
	if err != nil {
		return plumbing.ZeroHash, err
	}

	commit, err := w.buildCommitObject(msg, opts, tree)
	if err != nil {
		return plumbing.ZeroHash, err
	}

	return commit, w.updateHEAD(commit)
}

func (w *Worktree) autoAddModifiedAndDeleted() error {
	s, err := w.Status()
	if err != nil {
		return err
	}

	for path, fs := range s {
		if fs.Worktree != Modified && fs.Worktree != Deleted {
			continue
		}

		if _, err := w.Add(path); err != nil {
			return err
		}
	}

	return nil
}

func (w *Worktree) updateHEAD(commit plumbing.Hash) error {
	head, err := w.r.Storer.Reference(plumbing.HEAD)
	if err != nil {
		return err
	}

	name := plumbing.HEAD
	if head.Type() != plumbing.HashReference {
		name = head.Target()
	}

	ref := plumbing.NewHashReference(name, commit)
	return w.r.Storer.SetReference(ref)
}

func (w *Worktree) buildCommitObject(msg string, opts *CommitOptions, tree plumbing.Hash) (plumbing.Hash, error) {
	commit := &object.Commit{
		Author:       *opts.Author,
		Committer:    *opts.Committer,
		Message:      msg,
		TreeHash:     tree,
		ParentHashes: opts.Parents,
	}

	obj := w.r.Storer.NewEncodedObject()
	if err := commit.Encode(obj); err != nil {
		return plumbing.ZeroHash, err
	}
	return w.r.Storer.SetEncodedObject(obj)
}

// buildTreeHelper converts a given index.Index file into multiple git objects
// reading the blobs from the given filesystem and creating the trees from the
// index structure. The created objects are pushed to a given Storer.
type buildTreeHelper struct {
	fs billy.Filesystem
	s  storage.Storer

	trees   map[string]*object.Tree
	entries map[string]*object.TreeEntry
}

// BuildTree builds the tree objects and push its to the storer, the hash
// of the root tree is returned.
func (h *buildTreeHelper) BuildTree(idx *index.Index) (plumbing.Hash, error) {
	const rootNode = ""
	h.trees = map[string]*object.Tree{rootNode: {}}
	h.entries = map[string]*object.TreeEntry{}

	for _, e := range idx.Entries {
		if err := h.commitIndexEntry(e); err != nil {
			return plumbing.ZeroHash, err
		}
	}

	return h.copyTreeToStorageRecursive(rootNode, h.trees[rootNode])
}

func (h *buildTreeHelper) commitIndexEntry(e *index.Entry) error {
	parts := strings.Split(e.Name, "/")

	var fullpath string
	for _, part := range parts {
		parent := fullpath
		fullpath = path.Join(fullpath, part)

		h.doBuildTree(e, parent, fullpath)
	}

	return nil
}

func (h *buildTreeHelper) doBuildTree(e *index.Entry, parent, fullpath string) {
	if _, ok := h.trees[fullpath]; ok {
		return
	}

	if _, ok := h.entries[fullpath]; ok {
		return
	}

	te := object.TreeEntry{Name: path.Base(fullpath)}

	if fullpath == e.Name {
		te.Mode = e.Mode
		te.Hash = e.Hash
	} else {
		te.Mode = filemode.Dir
		h.trees[fullpath] = &object.Tree{}
	}

	h.trees[parent].Entries = append(h.trees[parent].Entries, te)
}

func (h *buildTreeHelper) copyTreeToStorageRecursive(parent string, t *object.Tree) (plumbing.Hash, error) {
	for i, e := range t.Entries {
		if e.Mode != filemode.Dir && !e.Hash.IsZero() {
			continue
		}

		path := path.Join(parent, e.Name)

		var err error
		e.Hash, err = h.copyTreeToStorageRecursive(path, h.trees[path])
		if err != nil {
			return plumbing.ZeroHash, err
		}

		t.Entries[i] = e
	}

	o := h.s.NewEncodedObject()
	if err := t.Encode(o); err != nil {
		return plumbing.ZeroHash, err
	}

	return h.s.SetEncodedObject(o)
}
<commit_msg>worktree: sort the tree object.  Fixes #881<commit_after>package git

import (
	"path"
	"strings"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/filemode"
	"gopkg.in/src-d/go-git.v4/plumbing/format/index"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/storage"

	"gopkg.in/src-d/go-billy.v4"
	"sort"
)

// Commit stores the current contents of the index in a new commit along with
// a log message from the user describing the changes.
func (w *Worktree) Commit(msg string, opts *CommitOptions) (plumbing.Hash, error) {
	if err := opts.Validate(w.r); err != nil {
		return plumbing.ZeroHash, err
	}

	if opts.All {
		if err := w.autoAddModifiedAndDeleted(); err != nil {
			return plumbing.ZeroHash, err
		}
	}

	idx, err := w.r.Storer.Index()
	if err != nil {
		return plumbing.ZeroHash, err
	}

	h := &buildTreeHelper{
		fs: w.Filesystem,
		s:  w.r.Storer,
	}

	tree, err := h.BuildTree(idx)
	if err != nil {
		return plumbing.ZeroHash, err
	}

	commit, err := w.buildCommitObject(msg, opts, tree)
	if err != nil {
		return plumbing.ZeroHash, err
	}

	return commit, w.updateHEAD(commit)
}

func (w *Worktree) autoAddModifiedAndDeleted() error {
	s, err := w.Status()
	if err != nil {
		return err
	}

	for path, fs := range s {
		if fs.Worktree != Modified && fs.Worktree != Deleted {
			continue
		}

		if _, err := w.Add(path); err != nil {
			return err
		}
	}

	return nil
}

func (w *Worktree) updateHEAD(commit plumbing.Hash) error {
	head, err := w.r.Storer.Reference(plumbing.HEAD)
	if err != nil {
		return err
	}

	name := plumbing.HEAD
	if head.Type() != plumbing.HashReference {
		name = head.Target()
	}

	ref := plumbing.NewHashReference(name, commit)
	return w.r.Storer.SetReference(ref)
}

func (w *Worktree) buildCommitObject(msg string, opts *CommitOptions, tree plumbing.Hash) (plumbing.Hash, error) {
	commit := &object.Commit{
		Author:       *opts.Author,
		Committer:    *opts.Committer,
		Message:      msg,
		TreeHash:     tree,
		ParentHashes: opts.Parents,
	}

	obj := w.r.Storer.NewEncodedObject()
	if err := commit.Encode(obj); err != nil {
		return plumbing.ZeroHash, err
	}
	return w.r.Storer.SetEncodedObject(obj)
}

// buildTreeHelper converts a given index.Index file into multiple git objects
// reading the blobs from the given filesystem and creating the trees from the
// index structure. The created objects are pushed to a given Storer.
type buildTreeHelper struct {
	fs billy.Filesystem
	s  storage.Storer

	trees   map[string]*object.Tree
	entries map[string]*object.TreeEntry
}

// BuildTree builds the tree objects and push its to the storer, the hash
// of the root tree is returned.
func (h *buildTreeHelper) BuildTree(idx *index.Index) (plumbing.Hash, error) {
	const rootNode = ""
	h.trees = map[string]*object.Tree{rootNode: {}}
	h.entries = map[string]*object.TreeEntry{}

	for _, e := range idx.Entries {
		if err := h.commitIndexEntry(e); err != nil {
			return plumbing.ZeroHash, err
		}
	}

	return h.copyTreeToStorageRecursive(rootNode, h.trees[rootNode])
}

func (h *buildTreeHelper) commitIndexEntry(e *index.Entry) error {
	parts := strings.Split(e.Name, "/")

	var fullpath string
	for _, part := range parts {
		parent := fullpath
		fullpath = path.Join(fullpath, part)

		h.doBuildTree(e, parent, fullpath)
	}

	return nil
}

func (h *buildTreeHelper) doBuildTree(e *index.Entry, parent, fullpath string) {
	if _, ok := h.trees[fullpath]; ok {
		return
	}

	if _, ok := h.entries[fullpath]; ok {
		return
	}

	te := object.TreeEntry{Name: path.Base(fullpath)}

	if fullpath == e.Name {
		te.Mode = e.Mode
		te.Hash = e.Hash
	} else {
		te.Mode = filemode.Dir
		h.trees[fullpath] = &object.Tree{}
	}

	h.trees[parent].Entries = append(h.trees[parent].Entries, te)
}

type sortableEntries []object.TreeEntry

func (sortableEntries) sortName(te object.TreeEntry) string {
	if te.Mode == filemode.Dir {
		return te.Name + "/"
	}
	return te.Name
}
func (se sortableEntries) Len() int               { return len(se) }
func (se sortableEntries) Less(i int, j int) bool { return se.sortName(se[i]) < se.sortName(se[j]) }
func (se sortableEntries) Swap(i int, j int)      { se[i], se[j] = se[j], se[i] }

func (h *buildTreeHelper) copyTreeToStorageRecursive(parent string, t *object.Tree) (plumbing.Hash, error) {
	sort.Sort(sortableEntries(t.Entries))
	for i, e := range t.Entries {
		if e.Mode != filemode.Dir && !e.Hash.IsZero() {
			continue
		}

		path := path.Join(parent, e.Name)

		var err error
		e.Hash, err = h.copyTreeToStorageRecursive(path, h.trees[path])
		if err != nil {
			return plumbing.ZeroHash, err
		}

		t.Entries[i] = e
	}

	o := h.s.NewEncodedObject()
	if err := t.Encode(o); err != nil {
		return plumbing.ZeroHash, err
	}

	return h.s.SetEncodedObject(o)
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tika

import (
	"context"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"time"

	"golang.org/x/net/context/ctxhttp"
)

// Server represents a Tika server. Create a new Server with NewServer,
// start it with StartServer, and shut it down with Close.
// There is no need to create a Server for an already running Tika Server
// since you can pass its URL directly to a Client.
type Server struct {
	jar      string
	url      string // url is derived from port and hostname.
	port     string
	hostname string
	cancel   func()
	timeout  time.Duration
}

// URL returns the URL of this Server.
func (s *Server) URL() string {
	return s.url
}

// An Option can be passed to NewServer to configure the Server.
type Option func(*Server)

// WithHostname returns an Option to set the host of the Server (default localhost).
func WithHostname(h string) Option {
	return func(s *Server) {
		s.hostname = h
	}
}

// WithPort returns an Option to set the port of the Server (default 9998).
func WithPort(p string) Option {
	return func(s *Server) {
		s.port = p
	}
}

// WithStartupTimeout returns an Option to set the timeout for how long to wait
// for the Server to start.
func WithStartupTimeout(d time.Duration) Option {
	return func(s *Server) {
		s.timeout = d
	}
}

// NewServer creates a new Server.
func NewServer(jar string, options ...Option) (*Server, error) {
	if jar == "" {
		return nil, fmt.Errorf("no jar file specified")
	}
	if _, err := os.Stat(jar); err != nil {
		return nil, fmt.Errorf("jar file not found: %s", jar)
	}
	s := &Server{
		jar:      jar,
		port:     "9998",
		timeout:  10 * time.Second,
		hostname: "localhost",
	}
	for _, o := range options {
		o(s)
	}
	urlString := "http://" + s.hostname + ":" + s.port
	u, err := url.Parse(urlString)
	if err != nil {
		return nil, fmt.Errorf("invalid hostname %q or port %q: %v", s.hostname, s.port, err)
	}
	s.url = u.String()
	return s, nil
}

type commander func(context.Context, string, ...string) *exec.Cmd

// cmder is used to stub out *exec.Cmd for testing.
var cmder commander = exec.CommandContext

// Start starts the given server. Start will start a new Java process. The
// caller must call cancel() to shut down the process when finished with the
// Server. The given Context is used for the Java process, not for cancellation
// of startup.
func (s *Server) Start(ctx context.Context) (cancel func(), err error) {
	ctx, cancel = context.WithCancel(ctx)
	cmd := cmder(ctx, "java", "-jar", s.jar, "-p", s.port)

	stderr, err := cmd.StderrPipe()
	if err != nil {
		cancel()
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		cancel()
		return nil, err
	}

	if err := s.waitForStart(); err != nil {
		cancel()
		buf, err := ioutil.ReadAll(stderr)
		if err != nil {
			return nil, fmt.Errorf("error reading stderr: %v", err)
		}
		return nil, fmt.Errorf("error starting server: %v: %v", err, string(buf))
	}
	return cancel, nil
}

// waitForServer waits until the given Server is responding to requests.
// waitForStart returns an error if the server does not respond within the timeout.
func (s Server) waitForStart() error {
	c := NewClient(nil, s.url)
	var err error
	for i := time.Duration(0); i < s.timeout; i += time.Second {
		if _, err = c.Version(context.Background()); err == nil {
			return nil
		}
		time.Sleep(time.Second)
	}
	return err
}

func validateFileMD5(path, wantH string) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return false
	}

	return fmt.Sprintf("%x", h.Sum(nil)) == wantH
}

// A Version represents a Tika Server version.
type Version string

// Supported versions of Tika Server.
const (
	Version114 Version = "1.14"
)

var md5s = map[Version]string{
	Version114: "39055fc71358d774b9da066f80b1141c",
}

// DownloadServer downloads and validates the given server version,
// saving it at path. DownloadServer returns an error if it could
// not be downloaded/validated. Valid values for the version are 1.14.
// It is the callers responsibility to remove the file when no longer needed.
// If the file already exists and has the correct MD5, DownloadServer will
// do nothing.
func DownloadServer(ctx context.Context, version Version, path string) error {
	wantH := md5s[version]
	if wantH == "" {
		return fmt.Errorf("unsupported Tika version: %s", version)
	}

	if _, err := os.Stat(path); err == nil {
		if validateFileMD5(path, wantH) {
			return nil
		}
	}
	out, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("error creating file: %v", err)
	}
	defer out.Close()

	url := fmt.Sprintf("http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server/%s/tika-server-%s.jar", version, version)
	resp, err := ctxhttp.Get(ctx, nil, url)
	if err != nil {
		return fmt.Errorf("unable to download %q: %v", url, err)
	}
	defer resp.Body.Close()

	if _, err = io.Copy(out, resp.Body); err != nil {
		return fmt.Errorf("error saving download: %v", err)
	}

	if !validateFileMD5(path, wantH) {
		if err := os.Remove(path); err != nil {
			return fmt.Errorf("invalid md5: error removing %s: %v", path, err)
		}
		return fmt.Errorf("invalid md5")
	}
	return nil
}
<commit_msg>Make waitForStart take a context<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tika

import (
	"context"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"time"

	"golang.org/x/net/context/ctxhttp"
)

// Server represents a Tika server. Create a new Server with NewServer,
// start it with StartServer, and shut it down with Close.
// There is no need to create a Server for an already running Tika Server
// since you can pass its URL directly to a Client.
type Server struct {
	jar            string
	url            string // url is derived from port and hostname.
	port           string
	hostname       string
	cancel         func()
	startupTimeout time.Duration
}

// URL returns the URL of this Server.
func (s *Server) URL() string {
	return s.url
}

// An Option can be passed to NewServer to configure the Server.
type Option func(*Server)

// WithHostname returns an Option to set the host of the Server (default localhost).
func WithHostname(h string) Option {
	return func(s *Server) {
		s.hostname = h
	}
}

// WithPort returns an Option to set the port of the Server (default 9998).
func WithPort(p string) Option {
	return func(s *Server) {
		s.port = p
	}
}

// WithStartupTimeout returns an Option to set the timeout for how long to wait
// for the Server to start (default 10s).
func WithStartupTimeout(d time.Duration) Option {
	return func(s *Server) {
		s.startupTimeout = d
	}
}

// NewServer creates a new Server.
func NewServer(jar string, options ...Option) (*Server, error) {
	if jar == "" {
		return nil, fmt.Errorf("no jar file specified")
	}
	if _, err := os.Stat(jar); err != nil {
		return nil, fmt.Errorf("jar file not found: %s", jar)
	}
	s := &Server{
		jar:            jar,
		port:           "9998",
		startupTimeout: 10 * time.Second,
		hostname:       "localhost",
	}
	for _, o := range options {
		o(s)
	}
	urlString := "http://" + s.hostname + ":" + s.port
	u, err := url.Parse(urlString)
	if err != nil {
		return nil, fmt.Errorf("invalid hostname %q or port %q: %v", s.hostname, s.port, err)
	}
	s.url = u.String()
	return s, nil
}

type commander func(context.Context, string, ...string) *exec.Cmd

// cmder is used to stub out *exec.Cmd for testing.
var cmder commander = exec.CommandContext

// Start starts the given server. Start will start a new Java process. The
// caller must call cancel() to shut down the process when finished with the
// Server. The given Context is used for the Java process, not for cancellation
// of startup.
func (s *Server) Start(ctx context.Context) (cancel func(), err error) {
	ctx, cancel = context.WithCancel(ctx)
	cmd := cmder(ctx, "java", "-jar", s.jar, "-p", s.port)

	stderr, err := cmd.StderrPipe()
	if err != nil {
		cancel()
		return nil, err
	}

	if err := cmd.Start(); err != nil {
		cancel()
		return nil, err
	}

	if err := s.waitForStart(ctx); err != nil {
		cancel()
		buf, readErr := ioutil.ReadAll(stderr)
		if readErr != nil {
			return nil, fmt.Errorf("error reading stderr: %v", err)
		}
		// Report stderr since sometimes the server says why it failed to start.
		return nil, fmt.Errorf("error starting server: %v\nserver stderr:\n\n%v", err, string(buf))
	}
	return cancel, nil
}

// waitForServer waits until the given Server is responding to requests.
// waitForStart returns an error if the server does not respond within the
// timeout set by WithStartupTimeout or if ctx is Done() first.
func (s Server) waitForStart(ctx context.Context) error {
	c := NewClient(nil, s.url)
	ctx, cancel := context.WithTimeout(ctx, s.startupTimeout)
	defer cancel()
	select {
	case <-time.Tick(500 * time.Millisecond):
		if _, err := c.Version(ctx); err == nil {
			return nil
		}
	case <-ctx.Done():
		return ctx.Err()
	}
	return fmt.Errorf("could not reach server")
}

func validateFileMD5(path, wantH string) bool {
	f, err := os.Open(path)
	if err != nil {
		return false
	}
	defer f.Close()

	h := md5.New()
	if _, err := io.Copy(h, f); err != nil {
		return false
	}

	return fmt.Sprintf("%x", h.Sum(nil)) == wantH
}

// A Version represents a Tika Server version.
type Version string

// Supported versions of Tika Server.
const (
	Version114 Version = "1.14"
)

var md5s = map[Version]string{
	Version114: "39055fc71358d774b9da066f80b1141c",
}

// DownloadServer downloads and validates the given server version,
// saving it at path. DownloadServer returns an error if it could
// not be downloaded/validated. Valid values for the version are 1.14.
// It is the callers responsibility to remove the file when no longer needed.
// If the file already exists and has the correct MD5, DownloadServer will
// do nothing.
func DownloadServer(ctx context.Context, version Version, path string) error {
	wantH := md5s[version]
	if wantH == "" {
		return fmt.Errorf("unsupported Tika version: %s", version)
	}

	if _, err := os.Stat(path); err == nil {
		if validateFileMD5(path, wantH) {
			return nil
		}
	}
	out, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("error creating file: %v", err)
	}
	defer out.Close()

	url := fmt.Sprintf("http://search.maven.org/remotecontent?filepath=org/apache/tika/tika-server/%s/tika-server-%s.jar", version, version)
	resp, err := ctxhttp.Get(ctx, nil, url)
	if err != nil {
		return fmt.Errorf("unable to download %q: %v", url, err)
	}
	defer resp.Body.Close()

	if _, err = io.Copy(out, resp.Body); err != nil {
		return fmt.Errorf("error saving download: %v", err)
	}

	if !validateFileMD5(path, wantH) {
		if err := os.Remove(path); err != nil {
			return fmt.Errorf("invalid md5: error removing %s: %v", path, err)
		}
		return fmt.Errorf("invalid md5")
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package storage implements storage and retrieval of topic maps in a mounted
// file system using the Badger key-value database libraries.
package storage

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"fmt"
	"time"

	"github.com/dgraph-io/badger"
)

// makeKey returns a storage key built from a global single-byte prefix followed by
// a sequence of entities.
func makeKey(buf []byte, prefix byte, entities ...uint64) []byte {
	length := 1 + 8*len(entities)
	if len(buf) < length {
		buf = make([]byte, 1+8*len(entities))
	}
	buf[0] = prefix
	for i, entity := range entities {
		binary.BigEndian.PutUint64(buf[i*8+1:(i+1)*8+1], entity)
	}
	return buf
}

// To allow complex values may be encoded differently in later versions, a
// single byte prefix is included in serialized values to identify the
// encoding.
const (
	// The only format currently supported.
	GobFormat byte = iota
)

const (
	// MetaPrefix is the first byte in keys for metadata about a database.
	//
	// {MetaPrefix, Meta*} : see documentation for Meta* constant.
	MetaPrefix byte = iota

	// TopicMapPrefix is the first byte in keys where the following eight bytes
	// identify a topic map.
	//
	// {TopicMapPrefix, TopicMap} : {GobFormat, TopicMapInfo}
	TopicMapPrefix

	// TopicMapSequence is the badger.Sequence prefix for generating TopicMap
	// values.
	TopicMapSequence
)

// TopicMap is how this package identifies topic maps.
type TopicMap uint64

// Storage is a thin wrapper around a Badger database.
//
// Remember to call Close() on any Storage when finished with it.
type Storage struct {
	db *badger.DB
}

// Open returns a new Storage based on a Badger database stored at dir.
func Open(dir string) (*Storage, error) {
	opts := badger.DefaultOptions
	opts.Dir = dir
	opts.ValueDir = dir
	db, err := badger.Open(opts)
	if err != nil {
		return nil, err
	}
	return &Storage{db: db}, nil
}

// Close closes the underlying Badger database, blocking as necessary to flush
// pending updates to disk.
func (s *Storage) Close() error { return s.db.Close() }

// NewTransaction returns a transaction that can be used to read and, if update
// is true, to write to the underlying store.
func (s *Storage) NewTransaction(update bool) *Transaction {
	return &Transaction{
		s:   s,
		txn: s.db.NewTransaction(update),
	}
}

// Transaction is how all read and write operations are executed.
//
// Remember to call Discard() on any Transaction when finished with it.
type Transaction struct {
	s   *Storage
	txn *badger.Txn
}

// Commit commits changes or returns an error.
//
// Returns nil error if there were no writes.
func (t *Transaction) Commit() error { return t.txn.Commit() }

// Discard is idempotent and must be called for every Transaction when finished
// with it.
func (t *Transaction) Discard() { t.txn.Discard() }

// TopicMapInfo holds some metadata about a topic map.
//
// TODO: Add an identifier for the topicmaps.TopicMap item within the
// associated topic map, whose characteristics and reifying topic can provide
// much more information.
type TopicMapInfo struct {
	TopicMap TopicMap
	Created  time.Time
}

// CreateTopicMap creates a new topic map in transaction t and returns a copy
// of the topic map's new metadata.
func (t *Transaction) CreateTopicMap() (*TopicMapInfo, error) {
	sequence, err := t.s.db.GetSequence(makeKey(nil, TopicMapSequence), 1)
	if err != nil {
		return nil, err
	}
	entity, err := sequence.Next()
	if err != nil {
		return nil, err
	}
	info := TopicMapInfo{
		TopicMap: TopicMap(entity),
		Created:  time.Now().Truncate(0),
	}
	var value bytes.Buffer
	value.WriteByte(GobFormat)
	gob.NewEncoder(&value).Encode(&info)
	return &info, t.txn.Set(
		makeKey(nil, TopicMapPrefix, uint64(info.TopicMap)),
		value.Bytes())
}

// TopicMaps creates and returns a TopicMapsCursor that will iterate over topic
// maps according to query.
func (t *Transaction) TopicMaps(query TopicMapsQuery) *TopicMapsCursor {
	return &TopicMapsCursor{
		iter: t.txn.NewIterator(badger.DefaultIteratorOptions),
	}
}

// TopicMapsQuery describes how to fetch topic maps and then creates a cursor
// to do so.
type TopicMapsQuery struct{}

// TopicMapsCursor supports iterating over a set of topic maps.
//
// Remember to call Discard() on any TopicMapsCursor when finished with it.
type TopicMapsCursor struct {
	iter    *badger.Iterator
	started bool
}

// Next advances the cursor to the next topic map, which is the first topic map
// if it has not been called before, and returns true if and only if a topic
// map is found.
func (c *TopicMapsCursor) Next() bool {
	prefix := makeKey(nil, TopicMapPrefix)
	if !c.started {
		c.iter.Seek(prefix)
		c.started = true
	} else {
		c.iter.Next()
	}
	return c.iter.ValidForPrefix(prefix)
}

// Info decodes the TopicMapInfo associated with the current topic map.
//
// Panics unless the most recent call to c.Next() returned true.
func (c *TopicMapsCursor) Info() (*TopicMapInfo, error) {
	var info TopicMapInfo
	err := c.iter.Item().Value(func(val []byte) error {
		if val[0] != GobFormat {
			return UnsupportedFormatError(val[0])
		}
		return gob.NewDecoder(bytes.NewReader(val[1:])).Decode(&info)
	})
	return &info, err
}

// Discard must be called when the cursor is no longer needed.
func (c *TopicMapsCursor) Discard() {
	c.iter.Close()
}

// UnsupportedFormatError indicates that a value was found in the key-value
// backing store with an unsupported format code, perhaps due to data
// corruption.
type UnsupportedFormatError byte

func (e UnsupportedFormatError) Error() string {
	return fmt.Sprintf("unsupported format code 0x%x", byte(e))
}
<commit_msg>Update storage for new Badger Options API<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package storage implements storage and retrieval of topic maps in a mounted
// file system using the Badger key-value database libraries.
package storage

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"fmt"
	"time"

	"github.com/dgraph-io/badger"
)

// makeKey returns a storage key built from a global single-byte prefix followed by
// a sequence of entities.
func makeKey(buf []byte, prefix byte, entities ...uint64) []byte {
	length := 1 + 8*len(entities)
	if len(buf) < length {
		buf = make([]byte, 1+8*len(entities))
	}
	buf[0] = prefix
	for i, entity := range entities {
		binary.BigEndian.PutUint64(buf[i*8+1:(i+1)*8+1], entity)
	}
	return buf
}

// To allow complex values may be encoded differently in later versions, a
// single byte prefix is included in serialized values to identify the
// encoding.
const (
	// The only format currently supported.
	GobFormat byte = iota
)

const (
	// MetaPrefix is the first byte in keys for metadata about a database.
	//
	// {MetaPrefix, Meta*} : see documentation for Meta* constant.
	MetaPrefix byte = iota

	// TopicMapPrefix is the first byte in keys where the following eight bytes
	// identify a topic map.
	//
	// {TopicMapPrefix, TopicMap} : {GobFormat, TopicMapInfo}
	TopicMapPrefix

	// TopicMapSequence is the badger.Sequence prefix for generating TopicMap
	// values.
	TopicMapSequence
)

// TopicMap is how this package identifies topic maps.
type TopicMap uint64

// Storage is a thin wrapper around a Badger database.
//
// Remember to call Close() on any Storage when finished with it.
type Storage struct {
	db *badger.DB
}

// Open returns a new Storage based on a Badger database stored at dir.
func Open(dir string) (*Storage, error) {
	opts := badger.DefaultOptions(dir)
	db, err := badger.Open(opts)
	if err != nil {
		return nil, err
	}
	return &Storage{db: db}, nil
}

// Close closes the underlying Badger database, blocking as necessary to flush
// pending updates to disk.
func (s *Storage) Close() error { return s.db.Close() }

// NewTransaction returns a transaction that can be used to read and, if update
// is true, to write to the underlying store.
func (s *Storage) NewTransaction(update bool) *Transaction {
	return &Transaction{
		s:   s,
		txn: s.db.NewTransaction(update),
	}
}

// Transaction is how all read and write operations are executed.
//
// Remember to call Discard() on any Transaction when finished with it.
type Transaction struct {
	s   *Storage
	txn *badger.Txn
}

// Commit commits changes or returns an error.
//
// Returns nil error if there were no writes.
func (t *Transaction) Commit() error { return t.txn.Commit() }

// Discard is idempotent and must be called for every Transaction when finished
// with it.
func (t *Transaction) Discard() { t.txn.Discard() }

// TopicMapInfo holds some metadata about a topic map.
//
// TODO: Add an identifier for the topicmaps.TopicMap item within the
// associated topic map, whose characteristics and reifying topic can provide
// much more information.
type TopicMapInfo struct {
	TopicMap TopicMap
	Created  time.Time
}

// CreateTopicMap creates a new topic map in transaction t and returns a copy
// of the topic map's new metadata.
func (t *Transaction) CreateTopicMap() (*TopicMapInfo, error) {
	sequence, err := t.s.db.GetSequence(makeKey(nil, TopicMapSequence), 1)
	if err != nil {
		return nil, err
	}
	entity, err := sequence.Next()
	if err != nil {
		return nil, err
	}
	info := TopicMapInfo{
		TopicMap: TopicMap(entity),
		Created:  time.Now().Truncate(0),
	}
	var value bytes.Buffer
	value.WriteByte(GobFormat)
	gob.NewEncoder(&value).Encode(&info)
	return &info, t.txn.Set(
		makeKey(nil, TopicMapPrefix, uint64(info.TopicMap)),
		value.Bytes())
}

// TopicMaps creates and returns a TopicMapsCursor that will iterate over topic
// maps according to query.
func (t *Transaction) TopicMaps(query TopicMapsQuery) *TopicMapsCursor {
	return &TopicMapsCursor{
		iter: t.txn.NewIterator(badger.DefaultIteratorOptions),
	}
}

// TopicMapsQuery describes how to fetch topic maps and then creates a cursor
// to do so.
type TopicMapsQuery struct{}

// TopicMapsCursor supports iterating over a set of topic maps.
//
// Remember to call Discard() on any TopicMapsCursor when finished with it.
type TopicMapsCursor struct {
	iter    *badger.Iterator
	started bool
}

// Next advances the cursor to the next topic map, which is the first topic map
// if it has not been called before, and returns true if and only if a topic
// map is found.
func (c *TopicMapsCursor) Next() bool {
	prefix := makeKey(nil, TopicMapPrefix)
	if !c.started {
		c.iter.Seek(prefix)
		c.started = true
	} else {
		c.iter.Next()
	}
	return c.iter.ValidForPrefix(prefix)
}

// Info decodes the TopicMapInfo associated with the current topic map.
//
// Panics unless the most recent call to c.Next() returned true.
func (c *TopicMapsCursor) Info() (*TopicMapInfo, error) {
	var info TopicMapInfo
	err := c.iter.Item().Value(func(val []byte) error {
		if val[0] != GobFormat {
			return UnsupportedFormatError(val[0])
		}
		return gob.NewDecoder(bytes.NewReader(val[1:])).Decode(&info)
	})
	return &info, err
}

// Discard must be called when the cursor is no longer needed.
func (c *TopicMapsCursor) Discard() {
	c.iter.Close()
}

// UnsupportedFormatError indicates that a value was found in the key-value
// backing store with an unsupported format code, perhaps due to data
// corruption.
type UnsupportedFormatError byte

func (e UnsupportedFormatError) Error() string {
	return fmt.Sprintf("unsupported format code 0x%x", byte(e))
}
<|endoftext|>