<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"bytes"
	"context"
	"fmt"
	"io"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/kaniko"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	kubernetesclient "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/client"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

const initContainer = "kaniko-init-container"

func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, tag string, requiredImages map[string]*string, platforms platform.Matcher) (string, error) {
	// TODO: Implement building multi-platform images for cluster builder
	if platforms.IsMultiPlatform() {
		log.Entry(ctx).Warnf("multiple target platforms %q found for artifact %q. Skaffold doesn't yet support multi-platform builds for the docker builder. Consider specifying a single target platform explicitly. See https://skaffold.dev/docs/pipeline-stages/builders/#cross-platform-build-support", platforms.String(), artifactName)
	}

	generatedEnvs, err := generateEnvFromImage(tag)
	if err != nil {
		return "", fmt.Errorf("error processing generated env variables from image uri: %w", err)
	}
	env, err := evaluateEnv(artifact.Env, generatedEnvs...)
	if err != nil {
		return "", fmt.Errorf("unable to evaluate env variables: %w", err)
	}
	artifact.Env = env

	buildArgs, err := docker.EvalBuildArgsWithEnv(b.cfg.Mode(), kaniko.GetContext(artifact, workspace), artifact.DockerfilePath, artifact.BuildArgs, requiredImages, envMapFromVars(artifact.Env))
	if err != nil {
		return "", fmt.Errorf("unable to evaluate build args: %w", err)
	}
	artifact.BuildArgs = buildArgs

	client, err := kubernetesclient.DefaultClient()
	if err != nil {
		return "", fmt.Errorf("getting Kubernetes client: %w", err)
	}
	pods := client.CoreV1().Pods(b.Namespace)

	podSpec, err := b.kanikoPodSpec(artifact, tag, platforms)
	if err != nil {
		return "", err
	}

	pod, err := pods.Create(ctx, podSpec, metav1.CreateOptions{})
	if err != nil {
		return "", fmt.Errorf("creating kaniko pod: %w", err)
	}
	defer func() {
		if err := pods.Delete(ctx, pod.Name, metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			log.Entry(ctx).Fatalf("deleting pod: %s", err)
		}
	}()

	if err := b.copyKanikoBuildContext(ctx, workspace, artifactName, artifact, pods, pod.Name); err != nil {
		return "", fmt.Errorf("copying sources: %w", err)
	}

	// Wait for the pods to succeed while streaming the logs
	waitForLogs := streamLogs(ctx, out, pod.Name, pods)

	if err := kubernetes.WaitForPodSucceeded(ctx, pods, pod.Name, b.timeout); err != nil {
		waitForLogs()
		return "", err
	}

	waitForLogs()

	return docker.RemoteDigest(tag, b.cfg, nil)
}

// first copy over the buildcontext tarball into the init container tmp dir via kubectl cp
// Via kubectl exec, we extract the tarball to the empty dir
// Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container
func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, artifactName string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error {
	if err := kubernetes.WaitForPodInitialized(ctx, pods, podName); err != nil {
		return fmt.Errorf("waiting for pod to initialize: %w", err)
	}

	errs := make(chan error, 1)
	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := docker.CreateDockerTarContext(ctx, buildCtxWriter, docker.NewBuildConfig(
			kaniko.GetContext(artifact, workspace), artifactName, artifact.DockerfilePath, artifact.BuildArgs), b.cfg)
		if err != nil {
			buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err))
			errs <- err
			return
		}
		buildCtxWriter.Close()
	}()

	// Send context by piping into `tar`.
	// In case of an error, print the command's output. (The `err` itself is useless: exit status 1).
	var out bytes.Buffer
	if err := b.kubectlcli.Run(ctx, buildCtx, &out, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-xf", "-", "-C", kaniko.DefaultEmptyDirMountPath); err != nil {
		errRun := fmt.Errorf("uploading build context: %s", out.String())
		errTar := <-errs
		if errTar != nil {
			errRun = fmt.Errorf("%v\ntar errors: %w", errRun, errTar)
		}
		return errRun
	}

	// Generate a file to successfully terminate the init container.
	if out, err := b.kubectlcli.RunOut(ctx, "exec", podName, "-c", initContainer, "-n", b.Namespace, "--", "touch", "/tmp/complete"); err != nil {
		return fmt.Errorf("finishing upload of the build context: %s", out)
	}

	return nil
}

func evaluateEnv(env []v1.EnvVar, additional ...v1.EnvVar) ([]v1.EnvVar, error) {
	// Prepare additional envs
	addEnv := make(map[string]string)
	for _, addEnvVar := range additional {
		addEnv[addEnvVar.Name] = addEnvVar.Value
	}

	// Evaluate provided env variables
	var evaluated []v1.EnvVar
	for _, envVar := range env {
		val, err := util.ExpandEnvTemplate(envVar.Value, nil)
		if err != nil {
			return nil, fmt.Errorf("unable to get value for env variable %q: %w", envVar.Name, err)
		}

		evaluated = append(evaluated, v1.EnvVar{Name: envVar.Name, Value: val})

		// Provided env variables have higher priority than additional (generated) ones
		delete(addEnv, envVar.Name)
	}

	// Append additional (generated) env variables
	for name, value := range addEnv {
		if value != "" {
			evaluated = append(evaluated, v1.EnvVar{Name: name, Value: value})
		}
	}

	return evaluated, nil
}

func envMapFromVars(env []v1.EnvVar) map[string]string {
	envMap := make(map[string]string)
	for _, envVar := range env {
		envMap[envVar.Name] = envVar.Value
	}
	return envMap
}

func generateEnvFromImage(imageStr string) ([]v1.EnvVar, error) {
	imgRef, err := docker.ParseReference(imageStr)
	if err != nil {
		return nil, err
	}
	if imgRef.Tag == "" {
		imgRef.Tag = "latest"
	}
	var generatedEnvs []v1.EnvVar
	generatedEnvs = append(generatedEnvs, v1.EnvVar{Name: "IMAGE_REPO", Value: imgRef.Repo})
	generatedEnvs = append(generatedEnvs, v1.EnvVar{Name: "IMAGE_NAME", Value: imgRef.Name})
	generatedEnvs = append(generatedEnvs, v1.EnvVar{Name: "IMAGE_TAG", Value: imgRef.Tag})
	return generatedEnvs, nil
}
<commit_msg>fix: change log level of kaniko pod deletion error (#7621)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"bytes"
	"context"
	"fmt"
	"io"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/kaniko"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	kubernetesclient "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/client"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/platform"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
)

const initContainer = "kaniko-init-container"

func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, tag string, requiredImages map[string]*string, platforms platform.Matcher) (string, error) {
	// TODO: Implement building multi-platform images for cluster builder
	if platforms.IsMultiPlatform() {
		log.Entry(ctx).Warnf("multiple target platforms %q found for artifact %q. Skaffold doesn't yet support multi-platform builds for the docker builder. Consider specifying a single target platform explicitly. See https://skaffold.dev/docs/pipeline-stages/builders/#cross-platform-build-support", platforms.String(), artifactName)
	}

	generatedEnvs, err := generateEnvFromImage(tag)
	if err != nil {
		return "", fmt.Errorf("error processing generated env variables from image uri: %w", err)
	}
	env, err := evaluateEnv(artifact.Env, generatedEnvs...)
	if err != nil {
		return "", fmt.Errorf("unable to evaluate env variables: %w", err)
	}
	artifact.Env = env

	buildArgs, err := docker.EvalBuildArgsWithEnv(b.cfg.Mode(), kaniko.GetContext(artifact, workspace), artifact.DockerfilePath, artifact.BuildArgs, requiredImages, envMapFromVars(artifact.Env))
	if err != nil {
		return "", fmt.Errorf("unable to evaluate build args: %w", err)
	}
	artifact.BuildArgs = buildArgs

	client, err := kubernetesclient.DefaultClient()
	if err != nil {
		return "", fmt.Errorf("getting Kubernetes client: %w", err)
	}
	pods := client.CoreV1().Pods(b.Namespace)

	podSpec, err := b.kanikoPodSpec(artifact, tag, platforms)
	if err != nil {
		return "", err
	}

	pod, err := pods.Create(ctx, podSpec, metav1.CreateOptions{})
	if err != nil {
		return "", fmt.Errorf("creating kaniko pod: %w", err)
	}
	defer func() {
		if err := pods.Delete(ctx, pod.Name, metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			log.Entry(ctx).Errorf("deleting pod: %s", err)
		}
	}()

	if err := b.copyKanikoBuildContext(ctx, workspace, artifactName, artifact, pods, pod.Name); err != nil {
		return "", fmt.Errorf("copying sources: %w", err)
	}

	// Wait for the pods to succeed while streaming the logs
	waitForLogs := streamLogs(ctx, out, pod.Name, pods)

	if err := kubernetes.WaitForPodSucceeded(ctx, pods, pod.Name, b.timeout); err != nil {
		waitForLogs()
		return "", err
	}

	waitForLogs()

	return docker.RemoteDigest(tag, b.cfg, nil)
}

// first copy over the buildcontext tarball into the init container tmp dir via kubectl cp
// Via kubectl exec, we extract the tarball to the empty dir
// Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container
func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, artifactName string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error {
	if err := kubernetes.WaitForPodInitialized(ctx, pods, podName); err != nil {
		return fmt.Errorf("waiting for pod to initialize: %w", err)
	}

	errs := make(chan error, 1)
	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := docker.CreateDockerTarContext(ctx, buildCtxWriter, docker.NewBuildConfig(
			kaniko.GetContext(artifact, workspace), artifactName, artifact.DockerfilePath, artifact.BuildArgs), b.cfg)
		if err != nil {
			buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err))
			errs <- err
			return
		}
		buildCtxWriter.Close()
	}()

	// Send context by piping into `tar`.
	// In case of an error, print the command's output. (The `err` itself is useless: exit status 1).
	var out bytes.Buffer
	if err := b.kubectlcli.Run(ctx, buildCtx, &out, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-xf", "-", "-C", kaniko.DefaultEmptyDirMountPath); err != nil {
		errRun := fmt.Errorf("uploading build context: %s", out.String())
		errTar := <-errs
		if errTar != nil {
			errRun = fmt.Errorf("%v\ntar errors: %w", errRun, errTar)
		}
		return errRun
	}

	// Generate a file to successfully terminate the init container.
	if out, err := b.kubectlcli.RunOut(ctx, "exec", podName, "-c", initContainer, "-n", b.Namespace, "--", "touch", "/tmp/complete"); err != nil {
		return fmt.Errorf("finishing upload of the build context: %s", out)
	}

	return nil
}

func evaluateEnv(env []v1.EnvVar, additional ...v1.EnvVar) ([]v1.EnvVar, error) {
	// Prepare additional envs
	addEnv := make(map[string]string)
	for _, addEnvVar := range additional {
		addEnv[addEnvVar.Name] = addEnvVar.Value
	}

	// Evaluate provided env variables
	var evaluated []v1.EnvVar
	for _, envVar := range env {
		val, err := util.ExpandEnvTemplate(envVar.Value, nil)
		if err != nil {
			return nil, fmt.Errorf("unable to get value for env variable %q: %w", envVar.Name, err)
		}

		evaluated = append(evaluated, v1.EnvVar{Name: envVar.Name, Value: val})

		// Provided env variables have higher priority than additional (generated) ones
		delete(addEnv, envVar.Name)
	}

	// Append additional (generated) env variables
	for name, value := range addEnv {
		if value != "" {
			evaluated = append(evaluated, v1.EnvVar{Name: name, Value: value})
		}
	}

	return evaluated, nil
}

func envMapFromVars(env []v1.EnvVar) map[string]string {
	envMap := make(map[string]string)
	for _, envVar := range env {
		envMap[envVar.Name] = envVar.Value
	}
	return envMap
}

func generateEnvFromImage(imageStr string) ([]v1.EnvVar, error) {
	imgRef, err := docker.ParseReference(imageStr)
	if err != nil {
		return nil, err
	}
	if imgRef.Tag == "" {
		imgRef.Tag = "latest"
	}
	var generatedEnvs []v1.EnvVar
	generatedEnvs = append(generatedEnvs, v1.EnvVar{Name: "IMAGE_REPO", Value: imgRef.Repo})
	generatedEnvs = append(generatedEnvs, v1.EnvVar{Name: "IMAGE_NAME", Value: imgRef.Name})
	generatedEnvs = append(generatedEnvs, v1.EnvVar{Name: "IMAGE_TAG", Value: imgRef.Tag})
	return generatedEnvs, nil
}
<|endoftext|>


<commit_before>package config

import (
	"fmt"
	"reflect"
	"regexp"
	"strings"

	"github.com/mitchellh/reflectwalk"
)

// InterpSplitDelim is the delimeter that is looked for to split when
// it is returned. This is a comma right now but should eventually become
// a value that a user is very unlikely to use (such as UUID).
const InterpSplitDelim = `,`

// interpRegexp is a regexp that matches interpolations such as ${foo.bar}
var interpRegexp *regexp.Regexp = regexp.MustCompile(
	`(?i)(\$+)\{([\s*-.,\\/\(\)a-z0-9_"]+)\}`)

// interpolationWalker implements interfaces for the reflectwalk package
// (github.com/mitchellh/reflectwalk) that can be used to automatically
// execute a callback for an interpolation.
type interpolationWalker struct {
	F       interpolationWalkerFunc
	Replace bool

	key         []string
	lastValue   reflect.Value
	loc         reflectwalk.Location
	cs          []reflect.Value
	csKey       []reflect.Value
	csData      interface{}
	sliceIndex  int
	unknownKeys []string
}

// interpolationWalkerFunc is the callback called by interpolationWalk.
// It is called with any interpolation found. It should return a value
// to replace the interpolation with, along with any errors.
//
// If Replace is set to false in interpolationWalker, then the replace
// value can be anything as it will have no effect.
type interpolationWalkerFunc func(Interpolation) (string, error)

func (w *interpolationWalker) Enter(loc reflectwalk.Location) error {
	w.loc = loc
	return nil
}

func (w *interpolationWalker) Exit(loc reflectwalk.Location) error {
	w.loc = reflectwalk.None

	switch loc {
	case reflectwalk.Map:
		w.cs = w.cs[:len(w.cs)-1]
	case reflectwalk.MapValue:
		w.key = w.key[:len(w.key)-1]
		w.csKey = w.csKey[:len(w.csKey)-1]
	case reflectwalk.Slice:
		// Split any values that need to be split
		w.splitSlice()
		w.cs = w.cs[:len(w.cs)-1]
	case reflectwalk.SliceElem:
		w.csKey = w.csKey[:len(w.csKey)-1]
	}

	return nil
}

func (w *interpolationWalker) Map(m reflect.Value) error {
	w.cs = append(w.cs, m)
	return nil
}

func (w *interpolationWalker) MapElem(m, k, v reflect.Value) error {
	w.csData = k
	w.csKey = append(w.csKey, k)
	w.key = append(w.key, k.String())
	w.lastValue = v
	return nil
}

func (w *interpolationWalker) Slice(s reflect.Value) error {
	w.cs = append(w.cs, s)
	return nil
}

func (w *interpolationWalker) SliceElem(i int, elem reflect.Value) error {
	w.csKey = append(w.csKey, reflect.ValueOf(i))
	w.sliceIndex = i
	return nil
}

func (w *interpolationWalker) Primitive(v reflect.Value) error {
	setV := v

	// We only care about strings
	if v.Kind() == reflect.Interface {
		setV = v
		v = v.Elem()
	}
	if v.Kind() != reflect.String {
		return nil
	}

	// XXX: This can be a lot more efficient if we used a real
	// parser. A regexp is a hammer though that will get this working.

	matches := interpRegexp.FindAllStringSubmatch(v.String(), -1)
	if len(matches) == 0 {
		return nil
	}

	result := v.String()
	for _, match := range matches {
		dollars := len(match[1])

		// If there are even amounts of dollar signs, then it is escaped
		if dollars%2 == 0 {
			continue
		}

		// Interpolation found, instantiate it
		key := match[2]

		i, err := ExprParse(key)
		if err != nil {
			return err
		}

		replaceVal, err := w.F(i)
		if err != nil {
			return fmt.Errorf(
				"%s: %s",
				key,
				err)
		}

		if w.Replace {
			// We need to determine if we need to remove this element
			// if the result contains any "UnknownVariableValue" which is
			// set if it is computed. This behavior is different if we're
			// splitting (in a SliceElem) or not.
			remove := false
			if w.loc == reflectwalk.SliceElem {
				parts := strings.Split(replaceVal, InterpSplitDelim)
				for _, p := range parts {
					if p == UnknownVariableValue {
						remove = true
						break
					}
				}
			} else if replaceVal == UnknownVariableValue {
				remove = true
			}
			if remove {
				w.removeCurrent()
				return nil
			}

			// Replace in our interpolation and continue on.
			result = strings.Replace(result, match[0], replaceVal, -1)
		}
	}

	if w.Replace {
		resultVal := reflect.ValueOf(result)
		switch w.loc {
		case reflectwalk.MapKey:
			m := w.cs[len(w.cs)-1]

			// Delete the old value
			var zero reflect.Value
			m.SetMapIndex(w.csData.(reflect.Value), zero)

			// Set the new key with the existing value
			m.SetMapIndex(resultVal, w.lastValue)

			// Set the key to be the new key
			w.csData = resultVal
		case reflectwalk.MapValue:
			// If we're in a map, then the only way to set a map value is
			// to set it directly.
			m := w.cs[len(w.cs)-1]
			mk := w.csData.(reflect.Value)
			m.SetMapIndex(mk, resultVal)
		default:
			// Otherwise, we should be addressable
			setV.Set(resultVal)
		}
	}

	return nil
}

func (w *interpolationWalker) removeCurrent() {
	c := w.cs[len(w.cs)-1]
	switch c.Kind() {
	case reflect.Map:
		// Zero value so that we delete the map key
		var val reflect.Value

		// Get the key and delete it
		k := w.csData.(reflect.Value)
		c.SetMapIndex(k, val)
	}

	// Append the key to the unknown keys
	w.unknownKeys = append(w.unknownKeys, strings.Join(w.key, "."))
}

func (w *interpolationWalker) replace(v reflect.Value, offset int) {
	c := w.cs[len(w.cs)-1+offset]
	switch c.Kind() {
	case reflect.Map:
		// Get the key and delete it
		k := w.csKey[len(w.csKey)-1]
		c.SetMapIndex(k, v)
	}
}

func (w *interpolationWalker) splitSlice() {
	// Get the []interface{} slice so we can do some operations on
	// it without dealing with reflection. We'll document each step
	// here to be clear.
	var s []interface{}
	raw := w.cs[len(w.cs)-1]
	switch v := raw.Interface().(type) {
	case []interface{}:
		s = v
	case []map[string]interface{}:
		return
	default:
		panic("Unknown kind: " + raw.Kind().String())
	}

	// Check if we have any elements that we need to split. If not, then
	// just return since we're done.
	split := false
	for _, v := range s {
		sv, ok := v.(string)
		if !ok {
			continue
		}
		if idx := strings.Index(sv, InterpSplitDelim); idx >= 0 {
			split = true
			break
		}
	}
	if !split {
		return
	}

	// Make a new result slice that is twice the capacity to fit our growth.
	result := make([]interface{}, 0, len(s)*2)

	// Go over each element of the original slice and start building up
	// the resulting slice by splitting where we have to.
	for _, v := range s {
		sv, ok := v.(string)
		if !ok {
			// Not a string, so just set it
			result = append(result, v)
			continue
		}

		// Split on the delimiter
		for _, p := range strings.Split(sv, InterpSplitDelim) {
			result = append(result, p)
		}
	}

	// Our slice is now done, we have to replace the slice now
	// with this new one that we have.
	w.replace(reflect.ValueOf(result), -1)
}
<commit_msg>config: make the replace function marginally more understandable<commit_after>package config

import (
	"fmt"
	"reflect"
	"regexp"
	"strings"

	"github.com/mitchellh/reflectwalk"
)

// InterpSplitDelim is the delimeter that is looked for to split when
// it is returned. This is a comma right now but should eventually become
// a value that a user is very unlikely to use (such as UUID).
const InterpSplitDelim = `,`

// interpRegexp is a regexp that matches interpolations such as ${foo.bar}
var interpRegexp *regexp.Regexp = regexp.MustCompile(
	`(?i)(\$+)\{([\s*-.,\\/\(\)a-z0-9_"]+)\}`)

// interpolationWalker implements interfaces for the reflectwalk package
// (github.com/mitchellh/reflectwalk) that can be used to automatically
// execute a callback for an interpolation.
type interpolationWalker struct {
	F       interpolationWalkerFunc
	Replace bool

	key         []string
	lastValue   reflect.Value
	loc         reflectwalk.Location
	cs          []reflect.Value
	csKey       []reflect.Value
	csData      interface{}
	sliceIndex  int
	unknownKeys []string
}

// interpolationWalkerFunc is the callback called by interpolationWalk.
// It is called with any interpolation found. It should return a value
// to replace the interpolation with, along with any errors.
//
// If Replace is set to false in interpolationWalker, then the replace
// value can be anything as it will have no effect.
type interpolationWalkerFunc func(Interpolation) (string, error)

func (w *interpolationWalker) Enter(loc reflectwalk.Location) error {
	w.loc = loc
	return nil
}

func (w *interpolationWalker) Exit(loc reflectwalk.Location) error {
	w.loc = reflectwalk.None

	switch loc {
	case reflectwalk.Map:
		w.cs = w.cs[:len(w.cs)-1]
	case reflectwalk.MapValue:
		w.key = w.key[:len(w.key)-1]
		w.csKey = w.csKey[:len(w.csKey)-1]
	case reflectwalk.Slice:
		// Split any values that need to be split
		w.splitSlice()
		w.cs = w.cs[:len(w.cs)-1]
	case reflectwalk.SliceElem:
		w.csKey = w.csKey[:len(w.csKey)-1]
	}

	return nil
}

func (w *interpolationWalker) Map(m reflect.Value) error {
	w.cs = append(w.cs, m)
	return nil
}

func (w *interpolationWalker) MapElem(m, k, v reflect.Value) error {
	w.csData = k
	w.csKey = append(w.csKey, k)
	w.key = append(w.key, k.String())
	w.lastValue = v
	return nil
}

func (w *interpolationWalker) Slice(s reflect.Value) error {
	w.cs = append(w.cs, s)
	return nil
}

func (w *interpolationWalker) SliceElem(i int, elem reflect.Value) error {
	w.csKey = append(w.csKey, reflect.ValueOf(i))
	w.sliceIndex = i
	return nil
}

func (w *interpolationWalker) Primitive(v reflect.Value) error {
	setV := v

	// We only care about strings
	if v.Kind() == reflect.Interface {
		setV = v
		v = v.Elem()
	}
	if v.Kind() != reflect.String {
		return nil
	}

	// XXX: This can be a lot more efficient if we used a real
	// parser. A regexp is a hammer though that will get this working.

	matches := interpRegexp.FindAllStringSubmatch(v.String(), -1)
	if len(matches) == 0 {
		return nil
	}

	result := v.String()
	for _, match := range matches {
		dollars := len(match[1])

		// If there are even amounts of dollar signs, then it is escaped
		if dollars%2 == 0 {
			continue
		}

		// Interpolation found, instantiate it
		key := match[2]

		i, err := ExprParse(key)
		if err != nil {
			return err
		}

		replaceVal, err := w.F(i)
		if err != nil {
			return fmt.Errorf(
				"%s: %s",
				key,
				err)
		}

		if w.Replace {
			// We need to determine if we need to remove this element
			// if the result contains any "UnknownVariableValue" which is
			// set if it is computed. This behavior is different if we're
			// splitting (in a SliceElem) or not.
			remove := false
			if w.loc == reflectwalk.SliceElem {
				parts := strings.Split(replaceVal, InterpSplitDelim)
				for _, p := range parts {
					if p == UnknownVariableValue {
						remove = true
						break
					}
				}
			} else if replaceVal == UnknownVariableValue {
				remove = true
			}
			if remove {
				w.removeCurrent()
				return nil
			}

			// Replace in our interpolation and continue on.
			result = strings.Replace(result, match[0], replaceVal, -1)
		}
	}

	if w.Replace {
		resultVal := reflect.ValueOf(result)
		switch w.loc {
		case reflectwalk.MapKey:
			m := w.cs[len(w.cs)-1]

			// Delete the old value
			var zero reflect.Value
			m.SetMapIndex(w.csData.(reflect.Value), zero)

			// Set the new key with the existing value
			m.SetMapIndex(resultVal, w.lastValue)

			// Set the key to be the new key
			w.csData = resultVal
		case reflectwalk.MapValue:
			// If we're in a map, then the only way to set a map value is
			// to set it directly.
			m := w.cs[len(w.cs)-1]
			mk := w.csData.(reflect.Value)
			m.SetMapIndex(mk, resultVal)
		default:
			// Otherwise, we should be addressable
			setV.Set(resultVal)
		}
	}

	return nil
}

func (w *interpolationWalker) removeCurrent() {
	c := w.cs[len(w.cs)-1]
	switch c.Kind() {
	case reflect.Map:
		// Zero value so that we delete the map key
		var val reflect.Value

		// Get the key and delete it
		k := w.csData.(reflect.Value)
		c.SetMapIndex(k, val)
	}

	// Append the key to the unknown keys
	w.unknownKeys = append(w.unknownKeys, strings.Join(w.key, "."))
}

func (w *interpolationWalker) replaceCurrent(v reflect.Value) {
	c := w.cs[len(w.cs)-2]
	switch c.Kind() {
	case reflect.Map:
		// Get the key and delete it
		k := w.csKey[len(w.csKey)-1]
		c.SetMapIndex(k, v)
	}
}

func (w *interpolationWalker) splitSlice() {
	// Get the []interface{} slice so we can do some operations on
	// it without dealing with reflection. We'll document each step
	// here to be clear.
	var s []interface{}
	raw := w.cs[len(w.cs)-1]
	switch v := raw.Interface().(type) {
	case []interface{}:
		s = v
	case []map[string]interface{}:
		return
	default:
		panic("Unknown kind: " + raw.Kind().String())
	}

	// Check if we have any elements that we need to split. If not, then
	// just return since we're done.
	split := false
	for _, v := range s {
		sv, ok := v.(string)
		if !ok {
			continue
		}
		if idx := strings.Index(sv, InterpSplitDelim); idx >= 0 {
			split = true
			break
		}
	}
	if !split {
		return
	}

	// Make a new result slice that is twice the capacity to fit our growth.
	result := make([]interface{}, 0, len(s)*2)

	// Go over each element of the original slice and start building up
	// the resulting slice by splitting where we have to.
	for _, v := range s {
		sv, ok := v.(string)
		if !ok {
			// Not a string, so just set it
			result = append(result, v)
			continue
		}

		// Split on the delimiter
		for _, p := range strings.Split(sv, InterpSplitDelim) {
			result = append(result, p)
		}
	}

	// Our slice is now done, we have to replace the slice now
	// with this new one that we have.
	w.replaceCurrent(reflect.ValueOf(result))
}
<|endoftext|>


<commit_before>// Package lock implements user locking after N bad sign-in attempts.
package lock

import (
	"errors"
	"time"

	"gopkg.in/authboss.v0"
)

// Storage key constants
const (
	StoreAttemptNumber = "attempt_number"
	StoreAttemptTime   = "attempt_time"
	StoreLocked        = "locked"
)

var (
	errUserMissing = errors.New("lock: user not loaded in BeforeAuth callback")
)

func init() {
	authboss.RegisterModule("lock", &Lock{})
}

// Lock module
type Lock struct {
	*authboss.Authboss
}

// Initialize the module
func (l *Lock) Initialize(ab *authboss.Authboss) error {
	l.Authboss = ab
	if l.Storer == nil && l.StoreMaker == nil {
		return errors.New("lock: Need a Storer")
	}

	// Events
	l.Callbacks.After(authboss.EventGetUser, func(ctx *authboss.Context) error {
		_, err := l.beforeAuth(ctx)
		return err
	})
	l.Callbacks.Before(authboss.EventAuth, l.beforeAuth)
	l.Callbacks.After(authboss.EventAuth, l.afterAuth)
	l.Callbacks.After(authboss.EventAuthFail, l.afterAuthFail)

	return nil
}

// Routes for the module
func (l *Lock) Routes() authboss.RouteTable {
	return nil
}

// Storage requirements
func (l *Lock) Storage() authboss.StorageOptions {
	return authboss.StorageOptions{
		l.PrimaryID:        authboss.String,
		StoreAttemptNumber: authboss.Integer,
		StoreAttemptTime:   authboss.DateTime,
		StoreLocked:        authboss.DateTime,
	}
}

// beforeAuth ensures the account is not locked.
func (l *Lock) beforeAuth(ctx *authboss.Context) (authboss.Interrupt, error) {
	if ctx.User == nil {
		return authboss.InterruptNone, errUserMissing
	}

	if locked, ok := ctx.User.DateTime(StoreLocked); ok && locked.After(time.Now().UTC()) {
		return authboss.InterruptAccountLocked, nil
	}

	return authboss.InterruptNone, nil
}

// afterAuth resets the attempt number field.
func (l *Lock) afterAuth(ctx *authboss.Context) error {
	if ctx.User == nil {
		return errUserMissing
	}

	ctx.User[StoreAttemptNumber] = int64(0)
	ctx.User[StoreAttemptTime] = time.Now().UTC()

	if err := ctx.SaveUser(); err != nil {
		return err
	}

	return nil
}

// afterAuthFail adjusts the attempt number and time.
func (l *Lock) afterAuthFail(ctx *authboss.Context) error {
	if ctx.User == nil {
		return errUserMissing
	}

	lastAttempt := time.Now().UTC()
	if attemptTime, ok := ctx.User.DateTime(StoreAttemptTime); ok {
		lastAttempt = attemptTime
	}

	var nAttempts int64
	if attempts, ok := ctx.User.Int64(StoreAttemptNumber); ok {
		nAttempts = attempts
	}

	nAttempts++

	if time.Now().UTC().Sub(lastAttempt) <= l.LockWindow {
		if nAttempts >= int64(l.LockAfter) {
			ctx.User[StoreLocked] = time.Now().UTC().Add(l.LockDuration)
		}

		ctx.User[StoreAttemptNumber] = nAttempts
	} else {
		ctx.User[StoreAttemptNumber] = int64(0)
	}
	ctx.User[StoreAttemptTime] = time.Now().UTC()

	if err := ctx.SaveUser(); err != nil {
		return err
	}

	return nil
}

// Lock a user manually.
func (l *Lock) Lock(key string) error {
	user, err := l.Storer.Get(key)
	if err != nil {
		return err
	}

	attr := authboss.Unbind(user)
	if err != nil {
		return err
	}

	attr[StoreLocked] = time.Now().UTC().Add(l.LockDuration)

	return l.Storer.Put(key, attr)
}

// Unlock a user that was locked by this module.
func (l *Lock) Unlock(key string) error {
	user, err := l.Storer.Get(key)
	if err != nil {
		return err
	}

	attr := authboss.Unbind(user)
	if err != nil {
		return err
	}

	// Set the last attempt to be -window*2 to avoid immediately
	// giving another login failure.
	attr[StoreAttemptTime] = time.Now().UTC().Add(-l.LockWindow * 2)
	attr[StoreAttemptNumber] = int64(0)
	attr[StoreLocked] = time.Now().UTC().Add(-l.LockDuration)

	return l.Storer.Put(key, attr)
}
<commit_msg>AttemptNumber should be set to 1 when login fails again after the lock windown is passed<commit_after>// Package lock implements user locking after N bad sign-in attempts.
package lock

import (
	"errors"
	"time"

	"gopkg.in/authboss.v0"
)

// Storage key constants
const (
	StoreAttemptNumber = "attempt_number"
	StoreAttemptTime   = "attempt_time"
	StoreLocked        = "locked"
)

var (
	errUserMissing = errors.New("lock: user not loaded in BeforeAuth callback")
)

func init() {
	authboss.RegisterModule("lock", &Lock{})
}

// Lock module
type Lock struct {
	*authboss.Authboss
}

// Initialize the module
func (l *Lock) Initialize(ab *authboss.Authboss) error {
	l.Authboss = ab
	if l.Storer == nil && l.StoreMaker == nil {
		return errors.New("lock: Need a Storer")
	}

	// Events
	l.Callbacks.After(authboss.EventGetUser, func(ctx *authboss.Context) error {
		_, err := l.beforeAuth(ctx)
		return err
	})
	l.Callbacks.Before(authboss.EventAuth, l.beforeAuth)
	l.Callbacks.After(authboss.EventAuth, l.afterAuth)
	l.Callbacks.After(authboss.EventAuthFail, l.afterAuthFail)

	return nil
}

// Routes for the module
func (l *Lock) Routes() authboss.RouteTable {
	return nil
}

// Storage requirements
func (l *Lock) Storage() authboss.StorageOptions {
	return authboss.StorageOptions{
		l.PrimaryID:        authboss.String,
		StoreAttemptNumber: authboss.Integer,
		StoreAttemptTime:   authboss.DateTime,
		StoreLocked:        authboss.DateTime,
	}
}

// beforeAuth ensures the account is not locked.
func (l *Lock) beforeAuth(ctx *authboss.Context) (authboss.Interrupt, error) {
	if ctx.User == nil {
		return authboss.InterruptNone, errUserMissing
	}

	if locked, ok := ctx.User.DateTime(StoreLocked); ok && locked.After(time.Now().UTC()) {
		return authboss.InterruptAccountLocked, nil
	}

	return authboss.InterruptNone, nil
}

// afterAuth resets the attempt number field.
func (l *Lock) afterAuth(ctx *authboss.Context) error {
	if ctx.User == nil {
		return errUserMissing
	}

	ctx.User[StoreAttemptNumber] = int64(0)
	ctx.User[StoreAttemptTime] = time.Now().UTC()

	if err := ctx.SaveUser(); err != nil {
		return err
	}

	return nil
}

// afterAuthFail adjusts the attempt number and time.
func (l *Lock) afterAuthFail(ctx *authboss.Context) error {
	if ctx.User == nil {
		return errUserMissing
	}

	lastAttempt := time.Now().UTC()
	if attemptTime, ok := ctx.User.DateTime(StoreAttemptTime); ok {
		lastAttempt = attemptTime
	}

	var nAttempts int64
	if attempts, ok := ctx.User.Int64(StoreAttemptNumber); ok {
		nAttempts = attempts
	}

	nAttempts++

	if time.Now().UTC().Sub(lastAttempt) <= l.LockWindow {
		if nAttempts >= int64(l.LockAfter) {
			ctx.User[StoreLocked] = time.Now().UTC().Add(l.LockDuration)
		}

		ctx.User[StoreAttemptNumber] = nAttempts
	} else {
		ctx.User[StoreAttemptNumber] = int64(1)
	}
	ctx.User[StoreAttemptTime] = time.Now().UTC()

	if err := ctx.SaveUser(); err != nil {
		return err
	}

	return nil
}

// Lock a user manually.
func (l *Lock) Lock(key string) error {
	user, err := l.Storer.Get(key)
	if err != nil {
		return err
	}

	attr := authboss.Unbind(user)
	if err != nil {
		return err
	}

	attr[StoreLocked] = time.Now().UTC().Add(l.LockDuration)

	return l.Storer.Put(key, attr)
}

// Unlock a user that was locked by this module.
func (l *Lock) Unlock(key string) error {
	user, err := l.Storer.Get(key)
	if err != nil {
		return err
	}

	attr := authboss.Unbind(user)
	if err != nil {
		return err
	}

	// Set the last attempt to be -window*2 to avoid immediately
	// giving another login failure.
	attr[StoreAttemptTime] = time.Now().UTC().Add(-l.LockWindow * 2)
	attr[StoreAttemptNumber] = int64(0)
	attr[StoreLocked] = time.Now().UTC().Add(-l.LockDuration)

	return l.Storer.Put(key, attr)
}
<|endoftext|>


<commit_before>package steam

import (
	"encoding/json"
	"errors"
	"io"
	"net/http"

	"fmt"
	"net/url"
	"strconv"

	"github.com/PuerkitoBio/goquery"
)

type Confirmation struct {
	ID        uint64
	Key       uint64
	Title     string
	Receiving string
	Since     string
	OfferID   uint64
}

const offerIDPart = "tradeofferid_"

var (
	ErrConfirmationsUnknownError = errors.New("unknown error occurered finding confirmations")
	ErrCannotFindConfirmations   = errors.New("unable to find confirmations")
	ErrCannotFindDescriptions    = errors.New("unable to find confirmation descriptions")
	ErrConfiramtionsDescMismatch = errors.New("cannot match confirmations with their respective descriptions")
	ErrConfirmationOfferIDFail   = errors.New("unable to get confirmation offer id")
	ErrCannotFindTradeOffer      = errors.New("unable to find tradeoffer div to get offer id for confirmation")
	ErrCannotFindOfferIDAttr     = errors.New("unable to find offer ID attribute")
)

func (session *Session) execConfirmationRequest(request, key, tag string, current int64, values map[string]interface{}) (*http.Response, error) {
	params := url.Values{
		"p":   {session.deviceID},
		"a":   {session.oauth.SteamID.ToString()},
		"k":   {key},
		"t":   {strconv.FormatInt(current, 10)},
		"m":   {"android"},
		"tag": {tag},
	}

	if values != nil {
		for k, v := range values {
			switch v := v.(type) {
			case string:
				params.Add(k, v)
			case uint64:
				params.Add(k, strconv.FormatUint(v, 10))
			default:
				return nil, fmt.Errorf("execConfirmationRequest: missing implementation for type %v", v)
			}
		}
	}

	return session.client.Get("https://steamcommunity.com/mobileconf/" + request + params.Encode())
}

func (session *Session) GetConfirmations(identitySecret string, current int64) ([]*Confirmation, error) {
	key, err := GenerateConfirmationCode(identitySecret, "conf", current)
	if err != nil {
		return nil, err
	}

	resp, err := session.execConfirmationRequest("conf?", key, "conf", current, nil)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(io.Reader(resp.Body))
	if err != nil {
		return nil, err
	}

	/* FIXME: broken
	if empty := doc.Find(".mobileconf_empty"); empty != nil {
		if done := doc.Find(".mobileconf_done"); done != nil {
			return nil, nil
		}

		return nil, ErrConfirmationsUnknownError // FIXME
	}
	*/

	entries := doc.Find(".mobileconf_list_entry")
	if entries == nil {
		return nil, ErrCannotFindConfirmations
	}

	descriptions := doc.Find(".mobileconf_list_entry_description")
	if descriptions == nil {
		return nil, ErrCannotFindDescriptions
	}

	if len(entries.Nodes) != len(descriptions.Nodes) {
		return nil, ErrConfiramtionsDescMismatch
	}

	confirmations := []*Confirmation{}
	for k, sel := range entries.Nodes {
		confirmation := &Confirmation{}
		for _, attr := range sel.Attr {
			if attr.Key == "data-confid" {
				confirmation.ID, _ = strconv.ParseUint(attr.Val, 10, 32)
			} else if attr.Key == "data-key" {
				confirmation.Key, _ = strconv.ParseUint(attr.Val, 10, 64)
			} else if attr.Key == "data-creator" {
				confirmation.OfferID, _ = strconv.ParseUint(attr.Val, 10, 64)
			}
		}

		descSel := descriptions.Nodes[k]
		depth := 0
		for child := descSel.FirstChild; child != nil; child = child.NextSibling {
			for n := child.FirstChild; n != nil; n = n.NextSibling {
				switch depth {
				case 0:
					confirmation.Title = n.Data
				case 1:
					confirmation.Receiving = n.Data
				case 2:
					confirmation.Since = n.Data
				}
				depth++
			}
		}

		confirmations = append(confirmations, confirmation)
	}

	return confirmations, nil
}

func (session *Session) AnswerConfirmation(confirmation *Confirmation, identitySecret, answer string, current int64) error {
	key, err := GenerateConfirmationCode(identitySecret, answer, current)
	if err != nil {
		return err
	}

	op := map[string]interface{}{
		"op":  answer,
		"cid": uint64(confirmation.ID),
		"ck":  confirmation.Key,
	}

	resp, err := session.execConfirmationRequest("ajaxop?", key, answer, current, op)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	type Response struct {
		Success bool   `json:"success"`
		Message string `json:"message"`
	}

	var response Response
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return err
	}

	if !response.Success {
		return errors.New(response.Message)
	}

	return nil
}

func (confirmation *Confirmation) Answer(session *Session, key, answer string, current int64) error {
	return session.AnswerConfirmation(confirmation, key, answer, current)
}
<commit_msg>Fix confirmations bug and remove unused variables<commit_after>package steam

import (
	"encoding/json"
	"errors"
	"io"
	"net/http"

	"fmt"
	"net/url"
	"strconv"

	"github.com/PuerkitoBio/goquery"
)

type Confirmation struct {
	ID        uint64
	Key       uint64
	Title     string
	Receiving string
	Since     string
	OfferID   uint64
}

var (
	ErrConfirmationsUnknownError = errors.New("unknown error occurered finding confirmations")
	ErrCannotFindConfirmations   = errors.New("unable to find confirmations")
	ErrCannotFindDescriptions    = errors.New("unable to find confirmation descriptions")
	ErrConfiramtionsDescMismatch = errors.New("cannot match confirmations with their respective descriptions")
)

func (session *Session) execConfirmationRequest(request, key, tag string, current int64, values map[string]interface{}) (*http.Response, error) {
	params := url.Values{
		"p":   {session.deviceID},
		"a":   {session.oauth.SteamID.ToString()},
		"k":   {key},
		"t":   {strconv.FormatInt(current, 10)},
		"m":   {"android"},
		"tag": {tag},
	}

	if values != nil {
		for k, v := range values {
			switch v := v.(type) {
			case string:
				params.Add(k, v)
			case uint64:
				params.Add(k, strconv.FormatUint(v, 10))
			default:
				return nil, fmt.Errorf("execConfirmationRequest: missing implementation for type %v", v)
			}
		}
	}

	return session.client.Get("https://steamcommunity.com/mobileconf/" + request + params.Encode())
}

func (session *Session) GetConfirmations(identitySecret string, current int64) ([]*Confirmation, error) {
	key, err := GenerateConfirmationCode(identitySecret, "conf", current)
	if err != nil {
		return nil, err
	}

	resp, err := session.execConfirmationRequest("conf?", key, "conf", current, nil)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(io.Reader(resp.Body))
	if err != nil {
		return nil, err
	}

	/* FIXME: broken
	if empty := doc.Find(".mobileconf_empty"); empty != nil {
		if done := doc.Find(".mobileconf_done"); done != nil {
			return nil, nil
		}

		return nil, ErrConfirmationsUnknownError // FIXME
	}
	*/

	entries := doc.Find(".mobileconf_list_entry")
	if entries == nil {
		return nil, ErrCannotFindConfirmations
	}

	descriptions := doc.Find(".mobileconf_list_entry_description")
	if descriptions == nil {
		return nil, ErrCannotFindDescriptions
	}

	if len(entries.Nodes) != len(descriptions.Nodes) {
		return nil, ErrConfiramtionsDescMismatch
	}

	confirmations := []*Confirmation{}
	for k, sel := range entries.Nodes {
		confirmation := &Confirmation{}
		for _, attr := range sel.Attr {
			if attr.Key == "data-confid" {
				confirmation.ID, _ = strconv.ParseUint(attr.Val, 10, 64)
			} else if attr.Key == "data-key" {
				confirmation.Key, _ = strconv.ParseUint(attr.Val, 10, 64)
			} else if attr.Key == "data-creator" {
				confirmation.OfferID, _ = strconv.ParseUint(attr.Val, 10, 64)
			}
		}

		descSel := descriptions.Nodes[k]
		depth := 0
		for child := descSel.FirstChild; child != nil; child = child.NextSibling {
			for n := child.FirstChild; n != nil; n = n.NextSibling {
				switch depth {
				case 0:
					confirmation.Title = n.Data
				case 1:
					confirmation.Receiving = n.Data
				case 2:
					confirmation.Since = n.Data
				}
				depth++
			}
		}

		confirmations = append(confirmations, confirmation)
	}

	return confirmations, nil
}

func (session *Session) AnswerConfirmation(confirmation *Confirmation, identitySecret, answer string, current int64) error {
	key, err := GenerateConfirmationCode(identitySecret, answer, current)
	if err != nil {
		return err
	}

	op := map[string]interface{}{
		"op":  answer,
		"cid": uint64(confirmation.ID),
		"ck":  confirmation.Key,
	}

	resp, err := session.execConfirmationRequest("ajaxop?", key, answer, current, op)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	type Response struct {
		Success bool   `json:"success"`
		Message string `json:"message"`
	}

	var response Response
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return err
	}

	if !response.Success {
		return errors.New(response.Message)
	}

	return nil
}

func (confirmation *Confirmation) Answer(session *Session, key, answer string, current int64) error {
	return session.AnswerConfirmation(confirmation, key, answer, current)
}
<|endoftext|>


<commit_before>/**
  Steam Library For Go
  Copyright (C) 2016 Ahmed Samy <f.fallen45@gmail.com>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/
package steam

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

type Confirmation struct {
	ID        uint64
	Key       uint64
	Title     string
	Receiving string
	Since     string
}

const offerIDPart = "tradeofferid_"

var (
	ErrConfirmationsUnknownError = errors.New("unknown error occurered finding confirmations")
	ErrCannotFindConfirmations   = errors.New("unable to find confirmations")
	ErrCannotFindDescriptions    = errors.New("unable to find confirmation descriptions")
	ErrConfiramtionsDescMismatch = errors.New("cannot match confirmations with their respective descriptions")
	ErrConfirmationOfferIDFail   = errors.New("unable to get confirmation offer id")
	ErrCannotFindTradeOffer      = errors.New("unable to find tradeoffer div to get offer id for confirmation")
	ErrCannotFindOfferIDAttr     = errors.New("unable to find offer ID attribute")
)

func (community *Community) execConfirmationRequest(request, key, tag string, current int64, values map[string]interface{}) (*http.Response, error) {
	params := url.Values{
		"p":   {community.deviceID},
		"a":   {community.steamID.ToString()},
		"k":   {key},
		"t":   {strconv.FormatInt(current, 10)},
		"m":   {"android"},
		"tag": {tag},
	}

	if values != nil {
		for k, v := range values {
			switch v.(type) {
			case string:
				params.Add(k, v.(string))
			case uint64:
				params.Add(k, strconv.FormatUint(v.(uint64), 10))
			default:
				return nil, fmt.Errorf("execConfirmationRequest: Please implement case for this type %v", v)
			}
		}
	}

	req, err := http.NewRequest(http.MethodGet, "https://steamcommunity.com/mobileconf/"+request+"?"+params.Encode(), nil)
	if err != nil {
		return nil, err
	}

	return community.client.Do(req)
}

func (community *Community) GetConfirmations(key string) ([]*Confirmation, error) {
	resp, err := community.execConfirmationRequest("conf", key, "conf", time.Now().Unix(), nil)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(io.Reader(resp.Body))
	if err != nil {
		return nil, err
	}

	if empty := doc.Find(".mobileconf_empty"); empty != nil {
		if done := doc.Find(".mobileconf_done"); done != nil {
			return nil, nil
		}

		return nil, ErrConfirmationsUnknownError // FIXME
	}

	entries := doc.Find(".mobileconf_list_entry")
	if entries == nil {
		return nil, ErrCannotFindConfirmations
	}

	descriptions := doc.Find(".mobileconf_list_entry_description")
	if descriptions == nil {
		return nil, ErrCannotFindDescriptions
	}

	if len(entries.Nodes) != len(descriptions.Nodes) {
		return nil, ErrConfiramtionsDescMismatch
	}

	confirmations := []*Confirmation{}
	for k, sel := range entries.Nodes {
		confirmation := &Confirmation{}
		for _, attr := range sel.Attr {
			if attr.Key == "data-confid" {
				confirmation.ID, _ = strconv.ParseUint(attr.Val, 10, 32)
			} else if attr.Key == "data-key" {
				confirmation.Key, _ = strconv.ParseUint(attr.Val, 10, 64)
			}
		}

		descSel := descriptions.Nodes[k]
		depth := 0
		for child := descSel.FirstChild; child != nil; child = child.NextSibling {
			for n := child.FirstChild; n != nil; n = n.NextSibling {
				switch depth {
				case 0:
					confirmation.Title = n.Data
				case 1:
					confirmation.Receiving = n.Data
				case 2:
					confirmation.Since = n.Data
				}
				depth++
			}
		}

		confirmations = append(confirmations, confirmation)
	}

	return confirmations, nil
}

func (community *Community) GetConfirmationOfferID(key string, cid uint64) (uint64, error) {
	resp, err := community.execConfirmationRequest(fmt.Sprintf("details/%d", cid), key, "details", time.Now().Unix(), nil)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return 0, err
	}

	type Response struct {
		Success bool   `json:"success"`
		HTML    string `json:"html"`
	}

	var r Response
	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return 0, err
	}

	if !r.Success {
		return 0, ErrConfirmationOfferIDFail
	}

	doc, err := goquery.NewDocumentFromReader(strings.NewReader(r.HTML))
	if err != nil {
		return 0, err
	}

	offer := doc.Find(".tradeoffer")
	if offer == nil {
		return 0, ErrCannotFindTradeOffer
	}

	val, ok := offer.Attr("id")
	if !ok || len(val) <= len(offerIDPart) || val[:len(offerIDPart)] != offerIDPart {
		return 0, ErrCannotFindOfferIDAttr
	}

	id := val[len(offerIDPart):]
	raw, err := strconv.ParseUint(id, 10, 64)
	if err != nil {
		return 0, err
	}

	return raw, nil
}

func (community *Community) AnswerConfirmation(confirmation *Confirmation, key, answer string) error {
	op := map[string]interface{}{
		"op":  answer,
		"cid": uint64(confirmation.ID),
		"ck":  confirmation.Key,
	}

	resp, err := community.execConfirmationRequest("ajaxop", key, answer, time.Now().Unix(), op)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	type Response struct {
		Success bool   `json:"success"`
		Message string `json:"message"`
	}

	var r Response
	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
		return err
	}

	if !r.Success {
		return errors.New(r.Message)
	}

	return nil
}

func (confirmation *Confirmation) GetOfferID(community *Community, key string) (uint64, error) {
	return community.GetConfirmationOfferID(key, confirmation.ID)
}

func (confirmation *Confirmation) Answer(community *Community, key, answer string) error {
	return community.AnswerConfirmation(confirmation, key, answer)
}
<commit_msg>confirmations: sync with style<commit_after>/**
  Steam Library For Go
  Copyright (C) 2016 Ahmed Samy <f.fallen45@gmail.com>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
*/
package steam

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
)

type Confirmation struct {
	ID        uint64
	Key       uint64
	Title     string
	Receiving string
	Since     string
}

const offerIDPart = "tradeofferid_"

var (
	ErrConfirmationsUnknownError = errors.New("unknown error occurered finding confirmations")
	ErrCannotFindConfirmations   = errors.New("unable to find confirmations")
	ErrCannotFindDescriptions    = errors.New("unable to find confirmation descriptions")
	ErrConfiramtionsDescMismatch = errors.New("cannot match confirmations with their respective descriptions")
	ErrConfirmationOfferIDFail   = errors.New("unable to get confirmation offer id")
	ErrCannotFindTradeOffer      = errors.New("unable to find tradeoffer div to get offer id for confirmation")
	ErrCannotFindOfferIDAttr     = errors.New("unable to find offer ID attribute")
)

func (community *Community) execConfirmationRequest(request, key, tag string, current int64, values map[string]interface{}) (*http.Response, error) {
	params := url.Values{
		"p":   {community.deviceID},
		"a":   {community.steamID.ToString()},
		"k":   {key},
		"t":   {strconv.FormatInt(current, 10)},
		"m":   {"android"},
		"tag": {tag},
	}

	if values != nil {
		for k, v := range values {
			switch v.(type) {
			case string:
				params.Add(k, v.(string))
			case uint64:
				params.Add(k, strconv.FormatUint(v.(uint64), 10))
			default:
				return nil, fmt.Errorf("execConfirmationRequest: Please implement case for this type %v", v)
			}
		}
	}

	req, err := http.NewRequest(http.MethodGet, "https://steamcommunity.com/mobileconf/"+request+"?"+params.Encode(), nil)
	if err != nil {
		return nil, err
	}

	return community.client.Do(req)
}

func (community *Community) GetConfirmations(key string) ([]*Confirmation, error) {
	resp, err := community.execConfirmationRequest("conf", key, "conf", time.Now().Unix(), nil)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromReader(io.Reader(resp.Body))
	if err != nil {
		return nil, err
	}

	if empty := doc.Find(".mobileconf_empty"); empty != nil {
		if done := doc.Find(".mobileconf_done"); done != nil {
			return nil, nil
		}

		return nil, ErrConfirmationsUnknownError // FIXME
	}

	entries := doc.Find(".mobileconf_list_entry")
	if entries == nil {
		return nil, ErrCannotFindConfirmations
	}

	descriptions := doc.Find(".mobileconf_list_entry_description")
	if descriptions == nil {
		return nil, ErrCannotFindDescriptions
	}

	if len(entries.Nodes) != len(descriptions.Nodes) {
		return nil, ErrConfiramtionsDescMismatch
	}

	confirmations := []*Confirmation{}
	for k, sel := range entries.Nodes {
		confirmation := &Confirmation{}
		for _, attr := range sel.Attr {
			if attr.Key == "data-confid" {
				confirmation.ID, _ = strconv.ParseUint(attr.Val, 10, 32)
			} else if attr.Key == "data-key" {
				confirmation.Key, _ = strconv.ParseUint(attr.Val, 10, 64)
			}
		}

		descSel := descriptions.Nodes[k]
		depth := 0
		for child := descSel.FirstChild; child != nil; child = child.NextSibling {
			for n := child.FirstChild; n != nil; n = n.NextSibling {
				switch depth {
				case 0:
					confirmation.Title = n.Data
				case 1:
					confirmation.Receiving = n.Data
				case 2:
					confirmation.Since = n.Data
				}
				depth++
			}
		}

		confirmations = append(confirmations, confirmation)
	}

	return confirmations, nil
}

func (community *Community) GetConfirmationOfferID(key string, cid uint64) (uint64, error) {
	resp, err := community.execConfirmationRequest(fmt.Sprintf("details/%d", cid), key, "details", time.Now().Unix(), nil)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return 0, err
	}

	type Response struct {
		Success bool   `json:"success"`
		HTML    string `json:"html"`
	}

	var response Response
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return 0, err
	}

	if !response.Success {
		return 0, ErrConfirmationOfferIDFail
	}

	doc, err := goquery.NewDocumentFromReader(strings.NewReader(response.HTML))
	if err != nil {
		return 0, err
	}

	offer := doc.Find(".tradeoffer")
	if offer == nil {
		return 0, ErrCannotFindTradeOffer
	}

	val, ok := offer.Attr("id")
	if !ok || len(val) <= len(offerIDPart) || val[:len(offerIDPart)] != offerIDPart {
		return 0, ErrCannotFindOfferIDAttr
	}

	id := val[len(offerIDPart):]
	raw, err := strconv.ParseUint(id, 10, 64)
	if err != nil {
		return 0, err
	}

	return raw, nil
}

func (community *Community) AnswerConfirmation(confirmation *Confirmation, key, answer string) error {
	op := map[string]interface{}{
		"op":  answer,
		"cid": uint64(confirmation.ID),
		"ck":  confirmation.Key,
	}

	resp, err := community.execConfirmationRequest("ajaxop", key, answer, time.Now().Unix(), op)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		return err
	}

	type Response struct {
		Success bool   `json:"success"`
		Message string `json:"message"`
	}

	var response Response
	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
		return err
	}

	if !response.Success {
		return errors.New(response.Message)
	}

	return nil
}

func (confirmation *Confirmation) GetOfferID(community *Community, key string) (uint64, error) {
	return community.GetConfirmationOfferID(key, confirmation.ID)
}

func (confirmation *Confirmation) Answer(community *Community, key, answer string) error {
	return community.AnswerConfirmation(confirmation, key, answer)
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"log"
	"net/url"
	"strings"

	"github.com/gliderlabs/registrator/bridge"
	consulapi "github.com/hashicorp/consul/api"
)

const DefaultInterval = "10s"

func init() {
	f := new(Factory)
	bridge.Register(f, "consul")
	bridge.Register(f, "consul-unix")
}

func (r *ConsulAdapter) interpolateService(script string, service *bridge.Service) string {
	withIp := strings.Replace(script, "$SERVICE_IP", service.Origin.HostIP, -1)
	withPort := strings.Replace(withIp, "$SERVICE_PORT", service.Origin.HostPort, -1)
	return withPort
}

type Factory struct{}

func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	config := consulapi.DefaultConfig()
	if uri.Scheme == "consul-unix" {
		config.Address = strings.TrimPrefix(uri.String(), "consul-")
	} else if uri.Host != "" {
		config.Address = uri.Host
	}
	client, err := consulapi.NewClient(config)
	if err != nil {
		log.Fatal("consul: ", uri.Scheme)
	}
	return &ConsulAdapter{client: client}
}

type ConsulAdapter struct {
	client *consulapi.Client
}

// Ping will try to connect to consul by attempting to retrieve the current leader.
func (r *ConsulAdapter) Ping() error {
	status := r.client.Status()
	leader, err := status.Leader()
	if err != nil {
		return err
	}
	log.Println("consul: current leader ", leader)

	return nil
}

func (r *ConsulAdapter) Register(service *bridge.Service) error {
	registration := new(consulapi.AgentServiceRegistration)
	registration.ID = service.ID
	registration.Name = service.Name
	registration.Port = service.Port
	registration.Tags = service.Tags
	registration.Address = service.IP
	registration.Check = r.buildCheck(service)
	return r.client.Agent().ServiceRegister(registration)
}

func (r *ConsulAdapter) buildCheck(service *bridge.Service) *consulapi.AgentServiceCheck {
	check := new(consulapi.AgentServiceCheck)
	if path := service.Attrs["check_http"]; path != "" {
		check.HTTP = fmt.Sprintf("http://%s:%d%s", service.IP, service.Port, path)
		if timeout := service.Attrs["check_timeout"]; timeout != "" {
			check.Timeout = timeout
		}
	} else if cmd := service.Attrs["check_cmd"]; cmd != "" {
		check.Script = fmt.Sprintf("check-cmd %s %s %s", service.Origin.ContainerID[:12], service.Origin.ExposedPort, cmd)
	} else if script := service.Attrs["check_script"]; script != "" {
		check.Script = r.interpolateService(script, service)
	} else if ttl := service.Attrs["check_ttl"]; ttl != "" {
		check.TTL = ttl
	} else if tcp := service.Attrs["check_tcp"]; tcp != "" {
		check.TCP = fmt.Sprintf("%s:%d", service.IP, service.Port)
		if timeout := service.Attrs["check_timeout"]; timeout != "" {
			check.Timeout = timeout
		}
	} else {
		return nil
	}
	if check.Script != "" || check.HTTP != "" || check.TCP != "" {
		if interval := service.Attrs["check_interval"]; interval != "" {
			check.Interval = interval
		} else {
			check.Interval = DefaultInterval
		}
	}
	return check
}

func (r *ConsulAdapter) Deregister(service *bridge.Service) error {
	return r.client.Agent().ServiceDeregister(service.ID)
}

func (r *ConsulAdapter) Refresh(service *bridge.Service) error {
	return nil
}

func (r *ConsulAdapter) Services() ([]*bridge.Service, error) {
	services, err := r.client.Agent().Services()
	if err != nil {
		return []*bridge.Service{}, err
	}
	out := make([]*bridge.Service, len(services))
	i := 0
	for _, v := range services {
		s := &bridge.Service{
			ID:   v.ID,
			Name: v.Service,
			Port: v.Port,
			Tags: v.Tags,
			IP:   v.Address,
		}
		out[i] = s
		i++
	}
	return out, nil
}
<commit_msg>adding TLS support for consul backend<commit_after>package consul

import (
	"fmt"
	"log"
	"net/url"
	"strings"
	"os"
	"github.com/gliderlabs/registrator/bridge"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
)

const DefaultInterval = "10s"

func init() {
	f := new(Factory)
	bridge.Register(f, "consul-tls")
	bridge.Register(f, "consul")
	bridge.Register(f, "consul-unix")
}

func (r *ConsulAdapter) interpolateService(script string, service *bridge.Service) string {
	withIp := strings.Replace(script, "$SERVICE_IP", service.Origin.HostIP, -1)
	withPort := strings.Replace(withIp, "$SERVICE_PORT", service.Origin.HostPort, -1)
	return withPort
}

type Factory struct{}

func (f *Factory) New(uri *url.URL) bridge.RegistryAdapter {
	config := consulapi.DefaultConfig()
	if uri.Scheme == "consul-unix" {
		config.Address = strings.TrimPrefix(uri.String(), "consul-")
	} else if uri.Scheme == "consul-tls" {
	        tlsConfigDesc := &consulapi.TLSConfig {
			  Address: uri.Host,
			  CAFile: os.Getenv("CONSUL_CACERT"),
  			  CertFile: os.Getenv("CONSUL_TLSCERT"),
  			  KeyFile: os.Getenv("CONSUL_TLSKEY"),
			  InsecureSkipVerify: false,
		}
		tlsConfig, err := consulapi.SetupTLSConfig(tlsConfigDesc)
		if err != nil {
		   log.Fatal("Cannot set up Consul TLSConfig", err)
		}
		config.Scheme = "https"
		transport := cleanhttp.DefaultPooledTransport()
		transport.TLSClientConfig = tlsConfig
		config.HttpClient.Transport = transport
		config.Address = uri.Host
	} else if uri.Host != "" {
		config.Address = uri.Host
	}
	client, err := consulapi.NewClient(config)
	if err != nil {
		log.Fatal("consul: ", uri.Scheme)
	}
	return &ConsulAdapter{client: client}
}

type ConsulAdapter struct {
	client *consulapi.Client
}

// Ping will try to connect to consul by attempting to retrieve the current leader.
func (r *ConsulAdapter) Ping() error {
	status := r.client.Status()
	leader, err := status.Leader()
	if err != nil {
		return err
	}
	log.Println("consul: current leader ", leader)

	return nil
}

func (r *ConsulAdapter) Register(service *bridge.Service) error {
	registration := new(consulapi.AgentServiceRegistration)
	registration.ID = service.ID
	registration.Name = service.Name
	registration.Port = service.Port
	registration.Tags = service.Tags
	registration.Address = service.IP
	registration.Check = r.buildCheck(service)
	return r.client.Agent().ServiceRegister(registration)
}

func (r *ConsulAdapter) buildCheck(service *bridge.Service) *consulapi.AgentServiceCheck {
	check := new(consulapi.AgentServiceCheck)
	if path := service.Attrs["check_http"]; path != "" {
		check.HTTP = fmt.Sprintf("http://%s:%d%s", service.IP, service.Port, path)
		if timeout := service.Attrs["check_timeout"]; timeout != "" {
			check.Timeout = timeout
		}
	} else if cmd := service.Attrs["check_cmd"]; cmd != "" {
		check.Script = fmt.Sprintf("check-cmd %s %s %s", service.Origin.ContainerID[:12], service.Origin.ExposedPort, cmd)
	} else if script := service.Attrs["check_script"]; script != "" {
		check.Script = r.interpolateService(script, service)
	} else if ttl := service.Attrs["check_ttl"]; ttl != "" {
		check.TTL = ttl
	} else if tcp := service.Attrs["check_tcp"]; tcp != "" {
		check.TCP = fmt.Sprintf("%s:%d", service.IP, service.Port)
		if timeout := service.Attrs["check_timeout"]; timeout != "" {
			check.Timeout = timeout
		}
	} else {
		return nil
	}
	if check.Script != "" || check.HTTP != "" || check.TCP != "" {
		if interval := service.Attrs["check_interval"]; interval != "" {
			check.Interval = interval
		} else {
			check.Interval = DefaultInterval
		}
	}
	return check
}

func (r *ConsulAdapter) Deregister(service *bridge.Service) error {
	return r.client.Agent().ServiceDeregister(service.ID)
}

func (r *ConsulAdapter) Refresh(service *bridge.Service) error {
	return nil
}

func (r *ConsulAdapter) Services() ([]*bridge.Service, error) {
	services, err := r.client.Agent().Services()
	if err != nil {
		return []*bridge.Service{}, err
	}
	out := make([]*bridge.Service, len(services))
	i := 0
	for _, v := range services {
		s := &bridge.Service{
			ID:   v.ID,
			Name: v.Service,
			Port: v.Port,
			Tags: v.Tags,
			IP:   v.Address,
		}
		out[i] = s
		i++
	}
	return out, nil
}
<|endoftext|>


<commit_before>package tq

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/rubyist/tracerx"
)

const (
	TusAdapterName = "tus"
	TusVersion     = "1.0.0"
)

// Adapter for tus.io protocol resumaable uploads
type tusUploadAdapter struct {
	*adapterBase
}

func (a *tusUploadAdapter) ClearTempStorage() error {
	// nothing to do, all temp state is on the server end
	return nil
}

func (a *tusUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *tusUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	rel, err := t.Rel("upload")
	if err != nil {
		return err
	}
	if rel == nil {
		return errors.Errorf("No upload action for object: %s", t.Oid)
	}

	// Note not supporting the Creation extension since the batch API generates URLs
	// Also not supporting Concatenation to support parallel uploads of chunks; forward only

	// 1. Send HEAD request to determine upload start point
	//    Request must include Tus-Resumable header (version)
	tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Oid)
	req, err := a.newHTTPRequest("HEAD", rel)
	if err != nil {
		return err
	}

	req.Header.Set("Tus-Resumable", TusVersion)

	res, err := a.doHTTP(t, req)
	if err != nil {
		return errors.NewRetriableError(err)
	}

	//    Response will contain Upload-Offset if supported
	offHdr := res.Header.Get("Upload-Offset")
	if len(offHdr) == 0 {
		return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
	}
	offset, err := strconv.ParseInt(offHdr, 10, 64)
	if err != nil || offset < 0 {
		return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
	}
	// Upload-Offset=size means already completed (skip)
	// Batch API will probably already detect this, but handle just in case
	if offset >= t.Size {
		tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Oid)
		advanceCallbackProgress(cb, t, t.Size)
		return nil
	}

	// Open file for uploading
	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errors.Wrap(err, "tus upload")
	}
	defer f.Close()

	// Upload-Offset=0 means start from scratch, but still send PATCH
	if offset == 0 {
		tracerx.Printf("xfer: tus.io uploading %q from start", t.Oid)
	} else {
		tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Oid, offset)
		advanceCallbackProgress(cb, t, offset)
	}

	// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
	//    Response status must be 204
	//    Response Upload-Offset must be request Upload-Offset plus sent bytes
	//    Response may include Upload-Expires header in which case check not passed

	tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Oid)
	req, err = a.newHTTPRequest("PATCH", rel)
	if err != nil {
		return err
	}

	req.Header.Set("Tus-Resumable", TusVersion)
	req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
	req.Header.Set("Content-Type", "application/offset+octet-stream")
	req.Header.Set("Content-Length", strconv.FormatInt(t.Size-offset, 10))
	req.ContentLength = t.Size - offset

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar, readSinceLast)
		}
		return nil
	}

	var reader lfsapi.ReadSeekCloser = progress.NewBodyWithCallback(f, t.Size, ccb)
	reader = newStartCallbackReader(reader, func() error {
		// seek to the offset since lfsapi.Client rewinds the body
		if _, err := f.Seek(offset, os.SEEK_CUR); err != nil {
			return err
		}
		// Signal auth was ok on first read; this frees up other workers to start
		if authOkFunc != nil {
			authOkFunc()
		}
		return nil
	})

	req.Body = reader

	res, err = a.doHTTP(t, req)
	if err != nil {
		return errors.NewRetriableError(err)
	}

	a.apiClient.LogResponse("lfs.data.upload", res)

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		err = errors.New("http: received status 403")
		return errors.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errors.Wrapf(nil, "Invalid status for %s %s: %d",
			req.Method,
			strings.SplitN(req.URL.String(), "?", 2)[0],
			res.StatusCode,
		)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	cli := &lfsapi.Client{}
	return verifyUpload(cli, t)
}

func configureTusAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(TusAdapterName, Upload, func(name string, dir Direction) Adapter {
		switch dir {
		case Upload:
			bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)}
			// self implements impl
			bu.transferImpl = bu
			return bu
		case Download:
			panic("Should never ask tus.io to download")
		}
		return nil
	})
}
<commit_msg>tq/tus_upload: use initialized lfsapi.Client<commit_after>package tq

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strconv"
	"strings"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/rubyist/tracerx"
)

const (
	TusAdapterName = "tus"
	TusVersion     = "1.0.0"
)

// Adapter for tus.io protocol resumaable uploads
type tusUploadAdapter struct {
	*adapterBase
}

func (a *tusUploadAdapter) ClearTempStorage() error {
	// nothing to do, all temp state is on the server end
	return nil
}

func (a *tusUploadAdapter) WorkerStarting(workerNum int) (interface{}, error) {
	return nil, nil
}
func (a *tusUploadAdapter) WorkerEnding(workerNum int, ctx interface{}) {
}

func (a *tusUploadAdapter) DoTransfer(ctx interface{}, t *Transfer, cb ProgressCallback, authOkFunc func()) error {
	rel, err := t.Rel("upload")
	if err != nil {
		return err
	}
	if rel == nil {
		return errors.Errorf("No upload action for object: %s", t.Oid)
	}

	// Note not supporting the Creation extension since the batch API generates URLs
	// Also not supporting Concatenation to support parallel uploads of chunks; forward only

	// 1. Send HEAD request to determine upload start point
	//    Request must include Tus-Resumable header (version)
	tracerx.Printf("xfer: sending tus.io HEAD request for %q", t.Oid)
	req, err := a.newHTTPRequest("HEAD", rel)
	if err != nil {
		return err
	}

	req.Header.Set("Tus-Resumable", TusVersion)

	res, err := a.doHTTP(t, req)
	if err != nil {
		return errors.NewRetriableError(err)
	}

	//    Response will contain Upload-Offset if supported
	offHdr := res.Header.Get("Upload-Offset")
	if len(offHdr) == 0 {
		return fmt.Errorf("Missing Upload-Offset header from tus.io HEAD response at %q, contact server admin", rel.Href)
	}
	offset, err := strconv.ParseInt(offHdr, 10, 64)
	if err != nil || offset < 0 {
		return fmt.Errorf("Invalid Upload-Offset value %q in response from tus.io HEAD at %q, contact server admin", offHdr, rel.Href)
	}
	// Upload-Offset=size means already completed (skip)
	// Batch API will probably already detect this, but handle just in case
	if offset >= t.Size {
		tracerx.Printf("xfer: tus.io HEAD offset %d indicates %q is already fully uploaded, skipping", offset, t.Oid)
		advanceCallbackProgress(cb, t, t.Size)
		return nil
	}

	// Open file for uploading
	f, err := os.OpenFile(t.Path, os.O_RDONLY, 0644)
	if err != nil {
		return errors.Wrap(err, "tus upload")
	}
	defer f.Close()

	// Upload-Offset=0 means start from scratch, but still send PATCH
	if offset == 0 {
		tracerx.Printf("xfer: tus.io uploading %q from start", t.Oid)
	} else {
		tracerx.Printf("xfer: tus.io resuming upload %q from %d", t.Oid, offset)
		advanceCallbackProgress(cb, t, offset)
	}

	// 2. Send PATCH request with byte start point (even if 0) in Upload-Offset
	//    Response status must be 204
	//    Response Upload-Offset must be request Upload-Offset plus sent bytes
	//    Response may include Upload-Expires header in which case check not passed

	tracerx.Printf("xfer: sending tus.io PATCH request for %q", t.Oid)
	req, err = a.newHTTPRequest("PATCH", rel)
	if err != nil {
		return err
	}

	req.Header.Set("Tus-Resumable", TusVersion)
	req.Header.Set("Upload-Offset", strconv.FormatInt(offset, 10))
	req.Header.Set("Content-Type", "application/offset+octet-stream")
	req.Header.Set("Content-Length", strconv.FormatInt(t.Size-offset, 10))
	req.ContentLength = t.Size - offset

	// Ensure progress callbacks made while uploading
	// Wrap callback to give name context
	ccb := func(totalSize int64, readSoFar int64, readSinceLast int) error {
		if cb != nil {
			return cb(t.Name, totalSize, readSoFar, readSinceLast)
		}
		return nil
	}

	var reader lfsapi.ReadSeekCloser = progress.NewBodyWithCallback(f, t.Size, ccb)
	reader = newStartCallbackReader(reader, func() error {
		// seek to the offset since lfsapi.Client rewinds the body
		if _, err := f.Seek(offset, os.SEEK_CUR); err != nil {
			return err
		}
		// Signal auth was ok on first read; this frees up other workers to start
		if authOkFunc != nil {
			authOkFunc()
		}
		return nil
	})

	req.Body = reader

	res, err = a.doHTTP(t, req)
	if err != nil {
		return errors.NewRetriableError(err)
	}

	a.apiClient.LogResponse("lfs.data.upload", res)

	// A status code of 403 likely means that an authentication token for the
	// upload has expired. This can be safely retried.
	if res.StatusCode == 403 {
		err = errors.New("http: received status 403")
		return errors.NewRetriableError(err)
	}

	if res.StatusCode > 299 {
		return errors.Wrapf(nil, "Invalid status for %s %s: %d",
			req.Method,
			strings.SplitN(req.URL.String(), "?", 2)[0],
			res.StatusCode,
		)
	}

	io.Copy(ioutil.Discard, res.Body)
	res.Body.Close()

	return verifyUpload(a.apiClient, t)
}

func configureTusAdapter(m *Manifest) {
	m.RegisterNewAdapterFunc(TusAdapterName, Upload, func(name string, dir Direction) Adapter {
		switch dir {
		case Upload:
			bu := &tusUploadAdapter{newAdapterBase(name, dir, nil)}
			// self implements impl
			bu.transferImpl = bu
			return bu
		case Download:
			panic("Should never ask tus.io to download")
		}
		return nil
	})
}
<|endoftext|>


<commit_before>package gops

import (
	"fmt"

	"github.com/shirou/gopsutil/process"
)

// Build a hash pid -> ppid
func buildPIDHash(processInfos []*ProcessInfo) (hash map[int32]int32) {
	hash = make(map[int32]int32)
	for _, processInfo := range processInfos {
		hash[processInfo.PID] = processInfo.PPID
	}
	return
}

// Return whether the PID is of a kernel thread, based on whether it has
// the init process (PID 1) as ancestor
func isKernelThread(pid int32, pidHash map[int32]int32) bool {
	if pid == 1 {
		return false
	}
	ppid, ok := pidHash[pid]
	if !ok {
		return true
	}

	return isKernelThread(ppid, pidHash)
}

// Name processes "kernel" if they're a kernel thread
func postProcess(processInfos []*ProcessInfo) {
	pidHash := buildPIDHash(processInfos)
	for _, processInfo := range processInfos {
		if isKernelThread(processInfo.PID, pidHash) {
			processInfo.Name = "kernel"
		}
	}
}
<commit_msg>[processes][linux] Avoid possible infinite recursion on ppid hash recursion<commit_after>package gops

// Build a hash pid -> ppid
func buildPIDHash(processInfos []*ProcessInfo) (hash map[int32]int32) {
	hash = make(map[int32]int32)
	for _, processInfo := range processInfos {
		hash[processInfo.PID] = processInfo.PPID
	}
	return
}

// Return whether the PID is of a kernel thread, based on whether it has
// the init process (PID 1) as ancestor
func isKernelThread(pid int32, pidHash map[int32]int32, depth int32) bool {
	if pid == 1 {
		return false
	}
	// Once we reach this level of recursion we know it's not a kernel thread.
	// Also, avoids a possible infinite recursion as our process list is not
	// necessarily consistent.
	if depth > 2 {
		return false
	}

	ppid, ok := pidHash[pid]
	if !ok {
		return true
	}

	return isKernelThread(ppid, pidHash, depth+1)
}

// Name processes "kernel" if they're a kernel thread
func postProcess(processInfos []*ProcessInfo) {
	pidHash := buildPIDHash(processInfos)
	for _, processInfo := range processInfos {
		if isKernelThread(processInfo.PID, pidHash, 0) {
			processInfo.Name = "kernel"
		}
	}
}
<|endoftext|>


<commit_before>package lshensemble

import (
	"math"
	"sort"
)

const (
	integrationPrecision = 0.01
)

// NewLshForest default constructor uses 32 bit hash value
var NewLshForest = NewLshForest32

type keys []interface{}

// For initial bootstrapping
type initHashTable map[string]keys

type bucket struct {
	hashKey string
	keys    keys
}

type hashTable []bucket

func (h hashTable) Len() int           { return len(h) }
func (h hashTable) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
func (h hashTable) Less(i, j int) bool { return h[i].hashKey < h[j].hashKey }

// LshForest represents a MinHash LSH implemented using LSH Forest
// (http://ilpubs.stanford.edu:8090/678/1/2005-14.pdf).
// It supports query-time setting of the MinHash LSH parameters
// L (number of bands) and
// K (number of hash functions per band).
type LshForest struct {
	k              int
	l              int
	initHashTables []initHashTable
	hashTables     []hashTable
	hashKeyFunc    hashKeyFunc
	hashValueSize  int
}

func newLshForest(k, l, hashValueSize int) *LshForest {
	if k < 0 || l < 0 {
		panic("k and l must be positive")
	}
	hashTables := make([]hashTable, l)
	for i := range hashTables {
		hashTables[i] = make(hashTable, 0)
	}
	initHashTables := make([]initHashTable, l)
	for i := range initHashTables {
		initHashTables[i] = make(initHashTable)
	}
	return &LshForest{
		k:              k,
		l:              l,
		hashValueSize:  hashValueSize,
		initHashTables: initHashTables,
		hashTables:     hashTables,
		hashKeyFunc:    hashKeyFuncGen(hashValueSize),
	}
}

// NewLshForest64 uses 64-bit hash values.
func NewLshForest64(k, l int) *LshForest {
	return newLshForest(k, l, 8)
}

// NewLshForest32 uses 32-bit hash values.
// MinHash signatures with 64 bit hash values will have
// their hash values trimed.
func NewLshForest32(k, l int) *LshForest {
	return newLshForest(k, l, 4)
}

// NewLshForest16 uses 16-bit hash values.
// MinHash signatures with 64 or 32 bit hash values will have
// their hash values trimed.
func NewLshForest16(k, l int) *LshForest {
	return newLshForest(k, l, 2)
}

// Add a key with MinHash signature into the index.
// The key won't be searchable until Index() is called.
func (f *LshForest) Add(key interface{}, sig []uint64) {
	// Generate hash keys
	Hs := make([]string, f.l)
	for i := 0; i < f.l; i++ {
		Hs[i] = f.hashKeyFunc(sig[i*f.k : (i+1)*f.k])
	}
	// Insert keys into the bootstrapping tables
	for i := range f.initHashTables {
		ht := f.initHashTables[i]
		hk := Hs[i]
		if _, exist := ht[hk]; exist {
			ht[hk] = append(ht[hk], key)
		} else {
			ht[hk] = make(keys, 1)
			ht[hk][0] = key
		}
	}
}

// Index makes all the keys added searchable.
func (f *LshForest) Index() {
	for i := range f.hashTables {
		// Build sorted hash table using buckets from init hash tables
		initHt := f.initHashTables[i]
		ht := f.hashTables[i]
		for hashKey := range initHt {
			ks, _ := initHt[hashKey]
			ht = append(ht, bucket{
				hashKey: hashKey,
				keys:    ks,
			})
		}
		sort.Sort(ht)
		f.hashTables[i] = ht
		// Reset the init hash tables
		f.initHashTables[i] = make(initHashTable)
	}
}

// Query returns candidate keys given the query signature and parameters.
func (f *LshForest) Query(sig []uint64, K, L int, out chan<- interface{}, done <-chan struct{}) {
	if K == -1 {
		K = f.k
	}
	if L == -1 {
		L = f.l
	}
	prefixSize := f.hashValueSize * K
	// Generate hash keys
	Hs := make([]string, L)
	for i := 0; i < L; i++ {
		Hs[i] = f.hashKeyFunc(sig[i*f.k : i*f.k+K])
	}
	seens := make(map[interface{}]bool)
	for i := 0; i < L; i++ {
		ht := f.hashTables[i]
		hk := Hs[i]
		k := sort.Search(len(ht), func(x int) bool {
			return ht[x].hashKey[:prefixSize] >= hk
		})
		if k < len(ht) && ht[k].hashKey[:prefixSize] == hk {
			for j := k; j < len(ht) && ht[j].hashKey[:prefixSize] == hk; j++ {
				for _, key := range ht[j].keys {
					if _, seen := seens[key]; seen {
						continue
					}
					seens[key] = true
					select {
					case out <- key:
					case <-done:
						return
					}
				}
			}
		}
	}
}

// OptimalKL returns the optimal K and L for containment search,
// and the false positive and negative probabilities.
// where x is the indexed domain size, q is the query domain size,
// and t is the containment threshold.
func (f *LshForest) OptimalKL(x, q int, t float64) (optK, optL int, fp, fn float64) {
	minError := math.MaxFloat64
	for l := 1; l <= f.l; l++ {
		for k := 1; k <= f.k; k++ {
			currFp := probFalsePositive(x, q, l, k, t, integrationPrecision)
			currFn := probFalseNegative(x, q, l, k, t, integrationPrecision)
			currErr := currFn + currFp
			if minError > currErr {
				minError = currErr
				optK = k
				optL = l
				fp = currFp
				fn = currFn
			}
		}
	}
	return
}
<commit_msg>make forest indexing more efficient<commit_after>package lshensemble

import (
	"math"
	"sort"
)

const (
	integrationPrecision = 0.01
)

// NewLshForest default constructor uses 32 bit hash value
var NewLshForest = NewLshForest32

type keys []interface{}

// For initial bootstrapping
type initHashTable map[string]keys

type bucket struct {
	hashKey string
	keys    keys
}

type hashTable []bucket

func (h hashTable) Len() int           { return len(h) }
func (h hashTable) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
func (h hashTable) Less(i, j int) bool { return h[i].hashKey < h[j].hashKey }

// LshForest represents a MinHash LSH implemented using LSH Forest
// (http://ilpubs.stanford.edu:8090/678/1/2005-14.pdf).
// It supports query-time setting of the MinHash LSH parameters
// L (number of bands) and
// K (number of hash functions per band).
type LshForest struct {
	k              int
	l              int
	initHashTables []initHashTable
	hashTables     []hashTable
	hashKeyFunc    hashKeyFunc
	hashValueSize  int
}

func newLshForest(k, l, hashValueSize int) *LshForest {
	if k < 0 || l < 0 {
		panic("k and l must be positive")
	}
	hashTables := make([]hashTable, l)
	initHashTables := make([]initHashTable, l)
	for i := range initHashTables {
		initHashTables[i] = make(initHashTable)
	}
	return &LshForest{
		k:              k,
		l:              l,
		hashValueSize:  hashValueSize,
		initHashTables: initHashTables,
		hashTables:     hashTables,
		hashKeyFunc:    hashKeyFuncGen(hashValueSize),
	}
}

// NewLshForest64 uses 64-bit hash values.
func NewLshForest64(k, l int) *LshForest {
	return newLshForest(k, l, 8)
}

// NewLshForest32 uses 32-bit hash values.
// MinHash signatures with 64 bit hash values will have
// their hash values trimed.
func NewLshForest32(k, l int) *LshForest {
	return newLshForest(k, l, 4)
}

// NewLshForest16 uses 16-bit hash values.
// MinHash signatures with 64 or 32 bit hash values will have
// their hash values trimed.
func NewLshForest16(k, l int) *LshForest {
	return newLshForest(k, l, 2)
}

// Add a key with MinHash signature into the index.
// The key won't be searchable until Index() is called.
func (f *LshForest) Add(key interface{}, sig []uint64) {
	// Generate hash keys
	Hs := make([]string, f.l)
	for i := 0; i < f.l; i++ {
		Hs[i] = f.hashKeyFunc(sig[i*f.k : (i+1)*f.k])
	}
	// Insert keys into the bootstrapping tables
	for i := range f.initHashTables {
		ht := f.initHashTables[i]
		hk := Hs[i]
		if _, exist := ht[hk]; exist {
			ht[hk] = append(ht[hk], key)
		} else {
			ht[hk] = make(keys, 1)
			ht[hk][0] = key
		}
	}
}

// Index makes all the keys added searchable.
func (f *LshForest) Index() {
	for i := range f.hashTables {
		ht := make(hashTable, 0, len(f.initHashTables[i]))
		// Build sorted hash table using buckets from init hash tables
		for hashKey, keys := range f.initHashTables[i] {
			ht = append(ht, bucket{
				hashKey: hashKey,
				keys:    keys,
			})
		}
		sort.Sort(ht)
		f.hashTables[i] = ht
		// Reset the init hash tables
		f.initHashTables[i] = make(initHashTable)
	}
}

// Query returns candidate keys given the query signature and parameters.
func (f *LshForest) Query(sig []uint64, K, L int, out chan<- interface{}, done <-chan struct{}) {
	if K == -1 {
		K = f.k
	}
	if L == -1 {
		L = f.l
	}
	prefixSize := f.hashValueSize * K
	// Generate hash keys
	Hs := make([]string, L)
	for i := 0; i < L; i++ {
		Hs[i] = f.hashKeyFunc(sig[i*f.k : i*f.k+K])
	}
	seens := make(map[interface{}]bool)
	for i := 0; i < L; i++ {
		ht := f.hashTables[i]
		hk := Hs[i]
		k := sort.Search(len(ht), func(x int) bool {
			return ht[x].hashKey[:prefixSize] >= hk
		})
		if k < len(ht) && ht[k].hashKey[:prefixSize] == hk {
			for j := k; j < len(ht) && ht[j].hashKey[:prefixSize] == hk; j++ {
				for _, key := range ht[j].keys {
					if _, seen := seens[key]; seen {
						continue
					}
					seens[key] = true
					select {
					case out <- key:
					case <-done:
						return
					}
				}
			}
		}
	}
}

// OptimalKL returns the optimal K and L for containment search,
// and the false positive and negative probabilities.
// where x is the indexed domain size, q is the query domain size,
// and t is the containment threshold.
func (f *LshForest) OptimalKL(x, q int, t float64) (optK, optL int, fp, fn float64) {
	minError := math.MaxFloat64
	for l := 1; l <= f.l; l++ {
		for k := 1; k <= f.k; k++ {
			currFp := probFalsePositive(x, q, l, k, t, integrationPrecision)
			currFn := probFalseNegative(x, q, l, k, t, integrationPrecision)
			currErr := currFn + currFp
			if minError > currErr {
				minError = currErr
				optK = k
				optL = l
				fp = currFp
				fn = currFn
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package config
import (
	"fmt"
	"strings"
	"os"
	"encoding/json"
)

type AppConfig struct {
	Mongodb string
	EncryptResponse bool
	CryptedRequest bool
}

type ServerConfig struct {
	Port int
	Host string
}

var (
	app  AppConfig
	server ServerConfig
	configPath string
)

func init() {
	configPath = "./config"
}
func ServerConf() ServerConfig {
	return server
}

func AppConf() AppConfig {
	return app
}

func ReadConfig(path string) {
	configPath = strings.TrimRight(path, "/")
	merr := ReadAppConfig(fmt.Sprintf("%s/app.json", configPath))
	serr := ReadServerConfig(fmt.Sprintf("%s/conf.json", configPath))
	if merr != nil || serr != nil {
		panic(fmt.Sprintf("Can not load configuration"))
	}
}

func ReadAppConfig(filename string) error {
	file, _ := os.Open(filename)
	decoder := json.NewDecoder(file)
	return decoder.Decode(&app)
}

func ReadServerConfig(filename string) error {
	file, _ := os.Open(filename)
	decoder := json.NewDecoder(file)
	return decoder.Decode(&server)
}
<commit_msg>Panic with path to config files to show what we trying to load<commit_after>package config
import (
	"fmt"
	"strings"
	"os"
	"encoding/json"
)

type AppConfig struct {
	Mongodb string
	EncryptResponse bool
	CryptedRequest bool
}

type ServerConfig struct {
	Port int
	Host string
}

var (
	app  AppConfig
	server ServerConfig
	configPath string
)

func init() {
	configPath = "./config"
}
func ServerConf() ServerConfig {
	return server
}

func AppConf() AppConfig {
	return app
}

func ReadConfig(path string) {
	configPath = strings.TrimRight(path, "/")
	merr := ReadAppConfig(fmt.Sprintf("%s/app.json", configPath))
	serr := ReadServerConfig(fmt.Sprintf("%s/conf.json", configPath))
	if merr != nil || serr != nil {
		panic(fmt.Sprintf("Can not load configuration:%s/app.json or %s/conf.json", configPath, configPath))
	}
}

func ReadAppConfig(filename string) error {
	file, _ := os.Open(filename)
	decoder := json.NewDecoder(file)
	return decoder.Decode(&app)
}

func ReadServerConfig(filename string) error {
	file, _ := os.Open(filename)
	decoder := json.NewDecoder(file)
	return decoder.Decode(&server)
}
<|endoftext|>


<commit_before>package fabric

import (
	"context"
	"fmt"
	"net"

	"go.uber.org/zap"
)

// NewTransportTCP returns a new TCP transport
func NewTransportTCP(addr string) Transport {
	return &TCP{address: addr}
}

// TCP transport
type TCP struct {
	address string
}

// Name of the transport
func (t *TCP) Name() string {
	return "tcp"
}

// DialContext attemps to dial to the peer with the given addr
func (t *TCP) DialContext(ctx context.Context, addr Address) (
	net.Conn, error) {
	pr := addr.CurrentParams()
	tcon, err := net.Dial("tcp", pr)
	if err != nil {
		return nil, err
	}

	return tcon, nil
}

// CanDial checks if address can be dialed by this transport
func (t *TCP) CanDial(addr Address) (bool, error) {
	return addr.CurrentProtocol() == "tcp", nil
}

// Listen handles the transports
func (t *TCP) Listen(ctx context.Context, handler func(context.Context, net.Conn) error) error {
	// TODO read the address from the struct
	l, err := net.Listen("tcp", t.address)
	if err != nil {
		return err
	}

	go func() {
		for {
			// Listen for an incoming connection.
			conn, err := l.Accept()
			if err != nil {
				Logger(ctx).Error("Could not accept TCP connection", zap.Error(err))
				continue
			}
			go t.handleListen(ctx, conn, handler)
		}
	}()

	return nil
}

func (t *TCP) handleListen(ctx context.Context, conn net.Conn, handler func(context.Context, net.Conn) error) {
	defer func() {
		if err := conn.Close(); err != nil {
			fmt.Println("Could not close conn", err)
		}
	}()
	if err := handler(ctx, conn); err != nil {
		fmt.Println("Listen: Could not handle request. error:", err)
	}
}

// Address returns the address the transport is listening to
func (t *TCP) Address() string {
	return t.address
}
<commit_msg>Add TCP transport initial tests<commit_after>package fabric

import (
	"context"
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"

	"go.uber.org/zap"
)

// NewTransportTCP returns a new TCP transport
func NewTransportTCP(addr string) Transport {
	return &TCP{address: addr}
}

// TCP transport
type TCP struct {
	address string
}

// Name of the transport
func (t *TCP) Name() string {
	return "tcp"
}

// DialContext attemps to dial to the peer with the given addr
func (t *TCP) DialContext(ctx context.Context, addr Address) (
	net.Conn, error) {
	pr := addr.CurrentParams()
	tcon, err := net.Dial("tcp", pr)
	if err != nil {
		return nil, err
	}

	return tcon, nil
}

// CanDial checks if address can be dialed by this transport
func (t *TCP) CanDial(addr Address) (bool, error) {
	if addr.CurrentProtocol() != "tcp" {
		return false, nil
	}

	params := strings.Split(addr.CurrentParams(), ":")
	if len(params) != 2 {
		return false, errors.New("Invalid number of parameters")
	}

	if len(params[0]) == 0 {
		return false, errors.New("Missing destination host/ip")
	}

	port, err := strconv.Atoi(params[1])
	if err != nil {
		return false, errors.New("Invalid port")
	}

	if port == 0 || port > 65535 {
		return false, errors.New("Invalid port")
	}

	return true, nil
}

// Listen handles the transports
func (t *TCP) Listen(ctx context.Context, handler func(context.Context, net.Conn) error) error {
	// TODO read the address from the struct
	l, err := net.Listen("tcp", t.address)
	if err != nil {
		return err
	}

	go func() {
		for {
			// Listen for an incoming connection.
			conn, err := l.Accept()
			if err != nil {
				Logger(ctx).Error("Could not accept TCP connection", zap.Error(err))
				continue
			}
			go t.handleListen(ctx, conn, handler)
		}
	}()

	return nil
}

func (t *TCP) handleListen(ctx context.Context, conn net.Conn, handler func(context.Context, net.Conn) error) {
	defer func() {
		if err := conn.Close(); err != nil {
			fmt.Println("Could not close conn", err)
		}
	}()
	if err := handler(ctx, conn); err != nil {
		fmt.Println("Listen: Could not handle request. error:", err)
	}
}

// Address returns the address the transport is listening to
func (t *TCP) Address() string {
	return t.address
}
<|endoftext|>