<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main // import "helm.sh/helm/cmd/helm"

import (
	"context"
	"io"
	"path/filepath"

	auth "github.com/deislabs/oras/pkg/auth/docker"
	"github.com/spf13/cobra"

	"helm.sh/helm/cmd/helm/require"
	"helm.sh/helm/pkg/action"
	"helm.sh/helm/pkg/registry"
)

var globalUsage = `The Kubernetes package manager

To begin working with Helm, run the 'helm init' command:

	$ helm init

This will set up any necessary local configuration.

Common actions from this point include:

- helm search:    search for charts
- helm fetch:     download a chart to your local directory to view
- helm install:   upload the chart to Kubernetes
- helm list:      list releases of charts

Environment:
  $HELM_HOME          set an alternative location for Helm files. By default, these are stored in ~/.helm
  $HELM_DRIVER        set the backend storage driver. Values are: configmap, secret, memory
  $HELM_NO_PLUGINS    disable plugins. Set HELM_NO_PLUGINS=1 to disable plugins.
  $KUBECONFIG         set an alternative Kubernetes configuration file (default "~/.kube/config")
`

func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string) *cobra.Command {
	cmd := &cobra.Command{
		Use:          "helm",
		Short:        "The Helm package manager for Kubernetes.",
		Long:         globalUsage,
		SilenceUsage: true,
		Args:         require.NoArgs,
	}
	flags := cmd.PersistentFlags()

	settings.AddFlags(flags)

	flags.Parse(args)

	// set defaults from environment
	settings.Init(flags)

	// Add the registry client based on settings
	// TODO: Move this elsewhere (first, settings.Init() must move)
	// TODO: handle errors, dont panic
	credentialsFile := filepath.Join(settings.Home.Registry(), registry.CredentialsFileBasename)
	client, err := auth.NewClient(credentialsFile)
	if err != nil {
		panic(err)
	}
	resolver, err := client.Resolver(context.Background())
	if err != nil {
		panic(err)
	}
	actionConfig.RegistryClient = registry.NewClient(&registry.ClientOptions{
		Debug: settings.Debug,
		Out:   out,
		Authorizer: registry.Authorizer{
			Client: client,
		},
		Resolver: registry.Resolver{
			Resolver: resolver,
		},
		CacheRootDir: settings.Home.Registry(),
	})

	cmd.AddCommand(
		// chart commands
		newCreateCmd(out),
		newDependencyCmd(out),
		newPullCmd(out),
		newShowCmd(out),
		newLintCmd(out),
		newPackageCmd(out),
		newRepoCmd(out),
		newSearchCmd(out),
		newVerifyCmd(out),

		// registry/chart cache commands
		newRegistryCmd(actionConfig, out),
		newChartCmd(actionConfig, out),

		// release commands
		newGetCmd(actionConfig, out),
		newHistoryCmd(actionConfig, out),
		newInstallCmd(actionConfig, out),
		newListCmd(actionConfig, out),
		newReleaseTestCmd(actionConfig, out),
		newRollbackCmd(actionConfig, out),
		newStatusCmd(actionConfig, out),
		newUninstallCmd(actionConfig, out),
		newUpgradeCmd(actionConfig, out),

		newCompletionCmd(out),
		newHomeCmd(out),
		newInitCmd(out),
		newPluginCmd(out),
		newTemplateCmd(out),
		newVersionCmd(out),

		// Hidden documentation generator command: 'helm docs'
		newDocsCmd(out),
	)

	// Find and add plugins
	loadPlugins(cmd, out)

	return cmd
}
<commit_msg>Add dynamic completion for release names<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main // import "helm.sh/helm/cmd/helm"

import (
	"context"
	"io"
	"path/filepath"

	auth "github.com/deislabs/oras/pkg/auth/docker"
	"github.com/spf13/cobra"

	"helm.sh/helm/cmd/helm/require"
	"helm.sh/helm/pkg/action"
	"helm.sh/helm/pkg/registry"
)

const (
	bashCompletionFunc = `
__helm_override_flag_list=(--kubeconfig --kube-context --home --namespace -n)
__helm_override_flags()
{
    local ${__helm_override_flag_list[*]##*-} two_word_of of var
    for w in "${words[@]}"; do
        if [ -n "${two_word_of}" ]; then
            eval "${two_word_of##*-}=\"${two_word_of}=\${w}\""
            two_word_of=
            continue
        fi
        for of in "${__helm_override_flag_list[@]}"; do
            case "${w}" in
                ${of}=*)
                    eval "${of##*-}=\"${w}\""
                    ;;
                ${of})
                    two_word_of="${of}"
                    ;;
            esac
        done
    done
    for var in "${__helm_override_flag_list[@]##*-}"; do
        if eval "test -n \"\$${var}\""; then
            eval "echo \${${var}}"
        fi
    done
}
__helm_list_releases()
{
	__helm_debug "${FUNCNAME[0]}: c is $c words[c] is ${words[c]}"
	local out filter
	# Use ^ to map from the start of the release name
	filter="^${words[c]}"
    if out=$(helm list $(__helm_override_flags) -a -q -m 1000 -f ${filter} 2>/dev/null); then
        COMPREPLY=( $( compgen -W "${out[*]}" -- "$cur" ) )
    fi
}
__helm_custom_func()
{
	__helm_debug "${FUNCNAME[0]}: last_command is $last_command"
    case ${last_command} in
		helm_uninstall | helm_history | helm_status | helm_test_run |\
	    helm_upgrade | helm_rollback | helm_get_*)
            __helm_list_releases
            return
            ;;
        *)
            ;;
    esac
}
`
)

var globalUsage = `The Kubernetes package manager

To begin working with Helm, run the 'helm init' command:

	$ helm init

This will set up any necessary local configuration.

Common actions from this point include:

- helm search:    search for charts
- helm fetch:     download a chart to your local directory to view
- helm install:   upload the chart to Kubernetes
- helm list:      list releases of charts

Environment:
  $HELM_HOME          set an alternative location for Helm files. By default, these are stored in ~/.helm
  $HELM_DRIVER        set the backend storage driver. Values are: configmap, secret, memory
  $HELM_NO_PLUGINS    disable plugins. Set HELM_NO_PLUGINS=1 to disable plugins.
  $KUBECONFIG         set an alternative Kubernetes configuration file (default "~/.kube/config")
`

func newRootCmd(actionConfig *action.Configuration, out io.Writer, args []string) *cobra.Command {
	cmd := &cobra.Command{
		Use:                    "helm",
		Short:                  "The Helm package manager for Kubernetes.",
		Long:                   globalUsage,
		SilenceUsage:           true,
		Args:                   require.NoArgs,
		BashCompletionFunction: bashCompletionFunc,
	}
	flags := cmd.PersistentFlags()

	settings.AddFlags(flags)

	flags.Parse(args)

	// set defaults from environment
	settings.Init(flags)

	// Add the registry client based on settings
	// TODO: Move this elsewhere (first, settings.Init() must move)
	// TODO: handle errors, dont panic
	credentialsFile := filepath.Join(settings.Home.Registry(), registry.CredentialsFileBasename)
	client, err := auth.NewClient(credentialsFile)
	if err != nil {
		panic(err)
	}
	resolver, err := client.Resolver(context.Background())
	if err != nil {
		panic(err)
	}
	actionConfig.RegistryClient = registry.NewClient(&registry.ClientOptions{
		Debug: settings.Debug,
		Out:   out,
		Authorizer: registry.Authorizer{
			Client: client,
		},
		Resolver: registry.Resolver{
			Resolver: resolver,
		},
		CacheRootDir: settings.Home.Registry(),
	})

	cmd.AddCommand(
		// chart commands
		newCreateCmd(out),
		newDependencyCmd(out),
		newPullCmd(out),
		newShowCmd(out),
		newLintCmd(out),
		newPackageCmd(out),
		newRepoCmd(out),
		newSearchCmd(out),
		newVerifyCmd(out),

		// registry/chart cache commands
		newRegistryCmd(actionConfig, out),
		newChartCmd(actionConfig, out),

		// release commands
		newGetCmd(actionConfig, out),
		newHistoryCmd(actionConfig, out),
		newInstallCmd(actionConfig, out),
		newListCmd(actionConfig, out),
		newReleaseTestCmd(actionConfig, out),
		newRollbackCmd(actionConfig, out),
		newStatusCmd(actionConfig, out),
		newUninstallCmd(actionConfig, out),
		newUpgradeCmd(actionConfig, out),

		newCompletionCmd(out),
		newHomeCmd(out),
		newInitCmd(out),
		newPluginCmd(out),
		newTemplateCmd(out),
		newVersionCmd(out),

		// Hidden documentation generator command: 'helm docs'
		newDocsCmd(out),
	)

	// Find and add plugins
	loadPlugins(cmd, out)

	return cmd
}
<|endoftext|>


<commit_before>// Copyright © 2017 Prometheus Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"testing"
)

func TestRepoLocation(t *testing.T) {
	var repoTests = []struct {
		s string // input
		expected string // expected result
	}{
		{"git@github.com:prometheus/promu.git", "github.com/prometheus/promu"},
		{"https://github.com/prometheus/promu.git", "github.com/prometheus/promu"},
		{"ssh://git@gitlab.fr:22443/prometheus/promu.git", "gitlab.fr/prometheus/promu"},
		{"https://sdurrheimer@gitlab.fr/prometheus/promu.git", "gitlab.fr/prometheus/promu"},
	}

	for _, tc := range repoTests {
		actual, err := repoLocation(tc.s)
		if err != nil {
			t.Errorf("repoLocation(%s): %+v", tc.s, err)
		}
		if actual != tc.expected {
			t.Errorf("repoLocation(%s): expected %s, got %s", tc.s, tc.expected, actual)
		}
	}
}
<commit_msg>Fix info_test.go formatting<commit_after>// Copyright © 2017 Prometheus Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"testing"
)

func TestRepoLocation(t *testing.T) {
	var repoTests = []struct {
		s        string // input
		expected string // expected result
	}{
		{"git@github.com:prometheus/promu.git", "github.com/prometheus/promu"},
		{"https://github.com/prometheus/promu.git", "github.com/prometheus/promu"},
		{"ssh://git@gitlab.fr:22443/prometheus/promu.git", "gitlab.fr/prometheus/promu"},
		{"https://sdurrheimer@gitlab.fr/prometheus/promu.git", "gitlab.fr/prometheus/promu"},
	}

	for _, tc := range repoTests {
		actual, err := repoLocation(tc.s)
		if err != nil {
			t.Errorf("repoLocation(%s): %+v", tc.s, err)
		}
		if actual != tc.expected {
			t.Errorf("repoLocation(%s): expected %s, got %s", tc.s, tc.expected, actual)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the pieces of the tool that use typechecking from the go/types package.

package main

import (
	"go/ast"
	"go/token"

	"code.google.com/p/go.tools/go/exact"
	"code.google.com/p/go.tools/go/types"
)

func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) error {
	pkg.idents = make(map[*ast.Ident]types.Object)
	pkg.spans = make(map[types.Object]Span)
	pkg.types = make(map[ast.Expr]types.Type)
	pkg.values = make(map[ast.Expr]exact.Value)
	// By providing a Config with our own error function, it will continue
	// past the first error. There is no need for that function to do anything.
	config := types.Config{
		Error: func(error) {},
	}
	info := &types.Info{
		Types:   pkg.types,
		Values:  pkg.values,
		Objects: pkg.idents,
	}
	_, err := config.Check(pkg.path, fs, astFiles, info)
	// update spans
	for id, obj := range pkg.idents {
		pkg.growSpan(id, obj)
	}
	return err
}

// isStruct reports whether the composite literal c is a struct.
// If it is not (probably a struct), it returns a printable form of the type.
func (pkg *Package) isStruct(c *ast.CompositeLit) (bool, string) {
	// Check that the CompositeLit's type is a slice or array (which needs no tag), if possible.
	typ := pkg.types[c]
	// If it's a named type, pull out the underlying type. If it's not, the Underlying
	// method returns the type itself.
	actual := typ
	if actual != nil {
		actual = actual.Underlying()
	}
	if actual == nil {
		// No type information available. Assume true, so we do the check.
		return true, ""
	}
	switch actual.(type) {
	case *types.Struct:
		return true, typ.String()
	default:
		return false, ""
	}
}

var (
	stringerMethodType = types.New("func() string")
	errorType          = types.New("interface{ Error() string }")
	stringerType       = types.New("interface{ String() string }")
)

// matchArgType reports an error if printf verb t is not appropriate
// for operand arg.
//
// typ is used only for recursive calls; external callers must supply nil.
//
// (Recursion arises from the compound types {map,chan,slice} which
// may be printed with %d etc. if that is appropriate for their element
// types.)
func (f *File) matchArgType(t printfArgType, typ types.Type, arg ast.Expr) bool {
	// %v, %T accept any argument type.
	if t == anyType {
		return true
	}
	if typ == nil {
		// external call
		typ = f.pkg.types[arg]
		if typ == nil {
			return true // probably a type check problem
		}
	}
	// If we can use a string, does arg implement the Stringer or Error interface?
	if t&argString != 0 {
		if types.IsAssignableTo(typ, errorType) || types.IsAssignableTo(typ, stringerType) {
			return true
		}
	}

	switch typ := typ.Underlying().(type) {
	case *types.Signature:
		return t&argPointer != 0

	case *types.Map:
		// Recurse: map[int]int matches %d.
		return t&argPointer != 0 ||
			(f.matchArgType(t, typ.Key(), arg) && f.matchArgType(t, typ.Elem(), arg))

	case *types.Chan:
		return t&argPointer != 0

	case *types.Slice:
		if types.IsIdentical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 {
			return true // %s matches []byte
		}
		// Recurse: []int matches %d.
		return t&argPointer != 0 || f.matchArgType(t, typ.Elem(), arg)

	case *types.Pointer:
		return t&argPointer != 0

	case *types.Basic:
		switch typ.Kind() {
		case types.UntypedBool,
			types.Bool:
			return t&argBool != 0

		case types.UntypedInt,
			types.Int,
			types.Int8,
			types.Int16,
			types.Int32,
			types.Int64,
			types.Uint,
			types.Uint8,
			types.Uint16,
			types.Uint32,
			types.Uint64,
			types.Uintptr:
			return t&argInt != 0

		case types.UntypedFloat,
			types.Float32,
			types.Float64:
			return t&argFloat != 0

		case types.UntypedComplex,
			types.Complex64,
			types.Complex128:
			return t&argComplex != 0

		case types.UntypedString,
			types.String:
			return t&argString != 0

		case types.UnsafePointer:
			return t&(argPointer|argInt) != 0

		case types.UntypedRune:
			return t&(argInt|argRune) != 0

		case types.UntypedNil:
			return t&argPointer != 0 // TODO?

		case types.Invalid:
			if *verbose {
				f.Warnf(arg.Pos(), "printf argument %v has invalid or unknown type", f.gofmt(arg))
			}
			return true // Probably a type check problem.
		}
		panic("unreachable")
	}

	return false
}

// numArgsInSignature tells how many formal arguments the function type
// being called has.
func (f *File) numArgsInSignature(call *ast.CallExpr) int {
	// Check the type of the function or method declaration
	typ := f.pkg.types[call.Fun]
	if typ == nil {
		return 0
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return 0
	}
	return sig.Params().Len()
}

// isErrorMethodCall reports whether the call is of a method with signature
//	func Error() string
// where "string" is the universe's string type. We know the method is called "Error".
func (f *File) isErrorMethodCall(call *ast.CallExpr) bool {
	typ := f.pkg.types[call]
	if typ != nil {
		// We know it's called "Error", so just check the function signature.
		return types.IsIdentical(f.pkg.types[call.Fun], stringerMethodType)
	}
	// Without types, we can still check by hand.
	// Is it a selector expression? Otherwise it's a function call, not a method call.
	sel, ok := call.Fun.(*ast.SelectorExpr)
	if !ok {
		return false
	}
	// The package is type-checked, so if there are no arguments, we're done.
	if len(call.Args) > 0 {
		return false
	}
	// Check the type of the method declaration
	typ = f.pkg.types[sel]
	if typ == nil {
		return false
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return false
	}
	// There must be a receiver for it to be a method call. Otherwise it is
	// a function, not something that satisfies the error interface.
	if sig.Recv == nil {
		return false
	}
	// There must be no arguments. Already verified by type checking, but be thorough.
	if sig.Params().Len() > 0 {
		return false
	}
	// Finally the real questions.
	// There must be one result.
	if sig.Results().Len() != 1 {
		return false
	}
	// It must have return type "string" from the universe.
	return sig.Results().At(0).Type() == types.Typ[types.String]
}
<commit_msg>cmd/vet: Avoid warning on pointer to invalid types.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file contains the pieces of the tool that use typechecking from the go/types package.

package main

import (
	"go/ast"
	"go/token"

	"code.google.com/p/go.tools/go/exact"
	"code.google.com/p/go.tools/go/types"
)

func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) error {
	pkg.idents = make(map[*ast.Ident]types.Object)
	pkg.spans = make(map[types.Object]Span)
	pkg.types = make(map[ast.Expr]types.Type)
	pkg.values = make(map[ast.Expr]exact.Value)
	// By providing a Config with our own error function, it will continue
	// past the first error. There is no need for that function to do anything.
	config := types.Config{
		Error: func(error) {},
	}
	info := &types.Info{
		Types:   pkg.types,
		Values:  pkg.values,
		Objects: pkg.idents,
	}
	_, err := config.Check(pkg.path, fs, astFiles, info)
	// update spans
	for id, obj := range pkg.idents {
		pkg.growSpan(id, obj)
	}
	return err
}

// isStruct reports whether the composite literal c is a struct.
// If it is not (probably a struct), it returns a printable form of the type.
func (pkg *Package) isStruct(c *ast.CompositeLit) (bool, string) {
	// Check that the CompositeLit's type is a slice or array (which needs no tag), if possible.
	typ := pkg.types[c]
	// If it's a named type, pull out the underlying type. If it's not, the Underlying
	// method returns the type itself.
	actual := typ
	if actual != nil {
		actual = actual.Underlying()
	}
	if actual == nil {
		// No type information available. Assume true, so we do the check.
		return true, ""
	}
	switch actual.(type) {
	case *types.Struct:
		return true, typ.String()
	default:
		return false, ""
	}
}

var (
	stringerMethodType = types.New("func() string")
	errorType          = types.New("interface{ Error() string }")
	stringerType       = types.New("interface{ String() string }")
)

// matchArgType reports an error if printf verb t is not appropriate
// for operand arg.
//
// typ is used only for recursive calls; external callers must supply nil.
//
// (Recursion arises from the compound types {map,chan,slice} which
// may be printed with %d etc. if that is appropriate for their element
// types.)
func (f *File) matchArgType(t printfArgType, typ types.Type, arg ast.Expr) bool {
	// %v, %T accept any argument type.
	if t == anyType {
		return true
	}
	if typ == nil {
		// external call
		typ = f.pkg.types[arg]
		if typ == nil {
			return true // probably a type check problem
		}
	}
	// If we can use a string, does arg implement the Stringer or Error interface?
	if t&argString != 0 {
		if types.IsAssignableTo(typ, errorType) || types.IsAssignableTo(typ, stringerType) {
			return true
		}
	}

	switch typ := typ.Underlying().(type) {
	case *types.Signature:
		return t&argPointer != 0

	case *types.Map:
		// Recurse: map[int]int matches %d.
		return t&argPointer != 0 ||
			(f.matchArgType(t, typ.Key(), arg) && f.matchArgType(t, typ.Elem(), arg))

	case *types.Chan:
		return t&argPointer != 0

	case *types.Slice:
		if types.IsIdentical(typ.Elem().Underlying(), types.Typ[types.Byte]) && t&argString != 0 {
			return true // %s matches []byte
		}
		// Recurse: []int matches %d.
		return t&argPointer != 0 || f.matchArgType(t, typ.Elem(), arg)

	case *types.Pointer:
		// Ugly, but dealing with an edge case: a known pointer to an invalid type,
		// probably something from a failed import.
		if typ.Elem().String() == "invalid type" {
			if *verbose {
				f.Warnf(arg.Pos(), "printf argument %v is pointer to invalid or unknown type", f.gofmt(arg))
			}
			return true // special case
		}
		return t&argPointer != 0

	case *types.Basic:
		switch typ.Kind() {
		case types.UntypedBool,
			types.Bool:
			return t&argBool != 0

		case types.UntypedInt,
			types.Int,
			types.Int8,
			types.Int16,
			types.Int32,
			types.Int64,
			types.Uint,
			types.Uint8,
			types.Uint16,
			types.Uint32,
			types.Uint64,
			types.Uintptr:
			return t&argInt != 0

		case types.UntypedFloat,
			types.Float32,
			types.Float64:
			return t&argFloat != 0

		case types.UntypedComplex,
			types.Complex64,
			types.Complex128:
			return t&argComplex != 0

		case types.UntypedString,
			types.String:
			return t&argString != 0

		case types.UnsafePointer:
			return t&(argPointer|argInt) != 0

		case types.UntypedRune:
			return t&(argInt|argRune) != 0

		case types.UntypedNil:
			return t&argPointer != 0 // TODO?

		case types.Invalid:
			if *verbose {
				f.Warnf(arg.Pos(), "printf argument %v has invalid or unknown type", f.gofmt(arg))
			}
			return true // Probably a type check problem.
		}
		panic("unreachable")
	}

	return false
}

// numArgsInSignature tells how many formal arguments the function type
// being called has.
func (f *File) numArgsInSignature(call *ast.CallExpr) int {
	// Check the type of the function or method declaration
	typ := f.pkg.types[call.Fun]
	if typ == nil {
		return 0
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return 0
	}
	return sig.Params().Len()
}

// isErrorMethodCall reports whether the call is of a method with signature
//	func Error() string
// where "string" is the universe's string type. We know the method is called "Error".
func (f *File) isErrorMethodCall(call *ast.CallExpr) bool {
	typ := f.pkg.types[call]
	if typ != nil {
		// We know it's called "Error", so just check the function signature.
		return types.IsIdentical(f.pkg.types[call.Fun], stringerMethodType)
	}
	// Without types, we can still check by hand.
	// Is it a selector expression? Otherwise it's a function call, not a method call.
	sel, ok := call.Fun.(*ast.SelectorExpr)
	if !ok {
		return false
	}
	// The package is type-checked, so if there are no arguments, we're done.
	if len(call.Args) > 0 {
		return false
	}
	// Check the type of the method declaration
	typ = f.pkg.types[sel]
	if typ == nil {
		return false
	}
	// The type must be a signature, but be sure for safety.
	sig, ok := typ.(*types.Signature)
	if !ok {
		return false
	}
	// There must be a receiver for it to be a method call. Otherwise it is
	// a function, not something that satisfies the error interface.
	if sig.Recv == nil {
		return false
	}
	// There must be no arguments. Already verified by type checking, but be thorough.
	if sig.Params().Len() > 0 {
		return false
	}
	// Finally the real questions.
	// There must be one result.
	if sig.Results().Len() != 1 {
		return false
	}
	// It must have return type "string" from the universe.
	return sig.Results().At(0).Type() == types.Typ[types.String]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"net/http"
	"os"

	"encoding/json"

	log "github.com/Sirupsen/logrus"
	"github.com/cheggaaa/pb"
	"github.com/codegangsta/cli"
	"github.com/mattn/go-isatty"
	"github.com/otium/ytdl"
)

type options struct {
	noProgress  bool
	outputFile  string
	infoOnly    bool
	silent      bool
	debug       bool
	append      bool
	filters     []string
	downloadURL bool
	byteRange   string
	json        bool
}

func main() {
	app := cli.NewApp()
	app.Name = "ytdl"
	app.HelpName = "ytdl"
	// Set our own custom args usage
	app.ArgsUsage = "[youtube url or video id]"
	app.Usage = "Download youtube videos"
	app.HideHelp = true
	app.Version = "0.3.0"

	app.Flags = []cli.Flag{
		cli.HelpFlag,
		cli.StringFlag{
			Name:  "output, o",
			Usage: "Write output to a file, passing - outputs to stdout",
			Value: "{{.Title}}.{{.Ext}}",
		},
		cli.BoolFlag{
			Name:  "info, i",
			Usage: "Only output video info",
		},
		cli.BoolFlag{
			Name:  "no-progress",
			Usage: "Disable the progress bar",
		},
		cli.BoolFlag{
			Name:  "silent, s",
			Usage: "Only output errors, also disables progress bar",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "Output debug log",
		},
		cli.BoolFlag{
			Name:  "append, a",
			Usage: "Append to output file instead of overwriting",
		},
		cli.StringSliceFlag{
			Name:  "filter, f",
			Usage: "Filter available formats, syntax: [format_key]:val1,val2",
			Value: &cli.StringSlice{
				fmt.Sprintf("%s:mp4", ytdl.FormatExtensionKey),
				fmt.Sprintf("%s:1080p,720p,480p,360p,240p,144p", ytdl.FormatResolutionKey),
				fmt.Sprintf("!%s:nil", ytdl.FormatVideoEncodingKey),
				fmt.Sprintf("!%s:nil", ytdl.FormatAudioEncodingKey),
			},
		},
		cli.StringFlag{
			Name:  "range, r",
			Usage: "Download a specific range of bytes of the video, [start]-[end]",
		},
		cli.BoolFlag{
			Name:  "download-url, u",
			Usage: "Prints download url to stdout",
		},
		cli.BoolFlag{
			Name:  "json, j",
			Usage: "Print info json to stdout",
		},
	}

	app.Action = func(c *cli.Context) {
		identifier := c.Args().First()
		if identifier == "" || c.Bool("help") {
			cli.ShowAppHelp(c)
		} else {
			options := options{
				noProgress:  c.Bool("no-progress"),
				outputFile:  c.String("output"),
				infoOnly:    c.Bool("info"),
				silent:      c.Bool("silent"),
				debug:       c.Bool("debug"),
				append:      c.Bool("append"),
				filters:     c.StringSlice("filter"),
				downloadURL: c.Bool("download-url"),
				byteRange:   c.String("range"),
				json:        c.Bool("json"),
			}
			handler(identifier, options)
		}
	}
	app.Run(os.Args)
}

func handler(identifier string, options options) {
	var err error
	defer func() {
		if err != nil {
			log.SetOutput(os.Stderr)
			log.Fatal(err.Error())
		}
	}()

	var out io.Writer
	var logOut io.Writer = os.Stdout
	// if downloading to stdout, set log output to stderr, not sure if this is correct
	if options.outputFile == "-" {
		out = os.Stdout
		logOut = os.Stderr
	}
	log.SetOutput(logOut)

	// ouput only errors or not
	silent := options.outputFile == "" || options.silent || options.infoOnly || options.downloadURL || options.json
	if silent {
		log.SetLevel(log.FatalLevel)
	} else if options.debug {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	// TODO: Show activity indicator
	log.Info("Fetching video info...")
	//fmt.Print("\u001b[0G")
	//fmt.Print("\u001b[2K")
	info, err := ytdl.GetVideoInfo(identifier)
	if err != nil {
		err = fmt.Errorf("Unable to fetch video info: %s", err.Error())
		return
	}

	// TODO: Get more info, and change structure
	// TODO: Allow json output
	if options.infoOnly {
		fmt.Println("Title:", info.Title)
		fmt.Println("Author:", info.Author)
		fmt.Println("Date Published:", info.DatePublished.Format("Jan 2 2006"))
		fmt.Println("Duration:", info.Duration)
		return
	} else if options.json {
		var data []byte
		data, err = json.MarshalIndent(info, "", "\t")
		if err != nil {
			err = fmt.Errorf("Unable to marshal json: %s", err.Error())
			return
		}
		fmt.Println(string(data))
		return
	}

	formats := info.Formats
	// parse filter arguments, and filter through formats
	for _, filter := range options.filters {
		key, values, exclude, err := parseFilter(filter)
		if err == nil {
			if exclude {
				formats = ytdl.FilterFormatsExclude(formats, ytdl.FormatKey(key), values)
			} else {
				formats = ytdl.FilterFormats(formats, ytdl.FormatKey(key), values)
			}
		}
	}
	if len(formats) == 0 {
		err = fmt.Errorf("No formats available that match criteria")
		return
	}
	format := formats[0]
	downloadURL, err := info.GetDownloadURL(format)
	if err != nil {
		err = fmt.Errorf("Unable to get download url: %s", err.Error())
		return
	}
	if options.downloadURL {
		fmt.Print(downloadURL.String())
		// print new line character if outputing to terminal
		if isatty.IsTerminal(os.Stdout.Fd()) {
			fmt.Println()
		}
		return
	}
	if out == nil {
		var fileName string
		fileName, err = createFileName(options.outputFile, outputFileName{
			Title:         info.Title,
			Ext:           format[ytdl.FormatExtensionKey].(string),
			DatePublished: info.DatePublished.Format("2006-01-02"),
			Resolution:    format[ytdl.FormatResolutionKey].(string),
			Author:        info.Author,
			Duration:      info.Duration.String(),
		})
		if err != nil {
			err = fmt.Errorf("Unable to parse output file file name: %s", err.Error())
			return
		}
		// Create file truncate if append flag is not set
		flags := os.O_CREATE | os.O_WRONLY
		if options.append {
			flags |= os.O_APPEND
		} else {
			flags |= os.O_TRUNC
		}
		var f *os.File
		// open as write only
		f, err = os.OpenFile(fileName, flags, 0666)
		if err != nil {
			err = fmt.Errorf("Unable to open output file: %s", err.Error())
			return
		}
		defer f.Close()
		out = f
	}

	log.Info("Downloading to ", out.(*os.File).Name())
	var req *http.Request
	req, err = http.NewRequest("GET", downloadURL.String(), nil)
	// if byte range flag is set, use http range header option
	if options.byteRange != "" {
		req.Header.Set("Range", "bytes="+options.byteRange)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 {
		if err == nil {
			err = fmt.Errorf("Received status code %d from download url", resp.StatusCode)
		}
		err = fmt.Errorf("Unable to start download: %s", err.Error())
		return
	}
	defer resp.Body.Close()

	// if we aren't in silent mode or the no progress flag wasn't set,
	// initialize progress bar
	if !silent && !options.noProgress {
		progressBar := pb.New64(resp.ContentLength)
		progressBar.SetUnits(pb.U_BYTES)
		progressBar.ShowTimeLeft = true
		progressBar.ShowSpeed = true
		//	progressBar.RefreshRate = time.Millisecond * 1
		progressBar.Output = logOut
		progressBar.Start()
		defer progressBar.Finish()
		out = io.MultiWriter(out, progressBar)
	}
	_, err = io.Copy(out, resp.Body)
}
<commit_msg>Bumped to version 0.3.1<commit_after>package main

import (
	"fmt"
	"io"
	"net/http"
	"os"

	"encoding/json"

	log "github.com/Sirupsen/logrus"
	"github.com/cheggaaa/pb"
	"github.com/codegangsta/cli"
	"github.com/mattn/go-isatty"
	"github.com/otium/ytdl"
)

type options struct {
	noProgress  bool
	outputFile  string
	infoOnly    bool
	silent      bool
	debug       bool
	append      bool
	filters     []string
	downloadURL bool
	byteRange   string
	json        bool
}

func main() {
	app := cli.NewApp()
	app.Name = "ytdl"
	app.HelpName = "ytdl"
	// Set our own custom args usage
	app.ArgsUsage = "[youtube url or video id]"
	app.Usage = "Download youtube videos"
	app.HideHelp = true
	app.Version = "0.3.1"

	app.Flags = []cli.Flag{
		cli.HelpFlag,
		cli.StringFlag{
			Name:  "output, o",
			Usage: "Write output to a file, passing - outputs to stdout",
			Value: "{{.Title}}.{{.Ext}}",
		},
		cli.BoolFlag{
			Name:  "info, i",
			Usage: "Only output video info",
		},
		cli.BoolFlag{
			Name:  "no-progress",
			Usage: "Disable the progress bar",
		},
		cli.BoolFlag{
			Name:  "silent, s",
			Usage: "Only output errors, also disables progress bar",
		},
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "Output debug log",
		},
		cli.BoolFlag{
			Name:  "append, a",
			Usage: "Append to output file instead of overwriting",
		},
		cli.StringSliceFlag{
			Name:  "filter, f",
			Usage: "Filter available formats, syntax: [format_key]:val1,val2",
			Value: &cli.StringSlice{
				fmt.Sprintf("%s:mp4", ytdl.FormatExtensionKey),
				fmt.Sprintf("%s:1080p,720p,480p,360p,240p,144p", ytdl.FormatResolutionKey),
				fmt.Sprintf("!%s:nil", ytdl.FormatVideoEncodingKey),
				fmt.Sprintf("!%s:nil", ytdl.FormatAudioEncodingKey),
			},
		},
		cli.StringFlag{
			Name:  "range, r",
			Usage: "Download a specific range of bytes of the video, [start]-[end]",
		},
		cli.BoolFlag{
			Name:  "download-url, u",
			Usage: "Prints download url to stdout",
		},
		cli.BoolFlag{
			Name:  "json, j",
			Usage: "Print info json to stdout",
		},
	}

	app.Action = func(c *cli.Context) {
		identifier := c.Args().First()
		if identifier == "" || c.Bool("help") {
			cli.ShowAppHelp(c)
		} else {
			options := options{
				noProgress:  c.Bool("no-progress"),
				outputFile:  c.String("output"),
				infoOnly:    c.Bool("info"),
				silent:      c.Bool("silent"),
				debug:       c.Bool("debug"),
				append:      c.Bool("append"),
				filters:     c.StringSlice("filter"),
				downloadURL: c.Bool("download-url"),
				byteRange:   c.String("range"),
				json:        c.Bool("json"),
			}
			handler(identifier, options)
		}
	}
	app.Run(os.Args)
}

func handler(identifier string, options options) {
	var err error
	defer func() {
		if err != nil {
			log.SetOutput(os.Stderr)
			log.Fatal(err.Error())
		}
	}()

	var out io.Writer
	var logOut io.Writer = os.Stdout
	// if downloading to stdout, set log output to stderr, not sure if this is correct
	if options.outputFile == "-" {
		out = os.Stdout
		logOut = os.Stderr
	}
	log.SetOutput(logOut)

	// ouput only errors or not
	silent := options.outputFile == "" || options.silent || options.infoOnly || options.downloadURL || options.json
	if silent {
		log.SetLevel(log.FatalLevel)
	} else if options.debug {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.InfoLevel)
	}

	// TODO: Show activity indicator
	log.Info("Fetching video info...")
	//fmt.Print("\u001b[0G")
	//fmt.Print("\u001b[2K")
	info, err := ytdl.GetVideoInfo(identifier)
	if err != nil {
		err = fmt.Errorf("Unable to fetch video info: %s", err.Error())
		return
	}

	// TODO: Get more info, and change structure
	// TODO: Allow json output
	if options.infoOnly {
		fmt.Println("Title:", info.Title)
		fmt.Println("Author:", info.Author)
		fmt.Println("Date Published:", info.DatePublished.Format("Jan 2 2006"))
		fmt.Println("Duration:", info.Duration)
		return
	} else if options.json {
		var data []byte
		data, err = json.MarshalIndent(info, "", "\t")
		if err != nil {
			err = fmt.Errorf("Unable to marshal json: %s", err.Error())
			return
		}
		fmt.Println(string(data))
		return
	}

	formats := info.Formats
	// parse filter arguments, and filter through formats
	for _, filter := range options.filters {
		key, values, exclude, err := parseFilter(filter)
		if err == nil {
			if exclude {
				formats = ytdl.FilterFormatsExclude(formats, ytdl.FormatKey(key), values)
			} else {
				formats = ytdl.FilterFormats(formats, ytdl.FormatKey(key), values)
			}
		}
	}
	if len(formats) == 0 {
		err = fmt.Errorf("No formats available that match criteria")
		return
	}
	format := formats[0]
	downloadURL, err := info.GetDownloadURL(format)
	if err != nil {
		err = fmt.Errorf("Unable to get download url: %s", err.Error())
		return
	}
	if options.downloadURL {
		fmt.Print(downloadURL.String())
		// print new line character if outputing to terminal
		if isatty.IsTerminal(os.Stdout.Fd()) {
			fmt.Println()
		}
		return
	}
	if out == nil {
		var fileName string
		fileName, err = createFileName(options.outputFile, outputFileName{
			Title:         info.Title,
			Ext:           format[ytdl.FormatExtensionKey].(string),
			DatePublished: info.DatePublished.Format("2006-01-02"),
			Resolution:    format[ytdl.FormatResolutionKey].(string),
			Author:        info.Author,
			Duration:      info.Duration.String(),
		})
		if err != nil {
			err = fmt.Errorf("Unable to parse output file file name: %s", err.Error())
			return
		}
		// Create file truncate if append flag is not set
		flags := os.O_CREATE | os.O_WRONLY
		if options.append {
			flags |= os.O_APPEND
		} else {
			flags |= os.O_TRUNC
		}
		var f *os.File
		// open as write only
		f, err = os.OpenFile(fileName, flags, 0666)
		if err != nil {
			err = fmt.Errorf("Unable to open output file: %s", err.Error())
			return
		}
		defer f.Close()
		out = f
	}

	log.Info("Downloading to ", out.(*os.File).Name())
	var req *http.Request
	req, err = http.NewRequest("GET", downloadURL.String(), nil)
	// if byte range flag is set, use http range header option
	if options.byteRange != "" {
		req.Header.Set("Range", "bytes="+options.byteRange)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 {
		if err == nil {
			err = fmt.Errorf("Received status code %d from download url", resp.StatusCode)
		}
		err = fmt.Errorf("Unable to start download: %s", err.Error())
		return
	}
	defer resp.Body.Close()

	// if we aren't in silent mode or the no progress flag wasn't set,
	// initialize progress bar
	if !silent && !options.noProgress {
		progressBar := pb.New64(resp.ContentLength)
		progressBar.SetUnits(pb.U_BYTES)
		progressBar.ShowTimeLeft = true
		progressBar.ShowSpeed = true
		//	progressBar.RefreshRate = time.Millisecond * 1
		progressBar.Output = logOut
		progressBar.Start()
		defer progressBar.Finish()
		out = io.MultiWriter(out, progressBar)
	}
	_, err = io.Copy(out, resp.Body)
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package golang

import (
	"bytes"
	"encoding/hex"
	"math/rand"
	"testing"
	"time"

	"github.com/hyperledger/fabric/core/util"
)

// TestHashContentChange changes a random byte in a content and checks for hash change
func TestHashContentChange(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	b2 := []byte("To be, or not to be- that is the question: Whether 'tis nobler in the mind to suffer The slings and arrows of outrageous fortune Or to take arms against a sea of troubles, And by opposing end them. To die- to sleep- No more; and by a sleep to say we end The heartache, and the thousand natural shocks That flesh is heir to. 'Tis a consummation Devoutly to be wish'd.")

	h1 := computeHash(b2, hash)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	randIndex := (int(r.Uint32())) % len(b2)

	randChar := (int(r.Uint32())) % 128

	//change a random byte with a random char
	b2[randIndex] = byte(randChar)

	//this is the core hash func under test
	h2 := computeHash(b2, hash)

	//the two hashes should be different
	if bytes.Compare(h1, h2) == 0 {
		t.Fail()
		t.Logf("Hash expected to be different but is same")
	}
}

// TestHashLenChange changes a random length of a content and checks for hash change
func TestHashLenChange(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	b2 := []byte("To be, or not to be-")

	h1 := computeHash(b2, hash)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	randIndex := (int(r.Uint32())) % len(b2)

	b2 = b2[0:randIndex]

	h2 := computeHash(b2, hash)

	//hash should be different
	if bytes.Compare(h1, h2) == 0 {
		t.Fail()
		t.Logf("Hash expected to be different but is same")
	}
}

// TestHashOrderChange changes a order of hash computation over a list of lines and checks for hash change
func TestHashOrderChange(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	b2 := [][]byte{[]byte("To be, or not to be- that is the question:"),
		[]byte("Whether 'tis nobler in the mind to suffer"),
		[]byte("The slings and arrows of outrageous fortune"),
		[]byte("Or to take arms against a sea of troubles,"),
		[]byte("And by opposing end them."),
		[]byte("To die- to sleep- No more; and by a sleep to say we end"),
		[]byte("The heartache, and the thousand natural shocks"),
		[]byte("That flesh is heir to."),
		[]byte("'Tis a consummation Devoutly to be wish'd.")}
	h1 := hash

	for _, l := range b2 {
		h1 = computeHash(l, h1)
	}

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	randIndex1 := (int(r.Uint32())) % len(b2)
	randIndex2 := (int(r.Uint32())) % len(b2)

	//switch two arbitrary lines
	tmp := b2[randIndex2]
	b2[randIndex2] = b2[randIndex1]
	b2[randIndex1] = tmp

	h2 := hash
	for _, l := range b2 {
		h2 = computeHash(l, hash)
	}

	//hash should be different
	if bytes.Compare(h1, h2) == 0 {
		t.Fail()
		t.Logf("Hash expected to be different but is same")
	}
}

// TestHashOverFiles computes hash over a directory and ensures it matches precomputed, hardcoded, hash
func TestHashOverFiles(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	hash, err := hashFilesInDir(".", "hashtestfiles", hash, nil)

	if err != nil {
		t.Fail()
		t.Logf("error : %s", err)
	}

	//as long as no files under "hashtestfiles" are changed, hash should always compute to the following
	expectedHash := "a4fe18bebf3d7e1c030c042903bdda9019b33829d03d9b95ab1edc8957be70dee6d786ab27b207210d29b5d9f88456ff753b8da5c244458cdcca6eb3c28a17ce"

	computedHash := hex.EncodeToString(hash[:])

	if expectedHash != computedHash {
		t.Fail()
		t.Logf("Hash expected to be different but is same")
	}
}
<commit_msg>make sure computed random quantity is also different<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package golang

import (
	"bytes"
	"encoding/hex"
	"math/rand"
	"testing"
	"time"

	"github.com/hyperledger/fabric/core/util"
)

// TestHashContentChange changes a random byte in a content and checks for hash change
func TestHashContentChange(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	b2 := []byte("To be, or not to be- that is the question: Whether 'tis nobler in the mind to suffer The slings and arrows of outrageous fortune Or to take arms against a sea of troubles, And by opposing end them. To die- to sleep- No more; and by a sleep to say we end The heartache, and the thousand natural shocks That flesh is heir to. 'Tis a consummation Devoutly to be wish'd.")

	h1 := computeHash(b2, hash)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	randIndex := (int(r.Uint32())) % len(b2)

	randByte := byte((int(r.Uint32())) % 128)

	//make sure the two bytes are different
	for {
		if randByte != b2[randIndex] {
			break
		}

		randByte = byte((int(r.Uint32())) % 128)
	}

	//change a random byte
	b2[randIndex] = randByte

	//this is the core hash func under test
	h2 := computeHash(b2, hash)

	//the two hashes should be different
	if bytes.Compare(h1, h2) == 0 {
		t.Fail()
		t.Logf("Hash expected to be different but is same")
	}
}

// TestHashLenChange changes a random length of a content and checks for hash change
func TestHashLenChange(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	b2 := []byte("To be, or not to be-")

	h1 := computeHash(b2, hash)

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	randIndex := (int(r.Uint32())) % len(b2)

	b2 = b2[0:randIndex]

	h2 := computeHash(b2, hash)

	//hash should be different
	if bytes.Compare(h1, h2) == 0 {
		t.Fail()
		t.Logf("Hash expected to be different but is same")
	}
}

// TestHashOrderChange changes a order of hash computation over a list of lines and checks for hash change
func TestHashOrderChange(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	b2 := [][]byte{[]byte("To be, or not to be- that is the question:"),
		[]byte("Whether 'tis nobler in the mind to suffer"),
		[]byte("The slings and arrows of outrageous fortune"),
		[]byte("Or to take arms against a sea of troubles,"),
		[]byte("And by opposing end them."),
		[]byte("To die- to sleep- No more; and by a sleep to say we end"),
		[]byte("The heartache, and the thousand natural shocks"),
		[]byte("That flesh is heir to."),
		[]byte("'Tis a consummation Devoutly to be wish'd.")}
	h1 := hash

	for _, l := range b2 {
		h1 = computeHash(l, h1)
	}

	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	randIndex1 := (int(r.Uint32())) % len(b2)
	randIndex2 := (int(r.Uint32())) % len(b2)

	//make sure the two indeces are different
	for {
		if randIndex2 != randIndex1 {
			break
		}

		randIndex2 = (int(r.Uint32())) % len(b2)
	}

	//switch two arbitrary lines
	tmp := b2[randIndex2]
	b2[randIndex2] = b2[randIndex1]
	b2[randIndex1] = tmp

	h2 := hash
	for _, l := range b2 {
		h2 = computeHash(l, hash)
	}

	//hash should be different
	if bytes.Compare(h1, h2) == 0 {
		t.Fail()
		t.Logf("Hash expected to be different but is same")
	}
}

// TestHashOverFiles computes hash over a directory and ensures it matches precomputed, hardcoded, hash
func TestHashOverFiles(t *testing.T) {
	b := []byte("firstcontent")
	hash := util.ComputeCryptoHash(b)

	hash, err := hashFilesInDir(".", "hashtestfiles", hash, nil)

	if err != nil {
		t.Fail()
		t.Logf("error : %s", err)
	}

	//as long as no files under "hashtestfiles" are changed, hash should always compute to the following
	expectedHash := "a4fe18bebf3d7e1c030c042903bdda9019b33829d03d9b95ab1edc8957be70dee6d786ab27b207210d29b5d9f88456ff753b8da5c244458cdcca6eb3c28a17ce"

	computedHash := hex.EncodeToString(hash[:])

	if expectedHash != computedHash {
		t.Fail()
		t.Logf("Hash expected to be unchanged")
	}
}
<|endoftext|>


<commit_before>package main

import "flag"
import "log"
import "github.com/brotherlogic/godiscogs"
import "github.com/brotherlogic/goserver"
import "google.golang.org/grpc"

import pb "github.com/brotherlogic/discogssyncer/server"

// Syncer the configuration for the syncer
type Syncer struct {
	*goserver.GoServer
	saveLocation string
	token        string
}

// DoRegister does RPC registration
func (s Syncer) DoRegister(server *grpc.Server) {
	pb.RegisterDiscogsServiceServer(server, &s)
}

// InitServer builds an initial server
func InitServer(token *string, folder *string) Syncer {
	syncer := Syncer{&goserver.GoServer{}, *folder, *token}
	syncer.Register = syncer
	return syncer
}

func main() {
	var folder = flag.String("folder", "/home/simon/.discogs/", "Location to store the records")
	var token = flag.String("token", "", "Discogs Token")
	var sync = flag.Bool("sync", true, "Flag to serve rather than sync")
	flag.Parse()

	syncer := InitServer(token, folder)

	log.Printf("HERE = %v", *sync)

	if *sync {
		retr := godiscogs.NewDiscogsRetriever(*token)
		syncer.SaveCollection(retr)
	} else {
		syncer.PrepServer()
		syncer.RegisterServer("discogssyncer", false)
		syncer.Serve()
	}
}
<commit_msg>Added cleaner to sync process. This closes #51.<commit_after>package main

import "flag"
import "log"
import "os"
import "path/filepath"
import "strings"
import "time"

import "github.com/brotherlogic/godiscogs"
import "github.com/brotherlogic/goserver"
import "google.golang.org/grpc"

import pb "github.com/brotherlogic/discogssyncer/server"

// Syncer the configuration for the syncer
type Syncer struct {
	*goserver.GoServer
	saveLocation string
	token        string
}

var (
    syncTime int64
)

// DoRegister does RPC registration
func (s Syncer) DoRegister(server *grpc.Server) {
	pb.RegisterDiscogsServiceServer(server, &s)
}

func doDelete(path string, f os.FileInfo, err error) error {
     if !strings.Contains(path, "metadata/") && !f.IsDir() && f.ModTime().Unix() < syncTime {
     	return os.Remove(path)
     }
     return nil
}

func (s Syncer) clean() {
     filepath.Walk(s.saveLocation, doDelete)
}

// InitServer builds an initial server
func InitServer(token *string, folder *string) Syncer {
	syncer := Syncer{&goserver.GoServer{}, *folder, *token}
	syncer.Register = syncer
	return syncer
}

func main() {
	var folder = flag.String("folder", "/home/simon/.discogs/", "Location to store the records")
	var token = flag.String("token", "", "Discogs Token")
	var sync = flag.Bool("sync", true, "Flag to serve rather than sync")
	flag.Parse()

	syncer := InitServer(token, folder)

	log.Printf("HERE = %v", *sync)

	if *sync {
		retr := godiscogs.NewDiscogsRetriever(*token)
		syncTime = time.Now().Unix()
		syncer.SaveCollection(retr)
		syncer.clean()
	} else {
		syncer.PrepServer()
		syncer.RegisterServer("discogssyncer", false)
		syncer.Serve()
	}
}
<|endoftext|>


<commit_before>package all

import (
	// The following are necessary as they register handlers in their init functions.

	// Required features. Can't remove unless there is replacements.
	_ "v2ray.com/core/app/dispatcher"
	_ "v2ray.com/core/app/proxyman/inbound"
	_ "v2ray.com/core/app/proxyman/outbound"

	// Default commander and all its services. This is an optional feature.
	_ "v2ray.com/core/app/commander"
	_ "v2ray.com/core/app/log/command"
	_ "v2ray.com/core/app/proxyman/command"
	_ "v2ray.com/core/app/stats/command"

	// Other optional features.
	_ "v2ray.com/core/app/dns"
	_ "v2ray.com/core/app/log"
	_ "v2ray.com/core/app/policy"
	_ "v2ray.com/core/app/reverse"
	_ "v2ray.com/core/app/router"
	_ "v2ray.com/core/app/stats"

	// Inbound and outbound proxies.
	_ "v2ray.com/core/proxy/blackhole"
	_ "v2ray.com/core/proxy/dokodemo"
	_ "v2ray.com/core/proxy/freedom"
	_ "v2ray.com/core/proxy/http"
	_ "v2ray.com/core/proxy/mtproto"
	_ "v2ray.com/core/proxy/shadowsocks"
	_ "v2ray.com/core/proxy/socks"
	_ "v2ray.com/core/proxy/vmess/inbound"
	_ "v2ray.com/core/proxy/vmess/outbound"

	// Transports
	_ "v2ray.com/core/transport/internet/domainsocket"
	_ "v2ray.com/core/transport/internet/http"
	_ "v2ray.com/core/transport/internet/kcp"
	_ "v2ray.com/core/transport/internet/quic"
	_ "v2ray.com/core/transport/internet/tcp"
	_ "v2ray.com/core/transport/internet/tls"
	_ "v2ray.com/core/transport/internet/udp"
	_ "v2ray.com/core/transport/internet/websocket"

	// Transport headers
	_ "v2ray.com/core/transport/internet/headers/http"
	_ "v2ray.com/core/transport/internet/headers/noop"
	_ "v2ray.com/core/transport/internet/headers/srtp"
	_ "v2ray.com/core/transport/internet/headers/tls"
	_ "v2ray.com/core/transport/internet/headers/utp"
	_ "v2ray.com/core/transport/internet/headers/wechat"
	_ "v2ray.com/core/transport/internet/headers/wireguard"

	// JSON config support. Choose only one from the two below.
	// The following line loads JSON from v2ctl
	_ "v2ray.com/core/main/json"
	// The following line loads JSON internally
	// _ "v2ray.com/core/main/jsonem"

	// Load config from file or http(s)
	_ "v2ray.com/core/main/confloader/external"
)
<commit_msg>enable dns outbound<commit_after>package all

import (
	// The following are necessary as they register handlers in their init functions.

	// Required features. Can't remove unless there is replacements.
	_ "v2ray.com/core/app/dispatcher"
	_ "v2ray.com/core/app/proxyman/inbound"
	_ "v2ray.com/core/app/proxyman/outbound"

	// Default commander and all its services. This is an optional feature.
	_ "v2ray.com/core/app/commander"
	_ "v2ray.com/core/app/log/command"
	_ "v2ray.com/core/app/proxyman/command"
	_ "v2ray.com/core/app/stats/command"

	// Other optional features.
	_ "v2ray.com/core/app/dns"
	_ "v2ray.com/core/app/log"
	_ "v2ray.com/core/app/policy"
	_ "v2ray.com/core/app/reverse"
	_ "v2ray.com/core/app/router"
	_ "v2ray.com/core/app/stats"

	// Inbound and outbound proxies.
	_ "v2ray.com/core/proxy/blackhole"
	_ "v2ray.com/core/proxy/dns"
	_ "v2ray.com/core/proxy/dokodemo"
	_ "v2ray.com/core/proxy/freedom"
	_ "v2ray.com/core/proxy/http"
	_ "v2ray.com/core/proxy/mtproto"
	_ "v2ray.com/core/proxy/shadowsocks"
	_ "v2ray.com/core/proxy/socks"
	_ "v2ray.com/core/proxy/vmess/inbound"
	_ "v2ray.com/core/proxy/vmess/outbound"

	// Transports
	_ "v2ray.com/core/transport/internet/domainsocket"
	_ "v2ray.com/core/transport/internet/http"
	_ "v2ray.com/core/transport/internet/kcp"
	_ "v2ray.com/core/transport/internet/quic"
	_ "v2ray.com/core/transport/internet/tcp"
	_ "v2ray.com/core/transport/internet/tls"
	_ "v2ray.com/core/transport/internet/udp"
	_ "v2ray.com/core/transport/internet/websocket"

	// Transport headers
	_ "v2ray.com/core/transport/internet/headers/http"
	_ "v2ray.com/core/transport/internet/headers/noop"
	_ "v2ray.com/core/transport/internet/headers/srtp"
	_ "v2ray.com/core/transport/internet/headers/tls"
	_ "v2ray.com/core/transport/internet/headers/utp"
	_ "v2ray.com/core/transport/internet/headers/wechat"
	_ "v2ray.com/core/transport/internet/headers/wireguard"

	// JSON config support. Choose only one from the two below.
	// The following line loads JSON from v2ctl
	_ "v2ray.com/core/main/json"
	// The following line loads JSON internally
	// _ "v2ray.com/core/main/jsonem"

	// Load config from file or http(s)
	_ "v2ray.com/core/main/confloader/external"
)
<|endoftext|>


<commit_before>/*
	The Cluster Telemetry Frontend.
*/

package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"net/http"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"google.golang.org/api/option"

	"go.skia.org/infra/ct/go/ctfe/admin_tasks"
	"go.skia.org/infra/ct/go/ctfe/capture_skps"
	"go.skia.org/infra/ct/go/ctfe/chromium_analysis"
	"go.skia.org/infra/ct/go/ctfe/chromium_builds"
	"go.skia.org/infra/ct/go/ctfe/chromium_perf"
	"go.skia.org/infra/ct/go/ctfe/lua_scripts"
	"go.skia.org/infra/ct/go/ctfe/metrics_analysis"
	"go.skia.org/infra/ct/go/ctfe/pending_tasks"
	"go.skia.org/infra/ct/go/ctfe/pixel_diff"
	"go.skia.org/infra/ct/go/ctfe/task_common"
	"go.skia.org/infra/ct/go/ctfe/task_types"
	ctfeutil "go.skia.org/infra/ct/go/ctfe/util"
	ctutil "go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/allowed"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/ds"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/login"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skiaversion"
	"go.skia.org/infra/go/sklog"
	skutil "go.skia.org/infra/go/util"
)

var (
	// flags
	host         = flag.String("host", "localhost", "HTTP service host")
	promPort     = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	port         = flag.String("port", ":8000", "HTTP service port (e.g., ':8000')")
	internalPort = flag.String("internal_port", ":9000", "HTTP service internal port (e.g., ':9000')")
	local        = flag.Bool("local", false, "Running locally if true. As opposed to in production.")

	resourcesDir           = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the current directory will be used.")
	tasksSchedulerWaitTime = flag.Duration("tasks_scheduler_wait_time", 5*time.Minute, "How often the repeated tasks scheduler should run.")

	// Email params
	emailClientSecretFile = flag.String("email_client_secret_file", "/etc/ct-email-secrets/client_secret.json", "OAuth client secret JSON file for sending email.")
	emailTokenCacheFile   = flag.String("email_token_cache_file", "/etc/ct-email-secrets/client_token.json", "OAuth token cache file for sending email.")

	// Datastore params
	namespace   = flag.String("namespace", "cluster-telemetry", "The Cloud Datastore namespace, such as 'cluster-telemetry'.")
	projectName = flag.String("project_name", "google.com:skia-buildbots", "The Google Cloud project name.")

	// authenticated http client
	client *http.Client
)

func reloadTemplates() {
	if *resourcesDir == "" {
		// If resourcesDir is not specified then consider the directory two directories up from this
		// source file as the resourcesDir.
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../..")
	}
	admin_tasks.ReloadTemplates(*resourcesDir)
	capture_skps.ReloadTemplates(*resourcesDir)
	chromium_analysis.ReloadTemplates(*resourcesDir)
	chromium_builds.ReloadTemplates(*resourcesDir)
	chromium_perf.ReloadTemplates(*resourcesDir)
	lua_scripts.ReloadTemplates(*resourcesDir)
	metrics_analysis.ReloadTemplates(*resourcesDir)
	pending_tasks.ReloadTemplates(*resourcesDir)
	pixel_diff.ReloadTemplates(*resourcesDir)
}

func Init() {
	reloadTemplates()
}

func getIntParam(name string, r *http.Request) (*int, error) {
	raw, ok := r.URL.Query()[name]
	if !ok {
		return nil, nil
	}
	v64, err := strconv.ParseInt(raw[0], 10, 32)
	if err != nil {
		return nil, fmt.Errorf("Invalid value for parameter %q: %s -- %v", name, raw, err)
	}
	v32 := int(v64)
	return &v32, nil
}

func runServer(serverURL string) {
	externalRouter := mux.NewRouter()
	externalRouter.PathPrefix("/res/").HandlerFunc(httputils.MakeResourceHandler(*resourcesDir))
	// Router for URLs that do not need to be exposed externally. Eg:
	// updating tasks, querying for pending tasks, terminating tasks.
	internalRouter := mux.NewRouter()

	admin_tasks.AddHandlers(externalRouter, internalRouter)
	capture_skps.AddHandlers(externalRouter, internalRouter)
	chromium_analysis.AddHandlers(externalRouter, internalRouter)
	chromium_builds.AddHandlers(externalRouter, internalRouter)
	chromium_perf.AddHandlers(externalRouter, internalRouter) // Note: chromium_perf adds a handler for "/".
	lua_scripts.AddHandlers(externalRouter, internalRouter)
	metrics_analysis.AddHandlers(externalRouter, internalRouter)
	pending_tasks.AddHandlers(externalRouter, internalRouter)
	pixel_diff.AddHandlers(externalRouter, internalRouter)

	task_common.AddHandlers(externalRouter, internalRouter)

	// Handler for displaying results stored in Google Storage.
	externalRouter.PathPrefix(ctfeutil.RESULTS_URI).HandlerFunc(resultsHandler)

	// Common handlers used by different pages.
	externalRouter.HandleFunc("/json/version", skiaversion.JsonHandler)
	externalRouter.HandleFunc("/loginstatus/", login.StatusHandler)

	// Do not log healthz requests.
	http.HandleFunc("/healthz", httputils.HealthCheckHandler)

	h := httputils.LoggingGzipRequestResponse(externalRouter)
	h = login.RestrictViewer(h)
	h = login.ForceAuth(h, login.DEFAULT_REDIRECT_URL)
	h = httputils.ForceHTTPS(h)
	http.Handle("/", h)

	go func() {
		sklog.Infof("Internal server is accessible via %s", *internalPort)
		sklog.Fatal(http.ListenAndServe(*internalPort, internalRouter))
	}()

	sklog.Infof("Ready to serve on %s", serverURL)
	sklog.Fatal(http.ListenAndServe(*port, nil))
}

// startCtfeMetrics registers metrics which indicate CT is running healthily
// and starts a goroutine to update them periodically.
func startCtfeMetrics(ctx context.Context) {
	pendingTasksGauge := metrics2.GetInt64Metric("num_pending_tasks")
	oldestPendingTaskAgeGauge := metrics2.GetFloat64Metric("oldest_pending_task_age")
	// 0=no tasks pending; 1=started; 2=not started
	oldestPendingTaskStatusGauge := metrics2.GetInt64Metric("oldest_pending_task_status")
	go func() {
		for range time.Tick(common.SAMPLE_PERIOD) {
			pendingTaskCount, err := pending_tasks.GetPendingTaskCount(ctx)
			if err != nil {
				sklog.Error(err)
			} else {
				pendingTasksGauge.Update(pendingTaskCount)
			}

			oldestPendingTask, err := pending_tasks.GetOldestPendingTask(ctx)
			if err != nil {
				sklog.Error(err)
			} else if oldestPendingTask == nil {
				oldestPendingTaskAgeGauge.Update(0)
				oldestPendingTaskStatusGauge.Update(0)
			} else {
				addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(oldestPendingTask.GetCommonCols().TsAdded, 10))
				oldestPendingTaskAgeGauge.Update(time.Since(addedTime).Seconds())
				if oldestPendingTask.GetCommonCols().TsStarted != 0 {
					oldestPendingTaskStatusGauge.Update(1)
				} else {
					oldestPendingTaskStatusGauge.Update(2)
				}
			}
		}
	}()
}

// repeatedTasksScheduler looks for all tasks that contain repeat_after_days
// set to > 0 and schedules them when the specified time comes.
// The function does the following:
// 1. Look for tasks that need to be scheduled in the next 5 minutes.
// 2. Loop over these tasks.
//   2.1 Schedule the task again and set repeat_after_days to what it
//       originally was.
//   2.2 Update the original task and set repeat_after_days to 0 since the
//       newly created task will now replace it.
func repeatedTasksScheduler(ctx context.Context) {

	for range time.Tick(*tasksSchedulerWaitTime) {
		// Loop over all tasks to find tasks which need to be scheduled.
		for _, prototype := range task_types.Prototypes() {

			it := task_common.DatastoreTaskQuery(ctx, prototype,
				task_common.QueryParams{
					FutureRunsOnly: true,
					Offset:         0,
					Size:           task_common.MAX_PAGE_SIZE,
				})
			data, err := prototype.Query(it)
			if err != nil {
				sklog.Errorf("Failed to query %s tasks: %v", prototype.GetTaskName(), err)
				continue
			}

			tasks := task_common.AsTaskSlice(data)
			for _, task := range tasks {
				addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(task.GetCommonCols().TsAdded, 10))
				scheduledTime := addedTime.Add(time.Duration(task.GetCommonCols().RepeatAfterDays) * time.Hour * 24)

				cutOffTime := time.Now().UTC().Add(*tasksSchedulerWaitTime)
				if scheduledTime.Before(cutOffTime) {
					addTaskVars, err := task.GetPopulatedAddTaskVars()
					if err != nil {
						sklog.Errorf("Failed to get populated addTaskVars %v: %s", task, err)
						continue
					}
					if _, err := task_common.AddTask(ctx, addTaskVars); err != nil {
						sklog.Errorf("Failed to add task %v: %s", task, err)
						continue
					}

					taskVars := task.GetUpdateTaskVars()
					taskVars.GetUpdateTaskCommonVars().Id = task.GetCommonCols().DatastoreKey.ID
					taskVars.GetUpdateTaskCommonVars().ClearRepeatAfterDays = true
					if err := task_common.UpdateTask(ctx, taskVars, task); err != nil {
						sklog.Errorf("Failed to update task %v: %s", task, err)
						continue
					}
				}
			}
		}
	}
}

func resultsHandler(w http.ResponseWriter, r *http.Request) {
	sklog.Infof("Requesting: %s", r.RequestURI)
	if login.LoggedInAs(r) == "" {
		http.Redirect(w, r, login.LoginURL(w, r), http.StatusSeeOther)
		return
	}
	if !login.IsGoogler(r) {
		sklog.Info("User is not a Googler.")
		http.Error(w, "Only Google accounts are allowed.", http.StatusUnauthorized)
		return
	}

	storageURL := fmt.Sprintf("https://storage.googleapis.com/%s", strings.TrimLeft(r.URL.Path, ctfeutil.RESULTS_URI))
	resp, err := client.Get(storageURL)
	if err != nil {
		sklog.Errorf("resultsHandler: Unable to get data from %s: %s", storageURL, err)
		httputils.ReportError(w, r, err, "Unable to get data from google storage")
		return
	}
	defer skutil.Close(resp.Body)
	if resp.StatusCode != 200 {
		sklog.Errorf("resultsHandler: %s returned %d", storageURL, resp.StatusCode)
		httputils.ReportError(w, r, nil, fmt.Sprintf("Google storage request returned %d", resp.StatusCode))
		return
	}
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	if _, err := io.Copy(w, resp.Body); err != nil {
		sklog.Errorf("Error when copying response from %s: %s", storageURL, err)
		httputils.ReportError(w, r, err, "Error when copying response from google storage")
		return
	}
}

func main() {

	ctfeutil.PreExecuteTemplateHook = func() {
		// Don't use cached templates in local mode.
		if *local {
			reloadTemplates()
		}
	}

	common.InitWithMust("ctfe", common.PrometheusOpt(promPort))
	skiaversion.MustLogVersion()

	Init()
	serverURL := "https://" + *host
	if *local {
		serverURL = "http://" + *host + *port
	}

	if !*local {
		// Initialize mailing library.
		if err := ctutil.MailInit(*emailClientSecretFile, *emailTokenCacheFile); err != nil {
			sklog.Fatalf("Could not initialize mailing library: %s", err)
		}
	}

	allow := allowed.NewAllowedFromList(ctfeutil.DomainsWithViewAccess)
	login.InitWithAllow(*port, *local, nil, nil, allow)

	// Initialize the datastore.
	dsTokenSource, err := auth.NewDefaultTokenSource(*local, "https://www.googleapis.com/auth/datastore")
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}
	if err := ds.InitWithOpt(*projectName, *namespace, option.WithTokenSource(dsTokenSource)); err != nil {
		sklog.Fatalf("Could not init datastore: %s", err)
	}

	// Create authenticated HTTP client.
	storageTokenSource, err := auth.NewDefaultTokenSource(*local, auth.SCOPE_READ_ONLY)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}
	client = auth.ClientFromTokenSource(storageTokenSource)

	ctx := context.Background()

	startCtfeMetrics(ctx)

	// Start the repeated tasks scheduler.
	go repeatedTasksScheduler(ctx)

	runServer(serverURL)
}
<commit_msg>[CTFE] Remove handler for healthz since it is now done in httputils<commit_after>/*
	The Cluster Telemetry Frontend.
*/

package main

import (
	"context"
	"flag"
	"fmt"
	"io"
	"net/http"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/mux"
	"google.golang.org/api/option"

	"go.skia.org/infra/ct/go/ctfe/admin_tasks"
	"go.skia.org/infra/ct/go/ctfe/capture_skps"
	"go.skia.org/infra/ct/go/ctfe/chromium_analysis"
	"go.skia.org/infra/ct/go/ctfe/chromium_builds"
	"go.skia.org/infra/ct/go/ctfe/chromium_perf"
	"go.skia.org/infra/ct/go/ctfe/lua_scripts"
	"go.skia.org/infra/ct/go/ctfe/metrics_analysis"
	"go.skia.org/infra/ct/go/ctfe/pending_tasks"
	"go.skia.org/infra/ct/go/ctfe/pixel_diff"
	"go.skia.org/infra/ct/go/ctfe/task_common"
	"go.skia.org/infra/ct/go/ctfe/task_types"
	ctfeutil "go.skia.org/infra/ct/go/ctfe/util"
	ctutil "go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/go/allowed"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/ds"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/login"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skiaversion"
	"go.skia.org/infra/go/sklog"
	skutil "go.skia.org/infra/go/util"
)

var (
	// flags
	host         = flag.String("host", "localhost", "HTTP service host")
	promPort     = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	port         = flag.String("port", ":8000", "HTTP service port (e.g., ':8000')")
	internalPort = flag.String("internal_port", ":9000", "HTTP service internal port (e.g., ':9000')")
	local        = flag.Bool("local", false, "Running locally if true. As opposed to in production.")

	resourcesDir           = flag.String("resources_dir", "", "The directory to find templates, JS, and CSS files. If blank the current directory will be used.")
	tasksSchedulerWaitTime = flag.Duration("tasks_scheduler_wait_time", 5*time.Minute, "How often the repeated tasks scheduler should run.")

	// Email params
	emailClientSecretFile = flag.String("email_client_secret_file", "/etc/ct-email-secrets/client_secret.json", "OAuth client secret JSON file for sending email.")
	emailTokenCacheFile   = flag.String("email_token_cache_file", "/etc/ct-email-secrets/client_token.json", "OAuth token cache file for sending email.")

	// Datastore params
	namespace   = flag.String("namespace", "cluster-telemetry", "The Cloud Datastore namespace, such as 'cluster-telemetry'.")
	projectName = flag.String("project_name", "google.com:skia-buildbots", "The Google Cloud project name.")

	// authenticated http client
	client *http.Client
)

func reloadTemplates() {
	if *resourcesDir == "" {
		// If resourcesDir is not specified then consider the directory two directories up from this
		// source file as the resourcesDir.
		_, filename, _, _ := runtime.Caller(0)
		*resourcesDir = filepath.Join(filepath.Dir(filename), "../..")
	}
	admin_tasks.ReloadTemplates(*resourcesDir)
	capture_skps.ReloadTemplates(*resourcesDir)
	chromium_analysis.ReloadTemplates(*resourcesDir)
	chromium_builds.ReloadTemplates(*resourcesDir)
	chromium_perf.ReloadTemplates(*resourcesDir)
	lua_scripts.ReloadTemplates(*resourcesDir)
	metrics_analysis.ReloadTemplates(*resourcesDir)
	pending_tasks.ReloadTemplates(*resourcesDir)
	pixel_diff.ReloadTemplates(*resourcesDir)
}

func Init() {
	reloadTemplates()
}

func getIntParam(name string, r *http.Request) (*int, error) {
	raw, ok := r.URL.Query()[name]
	if !ok {
		return nil, nil
	}
	v64, err := strconv.ParseInt(raw[0], 10, 32)
	if err != nil {
		return nil, fmt.Errorf("Invalid value for parameter %q: %s -- %v", name, raw, err)
	}
	v32 := int(v64)
	return &v32, nil
}

func runServer(serverURL string) {
	externalRouter := mux.NewRouter()
	externalRouter.PathPrefix("/res/").HandlerFunc(httputils.MakeResourceHandler(*resourcesDir))
	// Router for URLs that do not need to be exposed externally. Eg:
	// updating tasks, querying for pending tasks, terminating tasks.
	internalRouter := mux.NewRouter()

	admin_tasks.AddHandlers(externalRouter, internalRouter)
	capture_skps.AddHandlers(externalRouter, internalRouter)
	chromium_analysis.AddHandlers(externalRouter, internalRouter)
	chromium_builds.AddHandlers(externalRouter, internalRouter)
	chromium_perf.AddHandlers(externalRouter, internalRouter) // Note: chromium_perf adds a handler for "/".
	lua_scripts.AddHandlers(externalRouter, internalRouter)
	metrics_analysis.AddHandlers(externalRouter, internalRouter)
	pending_tasks.AddHandlers(externalRouter, internalRouter)
	pixel_diff.AddHandlers(externalRouter, internalRouter)

	task_common.AddHandlers(externalRouter, internalRouter)

	// Handler for displaying results stored in Google Storage.
	externalRouter.PathPrefix(ctfeutil.RESULTS_URI).HandlerFunc(resultsHandler)

	// Common handlers used by different pages.
	externalRouter.HandleFunc("/json/version", skiaversion.JsonHandler)
	externalRouter.HandleFunc("/loginstatus/", login.StatusHandler)

	h := httputils.LoggingGzipRequestResponse(externalRouter)
	h = login.RestrictViewer(h)
	h = login.ForceAuth(h, login.DEFAULT_REDIRECT_URL)
	h = httputils.ForceHTTPS(h)
	http.Handle("/", h)

	go func() {
		sklog.Infof("Internal server is accessible via %s", *internalPort)
		sklog.Fatal(http.ListenAndServe(*internalPort, internalRouter))
	}()

	sklog.Infof("Ready to serve on %s", serverURL)
	sklog.Fatal(http.ListenAndServe(*port, nil))
}

// startCtfeMetrics registers metrics which indicate CT is running healthily
// and starts a goroutine to update them periodically.
func startCtfeMetrics(ctx context.Context) {
	pendingTasksGauge := metrics2.GetInt64Metric("num_pending_tasks")
	oldestPendingTaskAgeGauge := metrics2.GetFloat64Metric("oldest_pending_task_age")
	// 0=no tasks pending; 1=started; 2=not started
	oldestPendingTaskStatusGauge := metrics2.GetInt64Metric("oldest_pending_task_status")
	go func() {
		for range time.Tick(common.SAMPLE_PERIOD) {
			pendingTaskCount, err := pending_tasks.GetPendingTaskCount(ctx)
			if err != nil {
				sklog.Error(err)
			} else {
				pendingTasksGauge.Update(pendingTaskCount)
			}

			oldestPendingTask, err := pending_tasks.GetOldestPendingTask(ctx)
			if err != nil {
				sklog.Error(err)
			} else if oldestPendingTask == nil {
				oldestPendingTaskAgeGauge.Update(0)
				oldestPendingTaskStatusGauge.Update(0)
			} else {
				addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(oldestPendingTask.GetCommonCols().TsAdded, 10))
				oldestPendingTaskAgeGauge.Update(time.Since(addedTime).Seconds())
				if oldestPendingTask.GetCommonCols().TsStarted != 0 {
					oldestPendingTaskStatusGauge.Update(1)
				} else {
					oldestPendingTaskStatusGauge.Update(2)
				}
			}
		}
	}()
}

// repeatedTasksScheduler looks for all tasks that contain repeat_after_days
// set to > 0 and schedules them when the specified time comes.
// The function does the following:
// 1. Look for tasks that need to be scheduled in the next 5 minutes.
// 2. Loop over these tasks.
//   2.1 Schedule the task again and set repeat_after_days to what it
//       originally was.
//   2.2 Update the original task and set repeat_after_days to 0 since the
//       newly created task will now replace it.
func repeatedTasksScheduler(ctx context.Context) {

	for range time.Tick(*tasksSchedulerWaitTime) {
		// Loop over all tasks to find tasks which need to be scheduled.
		for _, prototype := range task_types.Prototypes() {

			it := task_common.DatastoreTaskQuery(ctx, prototype,
				task_common.QueryParams{
					FutureRunsOnly: true,
					Offset:         0,
					Size:           task_common.MAX_PAGE_SIZE,
				})
			data, err := prototype.Query(it)
			if err != nil {
				sklog.Errorf("Failed to query %s tasks: %v", prototype.GetTaskName(), err)
				continue
			}

			tasks := task_common.AsTaskSlice(data)
			for _, task := range tasks {
				addedTime := ctutil.GetTimeFromTs(strconv.FormatInt(task.GetCommonCols().TsAdded, 10))
				scheduledTime := addedTime.Add(time.Duration(task.GetCommonCols().RepeatAfterDays) * time.Hour * 24)

				cutOffTime := time.Now().UTC().Add(*tasksSchedulerWaitTime)
				if scheduledTime.Before(cutOffTime) {
					addTaskVars, err := task.GetPopulatedAddTaskVars()
					if err != nil {
						sklog.Errorf("Failed to get populated addTaskVars %v: %s", task, err)
						continue
					}
					if _, err := task_common.AddTask(ctx, addTaskVars); err != nil {
						sklog.Errorf("Failed to add task %v: %s", task, err)
						continue
					}

					taskVars := task.GetUpdateTaskVars()
					taskVars.GetUpdateTaskCommonVars().Id = task.GetCommonCols().DatastoreKey.ID
					taskVars.GetUpdateTaskCommonVars().ClearRepeatAfterDays = true
					if err := task_common.UpdateTask(ctx, taskVars, task); err != nil {
						sklog.Errorf("Failed to update task %v: %s", task, err)
						continue
					}
				}
			}
		}
	}
}

func resultsHandler(w http.ResponseWriter, r *http.Request) {
	sklog.Infof("Requesting: %s", r.RequestURI)
	if login.LoggedInAs(r) == "" {
		http.Redirect(w, r, login.LoginURL(w, r), http.StatusSeeOther)
		return
	}
	if !login.IsGoogler(r) {
		sklog.Info("User is not a Googler.")
		http.Error(w, "Only Google accounts are allowed.", http.StatusUnauthorized)
		return
	}

	storageURL := fmt.Sprintf("https://storage.googleapis.com/%s", strings.TrimLeft(r.URL.Path, ctfeutil.RESULTS_URI))
	resp, err := client.Get(storageURL)
	if err != nil {
		sklog.Errorf("resultsHandler: Unable to get data from %s: %s", storageURL, err)
		httputils.ReportError(w, r, err, "Unable to get data from google storage")
		return
	}
	defer skutil.Close(resp.Body)
	if resp.StatusCode != 200 {
		sklog.Errorf("resultsHandler: %s returned %d", storageURL, resp.StatusCode)
		httputils.ReportError(w, r, nil, fmt.Sprintf("Google storage request returned %d", resp.StatusCode))
		return
	}
	w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
	if _, err := io.Copy(w, resp.Body); err != nil {
		sklog.Errorf("Error when copying response from %s: %s", storageURL, err)
		httputils.ReportError(w, r, err, "Error when copying response from google storage")
		return
	}
}

func main() {

	ctfeutil.PreExecuteTemplateHook = func() {
		// Don't use cached templates in local mode.
		if *local {
			reloadTemplates()
		}
	}

	common.InitWithMust("ctfe", common.PrometheusOpt(promPort))
	skiaversion.MustLogVersion()

	Init()
	serverURL := "https://" + *host
	if *local {
		serverURL = "http://" + *host + *port
	}

	if !*local {
		// Initialize mailing library.
		if err := ctutil.MailInit(*emailClientSecretFile, *emailTokenCacheFile); err != nil {
			sklog.Fatalf("Could not initialize mailing library: %s", err)
		}
	}

	allow := allowed.NewAllowedFromList(ctfeutil.DomainsWithViewAccess)
	login.InitWithAllow(*port, *local, nil, nil, allow)

	// Initialize the datastore.
	dsTokenSource, err := auth.NewDefaultTokenSource(*local, "https://www.googleapis.com/auth/datastore")
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}
	if err := ds.InitWithOpt(*projectName, *namespace, option.WithTokenSource(dsTokenSource)); err != nil {
		sklog.Fatalf("Could not init datastore: %s", err)
	}

	// Create authenticated HTTP client.
	storageTokenSource, err := auth.NewDefaultTokenSource(*local, auth.SCOPE_READ_ONLY)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}
	client = auth.ClientFromTokenSource(storageTokenSource)

	ctx := context.Background()

	startCtfeMetrics(ctx)

	// Start the repeated tasks scheduler.
	go repeatedTasksScheduler(ctx)

	runServer(serverURL)
}
<|endoftext|>


<commit_before>// chris 080415

package rebnf

import (
	"testing"

	"golang.org/x/exp/ebnf"
)

func testIsCapital(t *testing.T, x string, expect bool) {
	if IsCapital(x) != expect {
		t.Fail()
	}
}

func TestIsCapital(t *testing.T) {
	testIsCapital(t, "Hello", true)
	testIsCapital(t, "there", false)
	testIsCapital(t, "", false)
	testIsCapital(t, "Σ1", true)
}

func testIsTerminal(t *testing.T, expr ebnf.Expression, expect bool) {
	if IsTerminal(expr) != expect {
		t.Fail()
	}
}

func TestIsTerminal(t *testing.T) {
	var expr ebnf.Expression
	var t1, t2 ebnf.Token

	expr = &ebnf.Name{String: "Production"}
	testIsTerminal(t, expr, false)

	expr = &ebnf.Name{String: "token"}
	testIsTerminal(t, expr, true)

	expr = &ebnf.Token{String: "blah"}
	testIsTerminal(t, expr, true)

	t1 = ebnf.Token{String: "a"}
	t2 = ebnf.Token{String: "z"}
	expr = &ebnf.Range{Begin: &t1, End: &t2}
	testIsTerminal(t, expr, true)

	expr = &ebnf.Repetition{Body: &ebnf.Token{String: "foo"}}
	testIsTerminal(t, expr, false)
}

func sliceExprEquals(a, b []ebnf.Expression) bool {
	if len(a) != len(b) {
		return false
	}
	for i, _ := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

func testFindTerminals(t *testing.T, x []ebnf.Expression, expect []ebnf.Expression) {
	x = findTerminals(x)
	if !sliceExprEquals(x, expect) {
		t.Fail()
	}
}

func TestFindTerminals(t *testing.T) {
	var term ebnf.Expression
	var x, expect []ebnf.Expression

	term = &ebnf.Name{String: "some_terminal"}

	x = []ebnf.Expression{
		term,
		&ebnf.Name{String: "NonTerminal"},
		&ebnf.Option{Body: &ebnf.Name{String: "whatever"}},
	}
	expect = []ebnf.Expression{term}
	testFindTerminals(t, x, expect)
}
<commit_msg>Updates tests with more verbose failures.<commit_after>// chris 080415

package rebnf

import (
	"testing"

	"golang.org/x/exp/ebnf"
)

func testIsCapital(t *testing.T, x string, expect bool) {
	out := IsCapital(x)
	if out != expect {
		t.Errorf("IsCapital(%q) != %v (got %v)", x, out, expect)
	}
}

func TestIsCapital(t *testing.T) {
	testIsCapital(t, "Hello", true)
	testIsCapital(t, "there", false)
	testIsCapital(t, "", false)
	testIsCapital(t, "Σ1", true)
}

func testIsTerminal(t *testing.T, expr ebnf.Expression, expect bool) {
	out := IsTerminal(expr)
	if out != expect {
		t.Errorf("IsTerminal(%#v) != %v (got %v)", expr, out, expect)
	}
}

func TestIsTerminal(t *testing.T) {
	var expr ebnf.Expression
	var t1, t2 ebnf.Token

	expr = &ebnf.Name{String: "Production"}
	testIsTerminal(t, expr, false)

	expr = &ebnf.Name{String: "token"}
	testIsTerminal(t, expr, true)

	expr = &ebnf.Token{String: "blah"}
	testIsTerminal(t, expr, true)

	t1 = ebnf.Token{String: "a"}
	t2 = ebnf.Token{String: "z"}
	expr = &ebnf.Range{Begin: &t1, End: &t2}
	testIsTerminal(t, expr, true)

	expr = &ebnf.Repetition{Body: &ebnf.Token{String: "foo"}}
	testIsTerminal(t, expr, false)
}

func sliceExprEquals(a, b []ebnf.Expression) bool {
	if len(a) != len(b) {
		return false
	}
	for i, _ := range a {
		if a[i] != b[i] {
			return false
		}
	}
	return true
}

func testFindTerminals(t *testing.T, x []ebnf.Expression, expect []ebnf.Expression) {
	out := findTerminals(x)
	if !sliceExprEquals(out, expect) {
		t.Errorf("findTerminals(%#v) != %#v (got %#v)", x, expect, out)
	}
}

func TestFindTerminals(t *testing.T) {
	var term ebnf.Expression
	var x, expect []ebnf.Expression

	term = &ebnf.Name{String: "some_terminal"}

	x = []ebnf.Expression{
		term,
		&ebnf.Name{String: "NonTerminal"},
		&ebnf.Option{Body: &ebnf.Name{String: "whatever"}},
	}
	expect = []ebnf.Expression{term}
	testFindTerminals(t, x, expect)
}
<|endoftext|>


<commit_before>/*
 *
 * This file contains client side Javascript that is rendered
 * when a client hits the RelayR route with a GET request.
 *
 */

package relayr

const connectionClassScript = `

var RelayRConnection = {};

RelayRConnection = (function() {
	var web, transport;
	var route = '%v';
	transport = {
		websocket: {
			connect: function(c) {
				var s = this;
				s.socket = new WebSocket("ws://" + window.location.host + route + "/ws?connectionId=" + transport.ConnectionId);
				s.socket.onclose = function(evt) {
					setTimeout(function() {
						web.n(); // renegotiate
					}, 2000);
				};

				s.socket.onmessage = function(evt) {
					c(evt.data);
				};

				s.socket.onerror = function(evt) {
					setTimeout(function() {
						s.connect(cId, c);
					});
				};

				s.socket.onopen = function(evt) {
					RelayRConnection.r();
				};
			},
			send: function(data) {
				var s = this;
				s.socket.send(data);
			}
		},
		longpoll: {
			connect: function(c) {
				RelayRConnection.r();
				var retry;
				retry = function() {
					web.gj(route + '/longpoll?connectionId=' + transport.ConnectionId + '&_=' + new Date().getTime(), function(data) {
						if (data.responseText) {
							var reconn = JSON.parse(data.responseText);
							if (reconn.Z) {
								web.n();
							} else {
								c(data);
								retry();
							}
						} else {
							web.n();
						}
					});
				};

				retry();
			},
			send: function(data) {
				var s = this;
				web.p(route + '/call?connectionId=' + transport.ConnectionId + '&_=' + new Date().getTime(), data, null, "json", null);
			}
		}
	};
	web = (function() {
		return {
			x: function() {
				var xd;

				if (window.XMLHttpRequest) {
					xd = new XMLHttpRequest();
				}
				else {
					xd = new ActiveXObject("Microsoft.XMLHTTP");
				}

				return xd;
			},
			g: function(u, c) {
				var s = this;

				var xd = s.x();

				xd.open('GET', u, true);

				xd.onreadystatechange = function() {
					if (xd.readyState === 4 && xd.status === 200) {
						c(xd);
					}
				};

				xd.send();
			},
			gj: function(u, c) {
				var s = this;

				var xd = s.x();

				xd.open('GET', u, true);
				xd.setRequestHeader("Content-type", "application/json");

				xd.onreadystatechange = function() {
					if (xd.readyState === 4) {
						if (xd.status === 200) {
							c(xd);
						} else {
							s.n();
						}
					}
				};

				xd.send();
			},
			p: function(u, d, c, t, e) {
				var s = this;

				var xd = s.x();

				xd.open('POST', u, true);
				xd.setRequestHeader("Content-type", "application/" + t);

				xd.onreadystatechange = function() {
					if (xd.readyState === 4) {
						if (xd.status === 200) {
							if (c) {
								c(xd);
							}
						} else {
							s.n();
						}
					}
				};

				xd.onerror = function() {
					if (e) {
						e(xd);
					}
				};

				xd.send(d);

				window.onbeforeunload = function() {
					delete xd;
					xd = null;
				};
			},
			t: function() {
				if (!!window.WebSocket) {
					return "websocket";
				} else {
					return "longpoll";
				}

				// TODO: Implement SSE Circuit
				/*if (typeof EventSource !== 'undefined') {
					return "SSE";
				}*/
			},
			n: function() {
				var s = this;
				var t = s.t();
				web.p(route + "/negotiate?_=" + new Date().getTime(), JSON.stringify({ t: t }), function(result) {
					var obj = JSON.parse(result.responseText);
					transport.ConnectionId = obj.ConnectionID;
					setTimeout(function() {
						transport[t].connect(function(data) {
							var cobj;
							if (data.responseText && data.status && data.responseXML) {
								cobj = JSON.parse(data.responseText);
							} else {
								if (data.responseText == "") return;
								cobj = JSON.parse(data);
							}
							var lobj = RelayR[cobj.R].client;
							var args = [];
							for (var i = 0; i < cobj.A.length; i++) {
								args.push(cobj.A[i]);
							}
							lobj[cobj.M].apply(lobj||window, args);
						});
					}, 0);
				}, "json",
				function(result) {
					// error .. try again
					setTimeout(function() {
						s.n();
					}, 2000);
				});
			}
		};
	})();

	return {
		ready: function(r) {
			RelayRConnection.r = r;

			web.n();
		},
		callServer: function(r, f, a) {
			transport[web.t()].send(JSON.stringify({ S: true, C: transport.ConnectionId, R: r, M: f, A: a}));
		}
	};
})();

`

const relayClassBegin = `

var RelayR = (function() {
	return {

`

const relayBegin = `

%v: {

	client: {},

	server: {

`

// {0} == function name
// {1} == Relay name
// {2} == function name
const relayMethod = `

%v: function() {
	RelayRConnection.callServer('%v', '%v', Array.prototype.slice.call(arguments));
},

`

const relayEnd = `

},

},

`

const relayClassEnd = `

	};
})();

`
<commit_msg>Stopped bug where ready() was called every negotiation<commit_after>/*
 *
 * This file contains client side Javascript that is rendered
 * when a client hits the RelayR route with a GET request.
 *
 */

package relayr

const connectionClassScript = `

var RelayRConnection = {};

RelayRConnection = (function() {
	var readyCalled = false;
	var web, transport;
	var route = '%v';
	transport = {
		websocket: {
			connect: function(c) {
				var s = this;
				s.socket = new WebSocket("ws://" + window.location.host + route + "/ws?connectionId=" + transport.ConnectionId);
				s.socket.onclose = function(evt) {
					setTimeout(function() {
						web.n(); // renegotiate
					}, 2000);
				};

				s.socket.onmessage = function(evt) {
					c(evt.data);
				};

				s.socket.onerror = function(evt) {
					setTimeout(function() {
						s.connect(cId, c);
					});
				};

				s.socket.onopen = function(evt) {
					if (!readyCalled) {
						RelayRConnection.r();
						readyCalled = true;
					}
				};
			},
			send: function(data) {
				var s = this;
				s.socket.send(data);
			}
		},
		longpoll: {
			connect: function(c) {
				if (!readyCalled) {
					RelayRConnection.r();
					readyCalled = true;
				}
				var retry;
				retry = function() {
					web.gj(route + '/longpoll?connectionId=' + transport.ConnectionId + '&_=' + new Date().getTime(), function(data) {
						if (data.responseText) {
							var reconn = JSON.parse(data.responseText);
							if (reconn.Z) {
								web.n();
							} else {
								c(data);
								retry();
							}
						} else {
							web.n();
						}
					});
				};

				retry();
			},
			send: function(data) {
				var s = this;
				web.p(route + '/call?connectionId=' + transport.ConnectionId + '&_=' + new Date().getTime(), data, null, "json", null);
			}
		}
	};
	web = (function() {
		return {
			x: function() {
				var xd;

				if (window.XMLHttpRequest) {
					xd = new XMLHttpRequest();
				}
				else {
					xd = new ActiveXObject("Microsoft.XMLHTTP");
				}

				return xd;
			},
			g: function(u, c) {
				var s = this;

				var xd = s.x();

				xd.open('GET', u, true);

				xd.onreadystatechange = function() {
					if (xd.readyState === 4 && xd.status === 200) {
						c(xd);
					}
				};

				xd.send();
			},
			gj: function(u, c) {
				var s = this;

				var xd = s.x();

				xd.open('GET', u, true);
				xd.setRequestHeader("Content-type", "application/json");

				xd.onreadystatechange = function() {
					if (xd.readyState === 4) {
						if (xd.status === 200) {
							c(xd);
						} else {
							s.n();
						}
					}
				};

				xd.send();
			},
			p: function(u, d, c, t, e) {
				var s = this;

				var xd = s.x();

				xd.open('POST', u, true);
				xd.setRequestHeader("Content-type", "application/" + t);

				xd.onreadystatechange = function() {
					if (xd.readyState === 4) {
						if (xd.status === 200) {
							if (c) {
								c(xd);
							}
						} else {
							s.n();
						}
					}
				};

				xd.onerror = function() {
					if (e) {
						e(xd);
					}
				};

				xd.send(d);

				window.onbeforeunload = function() {
					delete xd;
					xd = null;
				};
			},
			t: function() {
				if (!!window.WebSocket) {
					return "websocket";
				} else {
					return "longpoll";
				}

				// TODO: Implement SSE Circuit
				/*if (typeof EventSource !== 'undefined') {
					return "SSE";
				}*/
			},
			n: function() {
				var s = this;
				var t = s.t();
				web.p(route + "/negotiate?_=" + new Date().getTime(), JSON.stringify({ t: t }), function(result) {
					var obj = JSON.parse(result.responseText);
					transport.ConnectionId = obj.ConnectionID;
					setTimeout(function() {
						transport[t].connect(function(data) {
							var cobj;
							if (data.responseText && data.status && data.responseXML) {
								cobj = JSON.parse(data.responseText);
							} else {
								if (data.responseText == "") return;
								cobj = JSON.parse(data);
							}
							var lobj = RelayR[cobj.R].client;
							var args = [];
							for (var i = 0; i < cobj.A.length; i++) {
								args.push(cobj.A[i]);
							}
							lobj[cobj.M].apply(lobj||window, args);
						});
					}, 0);
				}, "json",
				function(result) {
					// error .. try again
					setTimeout(function() {
						s.n();
					}, 2000);
				});
			}
		};
	})();

	return {
		ready: function(r) {
			RelayRConnection.r = r;

			web.n();
		},
		callServer: function(r, f, a) {
			transport[web.t()].send(JSON.stringify({ S: true, C: transport.ConnectionId, R: r, M: f, A: a}));
		}
	};
})();

`

const relayClassBegin = `

var RelayR = (function() {
	return {

`

const relayBegin = `

%v: {

	client: {},

	server: {

`

// {0} == function name
// {1} == Relay name
// {2} == function name
const relayMethod = `

%v: function() {
	RelayRConnection.callServer('%v', '%v', Array.prototype.slice.call(arguments));
},

`

const relayEnd = `

},

},

`

const relayClassEnd = `

	};
})();

`
<|endoftext|>


<commit_before>package gitmediaclient

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
)

func Put(filename string) error {
	sha := filepath.Base(filename)
	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", objectUrl(sha), file)
	if err != nil {
		return err
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	fmt.Printf("Sending %s from %s: %d\n", sha, filename, res.StatusCode)
	return nil
}

func Get(filename string) (io.ReadCloser, error) {
	sha := filepath.Base(filename)
	if stat, err := os.Stat(filename); err != nil || stat == nil {
		req, err := http.NewRequest("GET", objectUrl(sha), nil)
		if err != nil {
			return nil, err
		}

		req.Header.Set("Accept", "application/vnd.git-media")
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return nil, err
		}

		fmt.Printf("Downloading %s: %d\n", sha, res.StatusCode)
		return res.Body, nil
	}

	return os.Open(filename)
}

func objectUrl(sha string) string {
	return "http://localhost:8080/objects/" + sha
}
<commit_msg>ラララララ ラー ララララー<commit_after>package gitmediaclient

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"path/filepath"
)

func Put(filename string) error {
	sha := filepath.Base(filename)
	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("PUT", objectUrl(sha), file)
	if err != nil {
		return err
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	defer res.Body.Close()

	if res.StatusCode > 299 {
		apierr := &Error{}
		dec := json.NewDecoder(res.Body)
		if err = dec.Decode(apierr); err != nil {
			return err
		}
		return apierr
	}

	fmt.Printf("Sending %s from %s: %d\n", sha, filename, res.StatusCode)
	return nil
}

func Get(filename string) (io.ReadCloser, error) {
	sha := filepath.Base(filename)
	if stat, err := os.Stat(filename); err != nil || stat == nil {
		req, err := http.NewRequest("GET", objectUrl(sha), nil)
		if err != nil {
			return nil, err
		}

		req.Header.Set("Accept", "application/vnd.git-media")
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return nil, err
		}

		fmt.Printf("Downloading %s: %d\n", sha, res.StatusCode)
		return res.Body, nil
	}

	return os.Open(filename)
}

func objectUrl(sha string) string {
	return "http://localhost:8080/objects/" + sha
}

type Error struct {
	Message   string `json:"message"`
	RequestId string `json:"request_id,omitempty"`
}

func (e *Error) Error() string {
	return e.Message
}
<|endoftext|>


<commit_before>package condor

import (
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"tes"
	pbe "tes/ga4gh"
	sched "tes/scheduler"
	worker "tes/worker"
	"text/template"
)

func NewScheduler(c tes.Config) sched.Scheduler {
	return &scheduler{c}
}

type scheduler struct {
	conf tes.Config
}

func (s *scheduler) Schedule(j *pbe.Job) sched.Offer {
	log.Println("Running condor scheduler")

	w := sched.Worker{
		ID: sched.GenWorkerID(),
		Resources: sched.Resources{
			// TODO
			CPU:  1,
			RAM:  1.0,
			Disk: 10.0,
		},
	}
	o := sched.NewOffer(j, w)
	go s.observe(o)
	return o
}

func (s *scheduler) observe(o sched.Offer) {
	<-o.Wait()
	if o.Accepted() {
		s.startWorker(o.Worker().ID)
	} else if o.Rejected() {
		log.Println("Condor offer was rejected")
	}
}

func (s *scheduler) startWorker(workerID string) {
	log.Println("Start condor worker")
	// TODO document that these working dirs need manual cleanup
	workdir := path.Join(s.conf.WorkDir, "condor-scheduler", workerID)
	workdir, _ = filepath.Abs(workdir)
	os.MkdirAll(workdir, 0755)

	workerConf := worker.Config{
		ID:            workerID,
		ServerAddress: s.conf.ServerAddress,
		Timeout:       0,
		NumWorkers:    1,
		Storage:       s.conf.Storage,
		WorkDir:       workdir,
	}
	confPath := path.Join(workdir, "worker.conf.yml")
	workerConf.ToYamlFile(confPath)

	workerPath := sched.DetectWorkerPath()

	submitPath := path.Join(workdir, "condor.submit")
	f, _ := os.Create(submitPath)

	submitTpl, _ := template.New("condor.submit").Parse(`
		universe    = vanilla
		executable  = {{.Executable}}
		arguments   = -config worker.conf.yml
		environment = "PATH=/usr/bin"
		log         = {{.WorkDir}}/condor-event-log
		error       = {{.WorkDir}}/tes-worker-stderr
		output      = {{.WorkDir}}/tes-worker-stdout
    input       = {{.Config}}
    should_transfer_files   = YES
    when_to_transfer_output = ON_EXIT
		queue
	`)
	submitTpl.Execute(f, map[string]string{
		"Executable": workerPath,
		"WorkDir":    workdir,
		"Config":     confPath,
	})
	f.Close()

	cmd := exec.Command("condor_submit")
	stdin, _ := os.Open(submitPath)
	cmd.Stdin = stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}
<commit_msg>Use Condor scratch dir as default workdir for worker<commit_after>package condor

import (
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"tes"
	pbe "tes/ga4gh"
	sched "tes/scheduler"
	worker "tes/worker"
	"text/template"
)

func NewScheduler(c tes.Config) sched.Scheduler {
	return &scheduler{c}
}

type scheduler struct {
	conf tes.Config
}

func (s *scheduler) Schedule(j *pbe.Job) sched.Offer {
	log.Println("Running condor scheduler")

	w := sched.Worker{
		ID: sched.GenWorkerID(),
		Resources: sched.Resources{
			// TODO
			CPU:  1,
			RAM:  1.0,
			Disk: 10.0,
		},
	}
	o := sched.NewOffer(j, w)
	go s.observe(o)
	return o
}

func (s *scheduler) observe(o sched.Offer) {
	<-o.Wait()
	if o.Accepted() {
		s.startWorker(o.Worker().ID)
	} else if o.Rejected() {
		log.Println("Condor offer was rejected")
	}
}

func (s *scheduler) startWorker(workerID string) {
	log.Println("Start condor worker")
	// TODO document that these working dirs need manual cleanup
	workdir := path.Join(s.conf.WorkDir, "condor-scheduler", workerID)
	workdir, _ = filepath.Abs(workdir)
	os.MkdirAll(workdir, 0755)

	workerConf := worker.Config{
		ID:            workerID,
		ServerAddress: s.conf.ServerAddress,
		Timeout:       0,
		NumWorkers:    1,
		Storage:       s.conf.Storage,
		WorkDir:       "",
	}
	confPath := path.Join(workdir, "worker.conf.yml")
	workerConf.ToYamlFile(confPath)

	workerPath := sched.DetectWorkerPath()

	submitPath := path.Join(workdir, "condor.submit")
	f, _ := os.Create(submitPath)

	submitTpl, _ := template.New("condor.submit").Parse(`
		universe    = vanilla
		executable  = {{.Executable}}
		arguments   = -config worker.conf.yml
		environment = "PATH=/usr/bin"
		log         = {{.WorkDir}}/condor-event-log
		error       = {{.WorkDir}}/tes-worker-stderr
		output      = {{.WorkDir}}/tes-worker-stdout
    input       = {{.Config}}
    should_transfer_files   = YES
    when_to_transfer_output = ON_EXIT
		queue
	`)
	submitTpl.Execute(f, map[string]string{
		"Executable": workerPath,
		"WorkDir":    workdir,
		"Config":     confPath,
	})
	f.Close()

	cmd := exec.Command("condor_submit")
	stdin, _ := os.Open(submitPath)
	cmd.Stdin = stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Run()
}
<|endoftext|>