<commit_before>package sharing

import (
	"strings"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/permissions"
	"github.com/cozy/cozy-stack/pkg/vfs"
)

const (
	// ActionRuleNone is used when an add/update/remove should not be
	// replicated to the other cozys
	ActionRuleNone = "none"
	// ActionRulePush is used when an add/update/remove should be replicated
	// only if it happened on the owner's cozy
	ActionRulePush = "push"
	// ActionRuleSync is used when an add/update/remove should be always replicated
	ActionRuleSync = "sync"
	// ActionRuleRevoke is used when a remove should revoke the sharing
	ActionRuleRevoke = "revoke"
)

// Rule describes how the sharing behave when a document matching the rule is
// added, updated or deleted.
type Rule struct {
	Title    string   `json:"title"`
	DocType  string   `json:"doctype"`
	Selector string   `json:"selector,omitempty"`
	Values   []string `json:"values"`
	Local    bool     `json:"local,omitempty"`
	Add      string   `json:"add"`
	Update   string   `json:"update"`
	Remove   string   `json:"remove"`
}

// FilesByID returns true if the rule is for the files by doctype and the
// selector is an id (not a referenced_by). With such a rule, the identifiers
// must be xored before being sent to another cozy instance.
func (r Rule) FilesByID() bool {
	if r.DocType != consts.Files {
		return false
	}
	return r.Selector == "" || r.Selector == "id" || r.Selector == "_id"
}

// ValidateRules returns an error if the rules are invalid (the doctype is
// missing for example)
func (s *Sharing) ValidateRules() error {
	if len(s.Rules) == 0 {
		return ErrNoRules
	}
	for _, rule := range s.Rules {
		if rule.Title == "" || rule.DocType == "" || len(rule.Values) == 0 {
			return ErrInvalidRule
		}
		if rule.DocType == consts.Files {
			for _, val := range rule.Values {
				if val == consts.RootDirID ||
					val == consts.TrashDirID ||
					val == consts.SharedWithMeDirID {
					return ErrInvalidRule
				}
			}
			// XXX Currently, we only support one file/folder per rule for the id selector
			if rule.Selector == "" || rule.Selector == "id" || rule.Selector == "_id" {
				if len(rule.Values) > 1 {
					return ErrInvalidRule
				}
			}
			if rule.Selector == couchdb.SelectorReferencedBy {
				// For a referenced_by rule, values should be "doctype/docid"
				for _, val := range rule.Values {
					parts := strings.SplitN(val, "/", 2)
					if len(parts) != 2 {
						return ErrInvalidRule
					}
				}
			}
		} else if permissions.CheckWritable(rule.DocType) != nil {
			return ErrInvalidRule
		}
		if rule.Add == "" {
			rule.Add = ActionRuleNone
		}
		rule.Add = strings.ToLower(rule.Add)
		if rule.Add != ActionRuleNone &&
			rule.Add != ActionRulePush &&
			rule.Add != ActionRuleSync {
			return ErrInvalidRule
		}
		if rule.Update == "" {
			rule.Update = ActionRuleNone
		}
		rule.Update = strings.ToLower(rule.Update)
		if rule.Update != ActionRuleNone &&
			rule.Update != ActionRulePush &&
			rule.Update != ActionRuleSync {
			return ErrInvalidRule
		}
		if rule.Remove == "" {
			rule.Remove = ActionRuleNone
		}
		rule.Remove = strings.ToLower(rule.Remove)
		if rule.Remove != ActionRuleNone &&
			rule.Remove != ActionRulePush &&
			rule.Remove != ActionRuleSync &&
			rule.Remove != ActionRuleRevoke {
			return ErrInvalidRule
		}
	}
	return nil
}

// Accept returns true if the document matches the rule criteria
func (r Rule) Accept(doctype string, doc map[string]interface{}) bool {
	if r.Local || doctype != r.DocType {
		return false
	}
	var obj interface{} = doc
	if r.Selector == "" || r.Selector == "id" {
		obj = doc["_id"]
	} else if doctype == consts.Files && r.Selector == couchdb.SelectorReferencedBy {
		if o, k := doc[couchdb.SelectorReferencedBy].([]map[string]interface{}); k {
			refs := make([]string, len(o))
			for i, ref := range o {
				refs[i] = ref["type"].(string) + "/" + ref["id"].(string)
			}
			obj = refs
		}
	} else {
		keys := strings.Split(r.Selector, ".")
		for _, key := range keys {
			if o, k := obj.(map[string]interface{}); k {
				obj = o[key]
			} else {
				obj = nil
				break
			}
		}
	}
	if val, ok := obj.(string); ok {
		for _, v := range r.Values {
			if v == val {
				return true
			}
		}
	}
	if val, ok := obj.([]string); ok {
		for _, vv := range val {
			for _, v := range r.Values {
				if v == vv {
					return true
				}
			}
		}
	}
	return false
}

// TriggerArgs returns the string that can be used as an argument to create a
// trigger for this rule. The result can be an empty string if the rule doesn't
// need a trigger (a local or one-shot rule).
func (r Rule) TriggerArgs() string {
	if r.Local {
		return ""
	}
	verbs := make([]string, 0, 3)
	if r.Add == ActionRuleSync || r.Add == ActionRulePush {
		verbs = append(verbs, "CREATED")
	}
	if r.Update == ActionRuleSync || r.Update == ActionRulePush {
		verbs = append(verbs, "UPDATED")
	}
	if r.Remove == ActionRuleSync || r.Remove == ActionRulePush {
		verbs = append(verbs, "UPDATED")
	}
	if len(verbs) == 0 {
		return ""
	}
	args := r.DocType + ":" + strings.Join(verbs, ",")
	if len(r.Values) > 0 {
		args += ":" + strings.Join(r.Values, ",")
		if r.Selector != "" && r.Selector != "id" {
			args += ":" + r.Selector
		}
	}
	return args
}

// FirstFilesRule returns the first not-local rules for the files doctype
func (s *Sharing) FirstFilesRule() *Rule {
	for i, rule := range s.Rules {
		if !rule.Local && rule.DocType == consts.Files {
			return &s.Rules[i]
		}
	}
	return nil
}

func (s *Sharing) findRuleForNewDirectory(dir *vfs.DirDoc) (*Rule, int) {
	for i, rule := range s.Rules {
		if rule.Local || rule.DocType != consts.Files {
			continue
		}
		if rule.Selector != couchdb.SelectorReferencedBy {
			return &s.Rules[i], i
		}
		if len(dir.ReferencedBy) == 0 {
			continue
		}
		allFound := true
		for _, ref := range dir.ReferencedBy {
			if !rule.hasReferencedBy(ref) {
				allFound = false
				break
			}
		}
		if allFound {
			return &s.Rules[i], i
		}
	}
	return nil, 0
}

func (s *Sharing) findRuleForNewFile(file *vfs.FileDoc) (*Rule, int) {
	for i, rule := range s.Rules {
		if rule.Local || rule.DocType != consts.Files {
			continue
		}
		if rule.Selector != couchdb.SelectorReferencedBy {
			return &s.Rules[i], i
		}
		if len(file.ReferencedBy) == 0 {
			continue
		}
		allFound := true
		for _, ref := range file.ReferencedBy {
			if !rule.hasReferencedBy(ref) {
				allFound = false
				break
			}
		}
		if allFound {
			return &s.Rules[i], i
		}
	}
	return nil, 0
}

// HasSync returns true if the rule has a sync behaviour
func (r *Rule) HasSync() bool {
	return r.Add == ActionRuleSync || r.Update == ActionRuleSync ||
		r.Remove == ActionRuleSync
}

// HasPush returns true if the rule has a sync behaviour
func (r *Rule) HasPush() bool {
	return r.Add == ActionRulePush || r.Update == ActionRulePush ||
		r.Remove == ActionRulePush
}

// hasReferencedBy returns true if the rule matches a file that has this reference
func (r *Rule) hasReferencedBy(ref couchdb.DocReference) bool {
	if r.Selector != couchdb.SelectorReferencedBy {
		return false
	}
	v := ref.Type + "/" + ref.ID
	for _, val := range r.Values {
		if val == v {
			return true
		}
	}
	return false
}
<commit_msg>Fix default values for rules<commit_after>package sharing

import (
	"strings"

	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/permissions"
	"github.com/cozy/cozy-stack/pkg/vfs"
)

const (
	// ActionRuleNone is used when an add/update/remove should not be
	// replicated to the other cozys
	ActionRuleNone = "none"
	// ActionRulePush is used when an add/update/remove should be replicated
	// only if it happened on the owner's cozy
	ActionRulePush = "push"
	// ActionRuleSync is used when an add/update/remove should be always replicated
	ActionRuleSync = "sync"
	// ActionRuleRevoke is used when a remove should revoke the sharing
	ActionRuleRevoke = "revoke"
)

// Rule describes how the sharing behave when a document matching the rule is
// added, updated or deleted.
type Rule struct {
	Title    string   `json:"title"`
	DocType  string   `json:"doctype"`
	Selector string   `json:"selector,omitempty"`
	Values   []string `json:"values"`
	Local    bool     `json:"local,omitempty"`
	Add      string   `json:"add"`
	Update   string   `json:"update"`
	Remove   string   `json:"remove"`
}

// FilesByID returns true if the rule is for the files by doctype and the
// selector is an id (not a referenced_by). With such a rule, the identifiers
// must be xored before being sent to another cozy instance.
func (r Rule) FilesByID() bool {
	if r.DocType != consts.Files {
		return false
	}
	return r.Selector == "" || r.Selector == "id" || r.Selector == "_id"
}

// ValidateRules returns an error if the rules are invalid (the doctype is
// missing for example)
func (s *Sharing) ValidateRules() error {
	if len(s.Rules) == 0 {
		return ErrNoRules
	}
	for i, rule := range s.Rules {
		if rule.Title == "" || rule.DocType == "" || len(rule.Values) == 0 {
			return ErrInvalidRule
		}
		if rule.DocType == consts.Files {
			for _, val := range rule.Values {
				if val == consts.RootDirID ||
					val == consts.TrashDirID ||
					val == consts.SharedWithMeDirID {
					return ErrInvalidRule
				}
			}
			// XXX Currently, we only support one file/folder per rule for the id selector
			if rule.Selector == "" || rule.Selector == "id" || rule.Selector == "_id" {
				if len(rule.Values) > 1 {
					return ErrInvalidRule
				}
			}
			if rule.Selector == couchdb.SelectorReferencedBy {
				// For a referenced_by rule, values should be "doctype/docid"
				for _, val := range rule.Values {
					parts := strings.SplitN(val, "/", 2)
					if len(parts) != 2 {
						return ErrInvalidRule
					}
				}
			}
		} else if permissions.CheckWritable(rule.DocType) != nil {
			return ErrInvalidRule
		}
		if rule.Add == "" {
			s.Rules[i].Add = ActionRuleNone
			rule.Add = s.Rules[i].Add
		}
		rule.Add = strings.ToLower(rule.Add)
		if rule.Add != ActionRuleNone &&
			rule.Add != ActionRulePush &&
			rule.Add != ActionRuleSync {
			return ErrInvalidRule
		}
		if rule.Update == "" {
			s.Rules[i].Update = ActionRuleNone
			rule.Update = s.Rules[i].Update
		}
		rule.Update = strings.ToLower(rule.Update)
		if rule.Update != ActionRuleNone &&
			rule.Update != ActionRulePush &&
			rule.Update != ActionRuleSync {
			return ErrInvalidRule
		}
		if rule.Remove == "" {
			s.Rules[i].Remove = ActionRuleNone
			rule.Remove = s.Rules[i].Remove
		}
		rule.Remove = strings.ToLower(rule.Remove)
		if rule.Remove != ActionRuleNone &&
			rule.Remove != ActionRulePush &&
			rule.Remove != ActionRuleSync &&
			rule.Remove != ActionRuleRevoke {
			return ErrInvalidRule
		}
	}
	return nil
}

// Accept returns true if the document matches the rule criteria
func (r Rule) Accept(doctype string, doc map[string]interface{}) bool {
	if r.Local || doctype != r.DocType {
		return false
	}
	var obj interface{} = doc
	if r.Selector == "" || r.Selector == "id" {
		obj = doc["_id"]
	} else if doctype == consts.Files && r.Selector == couchdb.SelectorReferencedBy {
		if o, k := doc[couchdb.SelectorReferencedBy].([]map[string]interface{}); k {
			refs := make([]string, len(o))
			for i, ref := range o {
				refs[i] = ref["type"].(string) + "/" + ref["id"].(string)
			}
			obj = refs
		}
	} else {
		keys := strings.Split(r.Selector, ".")
		for _, key := range keys {
			if o, k := obj.(map[string]interface{}); k {
				obj = o[key]
			} else {
				obj = nil
				break
			}
		}
	}
	if val, ok := obj.(string); ok {
		for _, v := range r.Values {
			if v == val {
				return true
			}
		}
	}
	if val, ok := obj.([]string); ok {
		for _, vv := range val {
			for _, v := range r.Values {
				if v == vv {
					return true
				}
			}
		}
	}
	return false
}

// TriggerArgs returns the string that can be used as an argument to create a
// trigger for this rule. The result can be an empty string if the rule doesn't
// need a trigger (a local or one-shot rule).
func (r Rule) TriggerArgs() string {
	if r.Local {
		return ""
	}
	verbs := make([]string, 0, 3)
	if r.Add == ActionRuleSync || r.Add == ActionRulePush {
		verbs = append(verbs, "CREATED")
	}
	if r.Update == ActionRuleSync || r.Update == ActionRulePush {
		verbs = append(verbs, "UPDATED")
	}
	if r.Remove == ActionRuleSync || r.Remove == ActionRulePush {
		verbs = append(verbs, "UPDATED")
	}
	if len(verbs) == 0 {
		return ""
	}
	args := r.DocType + ":" + strings.Join(verbs, ",")
	if len(r.Values) > 0 {
		args += ":" + strings.Join(r.Values, ",")
		if r.Selector != "" && r.Selector != "id" {
			args += ":" + r.Selector
		}
	}
	return args
}

// FirstFilesRule returns the first not-local rules for the files doctype
func (s *Sharing) FirstFilesRule() *Rule {
	for i, rule := range s.Rules {
		if !rule.Local && rule.DocType == consts.Files {
			return &s.Rules[i]
		}
	}
	return nil
}

func (s *Sharing) findRuleForNewDirectory(dir *vfs.DirDoc) (*Rule, int) {
	for i, rule := range s.Rules {
		if rule.Local || rule.DocType != consts.Files {
			continue
		}
		if rule.Selector != couchdb.SelectorReferencedBy {
			return &s.Rules[i], i
		}
		if len(dir.ReferencedBy) == 0 {
			continue
		}
		allFound := true
		for _, ref := range dir.ReferencedBy {
			if !rule.hasReferencedBy(ref) {
				allFound = false
				break
			}
		}
		if allFound {
			return &s.Rules[i], i
		}
	}
	return nil, 0
}

func (s *Sharing) findRuleForNewFile(file *vfs.FileDoc) (*Rule, int) {
	for i, rule := range s.Rules {
		if rule.Local || rule.DocType != consts.Files {
			continue
		}
		if rule.Selector != couchdb.SelectorReferencedBy {
			return &s.Rules[i], i
		}
		if len(file.ReferencedBy) == 0 {
			continue
		}
		allFound := true
		for _, ref := range file.ReferencedBy {
			if !rule.hasReferencedBy(ref) {
				allFound = false
				break
			}
		}
		if allFound {
			return &s.Rules[i], i
		}
	}
	return nil, 0
}

// HasSync returns true if the rule has a sync behaviour
func (r *Rule) HasSync() bool {
	return r.Add == ActionRuleSync || r.Update == ActionRuleSync ||
		r.Remove == ActionRuleSync
}

// HasPush returns true if the rule has a sync behaviour
func (r *Rule) HasPush() bool {
	return r.Add == ActionRulePush || r.Update == ActionRulePush ||
		r.Remove == ActionRulePush
}

// hasReferencedBy returns true if the rule matches a file that has this reference
func (r *Rule) hasReferencedBy(ref couchdb.DocReference) bool {
	if r.Selector != couchdb.SelectorReferencedBy {
		return false
	}
	v := ref.Type + "/" + ref.ID
	for _, val := range r.Values {
		if val == v {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// If you are looking at these fields in the git tree, they look
// strange. They are modified on the fly by the build process. The
// in-tree values are dummy values used for "git archive", which also
// works for GitHub tar downloads.
//
// When releasing a new Kubernetes version, this file is updated by
// build/mark_new_version.sh to reflect the new version, and then a
// git annotated tag (using format vX.Y where X == Major version and Y
// == Minor version) is created to point to the commit that updates
// pkg/version/base.go
var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion
	// instead. First step in deprecation, keep the fields but make
	// them irrelevant. (Next we'll take it out, which may muck with
	// scripts consuming the kubectl version output - but most of
	// these should be looking at gitVersion already anyways.)
	gitMajor string = "1"  // major version, always numeric
	gitMinor string = "4+" // minor version, numeric possibly followed by "+"

	// semantic version, derived by build scripts (see
	// https://github.com/kubernetes/kubernetes/blob/master/docs/design/versioning.md
	// for a detailed discussion of this field)
	//
	// TODO: This field is still called "gitVersion" for legacy
	// reasons. For prerelease versions, the build metadata on the
	// semantic version is a git hash, but the version itself is no
	// longer the direct output of "git describe", but a slight
	// translation to be semver compliant.
	gitVersion   string = "v1.4.1-beta.0+$Format:%h$"
	gitCommit    string = "$Format:%H$"    // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"

	buildDate string = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
<commit_msg>Kubernetes version v1.4.1-beta.1<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package version

// Base version information.
//
// This is the fallback data used when version information from git is not
// provided via go ldflags. It provides an approximation of the Kubernetes
// version for ad-hoc builds (e.g. `go build`) that cannot get the version
// information from git.
//
// If you are looking at these fields in the git tree, they look
// strange. They are modified on the fly by the build process. The
// in-tree values are dummy values used for "git archive", which also
// works for GitHub tar downloads.
//
// When releasing a new Kubernetes version, this file is updated by
// build/mark_new_version.sh to reflect the new version, and then a
// git annotated tag (using format vX.Y where X == Major version and Y
// == Minor version) is created to point to the commit that updates
// pkg/version/base.go
var (
	// TODO: Deprecate gitMajor and gitMinor, use only gitVersion
	// instead. First step in deprecation, keep the fields but make
	// them irrelevant. (Next we'll take it out, which may muck with
	// scripts consuming the kubectl version output - but most of
	// these should be looking at gitVersion already anyways.)
	gitMajor string = "1"  // major version, always numeric
	gitMinor string = "4+" // minor version, numeric possibly followed by "+"

	// semantic version, derived by build scripts (see
	// https://github.com/kubernetes/kubernetes/blob/master/docs/design/versioning.md
	// for a detailed discussion of this field)
	//
	// TODO: This field is still called "gitVersion" for legacy
	// reasons. For prerelease versions, the build metadata on the
	// semantic version is a git hash, but the version itself is no
	// longer the direct output of "git describe", but a slight
	// translation to be semver compliant.
	gitVersion   string = "v1.4.1-beta.1+$Format:%h$"
	gitCommit    string = "$Format:%H$"    // sha1 from git, output of $(git rev-parse HEAD)
	gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty"

	buildDate string = "1970-01-01T00:00:00Z" // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ')
)
<|endoftext|>


<commit_before>package bassh

import (
	"crypto/x509"
	"encoding/pem"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"syscall"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
)

//SSHParams contains params to setup the Session
type SSHParams struct {
	Env    []string
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

//SSHClient wraps the ssh client configuration and the host/port information
type SSHClient struct {
	Session *ssh.Session
	Config  *ssh.ClientConfig
	Host    string
	Port    int
}

//InitSession returns a session initialised with the given params
func (client *SSHClient) InitSession(params *SSHParams) error {
	var (
		session *ssh.Session
		err     error
	)

	if session, err = client.newSession(); err != nil {
		return err
	}

	if err = client.prepareCommand(session, params); err != nil {
		return err
	}

	client.Session = session

	return nil
}

//InitStandardSession returns a session initialised with default params
//using stdIn, stdOut and stdErr to prepare the command
func (client *SSHClient) InitStandardSession() error {
	params := &SSHParams{
		Env:    []string{""},
		Stdin:  os.Stdin,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}

	return client.InitSession(params)
}

//CloseSession closes the session for the client
func (client *SSHClient) CloseSession() {
	client.Session.Close()
}

func (client *SSHClient) prepareCommand(session *ssh.Session, params *SSHParams) error {
	for _, env := range params.Env {
		variable := strings.Split(env, "=")
		if len(variable) != 2 {
			continue
		}
		fmt.Println("Setting env variable ", variable[0], " to ", variable[1])
		if err := session.Setenv(variable[0], variable[1]); err != nil {
			fmt.Println("The remote system doesn't accept the setEnv command: ", err)
			return err
		}
	}

	if params.Stdin != nil {
		stdin, err := session.StdinPipe()
		if err != nil {
			return fmt.Errorf("Unable to setup stdin for session: %v", err)
		}
		go io.Copy(stdin, params.Stdin)
	}

	if params.Stdout != nil {
		stdout, err := session.StdoutPipe()
		if err != nil {
			return fmt.Errorf("Unable to setup stdout for session: %v", err)
		}
		go io.Copy(params.Stdout, stdout)
	}

	if params.Stderr != nil {
		stderr, err := session.StderrPipe()
		if err != nil {
			return fmt.Errorf("Unable to setup stderr for session: %v", err)
		}
		go io.Copy(params.Stderr, stderr)
	}

	return nil
}

func (client *SSHClient) newSession() (*ssh.Session, error) {
	connection, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", client.Host, client.Port), client.Config)
	if err != nil {
		return nil, fmt.Errorf("Failed to dial: %s", err)
	}

	session, err := connection.NewSession()
	if err != nil {
		return nil, fmt.Errorf("Failed to create session: %s", err)
	}

	modes := ssh.TerminalModes{
		// ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		session.Close()
		return nil, fmt.Errorf("request for pseudo terminal failed: %s", err)
	}

	return session, nil
}

func decodeKeyForAuthMethod(file string) (ssh.AuthMethod, error) {
	fmt.Printf("Private key is at: %s\n", file)
	buffer, err := ioutil.ReadFile(file)
	if err != nil {
		fmt.Println("Error while reading the file: ", err)
		return nil, errors.New("Error while reading the .pem file from disk!")
	}
	decryptedBuffer, err := decryptIfEncrypted(buffer)
	if err != nil {
		return nil, err
	}
	key, err := ssh.ParsePrivateKey(decryptedBuffer)
	if err != nil {
		fmt.Println("Error while parsing private key: ", err)
		return nil, err
	}
	fmt.Println("Private key succesfully decripted and decoded.")

	return ssh.PublicKeys(key), nil
}

func decryptIfEncrypted(buffer []byte) ([]byte, error) {
	//  Decode the key extracting the pem.Block structure
	block, _ := pem.Decode(buffer)
	if block == nil {
		return nil, errors.New("Failed to decode certificate buffer")
	}
	//  Verify if the pem.block is Encrypted
	if x509.IsEncryptedPEMBlock(block) {
		fmt.Println("Key is encrypted, specify decrypt passphrase: ")
		bytePassword, err := terminal.ReadPassword(int(syscall.Stdin))
		if err == nil {
			fmt.Println("\nPassword typed: " + string(bytePassword))
		}
		passphrase := string(bytePassword)
		decryptedPem, err := x509.DecryptPEMBlock(block, []byte(strings.TrimSpace(passphrase)))
		if err != nil {
			fmt.Println("Error while reading the file: ", err)
			return nil, err
		}
		//  Recreating the decoded block to be returned
		var newBlock pem.Block
		newBlock.Type = block.Type
		newBlock.Headers = block.Headers
		newBlock.Bytes = decryptedPem
		//  Encoding block into []byte and returning
		return pem.EncodeToMemory(&newBlock), nil
	}
	fmt.Println("Key is not encrypted.")

	return buffer, nil
}

//func sshAgent() ssh.AuthMethod {
//	if sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
//		return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
//	}
//	return nil
//}

//ConfigureCredentials returns the ClientConfig struct to be used as part of the
//SSHClient definition
func ConfigureCredentials(username string, keypath string) (*ssh.ClientConfig, error) {
	var config ssh.ClientConfig
	config.User = username
	pemKeyCommand := keypath
	authMethod, err := decodeKeyForAuthMethod(pemKeyCommand)
	if err != nil {
		return nil, err
	}
	config.Auth = []ssh.AuthMethod{authMethod}
	return &config, nil
}

//CreateClient takes a *ssh.ClientConfig struct as input, ipAddress of the target
//machine and the ssh port, and returns an *SSHClient where a command can be Run on
func CreateClient(sshConfig *ssh.ClientConfig, ipAddr string, port int) *SSHClient {
	return &SSHClient{
		Config: sshConfig,
		Host:   ipAddr,
		Port:   port,
	}
}

//Run opens an SSH session and Runs the command passed as an argument
func (client *SSHClient) Run(command string) {

	if err := client.InitStandardSession(); err != nil {
		fmt.Errorf("Error while initialising SSH session! Error was: %s", err)
	}
	defer client.CloseSession()

	if err := client.Session.Run(command); err != nil {
		fmt.Fprintf(os.Stderr, "command run error: %s\n", err)
		if client.Session == nil {
			fmt.Println("Session not initialised.")
		}
		os.Exit(1)
	}
}

//RunBash runs /bin/bash on the client
func (client *SSHClient) RunBash() {
	client.Run("/bin/bash")
}
<commit_msg>Adding required config param for HostKeyCallback<commit_after>package bassh

import (
	"crypto/x509"
	"encoding/pem"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"syscall"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
)

//SSHParams contains params to setup the Session
type SSHParams struct {
	Env    []string
	Stdin  io.Reader
	Stdout io.Writer
	Stderr io.Writer
}

//SSHClient wraps the ssh client configuration and the host/port information
type SSHClient struct {
	Session *ssh.Session
	Config  *ssh.ClientConfig
	Host    string
	Port    int
}

//InitSession returns a session initialised with the given params
func (client *SSHClient) InitSession(params *SSHParams) error {
	var (
		session *ssh.Session
		err     error
	)

	if session, err = client.newSession(); err != nil {
		return err
	}

	if err = client.prepareCommand(session, params); err != nil {
		return err
	}

	client.Session = session

	return nil
}

//InitStandardSession returns a session initialised with default params
//using stdIn, stdOut and stdErr to prepare the command
func (client *SSHClient) InitStandardSession() error {
	params := &SSHParams{
		Env:    []string{""},
		Stdin:  os.Stdin,
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}

	return client.InitSession(params)
}

//CloseSession closes the session for the client
func (client *SSHClient) CloseSession() {
	if client.Session == nil {
		return
	}
	if err := client.Session.Close(); err != nil {
		fmt.Printf("Error while closing session: %s", err)
	}
}

func (client *SSHClient) prepareCommand(session *ssh.Session, params *SSHParams) error {
	for _, env := range params.Env {
		variable := strings.Split(env, "=")
		if len(variable) != 2 {
			continue
		}
		fmt.Println("Setting env variable ", variable[0], " to ", variable[1])
		if err := session.Setenv(variable[0], variable[1]); err != nil {
			fmt.Println("The remote system doesn't accept the setEnv command: ", err)
			return err
		}
	}

	if params.Stdin != nil {
		stdin, err := session.StdinPipe()
		if err != nil {
			return fmt.Errorf("Unable to setup stdin for session: %v", err)
		}
		go io.Copy(stdin, params.Stdin)
	}

	if params.Stdout != nil {
		stdout, err := session.StdoutPipe()
		if err != nil {
			return fmt.Errorf("Unable to setup stdout for session: %v", err)
		}
		go io.Copy(params.Stdout, stdout)
	}

	if params.Stderr != nil {
		stderr, err := session.StderrPipe()
		if err != nil {
			return fmt.Errorf("Unable to setup stderr for session: %v", err)
		}
		go io.Copy(params.Stderr, stderr)
	}

	return nil
}

func (client *SSHClient) newSession() (*ssh.Session, error) {
	connection, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", client.Host, client.Port), client.Config)
	if err != nil {
		return nil, fmt.Errorf("Failed to dial: %s", err)
	}

	session, err := connection.NewSession()
	if err != nil {
		return nil, fmt.Errorf("Failed to create session: %s", err)
	}

	modes := ssh.TerminalModes{
		// ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}

	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		session.Close()
		return nil, fmt.Errorf("request for pseudo terminal failed: %s", err)
	}

	return session, nil
}

func decodeKeyForAuthMethod(file string) (ssh.AuthMethod, error) {
	fmt.Printf("Private key is at: %s\n", file)
	buffer, err := ioutil.ReadFile(file)
	if err != nil {
		fmt.Println("Error while reading the file: ", err)
		return nil, errors.New("Error while reading the .pem file from disk!")
	}
	decryptedBuffer, err := decryptIfEncrypted(buffer)
	if err != nil {
		return nil, err
	}
	key, err := ssh.ParsePrivateKey(decryptedBuffer)
	if err != nil {
		fmt.Println("Error while parsing private key: ", err)
		return nil, err
	}
	fmt.Println("Private key succesfully decripted and decoded.")

	return ssh.PublicKeys(key), nil
}

func decryptIfEncrypted(buffer []byte) ([]byte, error) {
	//  Decode the key extracting the pem.Block structure
	block, _ := pem.Decode(buffer)
	if block == nil {
		return nil, errors.New("Failed to decode certificate buffer")
	}
	//  Verify if the pem.block is Encrypted
	if x509.IsEncryptedPEMBlock(block) {
		fmt.Println("Key is encrypted, specify decrypt passphrase: ")
		bytePassword, err := terminal.ReadPassword(int(syscall.Stdin))
		if err == nil {
			fmt.Println("\nPassword typed: " + string(bytePassword))
		}
		passphrase := string(bytePassword)
		decryptedPem, err := x509.DecryptPEMBlock(block, []byte(strings.TrimSpace(passphrase)))
		if err != nil {
			fmt.Println("Error while reading the file: ", err)
			return nil, err
		}
		//  Recreating the decoded block to be returned
		var newBlock pem.Block
		newBlock.Type = block.Type
		newBlock.Headers = block.Headers
		newBlock.Bytes = decryptedPem
		//  Encoding block into []byte and returning
		return pem.EncodeToMemory(&newBlock), nil
	}
	fmt.Println("Key is not encrypted.")

	return buffer, nil
}

//func sshAgent() ssh.AuthMethod {
//	if sshAgent, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK")); err == nil {
//		return ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers)
//	}
//	return nil
//}

//ConfigureCredentials returns the ClientConfig struct to be used as part of the
//SSHClient definition
func ConfigureCredentials(username string, keypath string) (*ssh.ClientConfig, error) {
	var config ssh.ClientConfig
	config.User = username
	pemKeyCommand := keypath
	authMethod, err := decodeKeyForAuthMethod(pemKeyCommand)
	if err != nil {
		return nil, err
	}
	config.Auth = []ssh.AuthMethod{authMethod}
	config.HostKeyCallback = ssh.InsecureIgnoreHostKey()
	return &config, nil
}

//CreateClient takes a *ssh.ClientConfig struct as input, ipAddress of the target
//machine and the ssh port, and returns an *SSHClient where a command can be Run on
func CreateClient(sshConfig *ssh.ClientConfig, ipAddr string, port int) *SSHClient {
	return &SSHClient{
		Config: sshConfig,
		Host:   ipAddr,
		Port:   port,
	}
}

//Run opens an SSH session and Runs the command passed as an argument
func (client *SSHClient) Run(command string) {

	if err := client.InitStandardSession(); err != nil {
		fmt.Printf("Error while initialising SSH session! Error was: %s\n", err)
		return
	}
	defer client.CloseSession()

	if err := client.Session.Run(command); err != nil {
		fmt.Fprintf(os.Stderr, "command run error: %s\n", err)
		if client.Session == nil {
			fmt.Println("Session not initialised.")
		}
		os.Exit(1)
	}
}

//RunBash runs /bin/bash on the client
func (client *SSHClient) RunBash() {
	client.Run("/bin/bash")
}
<|endoftext|>


<commit_before>package espapi

import "encoding/json"

type SubmissionBatchType string

type SubmissionBatch struct {
	SubmissionName        string `json:"submission_name,omitempty"`
	SubmissionType        string `json:"submission_type,omitempty"`
	Note                  string `json:"note,omitempty"`
	SaveExtractedMetadata bool   `json:"save_extracted_metadata,omitempty"`
	AssignmentId          string `json:"assignment_id,omitempty"`
	BriefId               string `json:"brief_id,omitempty"`
	EventId               string `json:"event_id,omitempty"`
}

func (s SubmissionBatch) Marshal() ([]byte, error) {
	return json.MarshalIndent(s, "", "  ")
}

func (b *SubmissionBatch) TypeIsValid() bool {
	return batchTypeIsValid[b.SubmissionType]
}

func (b SubmissionBatch) NameIsValid() bool {
	return len(b.SubmissionName) > 0
}

type SubmissionBatchUpdate struct {
	SubmissionBatch SubmissionBatchChanges `json:"submission_batch"`
}

func (s SubmissionBatchUpdate) Marshal() ([]byte, error) {
	return json.MarshalIndent(s, "", "  ")
}

type SubmissionBatchChanges struct {
	SubmissionName string `json:"submission_name,omitempty"`
	Note           string `json:"note,omitempty"`
}

func BatchTypes() []string {
	keys := make([]string, len(batchTypeIsValid))
	i := 0
	for k := range batchTypeIsValid {
		keys[i] = k
		i++
	}
	return keys
}

type ExtantSubmissionBatch struct {
	Id                               int
	submissionName                   string
	submissionType                   string
	userId                           string
	profileId                        int
	createdAt                        string
	updatedAt                        string
	status                           string
	note                             string
	createdBy                        string
	lastContributionSubmittedAt      string
	contributionsCount               int
	submittedContributionsCount      int
	istockExclusive                  bool
	contributionsAwaitingReviewCount int
	reviewedContributionsCount       int
	revisableContributionsCount      int
	lastSubmittedAt                  string
	isGetty                          bool
	isIstock                         bool
	totalProfileNotes                int
	saveExtractedMetadata            bool
	assignmentId                     string
	briefId                          string
	eventId                          string
}

var batchTypeIsValid = map[string]bool{
	"getty_creative_video":  true,
	"getty_editorial_video": true,
	"getty_creative_still":  true,
	"getty_editorial_still": true,
	"istock_creative_video": true,
}
<commit_msg>reformat<commit_after>package espapi

import "encoding/json"

type SubmissionBatchType string

type SubmissionBatch struct {
	SubmissionName        string `json:"submission_name,omitempty"`
	SubmissionType        string `json:"submission_type,omitempty"`
	Note                  string `json:"note,omitempty"`
	SaveExtractedMetadata bool   `json:"save_extracted_metadata,omitempty"`
	AssignmentId          string `json:"assignment_id,omitempty"`
	BriefId               string `json:"brief_id,omitempty"`
	EventId               string `json:"event_id,omitempty"`
}

func (s SubmissionBatch) Marshal() ([]byte, error) { return json.MarshalIndent(s, "", "  ") }
func (b *SubmissionBatch) TypeIsValid() bool       { return batchTypeIsValid[b.SubmissionType] }
func (b SubmissionBatch) NameIsValid() bool        { return len(b.SubmissionName) > 0 }

type SubmissionBatchUpdate struct {
	SubmissionBatch SubmissionBatchChanges `json:"submission_batch"`
}

func (s SubmissionBatchUpdate) Marshal() ([]byte, error) {
	return json.MarshalIndent(s, "", "  ")
}

type SubmissionBatchChanges struct {
	SubmissionName string `json:"submission_name,omitempty"`
	Note           string `json:"note,omitempty"`
}

func BatchTypes() []string {
	keys := make([]string, len(batchTypeIsValid))
	i := 0
	for k := range batchTypeIsValid {
		keys[i] = k
		i++
	}
	return keys
}

type ExtantSubmissionBatch struct {
	Id                               int
	submissionName                   string
	submissionType                   string
	userId                           string
	profileId                        int
	createdAt                        string
	updatedAt                        string
	status                           string
	note                             string
	createdBy                        string
	lastContributionSubmittedAt      string
	contributionsCount               int
	submittedContributionsCount      int
	istockExclusive                  bool
	contributionsAwaitingReviewCount int
	reviewedContributionsCount       int
	revisableContributionsCount      int
	lastSubmittedAt                  string
	isGetty                          bool
	isIstock                         bool
	totalProfileNotes                int
	saveExtractedMetadata            bool
	assignmentId                     string
	briefId                          string
	eventId                          string
}

var batchTypeIsValid = map[string]bool{
	"getty_creative_video":  true,
	"getty_editorial_video": true,
	"getty_creative_still":  true,
	"getty_editorial_still": true,
	"istock_creative_video": true,
}
<|endoftext|>


<commit_before>package bcast

/*
   bcast package for Go. Broadcasting on a set of channels.

   Copyright © 2013 Alexander I.Grafov <grafov@gmail.com>.
   All rights reserved.
   Use of this source code is governed by a BSD-style
   license that can be found in the LICENSE file.
*/

import (
	"container/heap"
	"errors"
	"sync"
	"time"
)

// Message is an internal structure to pack messages together with
// info about sender.
type Message struct {
	sender  *Member
	payload interface{}
	clock   int
}

// Member represents member of a Broadcast group.
type Member struct {
	group        *Group
	Read         chan interface{}
	clock        int
	messageQueue PriorityQueue
	send         chan Message
	close        chan bool
}

// Group provides a mechanism for the broadcast of messages to a
// collection of channels.
type Group struct {
	in         chan Message
	close      chan bool
	members    []*Member
	clock      int
	memberLock sync.Mutex
	clockLock  sync.Mutex
}

// NewGroup creates a new broadcast group.
func NewGroup() *Group {
	in := make(chan Message)
	close := make(chan bool)
	return &Group{in: in, close: close, clock: 0}
}

// MemberCount returns the number of members in the Broadcast Group.
func (g *Group) MemberCount() int {
	return len(g.Members())
}

// Members returns a slice of Members that are currently in the Group.
func (g *Group) Members() []*Member {
	g.memberLock.Lock()
	res := g.members[:]
	g.memberLock.Unlock()
	return res
}

// Join returns a new member object and handles the creation of its
// output channel.
func (g *Group) Join() *Member {
	memberChannel := make(chan interface{})
	return g.Add(memberChannel)
}

// Leave removes the provided member from the group
func (g *Group) Leave(leaving *Member) error {
	g.memberLock.Lock()
	memberIndex := -1
	for index, member := range g.members {
		if member == leaving {
			memberIndex = index
			break
		}
	}
	if memberIndex == -1 {
		return errors.New("Could not find provided memeber for removal")
	}
	g.members = append(g.members[:memberIndex], g.members[memberIndex+1:]...)
	leaving.close <- true // TODO: need to handle the case where there
	// is still stuff in this Members priorityQueue
	g.memberLock.Unlock()
	return nil
}

// Add adds a member to the group for the provided interface channel.
func (g *Group) Add(memberChannel chan interface{}) *Member {
	g.memberLock.Lock()
	g.clockLock.Lock()
	member := &Member{
		group:        g,
		Read:         memberChannel,
		clock:        g.clock,
		messageQueue: PriorityQueue{},
		send:         make(chan Message),
		close:        make(chan bool),
	}
	go member.listen()
	g.members = append(g.members, member)
	g.clockLock.Unlock()
	g.memberLock.Unlock()
	return member
}

// Close terminates the group immediately.
func (g *Group) Close() {
	g.close <- true
}

// Broadcast messages received from one group member to others.
// If incoming messages not arrived during `timeout` then function returns.
func (g *Group) Broadcast(timeout time.Duration) {
	var timeoutChannel <-chan time.Time
	if timeout != 0 {
		timeoutChannel = time.After(timeout)
	}
	for {
		select {
		case received := <-g.in:
			g.memberLock.Lock()
			g.clockLock.Lock()
			members := g.members[:]
			received.clock = g.clock
			g.clock++
			g.clockLock.Unlock()
			g.memberLock.Unlock()
			for _, member := range members {
				// This is done in a goroutine because if it
				// weren't it would be a blocking call
				go func(member *Member, received Message) {
					member.send <- received
				}(member, received)
			}
		case <-timeoutChannel:
			if timeout > 0 {
				return
			}
		case <-g.close:
			return
		}
	}
}

// Send broadcasts a message to every one of a Group's members.
func (g *Group) Send(val interface{}) {
	g.in <- Message{sender: nil, payload: val}
}

// Close removes the member it is called on from its broadcast group.
func (m *Member) Close() {
	m.group.Leave(m)
}

// Send broadcasts a message from one Member to the channels of all
// the other members in its group.
func (m *Member) Send(val interface{}) {
	m.group.in <- Message{sender: m, payload: val}
}

// Recv reads one value from the member's Read channel
func (m *Member) Recv() interface{} {
	return <-m.Read
}

func (m *Member) listen() {
	for {
		select {
		case message := <-m.send:
			m.handleMessage(&message)
		case <-m.close:
			return
		}
	}
}

func (m *Member) handleMessage(message *Message) {
	if !m.trySend(message) {
		heap.Push(&m.messageQueue, &Item{
			priority: message.clock,
			value:    message,
		})
		return
	}
	if m.messageQueue.Len() > 0 {
		nextMessage := m.messageQueue[0].value.(*Message)
		for m.trySend(nextMessage) {
			heap.Pop(&m.messageQueue)
			if m.messageQueue.Len() > 0 {
				nextMessage = m.messageQueue[0].value.(*Message)
			} else {
				break
			}
		}
	}
}

func (m *Member) trySend(message *Message) bool {
	shouldSend := message.clock == m.clock
	if shouldSend {
		if message.sender != m {
			m.Read <- message.payload
		}
		m.clock++
	}
	return shouldSend
}
<commit_msg>Add unlock when leave group with error<commit_after>package bcast

/*
   bcast package for Go. Broadcasting on a set of channels.

   Copyright © 2013 Alexander I.Grafov <grafov@gmail.com>.
   All rights reserved.
   Use of this source code is governed by a BSD-style
   license that can be found in the LICENSE file.
*/

import (
	"container/heap"
	"errors"
	"sync"
	"time"
)

// Message is an internal structure to pack messages together with
// info about sender.
type Message struct {
	sender  *Member
	payload interface{}
	clock   int
}

// Member represents member of a Broadcast group.
type Member struct {
	group        *Group
	Read         chan interface{}
	clock        int
	messageQueue PriorityQueue
	send         chan Message
	close        chan bool
}

// Group provides a mechanism for the broadcast of messages to a
// collection of channels.
type Group struct {
	in         chan Message
	close      chan bool
	members    []*Member
	clock      int
	memberLock sync.Mutex
	clockLock  sync.Mutex
}

// NewGroup creates a new broadcast group.
func NewGroup() *Group {
	in := make(chan Message)
	close := make(chan bool)
	return &Group{in: in, close: close, clock: 0}
}

// MemberCount returns the number of members in the Broadcast Group.
func (g *Group) MemberCount() int {
	return len(g.Members())
}

// Members returns a slice of Members that are currently in the Group.
func (g *Group) Members() []*Member {
	g.memberLock.Lock()
	res := g.members[:]
	g.memberLock.Unlock()
	return res
}

// Join returns a new member object and handles the creation of its
// output channel.
func (g *Group) Join() *Member {
	memberChannel := make(chan interface{})
	return g.Add(memberChannel)
}

// Leave removes the provided member from the group
func (g *Group) Leave(leaving *Member) error {
	g.memberLock.Lock()
	memberIndex := -1
	for index, member := range g.members {
		if member == leaving {
			memberIndex = index
			break
		}
	}
	if memberIndex == -1 {
		g.memberLock.Unlock()
		return errors.New("Could not find provided memeber for removal")
	}
	g.members = append(g.members[:memberIndex], g.members[memberIndex+1:]...)
	leaving.close <- true // TODO: need to handle the case where there
	// is still stuff in this Members priorityQueue
	g.memberLock.Unlock()
	return nil
}

// Add adds a member to the group for the provided interface channel.
func (g *Group) Add(memberChannel chan interface{}) *Member {
	g.memberLock.Lock()
	g.clockLock.Lock()
	member := &Member{
		group:        g,
		Read:         memberChannel,
		clock:        g.clock,
		messageQueue: PriorityQueue{},
		send:         make(chan Message),
		close:        make(chan bool),
	}
	go member.listen()
	g.members = append(g.members, member)
	g.clockLock.Unlock()
	g.memberLock.Unlock()
	return member
}

// Close terminates the group immediately.
func (g *Group) Close() {
	g.close <- true
}

// Broadcast messages received from one group member to others.
// If incoming messages not arrived during `timeout` then function returns.
func (g *Group) Broadcast(timeout time.Duration) {
	var timeoutChannel <-chan time.Time
	if timeout != 0 {
		timeoutChannel = time.After(timeout)
	}
	for {
		select {
		case received := <-g.in:
			g.memberLock.Lock()
			g.clockLock.Lock()
			members := g.members[:]
			received.clock = g.clock
			g.clock++
			g.clockLock.Unlock()
			g.memberLock.Unlock()
			for _, member := range members {
				// This is done in a goroutine because if it
				// weren't it would be a blocking call
				go func(member *Member, received Message) {
					member.send <- received
				}(member, received)
			}
		case <-timeoutChannel:
			if timeout > 0 {
				return
			}
		case <-g.close:
			return
		}
	}
}

// Send broadcasts a message to every one of a Group's members.
func (g *Group) Send(val interface{}) {
	g.in <- Message{sender: nil, payload: val}
}

// Close removes the member it is called on from its broadcast group.
func (m *Member) Close() {
	m.group.Leave(m)
}

// Send broadcasts a message from one Member to the channels of all
// the other members in its group.
func (m *Member) Send(val interface{}) {
	m.group.in <- Message{sender: m, payload: val}
}

// Recv reads one value from the member's Read channel
func (m *Member) Recv() interface{} {
	return <-m.Read
}

func (m *Member) listen() {
	for {
		select {
		case message := <-m.send:
			m.handleMessage(&message)
		case <-m.close:
			return
		}
	}
}

func (m *Member) handleMessage(message *Message) {
	if !m.trySend(message) {
		heap.Push(&m.messageQueue, &Item{
			priority: message.clock,
			value:    message,
		})
		return
	}
	if m.messageQueue.Len() > 0 {
		nextMessage := m.messageQueue[0].value.(*Message)
		for m.trySend(nextMessage) {
			heap.Pop(&m.messageQueue)
			if m.messageQueue.Len() > 0 {
				nextMessage = m.messageQueue[0].value.(*Message)
			} else {
				break
			}
		}
	}
}

func (m *Member) trySend(message *Message) bool {
	shouldSend := message.clock == m.clock
	if shouldSend {
		if message.sender != m {
			m.Read <- message.payload
		}
		m.clock++
	}
	return shouldSend
}
<|endoftext|>


<commit_before>package main

import (
    "net/http"
    "runtime"
    "sync"
    "fmt"
    "time"
    "io/ioutil"
    "flag"
    "os"
)

var wg sync.WaitGroup

func get(nb int, url string){
  tr := &http.Transport{
    DisableKeepAlives:  true,
    DisableCompression: true,
  }

  client := &http.Client{Transport: tr}

  for c := 0; c < nb; c++ {
	  response, err := client.Get(url)
    if err != nil {
      fmt.Printf("%s", err)
    } else {
      defer response.Body.Close()
      ioutil.ReadAll(response.Body)
    }
  }
  wg.Done()
}

func main() {
    concurrencyPtr := flag.Int("c", 4, "Number of multiple requests")
    processorNbPtr := flag.Int("proc", 1, "Number of processor to use. 1 should be sufficient")
    numberRequestPtr := flag.Int("n", 8000, "Number of requests to perform")

    flag.Parse()

    args := flag.Args()

    if len(args) != 1 {
	    fmt.Println("We must specify an url and that's it")
	    os.Exit(1)
    }

    url := args[0] // url to which we want to perform the benchmark

    runtime.GOMAXPROCS(*processorNbPtr)
    numberRequestGoRun := *numberRequestPtr / *concurrencyPtr

    fmt.Println("Number of multiple requests:", *concurrencyPtr)
    fmt.Println("Number of request by go runtime", numberRequestGoRun)
    fmt.Println("Number of processor used:", *processorNbPtr)
    fmt.Println("Number of total requests:", *numberRequestPtr)
    fmt.Println("Url to benchmark:", url)

    timestart := time.Now()

    wg.Add(*concurrencyPtr)

    fmt.Println("Starting");

    for c := 0; c < *concurrencyPtr; c++ {
	    go get(numberRequestGoRun, url)
    }

    fmt.Println("Performing");
    wg.Wait()
    fmt.Println("Finished")

    timeend := time.Now()
    fmt.Println("Total time:", timeend.Sub(timestart))
    fmt.Println("Requests by second", float64(*numberRequestPtr) / timeend.Sub(timestart).Seconds())
}
<commit_msg>remove again the read of the body if we get error 'Cannot assign requested address' read http://gwan.com/en_apachebench_httperf.html<commit_after>package main

import (
    "net/http"
    "runtime"
    "sync"
    "fmt"
    "time"
    "flag"
    "os"
)

var wg sync.WaitGroup

func get(nb int, url string){
  tr := &http.Transport{
    DisableKeepAlives:  true,
    DisableCompression: true,
  }

  client := &http.Client{Transport: tr}

  for c := 0; c < nb; c++ {
	  response, err := client.Get(url)
    if err != nil {
      fmt.Printf("%s", err)
    } else {
      response.Body.Close()
    }
  }
  wg.Done()
}

func main() {
    concurrencyPtr := flag.Int("c", 4, "Number of multiple requests")
    processorNbPtr := flag.Int("proc", 1, "Number of processor to use. 1 should be sufficient")
    numberRequestPtr := flag.Int("n", 8000, "Number of requests to perform")

    flag.Parse()

    args := flag.Args()

    if len(args) != 1 {
	    fmt.Println("We must specify an url and that's it")
	    os.Exit(1)
    }

    url := args[0] // url to which we want to perform the benchmark

    runtime.GOMAXPROCS(*processorNbPtr)
    numberRequestGoRun := *numberRequestPtr / *concurrencyPtr

    fmt.Println("Number of multiple requests:", *concurrencyPtr)
    fmt.Println("Number of request by go runtime", numberRequestGoRun)
    fmt.Println("Number of processor used:", *processorNbPtr)
    fmt.Println("Number of total requests:", *numberRequestPtr)
    fmt.Println("Url to benchmark:", url)

    timestart := time.Now()

    wg.Add(*concurrencyPtr)

    fmt.Println("Starting");

    for c := 0; c < *concurrencyPtr; c++ {
	    go get(numberRequestGoRun, url)
    }

    fmt.Println("Performing");
    wg.Wait()
    fmt.Println("Finished")

    timeend := time.Now()
    fmt.Println("Total time:", timeend.Sub(timestart))
    fmt.Println("Requests by second", float64(*numberRequestPtr) / timeend.Sub(timestart).Seconds())
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fetch provides a way to fetch modules from a proxy.
package fetch

import (
	"archive/zip"
	"bytes"
	"context"
	"errors"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path"
	"runtime"
	"strings"

	"go.opencensus.io/trace"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/config"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/godoc"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/source"
	"golang.org/x/pkgsite/internal/stdlib"
)

// BadPackageError represents an error loading a package
// because its contents do not make up a valid package.
//
// This can happen, for example, if the .go files fail
// to parse or declare different package names.
type BadPackageError struct {
	Err error // Not nil.
}

func (bpe *BadPackageError) Error() string { return bpe.Err.Error() }

// Go environments used to construct build contexts in loadPackage.
var goEnvs = []struct{ GOOS, GOARCH string }{
	{"linux", "amd64"},
	{"windows", "amd64"},
	{"darwin", "amd64"},
	{"js", "wasm"},
	{"linux", "js"},
}

// loadPackage loads a Go package by calling loadPackageWithBuildContext, trying
// several build contexts in turn. The first build context in the list to produce
// a non-empty package is used. If none of them result in a package, then
// loadPackage returns nil, nil.
//
// If the package is fine except that its documentation is too large, loadPackage
// returns both a package and a non-nil error with godoc.ErrTooLarge in its chain.
func loadPackage(ctx context.Context, zipGoFiles []*zip.File, innerPath string, sourceInfo *source.Info, modInfo *godoc.ModuleInfo) (_ *goPackage, err error) {
	defer derrors.Wrap(&err, "loadPackage(ctx, zipGoFiles, %q, sourceInfo, modInfo)", innerPath)
	ctx, span := trace.StartSpan(ctx, "fetch.loadPackage")
	defer span.End()
	for _, env := range goEnvs {
		pkg, err := loadPackageWithBuildContext(ctx, env.GOOS, env.GOARCH, zipGoFiles, innerPath, sourceInfo, modInfo)
		if err != nil && !errors.Is(err, godoc.ErrTooLarge) && !errors.Is(err, derrors.NotFound) {
			return nil, err
		}
		if pkg != nil {
			return pkg, err
		}
	}
	return nil, nil
}

// httpPost allows package fetch tests to stub out playground URL fetches.
var httpPost = http.Post

// loadPackageWithBuildContext loads a Go package made of .go files in zipGoFiles
// using a build context constructed from the given GOOS and GOARCH values.
// modulePath is stdlib.ModulePath for the Go standard library and the module
// path for all other modules. innerPath is the path of the Go package directory
// relative to the module root.
//
// zipGoFiles must contain only .go files that have been verified
// to be of reasonable size.
//
// The returned Package.Licenses field is not populated.
//
// It returns a nil Package if the directory doesn't contain a Go package
// or all .go files have been excluded by constraints.
// A *BadPackageError error is returned if the directory
// contains .go files but do not make up a valid package.
func loadPackageWithBuildContext(ctx context.Context, goos, goarch string, zipGoFiles []*zip.File, innerPath string, sourceInfo *source.Info, modInfo *godoc.ModuleInfo) (_ *goPackage, err error) {
	modulePath := modInfo.ModulePath
	defer derrors.Wrap(&err, "loadPackageWithBuildContext(%q, %q, zipGoFiles, %q, %q, %+v)",
		goos, goarch, innerPath, modulePath, sourceInfo)

	packageName, goFiles, fset, err := loadFilesWithBuildContext(innerPath, goos, goarch, zipGoFiles)
	if err != nil {
		return nil, err
	}
	docPkg := godoc.NewPackage(fset, goos, goarch, modInfo.ModulePackages)
	for _, pf := range goFiles {
		var removeNodes bool
		removeNodes = true
		// Don't strip the seemingly unexported functions from the builtin package;
		// they are actually Go builtins like make, new, etc.
		if modulePath == stdlib.ModulePath && innerPath == "builtin" {
			removeNodes = false
		}
		docPkg.AddFile(pf, removeNodes)
	}

	// Encode before rendering: both operations mess with the AST, but Encode restores
	// it enough to make Render work.
	src, err := docPkg.Encode(ctx)
	if err != nil {
		return nil, err
	}

	synopsis, imports, _, err := docPkg.Render(ctx, innerPath, sourceInfo, modInfo, goos, goarch)
	if err != nil && !errors.Is(err, godoc.ErrTooLarge) {
		return nil, err
	}
	importPath := path.Join(modulePath, innerPath)
	if modulePath == stdlib.ModulePath {
		importPath = innerPath
	}
	v1path := internal.V1Path(importPath, modulePath)
	return &goPackage{
		path:     importPath,
		name:     packageName,
		synopsis: synopsis,
		v1path:   v1path,
		imports:  imports,
		goos:     goos,
		goarch:   goarch,
		source:   src,
	}, err
}

// loadFilesWithBuildContext loads all the Go files at innerPath that match goos
// and goarch in the zip. It returns the package name as it occurs in the
// source, a map of the ASTs of all the Go files, and the token.FileSet used for
// parsing.
func loadFilesWithBuildContext(innerPath, goos, goarch string, zipGoFiles []*zip.File) (pkgName string, fileMap map[string]*ast.File, _ *token.FileSet, _ error) {
	// Apply build constraints to get a map from matching file names to their contents.
	files, err := matchingFiles(goos, goarch, zipGoFiles)
	if err != nil {
		return "", nil, nil, err
	}
	// Parse .go files and add them to the goFiles slice.
	var (
		fset            = token.NewFileSet()
		goFiles         = make(map[string]*ast.File)
		numNonTestFiles int
		packageName     string
		packageNameFile string // Name of file where packageName came from.
	)
	for name, b := range files {
		pf, err := parser.ParseFile(fset, name, b, parser.ParseComments)
		if err != nil {
			if pf == nil {
				return "", nil, nil, fmt.Errorf("internal error: the source couldn't be read: %v", err)
			}
			return "", nil, nil, &BadPackageError{Err: err}
		}
		// Remember all files, including test files for their examples.
		goFiles[name] = pf
		if strings.HasSuffix(name, "_test.go") {
			continue
		}
		// Keep track of the number of non-test files to check that the package name is the same.
		// and also because a directory with only test files doesn't count as a
		// Go package.
		numNonTestFiles++
		if numNonTestFiles == 1 {
			packageName = pf.Name.Name
			packageNameFile = name
		} else if pf.Name.Name != packageName {
			return "", nil, nil, &BadPackageError{Err: &build.MultiplePackageError{
				Dir:      innerPath,
				Packages: []string{packageName, pf.Name.Name},
				Files:    []string{packageNameFile, name},
			}}
		}
	}
	if numNonTestFiles == 0 {
		// This directory doesn't contain a package, or at least not one
		// that matches this build context.
		return "", nil, nil, derrors.NotFound
	}
	return packageName, goFiles, fset, nil
}

// matchingFiles returns a map from file names to their contents, read from zipGoFiles.
// It includes only those files that match the build context determined by goos and goarch.
func matchingFiles(goos, goarch string, zipGoFiles []*zip.File) (files map[string][]byte, err error) {
	defer derrors.Wrap(&err, "matchingFiles(%q, %q, zipGoFiles)", goos, goarch)
	// Populate the map with all the zip files.
	files = make(map[string][]byte)
	for _, f := range zipGoFiles {
		_, name := path.Split(f.Name)
		b, err := readZipFile(f, MaxFileSize)
		if err != nil {
			return nil, err
		}
		files[name] = b
	}

	// bctx is used to make decisions about which of the .go files are included
	// by build constraints.
	bctx := &build.Context{
		GOOS:        goos,
		GOARCH:      goarch,
		CgoEnabled:  true,
		Compiler:    build.Default.Compiler,
		ReleaseTags: build.Default.ReleaseTags,

		JoinPath: path.Join,
		OpenFile: func(name string) (io.ReadCloser, error) {
			return ioutil.NopCloser(bytes.NewReader(files[name])), nil
		},

		// If left nil, the default implementations of these read from disk,
		// which we do not want. None of these functions should be used
		// inside this function; it would be an internal error if they are.
		// Set them to non-nil values to catch if that happens.
		SplitPathList: func(string) []string { panic("internal error: unexpected call to SplitPathList") },
		IsAbsPath:     func(string) bool { panic("internal error: unexpected call to IsAbsPath") },
		IsDir:         func(string) bool { panic("internal error: unexpected call to IsDir") },
		HasSubdir:     func(string, string) (string, bool) { panic("internal error: unexpected call to HasSubdir") },
		ReadDir:       func(string) ([]os.FileInfo, error) { panic("internal error: unexpected call to ReadDir") },
	}

	for name := range files {
		match, err := bctx.MatchFile(".", name) // This will access the file we just added to files map above.
		if err != nil {
			return nil, &BadPackageError{Err: fmt.Errorf(`bctx.MatchFile(".", %q): %w`, name, err)}
		}
		if !match {
			// Excluded by build context.
			delete(files, name)
		}
	}
	return files, nil
}

// readZipFile decompresses zip file f and returns its uncompressed contents.
// The caller can check f.UncompressedSize64 before calling readZipFile to
// get the expected uncompressed size of f.
//
// limit is the maximum number of bytes to read.
func readZipFile(f *zip.File, limit int64) (_ []byte, err error) {
	defer derrors.Add(&err, "readZipFile(%q)", f.Name)

	r, err := f.Open()
	if err != nil {
		return nil, fmt.Errorf("f.Open(): %v", err)
	}
	b, err := ioutil.ReadAll(io.LimitReader(r, limit))
	if err != nil {
		r.Close()
		return nil, fmt.Errorf("ioutil.ReadAll(r): %v", err)
	}
	if err := r.Close(); err != nil {
		return nil, fmt.Errorf("closing: %v", err)
	}
	return b, nil
}

func allocMeg() int {
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)
	return int(ms.Alloc / (1024 * 1024))
}

// mib is the number of bytes in a mebibyte (Mi).
const mib = 1024 * 1024

// The largest module zip size we can comfortably process.
// We probably will OOM if we process a module whose zip is larger.
var maxModuleZipSize int64 = math.MaxInt64

func init() {
	v := config.GetEnvInt("GO_DISCOVERY_MAX_MODULE_ZIP_MI", -1)
	if v > 0 {
		maxModuleZipSize = int64(v) * mib
	}
}

var zipLoadShedder *loadShedder

func init() {
	ctx := context.Background()
	mebis := config.GetEnvInt("GO_DISCOVERY_MAX_IN_FLIGHT_ZIP_MI", -1)
	if mebis > 0 {
		log.Infof(ctx, "shedding load over %dMi", mebis)
		zipLoadShedder = &loadShedder{maxSizeInFlight: uint64(mebis) * mib}
	}
}

// ZipLoadShedStats returns a snapshot of the current LoadShedStats for zip files.
func ZipLoadShedStats() LoadShedStats {
	if zipLoadShedder != nil {
		return zipLoadShedder.stats()
	}
	return LoadShedStats{}
}
<commit_msg>internal/fetch: delete invalid platform in loadPackage<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package fetch provides a way to fetch modules from a proxy.
package fetch

import (
	"archive/zip"
	"bytes"
	"context"
	"errors"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path"
	"runtime"
	"strings"

	"go.opencensus.io/trace"
	"golang.org/x/pkgsite/internal"
	"golang.org/x/pkgsite/internal/config"
	"golang.org/x/pkgsite/internal/derrors"
	"golang.org/x/pkgsite/internal/godoc"
	"golang.org/x/pkgsite/internal/log"
	"golang.org/x/pkgsite/internal/source"
	"golang.org/x/pkgsite/internal/stdlib"
)

// BadPackageError represents an error loading a package
// because its contents do not make up a valid package.
//
// This can happen, for example, if the .go files fail
// to parse or declare different package names.
type BadPackageError struct {
	Err error // Not nil.
}

func (bpe *BadPackageError) Error() string { return bpe.Err.Error() }

// Go environments used to construct build contexts in loadPackage.
var goEnvs = []struct{ GOOS, GOARCH string }{
	{"linux", "amd64"},
	{"windows", "amd64"},
	{"darwin", "amd64"},
	{"js", "wasm"},
}

// loadPackage loads a Go package by calling loadPackageWithBuildContext, trying
// several build contexts in turn. The first build context in the list to produce
// a non-empty package is used. If none of them result in a package, then
// loadPackage returns nil, nil.
//
// If the package is fine except that its documentation is too large, loadPackage
// returns both a package and a non-nil error with godoc.ErrTooLarge in its chain.
func loadPackage(ctx context.Context, zipGoFiles []*zip.File, innerPath string, sourceInfo *source.Info, modInfo *godoc.ModuleInfo) (_ *goPackage, err error) {
	defer derrors.Wrap(&err, "loadPackage(ctx, zipGoFiles, %q, sourceInfo, modInfo)", innerPath)
	ctx, span := trace.StartSpan(ctx, "fetch.loadPackage")
	defer span.End()
	for _, env := range goEnvs {
		pkg, err := loadPackageWithBuildContext(ctx, env.GOOS, env.GOARCH, zipGoFiles, innerPath, sourceInfo, modInfo)
		if err != nil && !errors.Is(err, godoc.ErrTooLarge) && !errors.Is(err, derrors.NotFound) {
			return nil, err
		}
		if pkg != nil {
			return pkg, err
		}
	}
	return nil, nil
}

// httpPost allows package fetch tests to stub out playground URL fetches.
var httpPost = http.Post

// loadPackageWithBuildContext loads a Go package made of .go files in zipGoFiles
// using a build context constructed from the given GOOS and GOARCH values.
// modulePath is stdlib.ModulePath for the Go standard library and the module
// path for all other modules. innerPath is the path of the Go package directory
// relative to the module root.
//
// zipGoFiles must contain only .go files that have been verified
// to be of reasonable size.
//
// The returned Package.Licenses field is not populated.
//
// It returns a nil Package if the directory doesn't contain a Go package
// or all .go files have been excluded by constraints.
// A *BadPackageError error is returned if the directory
// contains .go files but do not make up a valid package.
func loadPackageWithBuildContext(ctx context.Context, goos, goarch string, zipGoFiles []*zip.File, innerPath string, sourceInfo *source.Info, modInfo *godoc.ModuleInfo) (_ *goPackage, err error) {
	modulePath := modInfo.ModulePath
	defer derrors.Wrap(&err, "loadPackageWithBuildContext(%q, %q, zipGoFiles, %q, %q, %+v)",
		goos, goarch, innerPath, modulePath, sourceInfo)

	packageName, goFiles, fset, err := loadFilesWithBuildContext(innerPath, goos, goarch, zipGoFiles)
	if err != nil {
		return nil, err
	}
	docPkg := godoc.NewPackage(fset, goos, goarch, modInfo.ModulePackages)
	for _, pf := range goFiles {
		var removeNodes bool
		removeNodes = true
		// Don't strip the seemingly unexported functions from the builtin package;
		// they are actually Go builtins like make, new, etc.
		if modulePath == stdlib.ModulePath && innerPath == "builtin" {
			removeNodes = false
		}
		docPkg.AddFile(pf, removeNodes)
	}

	// Encode before rendering: both operations mess with the AST, but Encode restores
	// it enough to make Render work.
	src, err := docPkg.Encode(ctx)
	if err != nil {
		return nil, err
	}

	synopsis, imports, _, err := docPkg.Render(ctx, innerPath, sourceInfo, modInfo, goos, goarch)
	if err != nil && !errors.Is(err, godoc.ErrTooLarge) {
		return nil, err
	}
	importPath := path.Join(modulePath, innerPath)
	if modulePath == stdlib.ModulePath {
		importPath = innerPath
	}
	v1path := internal.V1Path(importPath, modulePath)
	return &goPackage{
		path:     importPath,
		name:     packageName,
		synopsis: synopsis,
		v1path:   v1path,
		imports:  imports,
		goos:     goos,
		goarch:   goarch,
		source:   src,
	}, err
}

// loadFilesWithBuildContext loads all the Go files at innerPath that match goos
// and goarch in the zip. It returns the package name as it occurs in the
// source, a map of the ASTs of all the Go files, and the token.FileSet used for
// parsing.
func loadFilesWithBuildContext(innerPath, goos, goarch string, zipGoFiles []*zip.File) (pkgName string, fileMap map[string]*ast.File, _ *token.FileSet, _ error) {
	// Apply build constraints to get a map from matching file names to their contents.
	files, err := matchingFiles(goos, goarch, zipGoFiles)
	if err != nil {
		return "", nil, nil, err
	}
	// Parse .go files and add them to the goFiles slice.
	var (
		fset            = token.NewFileSet()
		goFiles         = make(map[string]*ast.File)
		numNonTestFiles int
		packageName     string
		packageNameFile string // Name of file where packageName came from.
	)
	for name, b := range files {
		pf, err := parser.ParseFile(fset, name, b, parser.ParseComments)
		if err != nil {
			if pf == nil {
				return "", nil, nil, fmt.Errorf("internal error: the source couldn't be read: %v", err)
			}
			return "", nil, nil, &BadPackageError{Err: err}
		}
		// Remember all files, including test files for their examples.
		goFiles[name] = pf
		if strings.HasSuffix(name, "_test.go") {
			continue
		}
		// Keep track of the number of non-test files to check that the package name is the same.
		// and also because a directory with only test files doesn't count as a
		// Go package.
		numNonTestFiles++
		if numNonTestFiles == 1 {
			packageName = pf.Name.Name
			packageNameFile = name
		} else if pf.Name.Name != packageName {
			return "", nil, nil, &BadPackageError{Err: &build.MultiplePackageError{
				Dir:      innerPath,
				Packages: []string{packageName, pf.Name.Name},
				Files:    []string{packageNameFile, name},
			}}
		}
	}
	if numNonTestFiles == 0 {
		// This directory doesn't contain a package, or at least not one
		// that matches this build context.
		return "", nil, nil, derrors.NotFound
	}
	return packageName, goFiles, fset, nil
}

// matchingFiles returns a map from file names to their contents, read from zipGoFiles.
// It includes only those files that match the build context determined by goos and goarch.
func matchingFiles(goos, goarch string, zipGoFiles []*zip.File) (files map[string][]byte, err error) {
	defer derrors.Wrap(&err, "matchingFiles(%q, %q, zipGoFiles)", goos, goarch)
	// Populate the map with all the zip files.
	files = make(map[string][]byte)
	for _, f := range zipGoFiles {
		_, name := path.Split(f.Name)
		b, err := readZipFile(f, MaxFileSize)
		if err != nil {
			return nil, err
		}
		files[name] = b
	}

	// bctx is used to make decisions about which of the .go files are included
	// by build constraints.
	bctx := &build.Context{
		GOOS:        goos,
		GOARCH:      goarch,
		CgoEnabled:  true,
		Compiler:    build.Default.Compiler,
		ReleaseTags: build.Default.ReleaseTags,

		JoinPath: path.Join,
		OpenFile: func(name string) (io.ReadCloser, error) {
			return ioutil.NopCloser(bytes.NewReader(files[name])), nil
		},

		// If left nil, the default implementations of these read from disk,
		// which we do not want. None of these functions should be used
		// inside this function; it would be an internal error if they are.
		// Set them to non-nil values to catch if that happens.
		SplitPathList: func(string) []string { panic("internal error: unexpected call to SplitPathList") },
		IsAbsPath:     func(string) bool { panic("internal error: unexpected call to IsAbsPath") },
		IsDir:         func(string) bool { panic("internal error: unexpected call to IsDir") },
		HasSubdir:     func(string, string) (string, bool) { panic("internal error: unexpected call to HasSubdir") },
		ReadDir:       func(string) ([]os.FileInfo, error) { panic("internal error: unexpected call to ReadDir") },
	}

	for name := range files {
		match, err := bctx.MatchFile(".", name) // This will access the file we just added to files map above.
		if err != nil {
			return nil, &BadPackageError{Err: fmt.Errorf(`bctx.MatchFile(".", %q): %w`, name, err)}
		}
		if !match {
			// Excluded by build context.
			delete(files, name)
		}
	}
	return files, nil
}

// readZipFile decompresses zip file f and returns its uncompressed contents.
// The caller can check f.UncompressedSize64 before calling readZipFile to
// get the expected uncompressed size of f.
//
// limit is the maximum number of bytes to read.
func readZipFile(f *zip.File, limit int64) (_ []byte, err error) {
	defer derrors.Add(&err, "readZipFile(%q)", f.Name)

	r, err := f.Open()
	if err != nil {
		return nil, fmt.Errorf("f.Open(): %v", err)
	}
	b, err := ioutil.ReadAll(io.LimitReader(r, limit))
	if err != nil {
		r.Close()
		return nil, fmt.Errorf("ioutil.ReadAll(r): %v", err)
	}
	if err := r.Close(); err != nil {
		return nil, fmt.Errorf("closing: %v", err)
	}
	return b, nil
}

func allocMeg() int {
	var ms runtime.MemStats
	runtime.ReadMemStats(&ms)
	return int(ms.Alloc / (1024 * 1024))
}

// mib is the number of bytes in a mebibyte (Mi).
const mib = 1024 * 1024

// The largest module zip size we can comfortably process.
// We probably will OOM if we process a module whose zip is larger.
var maxModuleZipSize int64 = math.MaxInt64

func init() {
	v := config.GetEnvInt("GO_DISCOVERY_MAX_MODULE_ZIP_MI", -1)
	if v > 0 {
		maxModuleZipSize = int64(v) * mib
	}
}

var zipLoadShedder *loadShedder

func init() {
	ctx := context.Background()
	mebis := config.GetEnvInt("GO_DISCOVERY_MAX_IN_FLIGHT_ZIP_MI", -1)
	if mebis > 0 {
		log.Infof(ctx, "shedding load over %dMi", mebis)
		zipLoadShedder = &loadShedder{maxSizeInFlight: uint64(mebis) * mib}
	}
}

// ZipLoadShedStats returns a snapshot of the current LoadShedStats for zip files.
func ZipLoadShedStats() LoadShedStats {
	if zipLoadShedder != nil {
		return zipLoadShedder.stats()
	}
	return LoadShedStats{}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pmezard/apec/jstruct"
	"github.com/pquerna/ffjson/ffjson"
)

type HTTPError struct {
	URL    string
	Code   int
	Status string
}

func (e *HTTPError) Error() string {
	return fmt.Sprintf("got %s fetching %s", e.Status, e.URL)
}

// doHTTP performs a single GET (or POST if input is not nil) and returns
// response data if any. It is the caller responsibility to close returned
// reader.
func doHTTP(url string, input io.Reader) (io.ReadCloser, error) {
	method := "GET"
	if input != nil {
		method = "POST"
	}
	rq, err := http.NewRequest(method, url, input)
	if err != nil {
		return nil, err
	}
	rq.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)")
	if input != nil {
		rq.Header.Set("Content-Type", "application/json")
	}
	rsp, err := http.DefaultClient.Do(rq)
	if err != nil {
		return nil, err
	}
	if rsp.StatusCode != http.StatusOK {
		rsp.Body.Close()
		err := &HTTPError{
			URL:    url,
			Code:   rsp.StatusCode,
			Status: rsp.Status,
		}
		return nil, err
	}
	return rsp.Body, nil
}

// tryHTTP performs a GET or POST with exponential backoff, with specified
// delay and maximum retry count.
func tryHTTP(url string, baseDelay time.Duration, loops int,
	input io.ReadSeeker) (io.ReadCloser, error) {

	delay := baseDelay
	for {
		if input != nil {
			_, err := input.Seek(0, 0)
			if err != nil {
				return nil, err
			}
		}
		output, err := doHTTP(url, input)
		if err == nil {
			return output, nil
		}
		if h, ok := err.(*HTTPError); ok && h.Code == http.StatusNotFound {
			return nil, err
		}
		fmt.Printf("fetching failed with: %s\n", err)
		loops -= 1
		if loops <= 0 {
			return nil, err
		}
		time.Sleep(delay)
		delay *= 2
	}
}

// doJson repeatedly POST input as JSON using tryHTTP. It expects a JSON
// response and decodes it into output.
func doJson(url string, baseDelay time.Duration, loops int, input interface{},
	output interface{}) error {

	var post io.ReadSeeker
	if input != nil {
		body := &bytes.Buffer{}
		err := json.NewEncoder(body).Encode(input)
		if err != nil {
			return err
		}
		post = bytes.NewReader(body.Bytes())
	}
	result, err := tryHTTP(url, baseDelay, loops, post)
	if err != nil {
		return err
	}
	defer result.Close()
	return json.NewDecoder(result).Decode(output)
}

type SearchPaging struct {
	Range      int `json:"range"`
	StartIndex int `json:"startIndex"`
}

type SearchSorts struct {
	Direction string `json:"direction"`
	Type      string `json:"type"`
}

// SearchFilters represents all filters of APEC API.
type SearchFilters struct {
	EnableFilter    bool          `json:"activeFiltre"`
	Functions       []int         `json:"fonctions"`
	Places          []int         `json:"lieux"`
	Keywords        string        `json:"motsCles"`
	Experience      []int         `json:"niveauxExperience"`
	Paging          SearchPaging  `json:"pagination"`
	MinSalary       int           `json:"salaireMinimum"`
	MaxSalary       int           `json:"salaireMaximum"`
	Sectors         []int         `json:"secteursActivite"`
	Sorts           []SearchSorts `json:"sorts"`
	ClientType      string        `json:"typeClient"`
	ContractTypes   []int         `json:"typesContrat"`
	ConventionTypes []int         `json:"typesConvention"`
}

// searchOffers returns the list of offer identifiers matching supplied conditions:
//  - start and count are used to page results
//  - minSalary: the minimum salary for returned offers
//  - locations: APEC internal location identifiers, can be empty
func searchOffers(start, count, minSalary int, locations []int) ([]string, error) {
	if locations == nil {
		locations = []int{}
	}
	filter := &SearchFilters{
		EnableFilter: true,
		Functions:    []int{},
		Places:       locations,
		Experience:   []int{},
		Paging: SearchPaging{
			Range:      count,
			StartIndex: start,
		},
		MinSalary: minSalary,
		MaxSalary: 1000,
		Sectors:   []int{},
		Sorts: []SearchSorts{
			{
				Direction: "DESCENDING",
				Type:      "DATE",
			},
		},
		ClientType:      "CADRE",
		ContractTypes:   []int{},
		ConventionTypes: []int{},
	}
	results := &struct {
		Results []struct {
			URI string `json:"@uriOffre"`
		} `json:"resultats"`
	}{}
	url := "https://cadres.apec.fr/cms/webservices/rechercheOffre/ids"
	err := doJson(url, 5*time.Second, 5, filter, results)
	if err != nil {
		return nil, err
	}
	ids := []string{}
	for _, uri := range results.Results {
		parts := strings.Split(uri.URI, "numeroOffre=")
		if len(parts) != 2 {
			return nil, fmt.Errorf("invalid offer identifier: %s", uri.URI)
		}
		ids = append(ids, parts[1])
	}
	return ids, nil
}

// getOffer returns the byte content of an offer document (theorically in JSON
// format). It may return nil without an error if the offer does not exist,
// which could happen with concurrent site updates.
func getOffer(id string) ([]byte, error) {
	u := "https://cadres.apec.fr/cms/webservices/offre/public?numeroOffre=" + id
	output, err := tryHTTP(u, time.Second, 5, nil)
	if err != nil {
		if h, ok := err.(*HTTPError); ok && h.Code == http.StatusNotFound {
			return nil, nil
		}
		return nil, err
	}
	defer output.Close()
	return ioutil.ReadAll(output)
}

// enumerateOffers search offers satisfying the minSalary and locations
// constraints and repeatedly calls callback with slices of offers identifiers.
// The enumeration is not atomic, there is no guarantee a value is returned
// only once.
func enumerateOffers(minSalary int, locations []int, callback func([]string) error) error {
	start := 0
	overlap := 5
	count := 250
	delay := 5 * time.Second
	for ; ; time.Sleep(delay) {
		fmt.Printf("fetching from %d to %d\n", start, start+count)
		ids, err := searchOffers(start, count, minSalary, locations)
		if err != nil {
			return err
		}
		start += (count - overlap)
		err = callback(ids)
		if err != nil {
			return err
		}
		if len(ids) < count {
			break
		}
	}
	return nil
}

func putOfferDate(store *Store, data []byte, deletedId uint64) error {
	js := &jstruct.JsonOffer{}
	err := ffjson.Unmarshal(data, js)
	if err != nil {
		return err
	}
	dateLayout := "2006-01-02T15:04:05.000+0000"
	date, err := time.Parse(dateLayout, js.Date)
	if err != nil {
		return fmt.Errorf("cannot parse offer date: %s", err)
	}
	age := OfferAge{
		Id:              js.Id,
		DeletedId:       deletedId,
		PublicationDate: date,
	}
	if deletedId != 0 {
		age.DeletionDate = time.Now()
	}
	return store.PutOfferDate(hashOffer(js), age)
}

// crawlOffers fetches specified offers and store their binary representation
// in the store. It returns the number of offers actually stored. Already
// fetched offers, or missing remote offers are ignored.
func crawlOffers(store *Store, ids []string) (int, int, error) {
	added := 0
	ageErrors := 0
	for _, id := range ids {
		ok, err := store.Has(id)
		if err != nil {
			return added, 0, err
		}
		if ok {
			continue
		}
		fmt.Printf("fetching %s\n", id)
		data, err := getOffer(id)
		if err != nil {
			return added, 0, err
		}
		time.Sleep(time.Second)
		if data == nil {
			fmt.Printf("could not find %s\n", id)
			continue
		}
		err = store.Put(id, data)
		if err != nil {
			return added, 0, err
		}
		added += 1
		err = putOfferDate(store, data, 0)
		if err != nil {
			ageErrors += 1
		}
	}
	return added, ageErrors, nil
}

func crawl(store *Store, minSalary int, locations []int) error {
	idsChan := make(chan []string)
	stopListing := make(chan bool)
	listingDone := make(chan error)
	crawlingDone := make(chan error)

	// List offers in one goroutine. Doing it as fast as possible without
	// waiting for offers to be fetched reduces the races between our
	// enumeration and possible web site updates.
	seen := map[string]bool{}
	go func() {
		pending := []string{}
		err := enumerateOffers(minSalary, locations, func(ids []string) error {
			for _, id := range ids {
				if !seen[id] {
					pending = append(pending, id)
				}
				seen[id] = true
			}
			select {
			case <-stopListing:
				return fmt.Errorf("offer enumeration was interrupted")
			case idsChan <- pending:
				pending = nil
			default:
			}
			return nil
		})
		close(idsChan)
		listingDone <- err
	}()

	// Crawl offers in another goroutine
	added := 0
	ageErrors := 0
	go func() {
		for ids := range idsChan {
			n, e, err := crawlOffers(store, ids)
			added += n
			ageErrors += e
			if n < len(ids) {
				fmt.Printf("%d known offers ignored\n", len(ids)-n)
			}
			if err != nil {
				crawlingDone <- err
				break
			}
		}
		close(crawlingDone)
	}()

	// Shutdown everything cleanly
	crawlingErr := <-crawlingDone
	close(stopListing)
	listingErr := <-listingDone
	if listingErr != nil {
		return listingErr
	}
	if crawlingErr != nil {
		return crawlingErr
	}

	// Delete unseen offers
	deleted := 0
	ids, err := store.List()
	if err != nil {
		return err
	}
	now := time.Now()
	for _, id := range ids {
		if seen[id] {
			continue
		}
		fmt.Printf("deleting %s\n", id)
		var offer []byte
		d, err := store.Get(id)
		if err == nil {
			offer = d
		} else {
			ageErrors += 1
		}
		deletedId, err := store.Delete(id, now)
		if err != nil {
			return fmt.Errorf("could not delete %s: %s\n", id, err)
		}
		if offer != nil {
			err = putOfferDate(store, offer, deletedId)
			if err != nil {
				ageErrors += 1
			}
		}
		deleted += 1
	}
	fmt.Printf("%d added, %d deleted, %d total\n", added, deleted, store.Size())
	if ageErrors > 0 {
		return fmt.Errorf("failed to compute %d offer age", ageErrors)
	}
	return nil
}

var (
	crawlCmd       = app.Command("crawl", "crawl APEC offers")
	crawlMinSalary = crawlCmd.Flag("min-salary", "minimum salary in kEUR").Default("0").Int()
	crawlLocations = crawlCmd.Flag("location", "offer location code").Ints()
)

func crawlFn(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	var closeErr error
	defer func() {
		closeErr = store.Close()
	}()
	err = crawl(store, *crawlMinSalary, *crawlLocations)
	if err != nil {
		return err
	}
	return closeErr
}
<commit_msg>crawl: reduce crawl window to 100 entries<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pmezard/apec/jstruct"
	"github.com/pquerna/ffjson/ffjson"
)

type HTTPError struct {
	URL    string
	Code   int
	Status string
}

func (e *HTTPError) Error() string {
	return fmt.Sprintf("got %s fetching %s", e.Status, e.URL)
}

// doHTTP performs a single GET (or POST if input is not nil) and returns
// response data if any. It is the caller responsibility to close returned
// reader.
func doHTTP(url string, input io.Reader) (io.ReadCloser, error) {
	method := "GET"
	if input != nil {
		method = "POST"
	}
	rq, err := http.NewRequest(method, url, input)
	if err != nil {
		return nil, err
	}
	rq.Header.Set("User-Agent", "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)")
	if input != nil {
		rq.Header.Set("Content-Type", "application/json")
	}
	rsp, err := http.DefaultClient.Do(rq)
	if err != nil {
		return nil, err
	}
	if rsp.StatusCode != http.StatusOK {
		rsp.Body.Close()
		err := &HTTPError{
			URL:    url,
			Code:   rsp.StatusCode,
			Status: rsp.Status,
		}
		return nil, err
	}
	return rsp.Body, nil
}

// tryHTTP performs a GET or POST with exponential backoff, with specified
// delay and maximum retry count.
func tryHTTP(url string, baseDelay time.Duration, loops int,
	input io.ReadSeeker) (io.ReadCloser, error) {

	delay := baseDelay
	for {
		if input != nil {
			_, err := input.Seek(0, 0)
			if err != nil {
				return nil, err
			}
		}
		output, err := doHTTP(url, input)
		if err == nil {
			return output, nil
		}
		if h, ok := err.(*HTTPError); ok && h.Code == http.StatusNotFound {
			return nil, err
		}
		fmt.Printf("fetching failed with: %s\n", err)
		loops -= 1
		if loops <= 0 {
			return nil, err
		}
		time.Sleep(delay)
		delay *= 2
	}
}

// doJson repeatedly POST input as JSON using tryHTTP. It expects a JSON
// response and decodes it into output.
func doJson(url string, baseDelay time.Duration, loops int, input interface{},
	output interface{}) error {

	var post io.ReadSeeker
	if input != nil {
		body := &bytes.Buffer{}
		err := json.NewEncoder(body).Encode(input)
		if err != nil {
			return err
		}
		post = bytes.NewReader(body.Bytes())
	}
	result, err := tryHTTP(url, baseDelay, loops, post)
	if err != nil {
		return err
	}
	defer result.Close()
	return json.NewDecoder(result).Decode(output)
}

type SearchPaging struct {
	Range      int `json:"range"`
	StartIndex int `json:"startIndex"`
}

type SearchSorts struct {
	Direction string `json:"direction"`
	Type      string `json:"type"`
}

// SearchFilters represents all filters of APEC API.
type SearchFilters struct {
	EnableFilter    bool          `json:"activeFiltre"`
	Functions       []int         `json:"fonctions"`
	Places          []int         `json:"lieux"`
	Keywords        string        `json:"motsCles"`
	Experience      []int         `json:"niveauxExperience"`
	Paging          SearchPaging  `json:"pagination"`
	MinSalary       int           `json:"salaireMinimum"`
	MaxSalary       int           `json:"salaireMaximum"`
	Sectors         []int         `json:"secteursActivite"`
	Sorts           []SearchSorts `json:"sorts"`
	ClientType      string        `json:"typeClient"`
	ContractTypes   []int         `json:"typesContrat"`
	ConventionTypes []int         `json:"typesConvention"`
}

// searchOffers returns the list of offer identifiers matching supplied conditions:
//  - start and count are used to page results
//  - minSalary: the minimum salary for returned offers
//  - locations: APEC internal location identifiers, can be empty
func searchOffers(start, count, minSalary int, locations []int) ([]string, error) {
	if locations == nil {
		locations = []int{}
	}
	filter := &SearchFilters{
		EnableFilter: true,
		Functions:    []int{},
		Places:       locations,
		Experience:   []int{},
		Paging: SearchPaging{
			Range:      count,
			StartIndex: start,
		},
		MinSalary: minSalary,
		MaxSalary: 1000,
		Sectors:   []int{},
		Sorts: []SearchSorts{
			{
				Direction: "DESCENDING",
				Type:      "DATE",
			},
		},
		ClientType:      "CADRE",
		ContractTypes:   []int{},
		ConventionTypes: []int{},
	}
	results := &struct {
		Results []struct {
			URI string `json:"@uriOffre"`
		} `json:"resultats"`
	}{}
	url := "https://cadres.apec.fr/cms/webservices/rechercheOffre/ids"
	err := doJson(url, 5*time.Second, 5, filter, results)
	if err != nil {
		return nil, err
	}
	ids := []string{}
	for _, uri := range results.Results {
		parts := strings.Split(uri.URI, "numeroOffre=")
		if len(parts) != 2 {
			return nil, fmt.Errorf("invalid offer identifier: %s", uri.URI)
		}
		ids = append(ids, parts[1])
	}
	return ids, nil
}

// getOffer returns the byte content of an offer document (theorically in JSON
// format). It may return nil without an error if the offer does not exist,
// which could happen with concurrent site updates.
func getOffer(id string) ([]byte, error) {
	u := "https://cadres.apec.fr/cms/webservices/offre/public?numeroOffre=" + id
	output, err := tryHTTP(u, time.Second, 5, nil)
	if err != nil {
		if h, ok := err.(*HTTPError); ok && h.Code == http.StatusNotFound {
			return nil, nil
		}
		return nil, err
	}
	defer output.Close()
	return ioutil.ReadAll(output)
}

// enumerateOffers search offers satisfying the minSalary and locations
// constraints and repeatedly calls callback with slices of offers identifiers.
// The enumeration is not atomic, there is no guarantee a value is returned
// only once.
func enumerateOffers(minSalary int, locations []int, callback func([]string) error) error {
	start := 0
	overlap := 5
	count := 100
	delay := 5 * time.Second
	for ; ; time.Sleep(delay) {
		fmt.Printf("fetching from %d to %d\n", start, start+count)
		ids, err := searchOffers(start, count, minSalary, locations)
		if err != nil {
			return err
		}
		start += (count - overlap)
		err = callback(ids)
		if err != nil {
			return err
		}
		if len(ids) < count {
			break
		}
	}
	return nil
}

func putOfferDate(store *Store, data []byte, deletedId uint64) error {
	js := &jstruct.JsonOffer{}
	err := ffjson.Unmarshal(data, js)
	if err != nil {
		return err
	}
	dateLayout := "2006-01-02T15:04:05.000+0000"
	date, err := time.Parse(dateLayout, js.Date)
	if err != nil {
		return fmt.Errorf("cannot parse offer date: %s", err)
	}
	age := OfferAge{
		Id:              js.Id,
		DeletedId:       deletedId,
		PublicationDate: date,
	}
	if deletedId != 0 {
		age.DeletionDate = time.Now()
	}
	return store.PutOfferDate(hashOffer(js), age)
}

// crawlOffers fetches specified offers and store their binary representation
// in the store. It returns the number of offers actually stored. Already
// fetched offers, or missing remote offers are ignored.
func crawlOffers(store *Store, ids []string) (int, int, error) {
	added := 0
	ageErrors := 0
	for _, id := range ids {
		ok, err := store.Has(id)
		if err != nil {
			return added, 0, err
		}
		if ok {
			continue
		}
		fmt.Printf("fetching %s\n", id)
		data, err := getOffer(id)
		if err != nil {
			return added, 0, err
		}
		time.Sleep(time.Second)
		if data == nil {
			fmt.Printf("could not find %s\n", id)
			continue
		}
		err = store.Put(id, data)
		if err != nil {
			return added, 0, err
		}
		added += 1
		err = putOfferDate(store, data, 0)
		if err != nil {
			ageErrors += 1
		}
	}
	return added, ageErrors, nil
}

func crawl(store *Store, minSalary int, locations []int) error {
	idsChan := make(chan []string)
	stopListing := make(chan bool)
	listingDone := make(chan error)
	crawlingDone := make(chan error)

	// List offers in one goroutine. Doing it as fast as possible without
	// waiting for offers to be fetched reduces the races between our
	// enumeration and possible web site updates.
	seen := map[string]bool{}
	go func() {
		pending := []string{}
		err := enumerateOffers(minSalary, locations, func(ids []string) error {
			for _, id := range ids {
				if !seen[id] {
					pending = append(pending, id)
				}
				seen[id] = true
			}
			select {
			case <-stopListing:
				return fmt.Errorf("offer enumeration was interrupted")
			case idsChan <- pending:
				pending = nil
			default:
			}
			return nil
		})
		close(idsChan)
		listingDone <- err
	}()

	// Crawl offers in another goroutine
	added := 0
	ageErrors := 0
	go func() {
		for ids := range idsChan {
			n, e, err := crawlOffers(store, ids)
			added += n
			ageErrors += e
			if n < len(ids) {
				fmt.Printf("%d known offers ignored\n", len(ids)-n)
			}
			if err != nil {
				crawlingDone <- err
				break
			}
		}
		close(crawlingDone)
	}()

	// Shutdown everything cleanly
	crawlingErr := <-crawlingDone
	close(stopListing)
	listingErr := <-listingDone
	if listingErr != nil {
		return listingErr
	}
	if crawlingErr != nil {
		return crawlingErr
	}

	// Delete unseen offers
	deleted := 0
	ids, err := store.List()
	if err != nil {
		return err
	}
	now := time.Now()
	for _, id := range ids {
		if seen[id] {
			continue
		}
		fmt.Printf("deleting %s\n", id)
		var offer []byte
		d, err := store.Get(id)
		if err == nil {
			offer = d
		} else {
			ageErrors += 1
		}
		deletedId, err := store.Delete(id, now)
		if err != nil {
			return fmt.Errorf("could not delete %s: %s\n", id, err)
		}
		if offer != nil {
			err = putOfferDate(store, offer, deletedId)
			if err != nil {
				ageErrors += 1
			}
		}
		deleted += 1
	}
	fmt.Printf("%d added, %d deleted, %d total\n", added, deleted, store.Size())
	if ageErrors > 0 {
		return fmt.Errorf("failed to compute %d offer age", ageErrors)
	}
	return nil
}

var (
	crawlCmd       = app.Command("crawl", "crawl APEC offers")
	crawlMinSalary = crawlCmd.Flag("min-salary", "minimum salary in kEUR").Default("0").Int()
	crawlLocations = crawlCmd.Flag("location", "offer location code").Ints()
)

func crawlFn(cfg *Config) error {
	store, err := OpenStore(cfg.Store())
	if err != nil {
		return err
	}
	var closeErr error
	defer func() {
		closeErr = store.Close()
	}()
	err = crawl(store, *crawlMinSalary, *crawlLocations)
	if err != nil {
		return err
	}
	return closeErr
}
<|endoftext|>


<commit_before>package irc

import (
	"testing"
	"time"
	"strings"
	"fmt"
	"github.com/kyleterry/tenyks/config"
	"github.com/kyleterry/tenyks/mockirc"
	_"code.google.com/p/gomock/gomock"
)

func TestNewConnNoDial(t *testing.T) {
	conf := config.ConnectionConfig{
		Name: "test",
		Ssl: true,
	}
	conn := NewConn(conf.Name, conf)
	if conn.Name != conf.Name {
		t.Error("Expected %s, got %s", conn.Name, conf.Name)
	}

	if !conn.usingSSL {
		t.Error("SSL is supposed to be enabled")
	}

	if conn.IsConnected() {
		t.Error("Connection is not supposed to be connected")
	}

	strMethodResult := fmt.Sprintf("%s", conn)
	if !strings.Contains(strMethodResult, "Disconnected") {
		t.Error("String method seems to be broken, Expected to contain 'Disconnected', got ", strMethodResult)
	}

	select {
	case <-conn.ConnectWait:
		t.Error("Channel is supposed to remain open and not recieve")
	case <-time.After(time.Second):
		break
	}
}

func MakeConnConfig() config.ConnectionConfig {
	return config.ConnectionConfig{
		Name: "mockirc",
		Host: "localhost",
		Port: 26661,
		FloodProtection: true,
		Retries: 5,
		Nicks: []string{"tenyks", "tenyks-"},
		Ident: "something",
		Realname: "tenyks",
		Admins: []string{"kyle"},
		Channels: []string{"#tenyks", "#test"},
	}
}

func TestCanConnectAndDisconnect(t *testing.T) {
	ircServer := mockirc.New("mockirc.tenyks.io", 26661)
	ircServer.Start()
	defer ircServer.Stop()

	conn := NewConn("mockirc", MakeConnConfig())
	wait := conn.Connect()
	<-wait

	if conn.GetRetries() > 0 {
		t.Error("Expected", 0, "got", conn.GetRetries())
	}

	if !conn.IsConnected() {
		t.Error("Expected", true, "got", false)
	}

	conn.Disconnect()

	if conn.IsConnected() {
		t.Error("Expected", false, "got", true)
	}
}

func TestCanHandshakeAndWorkWithIRC(t *testing.T) {
	ircServer := mockirc.New("mockirc.tenyks.io", 26661)
	ircServer.When("USER tenyks localhost something :tenyks").Respond(":101 :Welcome")
	ircServer.When("PING ").Respond(":PONG")
	ircServer.Start()
	defer ircServer.Stop()

	conn := NewConn("mockirc", MakeConnConfig())
	wait := conn.Connect()
	<-wait

	conn.BootstrapHandler(nil)
	<-conn.ConnectWait

	msg := string(<-conn.In)
	if msg != ":101 :Welcome\r\n" {
		t.Error("Expected :101 :Welcome", "got", msg)
	}

	conn.SendPing(nil)
	select {
	case msg := <-conn.In:
		if msg != ":PONG\r\n" {
			t.Error("Expected", ":PONG mockirc", "got", msg)
		}
	case <-time.After(time.Second * 5):
		t.Error("Timed out before getting a response back from mockirc")
	}

	conn.Disconnect()
}
<commit_msg>No need.<commit_after>package irc

import (
	"testing"
	"time"
	"strings"
	"fmt"
	"github.com/kyleterry/tenyks/config"
	"github.com/kyleterry/tenyks/mockirc"
)

func TestNewConnNoDial(t *testing.T) {
	conf := config.ConnectionConfig{
		Name: "test",
		Ssl: true,
	}
	conn := NewConn(conf.Name, conf)
	if conn.Name != conf.Name {
		t.Error("Expected %s, got %s", conn.Name, conf.Name)
	}

	if !conn.usingSSL {
		t.Error("SSL is supposed to be enabled")
	}

	if conn.IsConnected() {
		t.Error("Connection is not supposed to be connected")
	}

	strMethodResult := fmt.Sprintf("%s", conn)
	if !strings.Contains(strMethodResult, "Disconnected") {
		t.Error("String method seems to be broken, Expected to contain 'Disconnected', got ", strMethodResult)
	}

	select {
	case <-conn.ConnectWait:
		t.Error("Channel is supposed to remain open and not recieve")
	case <-time.After(time.Second):
		break
	}
}

func MakeConnConfig() config.ConnectionConfig {
	return config.ConnectionConfig{
		Name: "mockirc",
		Host: "localhost",
		Port: 26661,
		FloodProtection: true,
		Retries: 5,
		Nicks: []string{"tenyks", "tenyks-"},
		Ident: "something",
		Realname: "tenyks",
		Admins: []string{"kyle"},
		Channels: []string{"#tenyks", "#test"},
	}
}

func TestCanConnectAndDisconnect(t *testing.T) {
	ircServer := mockirc.New("mockirc.tenyks.io", 26661)
	ircServer.Start()
	defer ircServer.Stop()

	conn := NewConn("mockirc", MakeConnConfig())
	wait := conn.Connect()
	<-wait

	if conn.GetRetries() > 0 {
		t.Error("Expected", 0, "got", conn.GetRetries())
	}

	if !conn.IsConnected() {
		t.Error("Expected", true, "got", false)
	}

	conn.Disconnect()

	if conn.IsConnected() {
		t.Error("Expected", false, "got", true)
	}
}

func TestCanHandshakeAndWorkWithIRC(t *testing.T) {
	ircServer := mockirc.New("mockirc.tenyks.io", 26661)
	ircServer.When("USER tenyks localhost something :tenyks").Respond(":101 :Welcome")
	ircServer.When("PING ").Respond(":PONG")
	ircServer.Start()
	defer ircServer.Stop()

	conn := NewConn("mockirc", MakeConnConfig())
	wait := conn.Connect()
	<-wait

	conn.BootstrapHandler(nil)
	<-conn.ConnectWait

	msg := string(<-conn.In)
	if msg != ":101 :Welcome\r\n" {
		t.Error("Expected :101 :Welcome", "got", msg)
	}

	conn.SendPing(nil)
	select {
	case msg := <-conn.In:
		if msg != ":PONG\r\n" {
			t.Error("Expected", ":PONG mockirc", "got", msg)
		}
	case <-time.After(time.Second * 5):
		t.Error("Timed out before getting a response back from mockirc")
	}

	conn.Disconnect()
}
<|endoftext|>