<commit_before>// Copyright 2020 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package namespacelabeler

import (
	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"

	"k8s.io/apimachinery/pkg/util/intstr"

	"github.com/pkg/errors"
	"github.com/prometheus-community/prom-label-proxy/injectproxy"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/promql/parser"
)

// Labeler enables to enforce adding namespace labels to PrometheusRules and to metrics used in them
type Labeler struct {
	enforcedNsLabel       string
	prometheusRuleLabeler bool
	excludeList           map[string]map[string]struct{}
}

// New - creates new Labeler
// enforcedNsLabel - label name to be enforced for namespace
// excludeConfig - list of namespace + PrometheusRule names to be excluded while enforcing adding namespace label
func New(enforcedNsLabel string, excludeConfig []monitoringv1.PrometheusRuleExcludeConfig, prometheusRuleLabeler bool) *Labeler {

	if enforcedNsLabel == "" {
		return &Labeler{} // no-op labeler
	}

	if len(excludeConfig) == 0 {
		return &Labeler{enforcedNsLabel: enforcedNsLabel}
	}

	ruleExcludeList := make(map[string]map[string]struct{})

	for _, r := range excludeConfig {
		if r.RuleNamespace == "" || r.RuleName == "" {
			continue
		}
		if _, ok := ruleExcludeList[r.RuleNamespace]; !ok {
			ruleExcludeList[r.RuleNamespace] = make(map[string]struct{})
		}
		ruleExcludeList[r.RuleNamespace][r.RuleName] = struct{}{}
	}

	return &Labeler{
		excludeList:           ruleExcludeList,
		enforcedNsLabel:       enforcedNsLabel,
		prometheusRuleLabeler: prometheusRuleLabeler,
	}
}

func (l *Labeler) isExcludedRule(namespace, name string) bool {
	if l.excludeList == nil {
		return false
	}
	nsRules, ok := l.excludeList[namespace]
	if !ok {
		return false
	}
	_, ok = nsRules[name]
	return ok
}

// EnforceNamespaceLabel - adds(or modifies) namespace label to promRule labels with specified namespace
// and also adds namespace label to all the metrics used in promRule
func (l *Labeler) EnforceNamespaceLabel(rule *monitoringv1.PrometheusRule) error {

	if l.enforcedNsLabel == "" || l.isExcludedRule(rule.Namespace, rule.Name) {
		return nil
	}

	for gi, group := range rule.Spec.Groups {
		if l.prometheusRuleLabeler {
			group.PartialResponseStrategy = ""
		}
		for ri, r := range group.Rules {
			if len(rule.Spec.Groups[gi].Rules[ri].Labels) == 0 {
				rule.Spec.Groups[gi].Rules[ri].Labels = map[string]string{}
			}
			rule.Spec.Groups[gi].Rules[ri].Labels[l.enforcedNsLabel] = rule.Namespace

			expr := r.Expr.String()
			parsedExpr, err := parser.ParseExpr(expr)
			if err != nil {
				return errors.Wrap(err, "failed to parse promql expression")
			}
			err = injectproxy.SetRecursive(parsedExpr, []*labels.Matcher{{
				Name:  l.enforcedNsLabel,
				Type:  labels.MatchEqual,
				Value: rule.Namespace,
			}})
			if err != nil {
				return errors.Wrap(err, "failed to inject labels to expression")
			}

			rule.Spec.Groups[gi].Rules[ri].Expr = intstr.FromString(parsedExpr.String())
		}
	}
	return nil
}
<commit_msg>pkg/namespace-labeler: fix whitespace<commit_after>// Copyright 2020 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package namespacelabeler

import (
	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"

	"k8s.io/apimachinery/pkg/util/intstr"

	"github.com/pkg/errors"
	"github.com/prometheus-community/prom-label-proxy/injectproxy"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/promql/parser"
)

// Labeler enables to enforce adding namespace labels to PrometheusRules and to metrics used in them
type Labeler struct {
	enforcedNsLabel       string
	prometheusRuleLabeler bool
	excludeList           map[string]map[string]struct{}
}

// New - creates new Labeler
// enforcedNsLabel - label name to be enforced for namespace
// excludeConfig - list of namespace + PrometheusRule names to be excluded while enforcing adding namespace label
func New(enforcedNsLabel string, excludeConfig []monitoringv1.PrometheusRuleExcludeConfig, prometheusRuleLabeler bool) *Labeler {

	if enforcedNsLabel == "" {
		return &Labeler{} // no-op labeler
	}

	if len(excludeConfig) == 0 {
		return &Labeler{enforcedNsLabel: enforcedNsLabel}
	}

	ruleExcludeList := make(map[string]map[string]struct{})

	for _, r := range excludeConfig {
		if r.RuleNamespace == "" || r.RuleName == "" {
			continue
		}
		if _, ok := ruleExcludeList[r.RuleNamespace]; !ok {
			ruleExcludeList[r.RuleNamespace] = make(map[string]struct{})
		}
		ruleExcludeList[r.RuleNamespace][r.RuleName] = struct{}{}
	}

	return &Labeler{
		excludeList:           ruleExcludeList,
		enforcedNsLabel:       enforcedNsLabel,
		prometheusRuleLabeler: prometheusRuleLabeler,
	}
}

func (l *Labeler) isExcludedRule(namespace, name string) bool {
	if l.excludeList == nil {
		return false
	}
	nsRules, ok := l.excludeList[namespace]
	if !ok {
		return false
	}
	_, ok = nsRules[name]
	return ok
}

// EnforceNamespaceLabel - adds(or modifies) namespace label to promRule labels with specified namespace
// and also adds namespace label to all the metrics used in promRule
func (l *Labeler) EnforceNamespaceLabel(rule *monitoringv1.PrometheusRule) error {

	if l.enforcedNsLabel == "" || l.isExcludedRule(rule.Namespace, rule.Name) {
		return nil
	}

	for gi, group := range rule.Spec.Groups {
		if l.prometheusRuleLabeler {
			group.PartialResponseStrategy = ""
		}
		for ri, r := range group.Rules {
			if len(rule.Spec.Groups[gi].Rules[ri].Labels) == 0 {
				rule.Spec.Groups[gi].Rules[ri].Labels = map[string]string{}
			}
			rule.Spec.Groups[gi].Rules[ri].Labels[l.enforcedNsLabel] = rule.Namespace

			expr := r.Expr.String()
			parsedExpr, err := parser.ParseExpr(expr)
			if err != nil {
				return errors.Wrap(err, "failed to parse promql expression")
			}
			err = injectproxy.SetRecursive(parsedExpr, []*labels.Matcher{{
				Name:  l.enforcedNsLabel,
				Type:  labels.MatchEqual,
				Value: rule.Namespace,
			}})
			if err != nil {
				return errors.Wrap(err, "failed to inject labels to expression")
			}

			rule.Spec.Groups[gi].Rules[ri].Expr = intstr.FromString(parsedExpr.String())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	"github.com/openshift/origin/pkg/test/ginkgo"
)

type testCategorizer struct {
	by        string
	substring string
}

func testPodSandboxCreation(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	const testName = "[sig-network] pods should successfully create sandboxes"
	// we can further refine this signal by subdividing different failure modes if it is pertinent.  Right now I'm seeing
	// 1. error reading container (probably exited) json message: EOF
	// 2. dial tcp 10.0.76.225:6443: i/o timeout
	// 3. error getting pod: pods "terminate-cmd-rpofb45fa14c-96bb-40f7-bd9e-346721740cac" not found
	// 4. write child: broken pipe
	bySubStrings := []testCategorizer{
		{by: " by reading container", substring: "error reading container (probably exited) json message: EOF"},
		{by: " by not timing out", substring: "i/o timeout"},
		{by: " by writing network status", substring: "error setting the networks status"},
		{by: " by getting pod", substring: " error getting pod: pods"},
		{by: " by writing child", substring: "write child: broken pipe"},
		{by: " by other", substring: " "}, // always matches
	}

	failures := []string{}
	flakes := []string{}
	eventsForPods := getEventsByPod(events)
	for _, event := range events {
		if !strings.Contains(event.Message, "reason/FailedCreatePodSandBox Failed to create pod sandbox") {
			continue
		}
		if strings.Contains(event.Message, "Multus") && strings.Contains(event.Message, "error getting pod") && (strings.Contains(event.Message, "connection refused") || strings.Contains(event.Message, "i/o timeout")) {
			flakes = append(flakes, fmt.Sprintf("%v - multus is unable to get pods due to LB disruption https://bugzilla.redhat.com/show_bug.cgi?id=1927264 - %v", event.Locator, event.Message))
			continue
		}
		deletionTime := getPodDeletionTime(eventsForPods[event.Locator], event.Locator)
		if deletionTime == nil {
			// this indicates a failure to create the sandbox that should not happen
			failures = append(failures, fmt.Sprintf("%v - never deleted - %v", event.Locator, event.Message))
		} else {
			timeBetweenDeleteAndFailure := event.From.Sub(*deletionTime)
			switch {
			case timeBetweenDeleteAndFailure < 1*time.Second:
				// nothing here, one second is close enough to be ok, the kubelet and CNI just didn't know
			case timeBetweenDeleteAndFailure < 5*time.Second:
				// withing five seconds, it ought to be long enough to know, but it's close enough to flake and not fail
				flakes = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			case deletionTime.Before(event.From):
				// something went wrong.  More than five seconds after the pod ws deleted, the CNI is trying to set up pod sandboxes and can't
				failures = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			default:
				// something went wrong.  deletion happend after we had a failure to create the pod sandbox
				failures = append(failures, fmt.Sprintf("%v - deletion came AFTER sandbox failure - %v", event.Locator, event.Message))
			}
		}
	}
	if len(failures) == 0 && len(flakes) == 0 {
		successes := []*ginkgo.JUnitTestCase{}
		for _, by := range bySubStrings {
			successes = append(successes, &ginkgo.JUnitTestCase{Name: testName + by.by})
		}
		return successes
	}

	ret := []*ginkgo.JUnitTestCase{}
	failuresBySubtest, flakesBySubtest := categorizeBySubset(bySubStrings, failures, flakes)

	// now iterate the individual failures to create failure entries
	for by, subFailures := range failuresBySubtest {
		failure := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFailures, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFailures), strings.Join(subFailures, "\n")),
			},
		}
		ret = append(ret, failure)
	}
	for by, subFlakes := range flakesBySubtest {
		flake := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFlakes, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFlakes), strings.Join(subFlakes, "\n")),
			},
		}
		ret = append(ret, flake)
		// write a passing test to trigger detection of this issue as a flake. Doing this first to try to see how frequent the issue actually is
		success := &ginkgo.JUnitTestCase{
			Name: testName + by,
		}
		ret = append(ret, success)
	}

	return append(ret)
}

// categorizeBySubset returns a map keyed by category for failures and flakes.  If a category is present in both failures and flakes, all are listed under failures.
func categorizeBySubset(categorizers []testCategorizer, failures, flakes []string) (map[string][]string, map[string][]string) {
	failuresBySubtest := map[string][]string{}
	flakesBySubtest := map[string][]string{}
	for _, failure := range failures {
		for _, by := range categorizers {
			if strings.Contains(failure, by.substring) {
				failuresBySubtest[by.by] = append(failuresBySubtest[by.by], failure)
				break // break after first match so we only add each failure one bucket
			}
		}
	}

	for _, flake := range flakes {
		for _, by := range categorizers {
			if strings.Contains(flake, by.substring) {
				if _, isFailure := failuresBySubtest[by.by]; isFailure {
					failuresBySubtest[by.by] = append(failuresBySubtest[by.by], flake)
				} else {
					flakesBySubtest[by.by] = append(flakesBySubtest[by.by], flake)
				}
				break // break after first match so we only add each failure one bucket
			}
		}
	}
	return failuresBySubtest, flakesBySubtest
}

// getEventsByPod returns map keyed by pod locator with all events associated with it.
func getEventsByPod(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByPods := map[string]monitorapi.Intervals{}
	for _, event := range events {
		if !strings.Contains(event.Locator, "pod/") {
			continue
		}
		eventsByPods[event.Locator] = append(eventsByPods[event.Locator], event)
	}
	return eventsByPods
}

func getPodDeletionTime(events monitorapi.Intervals, podLocator string) *time.Time {
	for _, event := range events {
		if event.Locator == podLocator && event.Message == "reason/Deleted" {
			return &event.From
		}
	}
	return nil
}
<commit_msg>pkg/synthetictests/networking: Grant temporary exception for "error getting pod: Unauthorized"<commit_after>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	"github.com/openshift/origin/pkg/test/ginkgo"
)

type testCategorizer struct {
	by        string
	substring string
}

func testPodSandboxCreation(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	const testName = "[sig-network] pods should successfully create sandboxes"
	// we can further refine this signal by subdividing different failure modes if it is pertinent.  Right now I'm seeing
	// 1. error reading container (probably exited) json message: EOF
	// 2. dial tcp 10.0.76.225:6443: i/o timeout
	// 3. error getting pod: pods "terminate-cmd-rpofb45fa14c-96bb-40f7-bd9e-346721740cac" not found
	// 4. write child: broken pipe
	bySubStrings := []testCategorizer{
		{by: " by reading container", substring: "error reading container (probably exited) json message: EOF"},
		{by: " by not timing out", substring: "i/o timeout"},
		{by: " by writing network status", substring: "error setting the networks status"},
		{by: " by getting pod", substring: " error getting pod: pods"},
		{by: " by writing child", substring: "write child: broken pipe"},
		{by: " by other", substring: " "}, // always matches
	}

	failures := []string{}
	flakes := []string{}
	eventsForPods := getEventsByPod(events)
	for _, event := range events {
		if !strings.Contains(event.Message, "reason/FailedCreatePodSandBox Failed to create pod sandbox") {
			continue
		}
		if strings.Contains(event.Message, "Multus") && strings.Contains(event.Message, "error getting pod") && (strings.Contains(event.Message, "connection refused") || strings.Contains(event.Message, "i/o timeout")) {
			flakes = append(flakes, fmt.Sprintf("%v - multus is unable to get pods due to LB disruption https://bugzilla.redhat.com/show_bug.cgi?id=1927264 - %v", event.Locator, event.Message))
			continue
		}
		if strings.Contains(event.Message, "Multus") && strings.Contains(event.Message, "error getting pod: Unauthorized") {
			flakes = append(flakes, fmt.Sprintf("%v - multus is unable to get pods due to authorization https://bugzilla.redhat.com/show_bug.cgi?id=1972490 - %v", event.Locator, event.Message))
			continue
		}
		deletionTime := getPodDeletionTime(eventsForPods[event.Locator], event.Locator)
		if deletionTime == nil {
			// this indicates a failure to create the sandbox that should not happen
			failures = append(failures, fmt.Sprintf("%v - never deleted - %v", event.Locator, event.Message))
		} else {
			timeBetweenDeleteAndFailure := event.From.Sub(*deletionTime)
			switch {
			case timeBetweenDeleteAndFailure < 1*time.Second:
				// nothing here, one second is close enough to be ok, the kubelet and CNI just didn't know
			case timeBetweenDeleteAndFailure < 5*time.Second:
				// withing five seconds, it ought to be long enough to know, but it's close enough to flake and not fail
				flakes = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			case deletionTime.Before(event.From):
				// something went wrong.  More than five seconds after the pod ws deleted, the CNI is trying to set up pod sandboxes and can't
				failures = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			default:
				// something went wrong.  deletion happend after we had a failure to create the pod sandbox
				failures = append(failures, fmt.Sprintf("%v - deletion came AFTER sandbox failure - %v", event.Locator, event.Message))
			}
		}
	}
	if len(failures) == 0 && len(flakes) == 0 {
		successes := []*ginkgo.JUnitTestCase{}
		for _, by := range bySubStrings {
			successes = append(successes, &ginkgo.JUnitTestCase{Name: testName + by.by})
		}
		return successes
	}

	ret := []*ginkgo.JUnitTestCase{}
	failuresBySubtest, flakesBySubtest := categorizeBySubset(bySubStrings, failures, flakes)

	// now iterate the individual failures to create failure entries
	for by, subFailures := range failuresBySubtest {
		failure := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFailures, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFailures), strings.Join(subFailures, "\n")),
			},
		}
		ret = append(ret, failure)
	}
	for by, subFlakes := range flakesBySubtest {
		flake := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFlakes, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFlakes), strings.Join(subFlakes, "\n")),
			},
		}
		ret = append(ret, flake)
		// write a passing test to trigger detection of this issue as a flake. Doing this first to try to see how frequent the issue actually is
		success := &ginkgo.JUnitTestCase{
			Name: testName + by,
		}
		ret = append(ret, success)
	}

	return append(ret)
}

// categorizeBySubset returns a map keyed by category for failures and flakes.  If a category is present in both failures and flakes, all are listed under failures.
func categorizeBySubset(categorizers []testCategorizer, failures, flakes []string) (map[string][]string, map[string][]string) {
	failuresBySubtest := map[string][]string{}
	flakesBySubtest := map[string][]string{}
	for _, failure := range failures {
		for _, by := range categorizers {
			if strings.Contains(failure, by.substring) {
				failuresBySubtest[by.by] = append(failuresBySubtest[by.by], failure)
				break // break after first match so we only add each failure one bucket
			}
		}
	}

	for _, flake := range flakes {
		for _, by := range categorizers {
			if strings.Contains(flake, by.substring) {
				if _, isFailure := failuresBySubtest[by.by]; isFailure {
					failuresBySubtest[by.by] = append(failuresBySubtest[by.by], flake)
				} else {
					flakesBySubtest[by.by] = append(flakesBySubtest[by.by], flake)
				}
				break // break after first match so we only add each failure one bucket
			}
		}
	}
	return failuresBySubtest, flakesBySubtest
}

// getEventsByPod returns map keyed by pod locator with all events associated with it.
func getEventsByPod(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByPods := map[string]monitorapi.Intervals{}
	for _, event := range events {
		if !strings.Contains(event.Locator, "pod/") {
			continue
		}
		eventsByPods[event.Locator] = append(eventsByPods[event.Locator], event)
	}
	return eventsByPods
}

func getPodDeletionTime(events monitorapi.Intervals, podLocator string) *time.Time {
	for _, event := range events {
		if event.Locator == podLocator && event.Message == "reason/Deleted" {
			return &event.From
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package cell_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"code.cloudfoundry.org/archiver/compressor"
	archive_helper "code.cloudfoundry.org/archiver/extractor/test_helper"
	"code.cloudfoundry.org/bbs/models"
	ssh_helpers "code.cloudfoundry.org/diego-ssh/helpers"
	"code.cloudfoundry.org/diego-ssh/routes"
	"code.cloudfoundry.org/inigo/fixtures"
	"code.cloudfoundry.org/inigo/helpers"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"github.com/tedsuo/ifrit/grouper"
	"golang.org/x/crypto/ssh"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("SSH", func() {
	verifySSH := func(address, processGuid string, index int) {
		clientConfig := &ssh.ClientConfig{
			User: fmt.Sprintf("diego:%s/%d", processGuid, index),
			Auth: []ssh.AuthMethod{ssh.Password("")},
		}

		client, err := ssh.Dial("tcp", address, clientConfig)
		Expect(err).NotTo(HaveOccurred())

		session, err := client.NewSession()
		Expect(err).NotTo(HaveOccurred())

		output, err := session.Output("env")
		Expect(err).NotTo(HaveOccurred())

		Expect(string(output)).To(ContainSubstring("USER=root"))
		Expect(string(output)).To(ContainSubstring("TEST=foobar"))
		Expect(string(output)).To(ContainSubstring(fmt.Sprintf("INSTANCE_INDEX=%d", index)))
	}

	var (
		processGuid         string
		fileServerStaticDir string

		runtime ifrit.Process
		address string

		lrp models.DesiredLRP
	)

	BeforeEach(func() {
		processGuid = helpers.GenerateGuid()
		address = componentMaker.Addresses.SSHProxy

		var fileServer ifrit.Runner
		fileServer, fileServerStaticDir = componentMaker.FileServer()
		runtime = ginkgomon.Invoke(grouper.NewParallel(os.Kill, grouper.Members{
			{"router", componentMaker.Router()},
			{"file-server", fileServer},
			{"rep", componentMaker.Rep()},
			{"auctioneer", componentMaker.Auctioneer()},
			{"route-emitter", componentMaker.RouteEmitter()},
			{"ssh-proxy", componentMaker.SSHProxy()},
		}))

		tgCompressor := compressor.NewTgz()
		err := tgCompressor.Compress(componentMaker.Artifacts.Executables["sshd"], filepath.Join(fileServerStaticDir, "sshd.tgz"))
		Expect(err).NotTo(HaveOccurred())

		archive_helper.CreateZipArchive(
			filepath.Join(fileServerStaticDir, "lrp.zip"),
			fixtures.GoServerApp(),
		)

		sshRoute := routes.SSHRoute{
			ContainerPort:   3456,
			PrivateKey:      componentMaker.SSHConfig.PrivateKeyPem,
			HostFingerprint: ssh_helpers.MD5Fingerprint(componentMaker.SSHConfig.HostKey.PublicKey()),
		}

		sshRoutePayload, err := json.Marshal(sshRoute)
		Expect(err).NotTo(HaveOccurred())

		sshRouteMessage := json.RawMessage(sshRoutePayload)

		envVars := []*models.EnvironmentVariable{
			{Name: "TEST", Value: "foobar"},
		}

		lrp = models.DesiredLRP{
			ProcessGuid:        processGuid,
			Domain:             "inigo",
			Instances:          2,
			Privileged:         true,
			LegacyDownloadUser: "vcap",
			Setup: models.WrapAction(models.Serial(
				&models.DownloadAction{
					Artifact: "sshd",
					From:     fmt.Sprintf("http://%s/v1/static/%s", componentMaker.Addresses.FileServer, "sshd.tgz"),
					To:       "/tmp/diego",
					CacheKey: "sshd",
					User:     "root",
				},
				&models.DownloadAction{
					Artifact: "go-server",
					From:     fmt.Sprintf("http://%s/v1/static/%s", componentMaker.Addresses.FileServer, "lrp.zip"),
					To:       "/tmp/diego",
					CacheKey: "lrp-cache-key",
					User:     "root",
				},
			)),
			Action: models.WrapAction(models.Codependent(
				&models.RunAction{
					User: "root",
					Path: "/tmp/diego/sshd",
					Args: []string{
						"-address=0.0.0.0:3456",
						"-hostKey=" + componentMaker.SSHConfig.HostKeyPem,
						"-authorizedKey=" + componentMaker.SSHConfig.AuthorizedKey,
						"-inheritDaemonEnv",
					},
				},
				&models.RunAction{
					User: "root",
					Path: "/tmp/diego/go-server",
					Env:  []*models.EnvironmentVariable{{"PORT", "9999"}},
				},
			)),
			Monitor: models.WrapAction(&models.RunAction{
				User: "root",
				Path: "nc",
				Args: []string{"-z", "127.0.0.1", "3456"},
			}),
			StartTimeoutMs: 60000,
			RootFs:         "preloaded:" + helpers.PreloadedStacks[0],
			MemoryMb:       128,
			DiskMb:         128,
			Ports:          []uint32{3456},
			Routes: &models.Routes{
				routes.DIEGO_SSH: &sshRouteMessage,
			},
			EnvironmentVariables: envVars,
		}
	})

	JustBeforeEach(func() {
		logger := lagertest.NewTestLogger("test")
		logger.Info("desired-ssh-lrp", lager.Data{"lrp": lrp})

		err := bbsClient.DesireLRP(logger, &lrp)
		Expect(err).NotTo(HaveOccurred())

		Eventually(func() []*models.ActualLRPGroup {
			lrps, err := bbsClient.ActualLRPGroupsByProcessGuid(logger, processGuid)
			Expect(err).NotTo(HaveOccurred())
			return lrps
		}).Should(HaveLen(2))

		Eventually(
			helpers.LRPInstanceStatePoller(logger, bbsClient, processGuid, 0, nil),
		).Should(Equal(models.ActualLRPStateRunning))

		Eventually(
			helpers.LRPInstanceStatePoller(logger, bbsClient, processGuid, 1, nil),
		).Should(Equal(models.ActualLRPStateRunning))
	})

	AfterEach(func() {
		helpers.StopProcesses(runtime)
	})

	Context("when valid process guid and index are used in the username", func() {
		It("can ssh to appropriate app instance container", func() {
			verifySSH(address, processGuid, 0)
			verifySSH(address, processGuid, 1)
		})

		It("supports local port fowarding", func() {
			clientConfig := &ssh.ClientConfig{
				User: fmt.Sprintf("diego:%s/%d", processGuid, 0),
				Auth: []ssh.AuthMethod{ssh.Password("")},
			}

			client, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).NotTo(HaveOccurred())

			httpClient := &http.Client{
				Transport: &http.Transport{
					Dial: client.Dial,
				},
				Timeout: 10 * time.Second,
			}

			resp, err := httpClient.Get("http://localhost:9999/yo")
			Expect(err).NotTo(HaveOccurred())
			defer resp.Body.Close()
			Expect(resp.StatusCode).To(Equal(http.StatusOK))

			contents, err := ioutil.ReadAll(resp.Body)
			Expect(err).NotTo(HaveOccurred())
			Expect(contents).To(ContainSubstring("sup dawg"))
		})

		Context("when invalid password is used", func() {
			var clientConfig *ssh.ClientConfig

			BeforeEach(func() {
				clientConfig = &ssh.ClientConfig{
					User: "diego:" + processGuid + "/0",
					Auth: []ssh.AuthMethod{ssh.Password("invalid:password")},
				}
			})

			It("returns an error", func() {
				Eventually(
					helpers.LRPInstanceStatePoller(logger, bbsClient, processGuid, 0, nil),
				).Should(Equal(models.ActualLRPStateRunning))

				_, err := ssh.Dial("tcp", address, clientConfig)
				Expect(err).To(HaveOccurred())
			})
		})

		Context("when a bare-bones docker image is used as the root filesystem", func() {
			BeforeEach(func() {
				lrp.StartTimeoutMs = 120000
				lrp.RootFs = "docker:///cloudfoundry/diego-docker-app"

				// busybox nc doesn't support -z
				lrp.Monitor = models.WrapAction(&models.RunAction{
					User: "root",
					Path: "sh",
					Args: []string{
						"-c",
						"echo -n '' | telnet localhost 3456 >/dev/null 2>&1 && true",
					},
				})
			})

			It("can ssh to appropriate app instance container", func() {
				verifySSH(address, processGuid, 0)
				verifySSH(address, processGuid, 1)
			})

			It("supports local port fowarding", func() {
				clientConfig := &ssh.ClientConfig{
					User: fmt.Sprintf("diego:%s/%d", processGuid, 0),
					Auth: []ssh.AuthMethod{ssh.Password("")},
				}

				client, err := ssh.Dial("tcp", address, clientConfig)
				Expect(err).NotTo(HaveOccurred())

				httpClient := &http.Client{
					Transport: &http.Transport{
						Dial: client.Dial,
					},
					Timeout: 10 * time.Second,
				}

				resp, err := httpClient.Get("http://localhost:9999/yo")
				Expect(err).NotTo(HaveOccurred())
				defer resp.Body.Close()
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				contents, err := ioutil.ReadAll(resp.Body)
				Expect(err).NotTo(HaveOccurred())
				Expect(contents).To(ContainSubstring("sup dawg"))
			})
		})
	})

	Context("when non-existent index is used as part of username", func() {
		var clientConfig *ssh.ClientConfig

		BeforeEach(func() {
			clientConfig = &ssh.ClientConfig{
				User: "diego:" + processGuid + "/3",
				Auth: []ssh.AuthMethod{ssh.Password("")},
			}
		})

		It("returns an error", func() {
			_, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("when non-existent process guid is used as part of username", func() {
		var clientConfig *ssh.ClientConfig

		BeforeEach(func() {
			clientConfig = &ssh.ClientConfig{
				User: "diego:not-existing-process-guid/0",
				Auth: []ssh.AuthMethod{ssh.Password("")},
			}
		})

		It("returns an error", func() {
			_, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("when invalid username format is used", func() {
		var clientConfig *ssh.ClientConfig

		BeforeEach(func() {
			clientConfig = &ssh.ClientConfig{
				User: "root",
				Auth: []ssh.AuthMethod{ssh.Password("some-password")},
			}
		})

		It("returns an error", func() {
			_, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).To(HaveOccurred())
		})
	})
})
<commit_msg>increase client timeout for port forwarding test<commit_after>package cell_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"code.cloudfoundry.org/archiver/compressor"
	archive_helper "code.cloudfoundry.org/archiver/extractor/test_helper"
	"code.cloudfoundry.org/bbs/models"
	ssh_helpers "code.cloudfoundry.org/diego-ssh/helpers"
	"code.cloudfoundry.org/diego-ssh/routes"
	"code.cloudfoundry.org/inigo/fixtures"
	"code.cloudfoundry.org/inigo/helpers"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
	"github.com/tedsuo/ifrit/grouper"
	"golang.org/x/crypto/ssh"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("SSH", func() {
	verifySSH := func(address, processGuid string, index int) {
		clientConfig := &ssh.ClientConfig{
			User: fmt.Sprintf("diego:%s/%d", processGuid, index),
			Auth: []ssh.AuthMethod{ssh.Password("")},
		}

		client, err := ssh.Dial("tcp", address, clientConfig)
		Expect(err).NotTo(HaveOccurred())

		session, err := client.NewSession()
		Expect(err).NotTo(HaveOccurred())

		output, err := session.Output("env")
		Expect(err).NotTo(HaveOccurred())

		Expect(string(output)).To(ContainSubstring("USER=root"))
		Expect(string(output)).To(ContainSubstring("TEST=foobar"))
		Expect(string(output)).To(ContainSubstring(fmt.Sprintf("INSTANCE_INDEX=%d", index)))
	}

	var (
		processGuid         string
		fileServerStaticDir string

		runtime ifrit.Process
		address string

		lrp models.DesiredLRP
	)

	BeforeEach(func() {
		processGuid = helpers.GenerateGuid()
		address = componentMaker.Addresses.SSHProxy

		var fileServer ifrit.Runner
		fileServer, fileServerStaticDir = componentMaker.FileServer()
		runtime = ginkgomon.Invoke(grouper.NewParallel(os.Kill, grouper.Members{
			{"router", componentMaker.Router()},
			{"file-server", fileServer},
			{"rep", componentMaker.Rep()},
			{"auctioneer", componentMaker.Auctioneer()},
			{"route-emitter", componentMaker.RouteEmitter()},
			{"ssh-proxy", componentMaker.SSHProxy()},
		}))

		tgCompressor := compressor.NewTgz()
		err := tgCompressor.Compress(componentMaker.Artifacts.Executables["sshd"], filepath.Join(fileServerStaticDir, "sshd.tgz"))
		Expect(err).NotTo(HaveOccurred())

		archive_helper.CreateZipArchive(
			filepath.Join(fileServerStaticDir, "lrp.zip"),
			fixtures.GoServerApp(),
		)

		sshRoute := routes.SSHRoute{
			ContainerPort:   3456,
			PrivateKey:      componentMaker.SSHConfig.PrivateKeyPem,
			HostFingerprint: ssh_helpers.MD5Fingerprint(componentMaker.SSHConfig.HostKey.PublicKey()),
		}

		sshRoutePayload, err := json.Marshal(sshRoute)
		Expect(err).NotTo(HaveOccurred())

		sshRouteMessage := json.RawMessage(sshRoutePayload)

		envVars := []*models.EnvironmentVariable{
			{Name: "TEST", Value: "foobar"},
		}

		lrp = models.DesiredLRP{
			ProcessGuid:        processGuid,
			Domain:             "inigo",
			Instances:          2,
			Privileged:         true,
			LegacyDownloadUser: "vcap",
			Setup: models.WrapAction(models.Serial(
				&models.DownloadAction{
					Artifact: "sshd",
					From:     fmt.Sprintf("http://%s/v1/static/%s", componentMaker.Addresses.FileServer, "sshd.tgz"),
					To:       "/tmp/diego",
					CacheKey: "sshd",
					User:     "root",
				},
				&models.DownloadAction{
					Artifact: "go-server",
					From:     fmt.Sprintf("http://%s/v1/static/%s", componentMaker.Addresses.FileServer, "lrp.zip"),
					To:       "/tmp/diego",
					CacheKey: "lrp-cache-key",
					User:     "root",
				},
			)),
			Action: models.WrapAction(models.Codependent(
				&models.RunAction{
					User: "root",
					Path: "/tmp/diego/sshd",
					Args: []string{
						"-address=0.0.0.0:3456",
						"-hostKey=" + componentMaker.SSHConfig.HostKeyPem,
						"-authorizedKey=" + componentMaker.SSHConfig.AuthorizedKey,
						"-inheritDaemonEnv",
					},
				},
				&models.RunAction{
					User: "root",
					Path: "/tmp/diego/go-server",
					Env:  []*models.EnvironmentVariable{{"PORT", "9999"}},
				},
			)),
			Monitor: models.WrapAction(&models.RunAction{
				User: "root",
				Path: "nc",
				Args: []string{"-z", "127.0.0.1", "3456"},
			}),
			StartTimeoutMs: 60000,
			RootFs:         "preloaded:" + helpers.PreloadedStacks[0],
			MemoryMb:       128,
			DiskMb:         128,
			Ports:          []uint32{3456},
			Routes: &models.Routes{
				routes.DIEGO_SSH: &sshRouteMessage,
			},
			EnvironmentVariables: envVars,
		}
	})

	JustBeforeEach(func() {
		logger := lagertest.NewTestLogger("test")
		logger.Info("desired-ssh-lrp", lager.Data{"lrp": lrp})

		err := bbsClient.DesireLRP(logger, &lrp)
		Expect(err).NotTo(HaveOccurred())

		Eventually(func() []*models.ActualLRPGroup {
			lrps, err := bbsClient.ActualLRPGroupsByProcessGuid(logger, processGuid)
			Expect(err).NotTo(HaveOccurred())
			return lrps
		}).Should(HaveLen(2))

		Eventually(
			helpers.LRPInstanceStatePoller(logger, bbsClient, processGuid, 0, nil),
		).Should(Equal(models.ActualLRPStateRunning))

		Eventually(
			helpers.LRPInstanceStatePoller(logger, bbsClient, processGuid, 1, nil),
		).Should(Equal(models.ActualLRPStateRunning))
	})

	AfterEach(func() {
		helpers.StopProcesses(runtime)
	})

	Context("when valid process guid and index are used in the username", func() {
		It("can ssh to appropriate app instance container", func() {
			verifySSH(address, processGuid, 0)
			verifySSH(address, processGuid, 1)
		})

		It("supports local port fowarding", func() {
			clientConfig := &ssh.ClientConfig{
				User: fmt.Sprintf("diego:%s/%d", processGuid, 0),
				Auth: []ssh.AuthMethod{ssh.Password("")},
			}

			client, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).NotTo(HaveOccurred())

			httpClient := &http.Client{
				Transport: &http.Transport{
					Dial: client.Dial,
				},
				Timeout: 20 * time.Second,
			}

			resp, err := httpClient.Get("http://localhost:9999/yo")
			Expect(err).NotTo(HaveOccurred())
			defer resp.Body.Close()
			Expect(resp.StatusCode).To(Equal(http.StatusOK))

			contents, err := ioutil.ReadAll(resp.Body)
			Expect(err).NotTo(HaveOccurred())
			Expect(contents).To(ContainSubstring("sup dawg"))
		})

		Context("when invalid password is used", func() {
			var clientConfig *ssh.ClientConfig

			BeforeEach(func() {
				clientConfig = &ssh.ClientConfig{
					User: "diego:" + processGuid + "/0",
					Auth: []ssh.AuthMethod{ssh.Password("invalid:password")},
				}
			})

			It("returns an error", func() {
				Eventually(
					helpers.LRPInstanceStatePoller(logger, bbsClient, processGuid, 0, nil),
				).Should(Equal(models.ActualLRPStateRunning))

				_, err := ssh.Dial("tcp", address, clientConfig)
				Expect(err).To(HaveOccurred())
			})
		})

		Context("when a bare-bones docker image is used as the root filesystem", func() {
			BeforeEach(func() {
				lrp.StartTimeoutMs = 120000
				lrp.RootFs = "docker:///cloudfoundry/diego-docker-app"

				// busybox nc doesn't support -z
				lrp.Monitor = models.WrapAction(&models.RunAction{
					User: "root",
					Path: "sh",
					Args: []string{
						"-c",
						"echo -n '' | telnet localhost 3456 >/dev/null 2>&1 && echo -n '' | telnet localhost 9999 >/dev/null 2>&1 && true",
					},
				})
			})

			It("can ssh to appropriate app instance container", func() {
				verifySSH(address, processGuid, 0)
				verifySSH(address, processGuid, 1)
			})

			It("supports local port fowarding", func() {
				clientConfig := &ssh.ClientConfig{
					User: fmt.Sprintf("diego:%s/%d", processGuid, 0),
					Auth: []ssh.AuthMethod{ssh.Password("")},
				}

				client, err := ssh.Dial("tcp", address, clientConfig)
				Expect(err).NotTo(HaveOccurred())

				httpClient := &http.Client{
					Transport: &http.Transport{
						Dial: client.Dial,
					},
					Timeout: 20 * time.Second,
				}

				resp, err := httpClient.Get("http://localhost:9999/yo")
				Expect(err).NotTo(HaveOccurred())
				defer resp.Body.Close()
				Expect(resp.StatusCode).To(Equal(http.StatusOK))

				contents, err := ioutil.ReadAll(resp.Body)
				Expect(err).NotTo(HaveOccurred())
				Expect(contents).To(ContainSubstring("sup dawg"))
			})
		})
	})

	Context("when non-existent index is used as part of username", func() {
		var clientConfig *ssh.ClientConfig

		BeforeEach(func() {
			clientConfig = &ssh.ClientConfig{
				User: "diego:" + processGuid + "/3",
				Auth: []ssh.AuthMethod{ssh.Password("")},
			}
		})

		It("returns an error", func() {
			_, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("when non-existent process guid is used as part of username", func() {
		var clientConfig *ssh.ClientConfig

		BeforeEach(func() {
			clientConfig = &ssh.ClientConfig{
				User: "diego:not-existing-process-guid/0",
				Auth: []ssh.AuthMethod{ssh.Password("")},
			}
		})

		It("returns an error", func() {
			_, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).To(HaveOccurred())
		})
	})

	Context("when invalid username format is used", func() {
		var clientConfig *ssh.ClientConfig

		BeforeEach(func() {
			clientConfig = &ssh.ClientConfig{
				User: "root",
				Auth: []ssh.AuthMethod{ssh.Password("some-password")},
			}
		})

		It("returns an error", func() {
			_, err := ssh.Dial("tcp", address, clientConfig)
			Expect(err).To(HaveOccurred())
		})
	})
})
<|endoftext|>


<commit_before>// Copyright (c) 2018, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package minifier

import (
	"io"
	"net/http"
	"regexp"

	"github.com/tdewolff/minify/v2"
	"github.com/tdewolff/minify/v2/css"
	"github.com/tdewolff/minify/v2/html"
	"github.com/tdewolff/minify/v2/js"
	"github.com/tdewolff/minify/v2/json"
	"github.com/tdewolff/minify/v2/svg"
	"github.com/tdewolff/minify/v2/xml"
)

var minifier = minify.New()

func init() {
	htmlMinify := &html.Minifier{
		KeepWhitespace: true,
	}
	minifier.AddFunc("text/css", css.Minify)
	minifier.Add("text/html", htmlMinify)
	minifier.Add("text/x-template", htmlMinify)
	minifier.AddFunc("text/javascript", js.Minify)
	minifier.AddFunc("application/javascript", js.Minify)
	minifier.AddFunc("application/x-javascript", js.Minify)
	minifier.AddFunc("image/svg+xml", svg.Minify)
	minifier.AddFuncRegexp(regexp.MustCompile("[/+]json$"), json.Minify)
	minifier.AddFuncRegexp(regexp.MustCompile("[/+]xml$"), xml.Minify)
}

// LogFunc defines the function that is used by NewHandler
// to log error messages from minifier.
type LogFunc func(format string, a ...interface{})

// NewHandler returns a minifer http handler that implements common http.Hander
// related interfaces and uses a common configuration for github.com/tdewolff/minify.
func NewHandler(h http.Handler, logFunc LogFunc) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mw := &minifyResponseWriter{mw: minifier.ResponseWriter(w, r), w: w}
		h.ServeHTTP(mw, r)
		if err := mw.Close(); err != nil && err != minify.ErrNotExist && logFunc != nil {
			logFunc("minifier %q: %v", r.URL.String(), err)
		}
	})
}

type minifyResponseWriter struct {
	mw http.ResponseWriter
	w  http.ResponseWriter
}

func (w *minifyResponseWriter) Header() http.Header {
	return w.mw.Header()
}

func (w *minifyResponseWriter) Flush() {
	w.w.(http.Flusher).Flush()
}

func (w *minifyResponseWriter) Write(b []byte) (int, error) {
	return w.mw.Write(b)
}

func (w *minifyResponseWriter) WriteHeader(s int) {
	w.mw.WriteHeader(s)
}

func (w *minifyResponseWriter) Push(target string, opts *http.PushOptions) error {
	return w.w.(http.Pusher).Push(target, opts)
}

func (w *minifyResponseWriter) Close() error {
	if c, ok := w.mw.(io.Closer); ok {
		return c.Close()
	}
	return nil
}
<commit_msg>Set KeepDocumentTags for HTML minifier<commit_after>// Copyright (c) 2018, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package minifier

import (
	"io"
	"net/http"
	"regexp"

	"github.com/tdewolff/minify/v2"
	"github.com/tdewolff/minify/v2/css"
	"github.com/tdewolff/minify/v2/html"
	"github.com/tdewolff/minify/v2/js"
	"github.com/tdewolff/minify/v2/json"
	"github.com/tdewolff/minify/v2/svg"
	"github.com/tdewolff/minify/v2/xml"
)

var minifier = minify.New()

func init() {
	htmlMinify := &html.Minifier{
		KeepWhitespace:   true,
		KeepDocumentTags: true,
	}
	minifier.AddFunc("text/css", css.Minify)
	minifier.Add("text/html", htmlMinify)
	minifier.Add("text/x-template", htmlMinify)
	minifier.AddFunc("text/javascript", js.Minify)
	minifier.AddFunc("application/javascript", js.Minify)
	minifier.AddFunc("application/x-javascript", js.Minify)
	minifier.AddFunc("image/svg+xml", svg.Minify)
	minifier.AddFuncRegexp(regexp.MustCompile("[/+]json$"), json.Minify)
	minifier.AddFuncRegexp(regexp.MustCompile("[/+]xml$"), xml.Minify)
}

// LogFunc defines the function that is used by NewHandler
// to log error messages from minifier.
type LogFunc func(format string, a ...interface{})

// NewHandler returns a minifer http handler that implements common http.Hander
// related interfaces and uses a common configuration for github.com/tdewolff/minify.
func NewHandler(h http.Handler, logFunc LogFunc) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		mw := &minifyResponseWriter{mw: minifier.ResponseWriter(w, r), w: w}
		h.ServeHTTP(mw, r)
		if err := mw.Close(); err != nil && err != minify.ErrNotExist && logFunc != nil {
			logFunc("minifier %q: %v", r.URL.String(), err)
		}
	})
}

type minifyResponseWriter struct {
	mw http.ResponseWriter
	w  http.ResponseWriter
}

func (w *minifyResponseWriter) Header() http.Header {
	return w.mw.Header()
}

func (w *minifyResponseWriter) Flush() {
	w.w.(http.Flusher).Flush()
}

func (w *minifyResponseWriter) Write(b []byte) (int, error) {
	return w.mw.Write(b)
}

func (w *minifyResponseWriter) WriteHeader(s int) {
	w.mw.WriteHeader(s)
}

func (w *minifyResponseWriter) Push(target string, opts *http.PushOptions) error {
	return w.w.(http.Pusher).Push(target, opts)
}

func (w *minifyResponseWriter) Close() error {
	if c, ok := w.mw.(io.Closer); ok {
		return c.Close()
	}
	return nil
}
<|endoftext|>


<commit_before>package misc

import (
	"fmt"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/ChimeraCoder/anaconda"
	"github.com/coopernurse/gorp"
	"github.com/zachlatta/southbayfession/models"
)

func FetchLatestTweetsManager() {
	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))
	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_ACCESS_TOKEN"),
		os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))

	db := models.Dbm

	for {
		FetchAndCommitLatestTweets(api, db)
		time.Sleep(5 * time.Second)
	}
}

func FetchAndCommitLatestTweets(api *anaconda.TwitterApi, db gorp.SqlExecutor) {
	lastTweet, err := GetLastTweet(api, db)
	if err != nil {
		return
	}

	tweets, err := TweetsAfter(api, lastTweet)
	if err != nil {
		return
	}

	for _, tweet := range tweets {
		err := db.Insert(&tweet)
		if err != nil {
			fmt.Println(err)
			return
		}
	}
}

func GetLastTweet(api *anaconda.TwitterApi, db gorp.SqlExecutor) (*models.Tweet, error) {
	var tweets []models.Tweet
	_, err := db.Select(&tweets, "select * from tweets order by id limit 1")
	if err != nil {
		return nil, err
	}

	var tweet *models.Tweet
	if len(tweets) == 0 {
		tweet = &models.Tweet{TwitterId: 1}
	} else {
		tweet = &tweets[0]
	}
	return tweet, nil
}

func TweetsAfter(api *anaconda.TwitterApi, tweet *models.Tweet) (
	[]models.Tweet, error) {
	anacondaTweets, err := api.GetUserTimeline(url.Values{
		"screen_name": []string{"Southbayfession"},
		"since_id":    []string{strconv.FormatInt(tweet.TwitterId, 10)},
	})
	if err != nil {
		return nil, err
	}

	tweets := make([]models.Tweet, len(anacondaTweets))
	for i, t := range anacondaTweets {
		tweets[i] = models.Tweet{
			CreatedAt: t.CreatedAt,
			TwitterId: t.Id,
			Text:      t.Text,
			School:    "ESHS",
		}
	}

	return tweets, nil
}
<commit_msg>Don't print error on DB insertion error.<commit_after>package misc

import (
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/ChimeraCoder/anaconda"
	"github.com/coopernurse/gorp"
	"github.com/zachlatta/southbayfession/models"
)

func FetchLatestTweetsManager() {
	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))
	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_ACCESS_TOKEN"),
		os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))

	db := models.Dbm

	for {
		FetchAndCommitLatestTweets(api, db)
		time.Sleep(5 * time.Second)
	}
}

func FetchAndCommitLatestTweets(api *anaconda.TwitterApi, db gorp.SqlExecutor) {
	lastTweet, err := GetLastTweet(api, db)
	if err != nil {
		return
	}

	tweets, err := TweetsAfter(api, lastTweet)
	if err != nil {
		return
	}

	for _, tweet := range tweets {
		err := db.Insert(&tweet)
		if err != nil {
			return
		}
	}
}

func GetLastTweet(api *anaconda.TwitterApi, db gorp.SqlExecutor) (*models.Tweet, error) {
	var tweets []models.Tweet
	_, err := db.Select(&tweets, "select * from tweets order by id limit 1")
	if err != nil {
		return nil, err
	}

	var tweet *models.Tweet
	if len(tweets) == 0 {
		tweet = &models.Tweet{TwitterId: 1}
	} else {
		tweet = &tweets[0]
	}
	return tweet, nil
}

func TweetsAfter(api *anaconda.TwitterApi, tweet *models.Tweet) (
	[]models.Tweet, error) {
	anacondaTweets, err := api.GetUserTimeline(url.Values{
		"screen_name": []string{"Southbayfession"},
		"since_id":    []string{strconv.FormatInt(tweet.TwitterId, 10)},
	})
	if err != nil {
		return nil, err
	}

	tweets := make([]models.Tweet, len(anacondaTweets))
	for i, t := range anacondaTweets {
		tweets[i] = models.Tweet{
			CreatedAt: t.CreatedAt,
			TwitterId: t.Id,
			Text:      t.Text,
			School:    "ESHS",
		}
	}

	return tweets, nil
}
<|endoftext|>


<commit_before>package proto

import (
	"errors"
	"fmt"
	"io"
	"github.com/vapourismo/knx-go/knx/encoding"
)

// A ConnReq requests a connection to a gateway.
type ConnReq struct {
	Control HostInfo
	Tunnel  HostInfo
}

var connReqInfo = [4]byte{4, 4, 2, 0}

// WriteTo serializes the structure and writes it to the given Writer.
func (req *ConnReq) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, &req.Control, &req.Tunnel, connReqInfo)
}

// ConnResStatus is the type of status code carried in a connection response.
type ConnResStatus uint8

// Therese are known connection response status codes.
const (
	ConnResOk                ConnResStatus = 0x00
	ConnResUnsupportedType   ConnResStatus = 0x22
	ConnResUnsupportedOption ConnResStatus = 0x23
	ConnResBusy              ConnResStatus = 0x24
)

// String describes the status code.
func (status ConnResStatus) String() string {
	switch status {
	case ConnResOk:
		return "Connection established"

	case ConnResUnsupportedType:
		return "Requested connection type is unsupported"

	case ConnResUnsupportedOption:
		return "One of the requested options is unsupported"

	case ConnResBusy:
		return "No data channel is available"

	default:
		return fmt.Sprintf("Unknown status code %#x", uint8(status))
	}
}

// Error implements the error Error method.
func (status ConnResStatus) Error() string {
	return status.String()
}

// ConnRes is a response to a ConnReq.
type ConnRes struct {
	Channel uint8
	Status  ConnResStatus
	Control HostInfo
}

// ReadFrom initializes the structure by reading from the given Reader.
func (res *ConnRes) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &res.Channel, &res.Status, &res.Control)
}

// A ConnStateReq requests the the connection state from a gateway.
type ConnStateReq struct {
	Channel uint8
	Status  uint8
	Control HostInfo
}

// WriteTo serializes the structure and writes it to the given Writer.
func (req *ConnStateReq) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, req.Channel, req.Status, &req.Control)
}

// A ConnState represents the state of a connection.
type ConnState uint8

// These are known connection states.
const (
	ConnStateNormal    ConnState = 0x00
	ConnStateInactive  ConnState = 0x21
	ConnStateDataError ConnState = 0x26
	ConnStateKNXError  ConnState = 0x27
)

// String converts the connection state to a string.
func (state ConnState) String() string {
	switch state {
	case ConnStateNormal:
		return "Connection is intact"

	case ConnStateInactive:
		return "Connection is inactive"

	case ConnStateDataError:
		return "Gateway encountered a data error"

	case ConnStateKNXError:
		return "Gateway encountered a KNX error"

	default:
		return fmt.Sprintf("Unknown connection state %#x", uint8(state))
	}
}

// Error implements the error Error method.
func (state ConnState) Error() string {
	return state.String()
}

// A ConnStateRes is a response to a ConnStateReq.
type ConnStateRes struct {
	Channel uint8
	Status  ConnState
}

// ReadFrom initializes the structure by reading from the given Reader.
func (res *ConnStateRes) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &res.Channel, &res.Status)
}

// A DiscReq requests a connection to be terminated.
type DiscReq struct {
	Channel uint8
	Status  uint8
	Control HostInfo
}

// ReadFrom initializes the structure by reading from the given Reader.
func (req *DiscReq) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &req.Channel, &req.Status, &req.Control)
}

// WriteTo serializes the structure and writes it to the given Writer.
func (req *DiscReq) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, req.Channel, req.Status, &req.Control)
}

// A DiscRes is a response to a DiscReq..
type DiscRes struct {
	Channel uint8
	Status  uint8
}

// ReadFrom initializes the structure by reading from the given Reader.
func (res *DiscRes) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &res.Channel, &res.Status)
}

// WriteTo serializes the structure and writes it to the given Writer.
func (res *DiscRes) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, res.Channel, res.Status)
}
<commit_msg>Extend ReadFrom and WriteTo where possible<commit_after>package proto

import (
	"fmt"
	"io"
	"github.com/vapourismo/knx-go/knx/encoding"
)

// A ConnReq requests a connection to a gateway.
type ConnReq struct {
	Control HostInfo
	Tunnel  HostInfo
}

var connReqInfo = [4]byte{4, 4, 2, 0}

// WriteTo serializes the structure and writes it to the given Writer.
func (req *ConnReq) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, &req.Control, &req.Tunnel, connReqInfo)
}

// ConnResStatus is the type of status code carried in a connection response.
type ConnResStatus uint8

// Therese are known connection response status codes.
const (
	ConnResOk                ConnResStatus = 0x00
	ConnResUnsupportedType   ConnResStatus = 0x22
	ConnResUnsupportedOption ConnResStatus = 0x23
	ConnResBusy              ConnResStatus = 0x24
)

// String describes the status code.
func (status ConnResStatus) String() string {
	switch status {
	case ConnResOk:
		return "Connection established"

	case ConnResUnsupportedType:
		return "Requested connection type is unsupported"

	case ConnResUnsupportedOption:
		return "One of the requested options is unsupported"

	case ConnResBusy:
		return "No data channel is available"

	default:
		return fmt.Sprintf("Unknown status code %#x", uint8(status))
	}
}

// Error implements the error Error method.
func (status ConnResStatus) Error() string {
	return status.String()
}

// ConnRes is a response to a ConnReq.
type ConnRes struct {
	Channel uint8
	Status  ConnResStatus
	Control HostInfo
}

// ReadFrom initializes the structure by reading from the given Reader.
func (res *ConnRes) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &res.Channel, &res.Status, &res.Control)
}

// A ConnStateReq requests the the connection state from a gateway.
type ConnStateReq struct {
	Channel uint8
	Status  uint8
	Control HostInfo
}

// ReadFrom initializes the structure by reading from the given Reader.
func (req *ConnStateReq) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &req.Channel, &req.Status, &req.Control)
}

// WriteTo serializes the structure and writes it to the given Writer.
func (req *ConnStateReq) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, req.Channel, req.Status, &req.Control)
}

// A ConnState represents the state of a connection.
type ConnState uint8

// These are known connection states.
const (
	ConnStateNormal    ConnState = 0x00
	ConnStateInactive  ConnState = 0x21
	ConnStateDataError ConnState = 0x26
	ConnStateKNXError  ConnState = 0x27
)

// String converts the connection state to a string.
func (state ConnState) String() string {
	switch state {
	case ConnStateNormal:
		return "Connection is intact"

	case ConnStateInactive:
		return "Connection is inactive"

	case ConnStateDataError:
		return "Gateway encountered a data error"

	case ConnStateKNXError:
		return "Gateway encountered a KNX error"

	default:
		return fmt.Sprintf("Unknown connection state %#x", uint8(state))
	}
}

// Error implements the error Error method.
func (state ConnState) Error() string {
	return state.String()
}

// A ConnStateRes is a response to a ConnStateReq.
type ConnStateRes struct {
	Channel uint8
	Status  ConnState
}

// ReadFrom initializes the structure by reading from the given Reader.
func (res *ConnStateRes) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &res.Channel, &res.Status)
}

// WriteTo serializes the structure and writes it to the given Writer.
func (res *ConnStateRes) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, res.Channel, res.Status)
}

// A DiscReq requests a connection to be terminated.
type DiscReq struct {
	Channel uint8
	Status  uint8
	Control HostInfo
}

// ReadFrom initializes the structure by reading from the given Reader.
func (req *DiscReq) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &req.Channel, &req.Status, &req.Control)
}

// WriteTo serializes the structure and writes it to the given Writer.
func (req *DiscReq) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, req.Channel, req.Status, &req.Control)
}

// A DiscRes is a response to a DiscReq..
type DiscRes struct {
	Channel uint8
	Status  uint8
}

// ReadFrom initializes the structure by reading from the given Reader.
func (res *DiscRes) ReadFrom(r io.Reader) (int64, error) {
	return encoding.ReadSome(r, &res.Channel, &res.Status)
}

// WriteTo serializes the structure and writes it to the given Writer.
func (res *DiscRes) WriteTo(w io.Writer) (int64, error) {
	return encoding.WriteSome(w, res.Channel, res.Status)
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package v2

import (
	"bytes"
	"context"
	"io"
	"os"
	gruntime "runtime"
	"strings"

	"github.com/containerd/containerd/events/exchange"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/runtime"
	client "github.com/containerd/containerd/runtime/v2/shim"
	"github.com/containerd/containerd/runtime/v2/task"
	"github.com/containerd/ttrpc"
	"github.com/gogo/protobuf/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func shimBinary(ctx context.Context, bundle *Bundle, runtime, containerdAddress string, containerdTTRPCAddress string, events *exchange.Exchange, rt *runtime.TaskList) *binary {
	return &binary{
		bundle:                 bundle,
		runtime:                runtime,
		containerdAddress:      containerdAddress,
		containerdTTRPCAddress: containerdTTRPCAddress,
		events:                 events,
		rtTasks:                rt,
	}
}

type binary struct {
	runtime                string
	containerdAddress      string
	containerdTTRPCAddress string
	bundle                 *Bundle
	events                 *exchange.Exchange
	rtTasks                *runtime.TaskList
}

func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_ *shim, err error) {
	args := []string{"-id", b.bundle.ID}
	if logrus.GetLevel() == logrus.DebugLevel {
		args = append(args, "-debug")
	}
	args = append(args, "start")

	cmd, err := client.Command(
		ctx,
		b.runtime,
		b.containerdAddress,
		b.containerdTTRPCAddress,
		b.bundle.Path,
		opts,
		args...,
	)
	if err != nil {
		return nil, err
	}
	f, err := openShimLog(context.Background(), b.bundle, client.AnonDialer)
	if err != nil {
		return nil, errors.Wrap(err, "open shim log pipe")
	}
	defer func() {
		if err != nil {
			f.Close()
		}
	}()
	// open the log pipe and block until the writer is ready
	// this helps with synchronization of the shim
	// copy the shim's logs to containerd's output
	go func() {
		defer f.Close()
		_, err := io.Copy(os.Stderr, f)
		err = checkCopyShimLogError(ctx, err)
		if err != nil {
			log.G(ctx).WithError(err).Error("copy shim log")
		}
	}()
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, errors.Wrapf(err, "%s", out)
	}
	address := strings.TrimSpace(string(out))
	conn, err := client.Connect(address, client.AnonDialer)
	if err != nil {
		return nil, err
	}
	client := ttrpc.NewClient(conn, ttrpc.WithOnClose(onClose))
	return &shim{
		bundle:  b.bundle,
		client:  client,
		task:    task.NewTaskClient(client),
		events:  b.events,
		rtTasks: b.rtTasks,
	}, nil
}

func (b *binary) Delete(ctx context.Context) (*runtime.Exit, error) {
	log.G(ctx).Info("cleaning up dead shim")

	// Windows cannot delete the current working directory while an
	// executable is in use with it. For the cleanup case we invoke with the
	// default work dir and forward the bundle path on the cmdline.
	var bundlePath string
	if gruntime.GOOS != "windows" {
		bundlePath = b.bundle.Path
	}

	cmd, err := client.Command(ctx,
		b.runtime,
		b.containerdAddress,
		b.containerdTTRPCAddress,
		bundlePath,
		nil,
		"-id", b.bundle.ID,
		"-bundle", b.bundle.Path,
		"delete")
	if err != nil {
		return nil, err
	}
	var (
		out  = bytes.NewBuffer(nil)
		errb = bytes.NewBuffer(nil)
	)
	cmd.Stdout = out
	cmd.Stderr = errb
	if err := cmd.Run(); err != nil {
		return nil, errors.Wrapf(err, "%s", errb.String())
	}
	s := errb.String()
	if s != "" {
		log.G(ctx).Warnf("cleanup warnings %s", s)
	}
	var response task.DeleteResponse
	if err := response.Unmarshal(out.Bytes()); err != nil {
		return nil, err
	}
	if err := b.bundle.Delete(); err != nil {
		return nil, err
	}
	return &runtime.Exit{
		Status:    response.ExitStatus,
		Timestamp: response.ExitedAt,
		Pid:       response.Pid,
	}, nil
}
<commit_msg>v2: Fix missing ns when openShimLog on windows<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package v2

import (
	"bytes"
	"context"
	"io"
	"os"
	gruntime "runtime"
	"strings"

	"github.com/containerd/containerd/events/exchange"
	"github.com/containerd/containerd/log"
	"github.com/containerd/containerd/namespaces"
	"github.com/containerd/containerd/runtime"
	client "github.com/containerd/containerd/runtime/v2/shim"
	"github.com/containerd/containerd/runtime/v2/task"
	"github.com/containerd/ttrpc"
	"github.com/gogo/protobuf/types"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func shimBinary(ctx context.Context, bundle *Bundle, runtime, containerdAddress string, containerdTTRPCAddress string, events *exchange.Exchange, rt *runtime.TaskList) *binary {
	return &binary{
		bundle:                 bundle,
		runtime:                runtime,
		containerdAddress:      containerdAddress,
		containerdTTRPCAddress: containerdTTRPCAddress,
		events:                 events,
		rtTasks:                rt,
	}
}

type binary struct {
	runtime                string
	containerdAddress      string
	containerdTTRPCAddress string
	bundle                 *Bundle
	events                 *exchange.Exchange
	rtTasks                *runtime.TaskList
}

func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_ *shim, err error) {
	args := []string{"-id", b.bundle.ID}
	if logrus.GetLevel() == logrus.DebugLevel {
		args = append(args, "-debug")
	}
	args = append(args, "start")

	cmd, err := client.Command(
		ctx,
		b.runtime,
		b.containerdAddress,
		b.containerdTTRPCAddress,
		b.bundle.Path,
		opts,
		args...,
	)
	if err != nil {
		return nil, err
	}
	// Windows needs a namespace when openShimLog
	ns, _ := namespaces.Namespace(ctx)
	f, err := openShimLog(namespaces.WithNamespace(context.Background(), ns), b.bundle, client.AnonDialer)
	if err != nil {
		return nil, errors.Wrap(err, "open shim log pipe")
	}
	defer func() {
		if err != nil {
			f.Close()
		}
	}()
	// open the log pipe and block until the writer is ready
	// this helps with synchronization of the shim
	// copy the shim's logs to containerd's output
	go func() {
		defer f.Close()
		_, err := io.Copy(os.Stderr, f)
		err = checkCopyShimLogError(ctx, err)
		if err != nil {
			log.G(ctx).WithError(err).Error("copy shim log")
		}
	}()
	out, err := cmd.CombinedOutput()
	if err != nil {
		return nil, errors.Wrapf(err, "%s", out)
	}
	address := strings.TrimSpace(string(out))
	conn, err := client.Connect(address, client.AnonDialer)
	if err != nil {
		return nil, err
	}
	client := ttrpc.NewClient(conn, ttrpc.WithOnClose(onClose))
	return &shim{
		bundle:  b.bundle,
		client:  client,
		task:    task.NewTaskClient(client),
		events:  b.events,
		rtTasks: b.rtTasks,
	}, nil
}

func (b *binary) Delete(ctx context.Context) (*runtime.Exit, error) {
	log.G(ctx).Info("cleaning up dead shim")

	// Windows cannot delete the current working directory while an
	// executable is in use with it. For the cleanup case we invoke with the
	// default work dir and forward the bundle path on the cmdline.
	var bundlePath string
	if gruntime.GOOS != "windows" {
		bundlePath = b.bundle.Path
	}

	cmd, err := client.Command(ctx,
		b.runtime,
		b.containerdAddress,
		b.containerdTTRPCAddress,
		bundlePath,
		nil,
		"-id", b.bundle.ID,
		"-bundle", b.bundle.Path,
		"delete")
	if err != nil {
		return nil, err
	}
	var (
		out  = bytes.NewBuffer(nil)
		errb = bytes.NewBuffer(nil)
	)
	cmd.Stdout = out
	cmd.Stderr = errb
	if err := cmd.Run(); err != nil {
		return nil, errors.Wrapf(err, "%s", errb.String())
	}
	s := errb.String()
	if s != "" {
		log.G(ctx).Warnf("cleanup warnings %s", s)
	}
	var response task.DeleteResponse
	if err := response.Unmarshal(out.Bytes()); err != nil {
		return nil, err
	}
	if err := b.bundle.Delete(); err != nil {
		return nil, err
	}
	return &runtime.Exit{
		Status:    response.ExitStatus,
		Timestamp: response.ExitedAt,
		Pid:       response.Pid,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dmltest

import (
	"context"
	"database/sql"
	"io"
	"os"
	"testing"

	"github.com/DATA-DOG/go-sqlmock"
	"github.com/corestoreio/errors"
	"github.com/corestoreio/pkg/sql/dml"
	"github.com/fatih/color"
)

// EnvDSN is the name of the environment variable
const EnvDSN = "CS_DSN"

func getDSN(env string) (string, error) {
	dsn := os.Getenv(env)
	if dsn == "" {
		return "", errors.NotFound.Newf("DSN in environment variable %q not found.", EnvDSN)
	}
	return dsn, nil
}

// MustGetDSN returns the data source name from an environment variable or
// panics on error.
func MustGetDSN(t testing.TB) string {
	d, err := getDSN(EnvDSN)
	if err != nil {
		t.Skip(color.MagentaString("%s", err))
	}
	return d
}

// MustConnectDB is a helper function that creates a new database connection
// using a DSN from an environment variable found in the constant csdb.EnvDSN.
// If the DSN environment variable has not been set it skips the test.
// It creates a random database if the DSN database name is the word "random".
func MustConnectDB(t testing.TB, opts ...dml.ConnPoolOption) *dml.ConnPool {
	t.Helper()
	if _, err := getDSN(EnvDSN); errors.NotFound.Match(err) {
		t.Skip(color.MagentaString("%s", err))
	}

	cfg := []dml.ConnPoolOption{
		dml.WithDSN(MustGetDSN(t)),
		dml.WithCreateDatabase(context.Background(), ""), // empty DB name gets derived from the DSN
	}
	return dml.MustConnectAndVerify(append(cfg, opts...)...)
}

// Close for usage in conjunction with defer.
// 		defer dmltest.Close(t, db)
func Close(t testing.TB, c io.Closer) {
	t.Helper()
	if err := c.Close(); err != nil {
		t.Errorf("%+v", err)
	}
}

// MockDB creates a mocked database connection. Fatals on error.
func MockDB(t testing.TB, opts ...dml.ConnPoolOption) (*dml.ConnPool, sqlmock.Sqlmock) {
	if t != nil { // t can be nil in Example functions
		t.Helper()
	}
	db, sm, err := sqlmock.New()
	FatalIfError(t, err)
	cfg := []dml.ConnPoolOption{dml.WithDB(db)}
	dbc, err := dml.NewConnPool(append(cfg, opts...)...)
	FatalIfError(t, err)
	return dbc, sm
}

// MockClose for usage in conjunction with defer.
// 		defer dmltest.MockClose(t, db, dbMock)
func MockClose(t testing.TB, c io.Closer, m sqlmock.Sqlmock) {
	if t != nil { // t can be nil in Example functions
		t.Helper()
	}
	m.ExpectClose()
	FatalIfError(t, c.Close())
	FatalIfError(t, m.ExpectationsWereMet())
}

// FatalIfError fails the tests if an unexpected error occurred. If the error is
// gift wrapped, it prints the location. If `t` is nil, this function panics.
func FatalIfError(t testing.TB, err error) {
	if err != nil {
		if t != nil {
			t.Fatalf("%+v", err)
		} else {
			panic(err)
		}
	}
}

// CheckLastInsertID returns a function which accepts the return result from
// Exec*() and returns itself the last_insert_id or emits an error.
func CheckLastInsertID(t interface {
	Errorf(format string, args ...interface{})
}, msg ...string) func(sql.Result, error) int64 {
	return func(res sql.Result, err error) int64 {
		if err != nil {
			if len(msg) == 1 {
				t.Errorf("%q: %+v", msg[0], err)
			} else {
				t.Errorf("%+v", err)
			}
			return 0
		}
		lid, err := res.LastInsertId()
		if err != nil {
			if len(msg) == 1 {
				t.Errorf("%q: %+v", msg[0], err)
			} else {
				t.Errorf("%+v", err)
			}
			return 0
		}
		if lid < 0 {
			t.Errorf("Expecting Last Insert ID to be greater than zero, got %d", lid)
		}
		return lid
	}
}
<commit_msg>sql/dmltest: add MockDBCallBack<commit_after>// Copyright 2015-present, Cyrill @ Schumacher.fm and the CoreStore contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dmltest

import (
	"context"
	"database/sql"
	"io"
	"os"
	"testing"

	"github.com/DATA-DOG/go-sqlmock"
	"github.com/corestoreio/errors"
	"github.com/corestoreio/pkg/sql/dml"
	"github.com/fatih/color"
)

// EnvDSN is the name of the environment variable
const EnvDSN = "CS_DSN"

func getDSN(env string) (string, error) {
	dsn := os.Getenv(env)
	if dsn == "" {
		return "", errors.NotFound.Newf("DSN in environment variable %q not found.", EnvDSN)
	}
	return dsn, nil
}

// MustGetDSN returns the data source name from an environment variable or
// panics on error.
func MustGetDSN(t testing.TB) string {
	d, err := getDSN(EnvDSN)
	if err != nil {
		t.Skip(color.MagentaString("%s", err))
	}
	return d
}

// MustConnectDB is a helper function that creates a new database connection
// using a DSN from an environment variable found in the constant csdb.EnvDSN.
// If the DSN environment variable has not been set it skips the test.
// It creates a random database if the DSN database name is the word "random".
func MustConnectDB(t testing.TB, opts ...dml.ConnPoolOption) *dml.ConnPool {
	t.Helper()
	if _, err := getDSN(EnvDSN); errors.NotFound.Match(err) {
		t.Skip(color.MagentaString("%s", err))
	}

	cfg := []dml.ConnPoolOption{
		dml.WithDSN(MustGetDSN(t)),
		dml.WithCreateDatabase(context.Background(), ""), // empty DB name gets derived from the DSN
	}
	return dml.MustConnectAndVerify(append(cfg, opts...)...)
}

// Close for usage in conjunction with defer.
// 		defer dmltest.Close(t, db)
func Close(t testing.TB, c io.Closer) {
	t.Helper()
	if err := c.Close(); err != nil {
		t.Errorf("%+v", err)
	}
}

// MockDB creates a mocked database connection. Fatals on error.
func MockDB(t testing.TB, opts ...dml.ConnPoolOption) (*dml.ConnPool, sqlmock.Sqlmock) {
	if t != nil { // t can be nil in Example functions
		t.Helper()
	}
	db, sm, err := sqlmock.New()
	FatalIfError(t, err)
	cfg := []dml.ConnPoolOption{dml.WithDB(db)}
	dbc, err := dml.NewConnPool(append(cfg, opts...)...)
	FatalIfError(t, err)
	return dbc, sm
}

// MockDBCallBack same as MockDB but allows to add expectations early to the
// mock.
func MockDBCallBack(t testing.TB, mockCB func(sqlmock.Sqlmock), opts ...dml.ConnPoolOption) (*dml.ConnPool, sqlmock.Sqlmock) {
	if t != nil { // t can be nil in Example functions
		t.Helper()
	}
	db, sm, err := sqlmock.New()
	FatalIfError(t, err)
	if mockCB != nil {
		mockCB(sm)
	}
	cfg := []dml.ConnPoolOption{dml.WithDB(db)}
	dbc, err := dml.NewConnPool(append(cfg, opts...)...)
	FatalIfError(t, err)
	return dbc, sm
}

// MockClose for usage in conjunction with defer.
// 		defer dmltest.MockClose(t, db, dbMock)
func MockClose(t testing.TB, c io.Closer, m sqlmock.Sqlmock) {
	if t != nil { // t can be nil in Example functions
		t.Helper()
	}
	m.ExpectClose()
	FatalIfError(t, c.Close())
	FatalIfError(t, m.ExpectationsWereMet())
}

// FatalIfError fails the tests if an unexpected error occurred. If the error is
// gift wrapped, it prints the location. If `t` is nil, this function panics.
func FatalIfError(t testing.TB, err error) {
	if err != nil {
		if t != nil {
			t.Fatalf("%+v", err)
		} else {
			panic(err)
		}
	}
}

// CheckLastInsertID returns a function which accepts the return result from
// Exec*() and returns itself the last_insert_id or emits an error.
func CheckLastInsertID(t interface {
	Errorf(format string, args ...interface{})
}, msg ...string) func(sql.Result, error) int64 {
	return func(res sql.Result, err error) int64 {
		if err != nil {
			if len(msg) == 1 {
				t.Errorf("%q: %+v", msg[0], err)
			} else {
				t.Errorf("%+v", err)
			}
			return 0
		}
		lid, err := res.LastInsertId()
		if err != nil {
			if len(msg) == 1 {
				t.Errorf("%q: %+v", msg[0], err)
			} else {
				t.Errorf("%+v", err)
			}
			return 0
		}
		if lid < 0 {
			t.Errorf("Expecting Last Insert ID to be greater than zero, got %d", lid)
		}
		return lid
	}
}
<|endoftext|>


<commit_before>package headlessChrome

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/integrii/interactive"
)

// Debug enables debug output for this package to console
var Debug bool

// ChromePath is the command to execute chrome
var ChromePath = `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`

// Args are the args that will be used to start chrome
var Args = []string{
	"--headless",
	"--disable-gpu",
	"--repl",
	// "--dump-dom",
	// "--window-size=1024,768",
	// "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
	// "--verbose",
}

const expectedFirstLine = `Type a Javascript expression to evaluate or "quit" to exit.`
const promptPrefix = `>>>`

// OutputSanitizer puts output coming from the consolw that
// does not begin with the input prompt into the session
// output channel
func (cs *ChromeSession) OutputSanitizer() {
	for text := range cs.Session.Output {
		if !strings.HasPrefix(text, promptPrefix) {
			cs.Output <- text
		}
	}
}

// ChromeSession is an interactive console Session with a Chrome
// instance.
type ChromeSession struct {
	Session *interactive.Session
	Output  chan string
	Input   chan string
}

// Exit exits the running command out by ossuing a 'quit'
// to the chrome console
func (cs *ChromeSession) Exit() {
	cs.Session.Write(`quit`)
	cs.Session.Exit()
}

// Write writes to the Session
func (cs *ChromeSession) Write(s string) {
	if Debug {
		fmt.Println("Writing to console:")
		fmt.Println(s)
	}
	cs.Session.Write(s)
}

// OutputPrinter prints all outputs from the output channel to the cli
func (cs *ChromeSession) OutputPrinter() {
	for l := range cs.Session.Output {
		fmt.Println(l)
	}
}

// forceClose issues a force kill to the command
func (cs *ChromeSession) forceClose() {
	cs.Session.ForceClose()
}

// ClickSelector calls a click() on the supplied selector
func (cs *ChromeSession) ClickSelector(s string) {
	cs.Write(`document.querySelector("` + s + `").click()`)
}

// ClickItemWithInnerHTML clicks an item that has the matching inner html
func (cs *ChromeSession) ClickItemWithInnerHTML(elementType string, s string, itemIndex int) {
	cs.Write(`$("` + elementType + `").filter(function(idx) { return this.innerHTML.indexOf("` + s + `") == 0; })[` + strconv.Itoa(itemIndex) + `].click()`)
}

// GetItemWithInnerHTML fetches the item with the specified innerHTML content
func (cs *ChromeSession) GetItemWithInnerHTML(elementType string, s string, itemIndex int) {
	cs.Write(`$("` + elementType + `").filter(function(idx) { return this.innerHTML.indexOf("` + s + `") == 0; })[` + strconv.Itoa(itemIndex) + `]`)
}

// GetContentOfItemWithClasses fetches the content of the element with the specified classes
func (cs *ChromeSession) GetContentOfItemWithClasses(classes string, itemIndex int) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].innerHTML`)
}

// GetContentOfItemWithSelector gets the content of an element with the specified selector
func (cs *ChromeSession) GetContentOfItemWithSelector(selector string) {
	cs.Write(`document.querySelector("` + selector + `").innerHTML()`)
}

// ClickItemWithClasses clicks on the first item it finds with the provided classes.
// Multiple classes are separated by spaces
func (cs *ChromeSession) ClickItemWithClasses(classes string, itemIndex int) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].click()`)
}

// SetTextByID sets the text on the div with the specified id
func (cs *ChromeSession) SetTextByID(id string, text string) {
	cs.Write(`document.getElementById("` + id + `").innerHTML = "` + text + `"`)
}

// ClickItemWithID clicks an item with the specified id
func (cs *ChromeSession) ClickItemWithID(id string) {
	cs.Write(`document.getElementById("` + id + `").click()`)
}

// SetTextByClasses sets the text on the div with the specified id
func (cs *ChromeSession) SetTextByClasses(classes string, itemIndex int, text string) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].innerHTML = "` + text + `"`)
}

// SetInputTextByClasses sets the input text for an input field
func (cs *ChromeSession) SetInputTextByClasses(classes string, itemIndex int, text string) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].value = "` + text + `"`)
}

// NewBrowser starts a new chrome headless Session.
func NewBrowser(url string) (*ChromeSession, error) {
	var err error

	chromeSession := ChromeSession{}
	chromeSession.Output = make(chan string, 5000)

	// add url as last arg and create new Session
	args := append(Args, url)
	chromeSession.Session, err = interactive.NewSession(ChromePath, args)

	// map output and input channels for easy use
	chromeSession.Input = chromeSession.Session.Input

	go chromeSession.OutputSanitizer()

	firstLine := <-chromeSession.Output
	if !strings.Contains(firstLine, expectedFirstLine) {
		log.Println("ERROR: Unespected first line when initializing headless Chrome console:", firstLine)
	}

	return &chromeSession, err
}
<commit_msg>improved javascript operations<commit_after>package headlessChrome

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/integrii/interactive"
)

// Debug enables debug output for this package to console
var Debug bool

// ChromePath is the command to execute chrome
var ChromePath = `/Applications/Google Chrome.app/Contents/MacOS/Google Chrome`

// Args are the args that will be used to start chrome
var Args = []string{
	"--headless",
	"--disable-gpu",
	"--repl",
	// "--dump-dom",
	// "--window-size=1024,768",
	// "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
	// "--verbose",
}

const expectedFirstLine = `Type a Javascript expression to evaluate or "quit" to exit.`
const promptPrefix = `>>>`

// OutputSanitizer puts output coming from the consolw that
// does not begin with the input prompt into the session
// output channel
func (cs *ChromeSession) OutputSanitizer() {
	for text := range cs.Session.Output {
		if !strings.HasPrefix(text, promptPrefix) {
			cs.Output <- text
		}
	}
}

// ChromeSession is an interactive console Session with a Chrome
// instance.
type ChromeSession struct {
	Session *interactive.Session
	Output  chan string
	Input   chan string
}

// Exit exits the running command out by ossuing a 'quit'
// to the chrome console
func (cs *ChromeSession) Exit() {
	cs.Session.Write(`quit`)
	cs.Session.Exit()
}

// Write writes to the Session
func (cs *ChromeSession) Write(s string) {
	if Debug {
		fmt.Println("Writing to console:")
		fmt.Println(s)
	}
	cs.Session.Write(s)
}

// OutputPrinter prints all outputs from the output channel to the cli
func (cs *ChromeSession) OutputPrinter() {
	for l := range cs.Session.Output {
		fmt.Println(l)
	}
}

// forceClose issues a force kill to the command
func (cs *ChromeSession) forceClose() {
	cs.Session.ForceClose()
}

// ClickSelector calls a click() on the supplied selector
func (cs *ChromeSession) ClickSelector(s string) {
	cs.Write(`document.querySelector("` + s + `").click()`)
}

// ClickItemWithInnerHTML clicks an item that has the matching inner html
func (cs *ChromeSession) ClickItemWithInnerHTML(elementType string, s string, itemIndex int) {
	cs.Write(`var x = $("` + elementType + `").filter(function(idx) { return this.innerHTML.indexOf("` + s + `") == 0; });x[` + strconv.Itoa(itemIndex) + `].click()`)
}

// GetItemWithInnerHTML fetches the item with the specified innerHTML content
func (cs *ChromeSession) GetItemWithInnerHTML(elementType string, s string, itemIndex int) {
	cs.Write(`var x = $("` + elementType + `").filter(function(idx) { return this.innerHTML.indexOf("` + s + `") == 0; });x[` + strconv.Itoa(itemIndex) + `]`)
}

// GetContentOfItemWithClasses fetches the content of the element with the specified classes
func (cs *ChromeSession) GetContentOfItemWithClasses(classes string, itemIndex int) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].innerHTML`)
}

// GetContentOfItemWithSelector gets the content of an element with the specified selector
func (cs *ChromeSession) GetContentOfItemWithSelector(selector string) {
	cs.Write(`document.querySelector("` + selector + `").innerHTML()`)
}

// ClickItemWithClasses clicks on the first item it finds with the provided classes.
// Multiple classes are separated by spaces
func (cs *ChromeSession) ClickItemWithClasses(classes string, itemIndex int) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].click()`)
}

// SetTextByID sets the text on the div with the specified id
func (cs *ChromeSession) SetTextByID(id string, text string) {
	cs.Write(`document.getElementById("` + id + `").innerHTML = "` + text + `"`)
}

// ClickItemWithID clicks an item with the specified id
func (cs *ChromeSession) ClickItemWithID(id string) {
	cs.Write(`document.getElementById("` + id + `").click()`)
}

// SetTextByClasses sets the text on the div with the specified id
func (cs *ChromeSession) SetTextByClasses(classes string, itemIndex int, text string) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].innerHTML = "` + text + `"`)
}

// SetInputTextByClasses sets the input text for an input field
func (cs *ChromeSession) SetInputTextByClasses(classes string, itemIndex int, text string) {
	cs.Write(`document.getElementsByClassName("` + classes + `")[` + strconv.Itoa(itemIndex) + `].value = "` + text + `"`)
}

// NewBrowser starts a new chrome headless Session.
func NewBrowser(url string) (*ChromeSession, error) {
	var err error

	chromeSession := ChromeSession{}
	chromeSession.Output = make(chan string, 5000)

	// add url as last arg and create new Session
	args := append(Args, url)
	chromeSession.Session, err = interactive.NewSession(ChromePath, args)

	// map output and input channels for easy use
	chromeSession.Input = chromeSession.Session.Input

	go chromeSession.OutputSanitizer()

	firstLine := <-chromeSession.Output
	if !strings.Contains(firstLine, expectedFirstLine) {
		log.Println("ERROR: Unespected first line when initializing headless Chrome console:", firstLine)
	}

	return &chromeSession, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"encoding/json"

	"github.com/codegangsta/cli"
)

func doApply(c *cli.Context) error {
	inputFilename := c.String("input-filename")
	filterStatusCodeOk := c.Bool("filter-status-code-ok")
	jsonOutput := c.Bool("json-output")
	subsetSelection := c.Bool("subset-selection")
	sizeConstraint := c.Int("size-constraint")
	alpha := c.Float64("alpha")
	r := c.Float64("r")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "apply")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	cacheFilename := CacheFilename

	cache, err := LoadCache(cacheFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	examples, err := ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	AttachMetaData(cache, examples)
	if filterStatusCodeOk {
		examples = FilterStatusCodeOkExamples(examples)
	}
	model := TrainedModel(examples)

	result := Examples{}
	for _, e := range FilterUnlabeledExamples(examples) {
		e.Score = model.PredictScore(e.Fv)
		e.Title = strings.Replace(e.Title, "\n", " ", -1)
		if e.Score > 0.0 {
			result = append(result, e)
		}
	}

	if subsetSelection {
		result = SelectSubExamplesBySubModular(model, result, sizeConstraint, alpha, r)
	}

	for _, e := range result {
		e.Score = model.PredictScore(e.Fv)
		e.Title = strings.Replace(e.Title, "\n", " ", -1)
		if jsonOutput {
			b, err := json.Marshal(e)
			if err != nil {
				return err
			}
			fmt.Println(string(b))
		} else {
			fmt.Println(fmt.Sprintf("%0.03f\t%s", e.Score, e.Url))
		}
	}

	cache.Save(cacheFilename)
	return nil
}

var commandApply = cli.Command{
	Name:  "apply",
	Usage: "apply classifier to unlabeled examples",
	Description: `
Apply classifier to unlabeled examples, and print a pair of score and url.
`,
	Action: doApply,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
		cli.BoolFlag{Name: "filter-status-code-ok", Usage: "Use only examples with status code = 200"},
		cli.BoolFlag{Name: "json-output", Usage: "Make output with json format or not (tsv format)."},
		cli.BoolFlag{Name: "subset-selection", Usage: "Use subset selection algorithm (maximizing submodular function) to filter entries"},
		cli.Int64Flag{Name: "size-constraint", Value: 10, Usage: "Budget constraint. Max number of entries to be contained"},
		cli.Float64Flag{Name: "alpha", Value: 1.0},
		cli.Float64Flag{Name: "r", Value: 1.0, Usage: "Scaling factor for number of words"},
	},
}
<commit_msg>足切りの閾値を設定<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"encoding/json"

	"github.com/codegangsta/cli"
)

func doApply(c *cli.Context) error {
	inputFilename := c.String("input-filename")
	filterStatusCodeOk := c.Bool("filter-status-code-ok")
	jsonOutput := c.Bool("json-output")
	subsetSelection := c.Bool("subset-selection")
	sizeConstraint := c.Int("size-constraint")
	alpha := c.Float64("alpha")
	r := c.Float64("r")
	scoreThreshold := c.Float64("score-threshold")

	if inputFilename == "" {
		_ = cli.ShowCommandHelp(c, "apply")
		return cli.NewExitError("`input-filename` is a required field.", 1)
	}

	cacheFilename := CacheFilename

	cache, err := LoadCache(cacheFilename)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
	}

	examples, err := ReadExamples(inputFilename)
	if err != nil {
		return err
	}

	AttachMetaData(cache, examples)
	if filterStatusCodeOk {
		examples = FilterStatusCodeOkExamples(examples)
	}
	model := TrainedModel(examples)

	result := Examples{}
	for _, e := range FilterUnlabeledExamples(examples) {
		e.Score = model.PredictScore(e.Fv)
		e.Title = strings.Replace(e.Title, "\n", " ", -1)
		if e.Score > scoreThreshold {
			result = append(result, e)
		}
	}

	if subsetSelection {
		result = SelectSubExamplesBySubModular(model, result, sizeConstraint, alpha, r)
	}

	for _, e := range result {
		e.Score = model.PredictScore(e.Fv)
		e.Title = strings.Replace(e.Title, "\n", " ", -1)
		if jsonOutput {
			b, err := json.Marshal(e)
			if err != nil {
				return err
			}
			fmt.Println(string(b))
		} else {
			fmt.Println(fmt.Sprintf("%0.03f\t%s", e.Score, e.Url))
		}
	}

	cache.Save(cacheFilename)
	return nil
}

var commandApply = cli.Command{
	Name:  "apply",
	Usage: "apply classifier to unlabeled examples",
	Description: `
Apply classifier to unlabeled examples, and print a pair of score and url.
`,
	Action: doApply,
	Flags: []cli.Flag{
		cli.StringFlag{Name: "input-filename"},
		cli.BoolFlag{Name: "filter-status-code-ok", Usage: "Use only examples with status code = 200"},
		cli.BoolFlag{Name: "json-output", Usage: "Make output with json format or not (tsv format)."},
		cli.BoolFlag{Name: "subset-selection", Usage: "Use subset selection algorithm (maximizing submodular function) to filter entries"},
		cli.Int64Flag{Name: "size-constraint", Value: 10, Usage: "Budget constraint. Max number of entries to be contained"},
		cli.Float64Flag{Name: "alpha", Value: 1.0},
		cli.Float64Flag{Name: "r", Value: 1.0, Usage: "Scaling factor for number of words"},
		cli.Float64Flag{Name: "score-threshold", Value: 0.0},
	},
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"

	"github.com/docker/docker/api/client/bundlefile"
	"github.com/docker/docker/api/client/stack"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/network"
	"github.com/fatih/color"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
)

func apply(c *cli.Context) error {

	stacks, err := getStacks(c)
	if err != nil {
		return err
	}

	swarm, swarmErr := client.NewEnvClient()
	if swarmErr != nil {
		return cli.NewExitError(swarmErr.Error(), 3)
	}

	target := c.StringSlice("target")
	targetMap := map[string]bool{}

	for _, name := range target {
		targetMap[name] = true
	}

	for _, stack := range stacks {
		filter := filters.NewArgs()
		filter.Add("label", "com.docker.stack.namespace="+stack.Name)
		services, servicesErr := swarm.ServiceList(context.Background(), types.ServiceListOptions{Filter: filter})
		if servicesErr != nil {
			return cli.NewExitError(servicesErr.Error(), 3)
		}

		expected := getBundleServicesSpec(stack.Bundle, stack.Name)
		current := getSwarmServicesSpecForStack(services)

		err = updateNetworks(context.Background(), swarm, getUniqueNetworkNames(stack.Bundle.Services), stack.Name)

		if err != nil {
			log.Fatal("Error updating networks when creating services", err)
		}

		cyan := color.New(color.FgCyan)
		sp := NewServicePrinter(ioutil.Discard, false)
		for name, expectedService := range expected {
			if _, found := targetMap[expectedService.Spec.Name]; len(targetMap) == 0 || found {
				if currentService, found := current[name]; found {
					// service exists, need to update
					if sp.PrintServiceSpecDiff(currentService.Spec, expectedService.Spec) {
						cyan.Printf("Updating service %s\n", name)
						servicesErr := swarm.ServiceUpdate(context.Background(), currentService.ID, currentService.Version, expectedService.Spec, types.ServiceUpdateOptions{})
						if servicesErr != nil {
							return cli.NewExitError(servicesErr.Error(), 3)
						}
					}
				} else {
					// service doesn't exist, need to create a new one
					cyan.Printf("Creating service %s\n", name)
					_, servicesErr := swarm.ServiceCreate(context.Background(), expectedService.Spec, types.ServiceCreateOptions{})
					if servicesErr != nil {
						return cli.NewExitError(servicesErr.Error(), 3)
					}
				}
			}
		}
		for name, cs := range current {
			if _, found := targetMap[cs.Spec.Name]; len(targetMap) == 0 || found {
				if _, found := expected[name]; !found {
					// service exists but it's not expected, need to delete it
					cyan.Printf("Removing service %s\n", name)
					servicesErr := swarm.ServiceRemove(context.Background(), name)
					if servicesErr != nil {
						return cli.NewExitError(servicesErr.Error(), 3)
					}
				}
			}
		}
	}

	return nil
}

func updateNetworks(
	ctx context.Context,
	cli *client.Client,
	networks []string,
	namespace string,
) error {

	existingNetworks, err := stack.GetNetworks(ctx, cli, namespace)
	if err != nil {
		return err
	}

	existingNetworkMap := make(map[string]types.NetworkResource)
	for _, network := range existingNetworks {
		existingNetworkMap[network.Name] = network
	}

	createOpts := types.NetworkCreate{
		Labels: stack.GetStackLabels(namespace, nil),
		Driver: "overlay",
		IPAM:   network.IPAM{Driver: "default"},
	}

	for _, internalName := range networks {
		name := fmt.Sprintf("%s_%s", namespace, internalName)

		if _, exists := existingNetworkMap[name]; exists {
			continue
		}

		fmt.Printf("Creating network %s\n", name)
		if _, err := cli.NetworkCreate(ctx, name, createOpts); err != nil {
			return err
		}
	}
	return nil
}

func getUniqueNetworkNames(services map[string]bundlefile.Service) []string {
	networkSet := make(map[string]bool)
	for _, service := range services {
		for _, network := range service.Networks {
			networkSet[network] = true
		}
	}

	networks := []string{}
	for network := range networkSet {
		networks = append(networks, network)
	}
	return networks
}
<commit_msg>Only apply modified services<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"

	"github.com/docker/docker/api/client/bundlefile"
	"github.com/docker/docker/api/client/stack"
	"github.com/docker/engine-api/client"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/network"
	"github.com/fatih/color"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
)

func apply(c *cli.Context) error {

	stacks, err := getStacks(c)
	if err != nil {
		return err
	}

	swarm, swarmErr := client.NewEnvClient()
	if swarmErr != nil {
		return cli.NewExitError(swarmErr.Error(), 3)
	}

	target := c.StringSlice("target")
	targetMap := map[string]bool{}

	for _, name := range target {
		targetMap[name] = true
	}

	for _, stack := range stacks {
		filter := filters.NewArgs()
		filter.Add("label", "com.docker.stack.namespace="+stack.Name)
		services, servicesErr := swarm.ServiceList(context.Background(), types.ServiceListOptions{Filter: filter})
		if servicesErr != nil {
			return cli.NewExitError(servicesErr.Error(), 3)
		}

		expected := getBundleServicesSpec(stack.Bundle, stack.Name)
		translateNetworkToIds(&expected, swarm, stack.Name)
		current := getSwarmServicesSpecForStack(services)

		err = updateNetworks(context.Background(), swarm, getUniqueNetworkNames(stack.Bundle.Services), stack.Name)

		if err != nil {
			log.Fatal("Error updating networks when creating services", err)
		}

		cyan := color.New(color.FgCyan)
		sp := NewServicePrinter(ioutil.Discard, false)
		for name, expectedService := range expected {
			if _, found := targetMap[expectedService.Spec.Name]; len(targetMap) == 0 || found {
				if currentService, found := current[name]; found {
					if sp.PrintServiceSpecDiff(currentService.Spec, expectedService.Spec) {
						cyan.Printf("Updating service %s\n", name)
						servicesErr := swarm.ServiceUpdate(context.Background(), currentService.ID, currentService.Version, expectedService.Spec, types.ServiceUpdateOptions{})
						if servicesErr != nil {
							return cli.NewExitError(servicesErr.Error(), 3)
						}
					}
				} else {
					// service doesn't exist, need to create a new one
					cyan.Printf("Creating service %s\n", name)
					_, servicesErr := swarm.ServiceCreate(context.Background(), expectedService.Spec, types.ServiceCreateOptions{})
					if servicesErr != nil {
						return cli.NewExitError(servicesErr.Error(), 3)
					}
				}
			}
		}
		for name, cs := range current {
			if _, found := targetMap[cs.Spec.Name]; len(targetMap) == 0 || found {
				if _, found := expected[name]; !found {
					// service exists but it's not expected, need to delete it
					cyan.Printf("Removing service %s\n", name)
					servicesErr := swarm.ServiceRemove(context.Background(), name)
					if servicesErr != nil {
						return cli.NewExitError(servicesErr.Error(), 3)
					}
				}
			}
		}
	}

	return nil
}

func updateNetworks(
	ctx context.Context,
	cli *client.Client,
	networks []string,
	namespace string,
) error {

	existingNetworks, err := stack.GetNetworks(ctx, cli, namespace)
	if err != nil {
		return err
	}

	existingNetworkMap := make(map[string]types.NetworkResource)
	for _, network := range existingNetworks {
		existingNetworkMap[network.Name] = network
	}

	createOpts := types.NetworkCreate{
		Labels: stack.GetStackLabels(namespace, nil),
		Driver: "overlay",
		IPAM:   network.IPAM{Driver: "default"},
	}

	for _, internalName := range networks {
		name := fmt.Sprintf("%s_%s", namespace, internalName)

		if _, exists := existingNetworkMap[name]; exists {
			continue
		}

		fmt.Printf("Creating network %s\n", name)
		if _, err := cli.NetworkCreate(ctx, name, createOpts); err != nil {
			return err
		}
	}
	return nil
}

func getUniqueNetworkNames(services map[string]bundlefile.Service) []string {
	networkSet := make(map[string]bool)
	for _, service := range services {
		for _, network := range service.Networks {
			networkSet[network] = true
		}
	}

	networks := []string{}
	for network := range networkSet {
		networks = append(networks, network)
	}
	return networks
}
<|endoftext|>