<commit_before>package servergroup

import (
	"fmt"
	"time"

	config_util "github.com/prometheus/common/config"
	"github.com/prometheus/common/model"
	sd_config "github.com/prometheus/prometheus/discovery/config"
	"github.com/prometheus/prometheus/pkg/relabel"
)

var (
	// DefaultConfig is the Default base promxy configuration
	DefaultConfig = Config{
		AntiAffinity:   time.Second * 10,
		Scheme:         "http",
		RemoteReadPath: "api/v1/read",
		HTTPConfig: HTTPClientConfig{
			DialTimeout: time.Millisecond * 2000, // Default dial timeout of 200ms
		},
	}
)

// Config is the configuration for a ServerGroup that promxy will talk to.
// This is where the vast majority of options exist.
type Config struct {
	// RemoteRead directs promxy to load RAW data (meaning matrix selectors such as `foo[1h]`)
	// through the RemoteRead API on prom.
	// Pros:
	//  - StaleNaNs work
	//  - ~2x faster (in my local testing, more so if you are using default JSON marshaler in prom)
	//
	// Cons:
	//  - proto marshaling prom side doesn't stream, so the data being sent
	//      over the wire will be 2x its size in memory on the remote prom host.
	//  - "experimental" API (according to docs) -- meaning this might break
	//      without much (if any) warning
	//
	// Upstream prom added a StaleNan to determine if a given timeseries has gone
	// NaN -- the problem being that for range vectors they filter out all "stale" samples
	// meaning that it isn't possible to get a "raw" dump of data through the query/query_range v1 API
	// The only option that exists in reality is the "remote read" API -- which suffers
	// from the same memory-balooning problems that the HTTP+JSON API originally had.
	// It has **less** of a problem (its 2x memory instead of 14x) so it is a viable option.
	RemoteRead bool `yaml:"remote_read"`
	// RemoteReadPath sets the remote read path for the hosts in this servergroup
	RemoteReadPath string `yaml:"remote_read_path"`
	// HTTP client config for promxy to use when connecting to the various server_groups
	// this is the same config as prometheus
	HTTPConfig HTTPClientConfig `yaml:"http_client"`
	// Scheme defines how promxy talks to this server group (http, https, etc.)
	Scheme string `yaml:"scheme"`
	// Labels is a set of labels that will be added to all metrics retrieved
	// from this server group
	Labels model.LabelSet `json:"labels"`
	// RelabelConfigs are similar in function and identical in configuration as prometheus'
	// relabel config for scrape jobs. The difference here being that the source labels
	// you can pull from are from the downstream servergroup target and the labels you are
	// relabeling are that of the timeseries being returned. This allows you to mutate the
	// labelsets returned by that target at runtime.
	// To further illustrate the difference we'll look at an example:
	//
	//      relabel_configs:
	//    - source_labels: [__meta_consul_tags]
	//      regex: '.*,prod,.*'
	//      action: keep
	//    - source_labels: [__meta_consul_dc]
	//      regex: '.+'
	//      action: replace
	//      target_label: datacenter
	//
	// If we saw this in a scrape-config we would expect:
	//   (1) the scrape would only target hosts with a prod consul label
	//   (2) it would add a label to all returned series of datacenter with the value set to whatever the value of __meat_consul_dc was.
	//
	// If we saw this same config in promxy (pointing at prometheus hosts instead of some exporter), we'd expect a similar behavior:
	//   (1) only targets with the prod consul label would be included in the servergroup
	//   (2) it would add a label to all returned series of this servergroup of datacenter with the value set to whatever the value of __meat_consul_dc was.
	//
	// So in reality its "the same", the difference is in prometheus these apply to the labels/targets of a scrape job,
	// in promxy they apply to the prometheus hosts in the servergroup - but the behavior is the same.
	RelabelConfigs []*relabel.Config `yaml:"relabel_configs,omitempty"`
	// Hosts is a set of ServiceDiscoveryConfig options that allow promxy to discover
	// all hosts in the server_group
	Hosts sd_config.ServiceDiscoveryConfig `yaml:",inline"`
	// PathPrefix to prepend to all queries to hosts in this servergroup
	PathPrefix string `yaml:"path_prefix"`
	// QueryParams are a map of query params to add to all HTTP calls made to this downstream
	// the main use-case for this is to add `nocache=1` to VictoriaMetrics downstreams
	// (see https://github.com/jacksontj/promxy/issues/202)
	QueryParams map[string]string `yaml:"query_params"`
	// TODO cache this as a model.Time after unmarshal
	// AntiAffinity defines how large of a gap in the timeseries will cause promxy
	// to merge series from 2 hosts in a server_group. This required for a couple reasons
	// (1) Promxy cannot make assumptions on downstream clock-drift and
	// (2) two prometheus hosts scraping the same target may have different times
	// #2 is caused by prometheus storing the time of the scrape as the time the scrape **starts**.
	// in practice this is actually quite frequent as there are a variety of situations that
	// cause variable scrape completion time (slow exporter, serial exporter, network latency, etc.)
	// any one of these can cause the resulting data in prometheus to have the same time but in reality
	// come from different points in time. Best practice for this value is to set it to your scrape interval
	AntiAffinity time.Duration `yaml:"anti_affinity,omitempty"`

	// IgnoreError will hide all errors from this given servergroup effectively making
	// the responses from this servergroup "not required" for the result.
	// Note: this allows you to make the tradeoff between availability of queries and consistency of results
	IgnoreError bool `yaml:"ignore_error"`

	// RelativeTimeRangeConfig defines a relative time range that this servergroup will respond to
	// An example use-case would be if a specific servergroup was long-term storage, it might only
	// have data 3d old and retain 90d of data.
	*RelativeTimeRangeConfig `yaml:"relative_time_range"`

	// AbsoluteTimeRangeConfig defines an absolute time range that this servergroup will respond to
	// An example use-case would be if a specific servergroup was was "deprecated" and wasn't getting
	// any new data after a specific given point in time
	*AbsoluteTimeRangeConfig `yaml:"absolute_time_range"`
}

// GetScheme returns the scheme for this servergroup
func (c *Config) GetScheme() string {
	return c.Scheme
}

// GetAntiAffinity returns the AntiAffinity time for this servergroup
func (c *Config) GetAntiAffinity() model.Time {
	return model.TimeFromUnix(int64((c.AntiAffinity).Seconds()))
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
	*c = DefaultConfig
	// We want to set c to the defaults and then overwrite it with the input.
	// To make unmarshal fill the plain data struct rather than calling UnmarshalYAML
	// again, we have to hide it using a type indirection.
	type plain Config
	return unmarshal((*plain)(c))
}

// HTTPClientConfig extends prometheus' HTTPClientConfig
type HTTPClientConfig struct {
	DialTimeout time.Duration                `yaml:"dial_timeout"`
	HTTPConfig  config_util.HTTPClientConfig `yaml:",inline"`
}

// RelativeTimeRangeConfig configures durations relative from "now" to define
// a servergroup's time range
type RelativeTimeRangeConfig struct {
	Start *time.Duration `yaml:"start"`
	End   *time.Duration `yaml:"end"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (tr *RelativeTimeRangeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain RelativeTimeRangeConfig
	if err := unmarshal((*plain)(tr)); err != nil {
		return err
	}

	return tr.validate()
}

func (tr *RelativeTimeRangeConfig) validate() error {
	if tr.End != nil && tr.Start != nil && *tr.End < *tr.Start {
		return fmt.Errorf("RelativeTimeRangeConfig: End must be after start")
	}
	return nil
}

// AbsoluteTimeRangeConfig contains absolute times to define a servergroup's time range
type AbsoluteTimeRangeConfig struct {
	Start time.Time `yaml:"start"`
	End   time.Time `yaml:"end"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (tr *AbsoluteTimeRangeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain AbsoluteTimeRangeConfig
	if err := unmarshal((*plain)(tr)); err != nil {
		return err
	}

	return tr.validate()
}

func (tr *AbsoluteTimeRangeConfig) validate() error {
	if !tr.Start.IsZero() && !tr.End.IsZero() && tr.End.Before(tr.Start) {
		return fmt.Errorf("AbsoluteTimeRangeConfig: End must be after start")
	}
	return nil
}
<commit_msg>fix typo dial_timeout default to 200ms<commit_after>package servergroup

import (
	"fmt"
	"time"

	config_util "github.com/prometheus/common/config"
	"github.com/prometheus/common/model"
	sd_config "github.com/prometheus/prometheus/discovery/config"
	"github.com/prometheus/prometheus/pkg/relabel"
)

var (
	// DefaultConfig is the Default base promxy configuration
	DefaultConfig = Config{
		AntiAffinity:   time.Second * 10,
		Scheme:         "http",
		RemoteReadPath: "api/v1/read",
		HTTPConfig: HTTPClientConfig{
			DialTimeout: time.Millisecond * 200, // Default dial timeout of 200ms
		},
	}
)

// Config is the configuration for a ServerGroup that promxy will talk to.
// This is where the vast majority of options exist.
type Config struct {
	// RemoteRead directs promxy to load RAW data (meaning matrix selectors such as `foo[1h]`)
	// through the RemoteRead API on prom.
	// Pros:
	//  - StaleNaNs work
	//  - ~2x faster (in my local testing, more so if you are using default JSON marshaler in prom)
	//
	// Cons:
	//  - proto marshaling prom side doesn't stream, so the data being sent
	//      over the wire will be 2x its size in memory on the remote prom host.
	//  - "experimental" API (according to docs) -- meaning this might break
	//      without much (if any) warning
	//
	// Upstream prom added a StaleNan to determine if a given timeseries has gone
	// NaN -- the problem being that for range vectors they filter out all "stale" samples
	// meaning that it isn't possible to get a "raw" dump of data through the query/query_range v1 API
	// The only option that exists in reality is the "remote read" API -- which suffers
	// from the same memory-balooning problems that the HTTP+JSON API originally had.
	// It has **less** of a problem (its 2x memory instead of 14x) so it is a viable option.
	RemoteRead bool `yaml:"remote_read"`
	// RemoteReadPath sets the remote read path for the hosts in this servergroup
	RemoteReadPath string `yaml:"remote_read_path"`
	// HTTP client config for promxy to use when connecting to the various server_groups
	// this is the same config as prometheus
	HTTPConfig HTTPClientConfig `yaml:"http_client"`
	// Scheme defines how promxy talks to this server group (http, https, etc.)
	Scheme string `yaml:"scheme"`
	// Labels is a set of labels that will be added to all metrics retrieved
	// from this server group
	Labels model.LabelSet `json:"labels"`
	// RelabelConfigs are similar in function and identical in configuration as prometheus'
	// relabel config for scrape jobs. The difference here being that the source labels
	// you can pull from are from the downstream servergroup target and the labels you are
	// relabeling are that of the timeseries being returned. This allows you to mutate the
	// labelsets returned by that target at runtime.
	// To further illustrate the difference we'll look at an example:
	//
	//      relabel_configs:
	//    - source_labels: [__meta_consul_tags]
	//      regex: '.*,prod,.*'
	//      action: keep
	//    - source_labels: [__meta_consul_dc]
	//      regex: '.+'
	//      action: replace
	//      target_label: datacenter
	//
	// If we saw this in a scrape-config we would expect:
	//   (1) the scrape would only target hosts with a prod consul label
	//   (2) it would add a label to all returned series of datacenter with the value set to whatever the value of __meat_consul_dc was.
	//
	// If we saw this same config in promxy (pointing at prometheus hosts instead of some exporter), we'd expect a similar behavior:
	//   (1) only targets with the prod consul label would be included in the servergroup
	//   (2) it would add a label to all returned series of this servergroup of datacenter with the value set to whatever the value of __meat_consul_dc was.
	//
	// So in reality its "the same", the difference is in prometheus these apply to the labels/targets of a scrape job,
	// in promxy they apply to the prometheus hosts in the servergroup - but the behavior is the same.
	RelabelConfigs []*relabel.Config `yaml:"relabel_configs,omitempty"`
	// Hosts is a set of ServiceDiscoveryConfig options that allow promxy to discover
	// all hosts in the server_group
	Hosts sd_config.ServiceDiscoveryConfig `yaml:",inline"`
	// PathPrefix to prepend to all queries to hosts in this servergroup
	PathPrefix string `yaml:"path_prefix"`
	// QueryParams are a map of query params to add to all HTTP calls made to this downstream
	// the main use-case for this is to add `nocache=1` to VictoriaMetrics downstreams
	// (see https://github.com/jacksontj/promxy/issues/202)
	QueryParams map[string]string `yaml:"query_params"`
	// TODO cache this as a model.Time after unmarshal
	// AntiAffinity defines how large of a gap in the timeseries will cause promxy
	// to merge series from 2 hosts in a server_group. This required for a couple reasons
	// (1) Promxy cannot make assumptions on downstream clock-drift and
	// (2) two prometheus hosts scraping the same target may have different times
	// #2 is caused by prometheus storing the time of the scrape as the time the scrape **starts**.
	// in practice this is actually quite frequent as there are a variety of situations that
	// cause variable scrape completion time (slow exporter, serial exporter, network latency, etc.)
	// any one of these can cause the resulting data in prometheus to have the same time but in reality
	// come from different points in time. Best practice for this value is to set it to your scrape interval
	AntiAffinity time.Duration `yaml:"anti_affinity,omitempty"`

	// IgnoreError will hide all errors from this given servergroup effectively making
	// the responses from this servergroup "not required" for the result.
	// Note: this allows you to make the tradeoff between availability of queries and consistency of results
	IgnoreError bool `yaml:"ignore_error"`

	// RelativeTimeRangeConfig defines a relative time range that this servergroup will respond to
	// An example use-case would be if a specific servergroup was long-term storage, it might only
	// have data 3d old and retain 90d of data.
	*RelativeTimeRangeConfig `yaml:"relative_time_range"`

	// AbsoluteTimeRangeConfig defines an absolute time range that this servergroup will respond to
	// An example use-case would be if a specific servergroup was was "deprecated" and wasn't getting
	// any new data after a specific given point in time
	*AbsoluteTimeRangeConfig `yaml:"absolute_time_range"`
}

// GetScheme returns the scheme for this servergroup
func (c *Config) GetScheme() string {
	return c.Scheme
}

// GetAntiAffinity returns the AntiAffinity time for this servergroup
func (c *Config) GetAntiAffinity() model.Time {
	return model.TimeFromUnix(int64((c.AntiAffinity).Seconds()))
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
	*c = DefaultConfig
	// We want to set c to the defaults and then overwrite it with the input.
	// To make unmarshal fill the plain data struct rather than calling UnmarshalYAML
	// again, we have to hide it using a type indirection.
	type plain Config
	return unmarshal((*plain)(c))
}

// HTTPClientConfig extends prometheus' HTTPClientConfig
type HTTPClientConfig struct {
	DialTimeout time.Duration                `yaml:"dial_timeout"`
	HTTPConfig  config_util.HTTPClientConfig `yaml:",inline"`
}

// RelativeTimeRangeConfig configures durations relative from "now" to define
// a servergroup's time range
type RelativeTimeRangeConfig struct {
	Start *time.Duration `yaml:"start"`
	End   *time.Duration `yaml:"end"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (tr *RelativeTimeRangeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain RelativeTimeRangeConfig
	if err := unmarshal((*plain)(tr)); err != nil {
		return err
	}

	return tr.validate()
}

func (tr *RelativeTimeRangeConfig) validate() error {
	if tr.End != nil && tr.Start != nil && *tr.End < *tr.Start {
		return fmt.Errorf("RelativeTimeRangeConfig: End must be after start")
	}
	return nil
}

// AbsoluteTimeRangeConfig contains absolute times to define a servergroup's time range
type AbsoluteTimeRangeConfig struct {
	Start time.Time `yaml:"start"`
	End   time.Time `yaml:"end"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (tr *AbsoluteTimeRangeConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain AbsoluteTimeRangeConfig
	if err := unmarshal((*plain)(tr)); err != nil {
		return err
	}

	return tr.validate()
}

func (tr *AbsoluteTimeRangeConfig) validate() error {
	if !tr.Start.IsZero() && !tr.End.IsZero() && tr.End.Before(tr.Start) {
		return fmt.Errorf("AbsoluteTimeRangeConfig: End must be after start")
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
	"bytes"
	"io"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/kubernetes-incubator/cri-tools/pkg/framework"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/portforward"
	remoteclient "k8s.io/client-go/tools/remotecommand"
	"k8s.io/client-go/transport/spdy"
	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	defaultStreamServerAddress string = "127.0.0.1:10250"
	defaultStreamServerScheme  string = "http"
)

var _ = framework.KubeDescribe("Streaming", func() {
	f := framework.NewDefaultCRIFramework()

	var rc internalapi.RuntimeService
	var ic internalapi.ImageManagerService

	BeforeEach(func() {
		rc = f.CRIClient.CRIRuntimeClient
		ic = f.CRIClient.CRIImageClient
	})

	Context("runtime should support streaming interfaces", func() {
		var podID string
		var podConfig *runtimeapi.PodSandboxConfig

		AfterEach(func() {
			By("stop PodSandbox")
			rc.StopPodSandbox(podID)
			By("delete PodSandbox")
			rc.RemovePodSandbox(podID)
		})

		It("runtime should support exec [Conformance]", func() {
			podID, podConfig = framework.CreatePodSandboxForContainer(rc)

			By("create a default container")
			containerID := framework.CreateDefaultContainer(rc, ic, podID, podConfig, "container-for-exec-test")

			By("start container")
			startContainer(rc, containerID)

			req := createDefaultExec(rc, containerID)

			By("check the output of exec")
			checkExec(rc, req)
		})

		It("runtime should support attach [Conformance]", func() {
			podID, podConfig = framework.CreatePodSandboxForContainer(rc)

			By("create a default container")
			containerID := createShellContainer(rc, ic, podID, podConfig, "container-for-attach-test")

			By("start container")
			startContainer(rc, containerID)

			req := createDefaultAttach(rc, containerID)

			By("check the output of attach")
			checkAttach(rc, req)
		})

		It("runtime should support portforward [Conformance]", func() {
			By("create a PodSandbox with host port and container port port mapping")
			var podConfig *runtimeapi.PodSandboxConfig
			portMappings := []*runtimeapi.PortMapping{
				{
					ContainerPort: nginxContainerPort,
				},
			}
			podID, podConfig = createPodSandboxWithPortMapping(rc, portMappings)

			By("create a nginx container")
			containerID := createNginxContainer(rc, ic, podID, podConfig, "container-for-portforward-test")

			By("start the nginx container")
			startContainer(rc, containerID)

			req := createDefaultPortForward(rc, podID)

			By("check the output of portforward")
			checkPortForward(rc, req)
		})
	})
})

func createDefaultExec(c internalapi.RuntimeService, containerID string) string {
	By("exec default command in container: " + containerID)
	req := &runtimeapi.ExecRequest{
		ContainerId: containerID,
		Cmd:         []string{"echo", "hello"},
	}

	resp, err := c.Exec(req)
	framework.ExpectNoError(err, "failed to exec in container %q", containerID)
	framework.Logf("Get exec url: " + resp.Url)
	return resp.Url
}

func checkExec(c internalapi.RuntimeService, execServerURL string) {
	localOut := &bytes.Buffer{}
	localErr := &bytes.Buffer{}

	// Only http is supported now.
	// TODO: support streaming APIs via tls.
	url := parseURL(c, execServerURL)
	e, err := remoteclient.NewSPDYExecutor(&rest.Config{}, "POST", url)
	framework.ExpectNoError(err, "failed to create executor for %q", execServerURL)

	err = e.Stream(remoteclient.StreamOptions{
		Stdout: localOut,
		Stderr: localErr,
		Tty:    false,
	})
	framework.ExpectNoError(err, "failed to open streamer for %q", execServerURL)

	Expect(localOut.String()).To(Equal("hello\n"), "The stdout of exec should be hello")
	Expect(localErr.String()).To(BeEmpty(), "The stderr of exec should be empty")
	framework.Logf("Check exec url %q succeed", execServerURL)
}

func parseURL(c internalapi.RuntimeService, serverURL string) *url.URL {
	url, err := url.Parse(serverURL)
	framework.ExpectNoError(err, "failed to parse url:  %q", serverURL)

	version := getVersion(c)
	if version.RuntimeName == "docker" {
		if url.Host == "" {
			url.Host = defaultStreamServerAddress
		}
		if url.Scheme == "" {
			url.Scheme = defaultStreamServerScheme
		}
	}

	Expect(url.Host).NotTo(BeEmpty(), "The host of url should not be empty")
	framework.Logf("Parse url %q succeed", serverURL)
	return url
}

func createDefaultAttach(c internalapi.RuntimeService, containerID string) string {
	By("attach container: " + containerID)
	req := &runtimeapi.AttachRequest{
		ContainerId: containerID,
		Stdin:       true,
		Tty:         false,
	}

	resp, err := c.Attach(req)
	framework.ExpectNoError(err, "failed to attach in container %q", containerID)
	framework.Logf("Get attach url: " + resp.Url)
	return resp.Url
}

func checkAttach(c internalapi.RuntimeService, attachServerURL string) {
	localOut := &bytes.Buffer{}
	localErr := &bytes.Buffer{}
	reader, writer := io.Pipe()
	var out string

	go func() {
		defer GinkgoRecover()
		writer.Write([]byte("echo hello\n"))
		Eventually(func() string {
			out = localOut.String()
			return out
		}, time.Minute, time.Second).ShouldNot(BeEmpty())
		writer.Close()
	}()

	// Only http is supported now.
	// TODO: support streaming APIs via tls.
	url := parseURL(c, attachServerURL)
	e, err := remoteclient.NewSPDYExecutor(&rest.Config{}, "POST", url)
	framework.ExpectNoError(err, "failed to create executor for %q", attachServerURL)

	err = e.Stream(remoteclient.StreamOptions{
		Stdin:  reader,
		Stdout: localOut,
		Stderr: localErr,
		Tty:    false,
	})
	framework.ExpectNoError(err, "failed to open streamer for %q", attachServerURL)

	Expect(out).To(Equal("hello\n"), "The stdout of exec should be hello")
	Expect(localErr.String()).To(BeEmpty(), "The stderr of attach should be empty")
	framework.Logf("Check attach url %q succeed", attachServerURL)
}

func createDefaultPortForward(c internalapi.RuntimeService, podID string) string {
	By("port forward PodSandbox: " + podID)
	req := &runtimeapi.PortForwardRequest{
		PodSandboxId: podID,
	}

	resp, err := c.PortForward(req)
	framework.ExpectNoError(err, "failed to port forward PodSandbox %q", podID)
	framework.Logf("Get port forward url: " + resp.Url)
	return resp.Url
}

func checkPortForward(c internalapi.RuntimeService, portForwardSeverURL string) {
	stopChan := make(chan struct{}, 1)
	readyChan := make(chan struct{})
	defer close(stopChan)

	transport, upgrader, err := spdy.RoundTripperFor(&rest.Config{})
	framework.ExpectNoError(err, "failed to create spdy round tripper")
	url := parseURL(c, portForwardSeverURL)
	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", url)
	pf, err := portforward.New(dialer, []string{"8000:80"}, stopChan, readyChan, os.Stdout, os.Stderr)
	framework.ExpectNoError(err, "failed to create port forward for %q", portForwardSeverURL)

	go func() {
		defer GinkgoRecover()
		By("start port forward")
		err = pf.ForwardPorts()
		framework.ExpectNoError(err, "failed to start port forward for %q", portForwardSeverURL)
	}()

	By("check if we can get nginx main page via localhost:8000")
	checkNginxMainPage(c, "", true)
	framework.Logf("Check port forward url %q succeed", portForwardSeverURL)
}
<commit_msg>pass stdout and stderr in streaming client<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validate

import (
	"bytes"
	"io"
	"net/http"
	"net/url"
	"os"
	"time"

	"github.com/kubernetes-incubator/cri-tools/pkg/framework"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/portforward"
	remoteclient "k8s.io/client-go/tools/remotecommand"
	"k8s.io/client-go/transport/spdy"
	internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	defaultStreamServerAddress string = "127.0.0.1:10250"
	defaultStreamServerScheme  string = "http"
)

var _ = framework.KubeDescribe("Streaming", func() {
	f := framework.NewDefaultCRIFramework()

	var rc internalapi.RuntimeService
	var ic internalapi.ImageManagerService

	BeforeEach(func() {
		rc = f.CRIClient.CRIRuntimeClient
		ic = f.CRIClient.CRIImageClient
	})

	Context("runtime should support streaming interfaces", func() {
		var podID string
		var podConfig *runtimeapi.PodSandboxConfig

		AfterEach(func() {
			By("stop PodSandbox")
			rc.StopPodSandbox(podID)
			By("delete PodSandbox")
			rc.RemovePodSandbox(podID)
		})

		It("runtime should support exec [Conformance]", func() {
			podID, podConfig = framework.CreatePodSandboxForContainer(rc)

			By("create a default container")
			containerID := framework.CreateDefaultContainer(rc, ic, podID, podConfig, "container-for-exec-test")

			By("start container")
			startContainer(rc, containerID)

			req := createDefaultExec(rc, containerID)

			By("check the output of exec")
			checkExec(rc, req)
		})

		It("runtime should support attach [Conformance]", func() {
			podID, podConfig = framework.CreatePodSandboxForContainer(rc)

			By("create a default container")
			containerID := createShellContainer(rc, ic, podID, podConfig, "container-for-attach-test")

			By("start container")
			startContainer(rc, containerID)

			req := createDefaultAttach(rc, containerID)

			By("check the output of attach")
			checkAttach(rc, req)
		})

		It("runtime should support portforward [Conformance]", func() {
			By("create a PodSandbox with host port and container port port mapping")
			var podConfig *runtimeapi.PodSandboxConfig
			portMappings := []*runtimeapi.PortMapping{
				{
					ContainerPort: nginxContainerPort,
				},
			}
			podID, podConfig = createPodSandboxWithPortMapping(rc, portMappings)

			By("create a nginx container")
			containerID := createNginxContainer(rc, ic, podID, podConfig, "container-for-portforward-test")

			By("start the nginx container")
			startContainer(rc, containerID)

			req := createDefaultPortForward(rc, podID)

			By("check the output of portforward")
			checkPortForward(rc, req)
		})
	})
})

func createDefaultExec(c internalapi.RuntimeService, containerID string) string {
	By("exec default command in container: " + containerID)
	req := &runtimeapi.ExecRequest{
		ContainerId: containerID,
		Cmd:         []string{"echo", "hello"},
		Stdout:      true,
		Stderr:      true,
	}

	resp, err := c.Exec(req)
	framework.ExpectNoError(err, "failed to exec in container %q", containerID)
	framework.Logf("Get exec url: " + resp.Url)
	return resp.Url
}

func checkExec(c internalapi.RuntimeService, execServerURL string) {
	localOut := &bytes.Buffer{}
	localErr := &bytes.Buffer{}

	// Only http is supported now.
	// TODO: support streaming APIs via tls.
	url := parseURL(c, execServerURL)
	e, err := remoteclient.NewSPDYExecutor(&rest.Config{}, "POST", url)
	framework.ExpectNoError(err, "failed to create executor for %q", execServerURL)

	err = e.Stream(remoteclient.StreamOptions{
		Stdout: localOut,
		Stderr: localErr,
		Tty:    false,
	})
	framework.ExpectNoError(err, "failed to open streamer for %q", execServerURL)

	Expect(localOut.String()).To(Equal("hello\n"), "The stdout of exec should be hello")
	Expect(localErr.String()).To(BeEmpty(), "The stderr of exec should be empty")
	framework.Logf("Check exec url %q succeed", execServerURL)
}

func parseURL(c internalapi.RuntimeService, serverURL string) *url.URL {
	url, err := url.Parse(serverURL)
	framework.ExpectNoError(err, "failed to parse url:  %q", serverURL)

	version := getVersion(c)
	if version.RuntimeName == "docker" {
		if url.Host == "" {
			url.Host = defaultStreamServerAddress
		}
		if url.Scheme == "" {
			url.Scheme = defaultStreamServerScheme
		}
	}

	Expect(url.Host).NotTo(BeEmpty(), "The host of url should not be empty")
	framework.Logf("Parse url %q succeed", serverURL)
	return url
}

func createDefaultAttach(c internalapi.RuntimeService, containerID string) string {
	By("attach container: " + containerID)
	req := &runtimeapi.AttachRequest{
		ContainerId: containerID,
		Stdin:       true,
		Stdout:      true,
		Stderr:      true,
		Tty:         false,
	}

	resp, err := c.Attach(req)
	framework.ExpectNoError(err, "failed to attach in container %q", containerID)
	framework.Logf("Get attach url: " + resp.Url)
	return resp.Url
}

func checkAttach(c internalapi.RuntimeService, attachServerURL string) {
	localOut := &bytes.Buffer{}
	localErr := &bytes.Buffer{}
	reader, writer := io.Pipe()
	var out string

	go func() {
		defer GinkgoRecover()
		writer.Write([]byte("echo hello\n"))
		Eventually(func() string {
			out = localOut.String()
			return out
		}, time.Minute, time.Second).ShouldNot(BeEmpty())
		writer.Close()
	}()

	// Only http is supported now.
	// TODO: support streaming APIs via tls.
	url := parseURL(c, attachServerURL)
	e, err := remoteclient.NewSPDYExecutor(&rest.Config{}, "POST", url)
	framework.ExpectNoError(err, "failed to create executor for %q", attachServerURL)

	err = e.Stream(remoteclient.StreamOptions{
		Stdin:  reader,
		Stdout: localOut,
		Stderr: localErr,
		Tty:    false,
	})
	framework.ExpectNoError(err, "failed to open streamer for %q", attachServerURL)

	Expect(out).To(Equal("hello\n"), "The stdout of exec should be hello")
	Expect(localErr.String()).To(BeEmpty(), "The stderr of attach should be empty")
	framework.Logf("Check attach url %q succeed", attachServerURL)
}

func createDefaultPortForward(c internalapi.RuntimeService, podID string) string {
	By("port forward PodSandbox: " + podID)
	req := &runtimeapi.PortForwardRequest{
		PodSandboxId: podID,
	}

	resp, err := c.PortForward(req)
	framework.ExpectNoError(err, "failed to port forward PodSandbox %q", podID)
	framework.Logf("Get port forward url: " + resp.Url)
	return resp.Url
}

func checkPortForward(c internalapi.RuntimeService, portForwardSeverURL string) {
	stopChan := make(chan struct{}, 1)
	readyChan := make(chan struct{})
	defer close(stopChan)

	transport, upgrader, err := spdy.RoundTripperFor(&rest.Config{})
	framework.ExpectNoError(err, "failed to create spdy round tripper")
	url := parseURL(c, portForwardSeverURL)
	dialer := spdy.NewDialer(upgrader, &http.Client{Transport: transport}, "POST", url)
	pf, err := portforward.New(dialer, []string{"8000:80"}, stopChan, readyChan, os.Stdout, os.Stderr)
	framework.ExpectNoError(err, "failed to create port forward for %q", portForwardSeverURL)

	go func() {
		defer GinkgoRecover()
		By("start port forward")
		err = pf.ForwardPorts()
		framework.ExpectNoError(err, "failed to start port forward for %q", portForwardSeverURL)
	}()

	By("check if we can get nginx main page via localhost:8000")
	checkNginxMainPage(c, "", true)
	framework.Logf("Check port forward url %q succeed", portForwardSeverURL)
}
<|endoftext|>


<commit_before>package logs

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"github.com/dokku/dokku/plugins/common"
	"github.com/joncalhoun/qson"
)

type vectorConfig struct {
	Sources map[string]vectorSource `json:"sources"`
	Sinks   map[string]vectorSink   `json:"sinks"`
}

type vectorSource struct {
	Type          string   `json:"type"`
	IncludeLabels []string `json:"include_labels,omitempty"`
}

type vectorSink map[string]interface{}

const vectorContainerName = "vector"

func killVectorContainer() error {
	if !common.ContainerExists(vectorContainerName) {
		return nil
	}

	if err := stopVectorContainer(); err != nil {
		return err
	}

	time.Sleep(10 * time.Second)
	if err := removeVectorContainer(); err != nil {
		return err
	}

	return nil
}

func removeVectorContainer() error {
	if !common.ContainerExists(vectorContainerName) {
		return nil
	}

	cmd := common.NewShellCmd(strings.Join([]string{
		common.DockerBin(), "container", "rm", "-f", vectorContainerName}, " "))

	return common.SuppressOutput(func() error {
		if cmd.Execute() {
			return nil
		}

		if common.ContainerExists(vectorContainerName) {
			return errors.New("Unable to remove vector container")
		}

		return nil
	})
}

func startVectorContainer(vectorImage string) error {
	cmd := common.NewShellCmd(strings.Join([]string{
		common.DockerBin(),
		"container",
		"run", "--detach", "--name", vectorContainerName, common.MustGetEnv("DOKKU_GLOBAL_RUN_ARGS"),
		"--volume", "/var/lib/dokku/data/logs/vector.json:/etc/vector/vector.json",
		"--volume", "/var/run/docker.sock:/var/run/docker.sock",
		"--volume", common.MustGetEnv("DOKKU_LOGS_HOST_DIR") + ":/var/logs/dokku/apps",
		vectorImage,
		"--config", "/etc/vector/vector.json", "--watch-config"}, " "))

	if !cmd.Execute() {
		return errors.New("Unable to start vector container")
	}

	return nil
}

func stopVectorContainer() error {
	if !common.ContainerExists(vectorContainerName) {
		return nil
	}

	if !common.ContainerIsRunning(vectorContainerName) {
		return nil
	}

	cmd := common.NewShellCmd(strings.Join([]string{
		common.DockerBin(), "container", "stop", vectorContainerName}, " "))

	return common.SuppressOutput(func() error {
		if cmd.Execute() {
			return nil
		}

		if common.ContainerIsRunning(vectorContainerName) {
			return errors.New("Unable to stop vector container")
		}

		return nil
	})
}

func valueToConfig(appName string, value string) (vectorSink, error) {
	var data vectorSink
	u, err := url.Parse(value)
	if err != nil {
		return data, err
	}

	if u.Query().Get("sinks") != "" {
		return data, errors.New("Invalid option sinks")
	}

	t := fmt.Sprintf("type=%s", u.Scheme)
	i := fmt.Sprintf("inputs[]=docker-source:%s", appName)
	if appName == "--global" {
		i = "inputs[]=docker-global-source"
	}
	if appName == "--null" {
		i = "inputs[]=docker-null-source"
	}

	initialQuery := fmt.Sprintf("%s&%s", t, i)
	query := u.RawQuery
	if query == "" {
		query = initialQuery
	} else if strings.HasPrefix(query, "&") {
		query = fmt.Sprintf("%s%s", initialQuery, query)
	} else {
		query = fmt.Sprintf("%s&%s", initialQuery, query)
	}

	b, err := qson.ToJSON(query)
	if err != nil {
		return data, err
	}

	if err := json.Unmarshal(b, &data); err != nil {
		return data, err
	}

	return data, nil
}

func writeVectorConfig() error {
	apps, err := common.DokkuApps()
	if err != nil {
		return err
	}

	data := vectorConfig{
		Sources: map[string]vectorSource{},
		Sinks:   map[string]vectorSink{},
	}
	for _, appName := range apps {
		value := common.PropertyGet("logs", appName, "vector-sink")
		if value == "" {
			continue
		}

		sink, err := valueToConfig(appName, value)
		if err != nil {
			return err
		}

		data.Sources[fmt.Sprintf("docker-source:%s", appName)] = vectorSource{
			Type:          "docker_logs",
			IncludeLabels: []string{fmt.Sprintf("com.dokku.app-name=%s", appName)},
		}

		data.Sinks[fmt.Sprintf("docker-sink:%s", appName)] = sink
	}

	value := common.PropertyGet("logs", "--global", "vector-sink")
	if value != "" {
		sink, err := valueToConfig("--global", value)
		if err != nil {
			return err
		}

		data.Sources["docker-global-source"] = vectorSource{
			Type:          "docker_logs",
			IncludeLabels: []string{"com.dokku.app-name"},
		}

		data.Sinks["docker-global-sink"] = sink
	}

	if len(data.Sources) == 0 {
		// pull from no containers
		data.Sources["docker-null-source"] = vectorSource{
			Type:          "docker_logs",
			IncludeLabels: []string{"com.dokku.vector-null"},
		}
	}

	if len(data.Sinks) == 0 {
		// write logs to a blackhole
		sink, err := valueToConfig("--null", "blackhole://?print_amount=1")
		if err != nil {
			return err
		}

		data.Sinks["docker-null-sink"] = sink
	}

	b, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return err
	}

	vectorConfig := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "logs", "vector.json")
	if err := common.WriteSliceToFile(vectorConfig, []string{string(b)}); err != nil {
		return err
	}

	return nil
}
<commit_msg>fix: handle case where there are no apps<commit_after>package logs

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"github.com/dokku/dokku/plugins/common"
	"github.com/joncalhoun/qson"
)

type vectorConfig struct {
	Sources map[string]vectorSource `json:"sources"`
	Sinks   map[string]vectorSink   `json:"sinks"`
}

type vectorSource struct {
	Type          string   `json:"type"`
	IncludeLabels []string `json:"include_labels,omitempty"`
}

type vectorSink map[string]interface{}

const vectorContainerName = "vector"

func killVectorContainer() error {
	if !common.ContainerExists(vectorContainerName) {
		return nil
	}

	if err := stopVectorContainer(); err != nil {
		return err
	}

	time.Sleep(10 * time.Second)
	if err := removeVectorContainer(); err != nil {
		return err
	}

	return nil
}

func removeVectorContainer() error {
	if !common.ContainerExists(vectorContainerName) {
		return nil
	}

	cmd := common.NewShellCmd(strings.Join([]string{
		common.DockerBin(), "container", "rm", "-f", vectorContainerName}, " "))

	return common.SuppressOutput(func() error {
		if cmd.Execute() {
			return nil
		}

		if common.ContainerExists(vectorContainerName) {
			return errors.New("Unable to remove vector container")
		}

		return nil
	})
}

func startVectorContainer(vectorImage string) error {
	cmd := common.NewShellCmd(strings.Join([]string{
		common.DockerBin(),
		"container",
		"run", "--detach", "--name", vectorContainerName, common.MustGetEnv("DOKKU_GLOBAL_RUN_ARGS"),
		"--volume", "/var/lib/dokku/data/logs/vector.json:/etc/vector/vector.json",
		"--volume", "/var/run/docker.sock:/var/run/docker.sock",
		"--volume", common.MustGetEnv("DOKKU_LOGS_HOST_DIR") + ":/var/logs/dokku/apps",
		vectorImage,
		"--config", "/etc/vector/vector.json", "--watch-config"}, " "))

	if !cmd.Execute() {
		return errors.New("Unable to start vector container")
	}

	return nil
}

func stopVectorContainer() error {
	if !common.ContainerExists(vectorContainerName) {
		return nil
	}

	if !common.ContainerIsRunning(vectorContainerName) {
		return nil
	}

	cmd := common.NewShellCmd(strings.Join([]string{
		common.DockerBin(), "container", "stop", vectorContainerName}, " "))

	return common.SuppressOutput(func() error {
		if cmd.Execute() {
			return nil
		}

		if common.ContainerIsRunning(vectorContainerName) {
			return errors.New("Unable to stop vector container")
		}

		return nil
	})
}

func valueToConfig(appName string, value string) (vectorSink, error) {
	var data vectorSink
	u, err := url.Parse(value)
	if err != nil {
		return data, err
	}

	if u.Query().Get("sinks") != "" {
		return data, errors.New("Invalid option sinks")
	}

	t := fmt.Sprintf("type=%s", u.Scheme)
	i := fmt.Sprintf("inputs[]=docker-source:%s", appName)
	if appName == "--global" {
		i = "inputs[]=docker-global-source"
	}
	if appName == "--null" {
		i = "inputs[]=docker-null-source"
	}

	initialQuery := fmt.Sprintf("%s&%s", t, i)
	query := u.RawQuery
	if query == "" {
		query = initialQuery
	} else if strings.HasPrefix(query, "&") {
		query = fmt.Sprintf("%s%s", initialQuery, query)
	} else {
		query = fmt.Sprintf("%s&%s", initialQuery, query)
	}

	b, err := qson.ToJSON(query)
	if err != nil {
		return data, err
	}

	if err := json.Unmarshal(b, &data); err != nil {
		return data, err
	}

	return data, nil
}

func writeVectorConfig() error {
	apps, _ := common.DokkuApps()
	data := vectorConfig{
		Sources: map[string]vectorSource{},
		Sinks:   map[string]vectorSink{},
	}
	for _, appName := range apps {
		value := common.PropertyGet("logs", appName, "vector-sink")
		if value == "" {
			continue
		}

		sink, err := valueToConfig(appName, value)
		if err != nil {
			return err
		}

		data.Sources[fmt.Sprintf("docker-source:%s", appName)] = vectorSource{
			Type:          "docker_logs",
			IncludeLabels: []string{fmt.Sprintf("com.dokku.app-name=%s", appName)},
		}

		data.Sinks[fmt.Sprintf("docker-sink:%s", appName)] = sink
	}

	value := common.PropertyGet("logs", "--global", "vector-sink")
	if value != "" {
		sink, err := valueToConfig("--global", value)
		if err != nil {
			return err
		}

		data.Sources["docker-global-source"] = vectorSource{
			Type:          "docker_logs",
			IncludeLabels: []string{"com.dokku.app-name"},
		}

		data.Sinks["docker-global-sink"] = sink
	}

	if len(data.Sources) == 0 {
		// pull from no containers
		data.Sources["docker-null-source"] = vectorSource{
			Type:          "docker_logs",
			IncludeLabels: []string{"com.dokku.vector-null"},
		}
	}

	if len(data.Sinks) == 0 {
		// write logs to a blackhole
		sink, err := valueToConfig("--null", "blackhole://?print_amount=1")
		if err != nil {
			return err
		}

		data.Sinks["docker-null-sink"] = sink
	}

	b, err := json.MarshalIndent(data, "", "  ")
	if err != nil {
		return err
	}

	vectorConfig := filepath.Join(common.MustGetEnv("DOKKU_LIB_ROOT"), "data", "logs", "vector.json")
	if err := common.WriteSliceToFile(vectorConfig, []string{string(b)}); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Package object contains implementations of all Git objects and utility
// functions to work with them.
package object

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"time"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
)

// ErrUnsupportedObject trigger when a non-supported object is being decoded.
var ErrUnsupportedObject = errors.New("unsupported object type")

// Object is a generic representation of any git object. It is implemented by
// Commit, Tree, Blob, and Tag, and includes the functions that are common to
// them.
//
// Object is returned when an object can be of any type. It is frequently used
// with a type cast to acquire the specific type of object:
//
//   func process(obj Object) {
//   	switch o := obj.(type) {
//   	case *Commit:
//   		// o is a Commit
//   	case *Tree:
//   		// o is a Tree
//   	case *Blob:
//   		// o is a Blob
//   	case *Tag:
//   		// o is a Tag
//   	}
//   }
//
// This interface is intentionally different from plumbing.EncodedObject, which
// is a lower level interface used by storage implementations to read and write
// objects in its encoded form.
type Object interface {
	ID() plumbing.Hash
	Type() plumbing.ObjectType
	Decode(plumbing.EncodedObject) error
	Encode(plumbing.EncodedObject) error
}

// GetObject gets an object from an object storer and decodes it.
func GetObject(s storer.EncodedObjectStorer, h plumbing.Hash) (Object, error) {
	o, err := s.EncodedObject(plumbing.AnyObject, h)
	if err != nil {
		return nil, err
	}

	return DecodeObject(s, o)
}

// DecodeObject decodes an encoded object into an Object and associates it to
// the given object storer.
func DecodeObject(s storer.EncodedObjectStorer, o plumbing.EncodedObject) (Object, error) {
	switch o.Type() {
	case plumbing.CommitObject:
		return DecodeCommit(s, o)
	case plumbing.TreeObject:
		return DecodeTree(s, o)
	case plumbing.BlobObject:
		return DecodeBlob(o)
	case plumbing.TagObject:
		return DecodeTag(s, o)
	default:
		return nil, plumbing.ErrInvalidType
	}
}

// DateFormat is the format being used in the original git implementation
const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"

// Signature is used to identify who and when created a commit or tag.
type Signature struct {
	// Name represents a person name. It is an arbitrary string.
	Name string
	// Email is an email, but it cannot be assumed to be well-formed.
	Email string
	// When is the timestamp of the signature.
	When time.Time
}

// Decode decodes a byte slice into a signature
func (s *Signature) Decode(b []byte) {
	open := bytes.LastIndexByte(b, '<')
	close := bytes.LastIndexByte(b, '>')
	if open == -1 || close == -1 {
		return
	}

	if close < open {
		return
	}

	s.Name = string(bytes.Trim(b[:open], " "))
	s.Email = string(b[open+1 : close])

	hasTime := close+2 < len(b)
	if hasTime {
		s.decodeTimeAndTimeZone(b[close+2:])
	}
}

// Encode encodes a Signature into a writer.
func (s *Signature) Encode(w io.Writer) error {
	if _, err := fmt.Fprintf(w, "%s <%s> ", s.Name, s.Email); err != nil {
		return err
	}
	if err := s.encodeTimeAndTimeZone(w); err != nil {
		return err
	}
	return nil
}

var timeZoneLength = 5

func (s *Signature) decodeTimeAndTimeZone(b []byte) {
	space := bytes.IndexByte(b, ' ')
	if space == -1 {
		space = len(b)
	}

	ts, err := strconv.ParseInt(string(b[:space]), 10, 64)
	if err != nil {
		return
	}

	s.When = time.Unix(ts, 0).In(time.UTC)
	var tzStart = space + 1
	if tzStart >= len(b) || tzStart+timeZoneLength > len(b) {
		return
	}

	timezone := string(b[tzStart : tzStart+timeZoneLength])
	tzhours, err := strconv.ParseInt(timezone[0:3], 10, 64)
	if err != nil {
		return
	}
	tzmins, err := strconv.ParseInt(timezone[3:], 10, 64)
	if err != nil {
		return
	}
	if tzhours < 0 {
		tzmins *= -1
	}

	tz := time.FixedZone("", int(tzhours*60*60+tzmins*60))

	s.When = s.When.In(tz)
}

func (s *Signature) encodeTimeAndTimeZone(w io.Writer) error {
	u := s.When.Unix()
	if u < 0 {
		u = 0
	}
	_, err := fmt.Fprintf(w, "%d %s", u, s.When.Format("-0700"))
	return err
}

func (s *Signature) String() string {
	return fmt.Sprintf("%s <%s>", s.Name, s.Email)
}

// ObjectIter provides an iterator for a set of objects.
type ObjectIter struct {
	storer.EncodedObjectIter
	s storer.EncodedObjectStorer
}

// NewObjectIter takes a storer.EncodedObjectStorer and a
// storer.EncodedObjectIter and returns an *ObjectIter that iterates over all
// objects contained in the storer.EncodedObjectIter.
func NewObjectIter(s storer.EncodedObjectStorer, iter storer.EncodedObjectIter) *ObjectIter {
	return &ObjectIter{iter, s}
}

// Next moves the iterator to the next object and returns a pointer to it. If
// there are no more objects, it returns io.EOF.
func (iter *ObjectIter) Next() (Object, error) {
	for {
		obj, err := iter.EncodedObjectIter.Next()
		if err != nil {
			return nil, err
		}

		o, err := iter.toObject(obj)
		if err == plumbing.ErrInvalidType {
			continue
		}

		if err != nil {
			return nil, err
		}

		return o, nil
	}
}

// ForEach call the cb function for each object contained on this iter until
// an error happens or the end of the iter is reached. If ErrStop is sent
// the iteration is stop but no error is returned. The iterator is closed.
func (iter *ObjectIter) ForEach(cb func(Object) error) error {
	return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error {
		o, err := iter.toObject(obj)
		if err == plumbing.ErrInvalidType {
			return nil
		}

		if err != nil {
			return err
		}

		return cb(o)
	})
}

func (iter *ObjectIter) toObject(obj plumbing.EncodedObject) (Object, error) {
	switch obj.Type() {
	case plumbing.BlobObject:
		blob := &Blob{}
		return blob, blob.Decode(obj)
	case plumbing.TreeObject:
		tree := &Tree{s: iter.s}
		return tree, tree.Decode(obj)
	case plumbing.CommitObject:
		commit := &Commit{}
		return commit, commit.Decode(obj)
	case plumbing.TagObject:
		tag := &Tag{}
		return tag, tag.Decode(obj)
	default:
		return nil, plumbing.ErrInvalidType
	}
}
<commit_msg>hack to prevent codecov from adding more codepaths<commit_after>// Package object contains implementations of all Git objects and utility
// functions to work with them.
package object

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"strconv"
	"time"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
)

// ErrUnsupportedObject trigger when a non-supported object is being decoded.
var ErrUnsupportedObject = errors.New("unsupported object type")

// Object is a generic representation of any git object. It is implemented by
// Commit, Tree, Blob, and Tag, and includes the functions that are common to
// them.
//
// Object is returned when an object can be of any type. It is frequently used
// with a type cast to acquire the specific type of object:
//
//   func process(obj Object) {
//   	switch o := obj.(type) {
//   	case *Commit:
//   		// o is a Commit
//   	case *Tree:
//   		// o is a Tree
//   	case *Blob:
//   		// o is a Blob
//   	case *Tag:
//   		// o is a Tag
//   	}
//   }
//
// This interface is intentionally different from plumbing.EncodedObject, which
// is a lower level interface used by storage implementations to read and write
// objects in its encoded form.
type Object interface {
	ID() plumbing.Hash
	Type() plumbing.ObjectType
	Decode(plumbing.EncodedObject) error
	Encode(plumbing.EncodedObject) error
}

// GetObject gets an object from an object storer and decodes it.
func GetObject(s storer.EncodedObjectStorer, h plumbing.Hash) (Object, error) {
	o, err := s.EncodedObject(plumbing.AnyObject, h)
	if err != nil {
		return nil, err
	}

	return DecodeObject(s, o)
}

// DecodeObject decodes an encoded object into an Object and associates it to
// the given object storer.
func DecodeObject(s storer.EncodedObjectStorer, o plumbing.EncodedObject) (Object, error) {
	switch o.Type() {
	case plumbing.CommitObject:
		return DecodeCommit(s, o)
	case plumbing.TreeObject:
		return DecodeTree(s, o)
	case plumbing.BlobObject:
		return DecodeBlob(o)
	case plumbing.TagObject:
		return DecodeTag(s, o)
	default:
		return nil, plumbing.ErrInvalidType
	}
}

// DateFormat is the format being used in the original git implementation
const DateFormat = "Mon Jan 02 15:04:05 2006 -0700"

// Signature is used to identify who and when created a commit or tag.
type Signature struct {
	// Name represents a person name. It is an arbitrary string.
	Name string
	// Email is an email, but it cannot be assumed to be well-formed.
	Email string
	// When is the timestamp of the signature.
	When time.Time
}

// Decode decodes a byte slice into a signature
func (s *Signature) Decode(b []byte) {
	open := bytes.LastIndexByte(b, '<')
	close := bytes.LastIndexByte(b, '>')
	if open == -1 || close == -1 {
		return
	}

	if close < open {
		return
	}

	s.Name = string(bytes.Trim(b[:open], " "))
	s.Email = string(b[open+1 : close])

	hasTime := close+2 < len(b)
	if hasTime {
		s.decodeTimeAndTimeZone(b[close+2:])
	}
}

// Encode encodes a Signature into a writer.
func (s *Signature) Encode(w io.Writer) error {
	if _, err := fmt.Fprintf(w, "%s <%s> ", s.Name, s.Email); err != nil {
		return err
	}
	if err := s.encodeTimeAndTimeZone(w); err != nil {
		return err
	}
	return nil
}

var timeZoneLength = 5

func (s *Signature) decodeTimeAndTimeZone(b []byte) {
	space := bytes.IndexByte(b, ' ')
	if space == -1 {
		space = len(b)
	}

	ts, err := strconv.ParseInt(string(b[:space]), 10, 64)
	if err != nil {
		return
	}

	s.When = time.Unix(ts, 0).In(time.UTC)
	var tzStart = space + 1
	if tzStart >= len(b) || tzStart+timeZoneLength > len(b) {
		return
	}

	timezone := string(b[tzStart : tzStart+timeZoneLength])
	tzhours, err1 := strconv.ParseInt(timezone[0:3], 10, 64)
	tzmins, err2 := strconv.ParseInt(timezone[3:], 10, 64)
	if err1 != nil || err2 != nil {
		return
	}
	if tzhours < 0 {
		tzmins *= -1
	}

	tz := time.FixedZone("", int(tzhours*60*60+tzmins*60))

	s.When = s.When.In(tz)
}

func (s *Signature) encodeTimeAndTimeZone(w io.Writer) error {
	u := s.When.Unix()
	if u < 0 {
		u = 0
	}
	_, err := fmt.Fprintf(w, "%d %s", u, s.When.Format("-0700"))
	return err
}

func (s *Signature) String() string {
	return fmt.Sprintf("%s <%s>", s.Name, s.Email)
}

// ObjectIter provides an iterator for a set of objects.
type ObjectIter struct {
	storer.EncodedObjectIter
	s storer.EncodedObjectStorer
}

// NewObjectIter takes a storer.EncodedObjectStorer and a
// storer.EncodedObjectIter and returns an *ObjectIter that iterates over all
// objects contained in the storer.EncodedObjectIter.
func NewObjectIter(s storer.EncodedObjectStorer, iter storer.EncodedObjectIter) *ObjectIter {
	return &ObjectIter{iter, s}
}

// Next moves the iterator to the next object and returns a pointer to it. If
// there are no more objects, it returns io.EOF.
func (iter *ObjectIter) Next() (Object, error) {
	for {
		obj, err := iter.EncodedObjectIter.Next()
		if err != nil {
			return nil, err
		}

		o, err := iter.toObject(obj)
		if err == plumbing.ErrInvalidType {
			continue
		}

		if err != nil {
			return nil, err
		}

		return o, nil
	}
}

// ForEach call the cb function for each object contained on this iter until
// an error happens or the end of the iter is reached. If ErrStop is sent
// the iteration is stop but no error is returned. The iterator is closed.
func (iter *ObjectIter) ForEach(cb func(Object) error) error {
	return iter.EncodedObjectIter.ForEach(func(obj plumbing.EncodedObject) error {
		o, err := iter.toObject(obj)
		if err == plumbing.ErrInvalidType {
			return nil
		}

		if err != nil {
			return err
		}

		return cb(o)
	})
}

func (iter *ObjectIter) toObject(obj plumbing.EncodedObject) (Object, error) {
	switch obj.Type() {
	case plumbing.BlobObject:
		blob := &Blob{}
		return blob, blob.Decode(obj)
	case plumbing.TreeObject:
		tree := &Tree{s: iter.s}
		return tree, tree.Decode(obj)
	case plumbing.CommitObject:
		commit := &Commit{}
		return commit, commit.Decode(obj)
	case plumbing.TagObject:
		tag := &Tag{}
		return tag, tag.Decode(obj)
	default:
		return nil, plumbing.ErrInvalidType
	}
}
<|endoftext|>


<commit_before>// EXAMPLE FROM: https://github.com/GoogleCloudPlatform/appengine-angular-gotodos
//
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// gotodos is an App Engine JSON backend for managing a todo list.
//
// It supports the following commands:
//
// - Create a new todo
// POST /todos
// > {"text": "do this"}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": false}
//
// - Update an existing todo
// POST /todos
// > {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
//
// - List existing todos:
// GET /todos
// >
// < [{"id": 1, "text": "do this", "created": 1356724843.0, "done": true},
//    {"id": 2, "text": "do that", "created": 1356724849.0, "done": false}]
//
// - Delete 'done' todos:
// DELETE /todos
// >
// <

package controllers

import (
	"encoding/json"
	"errors"
	"io"
	"log"
	"net/http"
	"regexp"
	"strings"

	"models"

	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"

	"github.com/PuerkitoBio/goquery"
	"github.com/astaxie/beegae"
	"github.com/astaxie/beego/validation"
)

type IOSAppController struct {
	beegae.Controller
}

func (this *IOSAppController) Get() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
}

type ErrorMessage struct {
	Status  string `json:"status"`
	Message string `json:"message"`
}

func (this *IOSAppController) Post() {
	iosapp, err := decodeIOSApp(this.Ctx.Input.Request.Body)
	if err != nil {
		errorMessage := ErrorMessage{Status: "error", Message: err.Error()}
		this.Data["json"] = errorMessage
		return
	}
	i, err := iosapp.Create(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) GetEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	} else {
		this.Data["json"] = &iosapp
	}
}
func (this *IOSAppController) UpdateEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	err = json.NewDecoder(this.Ctx.Input.Request.Body).Decode(&iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	i, err := iosapp.Update(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) DeleteEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	err := datastore.Delete(this.AppEngineCtx, key)
	if err == nil {
		this.Data["json"] = nil
	} else {
		this.Data["json"] = err
	}
}

func (this *IOSAppController) GetAppReview() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	client := urlfetch.Client(this.AppEngineCtx)
	req, err := http.NewRequest("GET", "https://itunes.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?pageNumber=0&sortOrdering=4&onlyLatestVersion=false&type=Purple+Software&id="+keyName, nil)
	req.Header.Add("X-Apple-Store-Front", iosapp.Region)
	req.Header.Add("User-Agent", "iTunes/9.2 (Macintosh; U; Mac OS X 10.6)")
	resp, err := client.Do(req)
	if err != nil {
		this.Data["json"] = err
		return
	}
	defer resp.Body.Close()
	regex_str := "([0-9]{4,}$)"
	re, err := regexp.Compile(regex_str)
	if err != nil {
		panic(err)
	}
	// regex_str_user_profile := "(userProfileId=[0-9]{4,}$)"
	// re_user_profile, err := regexp.Compile(regex_str_user_profile)
	// if err != nil {
	// 	panic(err)
	// }
	doc, _ := goquery.NewDocumentFromResponse(resp)
	doc.Find("Document View VBoxView View MatrixView VBoxView:nth-child(1) VBoxView VBoxView VBoxView").Each(func(_ int, s *goquery.Selection) {
		titleNode := s.Find("HBoxView>TextView>SetFontStyle>b").First()
		title := titleNode.Text()
		if title != "" {
			reviewIDURL, idExists := s.Find("HBoxView VBoxView GotoURL").First().Attr("url")
			if idExists {
				reviewID := re.FindString(reviewIDURL)
				var content string
				var versionAndDate string
				if len(reviewID) > 4 {
					num := 0
					log.Println(title)
					log.Println(reviewID)
					s.Find("TextView SetFontStyle").Each(func(_ int, sc *goquery.Selection) {
						num = num + 1
						if num == 4 {
							content = sc.Text()
							log.Println(content)
						}
					})
					userProfileNode := s.Find("HBoxView TextView SetFontStyle GotoURL").First()
					versionAndDate = userProfileNode.Parent().Text()
					versionAndDate = strings.Replace(versionAndDate, "\n", "", -1)
					versionAndDate = strings.Replace(versionAndDate, " ", "", -1)
					log.Printf("version and date: %v", versionAndDate)
					var appreview models.AppReview
					appreview.AppID = keyName
					appreview.ReviewID = reviewID
					appreview.Title = title
					appreview.Content = content
					appreview.Version = versionAndDate
					_, err = appreview.Create(this.AppEngineCtx)
					if err != nil {
						this.Data["json"] = err
						return
					}
				}

			}
		}

	})
}

func (this *IOSAppController) GetReviews() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	for i := 0; i < len(iosapps); i++ {
		log.Println(iosapps[i].AppID)
		t := taskqueue.NewPOSTTask("/admin/task/iosapp/getappreview/"+iosapps[i].AppID, nil)
		if _, err := taskqueue.Add(this.AppEngineCtx, t, ""); err != nil {
			this.Data["json"] = err
			return
		}
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
	// TODO set taskque to get app reviews
}

func (this *IOSAppController) Render() error {
	if _, ok := this.Data["json"].(error); ok {
		this.AppEngineCtx.Errorf("iosapp error: %v", this.Data["json"])
	}
	this.ServeJson()
	return nil
}

func decodeIOSApp(r io.ReadCloser) (*models.IOSApp, error) {
	defer r.Close()
	var iosapp models.IOSApp
	err := json.NewDecoder(r).Decode(&iosapp)
	valid := validation.Validation{}
	valid.Required(iosapp.AppID, "app_id")
	valid.Numeric(iosapp.AppID, "app_id")
	valid.Numeric(iosapp.Region, "region")
	regex_str := "^http"
	re, err := regexp.Compile(regex_str)
	valid.Match(iosapp.WebhookURL, re, "webhook_url")
	if valid.HasErrors() {
		for _, err := range valid.Errors {
			log.Println(err.Key, err.Message)
			return nil, errors.New(err.Key + " " + err.Message)
		}
	}
	return &iosapp, err
}
<commit_msg>fix validation message<commit_after>// EXAMPLE FROM: https://github.com/GoogleCloudPlatform/appengine-angular-gotodos
//
// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

// gotodos is an App Engine JSON backend for managing a todo list.
//
// It supports the following commands:
//
// - Create a new todo
// POST /todos
// > {"text": "do this"}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": false}
//
// - Update an existing todo
// POST /todos
// > {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
// < {"id": 1, "text": "do this", "created": 1356724843.0, "done": true}
//
// - List existing todos:
// GET /todos
// >
// < [{"id": 1, "text": "do this", "created": 1356724843.0, "done": true},
//    {"id": 2, "text": "do that", "created": 1356724849.0, "done": false}]
//
// - Delete 'done' todos:
// DELETE /todos
// >
// <

package controllers

import (
	"encoding/json"
	"errors"
	"io"
	"log"
	"net/http"
	"regexp"
	"strings"

	"models"

	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"

	"github.com/PuerkitoBio/goquery"
	"github.com/astaxie/beegae"
	"github.com/astaxie/beego/validation"
)

type IOSAppController struct {
	beegae.Controller
}

func (this *IOSAppController) Get() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
}

type ErrorMessage struct {
	Status  string `json:"status"`
	Message string `json:"message"`
}

func (this *IOSAppController) Post() {
	iosapp, err := decodeIOSApp(this.Ctx.Input.Request.Body)
	if err != nil {
		errorMessage := ErrorMessage{Status: "error", Message: err.Error()}
		this.Data["json"] = errorMessage
		// this.Ctx.Output.SetStatus(400)
		return
	}
	i, err := iosapp.Create(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) GetEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	} else {
		this.Data["json"] = &iosapp
	}
}
func (this *IOSAppController) UpdateEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	err = json.NewDecoder(this.Ctx.Input.Request.Body).Decode(&iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	i, err := iosapp.Update(this.AppEngineCtx)
	if err != nil {
		this.Data["json"] = err
	} else {
		this.Data["json"] = &i
	}
}

func (this *IOSAppController) DeleteEntity() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	err := datastore.Delete(this.AppEngineCtx, key)
	if err == nil {
		this.Data["json"] = nil
	} else {
		this.Data["json"] = err
	}
}

func (this *IOSAppController) GetAppReview() {
	keyName := this.Ctx.Input.Param(":key_name")
	key := datastore.NewKey(this.AppEngineCtx, "IOSApp", keyName, 0, nil)
	var iosapp models.IOSApp
	err := datastore.Get(this.AppEngineCtx, key, &iosapp)
	if err != nil {
		this.Data["json"] = err
		return
	}
	client := urlfetch.Client(this.AppEngineCtx)
	req, err := http.NewRequest("GET", "https://itunes.apple.com/WebObjects/MZStore.woa/wa/viewContentsUserReviews?pageNumber=0&sortOrdering=4&onlyLatestVersion=false&type=Purple+Software&id="+keyName, nil)
	req.Header.Add("X-Apple-Store-Front", iosapp.Region)
	req.Header.Add("User-Agent", "iTunes/9.2 (Macintosh; U; Mac OS X 10.6)")
	resp, err := client.Do(req)
	if err != nil {
		this.Data["json"] = err
		return
	}
	defer resp.Body.Close()
	regex_str := "([0-9]{4,}$)"
	re, err := regexp.Compile(regex_str)
	if err != nil {
		panic(err)
	}
	// regex_str_user_profile := "(userProfileId=[0-9]{4,}$)"
	// re_user_profile, err := regexp.Compile(regex_str_user_profile)
	// if err != nil {
	// 	panic(err)
	// }
	doc, _ := goquery.NewDocumentFromResponse(resp)
	doc.Find("Document View VBoxView View MatrixView VBoxView:nth-child(1) VBoxView VBoxView VBoxView").Each(func(_ int, s *goquery.Selection) {
		titleNode := s.Find("HBoxView>TextView>SetFontStyle>b").First()
		title := titleNode.Text()
		if title != "" {
			reviewIDURL, idExists := s.Find("HBoxView VBoxView GotoURL").First().Attr("url")
			if idExists {
				reviewID := re.FindString(reviewIDURL)
				var content string
				var versionAndDate string
				if len(reviewID) > 4 {
					num := 0
					log.Println(title)
					log.Println(reviewID)
					s.Find("TextView SetFontStyle").Each(func(_ int, sc *goquery.Selection) {
						num = num + 1
						if num == 4 {
							content = sc.Text()
							log.Println(content)
						}
					})
					userProfileNode := s.Find("HBoxView TextView SetFontStyle GotoURL").First()
					versionAndDate = userProfileNode.Parent().Text()
					versionAndDate = strings.Replace(versionAndDate, "\n", "", -1)
					versionAndDate = strings.Replace(versionAndDate, " ", "", -1)
					log.Printf("version and date: %v", versionAndDate)
					var appreview models.AppReview
					appreview.AppID = keyName
					appreview.ReviewID = reviewID
					appreview.Title = title
					appreview.Content = content
					appreview.Version = versionAndDate
					_, err = appreview.Create(this.AppEngineCtx)
					if err != nil {
						this.Data["json"] = err
						return
					}
				}

			}
		}

	})
}

func (this *IOSAppController) GetReviews() {
	iosapps := []models.IOSApp{}
	_, err := datastore.NewQuery("IOSApp").Order("-UpdatedAt").GetAll(this.AppEngineCtx, &iosapps)
	if err != nil {
		this.Data["json"] = err
		return
	}
	for i := 0; i < len(iosapps); i++ {
		log.Println(iosapps[i].AppID)
		t := taskqueue.NewPOSTTask("/admin/task/iosapp/getappreview/"+iosapps[i].AppID, nil)
		if _, err := taskqueue.Add(this.AppEngineCtx, t, ""); err != nil {
			this.Data["json"] = err
			return
		}
	}
	listDataSet := map[string]interface{}{"items": iosapps}
	this.Data["json"] = listDataSet
	// TODO set taskque to get app reviews
}

func (this *IOSAppController) Render() error {
	if _, ok := this.Data["json"].(error); ok {
		this.AppEngineCtx.Errorf("iosapp error: %v", this.Data["json"])
	}
	this.ServeJson()
	return nil
}

func decodeIOSApp(r io.ReadCloser) (*models.IOSApp, error) {
	defer r.Close()
	var iosapp models.IOSApp
	err := json.NewDecoder(r).Decode(&iosapp)
	valid := validation.Validation{}
	valid.Required(iosapp.AppID, "App ID")
	valid.Numeric(iosapp.AppID, "App ID")
	valid.Numeric(iosapp.Region, "Region Code")
	regex_str := "^http"
	re, err := regexp.Compile(regex_str)
	valid.Match(iosapp.WebhookURL, re, "Webhook URL")
	if valid.HasErrors() {
		for _, err := range valid.Errors {
			log.Println(err.Key, err.Message)
			return nil, errors.New(err.Key + " " + err.Message)
		}
	}
	return &iosapp, err
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"

	"github.com/ellcrys/util"
	"github.com/ncodes/cocoon/core/common"
	"github.com/ncodes/cocoon/core/config"
	"github.com/ncodes/cocoon/core/connector/launcher"
	"github.com/ncodes/cocoon/core/connector/server"
	"github.com/ncodes/cocoon/core/scheduler"
	logging "github.com/op/go-logging"
	"github.com/spf13/cobra"
)

func init() {
	config.ConfigureLogger()
}

// creates a deployment request with argument
// fetched from the environment.
func getRequest() (*launcher.Request, error) {

	// get cocoon code github link and language
	ccID := os.Getenv("COCOON_ID")
	ccURL := os.Getenv("COCOON_CODE_URL")
	ccTag := os.Getenv("COCOON_CODE_TAG")
	ccLang := os.Getenv("COCOON_CODE_LANG")
	diskLimit := util.Env("COCOON_DISK_LIMIT", "300")
	buildParam := os.Getenv("COCOON_BUILD_PARAMS")
	ccLink := os.Getenv("COCOON_LINK")
	ccAddr := scheduler.Getenv("ADDR_COCOON_RPC", "127.0.0.1:8000")

	if ccID == "" {
		return nil, fmt.Errorf("Cocoon code id not set @ $COCOON_ID")
	} else if ccURL == "" {
		return nil, fmt.Errorf("Cocoon code url not set @ $COCOON_CODE_URL")
	} else if ccLang == "" {
		return nil, fmt.Errorf("Cocoon code url not set @ $COCOON_CODE_LANG")
	}

	return &launcher.Request{
		ID:          ccID,
		URL:         ccURL,
		Tag:         ccTag,
		Lang:        ccLang,
		DiskLimit:   common.MBToByte(util.ToInt64(diskLimit)),
		BuildParams: buildParam,
		Link:        ccLink,
		CocoonAddr:  ccAddr,
	}, nil
}

// connectorCmd represents the connector command
var connectorCmd = &cobra.Command{
	Use:   "connector",
	Short: "Start the connector",
	Long:  `Starts the connector and launches a cocoon code.`,
	Run: func(cmd *cobra.Command, args []string) {

		waitCh := make(chan bool, 1)

		var log = logging.MustGetLogger("connector")
		log.Info("Connector started. Initiating cocoon code launch procedure.")

		// get request
		req, err := getRequest()
		if err != nil {
			log.Error(err.Error())
			return
		}

		// install cooncode
		lchr := launcher.NewLauncher(waitCh)
		lchr.AddLanguage(launcher.NewGo(req))
		go lchr.Launch(req)

		// start grpc API server
		grpcServer := server.NewAPIServer(lchr)
		addr := scheduler.Getenv("ADDR_CONNECTOR_RPC", "127.0.0.1:8002")
		go grpcServer.Start(addr, make(chan bool, 1))

		// httpServer := server.NewHTTPServer()
		// httpServerAddr := util.Env(scheduler.Getenv("IP_connector_http"), "")
		// httpServerPort := util.Env(scheduler.Getenv("PORT_connector_http"), "8003")
		// go httpServer.Start(fmt.Sprintf("%s:%s", httpServerAddr, httpServerPort))

		if <-waitCh {
			grpcServer.Stop(1)
			log.Fatal("launcher has failed")
		} else {
			grpcServer.Stop(0)
		}

		log.Info("launcher successfully exited")
	},
}

func init() {
	RootCmd.AddCommand(connectorCmd)
}
<commit_msg>cocoon addr should include the loopback interface ip. We don't need the allocated IP from the nomad<commit_after>package cmd

import (
	"fmt"
	"os"

	"github.com/ellcrys/util"
	"github.com/ncodes/cocoon/core/common"
	"github.com/ncodes/cocoon/core/config"
	"github.com/ncodes/cocoon/core/connector/launcher"
	"github.com/ncodes/cocoon/core/connector/server"
	"github.com/ncodes/cocoon/core/scheduler"
	logging "github.com/op/go-logging"
	"github.com/spf13/cobra"
)

func init() {
	config.ConfigureLogger()
}

// creates a deployment request with argument
// fetched from the environment.
func getRequest() (*launcher.Request, error) {

	// get cocoon code github link and language
	ccID := os.Getenv("COCOON_ID")
	ccURL := os.Getenv("COCOON_CODE_URL")
	ccTag := os.Getenv("COCOON_CODE_TAG")
	ccLang := os.Getenv("COCOON_CODE_LANG")
	diskLimit := util.Env("COCOON_DISK_LIMIT", "300")
	buildParam := os.Getenv("COCOON_BUILD_PARAMS")
	ccLink := os.Getenv("COCOON_LINK")
	ccAddr := fmt.Sprintf("127.0.0.1:%s", scheduler.Getenv("PORT_COCOON_RPC", "8000"))

	if ccID == "" {
		return nil, fmt.Errorf("Cocoon code id not set @ $COCOON_ID")
	} else if ccURL == "" {
		return nil, fmt.Errorf("Cocoon code url not set @ $COCOON_CODE_URL")
	} else if ccLang == "" {
		return nil, fmt.Errorf("Cocoon code url not set @ $COCOON_CODE_LANG")
	}

	return &launcher.Request{
		ID:          ccID,
		URL:         ccURL,
		Tag:         ccTag,
		Lang:        ccLang,
		DiskLimit:   common.MBToByte(util.ToInt64(diskLimit)),
		BuildParams: buildParam,
		Link:        ccLink,
		CocoonAddr:  ccAddr,
	}, nil
}

// connectorCmd represents the connector command
var connectorCmd = &cobra.Command{
	Use:   "connector",
	Short: "Start the connector",
	Long:  `Starts the connector and launches a cocoon code.`,
	Run: func(cmd *cobra.Command, args []string) {

		waitCh := make(chan bool, 1)

		var log = logging.MustGetLogger("connector")
		log.Info("Connector started. Initiating cocoon code launch procedure.")

		// get request
		req, err := getRequest()
		if err != nil {
			log.Error(err.Error())
			return
		}

		// install cooncode
		lchr := launcher.NewLauncher(waitCh)
		lchr.AddLanguage(launcher.NewGo(req))
		go lchr.Launch(req)

		// start grpc API server
		grpcServer := server.NewAPIServer(lchr)
		addr := scheduler.Getenv("ADDR_CONNECTOR_RPC", "127.0.0.1:8002")
		go grpcServer.Start(addr, make(chan bool, 1))

		// httpServer := server.NewHTTPServer()
		// httpServerAddr := util.Env(scheduler.Getenv("IP_connector_http"), "")
		// httpServerPort := util.Env(scheduler.Getenv("PORT_connector_http"), "8003")
		// go httpServer.Start(fmt.Sprintf("%s:%s", httpServerAddr, httpServerPort))

		if <-waitCh {
			grpcServer.Stop(1)
			log.Fatal("launcher has failed")
		} else {
			grpcServer.Stop(0)
		}

		log.Info("launcher successfully exited")
	},
}

func init() {
	RootCmd.AddCommand(connectorCmd)
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"time"

	humanize "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize"

	cmds "github.com/ipfs/go-ipfs/commands"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
	metrics "gx/ipfs/QmUBogf4nUefBjmYjn6jfsfPJRkmDGSeMhNj4usRKq69f4/go-libp2p/p2p/metrics"
	peer "gx/ipfs/QmUBogf4nUefBjmYjn6jfsfPJRkmDGSeMhNj4usRKq69f4/go-libp2p/p2p/peer"
	protocol "gx/ipfs/QmUBogf4nUefBjmYjn6jfsfPJRkmDGSeMhNj4usRKq69f4/go-libp2p/p2p/protocol"
)

var StatsCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline:          "Query IPFS statistics.",
		ShortDescription: ``,
	},

	Subcommands: map[string]*cmds.Command{
		"bw": statBwCmd,
	},
}

var statBwCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline:          "Print ipfs bandwidth information.",
		ShortDescription: ``,
	},
	Options: []cmds.Option{
		cmds.StringOption("peer", "p", "Specify a peer to print bandwidth for."),
		cmds.StringOption("proto", "t", "Specify a protocol to print bandwidth for."),
		cmds.BoolOption("poll", "Print bandwidth at an interval."),
		cmds.StringOption("interval", "i", "Time interval to wait between updating output."),
	},

	Run: func(req cmds.Request, res cmds.Response) {
		nd, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		// Must be online!
		if !nd.OnlineMode() {
			res.SetError(errNotOnline, cmds.ErrClient)
			return
		}

		pstr, pfound, err := req.Option("peer").String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		tstr, tfound, err := req.Option("proto").String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if pfound && tfound {
			res.SetError(errors.New("please only specify peer OR protocol"), cmds.ErrClient)
			return
		}

		var pid peer.ID
		if pfound {
			checkpid, err := peer.IDB58Decode(pstr)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			pid = checkpid
		}

		interval := time.Second
		timeS, found, err := req.Option("interval").String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if found {
			v, err := time.ParseDuration(timeS)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			interval = v
		}

		doPoll, _, err := req.Option("poll").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		out := make(chan interface{})
		res.SetOutput((<-chan interface{})(out))

		go func() {
			defer close(out)
			for {
				if pfound {
					stats := nd.Reporter.GetBandwidthForPeer(pid)
					out <- &stats
				} else if tfound {
					protoId := protocol.ID(tstr)
					stats := nd.Reporter.GetBandwidthForProtocol(protoId)
					out <- &stats
				} else {
					totals := nd.Reporter.GetBandwidthTotals()
					out <- &totals
				}
				if !doPoll {
					return
				}
				select {
				case <-time.After(interval):
				case <-req.Context().Done():
					return
				}
			}
		}()
	},
	Type: metrics.Stats{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			outCh, ok := res.Output().(<-chan interface{})
			if !ok {
				return nil, u.ErrCast()
			}

			polling, _, err := res.Request().Option("poll").Bool()
			if err != nil {
				return nil, err
			}

			first := true
			marshal := func(v interface{}) (io.Reader, error) {
				bs, ok := v.(*metrics.Stats)
				if !ok {
					return nil, u.ErrCast()
				}
				out := new(bytes.Buffer)
				if !polling {
					printStats(out, bs)
				} else {
					if first {
						fmt.Fprintln(out, "Total Up\t Total Down\t Rate Up\t Rate Down")
						first = false
					}
					fmt.Fprint(out, "\r")
					fmt.Fprintf(out, "%s \t\t", humanize.Bytes(uint64(bs.TotalOut)))
					fmt.Fprintf(out, " %s \t\t", humanize.Bytes(uint64(bs.TotalIn)))
					fmt.Fprintf(out, " %s/s   \t", humanize.Bytes(uint64(bs.RateOut)))
					fmt.Fprintf(out, " %s/s     ", humanize.Bytes(uint64(bs.RateIn)))
				}
				return out, nil

			}

			return &cmds.ChannelMarshaler{
				Channel:   outCh,
				Marshaler: marshal,
				Res:       res,
			}, nil
		},
	},
}

func printStats(out io.Writer, bs *metrics.Stats) {
	fmt.Fprintln(out, "Bandwidth")
	fmt.Fprintf(out, "TotalIn: %s\n", humanize.Bytes(uint64(bs.TotalIn)))
	fmt.Fprintf(out, "TotalOut: %s\n", humanize.Bytes(uint64(bs.TotalOut)))
	fmt.Fprintf(out, "RateIn: %s/s\n", humanize.Bytes(uint64(bs.RateIn)))
	fmt.Fprintf(out, "RateOut: %s/s\n", humanize.Bytes(uint64(bs.RateOut)))
}
<commit_msg>Added default false to stats bw poll<commit_after>package commands

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"time"

	humanize "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/go-humanize"

	cmds "github.com/ipfs/go-ipfs/commands"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
	metrics "gx/ipfs/QmUBogf4nUefBjmYjn6jfsfPJRkmDGSeMhNj4usRKq69f4/go-libp2p/p2p/metrics"
	peer "gx/ipfs/QmUBogf4nUefBjmYjn6jfsfPJRkmDGSeMhNj4usRKq69f4/go-libp2p/p2p/peer"
	protocol "gx/ipfs/QmUBogf4nUefBjmYjn6jfsfPJRkmDGSeMhNj4usRKq69f4/go-libp2p/p2p/protocol"
)

var StatsCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline:          "Query IPFS statistics.",
		ShortDescription: ``,
	},

	Subcommands: map[string]*cmds.Command{
		"bw": statBwCmd,
	},
}

var statBwCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline:          "Print ipfs bandwidth information.",
		ShortDescription: ``,
	},
	Options: []cmds.Option{
		cmds.StringOption("peer", "p", "Specify a peer to print bandwidth for."),
		cmds.StringOption("proto", "t", "Specify a protocol to print bandwidth for."),
		cmds.BoolOption("poll", "Print bandwidth at an interval. Default: false."),
		cmds.StringOption("interval", "i", "Time interval to wait between updating output."),
	},

	Run: func(req cmds.Request, res cmds.Response) {
		nd, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		// Must be online!
		if !nd.OnlineMode() {
			res.SetError(errNotOnline, cmds.ErrClient)
			return
		}

		pstr, pfound, err := req.Option("peer").String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		tstr, tfound, err := req.Option("proto").String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if pfound && tfound {
			res.SetError(errors.New("please only specify peer OR protocol"), cmds.ErrClient)
			return
		}

		var pid peer.ID
		if pfound {
			checkpid, err := peer.IDB58Decode(pstr)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			pid = checkpid
		}

		interval := time.Second
		timeS, found, err := req.Option("interval").String()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		if found {
			v, err := time.ParseDuration(timeS)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			interval = v
		}

		doPoll, _, err := req.Option("poll").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		out := make(chan interface{})
		res.SetOutput((<-chan interface{})(out))

		go func() {
			defer close(out)
			for {
				if pfound {
					stats := nd.Reporter.GetBandwidthForPeer(pid)
					out <- &stats
				} else if tfound {
					protoId := protocol.ID(tstr)
					stats := nd.Reporter.GetBandwidthForProtocol(protoId)
					out <- &stats
				} else {
					totals := nd.Reporter.GetBandwidthTotals()
					out <- &totals
				}
				if !doPoll {
					return
				}
				select {
				case <-time.After(interval):
				case <-req.Context().Done():
					return
				}
			}
		}()
	},
	Type: metrics.Stats{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			outCh, ok := res.Output().(<-chan interface{})
			if !ok {
				return nil, u.ErrCast()
			}

			polling, _, err := res.Request().Option("poll").Bool()
			if err != nil {
				return nil, err
			}

			first := true
			marshal := func(v interface{}) (io.Reader, error) {
				bs, ok := v.(*metrics.Stats)
				if !ok {
					return nil, u.ErrCast()
				}
				out := new(bytes.Buffer)
				if !polling {
					printStats(out, bs)
				} else {
					if first {
						fmt.Fprintln(out, "Total Up\t Total Down\t Rate Up\t Rate Down")
						first = false
					}
					fmt.Fprint(out, "\r")
					fmt.Fprintf(out, "%s \t\t", humanize.Bytes(uint64(bs.TotalOut)))
					fmt.Fprintf(out, " %s \t\t", humanize.Bytes(uint64(bs.TotalIn)))
					fmt.Fprintf(out, " %s/s   \t", humanize.Bytes(uint64(bs.RateOut)))
					fmt.Fprintf(out, " %s/s     ", humanize.Bytes(uint64(bs.RateIn)))
				}
				return out, nil

			}

			return &cmds.ChannelMarshaler{
				Channel:   outCh,
				Marshaler: marshal,
				Res:       res,
			}, nil
		},
	},
}

func printStats(out io.Writer, bs *metrics.Stats) {
	fmt.Fprintln(out, "Bandwidth")
	fmt.Fprintf(out, "TotalIn: %s\n", humanize.Bytes(uint64(bs.TotalIn)))
	fmt.Fprintf(out, "TotalOut: %s\n", humanize.Bytes(uint64(bs.TotalOut)))
	fmt.Fprintf(out, "RateIn: %s/s\n", humanize.Bytes(uint64(bs.RateIn)))
	fmt.Fprintf(out, "RateOut: %s/s\n", humanize.Bytes(uint64(bs.RateOut)))
}
<|endoftext|>


<commit_before>package formbuilder

func newPostError(err error) *postError {
	return &postError{err: err}
}

type postError struct {
	err error
}

func (e *postError) Unwrap() error {
	return e.err
}

func ErrPostFailed(err error) bool {
	_, ok := err.(*postError)
	return ok
}
<commit_msg>update<commit_after>package formbuilder

func newPostError(err error) *postError {
	return &postError{err: err}
}

type postError struct {
	err error
}

func (e *postError) Error() string {
	return e.err.Error()
}

func (e *postError) Unwrap() error {
	return e.err
}

func ErrPostFailed(err error) bool {
	_, ok := err.(*postError)
	return ok
}
<|endoftext|>


<commit_before>package core

import (
	"database/sql"
	"log"
	"time"

	"github.com/mdlayher/wavepipe/data"
)

// memFileSource represents a file source which indexes mock files from memory
type memFileSource struct{}

// mockFiles is a slice of mock files to index
var mockFiles = []data.Song{
	data.Song{
		Album: "Album",
		Artist: "Artist",
		Bitrate: 320,
		Channels: 2,
		FileName: "/mem/artist - song.mp3",
		FileSize: 1000,
		FileType: "MP3",
		LastModified: time.Now().Unix(),
		Length: 60,
		SampleRate: 44100,
		Title: "song",
		Track: 1,
		Year: 2014,
	},
}

// MediaScan adds mock media files to the database from memory
func (memFileSource) MediaScan(mediaFolder string, walkCancelChan chan struct{}) error {
	log.Println("mem: beginning mock media scan:", mediaFolder)

	// Iterate all media files and check for the matching prefix
	for _, song := range mockFiles {
		// Grab files with matching prefix
		if mediaFolder == song.FileName[0:len(mediaFolder)] {
			// Generate an artist model from this song's metadata
			artist := data.ArtistFromSong(&song)

			// Check for existing artist
			// Note: if the artist exists, this operation also loads necessary scanning information
			// such as their artist ID, for use in album and song generation
			if err := artist.Load(); err == sql.ErrNoRows {
				// Save new artist
				if err := artist.Save(); err != nil {
					log.Println(err)
				}
			}

			// Generate the album model from this song's metadata
			album := data.AlbumFromSong(&song)
			album.ArtistID = artist.ID

			// Check for existing album
			// Note: if the album exists, this operation also loads necessary scanning information
			// such as the album ID, for use in song generation
			if err := album.Load(); err == sql.ErrNoRows {
				// Save album
				if err := album.Save(); err != nil {
					log.Println(err)
				}
			}

			// Add ID fields to song
			song.ArtistID = artist.ID
			song.AlbumID = album.ID

			// Check for existing song
			if err := song.Load(); err == sql.ErrNoRows {
				// Save song (don't log these because they really slow things down)
				if err := song.Save(); err != nil {
					log.Println(err)
				}
			}
		}
	}

	log.Println("mem: mock media scan complete")
	return nil
}

// OrphanScan does nothing for mock media files, because the database is temporary anyway
func (memFileSource) OrphanScan(baseFolder string, subFolder string, orphanCancelChan chan struct{}) error {
	return nil
}
<commit_msg>go fmt memFileSource.go<commit_after>package core

import (
	"database/sql"
	"log"
	"time"

	"github.com/mdlayher/wavepipe/data"
)

// memFileSource represents a file source which indexes mock files from memory
type memFileSource struct{}

// mockFiles is a slice of mock files to index
var mockFiles = []data.Song{
	data.Song{
		Album:        "Album",
		Artist:       "Artist",
		Bitrate:      320,
		Channels:     2,
		FileName:     "/mem/artist - song.mp3",
		FileSize:     1000,
		FileType:     "MP3",
		LastModified: time.Now().Unix(),
		Length:       60,
		SampleRate:   44100,
		Title:        "song",
		Track:        1,
		Year:         2014,
	},
}

// MediaScan adds mock media files to the database from memory
func (memFileSource) MediaScan(mediaFolder string, walkCancelChan chan struct{}) error {
	log.Println("mem: beginning mock media scan:", mediaFolder)

	// Iterate all media files and check for the matching prefix
	for _, song := range mockFiles {
		// Grab files with matching prefix
		if mediaFolder == song.FileName[0:len(mediaFolder)] {
			// Generate an artist model from this song's metadata
			artist := data.ArtistFromSong(&song)

			// Check for existing artist
			// Note: if the artist exists, this operation also loads necessary scanning information
			// such as their artist ID, for use in album and song generation
			if err := artist.Load(); err == sql.ErrNoRows {
				// Save new artist
				if err := artist.Save(); err != nil {
					log.Println(err)
				}
			}

			// Generate the album model from this song's metadata
			album := data.AlbumFromSong(&song)
			album.ArtistID = artist.ID

			// Check for existing album
			// Note: if the album exists, this operation also loads necessary scanning information
			// such as the album ID, for use in song generation
			if err := album.Load(); err == sql.ErrNoRows {
				// Save album
				if err := album.Save(); err != nil {
					log.Println(err)
				}
			}

			// Add ID fields to song
			song.ArtistID = artist.ID
			song.AlbumID = album.ID

			// Check for existing song
			if err := song.Load(); err == sql.ErrNoRows {
				// Save song (don't log these because they really slow things down)
				if err := song.Save(); err != nil {
					log.Println(err)
				}
			}
		}
	}

	log.Println("mem: mock media scan complete")
	return nil
}

// OrphanScan does nothing for mock media files, because the database is temporary anyway
func (memFileSource) OrphanScan(baseFolder string, subFolder string, orphanCancelChan chan struct{}) error {
	return nil
}
<|endoftext|>