<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dataflowlib

import (
	"bytes"
	"fmt"
	"net/url"
	"path"

	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/mtime"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/exec"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/pipelinex"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/protox"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/stringx"
	pubsub_v1 "github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio/v1"
	pb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
	"github.com/golang/protobuf/proto"
	df "google.golang.org/api/dataflow/v1b3"
)

const (
	impulseKind    = "CreateCollection"
	parDoKind      = "ParallelDo"
	flattenKind    = "Flatten"
	gbkKind        = "GroupByKey"
	windowIntoKind = "Bucket"

	sideInputKind = "CollectionToSingleton"

	// Support for Dataflow native I/O, such as PubSub.
	readKind  = "ParallelRead"
	writeKind = "ParallelWrite"
)

// translate translates a pipeline into a sequence of Dataflow steps. The step
// representation and its semantics are complex. In particular, the service
// optimizes the steps (step fusing, etc.) and may move steps around. Our
// decorations of the steps must thus be robust against such changes, so that
// they can be properly decoded in the harness. There are multiple quirks and
// requirements of specific semi-opaque formats, such as base64 encoded blobs.
//
// Moreover, the harness sees pieces of the translated steps only -- not the
// full graph. Special steps are also inserted around GBK, for example, which
// makes the placement of the decoration somewhat tricky. The harness will
// also never see steps that the service executes directly, notably GBK/CoGBK.
func translate(p *pb.Pipeline) ([]*df.Step, error) {
	// NOTE: Dataflow apparently assumes that the steps are in topological order.
	// Otherwise, it fails with "Output out for step  was not found.". We assume
	// the pipeline has been normalized and each subtransform list is in such order.

	x := newTranslator(p.GetComponents())
	return x.translateTransforms("", p.GetRootTransformIds())
}

type translator struct {
	comp          *pb.Components
	pcollections  map[string]*outputReference
	coders        *graphx.CoderUnmarshaller
	bogusCoderRef *graphx.CoderRef
}

func newTranslator(comp *pb.Components) *translator {
	bytesCoderRef, _ := graphx.EncodeCoderRef(coder.NewW(coder.NewBytes(), coder.NewGlobalWindow()))

	return &translator{
		comp:          comp,
		pcollections:  makeOutputReferences(comp.GetTransforms()),
		coders:        graphx.NewCoderUnmarshaller(comp.GetCoders()),
		bogusCoderRef: bytesCoderRef,
	}
}

func (x *translator) translateTransforms(trunk string, ids []string) ([]*df.Step, error) {
	var steps []*df.Step
	for _, id := range ids {
		sub, err := x.translateTransform(trunk, id)
		if err != nil {
			return nil, err
		}
		steps = append(steps, sub...)
	}
	return steps, nil
}

func (x *translator) translateTransform(trunk string, id string) ([]*df.Step, error) {
	t := x.comp.Transforms[id]

	prop := properties{
		UserName:   userName(trunk, t.UniqueName),
		OutputInfo: x.translateOutputs(t.Outputs),
	}

	urn := t.GetSpec().GetUrn()
	switch urn {
	case graphx.URNImpulse:
		// NOTE: The impulse []data value is encoded in a special way as a
		// URL Query-escaped windowed _unnested_ value. It is read back in
		// a nested context at runtime.
		var buf bytes.Buffer
		if err := exec.EncodeWindowedValueHeader(exec.MakeWindowEncoder(coder.NewGlobalWindow()), window.SingleGlobalWindow, mtime.ZeroTimestamp, &buf); err != nil {
			return nil, err
		}
		value := string(append(buf.Bytes(), t.GetSpec().Payload...))
		// log.Printf("Impulse data: %v", url.QueryEscape(value))

		prop.Element = []string{url.QueryEscape(value)}
		return []*df.Step{x.newStep(id, impulseKind, prop)}, nil

	case graphx.URNParDo:
		var payload pb.ParDoPayload
		if err := proto.Unmarshal(t.Spec.Payload, &payload); err != nil {
			return nil, fmt.Errorf("invalid ParDo payload for %v: %v", t, err)
		}

		var steps []*df.Step
		rem := reflectx.ShallowClone(t.Inputs).(map[string]string)

		prop.NonParallelInputs = make(map[string]*outputReference)
		for key := range payload.SideInputs {
			// Side input require an additional conversion step, which must
			// be before the present one.
			delete(rem, key)

			pcol := x.comp.Pcollections[t.Inputs[key]]
			ref := x.pcollections[t.Inputs[key]]
			c := x.translateCoder(pcol, pcol.CoderId)

			side := &df.Step{
				Name: fmt.Sprintf("view%v_%v", id, key),
				Kind: sideInputKind,
				Properties: newMsg(properties{
					ParallelInput: ref,
					OutputInfo: []output{{
						UserName:   "i0",
						OutputName: "i0",
						Encoding:   graphx.WrapIterable(c),
					}},
					UserName: userName(trunk, fmt.Sprintf("AsView%v_%v", id, key)),
				}),
			}
			steps = append(steps, side)

			prop.NonParallelInputs[side.Name] = newOutputReference(side.Name, "i0")
		}

		in := stringx.SingleValue(rem)

		prop.ParallelInput = x.pcollections[in]
		prop.SerializedFn = id // == reference into the proto pipeline
		return append(steps, x.newStep(id, parDoKind, prop)), nil

	case graphx.URNFlatten:
		for _, in := range t.Inputs {
			prop.Inputs = append(prop.Inputs, x.pcollections[in])
		}
		return []*df.Step{x.newStep(id, flattenKind, prop)}, nil

	case graphx.URNGBK:
		in := stringx.SingleValue(t.Inputs)

		prop.ParallelInput = x.pcollections[in]
		prop.DisallowCombinerLifting = true
		prop.SerializedFn = encodeSerializedFn(x.extractWindowingStrategy(in))
		return []*df.Step{x.newStep(id, gbkKind, prop)}, nil

	case graphx.URNWindow:
		in := stringx.SingleValue(t.Inputs)
		out := stringx.SingleValue(t.Outputs)

		prop.ParallelInput = x.pcollections[in]
		prop.SerializedFn = encodeSerializedFn(x.extractWindowingStrategy(out))
		return []*df.Step{x.newStep(id, windowIntoKind, prop)}, nil

	case pubsub_v1.PubSubPayloadURN:
		// Translate to native handling of PubSub I/O.

		var msg pubsub_v1.PubSubPayload
		if err := proto.Unmarshal(t.Spec.Payload, &msg); err != nil {
			return nil, fmt.Errorf("bad pubsub payload: %v", err)
		}

		prop.Format = "pubsub"
		prop.PubSubTopic = msg.GetTopic()
		prop.PubSubSubscription = msg.GetSubscription()
		prop.PubSubIDLabel = msg.GetIdAttribute()
		prop.PubSubTimestampLabel = msg.GetTimestampAttribute()
		prop.PubSubWithAttributes = msg.GetWithAttributes()

		if prop.PubSubSubscription != "" {
			prop.PubSubTopic = ""
		}

		switch msg.Op {
		case pubsub_v1.PubSubPayload_READ:
			return []*df.Step{x.newStep(id, readKind, prop)}, nil

		case pubsub_v1.PubSubPayload_WRITE:
			in := stringx.SingleValue(t.Inputs)

			prop.ParallelInput = x.pcollections[in]
			prop.Encoding = x.wrapCoder(x.comp.Pcollections[in], coder.NewBytes())
			return []*df.Step{x.newStep(id, writeKind, prop)}, nil

		default:
			return nil, fmt.Errorf("bad pubsub op: %v", msg.Op)
		}

	default:
		// TODO: graphx.URNCombinePerKey:

		if len(t.Subtransforms) > 0 {
			return x.translateTransforms(fmt.Sprintf("%v%v/", trunk, path.Base(t.UniqueName)), t.Subtransforms)
		}

		return nil, fmt.Errorf("unexpected primitive urn: %v", t)
	}
}

func (x *translator) newStep(id, kind string, prop properties) *df.Step {
	step := &df.Step{
		Name:       id,
		Kind:       kind,
		Properties: newMsg(prop),
	}
	if prop.PubSubWithAttributes {
		// Hack to add a empty-value property for PubSub IO. This
		// will make PubSub send the entire message, not just
		// the payload.
		prop.PubSubWithAttributes = false
		step.Properties = newMsg(propertiesWithPubSubMessage{properties: prop})
	}
	return step
}

func (x *translator) translateOutputs(outputs map[string]string) []output {
	var ret []output
	for _, out := range outputs {
		pcol := x.comp.Pcollections[out]
		ref := x.pcollections[out]

		info := output{
			UserName:   ref.OutputName,
			OutputName: ref.OutputName,
			Encoding:   x.translateCoder(pcol, pcol.CoderId),
		}
		ret = append(ret, info)
	}
	if len(ret) == 0 {
		// Dataflow seems to require at least one output. We insert
		// a bogus one (named "bogus") and remove it in the harness.

		ret = []output{{
			UserName:   "bogus",
			OutputName: "bogus",
			Encoding:   x.bogusCoderRef,
		}}
	}
	return ret
}

func (x *translator) translateCoder(pcol *pb.PCollection, id string) *graphx.CoderRef {
	c, err := x.coders.Coder(id)
	if err != nil {
		panic(err)
	}
	return x.wrapCoder(pcol, c)
}

func (x *translator) wrapCoder(pcol *pb.PCollection, c *coder.Coder) *graphx.CoderRef {
	// TODO(herohde) 3/16/2018: ensure windowed values for Dataflow

	ws := x.comp.WindowingStrategies[pcol.WindowingStrategyId]
	wc, err := x.coders.WindowCoder(ws.WindowCoderId)
	if err != nil {
		panic(fmt.Sprintf("failed to decode window coder %v for windowing strategy %v: %v", ws.WindowCoderId, pcol.WindowingStrategyId, err))
	}
	ret, err := graphx.EncodeCoderRef(coder.NewW(c, wc))
	if err != nil {
		panic(fmt.Sprintf("failed to wrap coder %v for windowing strategy %v: %v", c, pcol.WindowingStrategyId, err))
	}
	return ret
}

// extractWindowingStrategy returns a self-contained windowing strategy from
// the given pcollection id.
func (x *translator) extractWindowingStrategy(pid string) *pb.MessageWithComponents {
	ws := x.comp.WindowingStrategies[x.comp.Pcollections[pid].WindowingStrategyId]

	msg := &pb.MessageWithComponents{
		Components: &pb.Components{
			Coders: pipelinex.TrimCoders(x.comp.Coders, ws.WindowCoderId),
		},
		Root: &pb.MessageWithComponents_WindowingStrategy{
			WindowingStrategy: ws,
		},
	}
	return msg
}

// makeOutputReferences builds a map from PCollection id to the Dataflow representation.
// Each output is named after the generating transform.
func makeOutputReferences(xforms map[string]*pb.PTransform) map[string]*outputReference {
	ret := make(map[string]*outputReference)
	for id, t := range xforms {
		if len(t.Subtransforms) > 0 {
			continue // ignore composites
		}
		for name, out := range t.Outputs {
			ret[out] = newOutputReference(id, name)
		}
	}
	return ret
}

// userName computes a Dataflow composite name understood by the Dataflow UI,
// determined by the scope nesting. Dataflow simply uses "/" to separate
// composite transforms, so we must remove them from the otherwise qualified
// package names of DoFns, etc. Assumes trunk ends in / or is empty.
func userName(trunk, name string) string {
	return fmt.Sprintf("%v%v", trunk, path.Base(name))
}

func encodeSerializedFn(in proto.Message) string {
	// The Beam Runner API uses percent-encoding for serialized fn messages.
	// See: https://en.wikipedia.org/wiki/Percent-encoding

	data := protox.MustEncode(in)
	return url.PathEscape(string(data))
}
<commit_msg>[BEAM-4276] Combiner lifting for Dataflow<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package dataflowlib

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/url"
	"path"

	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/coder"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/mtime"
	"github.com/apache/beam/sdks/go/pkg/beam/core/graph/window"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/exec"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/pipelinex"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/protox"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/reflectx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/stringx"
	pubsub_v1 "github.com/apache/beam/sdks/go/pkg/beam/io/pubsubio/v1"
	pb "github.com/apache/beam/sdks/go/pkg/beam/model/pipeline_v1"
	"github.com/golang/protobuf/proto"
	df "google.golang.org/api/dataflow/v1b3"
)

const (
	impulseKind    = "CreateCollection"
	parDoKind      = "ParallelDo"
	combineKind    = "CombineValues"
	flattenKind    = "Flatten"
	gbkKind        = "GroupByKey"
	windowIntoKind = "Bucket"

	sideInputKind = "CollectionToSingleton"

	// Support for Dataflow native I/O, such as PubSub.
	readKind  = "ParallelRead"
	writeKind = "ParallelWrite"
)

// translate translates a pipeline into a sequence of Dataflow steps. The step
// representation and its semantics are complex. In particular, the service
// optimizes the steps (step fusing, etc.) and may move steps around. Our
// decorations of the steps must thus be robust against such changes, so that
// they can be properly decoded in the harness. There are multiple quirks and
// requirements of specific semi-opaque formats, such as base64 encoded blobs.
//
// Moreover, the harness sees pieces of the translated steps only -- not the
// full graph. Special steps are also inserted around GBK, for example, which
// makes the placement of the decoration somewhat tricky. The harness will
// also never see steps that the service executes directly, notably GBK/CoGBK.
func translate(p *pb.Pipeline) ([]*df.Step, error) {
	// NOTE: Dataflow apparently assumes that the steps are in topological order.
	// Otherwise, it fails with "Output out for step  was not found.". We assume
	// the pipeline has been normalized and each subtransform list is in such order.

	x := newTranslator(p.GetComponents())
	return x.translateTransforms("", p.GetRootTransformIds())
}

type translator struct {
	comp          *pb.Components
	pcollections  map[string]*outputReference
	coders        *graphx.CoderUnmarshaller
	bogusCoderRef *graphx.CoderRef
}

func newTranslator(comp *pb.Components) *translator {
	bytesCoderRef, _ := graphx.EncodeCoderRef(coder.NewW(coder.NewBytes(), coder.NewGlobalWindow()))

	return &translator{
		comp:          comp,
		pcollections:  makeOutputReferences(comp.GetTransforms()),
		coders:        graphx.NewCoderUnmarshaller(comp.GetCoders()),
		bogusCoderRef: bytesCoderRef,
	}
}

func (x *translator) translateTransforms(trunk string, ids []string) ([]*df.Step, error) {
	var steps []*df.Step
	for _, id := range ids {
		sub, err := x.translateTransform(trunk, id)
		if err != nil {
			return nil, err
		}
		steps = append(steps, sub...)
	}
	return steps, nil
}

func (x *translator) translateTransform(trunk string, id string) ([]*df.Step, error) {
	t := x.comp.Transforms[id]

	prop := properties{
		UserName:   userName(trunk, t.UniqueName),
		OutputInfo: x.translateOutputs(t.Outputs),
	}

	urn := t.GetSpec().GetUrn()
	switch urn {
	case graphx.URNImpulse:
		// NOTE: The impulse []data value is encoded in a special way as a
		// URL Query-escaped windowed _unnested_ value. It is read back in
		// a nested context at runtime.
		var buf bytes.Buffer
		if err := exec.EncodeWindowedValueHeader(exec.MakeWindowEncoder(coder.NewGlobalWindow()), window.SingleGlobalWindow, mtime.ZeroTimestamp, &buf); err != nil {
			return nil, err
		}
		value := string(append(buf.Bytes(), t.GetSpec().Payload...))
		// log.Printf("Impulse data: %v", url.QueryEscape(value))

		prop.Element = []string{url.QueryEscape(value)}
		return []*df.Step{x.newStep(id, impulseKind, prop)}, nil

	case graphx.URNParDo:
		var payload pb.ParDoPayload
		if err := proto.Unmarshal(t.Spec.Payload, &payload); err != nil {
			return nil, fmt.Errorf("invalid ParDo payload for %v: %v", t, err)
		}

		var steps []*df.Step
		rem := reflectx.ShallowClone(t.Inputs).(map[string]string)

		prop.NonParallelInputs = make(map[string]*outputReference)
		for key := range payload.SideInputs {
			// Side input require an additional conversion step, which must
			// be before the present one.
			delete(rem, key)

			pcol := x.comp.Pcollections[t.Inputs[key]]
			ref := x.pcollections[t.Inputs[key]]
			c := x.translateCoder(pcol, pcol.CoderId)

			side := &df.Step{
				Name: fmt.Sprintf("view%v_%v", id, key),
				Kind: sideInputKind,
				Properties: newMsg(properties{
					ParallelInput: ref,
					OutputInfo: []output{{
						UserName:   "i0",
						OutputName: "i0",
						Encoding:   graphx.WrapIterable(c),
					}},
					UserName: userName(trunk, fmt.Sprintf("AsView%v_%v", id, key)),
				}),
			}
			steps = append(steps, side)

			prop.NonParallelInputs[side.Name] = newOutputReference(side.Name, "i0")
		}

		in := stringx.SingleValue(rem)

		prop.ParallelInput = x.pcollections[in]
		prop.SerializedFn = id // == reference into the proto pipeline
		return append(steps, x.newStep(id, parDoKind, prop)), nil
	case graphx.URNCombinePerKey:
		// Dataflow uses a GBK followed by a CombineValues to determine when it can lift.
		// To achieve this, we use the combine composite's subtransforms, and modify the
		// Combine ParDo with the CombineValues kind, set its SerializedFn to map to the
		// composite payload, and the accumulator coding.
		if len(t.Subtransforms) != 2 {
			return nil, fmt.Errorf("invalid CombinePerKey, expected 2 subtransforms but got %d in %v", len(t.Subtransforms), t)
		}
		steps, err := x.translateTransforms(fmt.Sprintf("%v%v/", trunk, path.Base(t.UniqueName)), t.Subtransforms)
		if err != nil {
			return nil, fmt.Errorf("invalid CombinePerKey, couldn't extract GBK from %v: %v", t, err)
		}
		var payload pb.CombinePayload
		if err := proto.Unmarshal(t.Spec.Payload, &payload); err != nil {
			return nil, fmt.Errorf("invalid Combine payload for %v: %v", t, err)
		}

		c, err := x.coders.Coder(payload.AccumulatorCoderId)
		if err != nil {
			return nil, fmt.Errorf("invalid Combine payload , missing Accumulator Coder %v: %v", t, err)
		}
		enc, err := graphx.EncodeCoderRef(c)
		if err != nil {
			return nil, fmt.Errorf("invalid Combine payload, couldn't encode Accumulator Coder %v: %v", t, err)
		}
		json.Unmarshal([]byte(steps[1].Properties), &prop)
		prop.Encoding = enc
		prop.SerializedFn = id
		steps[1].Kind = combineKind
		steps[1].Properties = newMsg(prop)
		return steps, nil

	case graphx.URNFlatten:
		for _, in := range t.Inputs {
			prop.Inputs = append(prop.Inputs, x.pcollections[in])
		}
		return []*df.Step{x.newStep(id, flattenKind, prop)}, nil

	case graphx.URNGBK:
		in := stringx.SingleValue(t.Inputs)

		prop.ParallelInput = x.pcollections[in]
		prop.SerializedFn = encodeSerializedFn(x.extractWindowingStrategy(in))
		return []*df.Step{x.newStep(id, gbkKind, prop)}, nil

	case graphx.URNWindow:
		in := stringx.SingleValue(t.Inputs)
		out := stringx.SingleValue(t.Outputs)

		prop.ParallelInput = x.pcollections[in]
		prop.SerializedFn = encodeSerializedFn(x.extractWindowingStrategy(out))
		return []*df.Step{x.newStep(id, windowIntoKind, prop)}, nil

	case pubsub_v1.PubSubPayloadURN:
		// Translate to native handling of PubSub I/O.

		var msg pubsub_v1.PubSubPayload
		if err := proto.Unmarshal(t.Spec.Payload, &msg); err != nil {
			return nil, fmt.Errorf("bad pubsub payload: %v", err)
		}

		prop.Format = "pubsub"
		prop.PubSubTopic = msg.GetTopic()
		prop.PubSubSubscription = msg.GetSubscription()
		prop.PubSubIDLabel = msg.GetIdAttribute()
		prop.PubSubTimestampLabel = msg.GetTimestampAttribute()
		prop.PubSubWithAttributes = msg.GetWithAttributes()

		if prop.PubSubSubscription != "" {
			prop.PubSubTopic = ""
		}

		switch msg.Op {
		case pubsub_v1.PubSubPayload_READ:
			return []*df.Step{x.newStep(id, readKind, prop)}, nil

		case pubsub_v1.PubSubPayload_WRITE:
			in := stringx.SingleValue(t.Inputs)

			prop.ParallelInput = x.pcollections[in]
			prop.Encoding = x.wrapCoder(x.comp.Pcollections[in], coder.NewBytes())
			return []*df.Step{x.newStep(id, writeKind, prop)}, nil

		default:
			return nil, fmt.Errorf("bad pubsub op: %v", msg.Op)
		}

	default:
		if len(t.Subtransforms) > 0 {
			return x.translateTransforms(fmt.Sprintf("%v%v/", trunk, path.Base(t.UniqueName)), t.Subtransforms)
		}

		return nil, fmt.Errorf("unexpected primitive urn: %v", t)
	}
}

func (x *translator) newStep(id, kind string, prop properties) *df.Step {
	step := &df.Step{
		Name:       id,
		Kind:       kind,
		Properties: newMsg(prop),
	}
	if prop.PubSubWithAttributes {
		// Hack to add a empty-value property for PubSub IO. This
		// will make PubSub send the entire message, not just
		// the payload.
		prop.PubSubWithAttributes = false
		step.Properties = newMsg(propertiesWithPubSubMessage{properties: prop})
	}
	return step
}

func (x *translator) translateOutputs(outputs map[string]string) []output {
	var ret []output
	for _, out := range outputs {
		pcol := x.comp.Pcollections[out]
		ref := x.pcollections[out]

		info := output{
			UserName:   ref.OutputName,
			OutputName: ref.OutputName,
			Encoding:   x.translateCoder(pcol, pcol.CoderId),
		}
		ret = append(ret, info)
	}
	if len(ret) == 0 {
		// Dataflow seems to require at least one output. We insert
		// a bogus one (named "bogus") and remove it in the harness.

		ret = []output{{
			UserName:   "bogus",
			OutputName: "bogus",
			Encoding:   x.bogusCoderRef,
		}}
	}
	return ret
}

func (x *translator) translateCoder(pcol *pb.PCollection, id string) *graphx.CoderRef {
	c, err := x.coders.Coder(id)
	if err != nil {
		panic(err)
	}
	return x.wrapCoder(pcol, c)
}

func (x *translator) wrapCoder(pcol *pb.PCollection, c *coder.Coder) *graphx.CoderRef {
	// TODO(herohde) 3/16/2018: ensure windowed values for Dataflow

	ws := x.comp.WindowingStrategies[pcol.WindowingStrategyId]
	wc, err := x.coders.WindowCoder(ws.WindowCoderId)
	if err != nil {
		panic(fmt.Sprintf("failed to decode window coder %v for windowing strategy %v: %v", ws.WindowCoderId, pcol.WindowingStrategyId, err))
	}
	ret, err := graphx.EncodeCoderRef(coder.NewW(c, wc))
	if err != nil {
		panic(fmt.Sprintf("failed to wrap coder %v for windowing strategy %v: %v", c, pcol.WindowingStrategyId, err))
	}
	return ret
}

// extractWindowingStrategy returns a self-contained windowing strategy from
// the given pcollection id.
func (x *translator) extractWindowingStrategy(pid string) *pb.MessageWithComponents {
	ws := x.comp.WindowingStrategies[x.comp.Pcollections[pid].WindowingStrategyId]

	msg := &pb.MessageWithComponents{
		Components: &pb.Components{
			Coders: pipelinex.TrimCoders(x.comp.Coders, ws.WindowCoderId),
		},
		Root: &pb.MessageWithComponents_WindowingStrategy{
			WindowingStrategy: ws,
		},
	}
	return msg
}

// makeOutputReferences builds a map from PCollection id to the Dataflow representation.
// Each output is named after the generating transform.
func makeOutputReferences(xforms map[string]*pb.PTransform) map[string]*outputReference {
	ret := make(map[string]*outputReference)
	for id, t := range xforms {
		if len(t.Subtransforms) > 0 {
			continue // ignore composites
		}
		for name, out := range t.Outputs {
			ret[out] = newOutputReference(id, name)
		}
	}
	return ret
}

// userName computes a Dataflow composite name understood by the Dataflow UI,
// determined by the scope nesting. Dataflow simply uses "/" to separate
// composite transforms, so we must remove them from the otherwise qualified
// package names of DoFns, etc. Assumes trunk ends in / or is empty.
func userName(trunk, name string) string {
	return fmt.Sprintf("%v%v", trunk, path.Base(name))
}

func encodeSerializedFn(in proto.Message) string {
	// The Beam Runner API uses percent-encoding for serialized fn messages.
	// See: https://en.wikipedia.org/wiki/Percent-encoding

	data := protox.MustEncode(in)
	return url.PathEscape(string(data))
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mysql

import "time"

var datetimeFormat = "2006-01-02 15:04:05.999999"

func toDatetime(t time.Time) string {
	return t.Format(datetimeFormat)
}

func parseDatetime(str string) (time.Time, error) {
	return time.Parse(datetimeFormat, str)
}
<commit_msg>storage/mysql: {to,parse}Datetime no longer needed<commit_after><|endoftext|>


<commit_before>// Package deep provides function deep.Equal which is like reflect.DeepEqual but
// retunrs a list of differences. This is helpful when comparing complex types
// like structures and maps.
package deep

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
)

var (
	// FloatPrecision is the number of decimal places to round float values
	// to when comparing.
	FloatPrecision = 10

	// MaxDiff specifies the maximum number of differences to return.
	MaxDiff = 10

	// MaxDepth specifies the maximum levels of a struct to recurse into.
	MaxDepth = 10

	// LogErrors causes errors to be logged to STDERR when true.
	LogErrors = false

	// CompareUnexportedFields causes unexported struct fields, like s in
	// T{s int}, to be comparsed when true.
	CompareUnexportedFields = false
)

var (
	// ErrMaxRecursion is logged when MaxDepth is reached.
	ErrMaxRecursion = errors.New("recursed to MaxDepth")

	// ErrTypeMismatch is logged when Equal passed two different types of values.
	ErrTypeMismatch = errors.New("variables are different reflect.Type")

	// ErrNotHandled is logged when a primitive Go kind is not handled.
	ErrNotHandled = errors.New("cannot compare the reflect.Kind")
)

type cmp struct {
	diff        []string
	buff        []string
	floatFormat string
}

var errorType = reflect.TypeOf((*error)(nil)).Elem()

// Equal compares variables a and b, recursing into their structure up to
// MaxDepth levels deep, and returns a list of differences, or nil if there are
// none. Some differences may not be found if an error is also returned.
//
// If a type has an Equal method, like time.Equal, it is called to check for
// equality.
func Equal(a, b interface{}) []string {
	aVal := reflect.ValueOf(a)
	bVal := reflect.ValueOf(b)
	c := &cmp{
		diff:        []string{},
		buff:        []string{},
		floatFormat: fmt.Sprintf("%%.%df", FloatPrecision),
	}
	if a == nil && b == nil {
		return nil
	} else if a == nil && b != nil {
		c.saveDiff(b, "<nil pointer>")
	} else if a != nil && b == nil {
		c.saveDiff(a, "<nil pointer>")
	}
	if len(c.diff) > 0 {
		return c.diff
	}

	c.equals(aVal, bVal, 0)
	if len(c.diff) > 0 {
		return c.diff // diffs
	}
	return nil // no diffs
}

func (c *cmp) equals(a, b reflect.Value, level int) {
	if level > MaxDepth {
		logError(ErrMaxRecursion)
		return
	}

	// If differenet types, they can't be equal
	aType := a.Type()
	bType := b.Type()
	if aType != bType {
		c.saveDiff(aType, bType)
		logError(ErrTypeMismatch)
		return
	}

	// Primitive https://golang.org/pkg/reflect/#Kind
	aKind := a.Kind()
	bKind := b.Kind()

	// If both types implement the error interface, compare the error strings.
	// This must be done before dereferencing because the interface is on a
	// pointer receiver.
	if aType.Implements(errorType) && bType.Implements(errorType) {
		if a.Elem().IsValid() && b.Elem().IsValid() { // both err != nil
			aString := a.MethodByName("Error").Call(nil)[0].String()
			bString := b.MethodByName("Error").Call(nil)[0].String()
			if aString != bString {
				c.saveDiff(aString, bString)
			}
			return
		}
	}

	// Dereference pointers and interface{}
	if aKind == reflect.Ptr || aKind == reflect.Interface {
		a = a.Elem()
		aKind = a.Kind()
		if a.IsValid() {
			aType = a.Type()
		}
	}
	if bKind == reflect.Ptr || bKind == reflect.Interface {
		b = b.Elem()
		if b.IsValid() {
			bType = b.Type()
		}
	}

	// Check if one value is nil, e.g. T{x: *X} and T.x is nil
	if !a.IsValid() || !b.IsValid() {
		if a.IsValid() && !b.IsValid() {
			c.saveDiff(aType, "<nil pointer>")
		} else if !a.IsValid() && b.IsValid() {
			c.saveDiff("<nil pointer>", bType)
		}
		return
	}

	// Types with an Equal(), like time.Time.
	eqFunc := a.MethodByName("Equal")
	if eqFunc.IsValid() {
		retVals := eqFunc.Call([]reflect.Value{b})
		if !retVals[0].Bool() {
			c.saveDiff(a, b)
		}
		return
	}

	switch aKind {

	/////////////////////////////////////////////////////////////////////
	// Iterable kinds
	/////////////////////////////////////////////////////////////////////

	case reflect.Struct:
		/*
			The variables are structs like:
				type T struct {
					FirstName string
					LastName  string
				}
			Type = <pkg>.T, Kind = reflect.Struct

			Iterate through the fields (FirstName, LastName), recurse into their values.
		*/
		for i := 0; i < a.NumField(); i++ {
			if aType.Field(i).PkgPath != "" && !CompareUnexportedFields {
				continue // skip unexported field, e.g. s in type T struct {s string}
			}

			c.push(aType.Field(i).Name) // push field name to buff

			// Get the Value for each field, e.g. FirstName has Type = string,
			// Kind = reflect.String.
			af := a.Field(i)
			bf := b.Field(i)

			// Recurse to compare the field values
			c.equals(af, bf, level+1)

			c.pop() // pop field name from buff

			if len(c.diff) >= MaxDiff {
				break
			}
		}
	case reflect.Map:
		/*
			The variables are maps like:
				map[string]int{
					"foo": 1,
					"bar": 2,
				}
			Type = map[string]int, Kind = reflect.Map

			Or:
				type T map[string]int{}
			Type = <pkg>.T, Kind = reflect.Map

			Iterate through the map keys (foo, bar), recurse into their values.
		*/

		if a.IsNil() || b.IsNil() {
			if a.IsNil() && !b.IsNil() {
				c.saveDiff("<nil map>", b)
			} else if !a.IsNil() && b.IsNil() {
				c.saveDiff(a, "<nil map>")
			}
			return
		}

		if a.Pointer() == b.Pointer() {
			return
		}

		for _, key := range a.MapKeys() {
			c.push(fmt.Sprintf("map[%s]", key))

			aVal := a.MapIndex(key)
			bVal := b.MapIndex(key)
			if bVal.IsValid() {
				c.equals(aVal, bVal, level+1)
			} else {
				c.saveDiff(aVal, "<does not have key>")
			}

			c.pop()

			if len(c.diff) >= MaxDiff {
				return
			}
		}

		for _, key := range b.MapKeys() {
			if aVal := a.MapIndex(key); aVal.IsValid() {
				continue
			}

			c.push(fmt.Sprintf("map[%s]", key))
			c.saveDiff("<does not have key>", b.MapIndex(key))
			c.pop()
			if len(c.diff) >= MaxDiff {
				return
			}
		}
	case reflect.Slice:
		if a.IsNil() || b.IsNil() {
			if a.IsNil() && !b.IsNil() {
				c.saveDiff("<nil slice>", b)
			} else if !a.IsNil() && b.IsNil() {
				c.saveDiff(a, "<nil slice>")
			}
			return
		}

		if a.Pointer() == b.Pointer() {
			return
		}

		aLen := a.Len()
		bLen := b.Len()
		n := aLen
		if bLen > aLen {
			n = bLen
		}
		for i := 0; i < n; i++ {
			c.push(fmt.Sprintf("slice[%d]", i))
			if i < aLen && i < bLen {
				c.equals(a.Index(i), b.Index(i), level+1)
			} else if i < aLen {
				c.saveDiff(a.Index(i), "<no value>")
			} else {
				c.saveDiff("<no value>", b.Index(i))
			}
			c.pop()
			if len(c.diff) >= MaxDiff {
				break
			}
		}

	/////////////////////////////////////////////////////////////////////
	// Primitive kinds
	/////////////////////////////////////////////////////////////////////

	case reflect.Float32, reflect.Float64:
		// Avoid 0.04147685731961082 != 0.041476857319611
		// 6 decimal places is close enough
		aval := fmt.Sprintf(c.floatFormat, a.Float())
		bval := fmt.Sprintf(c.floatFormat, b.Float())
		if aval != bval {
			c.saveDiff(a.Float(), b.Float())
		}
	case reflect.Bool:
		if a.Bool() != b.Bool() {
			c.saveDiff(a.Bool(), b.Bool())
		}
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if a.Int() != b.Int() {
			c.saveDiff(a.Int(), b.Int())
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		if a.Uint() != b.Uint() {
			c.saveDiff(a.Uint(), b.Uint())
		}
	case reflect.String:
		if a.String() != b.String() {
			c.saveDiff(a.String(), b.String())
		}

	default:
		logError(ErrNotHandled)
	}
}

func (c *cmp) push(name string) {
	c.buff = append(c.buff, name)
}

func (c *cmp) pop() {
	if len(c.buff) > 0 {
		c.buff = c.buff[0 : len(c.buff)-1]
	}
}

func (c *cmp) saveDiff(aval, bval interface{}) {
	if len(c.buff) > 0 {
		varName := strings.Join(c.buff, ".")
		c.diff = append(c.diff, fmt.Sprintf("%s: %v != %v", varName, aval, bval))
	} else {
		c.diff = append(c.diff, fmt.Sprintf("%v != %v", aval, bval))
	}
}

func init() {
	log.SetFlags(log.Lshortfile)
}

func logError(err error) {
	if LogErrors {
		log.Println(err)
	}
}
<commit_msg>Fix typo<commit_after>// Package deep provides function deep.Equal which is like reflect.DeepEqual but
// returns a list of differences. This is helpful when comparing complex types
// like structures and maps.
package deep

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"
)

var (
	// FloatPrecision is the number of decimal places to round float values
	// to when comparing.
	FloatPrecision = 10

	// MaxDiff specifies the maximum number of differences to return.
	MaxDiff = 10

	// MaxDepth specifies the maximum levels of a struct to recurse into.
	MaxDepth = 10

	// LogErrors causes errors to be logged to STDERR when true.
	LogErrors = false

	// CompareUnexportedFields causes unexported struct fields, like s in
	// T{s int}, to be comparsed when true.
	CompareUnexportedFields = false
)

var (
	// ErrMaxRecursion is logged when MaxDepth is reached.
	ErrMaxRecursion = errors.New("recursed to MaxDepth")

	// ErrTypeMismatch is logged when Equal passed two different types of values.
	ErrTypeMismatch = errors.New("variables are different reflect.Type")

	// ErrNotHandled is logged when a primitive Go kind is not handled.
	ErrNotHandled = errors.New("cannot compare the reflect.Kind")
)

type cmp struct {
	diff        []string
	buff        []string
	floatFormat string
}

var errorType = reflect.TypeOf((*error)(nil)).Elem()

// Equal compares variables a and b, recursing into their structure up to
// MaxDepth levels deep, and returns a list of differences, or nil if there are
// none. Some differences may not be found if an error is also returned.
//
// If a type has an Equal method, like time.Equal, it is called to check for
// equality.
func Equal(a, b interface{}) []string {
	aVal := reflect.ValueOf(a)
	bVal := reflect.ValueOf(b)
	c := &cmp{
		diff:        []string{},
		buff:        []string{},
		floatFormat: fmt.Sprintf("%%.%df", FloatPrecision),
	}
	if a == nil && b == nil {
		return nil
	} else if a == nil && b != nil {
		c.saveDiff(b, "<nil pointer>")
	} else if a != nil && b == nil {
		c.saveDiff(a, "<nil pointer>")
	}
	if len(c.diff) > 0 {
		return c.diff
	}

	c.equals(aVal, bVal, 0)
	if len(c.diff) > 0 {
		return c.diff // diffs
	}
	return nil // no diffs
}

func (c *cmp) equals(a, b reflect.Value, level int) {
	if level > MaxDepth {
		logError(ErrMaxRecursion)
		return
	}

	// If differenet types, they can't be equal
	aType := a.Type()
	bType := b.Type()
	if aType != bType {
		c.saveDiff(aType, bType)
		logError(ErrTypeMismatch)
		return
	}

	// Primitive https://golang.org/pkg/reflect/#Kind
	aKind := a.Kind()
	bKind := b.Kind()

	// If both types implement the error interface, compare the error strings.
	// This must be done before dereferencing because the interface is on a
	// pointer receiver.
	if aType.Implements(errorType) && bType.Implements(errorType) {
		if a.Elem().IsValid() && b.Elem().IsValid() { // both err != nil
			aString := a.MethodByName("Error").Call(nil)[0].String()
			bString := b.MethodByName("Error").Call(nil)[0].String()
			if aString != bString {
				c.saveDiff(aString, bString)
			}
			return
		}
	}

	// Dereference pointers and interface{}
	if aKind == reflect.Ptr || aKind == reflect.Interface {
		a = a.Elem()
		aKind = a.Kind()
		if a.IsValid() {
			aType = a.Type()
		}
	}
	if bKind == reflect.Ptr || bKind == reflect.Interface {
		b = b.Elem()
		if b.IsValid() {
			bType = b.Type()
		}
	}

	// Check if one value is nil, e.g. T{x: *X} and T.x is nil
	if !a.IsValid() || !b.IsValid() {
		if a.IsValid() && !b.IsValid() {
			c.saveDiff(aType, "<nil pointer>")
		} else if !a.IsValid() && b.IsValid() {
			c.saveDiff("<nil pointer>", bType)
		}
		return
	}

	// Types with an Equal(), like time.Time.
	eqFunc := a.MethodByName("Equal")
	if eqFunc.IsValid() {
		retVals := eqFunc.Call([]reflect.Value{b})
		if !retVals[0].Bool() {
			c.saveDiff(a, b)
		}
		return
	}

	switch aKind {

	/////////////////////////////////////////////////////////////////////
	// Iterable kinds
	/////////////////////////////////////////////////////////////////////

	case reflect.Struct:
		/*
			The variables are structs like:
				type T struct {
					FirstName string
					LastName  string
				}
			Type = <pkg>.T, Kind = reflect.Struct

			Iterate through the fields (FirstName, LastName), recurse into their values.
		*/
		for i := 0; i < a.NumField(); i++ {
			if aType.Field(i).PkgPath != "" && !CompareUnexportedFields {
				continue // skip unexported field, e.g. s in type T struct {s string}
			}

			c.push(aType.Field(i).Name) // push field name to buff

			// Get the Value for each field, e.g. FirstName has Type = string,
			// Kind = reflect.String.
			af := a.Field(i)
			bf := b.Field(i)

			// Recurse to compare the field values
			c.equals(af, bf, level+1)

			c.pop() // pop field name from buff

			if len(c.diff) >= MaxDiff {
				break
			}
		}
	case reflect.Map:
		/*
			The variables are maps like:
				map[string]int{
					"foo": 1,
					"bar": 2,
				}
			Type = map[string]int, Kind = reflect.Map

			Or:
				type T map[string]int{}
			Type = <pkg>.T, Kind = reflect.Map

			Iterate through the map keys (foo, bar), recurse into their values.
		*/

		if a.IsNil() || b.IsNil() {
			if a.IsNil() && !b.IsNil() {
				c.saveDiff("<nil map>", b)
			} else if !a.IsNil() && b.IsNil() {
				c.saveDiff(a, "<nil map>")
			}
			return
		}

		if a.Pointer() == b.Pointer() {
			return
		}

		for _, key := range a.MapKeys() {
			c.push(fmt.Sprintf("map[%s]", key))

			aVal := a.MapIndex(key)
			bVal := b.MapIndex(key)
			if bVal.IsValid() {
				c.equals(aVal, bVal, level+1)
			} else {
				c.saveDiff(aVal, "<does not have key>")
			}

			c.pop()

			if len(c.diff) >= MaxDiff {
				return
			}
		}

		for _, key := range b.MapKeys() {
			if aVal := a.MapIndex(key); aVal.IsValid() {
				continue
			}

			c.push(fmt.Sprintf("map[%s]", key))
			c.saveDiff("<does not have key>", b.MapIndex(key))
			c.pop()
			if len(c.diff) >= MaxDiff {
				return
			}
		}
	case reflect.Slice:
		if a.IsNil() || b.IsNil() {
			if a.IsNil() && !b.IsNil() {
				c.saveDiff("<nil slice>", b)
			} else if !a.IsNil() && b.IsNil() {
				c.saveDiff(a, "<nil slice>")
			}
			return
		}

		if a.Pointer() == b.Pointer() {
			return
		}

		aLen := a.Len()
		bLen := b.Len()
		n := aLen
		if bLen > aLen {
			n = bLen
		}
		for i := 0; i < n; i++ {
			c.push(fmt.Sprintf("slice[%d]", i))
			if i < aLen && i < bLen {
				c.equals(a.Index(i), b.Index(i), level+1)
			} else if i < aLen {
				c.saveDiff(a.Index(i), "<no value>")
			} else {
				c.saveDiff("<no value>", b.Index(i))
			}
			c.pop()
			if len(c.diff) >= MaxDiff {
				break
			}
		}

	/////////////////////////////////////////////////////////////////////
	// Primitive kinds
	/////////////////////////////////////////////////////////////////////

	case reflect.Float32, reflect.Float64:
		// Avoid 0.04147685731961082 != 0.041476857319611
		// 6 decimal places is close enough
		aval := fmt.Sprintf(c.floatFormat, a.Float())
		bval := fmt.Sprintf(c.floatFormat, b.Float())
		if aval != bval {
			c.saveDiff(a.Float(), b.Float())
		}
	case reflect.Bool:
		if a.Bool() != b.Bool() {
			c.saveDiff(a.Bool(), b.Bool())
		}
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		if a.Int() != b.Int() {
			c.saveDiff(a.Int(), b.Int())
		}
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		if a.Uint() != b.Uint() {
			c.saveDiff(a.Uint(), b.Uint())
		}
	case reflect.String:
		if a.String() != b.String() {
			c.saveDiff(a.String(), b.String())
		}

	default:
		logError(ErrNotHandled)
	}
}

func (c *cmp) push(name string) {
	c.buff = append(c.buff, name)
}

func (c *cmp) pop() {
	if len(c.buff) > 0 {
		c.buff = c.buff[0 : len(c.buff)-1]
	}
}

func (c *cmp) saveDiff(aval, bval interface{}) {
	if len(c.buff) > 0 {
		varName := strings.Join(c.buff, ".")
		c.diff = append(c.diff, fmt.Sprintf("%s: %v != %v", varName, aval, bval))
	} else {
		c.diff = append(c.diff, fmt.Sprintf("%v != %v", aval, bval))
	}
}

func init() {
	log.SetFlags(log.Lshortfile)
}

func logError(err error) {
	if LogErrors {
		log.Println(err)
	}
}
<|endoftext|>


<commit_before>package main

const Version = "0.11.0"
<commit_msg>bump to v0.12.0-dev<commit_after>package main

const Version = "0.12.0-dev"
<|endoftext|>


<commit_before>package golapa

import (
	"fmt"
	"bytes"
	"text/template"
	"os"
	"net/smtp"
	"crypto/tls"
	_ "crypto/sha512"
	"net"
	"time"
	"log"
	"encoding/base64"
)

type Message struct {
	Sender string
	To	string
	Subject string
	Body	string
}

func (msg *Message) ToMessage() ([]byte) {
	var b bytes.Buffer
	t := time.Now()
	curDateTime := t.UTC().Format(time.RFC1123Z)

	b.Write([]byte(fmt.Sprintf("Sender: %s\r\n", msg.Sender)))
	b.Write([]byte(fmt.Sprintf("Reply-To: %s\r\n", msg.Sender)))
	b.Write([]byte(fmt.Sprintf("Subject: =?utf-8?B?%s?=\r\n", base64.URLEncoding.EncodeToString([]byte(msg.Subject)))))
	b.Write([]byte(fmt.Sprintf("Date: %s\r\n", curDateTime)))
	b.Write([]byte(fmt.Sprintf("From: %s\r\n", msg.Sender)))
	b.Write([]byte(fmt.Sprintf("To: %s\r\n", msg.To)))

	return b.Bytes()
}

type Emailer interface {
	Send(msg *Message)
}

type StandardEmailer struct {
	TemplatePath	string
}

func (se *StandardEmailer) Send(msg *Message) {
	host := os.Getenv("EMAIL_HOST")
	password := os.Getenv("EMAIL_PASSWORD")
	from := os.Getenv("EMAIL_FROM")
	ssl := os.Getenv("EMAIL_SSL")

	hostPart, _, err := net.SplitHostPort(host)
	if err != nil {
		log.Print(err.Error())
	}
	auth := smtp.PlainAuth("", from, password, hostPart)

	if ssl == "" {
		err = smtp.SendMail(host, auth, from, []string{msg.To}, msg.ToMessage())
		if err != nil {
			log.Print(err.Error())
		}
	}

	if ssl == "true" {
		err = se.SendMailSSL(host, auth, from, []string{msg.To}, msg.ToMessage())
		if err != nil {
			log.Print(err.Error())
		}
	}
}

func (se *StandardEmailer) CreateMessage(sender string, subject string, to string, name string, email string) (*Message, error) {
	tpl, err := template.ParseFiles(fmt.Sprintf("%s/email.tpl", se.TemplatePath))
	if err != nil {
		return nil, err
	}
	kv := map[string]string{}
	kv["name"] = name
	kv["email"] = email

	var body bytes.Buffer
	tpl.Execute(&body, kv)
	log.Print(string(body.Bytes()))
	return &Message{
		Sender: sender,
		To: to,
		Subject: subject,
		Body: string(body.Bytes()),
	}, nil
}

func (se *StandardEmailer) SendMailSSL(host string, auth smtp.Auth, from string, recipients []string, msg []byte) (error) {
	conn, err := tls.Dial("tcp", host, nil)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	c, err := smtp.NewClient(conn, host)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	err = c.Auth(auth)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	err = c.Mail(from)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	for _, addr := range recipients {
		err = c.Rcpt(addr)
		if err != nil {
			log.Print(err.Error())
			return err
		}
	}

	d, err := c.Data()
	if err != nil {
		log.Print(err.Error())
		return err
	}

	_, err = d.Write(msg)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	err = d.Close()
	if err != nil {
		log.Print(err.Error())
		return err
	}

	c.Quit()
	return nil
}
<commit_msg>Commenting out unnecessary SplitHostPart call<commit_after>package golapa

import (
	"fmt"
	"bytes"
	"text/template"
	"os"
	"net/smtp"
	"crypto/tls"
	_ "crypto/sha512"
	"net"
	"time"
	"log"
	"encoding/base64"
)

type Message struct {
	Sender string
	To	string
	Subject string
	Body	string
}

func (msg *Message) ToMessage() ([]byte) {
	var b bytes.Buffer
	t := time.Now()
	curDateTime := t.UTC().Format(time.RFC1123Z)

	b.Write([]byte(fmt.Sprintf("Sender: %s\r\n", msg.Sender)))
	b.Write([]byte(fmt.Sprintf("Reply-To: %s\r\n", msg.Sender)))
	b.Write([]byte(fmt.Sprintf("Subject: =?utf-8?B?%s?=\r\n", base64.URLEncoding.EncodeToString([]byte(msg.Subject)))))
	b.Write([]byte(fmt.Sprintf("Date: %s\r\n", curDateTime)))
	b.Write([]byte(fmt.Sprintf("From: %s\r\n", msg.Sender)))
	b.Write([]byte(fmt.Sprintf("To: %s\r\n", msg.To)))

	return b.Bytes()
}

type Emailer interface {
	Send(msg *Message)
}

type StandardEmailer struct {
	TemplatePath	string
}

func (se *StandardEmailer) Send(msg *Message) {
	host := os.Getenv("EMAIL_HOST")
	password := os.Getenv("EMAIL_PASSWORD")
	from := os.Getenv("EMAIL_FROM")
	ssl := os.Getenv("EMAIL_SSL")

	//hostPart, _, err := net.SplitHostPort(host)
	if err != nil {
		log.Print(err.Error())
	}
	auth := smtp.PlainAuth("", from, password, host)

	if ssl == "" {
		err = smtp.SendMail(host, auth, from, []string{msg.To}, msg.ToMessage())
		if err != nil {
			log.Print(err.Error())
		}
	}

	if ssl == "true" {
		err = se.SendMailSSL(host, auth, from, []string{msg.To}, msg.ToMessage())
		if err != nil {
			log.Print(err.Error())
		}
	}
}

func (se *StandardEmailer) CreateMessage(sender string, subject string, to string, name string, email string) (*Message, error) {
	tpl, err := template.ParseFiles(fmt.Sprintf("%s/email.tpl", se.TemplatePath))
	if err != nil {
		return nil, err
	}
	kv := map[string]string{}
	kv["name"] = name
	kv["email"] = email

	var body bytes.Buffer
	tpl.Execute(&body, kv)
	log.Print(string(body.Bytes()))
	return &Message{
		Sender: sender,
		To: to,
		Subject: subject,
		Body: string(body.Bytes()),
	}, nil
}

func (se *StandardEmailer) SendMailSSL(host string, auth smtp.Auth, from string, recipients []string, msg []byte) (error) {
	conn, err := tls.Dial("tcp", host, nil)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	c, err := smtp.NewClient(conn, host)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	err = c.Auth(auth)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	err = c.Mail(from)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	for _, addr := range recipients {
		err = c.Rcpt(addr)
		if err != nil {
			log.Print(err.Error())
			return err
		}
	}

	d, err := c.Data()
	if err != nil {
		log.Print(err.Error())
		return err
	}

	_, err = d.Write(msg)
	if err != nil {
		log.Print(err.Error())
		return err
	}

	err = d.Close()
	if err != nil {
		log.Print(err.Error())
		return err
	}

	c.Quit()
	return nil
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"gopkg.in/mgo.v2/bson"
	"testing"
)

func TestIOmap(t *testing.T) {
	print("\nTestIOmap\n")
	i := NewIOmap()
	i.Add("qc.passed.fna", "http://shock.mcs.anl.gov:8000/node/fad5eabc7602b1fcebcaff518266805f", "ff3f41a91bbf135b38d0b35b1df3a42e", false)
	i.Add("qc.failed.fna", "http://shock.mcs.anl.gov:8000/node/f361be3e7a0914f82147bc1ba68df41e", "dff5aa75f124db423cda694c16254f69", false)
	m, _ := json.Marshal(i)
	print(string(m) + "\n")

	if !i.Has("qc.passed.fna") || !i.Has("qc.failed.fna") {
		t.Fail()
	}

	if v := i.Find("qc.passed.fna"); v == nil {
		t.Fail()
	}

	if v := i.Find("qc.failed.fna"); v == nil {
		t.Fail()
	}

	if v := i.Find("doesnotexist"); v != nil {
		t.Fail()
	}
}

func TestCommand(t *testing.T) {
	print("\nTestCommand\n")
	c := Command{Name: "superblat", Description: "", Options: "-p8 -o8", Template: "inputs::i1 inputs::i2 > outputs::o1"}
	c.Substitute(nil, nil)
	m, _ := json.Marshal(c)
	print(string(m) + "\n")
}

func TestTask(t *testing.T) {
	print("\nTestTask\n")
	nt, _ := NewTask()
	m, _ := json.Marshal(nt)
	print(string(m) + "\n")
}

func BenchmarkTask(b *testing.B) {
	for i := 0; i < b.N; i++ {
		nt, _ := NewTask()
		json.Marshal(nt)
	}
}

func TestPipeline(t *testing.T) {
	print("\nTestPipeline\n")
	p := NewPipeline()
	m, _ := json.Marshal(p)
	print(string(m) + "\n")
}

func BenchmarkPipeline(b *testing.B) {
	for i := 0; i < b.N; i++ {
		p := NewPipeline()
		json.Marshal(p)
	}
}

func BenchmarkPipelineB(b *testing.B) {
	for i := 0; i < b.N; i++ {
		p := NewPipeline()
		bson.Marshal(p)
	}
}
<commit_msg>deprecated<commit_after><|endoftext|>


<commit_before>package ipamplugin

import (
	"encoding/json"
	"fmt"
	"net"

	"github.com/appc/cni/pkg/skel"
	"github.com/appc/cni/pkg/types"
)

func (i *Ipam) CmdAdd(args *skel.CmdArgs) error {
	result, err := i.Allocate(args)
	if err != nil {
		return err
	}
	return result.Print()
}

func (i *Ipam) Allocate(args *skel.CmdArgs) (*types.Result, error) {
	// extract the things we care about
	conf, err := loadIPAMConf(args.StdinData)
	if err != nil {
		return nil, err
	}
	if conf == nil {
		conf = &ipamConf{}
	}
	containerID := args.ContainerID
	var ipnet *net.IPNet

	if conf.Subnet == "" {
		ipnet, err = i.weave.AllocateIP(containerID)
	} else {
		var subnet *net.IPNet
		subnet, err = types.ParseCIDR(conf.Subnet)
		if err != nil {
			return nil, fmt.Errorf("subnet given in config, but not parseable: %s", err)
		}
		ipnet, err = i.weave.AllocateIPInSubnet(containerID, subnet)
	}

	if err != nil {
		return nil, err
	}
	result := &types.Result{
		IP4: &types.IPConfig{
			IP:      *ipnet,
			Gateway: conf.Gateway,
			Routes:  conf.Routes,
		},
	}
	return result, nil
}

func (i *Ipam) CmdDel(args *skel.CmdArgs) error {
	return i.Release(args)
}

func (i *Ipam) Release(args *skel.CmdArgs) error {
	return i.weave.ReleaseIPsFor(args.ContainerID)
}

type ipamConf struct {
	Subnet  string        `json:"subnet,omitempty"`
	Gateway net.IP        `json:"gateway,omitempty"`
	Routes  []types.Route `json:"routes"`
}

type netConf struct {
	IPAM *ipamConf `json:"ipam"`
}

func loadIPAMConf(stdinData []byte) (*ipamConf, error) {
	var conf netConf
	return conf.IPAM, json.Unmarshal(stdinData, &conf)
}
<commit_msg>Sanity-check ipam args to make diagnosis easier<commit_after>package ipamplugin

import (
	"encoding/json"
	"fmt"
	"net"

	"github.com/appc/cni/pkg/skel"
	"github.com/appc/cni/pkg/types"
)

func (i *Ipam) CmdAdd(args *skel.CmdArgs) error {
	result, err := i.Allocate(args)
	if err != nil {
		return err
	}
	return result.Print()
}

func (i *Ipam) Allocate(args *skel.CmdArgs) (*types.Result, error) {
	// extract the things we care about
	conf, err := loadIPAMConf(args.StdinData)
	if err != nil {
		return nil, err
	}
	if conf == nil {
		conf = &ipamConf{}
	}
	containerID := args.ContainerID
	if containerID == "" {
		return nil, fmt.Errorf("Weave CNI Allocate: blank container name")
	}
	var ipnet *net.IPNet

	if conf.Subnet == "" {
		ipnet, err = i.weave.AllocateIP(containerID)
	} else {
		var subnet *net.IPNet
		subnet, err = types.ParseCIDR(conf.Subnet)
		if err != nil {
			return nil, fmt.Errorf("subnet given in config, but not parseable: %s", err)
		}
		ipnet, err = i.weave.AllocateIPInSubnet(containerID, subnet)
	}

	if err != nil {
		return nil, err
	}
	result := &types.Result{
		IP4: &types.IPConfig{
			IP:      *ipnet,
			Gateway: conf.Gateway,
			Routes:  conf.Routes,
		},
	}
	return result, nil
}

func (i *Ipam) CmdDel(args *skel.CmdArgs) error {
	return i.Release(args)
}

func (i *Ipam) Release(args *skel.CmdArgs) error {
	return i.weave.ReleaseIPsFor(args.ContainerID)
}

type ipamConf struct {
	Subnet  string        `json:"subnet,omitempty"`
	Gateway net.IP        `json:"gateway,omitempty"`
	Routes  []types.Route `json:"routes"`
}

type netConf struct {
	IPAM *ipamConf `json:"ipam"`
}

func loadIPAMConf(stdinData []byte) (*ipamConf, error) {
	var conf netConf
	return conf.IPAM, json.Unmarshal(stdinData, &conf)
}
<|endoftext|>


<commit_before>package example

import (
	"testing"

	"github.com/go-chat-bot/bot"
)

func TestHelloworld(t *testing.T) {
	want := "Hello nick"
	bot := &bot.Cmd{
		Command: "helloworld",
		User: &bot.User{
			Nick: "nick",
		},
	}
	got, error := hello(bot)

	if got != want {
		t.Errorf("Expected '%v' got '%v'", want, got)
	}

	if error != nil {
		t.Errorf("Expected '%v' got '%v'", nil, error)
	}
}
<commit_msg>Fix helloworld test<commit_after>package example

import (
	"testing"

	"github.com/go-chat-bot/bot"
)

func TestHelloworld(t *testing.T) {
	bot := &bot.Cmd{
		Command: "helloworld",
		User: &bot.User{
			Nick:     "nick",
			RealName: "Real Name",
		},
	}
	want := "Hello Real Name"
	got, error := hello(bot)

	if got != want {
		t.Errorf("Expected '%v' got '%v'", want, got)
	}

	if error != nil {
		t.Errorf("Expected '%v' got '%v'", nil, error)
	}
}
<|endoftext|>


<commit_before>package game

import (
	"container/heap"
	"fmt"
	"github.com/MichaelDiBernardo/srl/lib/math"
)

type Tile struct {
	Feature *Feature
	Actor   *Obj
	Items   *Inventory
	Pos     math.Point
	Visible bool
}

type Map [][]*Tile

type Level struct {
	Map       Map
	Bounds    math.Rectangle
	game      *Game
	scheduler *Scheduler
}

// Create a level that uses the given gametory to create game objects, and which
// generated by the given generator function.
func NewLevel(width, height int, game *Game, gen func(*Level) *Level) *Level {
	newmap := Map{}

	// Init all tiles.
	for y := 0; y < height; y++ {
		row := make([]*Tile, width, width)
		for x := 0; x < width; x++ {
			row[x] = &Tile{
				Pos:     math.Pt(x, y),
				Feature: FeatFloor,
				Items:   NewInventory(),
			}
		}
		newmap = append(newmap, row)
	}
	level := &Level{
		Map:       newmap,
		Bounds:    math.Rect(math.Origin, math.Pt(width, height)),
		game:      game,
		scheduler: NewScheduler(),
	}
	return gen(level)
}

func (l *Level) At(p math.Point) *Tile {
	return l.Map[p.Y][p.X]
}

func (l *Level) HasPoint(p math.Point) bool {
	return l.Bounds.HasPoint(p)
}

// Place `o` on the tile at `p`. Returns false if this is impossible (e.g.
// trying to put something on a solid square.)
// This will remove `o` from any tile on any map it was previously on.
func (l *Level) Place(o *Obj, p math.Point) bool {
	t := l.At(p)
	if t.Feature.Solid {
		return false
	}

	switch o.Spec.Family {
	case FamActor:
		return l.placeActor(o, t)
	case FamItem:
		return l.placeItem(o, t)
	default:
		panic(fmt.Sprintf("Tried to place object of family %v", o.Spec.Family))
	}
}

// Removes 'o' from the level.
func (l *Level) Remove(o *Obj) {
	switch o.Spec.Family {
	case FamActor:
		l.removeActor(o)
	default:
		panic(fmt.Sprintf("Tried to remove object of family %v", o.Spec.Family))
	}

}

// Advances the game until the player's next turn.
func (l *Level) Evolve() {
	// Advance schedule until we find the player.
	for {
		actor := l.scheduler.Next()
		if seer := actor.Seer; seer != nil {
			seer.CalcLOS()
		}
		if ai := actor.AI; ai != nil {
			ai.Act(l)
		}
		if actor.IsPlayer() {
			for _, row := range l.Map {
				for _, tile := range row {
					tile.Visible = false
				}
			}
			for _, pt := range actor.Seer.LOS() {
				l.At(pt).Visible = true
			}
			break
		}
	}
}

func (l *Level) placeActor(obj *Obj, tile *Tile) bool {
	if tile.Actor != nil {
		return false
	}

	// If this actor has been placed before, we need to clear the tile they
	// were on previously. If they haven't, we need to add them to the actor
	// list so we know who they are.
	if obj.Tile != nil {
		obj.Tile.Actor = nil
	} else {
		l.scheduler.Add(obj)
	}

	obj.Level = l
	obj.Tile = tile

	tile.Actor = obj

	return true
}

func (l *Level) placeItem(obj *Obj, tile *Tile) bool {
	return tile.Items.Add(obj)
}

func (l *Level) removeActor(obj *Obj) {
	obj.Tile.Actor = nil
	obj.Tile = nil
	obj.Level = nil
	l.scheduler.Remove(obj)
}

// Used to track which actor should be acting when.
type scheduled struct {
	actor *Obj
	delay int
}

// Shoehorning into container/heap's interface. SQ acts as the actual priority
// queue.
type SQ []*scheduled

// See https://golang.org/pkg/container/heap/#example__intHeap.
func (s *SQ) Len() int {
	return len(*s)
}

func (s *SQ) Less(i, j int) bool {
	return (*s)[i].delay < (*s)[j].delay
}

func (s *SQ) Swap(i, j int) {
	(*s)[i], (*s)[j] = (*s)[j], (*s)[i]
}

func (s *SQ) Push(x interface{}) {
	*s = append(*s, x.(*scheduled))
}

func (s *SQ) Pop() interface{} {
	old := *s
	n := len(old)
	x := old[n-1]
	*s = old[0 : n-1]
	return x
}

type Scheduler struct {
	pq *SQ
}

func NewScheduler() *Scheduler {
	pq := make(SQ, 0)
	return &Scheduler{pq: &pq}
}

func (s *Scheduler) Len() int {
	return len(*(s.pq))
}

// Add an actor to the schedule.
func (s *Scheduler) Add(actor *Obj) {
	delay := getdelay(actor.Sheet.Speed())

	// Player should always get the first turn.
	if actor.IsPlayer() {
		delay = 0
	}

	entry := &scheduled{actor: actor, delay: delay}
	heap.Push(s.pq, entry)
}

// Picks the next actor to act, and moves time forward for everyone else.
func (s *Scheduler) Next() *Obj {
	entry := heap.Pop(s.pq).(*scheduled)
	for _, e := range *(s.pq) {
		e.delay -= entry.delay
	}
	heap.Init(s.pq)

	actor := entry.actor
	entry.delay = getdelay(actor.Sheet.Speed())
	heap.Push(s.pq, entry)

	return actor
}

// Removes an actor from the scheduler.
func (s *Scheduler) Remove(actor *Obj) {
	index := -1
	for i, e := range *(s.pq) {
		if e.actor == actor {
			index = i
			break
		}
	}

	if index == -1 {
		panic("Tried to remove actor but wasn't in list.")
	}

	*(s.pq) = append((*(s.pq))[:index], (*(s.pq))[index+1:]...)
	heap.Init(s.pq)
}

// Given the speed of an actor, this will tell you how much delay to add after
// each of its turns.
func getdelay(spd int) int {
	switch spd {
	case 1:
		return 150
	case 2:
		return 100
	case 3:
		return 75
	case 4:
		return 50
	default:
		panic(fmt.Sprintf("Spd %d does not have a delay", spd))
	}
}
<commit_msg>Improve pointer messes.<commit_after>package game

import (
	"container/heap"
	"fmt"
	"github.com/MichaelDiBernardo/srl/lib/math"
)

type Tile struct {
	Feature *Feature
	Actor   *Obj
	Items   *Inventory
	Pos     math.Point
	Visible bool
}

type Map [][]*Tile

type Level struct {
	Map       Map
	Bounds    math.Rectangle
	game      *Game
	scheduler *Scheduler
}

// Create a level that uses the given gametory to create game objects, and which
// generated by the given generator function.
func NewLevel(width, height int, game *Game, gen func(*Level) *Level) *Level {
	newmap := Map{}

	// Init all tiles.
	for y := 0; y < height; y++ {
		row := make([]*Tile, width, width)
		for x := 0; x < width; x++ {
			row[x] = &Tile{
				Pos:     math.Pt(x, y),
				Feature: FeatFloor,
				Items:   NewInventory(),
			}
		}
		newmap = append(newmap, row)
	}
	level := &Level{
		Map:       newmap,
		Bounds:    math.Rect(math.Origin, math.Pt(width, height)),
		game:      game,
		scheduler: NewScheduler(),
	}
	return gen(level)
}

func (l *Level) At(p math.Point) *Tile {
	return l.Map[p.Y][p.X]
}

func (l *Level) HasPoint(p math.Point) bool {
	return l.Bounds.HasPoint(p)
}

// Place `o` on the tile at `p`. Returns false if this is impossible (e.g.
// trying to put something on a solid square.)
// This will remove `o` from any tile on any map it was previously on.
func (l *Level) Place(o *Obj, p math.Point) bool {
	t := l.At(p)
	if t.Feature.Solid {
		return false
	}

	switch o.Spec.Family {
	case FamActor:
		return l.placeActor(o, t)
	case FamItem:
		return l.placeItem(o, t)
	default:
		panic(fmt.Sprintf("Tried to place object of family %v", o.Spec.Family))
	}
}

// Removes 'o' from the level.
func (l *Level) Remove(o *Obj) {
	switch o.Spec.Family {
	case FamActor:
		l.removeActor(o)
	default:
		panic(fmt.Sprintf("Tried to remove object of family %v", o.Spec.Family))
	}

}

// Advances the game until the player's next turn.
func (l *Level) Evolve() {
	// Advance schedule until we find the player.
	for {
		actor := l.scheduler.Next()
		if seer := actor.Seer; seer != nil {
			seer.CalcLOS()
		}
		if ai := actor.AI; ai != nil {
			ai.Act(l)
		}
		if actor.IsPlayer() {
			for _, row := range l.Map {
				for _, tile := range row {
					tile.Visible = false
				}
			}
			for _, pt := range actor.Seer.LOS() {
				l.At(pt).Visible = true
			}
			break
		}
	}
}

func (l *Level) placeActor(obj *Obj, tile *Tile) bool {
	if tile.Actor != nil {
		return false
	}

	// If this actor has been placed before, we need to clear the tile they
	// were on previously. If they haven't, we need to add them to the actor
	// list so we know who they are.
	if obj.Tile != nil {
		obj.Tile.Actor = nil
	} else {
		l.scheduler.Add(obj)
	}

	obj.Level = l
	obj.Tile = tile

	tile.Actor = obj

	return true
}

func (l *Level) placeItem(obj *Obj, tile *Tile) bool {
	return tile.Items.Add(obj)
}

func (l *Level) removeActor(obj *Obj) {
	obj.Tile.Actor = nil
	obj.Tile = nil
	obj.Level = nil
	l.scheduler.Remove(obj)
}

// Used to track which actor should be acting when.
type scheduled struct {
	actor *Obj
	delay int
}

// Shoehorning into container/heap's interface. SQ acts as the actual priority
// queue.
type SQ []*scheduled

// See https://golang.org/pkg/container/heap/#example__intHeap.
func (s SQ) Len() int {
	return len(s)
}

func (s SQ) Less(i, j int) bool {
	return s[i].delay < s[j].delay
}

func (s SQ) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s *SQ) Push(x interface{}) {
	*s = append(*s, x.(*scheduled))
}

func (s *SQ) Pop() interface{} {
	old := *s
	n := len(old)
	x := old[n-1]
	*s = old[0 : n-1]
	return x
}

type Scheduler struct {
	pq *SQ
}

func NewScheduler() *Scheduler {
	pq := make(SQ, 0)
	return &Scheduler{pq: &pq}
}

func (s *Scheduler) Len() int {
	return s.pq.Len()
}

// Add an actor to the schedule.
func (s *Scheduler) Add(actor *Obj) {
	delay := getdelay(actor.Sheet.Speed())

	// Player should always get the first turn.
	if actor.IsPlayer() {
		delay = 0
	}

	entry := &scheduled{actor: actor, delay: delay}
	heap.Push(s.pq, entry)
}

// Picks the next actor to act, and moves time forward for everyone else.
func (s *Scheduler) Next() *Obj {
	entry := heap.Pop(s.pq).(*scheduled)
	for _, e := range *(s.pq) {
		e.delay -= entry.delay
	}
	heap.Init(s.pq)

	actor := entry.actor
	entry.delay = getdelay(actor.Sheet.Speed())
	heap.Push(s.pq, entry)

	return actor
}

// Removes an actor from the scheduler.
func (s *Scheduler) Remove(actor *Obj) {
	index := -1
	for i, e := range *(s.pq) {
		if e.actor == actor {
			index = i
			break
		}
	}

	if index == -1 {
		panic("Tried to remove actor but wasn't in list.")
	}

	*(s.pq) = append((*(s.pq))[:index], (*(s.pq))[index+1:]...)
	heap.Init(s.pq)
}

// Given the speed of an actor, this will tell you how much delay to add after
// each of its turns.
func getdelay(spd int) int {
	switch spd {
	case 1:
		return 150
	case 2:
		return 100
	case 3:
		return 75
	case 4:
		return 50
	default:
		panic(fmt.Sprintf("Spd %d does not have a delay", spd))
	}
}
<|endoftext|>


<commit_before>package dnsr

import (
	"fmt"
	"strings"
	"sync"
	"time"

	lru "github.com/hashicorp/golang-lru"
	"github.com/miekg/dns"
)

type Resolver struct {
	cache  *lru.Cache
	client *dns.Client
}

func New(size int) *Resolver {
	if size < 0 {
		size = 10000
	}
	cache, _ := lru.New(size)
	r := &Resolver{
		client: &dns.Client{},
		cache:  cache,
	}
	r.cacheRoot()
	return r
}

func (r *Resolver) Resolve(qname string, qtype uint16) <-chan dns.RR {
	c := make(chan dns.RR, 20)
	go func() {
		qname = toLowerFQDN(qname)
		// fmt.Printf("%s %s        ", qname, dns.TypeToString[qtype])
		// defer fmt.Printf("\n")
		defer close(c)
		if rrs := r.cacheGet(qname, qtype); rrs != nil {
			inject(c, rrs...)
			return
		}
		pname, ok := qname, true
		if qtype == dns.TypeNS {
			pname, ok = parent(qname)
			if !ok {
				return
			}
		}
	outer:
		for nrr := range r.Resolve(pname, dns.TypeNS) {
			ns, ok := nrr.(*dns.NS)
			if !ok {
				continue
			}
			for arr := range r.Resolve(ns.Ns, dns.TypeA) {
				a, ok := arr.(*dns.A)
				if !ok {
					continue
				}
				addr := a.A.String() + ":53"
				qmsg := &dns.Msg{}
				qmsg.SetQuestion(qname, qtype)
				qmsg.MsgHdr.RecursionDesired = false
				rmsg, _, err := r.client.Exchange(qmsg, addr)
				if err != nil {
					continue // FIXME: handle errors better from flaky/failing NS servers
				}
				if rmsg.Rcode == dns.RcodeNameError {
					r.cacheAdd(qname, qtype) // FIXME: cache NXDOMAIN responses responsibly
				}
				r.cacheSave(rmsg.Answer...)
				r.cacheSave(rmsg.Ns...)
				r.cacheSave(rmsg.Extra...)
				break outer
			}
		}

		if rrs := r.cacheGet(qname, qtype); rrs != nil {
			inject(c, rrs...)
			return
		}

		// Only check CNAMES for A and AAAA questions
		// if qtype != dns.TypeA && qtype != dns.TypeAAAA {
		// 	return
		// }
		if qtype == dns.TypeCNAME {
			return
		}

		for _, crr := range r.cacheGet(qname, dns.TypeCNAME) {
			cn, ok := crr.(*dns.CNAME)
			if !ok {
				continue
			}
			r.cacheAdd(qname, qtype, crr)
			c <- crr
			for rr := range r.Resolve(cn.Target, qtype) {
				fmt.Printf("%s\n", rr.String())
				r.cacheAdd(qname, qtype, rr)
				c <- rr
			}
			return
		}
	}()
	return c
}

func inject(c chan<- dns.RR, rrs ...dns.RR) {
	for _, rr := range rrs {
		c <- rr
	}
}

func parent(name string) (string, bool) {
	labels := dns.SplitDomainName(name)
	if labels == nil {
		return "", false
	}
	return toLowerFQDN(strings.Join(labels[1:], ".")), true
}

func toLowerFQDN(name string) string {
	return dns.Fqdn(strings.ToLower(name))
}

type key struct {
	qname string
	qtype uint16
}

type entry struct {
	m   sync.RWMutex
	exp time.Time
	rrs map[dns.RR]struct{}
}

func (r *Resolver) cacheRoot() {
	for t := range dns.ParseZone(strings.NewReader(root), "", "") {
		if t.Error == nil {
			r.cacheSave(t.RR)
		}
	}
}

// cacheGet returns a randomly ordered slice of DNS records.
func (r *Resolver) cacheGet(qname string, qtype uint16) []dns.RR {
	e := r.getEntry(qname, qtype)
	if e == nil {
		return nil
	}
	e.m.RLock()
	defer e.m.RUnlock()
	rrs := make([]dns.RR, 0, len(e.rrs))
	for rr, _ := range e.rrs {
		rrs = append(rrs, rr)
	}
	return rrs
}

// cacheSave saves 1 or more DNS records to the resolver cache.
func (r *Resolver) cacheSave(rrs ...dns.RR) {
	for _, rr := range rrs {
		h := rr.Header()
		r.cacheAdd(h.Name, h.Rrtype, rr)
	}
}

// cacheAdd adds 0 or more DNS records to the resolver cache for a specific
// domain name and record type. This ensures the cache entry exists, even
// if empty, for NXDOMAIN responses.
func (r *Resolver) cacheAdd(qname string, qtype uint16, rrs ...dns.RR) {
	qname = toLowerFQDN(qname)
	now := time.Now()
	e := r.getEntry(qname, qtype)
	if e == nil {
		e = &entry{
			exp: now.Add(24 * time.Hour),
			rrs: make(map[dns.RR]struct{}, 0),
		}
		r.cache.Add(key{qname, qtype}, e)
	}
	e.m.Lock()
	defer e.m.Unlock()
	for _, rr := range rrs {
		e.rrs[rr] = struct{}{}
		exp := now.Add(time.Duration(rr.Header().Ttl) * time.Second)
		if exp.Before(e.exp) {
			e.exp = exp
		}
	}
}

func (r *Resolver) getEntry(qname string, qtype uint16) *entry {
	c, ok := r.cache.Get(key{qname, qtype})
	if !ok {
		return nil
	}
	e := c.(*entry)
	if time.Now().After(e.exp) {
		return nil
	}
	return e
}
<commit_msg>cache NS records<commit_after>package dnsr

import (
	"fmt"
	"strings"
	"sync"
	"time"

	lru "github.com/hashicorp/golang-lru"
	"github.com/miekg/dns"
)

type Resolver struct {
	cache  *lru.Cache
	client *dns.Client
}

func New(size int) *Resolver {
	if size < 0 {
		size = 10000
	}
	cache, _ := lru.New(size)
	r := &Resolver{
		client: &dns.Client{},
		cache:  cache,
	}
	r.cacheRoot()
	return r
}

func (r *Resolver) Resolve(qname string, qtype uint16) <-chan dns.RR {
	c := make(chan dns.RR, 20)
	go func() {
		qname = toLowerFQDN(qname)
		// fmt.Printf("%s %s        ", qname, dns.TypeToString[qtype])
		// defer fmt.Printf("\n")
		defer close(c)
		if rrs := r.cacheGet(qname, qtype); rrs != nil {
			inject(c, rrs...)
			return
		}
		pname, ok := qname, true
		if qtype == dns.TypeNS {
			pname, ok = parent(qname)
			if !ok {
				return
			}
		}
	outer:
		for nrr := range r.Resolve(pname, dns.TypeNS) {
			ns, ok := nrr.(*dns.NS)
			if !ok {
				continue
			}
			for arr := range r.Resolve(ns.Ns, dns.TypeA) {
				a, ok := arr.(*dns.A)
				if !ok {
					continue
				}
				addr := a.A.String() + ":53"
				qmsg := &dns.Msg{}
				qmsg.SetQuestion(qname, qtype)
				qmsg.MsgHdr.RecursionDesired = false
				rmsg, _, err := r.client.Exchange(qmsg, addr)
				if err != nil {
					continue // FIXME: handle errors better from flaky/failing NS servers
				}
				if rmsg.Rcode == dns.RcodeNameError {
					r.cacheAdd(qname, qtype) // FIXME: cache NXDOMAIN responses responsibly
				}
				r.cacheSave(rmsg.Answer...)
				r.cacheSave(rmsg.Ns...)
				r.cacheSave(rmsg.Extra...)
				if qtype == dns.TypeNS && len(rmsg.Ns) > 0 {
					r.cacheAdd(qname, qtype, rmsg.Ns...)
				}
				break outer
			}
		}

		if rrs := r.cacheGet(qname, qtype); rrs != nil {
			inject(c, rrs...)
			return
		}

		// Only check CNAMES for A and AAAA questions
		// if qtype != dns.TypeA && qtype != dns.TypeAAAA {
		// 	return
		// }
		if qtype == dns.TypeCNAME {
			return
		}

		for _, crr := range r.cacheGet(qname, dns.TypeCNAME) {
			cn, ok := crr.(*dns.CNAME)
			if !ok {
				continue
			}
			r.cacheAdd(qname, qtype, crr)
			c <- crr
			for rr := range r.Resolve(cn.Target, qtype) {
				fmt.Printf("%s\n", rr.String())
				r.cacheAdd(qname, qtype, rr)
				c <- rr
			}
			return
		}
	}()
	return c
}

func inject(c chan<- dns.RR, rrs ...dns.RR) {
	for _, rr := range rrs {
		c <- rr
	}
}

func parent(name string) (string, bool) {
	labels := dns.SplitDomainName(name)
	if labels == nil {
		return "", false
	}
	return toLowerFQDN(strings.Join(labels[1:], ".")), true
}

func toLowerFQDN(name string) string {
	return dns.Fqdn(strings.ToLower(name))
}

type key struct {
	qname string
	qtype uint16
}

type entry struct {
	m   sync.RWMutex
	exp time.Time
	rrs map[dns.RR]struct{}
}

func (r *Resolver) cacheRoot() {
	for t := range dns.ParseZone(strings.NewReader(root), "", "") {
		if t.Error == nil {
			r.cacheSave(t.RR)
		}
	}
}

// cacheGet returns a randomly ordered slice of DNS records.
func (r *Resolver) cacheGet(qname string, qtype uint16) []dns.RR {
	e := r.getEntry(qname, qtype)
	if e == nil {
		return nil
	}
	e.m.RLock()
	defer e.m.RUnlock()
	rrs := make([]dns.RR, 0, len(e.rrs))
	for rr, _ := range e.rrs {
		rrs = append(rrs, rr)
	}
	return rrs
}

// cacheSave saves 1 or more DNS records to the resolver cache.
func (r *Resolver) cacheSave(rrs ...dns.RR) {
	for _, rr := range rrs {
		h := rr.Header()
		r.cacheAdd(h.Name, h.Rrtype, rr)
	}
}

// cacheAdd adds 0 or more DNS records to the resolver cache for a specific
// domain name and record type. This ensures the cache entry exists, even
// if empty, for NXDOMAIN responses.
func (r *Resolver) cacheAdd(qname string, qtype uint16, rrs ...dns.RR) {
	qname = toLowerFQDN(qname)
	now := time.Now()
	e := r.getEntry(qname, qtype)
	if e == nil {
		e = &entry{
			exp: now.Add(24 * time.Hour),
			rrs: make(map[dns.RR]struct{}, 0),
		}
		r.cache.Add(key{qname, qtype}, e)
	}
	e.m.Lock()
	defer e.m.Unlock()
	for _, rr := range rrs {
		e.rrs[rr] = struct{}{}
		exp := now.Add(time.Duration(rr.Header().Ttl) * time.Second)
		if exp.Before(e.exp) {
			e.exp = exp
		}
	}
}

func (r *Resolver) getEntry(qname string, qtype uint16) *entry {
	c, ok := r.cache.Get(key{qname, qtype})
	if !ok {
		return nil
	}
	e := c.(*entry)
	if time.Now().After(e.exp) {
		return nil
	}
	return e
}
<|endoftext|>


<commit_before>// +build linux

package journald

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/go-systemd/sdjournal"
	"github.com/segmentio/ecs-logs/lib"
)

func NewReader() (r ecslogs.Reader, err error) {
	var j *sdjournal.Journal

	if j, err = sdjournal.NewJournal(); err != nil {
		return
	}

	if err = j.SeekTail(); err != nil {
		j.Close()
		return
	}

	r = reader{j}
	return
}

type reader struct {
	*sdjournal.Journal
}

func (r reader) ReadMessage() (msg ecslogs.Message, err error) {
	for {
		var cur int
		var ok bool

		if cur, err = r.Next(); err != nil {
			return
		}

		if cur == 0 {
			r.Wait(sdjournal.IndefiniteWait)
			continue
		}

		if msg, ok, err = r.getMessage(); ok || err != nil {
			return
		}
	}
}

func (r reader) getMessage() (msg ecslogs.Message, ok bool, err error) {
	if msg.Group, err = r.GetDataValue("CONTAINER_TAG"); len(msg.Group) == 0 {
		// No CONTAINER_TAG, this must be a journal message from a process that
		// isn't running in a docker container.
		err = nil
		return
	}

	if msg.Stream, err = r.GetDataValue("CONTAINER_NAME"); err != nil {
		// There's a CONTAINER_TAG but no CONTAINER_NAME, something is seriously
		// wrong here, the log docker log driver is misbehaving.
		err = fmt.Errorf("missing CONTAINER_NAME in message with CONTAINER_TAG=%s", msg.Group)
		return
	}

	if s := r.getString("MESSAGE"); len(s) != 0 {
		d := json.NewDecoder(strings.NewReader(s))
		d.UseNumber()

		if d.Decode(&msg.Event) != nil {
			msg.Event = ecslogs.Event{
				Data: ecslogs.EventData{"message": s},
			}
		}
	}

	if msg.Event.Level == ecslogs.NONE {
		msg.Event.Level = r.getPriority()
	}

	if len(msg.Event.Info.Host) == 0 {
		msg.Event.Info.Host = r.getString("_HOSTNAME")
	}

	if len(msg.Event.Info.Source) == 0 {
		msg.Event.Info.Source = ecslogs.MessageSource(
			r.getString("CODE_FILE"),
			r.getInt("CODE_LINE"),
			r.getString("CODE_FUNC"),
		)
	}

	if len(msg.Event.Info.ID) == 0 {
		msg.Event.Info.ID = r.getString("MESSAGE_ID")
	}

	if msg.Event.Info.PID == 0 {
		msg.Event.Info.PID = r.getInt("_PID")
	}

	if msg.Event.Info.GID == 0 {
		msg.Event.Info.GID = r.getInt("_GID")
	}

	if msg.Event.Info.UID == 0 {
		msg.Event.Info.UID = r.getInt("_UID")
	}

	if msg.Event.Time == (time.Time{}) {
		msg.Event.Time = r.getTime()
	}

	ok = true
	return
}

func (r reader) getInt(k string) (v int) {
	v, _ = strconv.Atoi(r.getString(k))
	return
}

func (r reader) getTime() (t time.Time) {
	if u, e := r.GetRealtimeUsec(); e == nil {
		t = time.Unix(int64(u/1000000), int64((u%1000000)*1000))
	}
	return
}

func (r reader) getPriority() (p ecslogs.Level) {
	if v, e := strconv.Atoi(r.getString("PRIORITY")); e != nil {
		p = ecslogs.INFO
	} else {
		p = ecslogs.Level(v + 1)
	}
	return
}

func (r reader) getString(k string) (s string) {
	s, _ = r.GetDataValue(k)
	return
}
<commit_msg>fix journal reader<commit_after>// +build linux

package journald

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/coreos/go-systemd/sdjournal"
	"github.com/segmentio/ecs-logs/lib"
)

func NewReader() (r ecslogs.Reader, err error) {
	var j *sdjournal.Journal

	if j, err = sdjournal.NewJournal(); err != nil {
		return
	}

	if err = j.SeekTail(); err != nil {
		j.Close()
		return
	}

	r = reader{j}
	return
}

type reader struct {
	*sdjournal.Journal
}

func (r reader) ReadMessage() (msg ecslogs.Message, err error) {
	for {
		var cur int
		var ok bool

		if cur, err = r.Next(); err != nil {
			return
		}

		if cur == 0 {
			r.Wait(sdjournal.IndefiniteWait)
			continue
		}

		if msg, ok, err = r.getMessage(); ok || err != nil {
			return
		}
	}
}

func (r reader) getMessage() (msg ecslogs.Message, ok bool, err error) {
	if msg.Group, err = r.GetDataValue("CONTAINER_TAG"); len(msg.Group) == 0 {
		// No CONTAINER_TAG, this must be a journal message from a process that
		// isn't running in a docker container.
		err = nil
		return
	}

	if msg.Stream, err = r.GetDataValue("CONTAINER_NAME"); err != nil {
		// There's a CONTAINER_TAG but no CONTAINER_NAME, something is seriously
		// wrong here, the log docker log driver is misbehaving.
		err = fmt.Errorf("missing CONTAINER_NAME in message with CONTAINER_TAG=%s", msg.Group)
		return
	}

	if s := r.getString("MESSAGE"); len(s) != 0 {
		d := json.NewDecoder(strings.NewReader(s))
		d.UseNumber()

		if d.Decode(&msg.Event) != nil {
			msg.Event.Message = s
		}
	}

	if msg.Event.Level == ecslogs.NONE {
		msg.Event.Level = r.getPriority()
	}

	if len(msg.Event.Info.Host) == 0 {
		msg.Event.Info.Host = r.getString("_HOSTNAME")
	}

	if len(msg.Event.Info.Source) == 0 {
		msg.Event.Info.Source = ecslogs.MessageSource(
			r.getString("CODE_FILE"),
			r.getInt("CODE_LINE"),
			r.getString("CODE_FUNC"),
		)
	}

	if len(msg.Event.Info.ID) == 0 {
		msg.Event.Info.ID = r.getString("MESSAGE_ID")
	}

	if msg.Event.Info.PID == 0 {
		msg.Event.Info.PID = r.getInt("_PID")
	}

	if msg.Event.Info.GID == 0 {
		msg.Event.Info.GID = r.getInt("_GID")
	}

	if msg.Event.Info.UID == 0 {
		msg.Event.Info.UID = r.getInt("_UID")
	}

	if msg.Event.Time == (time.Time{}) {
		msg.Event.Time = r.getTime()
	}

	ok = true
	return
}

func (r reader) getInt(k string) (v int) {
	v, _ = strconv.Atoi(r.getString(k))
	return
}

func (r reader) getTime() (t time.Time) {
	if u, e := r.GetRealtimeUsec(); e == nil {
		t = time.Unix(int64(u/1000000), int64((u%1000000)*1000))
	}
	return
}

func (r reader) getPriority() (p ecslogs.Level) {
	if v, e := strconv.Atoi(r.getString("PRIORITY")); e != nil {
		p = ecslogs.INFO
	} else {
		p = ecslogs.Level(v + 1)
	}
	return
}

func (r reader) getString(k string) (s string) {
	s, _ = r.GetDataValue(k)
	return
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gzip implements reading and writing of gzip format compressed files,
// as specified in RFC 1952.
package gzip

import (
	"bufio"
	"compress/flate"
	"errors"
	"hash"
	"hash/crc32"
	"io"
	"time"
)

// BUG(nigeltao): Comments and Names don't properly map UTF-8 character codes outside of
// the 0x00-0x7f range to ISO 8859-1 (Latin-1).

const (
	gzipID1     = 0x1f
	gzipID2     = 0x8b
	gzipDeflate = 8
	flagText    = 1 << 0
	flagHdrCrc  = 1 << 1
	flagExtra   = 1 << 2
	flagName    = 1 << 3
	flagComment = 1 << 4
)

func makeReader(r io.Reader) flate.Reader {
	if rr, ok := r.(flate.Reader); ok {
		return rr
	}
	return bufio.NewReader(r)
}

var ErrHeader = errors.New("invalid gzip header")
var ErrChecksum = errors.New("gzip checksum error")

// The gzip file stores a header giving metadata about the compressed file.
// That header is exposed as the fields of the Compressor and Decompressor structs.
type Header struct {
	Comment string    // comment
	Extra   []byte    // "extra data"
	ModTime time.Time // modification time
	Name    string    // file name
	OS      byte      // operating system type
}

// An Decompressor is an io.Reader that can be read to retrieve
// uncompressed data from a gzip-format compressed file.
//
// In general, a gzip file can be a concatenation of gzip files,
// each with its own header.  Reads from the Decompressor
// return the concatenation of the uncompressed data of each.
// Only the first header is recorded in the Decompressor fields.
//
// Gzip files store a length and checksum of the uncompressed data.
// The Decompressor will return a ErrChecksum when Read
// reaches the end of the uncompressed data if it does not
// have the expected length or checksum.  Clients should treat data
// returned by Read as tentative until they receive the successful
// (zero length, nil error) Read marking the end of the data.
type Decompressor struct {
	Header
	r            flate.Reader
	decompressor io.ReadCloser
	digest       hash.Hash32
	size         uint32
	flg          byte
	buf          [512]byte
	err          error
}

// NewReader creates a new Decompressor reading the given reader.
// The implementation buffers input and may read more data than necessary from r.
// It is the caller's responsibility to call Close on the Decompressor when done.
func NewReader(r io.Reader) (*Decompressor, error) {
	z := new(Decompressor)
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	if err := z.readHeader(true); err != nil {
		z.err = err
		return nil, err
	}
	return z, nil
}

// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
func get4(p []byte) uint32 {
	return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24
}

func (z *Decompressor) readString() (string, error) {
	var err error
	needconv := false
	for i := 0; ; i++ {
		if i >= len(z.buf) {
			return "", ErrHeader
		}
		z.buf[i], err = z.r.ReadByte()
		if err != nil {
			return "", err
		}
		if z.buf[i] > 0x7f {
			needconv = true
		}
		if z.buf[i] == 0 {
			// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
			if needconv {
				s := make([]rune, 0, i)
				for _, v := range z.buf[0:i] {
					s = append(s, rune(v))
				}
				return string(s), nil
			}
			return string(z.buf[0:i]), nil
		}
	}
	panic("not reached")
}

func (z *Decompressor) read2() (uint32, error) {
	_, err := io.ReadFull(z.r, z.buf[0:2])
	if err != nil {
		return 0, err
	}
	return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil
}

func (z *Decompressor) readHeader(save bool) error {
	_, err := io.ReadFull(z.r, z.buf[0:10])
	if err != nil {
		return err
	}
	if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate {
		return ErrHeader
	}
	z.flg = z.buf[3]
	if save {
		z.ModTime = time.Unix(int64(get4(z.buf[4:8])), 0)
		// z.buf[8] is xfl, ignored
		z.OS = z.buf[9]
	}
	z.digest.Reset()
	z.digest.Write(z.buf[0:10])

	if z.flg&flagExtra != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		data := make([]byte, n)
		if _, err = io.ReadFull(z.r, data); err != nil {
			return err
		}
		if save {
			z.Extra = data
		}
	}

	var s string
	if z.flg&flagName != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Name = s
		}
	}

	if z.flg&flagComment != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Comment = s
		}
	}

	if z.flg&flagHdrCrc != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		sum := z.digest.Sum32() & 0xFFFF
		if n != sum {
			return ErrHeader
		}
	}

	z.digest.Reset()
	z.decompressor = flate.NewReader(z.r)
	return nil
}

func (z *Decompressor) Read(p []byte) (n int, err error) {
	if z.err != nil {
		return 0, z.err
	}
	if len(p) == 0 {
		return 0, nil
	}

	n, err = z.decompressor.Read(p)
	z.digest.Write(p[0:n])
	z.size += uint32(n)
	if n != 0 || err != io.EOF {
		z.err = err
		return
	}

	// Finished file; check checksum + size.
	if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
		z.err = err
		return 0, err
	}
	crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
	sum := z.digest.Sum32()
	if sum != crc32 || isize != z.size {
		z.err = ErrChecksum
		return 0, z.err
	}

	// File is ok; is there another?
	if err = z.readHeader(false); err != nil {
		z.err = err
		return
	}

	// Yes.  Reset and read from it.
	z.digest.Reset()
	z.size = 0
	return z.Read(p)
}

// Calling Close does not close the wrapped io.Reader originally passed to NewReader.
func (z *Decompressor) Close() error { return z.decompressor.Close() }
<commit_msg>compress/gzip: remove dead code<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gzip implements reading and writing of gzip format compressed files,
// as specified in RFC 1952.
package gzip

import (
	"bufio"
	"compress/flate"
	"errors"
	"hash"
	"hash/crc32"
	"io"
	"time"
)

// BUG(nigeltao): Comments and Names don't properly map UTF-8 character codes outside of
// the 0x00-0x7f range to ISO 8859-1 (Latin-1).

const (
	gzipID1     = 0x1f
	gzipID2     = 0x8b
	gzipDeflate = 8
	flagText    = 1 << 0
	flagHdrCrc  = 1 << 1
	flagExtra   = 1 << 2
	flagName    = 1 << 3
	flagComment = 1 << 4
)

func makeReader(r io.Reader) flate.Reader {
	if rr, ok := r.(flate.Reader); ok {
		return rr
	}
	return bufio.NewReader(r)
}

var ErrHeader = errors.New("invalid gzip header")
var ErrChecksum = errors.New("gzip checksum error")

// The gzip file stores a header giving metadata about the compressed file.
// That header is exposed as the fields of the Compressor and Decompressor structs.
type Header struct {
	Comment string    // comment
	Extra   []byte    // "extra data"
	ModTime time.Time // modification time
	Name    string    // file name
	OS      byte      // operating system type
}

// An Decompressor is an io.Reader that can be read to retrieve
// uncompressed data from a gzip-format compressed file.
//
// In general, a gzip file can be a concatenation of gzip files,
// each with its own header.  Reads from the Decompressor
// return the concatenation of the uncompressed data of each.
// Only the first header is recorded in the Decompressor fields.
//
// Gzip files store a length and checksum of the uncompressed data.
// The Decompressor will return a ErrChecksum when Read
// reaches the end of the uncompressed data if it does not
// have the expected length or checksum.  Clients should treat data
// returned by Read as tentative until they receive the successful
// (zero length, nil error) Read marking the end of the data.
type Decompressor struct {
	Header
	r            flate.Reader
	decompressor io.ReadCloser
	digest       hash.Hash32
	size         uint32
	flg          byte
	buf          [512]byte
	err          error
}

// NewReader creates a new Decompressor reading the given reader.
// The implementation buffers input and may read more data than necessary from r.
// It is the caller's responsibility to call Close on the Decompressor when done.
func NewReader(r io.Reader) (*Decompressor, error) {
	z := new(Decompressor)
	z.r = makeReader(r)
	z.digest = crc32.NewIEEE()
	if err := z.readHeader(true); err != nil {
		return nil, err
	}
	return z, nil
}

// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950).
func get4(p []byte) uint32 {
	return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24
}

func (z *Decompressor) readString() (string, error) {
	var err error
	needconv := false
	for i := 0; ; i++ {
		if i >= len(z.buf) {
			return "", ErrHeader
		}
		z.buf[i], err = z.r.ReadByte()
		if err != nil {
			return "", err
		}
		if z.buf[i] > 0x7f {
			needconv = true
		}
		if z.buf[i] == 0 {
			// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1).
			if needconv {
				s := make([]rune, 0, i)
				for _, v := range z.buf[0:i] {
					s = append(s, rune(v))
				}
				return string(s), nil
			}
			return string(z.buf[0:i]), nil
		}
	}
	panic("not reached")
}

func (z *Decompressor) read2() (uint32, error) {
	_, err := io.ReadFull(z.r, z.buf[0:2])
	if err != nil {
		return 0, err
	}
	return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil
}

func (z *Decompressor) readHeader(save bool) error {
	_, err := io.ReadFull(z.r, z.buf[0:10])
	if err != nil {
		return err
	}
	if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate {
		return ErrHeader
	}
	z.flg = z.buf[3]
	if save {
		z.ModTime = time.Unix(int64(get4(z.buf[4:8])), 0)
		// z.buf[8] is xfl, ignored
		z.OS = z.buf[9]
	}
	z.digest.Reset()
	z.digest.Write(z.buf[0:10])

	if z.flg&flagExtra != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		data := make([]byte, n)
		if _, err = io.ReadFull(z.r, data); err != nil {
			return err
		}
		if save {
			z.Extra = data
		}
	}

	var s string
	if z.flg&flagName != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Name = s
		}
	}

	if z.flg&flagComment != 0 {
		if s, err = z.readString(); err != nil {
			return err
		}
		if save {
			z.Comment = s
		}
	}

	if z.flg&flagHdrCrc != 0 {
		n, err := z.read2()
		if err != nil {
			return err
		}
		sum := z.digest.Sum32() & 0xFFFF
		if n != sum {
			return ErrHeader
		}
	}

	z.digest.Reset()
	z.decompressor = flate.NewReader(z.r)
	return nil
}

func (z *Decompressor) Read(p []byte) (n int, err error) {
	if z.err != nil {
		return 0, z.err
	}
	if len(p) == 0 {
		return 0, nil
	}

	n, err = z.decompressor.Read(p)
	z.digest.Write(p[0:n])
	z.size += uint32(n)
	if n != 0 || err != io.EOF {
		z.err = err
		return
	}

	// Finished file; check checksum + size.
	if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil {
		z.err = err
		return 0, err
	}
	crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8])
	sum := z.digest.Sum32()
	if sum != crc32 || isize != z.size {
		z.err = ErrChecksum
		return 0, z.err
	}

	// File is ok; is there another?
	if err = z.readHeader(false); err != nil {
		z.err = err
		return
	}

	// Yes.  Reset and read from it.
	z.digest.Reset()
	z.size = 0
	return z.Read(p)
}

// Calling Close does not close the wrapped io.Reader originally passed to NewReader.
func (z *Decompressor) Close() error { return z.decompressor.Close() }
<|endoftext|>