<commit_before>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kubernetes

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/pkg/errors"
	"github.com/tsuru/tsuru/provision/nodecontainer"
	"github.com/tsuru/tsuru/provision/provisioncommon"
	"github.com/tsuru/tsuru/provision/servicecommon"
	"k8s.io/client-go/kubernetes"
	k8sErrors "k8s.io/client-go/pkg/api/errors"
	"k8s.io/client-go/pkg/api/unversioned"
	"k8s.io/client-go/pkg/api/v1"
	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)

type nodeContainerManager struct {
	client kubernetes.Interface
}

func (m *nodeContainerManager) DeployNodeContainer(config *nodecontainer.NodeContainerConfig, pool string, filter servicecommon.PoolFilter, placementOnly bool) error {
	dsName := daemonSetName(config.Name, pool)
	oldDs, err := m.client.Extensions().DaemonSets(tsuruNamespace).Get(dsName)
	if err != nil {
		if !k8sErrors.IsNotFound(err) {
			return errors.WithStack(err)
		}
		oldDs = nil
	}
	nodeReq := v1.NodeSelectorRequirement{
		Key: labelNodePoolName,
	}
	if len(filter.Exclude) > 0 {
		nodeReq.Operator = v1.NodeSelectorOpNotIn
		nodeReq.Values = filter.Exclude
	} else {
		nodeReq.Operator = v1.NodeSelectorOpIn
		nodeReq.Values = filter.Include
	}
	affinityAnnotation := map[string]string{}
	if len(nodeReq.Values) != 0 {
		affinity := v1.Affinity{
			NodeAffinity: &v1.NodeAffinity{
				RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
					NodeSelectorTerms: []v1.NodeSelectorTerm{{
						MatchExpressions: []v1.NodeSelectorRequirement{nodeReq},
					}},
				},
			},
		}
		var affinityData []byte
		affinityData, err = json.Marshal(affinity)
		if err != nil {
			return errors.WithStack(err)
		}
		affinityAnnotation["scheduler.alpha.kubernetes.io/affinity"] = string(affinityData)
	}
	if oldDs != nil && placementOnly {
		oldDs.Spec.Template.ObjectMeta.Annotations = affinityAnnotation
		_, err = m.client.Extensions().DaemonSets(tsuruNamespace).Update(oldDs)
		return errors.WithStack(err)
	}
	ls := provisioncommon.NodeContainerLabels(config.Name, pool, provisionerName, config.Config.Labels)
	envVars := make([]v1.EnvVar, len(config.Config.Env))
	for i, v := range config.Config.Env {
		parts := strings.SplitN(v, "=", 2)
		envVars[i].Name = parts[0]
		if len(parts) > 1 {
			envVars[i].Value = parts[1]
		}
	}
	var volumes []v1.Volume
	var volumeMounts []v1.VolumeMount
	if config.Name == nodecontainer.BsDefaultName {
		config.HostConfig.Binds = append(config.HostConfig.Binds,
			"/var/log:/var/log:ro",
			"/var/lib/docker/containers:/var/lib/docker/containers:ro")
	}
	for i, b := range config.HostConfig.Binds {
		parts := strings.SplitN(b, ":", 3)
		vol := v1.Volume{
			Name: fmt.Sprintf("volume-%d", i),
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: parts[0],
				},
			},
		}
		mount := v1.VolumeMount{
			Name: vol.Name,
		}
		if len(parts) > 1 {
			mount.MountPath = parts[1]
		}
		if len(parts) > 2 {
			mount.ReadOnly = parts[2] == "ro"
		}
		volumes = append(volumes, vol)
		volumeMounts = append(volumeMounts, mount)
	}
	var secCtx *v1.SecurityContext
	if config.HostConfig.Privileged {
		trueVar := true
		secCtx = &v1.SecurityContext{
			Privileged: &trueVar,
		}
	}
	restartPolicy := v1.RestartPolicyAlways
	switch config.HostConfig.RestartPolicy.Name {
	case docker.RestartOnFailure(0).Name:
		restartPolicy = v1.RestartPolicyOnFailure
	case docker.NeverRestart().Name:
		restartPolicy = v1.RestartPolicyNever
	}
	ds := &extensions.DaemonSet{
		ObjectMeta: v1.ObjectMeta{
			Name:      dsName,
			Namespace: tsuruNamespace,
		},
		Spec: extensions.DaemonSetSpec{
			Selector: &unversioned.LabelSelector{
				MatchLabels: ls.ToNodeContainerSelector(),
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: v1.ObjectMeta{
					Labels:      ls.ToLabels(),
					Annotations: affinityAnnotation,
				},
				Spec: v1.PodSpec{
					Volumes:       volumes,
					RestartPolicy: restartPolicy,
					HostNetwork:   config.HostConfig.NetworkMode == "host",
					Containers: []v1.Container{
						{
							Name:            config.Name,
							Image:           config.Image(),
							Command:         config.Config.Entrypoint,
							Args:            config.Config.Cmd,
							Env:             envVars,
							WorkingDir:      config.Config.WorkingDir,
							TTY:             config.Config.Tty,
							VolumeMounts:    volumeMounts,
							SecurityContext: secCtx,
						},
					},
				},
			},
		},
	}
	if oldDs != nil {
		_, err = m.client.Extensions().DaemonSets(tsuruNamespace).Update(ds)
	} else {
		_, err = m.client.Extensions().DaemonSets(tsuruNamespace).Create(ds)
	}
	return errors.WithStack(err)
}
<commit_msg>provision/kubernetes: daemon sets do not recreate pods on update<commit_after>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kubernetes

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/fsouza/go-dockerclient"
	"github.com/pkg/errors"
	"github.com/tsuru/tsuru/provision/nodecontainer"
	"github.com/tsuru/tsuru/provision/provisioncommon"
	"github.com/tsuru/tsuru/provision/servicecommon"
	"k8s.io/client-go/kubernetes"
	k8sErrors "k8s.io/client-go/pkg/api/errors"
	"k8s.io/client-go/pkg/api/unversioned"
	"k8s.io/client-go/pkg/api/v1"
	extensions "k8s.io/client-go/pkg/apis/extensions/v1beta1"
)

type nodeContainerManager struct {
	client kubernetes.Interface
}

func (m *nodeContainerManager) DeployNodeContainer(config *nodecontainer.NodeContainerConfig, pool string, filter servicecommon.PoolFilter, placementOnly bool) error {
	dsName := daemonSetName(config.Name, pool)
	oldDs, err := m.client.Extensions().DaemonSets(tsuruNamespace).Get(dsName)
	if err != nil {
		if !k8sErrors.IsNotFound(err) {
			return errors.WithStack(err)
		}
		oldDs = nil
	}
	nodeReq := v1.NodeSelectorRequirement{
		Key: labelNodePoolName,
	}
	if len(filter.Exclude) > 0 {
		nodeReq.Operator = v1.NodeSelectorOpNotIn
		nodeReq.Values = filter.Exclude
	} else {
		nodeReq.Operator = v1.NodeSelectorOpIn
		nodeReq.Values = filter.Include
	}
	affinityAnnotation := map[string]string{}
	if len(nodeReq.Values) != 0 {
		affinity := v1.Affinity{
			NodeAffinity: &v1.NodeAffinity{
				RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{
					NodeSelectorTerms: []v1.NodeSelectorTerm{{
						MatchExpressions: []v1.NodeSelectorRequirement{nodeReq},
					}},
				},
			},
		}
		var affinityData []byte
		affinityData, err = json.Marshal(affinity)
		if err != nil {
			return errors.WithStack(err)
		}
		affinityAnnotation["scheduler.alpha.kubernetes.io/affinity"] = string(affinityData)
	}
	if oldDs != nil && placementOnly {
		oldDs.Spec.Template.ObjectMeta.Annotations = affinityAnnotation
		_, err = m.client.Extensions().DaemonSets(tsuruNamespace).Update(oldDs)
		return errors.WithStack(err)
	}
	ls := provisioncommon.NodeContainerLabels(config.Name, pool, provisionerName, config.Config.Labels)
	envVars := make([]v1.EnvVar, len(config.Config.Env))
	for i, v := range config.Config.Env {
		parts := strings.SplitN(v, "=", 2)
		envVars[i].Name = parts[0]
		if len(parts) > 1 {
			envVars[i].Value = parts[1]
		}
	}
	var volumes []v1.Volume
	var volumeMounts []v1.VolumeMount
	if config.Name == nodecontainer.BsDefaultName {
		config.HostConfig.Binds = append(config.HostConfig.Binds,
			"/var/log:/var/log:ro",
			"/var/lib/docker/containers:/var/lib/docker/containers:ro")
	}
	for i, b := range config.HostConfig.Binds {
		parts := strings.SplitN(b, ":", 3)
		vol := v1.Volume{
			Name: fmt.Sprintf("volume-%d", i),
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: parts[0],
				},
			},
		}
		mount := v1.VolumeMount{
			Name: vol.Name,
		}
		if len(parts) > 1 {
			mount.MountPath = parts[1]
		}
		if len(parts) > 2 {
			mount.ReadOnly = parts[2] == "ro"
		}
		volumes = append(volumes, vol)
		volumeMounts = append(volumeMounts, mount)
	}
	var secCtx *v1.SecurityContext
	if config.HostConfig.Privileged {
		trueVar := true
		secCtx = &v1.SecurityContext{
			Privileged: &trueVar,
		}
	}
	restartPolicy := v1.RestartPolicyAlways
	switch config.HostConfig.RestartPolicy.Name {
	case docker.RestartOnFailure(0).Name:
		restartPolicy = v1.RestartPolicyOnFailure
	case docker.NeverRestart().Name:
		restartPolicy = v1.RestartPolicyNever
	}
	ds := &extensions.DaemonSet{
		ObjectMeta: v1.ObjectMeta{
			Name:      dsName,
			Namespace: tsuruNamespace,
		},
		Spec: extensions.DaemonSetSpec{
			Selector: &unversioned.LabelSelector{
				MatchLabels: ls.ToNodeContainerSelector(),
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: v1.ObjectMeta{
					Labels:      ls.ToLabels(),
					Annotations: affinityAnnotation,
				},
				Spec: v1.PodSpec{
					Volumes:       volumes,
					RestartPolicy: restartPolicy,
					HostNetwork:   config.HostConfig.NetworkMode == "host",
					Containers: []v1.Container{
						{
							Name:            config.Name,
							Image:           config.Image(),
							Command:         config.Config.Entrypoint,
							Args:            config.Config.Cmd,
							Env:             envVars,
							WorkingDir:      config.Config.WorkingDir,
							TTY:             config.Config.Tty,
							VolumeMounts:    volumeMounts,
							SecurityContext: secCtx,
						},
					},
				},
			},
		},
	}
	if oldDs != nil {
		// TODO(cezarsa): This is only needed because kubernetes <=1.5 does not
		// support rolling updating daemon sets. Once 1.6 is out we can drop
		// the cleanup call and configure DaemonSetUpdateStrategy accordingly.
		err = cleanupDaemonSet(m.client, config.Name, pool)
		if err != nil {
			return err
		}
		_, err = m.client.Extensions().DaemonSets(tsuruNamespace).Create(ds)
	} else {
		_, err = m.client.Extensions().DaemonSets(tsuruNamespace).Create(ds)
	}
	return errors.WithStack(err)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"log"
	"bytes"
	"io"
	"time"
	"math"
	"github.com/metachord/flv.go/flv"
	"github.com/metachord/amf.go/amf0"
)

var inFile string
var outFile string

func init() {
	flag.StringVar(&inFile, "in", "", "input file")
	flag.StringVar(&outFile, "out", "", "output file")
}

type kfTimePos struct {
	Dts uint32
	Position int64
}

func main() {
	flag.Parse()

	log.Printf("Read from %s, write to %s\n", inFile, outFile)
	inF, err := os.Open(inFile)
	if err != nil {
		log.Fatal(err)
	}
	defer inF.Close()

	fi, err := inF.Stat()
	if err != nil {
		log.Fatal(err)
	}

	filesize := fi.Size()

	frReader := flv.NewReader(inF)

	header, err := frReader.ReadHeader()
	if err != nil {
		log.Fatal(err)
	}

	var lastKeyFrameTs, lastVTs, lastTs uint32
	var width, height uint16
	var audioRate uint32
	var videoFrameSize, audioFrameSize, dataFrameSize, metadataFrameSize uint64 = 0, 0, 0, 0
	var videoSize, audioSize uint64 = 0, 0
	var videoFrames, audioFrames uint32 = 0, 0
	var stereo bool = false
	var videoCodec, audioCodec uint8 = 0, 0
	var audioSampleSize uint32 = 0
	var hasVideo, hasAudio, hasMetadata, hasKeyframes bool = false, false, false, false

	var oldOnMetaDataSize uint64 = 0

	var kfs []kfTimePos

nextFrame:
	for {
		frame, err := frReader.ReadFrame()
		if (frame != nil) {
			switch frame.(type) {
			case flv.VideoFrame:
				tfr := frame.(flv.VideoFrame)
				if (width == 0) || (height == 0) {
					width, height = tfr.Width, tfr.Height
					//log.Printf("VideoCodec: %d, Width: %d, Height: %d", tfr.CodecId, tfr.Width, tfr.Height)
				}
				switch tfr.Flavor {
				case flv.KEYFRAME:
					lastKeyFrameTs = tfr.Dts
					hasKeyframes = true
					kfs = append(kfs, kfTimePos{Dts: tfr.Dts, Position: tfr.Position})
				default:
					videoFrames++
				}
				hasVideo = true
				lastVTs = tfr.Dts
				lastTs = tfr.Dts
				videoCodec = uint8(tfr.CodecId)
				videoFrameSize += uint64(tfr.PrevTagSize)
				videoSize += uint64(len(tfr.Body))
			case flv.AudioFrame:
				tfr := frame.(flv.AudioFrame)
				//log.Printf("AudioCodec: %d, Rate: %d, BitSize: %d, Channels: %d", tfr.CodecId, tfr.Rate, tfr.BitSize, tfr.Channels)
				lastTs = tfr.Dts
				audioRate = tfr.Rate
				audioFrameSize += uint64(tfr.PrevTagSize)
				audioSize += uint64(len(tfr.Body))
				if tfr.Channels == flv.AUDIO_TYPE_STEREO {
					stereo = true
				}
				switch tfr.BitSize {
				case flv.AUDIO_SIZE_8BIT:
					audioSampleSize = 8
				case flv.AUDIO_SIZE_16BIT:
					audioSampleSize = 16
				}
				hasAudio = true
				audioCodec = uint8(tfr.CodecId)
				audioFrames++
			case flv.MetaFrame:
				tfr := frame.(flv.MetaFrame)
				buf := bytes.NewReader(tfr.Body)
				dec := amf0.NewDecoder(buf)

				evName, err := dec.Decode()
				if err != nil {
					break nextFrame
				}
				switch evName {
				case amf0.StringType("onMetaData"):
					oldOnMetaDataSize = uint64(tfr.PrevTagSize)
					md, err := dec.Decode()
					if err != nil {
						break nextFrame
					}

					log.Printf("Old onMetaData")
					switch md.(type) {
					case *amf0.EcmaArrayType:
						ea := md.(*amf0.EcmaArrayType)
						for k, v := range (*ea) {
							log.Printf("%v = %v\n", k, v)
						}
						if width == 0 {width = uint16((*ea)["width"].(amf0.NumberType))}
						if height == 0 {height = uint16((*ea)["height"].(amf0.NumberType))}
					case *amf0.ObjectType:
						ea := md.(*amf0.ObjectType)
						for k, v := range (*ea) {
							log.Printf("%v = %v\n", k, v)
						}
						if width == 0 {width = uint16((*ea)["width"].(amf0.NumberType))}
						if height == 0 {height = uint16((*ea)["height"].(amf0.NumberType))}
					}
				default:
					log.Printf("Unknown event: %s\n", evName)
				}
				hasMetadata = true
				lastTs = tfr.Dts
				metadataFrameSize += uint64(tfr.PrevTagSize)
			}
		} else {
			break
		}
		if err != nil {
			break
		}
	}
	//log.Printf("KFS: %v", kfs)
	lastKeyFrameTsF := float32(lastKeyFrameTs)/1000
	lastVTsF := float32(lastVTs)/1000
	duration := float32(lastTs)/1000
	dataFrameSize = videoFrameSize + audioFrameSize + metadataFrameSize

	now := time.Now()
	metadatadate := float64(now.Unix() * 1000) + (float64(now.Nanosecond()) / 1000000)

	videoDataRate := (float32(videoSize) / float32(duration))*8/1000
	audioDataRate := (float32(audioSize) / float32(duration))*8/1000

	frameRate := uint8(math.Floor(float64(videoFrames) / float64(duration)))

	//log.Printf("oldOnMetaDataSize: %d, FileSize: %d, LastKeyFrameTS: %f, LastTS: %f, Width: %d, Height: %d, VideoSize: %d, AudioSize: %d, MetaDataSize: %d, DataSize: %d, Duration: %f, MetadataDate: %f, VideoDataRate: %f, AudioDataRate: %f, FrameRate: %d, AudioRate: %d", oldOnMetaDataSize, filesize, lastKeyFrameTsF, lastVTsF, width, height, videoFrameSize, audioFrameSize, metadataFrameSize, dataFrameSize, duration, metadatadate, videoDataRate, audioDataRate, frameRate, audioRate)

	kfTimes := make(amf0.StrictArrayType, 0)
	kfPositions := make(amf0.StrictArrayType, 0)

	for i := range(kfs) {
		kfTimes = append(kfTimes, amf0.NumberType((float64(kfs[i].Dts)/1000)))
		kfPositions = append(kfTimes, amf0.NumberType(kfs[i].Position))
	}

	keyFrames := amf0.ObjectType{
		"times": &kfTimes,
		"filepositions": &kfPositions,
	}

	metaMap := amf0.EcmaArrayType {
		"metadatacreator": amf0.StringType("Flvtag https://github.com/metachord/flvtag"),
		"metadatadate": amf0.DateType{TimeZone: 0, Date: metadatadate},

		"keyframes": &keyFrames,

		"hasVideo": amf0.BooleanType(hasVideo),
		"hasAudio": amf0.BooleanType(hasAudio),
		"hasMetadata": amf0.BooleanType(hasMetadata),
		"hasKeyframes": amf0.BooleanType(hasKeyframes),
		"hasCuePoints": amf0.BooleanType(false),

		"videocodecid": amf0.NumberType(videoCodec),
		"width": amf0.NumberType(width),
		"height": amf0.NumberType(height),
		"videosize": amf0.NumberType(videoFrameSize),
		"framerate": amf0.NumberType(frameRate),
		"videodatarate": amf0.NumberType(videoDataRate),

		"audiocodecid": amf0.NumberType(audioCodec),
		"stereo": amf0.BooleanType(stereo),
		"audiosamplesize": amf0.NumberType(audioSampleSize),
		"audiodelay": amf0.NumberType(0),
		"audiodatarate": amf0.NumberType(audioDataRate),
		"audiosize": amf0.NumberType(audioFrameSize),
		"audiosamplerate": amf0.NumberType(audioRate),

		"filesize": amf0.NumberType(filesize),
		"datasize": amf0.NumberType(dataFrameSize),
		"lasttimestamp": amf0.NumberType(lastVTsF),
		"lastkeyframetimestamp": amf0.NumberType(lastKeyFrameTsF),
		"cuePoints": &amf0.StrictArrayType{},
		"duration": amf0.NumberType(duration),
		"canSeekToEnd": amf0.BooleanType(false),
	}

	log.Printf("New onMetaData")
	for k, v := range (metaMap) {
		log.Printf("%v = %v\n", k, v)
	}

	buf := new(bytes.Buffer)
	enc := amf0.NewEncoder(buf)
	err = enc.Encode(&metaMap)
	if err != nil {
		log.Fatalf("%s", err)
	}

	newOnMetaDataSize := uint64(buf.Len()) + uint64(flv.TAG_HEADER_LENGTH) + uint64(flv.PREV_TAG_SIZE_LENGTH)
	//log.Printf("newOnMetaDataSize: %v", newOnMetaDataSize)
	//log.Printf("oldKeyFrames: %v", &keyFrames)

	newKfPositions := make(amf0.StrictArrayType, 0)

	for i := range(kfs) {
		newKfPositions = append(newKfPositions, amf0.NumberType(uint64(kfs[i].Position) + newOnMetaDataSize - oldOnMetaDataSize))
	}
	keyFrames["filepositions"] = &newKfPositions

	//log.Printf("newKeyFrames: %v", &keyFrames)

	newBuf := new(bytes.Buffer)
	newEnc := amf0.NewEncoder(newBuf)

	err = newEnc.Encode(amf0.StringType("onMetaData"))
	if err != nil {
		log.Fatalf("%s", err)
	}

	err = newEnc.Encode(&metaMap)
	if err != nil {
		log.Fatalf("%s", err)
	}

	outF, err := os.Create(outFile)
	if err != nil {
		log.Fatal(err)
	}
	defer outF.Close()

	frWriter := flv.NewWriter(outF)

	frWriter.WriteHeader(header)

	newMdFrame := flv.MetaFrame {
		flv.CFrame{
			Stream: 0,
			Dts: 0,
			Type: flv.TAG_TYPE_META,
			Flavor: flv.METADATA,
			Body: newBuf.Bytes(),
		},
	}

	frWriter.WriteFrame(newMdFrame)

	//log.Printf("NewMetaData: %v", newBuf)

	inStart := kfs[0].Position
	inF.Seek(inStart, os.SEEK_SET)

	bufn := 4096
	tmpbuf := make([]byte, bufn)
	for {
		rn, err := inF.Read(tmpbuf)
		if (err != nil) && (err != io.EOF) {
				log.Fatal(err)
		}
		if (rn != bufn) && ((err != nil) && (err != io.EOF)) {
			log.Fatal("cannot read bufn bytes")
		}

		wn, err := outF.Write(tmpbuf[:rn])
		if err != nil {
			log.Fatal(err)
		}
		if wn != bufn {
			break
		}
	}

	inF.Close()
	outF.Close()

}

<commit_msg>Remove duplicated code (thanks to Rémy Oudompheng from golang-nuts maillist)<commit_after>package main

import (
	"flag"
	"os"
	"log"
	"bytes"
	"io"
	"time"
	"math"
	"github.com/metachord/flv.go/flv"
	"github.com/metachord/amf.go/amf0"
)

var inFile string
var outFile string

func init() {
	flag.StringVar(&inFile, "in", "", "input file")
	flag.StringVar(&outFile, "out", "", "output file")
}

type kfTimePos struct {
	Dts uint32
	Position int64
}

func main() {
	flag.Parse()

	log.Printf("Read from %s, write to %s\n", inFile, outFile)
	inF, err := os.Open(inFile)
	if err != nil {
		log.Fatal(err)
	}
	defer inF.Close()

	fi, err := inF.Stat()
	if err != nil {
		log.Fatal(err)
	}

	filesize := fi.Size()

	frReader := flv.NewReader(inF)

	header, err := frReader.ReadHeader()
	if err != nil {
		log.Fatal(err)
	}

	var lastKeyFrameTs, lastVTs, lastTs uint32
	var width, height uint16
	var audioRate uint32
	var videoFrameSize, audioFrameSize, dataFrameSize, metadataFrameSize uint64 = 0, 0, 0, 0
	var videoSize, audioSize uint64 = 0, 0
	var videoFrames, audioFrames uint32 = 0, 0
	var stereo bool = false
	var videoCodec, audioCodec uint8 = 0, 0
	var audioSampleSize uint32 = 0
	var hasVideo, hasAudio, hasMetadata, hasKeyframes bool = false, false, false, false

	var oldOnMetaDataSize uint64 = 0

	var kfs []kfTimePos

nextFrame:
	for {
		frame, err := frReader.ReadFrame()
		if (frame != nil) {
			switch frame.(type) {
			case flv.VideoFrame:
				tfr := frame.(flv.VideoFrame)
				if (width == 0) || (height == 0) {
					width, height = tfr.Width, tfr.Height
					//log.Printf("VideoCodec: %d, Width: %d, Height: %d", tfr.CodecId, tfr.Width, tfr.Height)
				}
				switch tfr.Flavor {
				case flv.KEYFRAME:
					lastKeyFrameTs = tfr.Dts
					hasKeyframes = true
					kfs = append(kfs, kfTimePos{Dts: tfr.Dts, Position: tfr.Position})
				default:
					videoFrames++
				}
				hasVideo = true
				lastVTs = tfr.Dts
				lastTs = tfr.Dts
				videoCodec = uint8(tfr.CodecId)
				videoFrameSize += uint64(tfr.PrevTagSize)
				videoSize += uint64(len(tfr.Body))
			case flv.AudioFrame:
				tfr := frame.(flv.AudioFrame)
				//log.Printf("AudioCodec: %d, Rate: %d, BitSize: %d, Channels: %d", tfr.CodecId, tfr.Rate, tfr.BitSize, tfr.Channels)
				lastTs = tfr.Dts
				audioRate = tfr.Rate
				audioFrameSize += uint64(tfr.PrevTagSize)
				audioSize += uint64(len(tfr.Body))
				if tfr.Channels == flv.AUDIO_TYPE_STEREO {
					stereo = true
				}
				switch tfr.BitSize {
				case flv.AUDIO_SIZE_8BIT:
					audioSampleSize = 8
				case flv.AUDIO_SIZE_16BIT:
					audioSampleSize = 16
				}
				hasAudio = true
				audioCodec = uint8(tfr.CodecId)
				audioFrames++
			case flv.MetaFrame:
				tfr := frame.(flv.MetaFrame)
				buf := bytes.NewReader(tfr.Body)
				dec := amf0.NewDecoder(buf)

				evName, err := dec.Decode()
				if err != nil {
					break nextFrame
				}
				switch evName {
				case amf0.StringType("onMetaData"):
					oldOnMetaDataSize = uint64(tfr.PrevTagSize)
					md, err := dec.Decode()
					if err != nil {
						break nextFrame
					}

					log.Printf("Old onMetaData")
					var ea map[amf0.StringType]interface{}
					switch md := md.(type) {
					case *amf0.EcmaArrayType:
						ea = *md
					case *amf0.ObjectType:
						ea = *md
					}
					for k, v := range ea {
						log.Printf("%v = %v\n", k, v)
					}
					if width == 0 {width = uint16(((ea)["width"]).(amf0.NumberType))}
					if height == 0 {height = uint16(((ea)["height"]).(amf0.NumberType))}

				default:
					log.Printf("Unknown event: %s\n", evName)
				}
				hasMetadata = true
				lastTs = tfr.Dts
				metadataFrameSize += uint64(tfr.PrevTagSize)
			}
		} else {
			break
		}
		if err != nil {
			break
		}
	}
	//log.Printf("KFS: %v", kfs)
	lastKeyFrameTsF := float32(lastKeyFrameTs)/1000
	lastVTsF := float32(lastVTs)/1000
	duration := float32(lastTs)/1000
	dataFrameSize = videoFrameSize + audioFrameSize + metadataFrameSize

	now := time.Now()
	metadatadate := float64(now.Unix() * 1000) + (float64(now.Nanosecond()) / 1000000)

	videoDataRate := (float32(videoSize) / float32(duration))*8/1000
	audioDataRate := (float32(audioSize) / float32(duration))*8/1000

	frameRate := uint8(math.Floor(float64(videoFrames) / float64(duration)))

	//log.Printf("oldOnMetaDataSize: %d, FileSize: %d, LastKeyFrameTS: %f, LastTS: %f, Width: %d, Height: %d, VideoSize: %d, AudioSize: %d, MetaDataSize: %d, DataSize: %d, Duration: %f, MetadataDate: %f, VideoDataRate: %f, AudioDataRate: %f, FrameRate: %d, AudioRate: %d", oldOnMetaDataSize, filesize, lastKeyFrameTsF, lastVTsF, width, height, videoFrameSize, audioFrameSize, metadataFrameSize, dataFrameSize, duration, metadatadate, videoDataRate, audioDataRate, frameRate, audioRate)

	kfTimes := make(amf0.StrictArrayType, 0)
	kfPositions := make(amf0.StrictArrayType, 0)

	for i := range(kfs) {
		kfTimes = append(kfTimes, amf0.NumberType((float64(kfs[i].Dts)/1000)))
		kfPositions = append(kfTimes, amf0.NumberType(kfs[i].Position))
	}

	keyFrames := amf0.ObjectType{
		"times": &kfTimes,
		"filepositions": &kfPositions,
	}

	metaMap := amf0.EcmaArrayType {
		"metadatacreator": amf0.StringType("Flvtag https://github.com/metachord/flvtag"),
		"metadatadate": amf0.DateType{TimeZone: 0, Date: metadatadate},

		"keyframes": &keyFrames,

		"hasVideo": amf0.BooleanType(hasVideo),
		"hasAudio": amf0.BooleanType(hasAudio),
		"hasMetadata": amf0.BooleanType(hasMetadata),
		"hasKeyframes": amf0.BooleanType(hasKeyframes),
		"hasCuePoints": amf0.BooleanType(false),

		"videocodecid": amf0.NumberType(videoCodec),
		"width": amf0.NumberType(width),
		"height": amf0.NumberType(height),
		"videosize": amf0.NumberType(videoFrameSize),
		"framerate": amf0.NumberType(frameRate),
		"videodatarate": amf0.NumberType(videoDataRate),

		"audiocodecid": amf0.NumberType(audioCodec),
		"stereo": amf0.BooleanType(stereo),
		"audiosamplesize": amf0.NumberType(audioSampleSize),
		"audiodelay": amf0.NumberType(0),
		"audiodatarate": amf0.NumberType(audioDataRate),
		"audiosize": amf0.NumberType(audioFrameSize),
		"audiosamplerate": amf0.NumberType(audioRate),

		"filesize": amf0.NumberType(filesize),
		"datasize": amf0.NumberType(dataFrameSize),
		"lasttimestamp": amf0.NumberType(lastVTsF),
		"lastkeyframetimestamp": amf0.NumberType(lastKeyFrameTsF),
		"cuePoints": &amf0.StrictArrayType{},
		"duration": amf0.NumberType(duration),
		"canSeekToEnd": amf0.BooleanType(false),
	}

	log.Printf("New onMetaData")
	for k, v := range (metaMap) {
		log.Printf("%v = %v\n", k, v)
	}

	buf := new(bytes.Buffer)
	enc := amf0.NewEncoder(buf)
	err = enc.Encode(&metaMap)
	if err != nil {
		log.Fatalf("%s", err)
	}

	newOnMetaDataSize := uint64(buf.Len()) + uint64(flv.TAG_HEADER_LENGTH) + uint64(flv.PREV_TAG_SIZE_LENGTH)
	//log.Printf("newOnMetaDataSize: %v", newOnMetaDataSize)
	//log.Printf("oldKeyFrames: %v", &keyFrames)

	newKfPositions := make(amf0.StrictArrayType, 0)

	for i := range(kfs) {
		newKfPositions = append(newKfPositions, amf0.NumberType(uint64(kfs[i].Position) + newOnMetaDataSize - oldOnMetaDataSize))
	}
	keyFrames["filepositions"] = &newKfPositions

	//log.Printf("newKeyFrames: %v", &keyFrames)

	newBuf := new(bytes.Buffer)
	newEnc := amf0.NewEncoder(newBuf)

	err = newEnc.Encode(amf0.StringType("onMetaData"))
	if err != nil {
		log.Fatalf("%s", err)
	}

	err = newEnc.Encode(&metaMap)
	if err != nil {
		log.Fatalf("%s", err)
	}

	outF, err := os.Create(outFile)
	if err != nil {
		log.Fatal(err)
	}
	defer outF.Close()

	frWriter := flv.NewWriter(outF)

	frWriter.WriteHeader(header)

	newMdFrame := flv.MetaFrame {
		flv.CFrame{
			Stream: 0,
			Dts: 0,
			Type: flv.TAG_TYPE_META,
			Flavor: flv.METADATA,
			Body: newBuf.Bytes(),
		},
	}

	frWriter.WriteFrame(newMdFrame)

	//log.Printf("NewMetaData: %v", newBuf)

	inStart := kfs[0].Position
	inF.Seek(inStart, os.SEEK_SET)

	bufn := 4096
	tmpbuf := make([]byte, bufn)
	for {
		rn, err := inF.Read(tmpbuf)
		if (err != nil) && (err != io.EOF) {
				log.Fatal(err)
		}
		if (rn != bufn) && ((err != nil) && (err != io.EOF)) {
			log.Fatal("cannot read bufn bytes")
		}

		wn, err := outF.Write(tmpbuf[:rn])
		if err != nil {
			log.Fatal(err)
		}
		if wn != bufn {
			break
		}
	}

	inF.Close()
	outF.Close()

}

<|endoftext|>


<commit_before>package metrin

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
)

const (
	defaultRegion = "us-east-1"
)

// BuildParamsInput - the type for BuildParams function argument
type BuildParamsInput struct {
	Namespace          string
	MetricName         string
	StartTime          int64
	EndTime            int64
	Period             int64
	Unit               string
	Statistics         []string
	ExtendedStatistics []string
	Dimensions         []string
}

// BuildParams - used for create GetMetricStatisticsInput
func BuildParams(input BuildParamsInput) *cloudwatch.GetMetricStatisticsInput {
	var startTime time.Time
	var endTime time.Time

	if input.StartTime <= 0 {
		startTime = time.Unix((time.Now().Unix() + input.StartTime), 0)
	}

	if input.EndTime <= 0 {
		endTime = time.Unix((time.Now().Unix() + input.EndTime), 0)
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(input.Namespace),
		MetricName: aws.String(input.MetricName),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
		Period:     aws.Int64(input.Period),
	}

	if len(input.Unit) > 0 {
		params.Unit = aws.String(input.Unit)
	}

	if len(input.Statistics) > 0 {
		s := make([]*string, len(input.Statistics))

		for i := range input.Statistics {
			s[i] = aws.String(input.Statistics[i])
		}

		params.Statistics = s
	}

	if len(input.ExtendedStatistics) > 0 {
		s := make([]*string, len(input.ExtendedStatistics))

		for i := range input.ExtendedStatistics {
			s[i] = aws.String(input.ExtendedStatistics[i])
		}

		params.ExtendedStatistics = s
	}

	if len(input.Dimensions) > 0 {
		s := make([]*cloudwatch.Dimension, len(input.Dimensions))

		for i := range input.Dimensions {
			splitted := strings.Split(input.Dimensions[i], ":")

			s[i] = &cloudwatch.Dimension{
				Name:  aws.String(splitted[0]),
				Value: aws.String(splitted[1]),
			}
		}

		params.Dimensions = s
	}

	return params
}

// GetMetricStatistics wrapper
func GetMetricStatistics(params *cloudwatch.GetMetricStatisticsInput) *cloudwatch.GetMetricStatisticsOutput {
	if os.Getenv("AWS_REGION") == "" {
		os.Setenv("AWS_REGION", defaultRegion)
	}

	session, err := session.NewSession()
	if err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	service := cloudwatch.New(session)

	response, err := service.GetMetricStatistics(params)
	if err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	return response
}

// GetLastDatapoint - get latest datapoint from datapoints by timestamp
func GetLastDatapoint(datapoints []*cloudwatch.Datapoint) *cloudwatch.Datapoint {
	lastDatapoint := datapoints[0]

	for i := range datapoints {
		datapoint := datapoints[i]

		if datapoint.Timestamp.Unix() > lastDatapoint.Timestamp.Unix() {
			lastDatapoint = datapoint
		}
	}

	return lastDatapoint
}
<commit_msg>check empty datapoints<commit_after>package metrin

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
)

const (
	defaultRegion = "us-east-1"
)

// BuildParamsInput - the type for BuildParams function argument
type BuildParamsInput struct {
	Namespace          string
	MetricName         string
	StartTime          int64
	EndTime            int64
	Period             int64
	Unit               string
	Statistics         []string
	ExtendedStatistics []string
	Dimensions         []string
}

// BuildParams - used for create GetMetricStatisticsInput
func BuildParams(input BuildParamsInput) *cloudwatch.GetMetricStatisticsInput {
	var startTime time.Time
	var endTime time.Time

	if input.StartTime <= 0 {
		startTime = time.Unix((time.Now().Unix() + input.StartTime), 0)
	}

	if input.EndTime <= 0 {
		endTime = time.Unix((time.Now().Unix() + input.EndTime), 0)
	}

	params := &cloudwatch.GetMetricStatisticsInput{
		Namespace:  aws.String(input.Namespace),
		MetricName: aws.String(input.MetricName),
		StartTime:  aws.Time(startTime),
		EndTime:    aws.Time(endTime),
		Period:     aws.Int64(input.Period),
	}

	if len(input.Unit) > 0 {
		params.Unit = aws.String(input.Unit)
	}

	if len(input.Statistics) > 0 {
		s := make([]*string, len(input.Statistics))

		for i := range input.Statistics {
			s[i] = aws.String(input.Statistics[i])
		}

		params.Statistics = s
	}

	if len(input.ExtendedStatistics) > 0 {
		s := make([]*string, len(input.ExtendedStatistics))

		for i := range input.ExtendedStatistics {
			s[i] = aws.String(input.ExtendedStatistics[i])
		}

		params.ExtendedStatistics = s
	}

	if len(input.Dimensions) > 0 {
		s := make([]*cloudwatch.Dimension, len(input.Dimensions))

		for i := range input.Dimensions {
			splitted := strings.Split(input.Dimensions[i], ":")

			s[i] = &cloudwatch.Dimension{
				Name:  aws.String(splitted[0]),
				Value: aws.String(splitted[1]),
			}
		}

		params.Dimensions = s
	}

	return params
}

// GetMetricStatistics wrapper
func GetMetricStatistics(params *cloudwatch.GetMetricStatisticsInput) *cloudwatch.GetMetricStatisticsOutput {
	if os.Getenv("AWS_REGION") == "" {
		os.Setenv("AWS_REGION", defaultRegion)
	}

	session, err := session.NewSession()
	if err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	service := cloudwatch.New(session)

	response, err := service.GetMetricStatistics(params)
	if err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	return response
}

// GetLastDatapoint - get latest datapoint from datapoints by timestamp
func GetLastDatapoint(datapoints []*cloudwatch.Datapoint) *cloudwatch.Datapoint {
	if len(datapoints) < 1 {
		fmt.Println("No datapoint.")
		os.Exit(3)
	}

	lastDatapoint := datapoints[0]

	for i := range datapoints {
		datapoint := datapoints[i]

		if datapoint.Timestamp.Unix() > lastDatapoint.Timestamp.Unix() {
			lastDatapoint = datapoint
		}
	}

	return lastDatapoint
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package perfschema

import (
	"fmt"
	"reflect"
	"runtime"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/util/types"
)

// statementInfo defines statement instrument information.
type statementInfo struct {
	// The registered statement key
	key uint64
	// The name of the statement instrument to register
	name string
}

// StatementState provides temporary storage to a statement runtime statistics.
// TODO:
// 1. support statement digest.
// 2. support prepared statement.
type StatementState struct {
	// Connection identifier
	connID uint64
	// Statement information
	info *statementInfo
	// Statement type
	stmtType reflect.Type
	// Source file and line number
	source string
	// Timer name
	timerName enumTimerName
	// Timer start
	timerStart int64
	// Timer end
	timerEnd int64
	// Locked time
	lockTime int64
	// SQL statement string
	sqlText string
	// Current schema name
	schemaName string
	// Number of errors
	errNum uint32
	// Number of warnings
	warnNum uint32
	// Rows affected
	rowsAffected uint64
	// Rows sent
	rowsSent uint64
	// Rows examined
	rowsExamined uint64
	// Metric, temporary tables created on disk
	createdTmpDiskTables uint32
	// Metric, temproray tables created
	createdTmpTables uint32
	// Metric, number of select full join
	selectFullJoin uint32
	// Metric, number of select full range join
	selectFullRangeJoin uint32
	// Metric, number of select range
	selectRange uint32
	// Metric, number of select range check
	selectRangeCheck uint32
	// Metric, number of select scan
	selectScan uint32
	// Metric, number of sort merge passes
	sortMergePasses uint32
	// Metric, number of sort merge
	sortRange uint32
	// Metric, number of sort rows
	sortRows uint32
	// Metric, number of sort scans
	sortScan uint32
	// Metric, no index used flag
	noIndexUsed uint8
	// Metric, no good index used flag
	noGoodIndexUsed uint8
}

var (
	stmtInfos = make(map[reflect.Type]*statementInfo)
)

func (ps *perfSchema) RegisterStatement(category, name string, elem interface{}) {
	instrumentName := fmt.Sprintf("%s%s/%s", statementInstrumentPrefix, category, name)
	key, err := ps.addInstrument(instrumentName)
	if err != nil {
		// just ignore, do nothing else.
		log.Errorf("Unable to register instrument %s", instrumentName)
		return
	}

	stmtInfos[reflect.TypeOf(elem)] = &statementInfo{
		key:  key,
		name: instrumentName,
	}
}

func (ps *perfSchema) StartStatement(sql string, connID uint64, callerName EnumCallerName, elem interface{}) *StatementState {
	stmtType := reflect.TypeOf(elem)
	info, ok := stmtInfos[stmtType]
	if !ok {
		// just ignore, do nothing else.
		log.Errorf("No instrument registered for statement %s", stmtType)
		return nil
	}

	// check and apply the configuration parameter in table setup_timers.
	timerName, err := ps.getTimerName(flagStatement)
	if err != nil {
		// just ignore, do nothing else.
		log.Error("Unable to check setup_timers table")
		return nil
	}
	var timerStart int64
	switch timerName {
	case timerNameNanosec:
		timerStart = time.Now().UnixNano()
	case timerNameMicrosec:
		timerStart = time.Now().UnixNano() / int64(time.Microsecond)
	case timerNameMillisec:
		timerStart = time.Now().UnixNano() / int64(time.Millisecond)
	default:
		return nil
	}

	// TODO: check and apply the additional configuration parameters in:
	// - table setup_actors
	// - table setup_setup_consumers
	// - table setup_instruments
	// - table setup_objects

	var source string
	source, ok = callerNames[callerName]
	if !ok {
		_, fileName, fileLine, ok := runtime.Caller(1)
		if !ok {
			// just ignore, do nothing else.
			log.Error("Unable to get runtime.Caller(1)")
			return nil
		}
		source = fmt.Sprintf("%s:%d", fileName, fileLine)
		callerNames[callerName] = source
	}

	return &StatementState{
		connID:     connID,
		info:       info,
		stmtType:   stmtType,
		source:     source,
		timerName:  timerName,
		timerStart: timerStart,
		sqlText:    sql,
	}
}

func (ps *perfSchema) EndStatement(state *StatementState) {
	if state == nil {
		return
	}

	switch state.timerName {
	case timerNameNanosec:
		state.timerEnd = time.Now().UnixNano()
	case timerNameMicrosec:
		state.timerEnd = time.Now().UnixNano() / int64(time.Microsecond)
	case timerNameMillisec:
		state.timerEnd = time.Now().UnixNano() / int64(time.Millisecond)
	default:
		return
	}

	log.Debugf("EndStatement: sql %s, connection id %d, type %s", state.sqlText, state.connID, state.stmtType)

	record := state2Record(state)
	err := ps.updateEventsStmtsCurrent(state.connID, record)
	if err != nil {
		log.Error("Unable to update events_statements_current table")
	}
	err = ps.appendEventsStmtsHistory(record)
	if err != nil {
		log.Errorf("Unable to append to events_statements_history table %v", errors.ErrorStack(err))
	}
}

func state2Record(state *StatementState) []types.Datum {
	return types.MakeDatums(
		state.connID,             // THREAD_ID
		state.info.key,           // EVENT_ID
		nil,                      // END_EVENT_ID
		state.info.name,          // EVENT_NAME
		state.source,             // SOURCE
		uint64(state.timerStart), // TIMER_START
		uint64(state.timerEnd),   // TIMER_END
		nil, // TIMER_WAIT
		uint64(state.lockTime),             // LOCK_TIME
		state.sqlText,                      // SQL_TEXT
		nil,                                // DIGEST
		nil,                                // DIGEST_TEXT
		state.schemaName,                   // CURRENT_SCHEMA
		nil,                                // OBJECT_TYPE
		nil,                                // OBJECT_SCHEMA
		nil,                                // OBJECT_NAME
		nil,                                // OBJECT_INSTANCE_BEGIN
		nil,                                // MYSQL_ERRNO,
		nil,                                // RETURNED_SQLSTATE
		nil,                                // MESSAGE_TEXT
		uint64(state.errNum),               // ERRORS
		uint64(state.warnNum),              // WARNINGS
		state.rowsAffected,                 // ROWS_AFFECTED
		state.rowsSent,                     // ROWS_SENT
		state.rowsExamined,                 // ROWS_EXAMINED
		uint64(state.createdTmpDiskTables), // CREATED_TMP_DISK_TABLES
		uint64(state.createdTmpTables),     // CREATED_TMP_TABLES
		uint64(state.selectFullJoin),       // SELECT_FULL_JOIN
		uint64(state.selectFullRangeJoin),  // SELECT_FULL_RANGE_JOIN
		uint64(state.selectRange),          // SELECT_RANGE
		uint64(state.selectRangeCheck),     // SELECT_RANGE_CHECK
		uint64(state.selectScan),           // SELECT_SCAN
		uint64(state.sortMergePasses),      // SORT_MERGE_PASSES
		uint64(state.sortRange),            // SORT_RANGE
		uint64(state.sortRows),             // SORT_ROWS
		uint64(state.sortScan),             // SORT_SCAN
		uint64(state.noIndexUsed),          // NO_INDEX_USED
		uint64(state.noGoodIndexUsed),      // NO_GOOD_INDEX_USED
		nil, // NESTING_EVENT_ID
		nil, // NESTING_EVENT_TYPE
		nil, // NESTING_EVENT_LEVEL
	)
}

func (ps *perfSchema) updateEventsStmtsCurrent(connID uint64, record []types.Datum) error {
	tbl := ps.mTables[TableStmtsCurrent]
	handle, ok := ps.stmtHandles[connID]
	if !ok {
		handle, err := tbl.AddRecord(nil, record)
		if err != nil {
			return errors.Trace(err)
		}
		ps.stmtHandles[connID] = handle
		return nil
	}
	err := tbl.UpdateRecord(nil, handle, nil, record, nil)
	if err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (ps *perfSchema) appendEventsStmtsHistory(record []types.Datum) error {
	tbl := ps.mTables[TableStmtsHistory]
	_, err := tbl.AddRecord(nil, record)
	if err != nil {
		return errors.Trace(err)
	}
	return nil
}

func registerStatements() {
	// Existing instrument names are the same as MySQL 5.7
	PerfHandle.RegisterStatement("sql", "alter_table", (*ast.AlterTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "begin", (*ast.BeginStmt)(nil))
	PerfHandle.RegisterStatement("sql", "commit", (*ast.CommitStmt)(nil))
	PerfHandle.RegisterStatement("sql", "create_db", (*ast.CreateDatabaseStmt)(nil))
	PerfHandle.RegisterStatement("sql", "create_index", (*ast.CreateIndexStmt)(nil))
	PerfHandle.RegisterStatement("sql", "create_table", (*ast.CreateTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "deallocate", (*ast.DeallocateStmt)(nil))
	PerfHandle.RegisterStatement("sql", "delete", (*ast.DeleteStmt)(nil))
	PerfHandle.RegisterStatement("sql", "do", (*ast.DoStmt)(nil))
	PerfHandle.RegisterStatement("sql", "drop_db", (*ast.DropDatabaseStmt)(nil))
	PerfHandle.RegisterStatement("sql", "drop_table", (*ast.DropTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "drop_index", (*ast.DropIndexStmt)(nil))
	PerfHandle.RegisterStatement("sql", "execute", (*ast.ExecuteStmt)(nil))
	PerfHandle.RegisterStatement("sql", "explain", (*ast.ExplainStmt)(nil))
	PerfHandle.RegisterStatement("sql", "insert", (*ast.InsertStmt)(nil))
	PerfHandle.RegisterStatement("sql", "prepare", (*ast.PrepareStmt)(nil))
	PerfHandle.RegisterStatement("sql", "rollback", (*ast.RollbackStmt)(nil))
	PerfHandle.RegisterStatement("sql", "select", (*ast.SelectStmt)(nil))
	PerfHandle.RegisterStatement("sql", "set", (*ast.SetStmt)(nil))
	PerfHandle.RegisterStatement("sql", "show", (*ast.ShowStmt)(nil))
	PerfHandle.RegisterStatement("sql", "truncate", (*ast.TruncateTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "union", (*ast.UnionStmt)(nil))
	PerfHandle.RegisterStatement("sql", "update", (*ast.UpdateStmt)(nil))
	PerfHandle.RegisterStatement("sql", "use", (*ast.UseStmt)(nil))
}
<commit_msg>perfschema: make go vet happy. (#1110)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package perfschema

import (
	"fmt"
	"reflect"
	"runtime"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/tidb/ast"
	"github.com/pingcap/tidb/util/types"
)

// statementInfo defines statement instrument information.
type statementInfo struct {
	// The registered statement key
	key uint64
	// The name of the statement instrument to register
	name string
}

// StatementState provides temporary storage to a statement runtime statistics.
// TODO:
// 1. support statement digest.
// 2. support prepared statement.
type StatementState struct {
	// Connection identifier
	connID uint64
	// Statement information
	info *statementInfo
	// Statement type
	stmtType reflect.Type
	// Source file and line number
	source string
	// Timer name
	timerName enumTimerName
	// Timer start
	timerStart int64
	// Timer end
	timerEnd int64
	// Locked time
	lockTime int64
	// SQL statement string
	sqlText string
	// Current schema name
	schemaName string
	// Number of errors
	errNum uint32
	// Number of warnings
	warnNum uint32
	// Rows affected
	rowsAffected uint64
	// Rows sent
	rowsSent uint64
	// Rows examined
	rowsExamined uint64
	// Metric, temporary tables created on disk
	createdTmpDiskTables uint32
	// Metric, temproray tables created
	createdTmpTables uint32
	// Metric, number of select full join
	selectFullJoin uint32
	// Metric, number of select full range join
	selectFullRangeJoin uint32
	// Metric, number of select range
	selectRange uint32
	// Metric, number of select range check
	selectRangeCheck uint32
	// Metric, number of select scan
	selectScan uint32
	// Metric, number of sort merge passes
	sortMergePasses uint32
	// Metric, number of sort merge
	sortRange uint32
	// Metric, number of sort rows
	sortRows uint32
	// Metric, number of sort scans
	sortScan uint32
	// Metric, no index used flag
	noIndexUsed uint8
	// Metric, no good index used flag
	noGoodIndexUsed uint8
}

var (
	stmtInfos = make(map[reflect.Type]*statementInfo)
)

func (ps *perfSchema) RegisterStatement(category, name string, elem interface{}) {
	instrumentName := fmt.Sprintf("%s%s/%s", statementInstrumentPrefix, category, name)
	key, err := ps.addInstrument(instrumentName)
	if err != nil {
		// just ignore, do nothing else.
		log.Errorf("Unable to register instrument %s", instrumentName)
		return
	}

	stmtInfos[reflect.TypeOf(elem)] = &statementInfo{
		key:  key,
		name: instrumentName,
	}
}

func (ps *perfSchema) StartStatement(sql string, connID uint64, callerName EnumCallerName, elem interface{}) *StatementState {
	stmtType := reflect.TypeOf(elem)
	info, ok := stmtInfos[stmtType]
	if !ok {
		// just ignore, do nothing else.
		log.Errorf("No instrument registered for statement %s", stmtType)
		return nil
	}

	// check and apply the configuration parameter in table setup_timers.
	timerName, err := ps.getTimerName(flagStatement)
	if err != nil {
		// just ignore, do nothing else.
		log.Error("Unable to check setup_timers table")
		return nil
	}
	var timerStart int64
	switch timerName {
	case timerNameNanosec:
		timerStart = time.Now().UnixNano()
	case timerNameMicrosec:
		timerStart = time.Now().UnixNano() / int64(time.Microsecond)
	case timerNameMillisec:
		timerStart = time.Now().UnixNano() / int64(time.Millisecond)
	default:
		return nil
	}

	// TODO: check and apply the additional configuration parameters in:
	// - table setup_actors
	// - table setup_setup_consumers
	// - table setup_instruments
	// - table setup_objects

	var source string
	source, ok = callerNames[callerName]
	if !ok {
		_, fileName, fileLine, ok := runtime.Caller(1)
		if !ok {
			// just ignore, do nothing else.
			log.Error("Unable to get runtime.Caller(1)")
			return nil
		}
		source = fmt.Sprintf("%s:%d", fileName, fileLine)
		callerNames[callerName] = source
	}

	return &StatementState{
		connID:     connID,
		info:       info,
		stmtType:   stmtType,
		source:     source,
		timerName:  timerName,
		timerStart: timerStart,
		sqlText:    sql,
	}
}

func (ps *perfSchema) EndStatement(state *StatementState) {
	if state == nil {
		return
	}

	switch state.timerName {
	case timerNameNanosec:
		state.timerEnd = time.Now().UnixNano()
	case timerNameMicrosec:
		state.timerEnd = time.Now().UnixNano() / int64(time.Microsecond)
	case timerNameMillisec:
		state.timerEnd = time.Now().UnixNano() / int64(time.Millisecond)
	default:
		return
	}

	log.Debugf("EndStatement: sql %s, connection id %d, type %s", state.sqlText, state.connID, state.stmtType)

	record := state2Record(state)
	err := ps.updateEventsStmtsCurrent(state.connID, record)
	if err != nil {
		log.Error("Unable to update events_statements_current table")
	}
	err = ps.appendEventsStmtsHistory(record)
	if err != nil {
		log.Errorf("Unable to append to events_statements_history table %v", errors.ErrorStack(err))
	}
}

func state2Record(state *StatementState) []types.Datum {
	return types.MakeDatums(
		state.connID,             // THREAD_ID
		state.info.key,           // EVENT_ID
		nil,                      // END_EVENT_ID
		state.info.name,          // EVENT_NAME
		state.source,             // SOURCE
		uint64(state.timerStart), // TIMER_START
		uint64(state.timerEnd),   // TIMER_END
		nil, // TIMER_WAIT
		uint64(state.lockTime),             // LOCK_TIME
		state.sqlText,                      // SQL_TEXT
		nil,                                // DIGEST
		nil,                                // DIGEST_TEXT
		state.schemaName,                   // CURRENT_SCHEMA
		nil,                                // OBJECT_TYPE
		nil,                                // OBJECT_SCHEMA
		nil,                                // OBJECT_NAME
		nil,                                // OBJECT_INSTANCE_BEGIN
		nil,                                // MYSQL_ERRNO,
		nil,                                // RETURNED_SQLSTATE
		nil,                                // MESSAGE_TEXT
		uint64(state.errNum),               // ERRORS
		uint64(state.warnNum),              // WARNINGS
		state.rowsAffected,                 // ROWS_AFFECTED
		state.rowsSent,                     // ROWS_SENT
		state.rowsExamined,                 // ROWS_EXAMINED
		uint64(state.createdTmpDiskTables), // CREATED_TMP_DISK_TABLES
		uint64(state.createdTmpTables),     // CREATED_TMP_TABLES
		uint64(state.selectFullJoin),       // SELECT_FULL_JOIN
		uint64(state.selectFullRangeJoin),  // SELECT_FULL_RANGE_JOIN
		uint64(state.selectRange),          // SELECT_RANGE
		uint64(state.selectRangeCheck),     // SELECT_RANGE_CHECK
		uint64(state.selectScan),           // SELECT_SCAN
		uint64(state.sortMergePasses),      // SORT_MERGE_PASSES
		uint64(state.sortRange),            // SORT_RANGE
		uint64(state.sortRows),             // SORT_ROWS
		uint64(state.sortScan),             // SORT_SCAN
		uint64(state.noIndexUsed),          // NO_INDEX_USED
		uint64(state.noGoodIndexUsed),      // NO_GOOD_INDEX_USED
		nil, // NESTING_EVENT_ID
		nil, // NESTING_EVENT_TYPE
		nil, // NESTING_EVENT_LEVEL
	)
}

func (ps *perfSchema) updateEventsStmtsCurrent(connID uint64, record []types.Datum) error {
	tbl := ps.mTables[TableStmtsCurrent]
	handle, ok := ps.stmtHandles[connID]
	if !ok {
		newHandle, err := tbl.AddRecord(nil, record)
		if err != nil {
			return errors.Trace(err)
		}
		ps.stmtHandles[connID] = newHandle
		return nil
	}
	err := tbl.UpdateRecord(nil, handle, nil, record, nil)
	if err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (ps *perfSchema) appendEventsStmtsHistory(record []types.Datum) error {
	tbl := ps.mTables[TableStmtsHistory]
	_, err := tbl.AddRecord(nil, record)
	if err != nil {
		return errors.Trace(err)
	}
	return nil
}

func registerStatements() {
	// Existing instrument names are the same as MySQL 5.7
	PerfHandle.RegisterStatement("sql", "alter_table", (*ast.AlterTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "begin", (*ast.BeginStmt)(nil))
	PerfHandle.RegisterStatement("sql", "commit", (*ast.CommitStmt)(nil))
	PerfHandle.RegisterStatement("sql", "create_db", (*ast.CreateDatabaseStmt)(nil))
	PerfHandle.RegisterStatement("sql", "create_index", (*ast.CreateIndexStmt)(nil))
	PerfHandle.RegisterStatement("sql", "create_table", (*ast.CreateTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "deallocate", (*ast.DeallocateStmt)(nil))
	PerfHandle.RegisterStatement("sql", "delete", (*ast.DeleteStmt)(nil))
	PerfHandle.RegisterStatement("sql", "do", (*ast.DoStmt)(nil))
	PerfHandle.RegisterStatement("sql", "drop_db", (*ast.DropDatabaseStmt)(nil))
	PerfHandle.RegisterStatement("sql", "drop_table", (*ast.DropTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "drop_index", (*ast.DropIndexStmt)(nil))
	PerfHandle.RegisterStatement("sql", "execute", (*ast.ExecuteStmt)(nil))
	PerfHandle.RegisterStatement("sql", "explain", (*ast.ExplainStmt)(nil))
	PerfHandle.RegisterStatement("sql", "insert", (*ast.InsertStmt)(nil))
	PerfHandle.RegisterStatement("sql", "prepare", (*ast.PrepareStmt)(nil))
	PerfHandle.RegisterStatement("sql", "rollback", (*ast.RollbackStmt)(nil))
	PerfHandle.RegisterStatement("sql", "select", (*ast.SelectStmt)(nil))
	PerfHandle.RegisterStatement("sql", "set", (*ast.SetStmt)(nil))
	PerfHandle.RegisterStatement("sql", "show", (*ast.ShowStmt)(nil))
	PerfHandle.RegisterStatement("sql", "truncate", (*ast.TruncateTableStmt)(nil))
	PerfHandle.RegisterStatement("sql", "union", (*ast.UnionStmt)(nil))
	PerfHandle.RegisterStatement("sql", "update", (*ast.UpdateStmt)(nil))
	PerfHandle.RegisterStatement("sql", "use", (*ast.UseStmt)(nil))
}
<|endoftext|>


<commit_before>package fmutils

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
)

func NullFileMd5() [md5.Size]byte {
	data := []byte("null file md5")
	return md5.Sum(data)
}

func GetFilenameList(directory string) []string {

	log.Infof("this is mushroom, %d", 1)

	dirName, _ := ioutil.ReadDir(directory)
	log.Infof("CurrentDir is %s", directory)

	filenamelist := make([]string, 0)
	for i := 0; i < len(dirName); i++ {
		if dirName[i].IsDir() {

		} else {
			filename := dirName[i].Name()
			filenamelist = append(filenamelist, filename)
		}
	}
	fmt.Println("Filename_list:", filenamelist)

	return nil
}

func GetFileMd5(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		log.Errorf("GetFileMd5 Error: err=%s", err)
		return ""
	}

	fstat, err1 := f.Stat()
	if err1 != nil {
		log.Errorf("GetFileMd5 Error: err=%s", err1)
		return ""
	}

	content := make([]byte, int(fstat.Size()))
	n, err2 := f.Read(content)
	if err2 != nil {
		log.Errorf("GetFileMd5 Error: err=%s", err2)
		return ""
	}

	checkSum := md5.Sum(content)
	log.Infof("GetFileMd5 read %d bytes data, md5=%x", n, checkSum)
	digest := fmt.Sprintf("%x", checkSum)

	return digest
}

func DeleteFile(filename string) {
	err := os.Remove(filename)
	if err != nil {
		log.Errorf("Remove file failed ! %s", err)
	} else {
		log.Infof("Remove %s success", filename)
	}

}

// soft link: dst -> src
func MakeSoftLink(src, dst string) {
	err := os.Symlink(src,dst)
	if err != nil{
		log.Errorf("Creat link failed ! %s",err)
	}else{
		log.Infof("Creat link success")
	}
}
<commit_msg>2.19 bak<commit_after>package fmutils

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"

)

func NullFileMd5() [md5.Size]byte {
	data := []byte("null file md5")
	return md5.Sum(data)
}

func GetFilenameList(directory string) []string {

	log.Infof("this is mushroom, %d", 1)

	dirName, _ := ioutil.ReadDir(directory)
	log.Infof("CurrentDir is %s", directory)

	filenamelist := make([]string, 0)
	for i := 0; i < len(dirName); i++ {
		if dirName[i].IsDir() {

		} else {
			filename := dirName[i].Name()
			filenamelist = append(filenamelist, filename)
		}
	}
	fmt.Println("Filename_list:", filenamelist)

	return nil
}

func GetFileMd5(filename string) string {
	f, err := os.Open(filename)
	if err != nil {
		log.Errorf("GetFileMd5 Error: err=%s", err)
		return ""
	}

	fstat, err1 := f.Stat()
	if err1 != nil {
		log.Errorf("GetFileMd5 Error: err=%s", err1)
		return ""
	}

	content := make([]byte, int(fstat.Size()))
	n, err2 := f.Read(content)
	if err2 != nil {
		log.Errorf("GetFileMd5 Error: err=%s", err2)
		return ""
	}

	checkSum := md5.Sum(content)
	log.Infof("GetFileMd5 read %d bytes data, md5=%x", n, checkSum)
	digest := fmt.Sprintf("%x", checkSum)

	return digest
}

func DeleteFile(filename string) {
	err := os.Remove(filename)
	if err != nil {
		log.Errorf("Remove file failed ! %s", err)
	} else {
		log.Infof("Remove %s success", filename)
	}

}

// soft link: dst -> src
func MakeSoftLink(src, dst string) {
	err := os.Symlink(src,dst)
	if err != nil{
		log.Errorf("Creat link failed ! %s",err)
	}else{
		log.Infof("Creat link success")
	}
}
<|endoftext|>


<commit_before>// Package secureheader adds some HTTP header fields widely
// considered to improve safety of HTTP requests. These fields
// are documented as follows:
//
//   Strict Transport Security: https://tools.ietf.org/html/rfc6797
//   Frame Options:             https://tools.ietf.org/html/draft-ietf-websec-x-frame-options-00
//   Cross Site Scripting:      http://msdn.microsoft.com/en-us/library/dd565647%28v=vs.85%29.aspx
//   Content Type Options:      http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx
//
// The easiest way to use this package is to replace nil in your
// http.ListenAndServe call with secureheader.DefaultConfig.
// DefaultConfig is configured with conservative (safer and more
// restrictive) behavior. If you want to customize its behavior,
// assign different values to its fields before calling
// ListenAndServe. See the example code below.
//
// This package was inspired by Twitter's secureheaders Ruby
// library. See https://github.com/twitter/secureheaders.
package secureheader

// TODO(kr): figure out how to add this one:
//   Content Security Policy:   https://dvcs.w3.org/hg/content-security-policy/raw-file/tip/csp-specification.dev.html
// See https://github.com/kr/secureheader/issues/1.

import (
	"net/http"
	"strconv"
	"time"
)

// DefaultConfig wraps the default http handler with some
// conservative (safer and more restrictive) behavior.
var DefaultConfig = &Config{
	HTTPSRedirect: true,

	ContentTypeOptions: true,

	HSTS:                  true,
	HSTSMaxAge:            100 * 24 * time.Hour,
	HSTSIncludeSubdomains: true,

	FrameOptions:      true,
	FrameOpionsPolicy: Deny,

	XSSProtection:      true,
	XSSProtectionBlock: false,

	Next: http.DefaultServeMux,
}

type Config struct {
	// If true, redirects any request with scheme http to the
	// equivalent https URL.
	HTTPSRedirect bool

	// If true, sets X-Content-Type-Options to "nosniff".
	ContentTypeOptions bool

	// If true, sets the HTTP Strict Transport Security header
	// field, which instructs browsers to send future requests
	// over HTTPS, even if the URL uses the unencrypted http
	// scheme.
	HSTS                  bool
	HSTSMaxAge            time.Duration
	HSTSIncludeSubdomains bool

	// If true, adds X-Frame-Options, to control when the request
	// should be displayed inside an HTML frame.
	FrameOptions      bool
	FrameOpionsPolicy FramePolicy

	// If true, sets X-XSS-Protection to "1", optionally with
	// "mode=block". See the official documentation, linked above,
	// for the meaning of these values.
	XSSProtection      bool
	XSSProtectionBlock bool

	Next http.Handler
}

// ServeHTTP sets header fields on w according to the options in
// c, then calls c.Next(w, r). If c.HTTPSRedirect is true and r is
// an unencrypted request, ServeHTTP responds with status 301 and
// does not call c.Next.
func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if c.HTTPSRedirect && r.URL.Scheme != "https" {
		url := *r.URL
		url.Scheme = "https"
		http.Redirect(w, r, url.String(), http.StatusMovedPermanently)
		return
	}
	if c.ContentTypeOptions {
		w.Header().Set("X-Content-Type-Options", "nosniff")
	}
	if c.HSTS {
		v := "max-age=" + strconv.FormatInt(int64(c.HSTSMaxAge/time.Second), 10)
		if c.HSTSIncludeSubdomains {
			v += "; includeSubDomains"
		}
		w.Header().Set("Strict-Transport-Security", v)
	}
	if c.FrameOptions {
		w.Header().Set("X-Frame-Options", string(c.FrameOpionsPolicy))
	}
	if c.XSSProtection {
		v := "1"
		if c.XSSProtectionBlock {
			v += "; mode=block"
		}
		w.Header().Set("X-XSS-Protection", v)
	}
	c.Next.ServeHTTP(w, r)
}

// FramePolicy tells the browser under what circumstances to allow
// the response to be displayed inside an HTML frame. There are
// three options:
//
//   Deny            do not permit display in a frame
//   SameOrigin      permit display in a frame from the same origin
//   AllowFrom(url)  permit display in a frame from the given url
type FramePolicy string

const (
	Deny       FramePolicy = "DENY"
	SameOrigin FramePolicy = "SAMEORIGIN"
)

// AllowFrom returns a FramePolicy specifying that each requested
// resource can be included in a frame from only the given url.
func AllowFrom(url string) FramePolicy {
	return FramePolicy("ALLOW-FROM: " + url)
}
<commit_msg>clearer instructions<commit_after>// Package secureheader adds some HTTP header fields widely
// considered to improve safety of HTTP requests. These fields
// are documented as follows:
//
//   Strict Transport Security: https://tools.ietf.org/html/rfc6797
//   Frame Options:             https://tools.ietf.org/html/draft-ietf-websec-x-frame-options-00
//   Cross Site Scripting:      http://msdn.microsoft.com/en-us/library/dd565647%28v=vs.85%29.aspx
//   Content Type Options:      http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx
//
// The easiest way to use this package:
//
//   http.ListenAndServe(addr, secureheader.DefaultConfig)
//
// DefaultConfig is initialized with conservative (safer and more
// restrictive) behavior. If you want to change that, set its
// fields to different values before calling ListenAndServe. See
// the example code below.
//
// This package was inspired by Twitter's secureheaders Ruby
// library. See https://github.com/twitter/secureheaders.
package secureheader

// TODO(kr): figure out how to add this one:
//   Content Security Policy:   https://dvcs.w3.org/hg/content-security-policy/raw-file/tip/csp-specification.dev.html
// See https://github.com/kr/secureheader/issues/1.

import (
	"net/http"
	"strconv"
	"time"
)

// DefaultConfig wraps the default http handler with some
// conservative (safer and more restrictive) behavior.
var DefaultConfig = &Config{
	HTTPSRedirect: true,

	ContentTypeOptions: true,

	HSTS:                  true,
	HSTSMaxAge:            100 * 24 * time.Hour,
	HSTSIncludeSubdomains: true,

	FrameOptions:      true,
	FrameOpionsPolicy: Deny,

	XSSProtection:      true,
	XSSProtectionBlock: false,

	Next: http.DefaultServeMux,
}

type Config struct {
	// If true, redirects any request with scheme http to the
	// equivalent https URL.
	HTTPSRedirect bool

	// If true, sets X-Content-Type-Options to "nosniff".
	ContentTypeOptions bool

	// If true, sets the HTTP Strict Transport Security header
	// field, which instructs browsers to send future requests
	// over HTTPS, even if the URL uses the unencrypted http
	// scheme.
	HSTS                  bool
	HSTSMaxAge            time.Duration
	HSTSIncludeSubdomains bool

	// If true, adds X-Frame-Options, to control when the request
	// should be displayed inside an HTML frame.
	FrameOptions      bool
	FrameOpionsPolicy FramePolicy

	// If true, sets X-XSS-Protection to "1", optionally with
	// "mode=block". See the official documentation, linked above,
	// for the meaning of these values.
	XSSProtection      bool
	XSSProtectionBlock bool

	Next http.Handler
}

// ServeHTTP sets header fields on w according to the options in
// c, then calls c.Next(w, r). If c.HTTPSRedirect is true and r is
// an unencrypted request, ServeHTTP responds with status 301 and
// does not call c.Next.
func (c *Config) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if c.HTTPSRedirect && r.URL.Scheme != "https" {
		url := *r.URL
		url.Scheme = "https"
		http.Redirect(w, r, url.String(), http.StatusMovedPermanently)
		return
	}
	if c.ContentTypeOptions {
		w.Header().Set("X-Content-Type-Options", "nosniff")
	}
	if c.HSTS {
		v := "max-age=" + strconv.FormatInt(int64(c.HSTSMaxAge/time.Second), 10)
		if c.HSTSIncludeSubdomains {
			v += "; includeSubDomains"
		}
		w.Header().Set("Strict-Transport-Security", v)
	}
	if c.FrameOptions {
		w.Header().Set("X-Frame-Options", string(c.FrameOpionsPolicy))
	}
	if c.XSSProtection {
		v := "1"
		if c.XSSProtectionBlock {
			v += "; mode=block"
		}
		w.Header().Set("X-XSS-Protection", v)
	}
	c.Next.ServeHTTP(w, r)
}

// FramePolicy tells the browser under what circumstances to allow
// the response to be displayed inside an HTML frame. There are
// three options:
//
//   Deny            do not permit display in a frame
//   SameOrigin      permit display in a frame from the same origin
//   AllowFrom(url)  permit display in a frame from the given url
type FramePolicy string

const (
	Deny       FramePolicy = "DENY"
	SameOrigin FramePolicy = "SAMEORIGIN"
)

// AllowFrom returns a FramePolicy specifying that each requested
// resource can be included in a frame from only the given url.
func AllowFrom(url string) FramePolicy {
	return FramePolicy("ALLOW-FROM: " + url)
}
<|endoftext|>


<commit_before>package sendgrid

import (
	"context"
	"errors"
	"net/http"
	"strconv"
	"time"

	"github.com/sendgrid/rest"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
)

// Version is this client library's current version
const (
	Version        = "3.10.5"
	rateLimitRetry = 5
	rateLimitSleep = 1100
)

type options struct {
	Auth     string
	Endpoint string
	Host     string
	Subuser  string
}

// Client is the Twilio SendGrid Go client
type Client struct {
	rest.Request
}

func (o *options) baseURL() string {
	return o.Host + o.Endpoint
}

// requestNew create Request
// @return [Request] a default request object
func requestNew(options options) rest.Request {
	requestHeaders := map[string]string{
		"Authorization": options.Auth,
		"User-Agent":    "sendgrid/" + Version + ";go",
		"Accept":        "application/json",
	}

	if len(options.Subuser) != 0 {
		requestHeaders["On-Behalf-Of"] = options.Subuser
	}

	return rest.Request{
		BaseURL: options.baseURL(),
		Headers: requestHeaders,
	}
}

// Send sends an email through Twilio SendGrid
func (cl *Client) Send(email *mail.SGMailV3) (*rest.Response, error) {
	return cl.SendWithContext(context.Background(), email)
}

// SendWithContext sends an email through Twilio SendGrid with context.Context.
func (cl *Client) SendWithContext(ctx context.Context, email *mail.SGMailV3) (*rest.Response, error) {
	cl.Body = mail.GetRequestBody(email)
	return MakeRequestWithContext(ctx, cl.Request)
}

// DefaultClient is used if no custom HTTP client is defined
var DefaultClient = rest.DefaultClient

// API sets up the request to the Twilio SendGrid API, this is main interface.
// Please use the MakeRequest or MakeRequestAsync functions instead.
// (deprecated)
func API(request rest.Request) (*rest.Response, error) {
	return MakeRequest(request)
}

// MakeRequest attempts a Twilio SendGrid request synchronously.
func MakeRequest(request rest.Request) (*rest.Response, error) {
	return MakeRequestWithContext(context.Background(), request)
}

// MakeRequestWithContext attempts a Twilio SendGrid request synchronously with context.Context.
func MakeRequestWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	return DefaultClient.SendWithContext(ctx, request)
}

// MakeRequestRetry a synchronous request, but retry in the event of a rate
// limited response.
func MakeRequestRetry(request rest.Request) (*rest.Response, error) {
	return MakeRequestRetryWithContext(context.Background(), request)
}

// MakeRequestRetryWithContext a synchronous request with context.Context, but retry in the event of a rate
// limited response.
func MakeRequestRetryWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	retry := 0
	var response *rest.Response
	var err error

	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		response, err = MakeRequestWithContext(ctx, request)
		if err != nil {
			return nil, err
		}

		if response.StatusCode != http.StatusTooManyRequests {
			return response, nil
		}

		if retry > rateLimitRetry {
			return nil, errors.New("rate limit retry exceeded")
		}
		retry++

		resetTime := time.Now().Add(rateLimitSleep * time.Millisecond)

		reset, ok := response.Headers["X-RateLimit-Reset"]
		if ok && len(reset) > 0 {
			t, err := strconv.Atoi(reset[0])
			if err == nil {
				resetTime = time.Unix(int64(t), 0)
			}
		}
		time.Sleep(resetTime.Sub(time.Now()))
	}
}

// MakeRequestAsync attempts a request asynchronously in a new go
// routine. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsync(request rest.Request) (chan *rest.Response, chan error) {
	return MakeRequestAsyncWithContext(context.Background(), request)
}

// MakeRequestAsyncWithContext attempts a request asynchronously in a new go
// routine with context.Context. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsyncWithContext(ctx context.Context, request rest.Request) (chan *rest.Response, chan error) {
	r := make(chan *rest.Response)
	e := make(chan error)

	go func() {
		response, err := MakeRequestRetryWithContext(ctx, request)
		if err != nil {
			e <- err
		}
		if response != nil {
			r <- response
		}
	}()

	return r, e
}
<commit_msg>Release v3.11.0<commit_after>package sendgrid

import (
	"context"
	"errors"
	"net/http"
	"strconv"
	"time"

	"github.com/sendgrid/rest"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
)

// Version is this client library's current version
const (
	Version        = "3.11.0"
	rateLimitRetry = 5
	rateLimitSleep = 1100
)

type options struct {
	Auth     string
	Endpoint string
	Host     string
	Subuser  string
}

// Client is the Twilio SendGrid Go client
type Client struct {
	rest.Request
}

func (o *options) baseURL() string {
	return o.Host + o.Endpoint
}

// requestNew create Request
// @return [Request] a default request object
func requestNew(options options) rest.Request {
	requestHeaders := map[string]string{
		"Authorization": options.Auth,
		"User-Agent":    "sendgrid/" + Version + ";go",
		"Accept":        "application/json",
	}

	if len(options.Subuser) != 0 {
		requestHeaders["On-Behalf-Of"] = options.Subuser
	}

	return rest.Request{
		BaseURL: options.baseURL(),
		Headers: requestHeaders,
	}
}

// Send sends an email through Twilio SendGrid
func (cl *Client) Send(email *mail.SGMailV3) (*rest.Response, error) {
	return cl.SendWithContext(context.Background(), email)
}

// SendWithContext sends an email through Twilio SendGrid with context.Context.
func (cl *Client) SendWithContext(ctx context.Context, email *mail.SGMailV3) (*rest.Response, error) {
	cl.Body = mail.GetRequestBody(email)
	return MakeRequestWithContext(ctx, cl.Request)
}

// DefaultClient is used if no custom HTTP client is defined
var DefaultClient = rest.DefaultClient

// API sets up the request to the Twilio SendGrid API, this is main interface.
// Please use the MakeRequest or MakeRequestAsync functions instead.
// (deprecated)
func API(request rest.Request) (*rest.Response, error) {
	return MakeRequest(request)
}

// MakeRequest attempts a Twilio SendGrid request synchronously.
func MakeRequest(request rest.Request) (*rest.Response, error) {
	return MakeRequestWithContext(context.Background(), request)
}

// MakeRequestWithContext attempts a Twilio SendGrid request synchronously with context.Context.
func MakeRequestWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	return DefaultClient.SendWithContext(ctx, request)
}

// MakeRequestRetry a synchronous request, but retry in the event of a rate
// limited response.
func MakeRequestRetry(request rest.Request) (*rest.Response, error) {
	return MakeRequestRetryWithContext(context.Background(), request)
}

// MakeRequestRetryWithContext a synchronous request with context.Context, but retry in the event of a rate
// limited response.
func MakeRequestRetryWithContext(ctx context.Context, request rest.Request) (*rest.Response, error) {
	retry := 0
	var response *rest.Response
	var err error

	for {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		default:
		}
		response, err = MakeRequestWithContext(ctx, request)
		if err != nil {
			return nil, err
		}

		if response.StatusCode != http.StatusTooManyRequests {
			return response, nil
		}

		if retry > rateLimitRetry {
			return nil, errors.New("rate limit retry exceeded")
		}
		retry++

		resetTime := time.Now().Add(rateLimitSleep * time.Millisecond)

		reset, ok := response.Headers["X-RateLimit-Reset"]
		if ok && len(reset) > 0 {
			t, err := strconv.Atoi(reset[0])
			if err == nil {
				resetTime = time.Unix(int64(t), 0)
			}
		}
		time.Sleep(resetTime.Sub(time.Now()))
	}
}

// MakeRequestAsync attempts a request asynchronously in a new go
// routine. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsync(request rest.Request) (chan *rest.Response, chan error) {
	return MakeRequestAsyncWithContext(context.Background(), request)
}

// MakeRequestAsyncWithContext attempts a request asynchronously in a new go
// routine with context.Context. This function returns two channels: responses
// and errors. This function will retry in the case of a
// rate limit.
func MakeRequestAsyncWithContext(ctx context.Context, request rest.Request) (chan *rest.Response, chan error) {
	r := make(chan *rest.Response)
	e := make(chan error)

	go func() {
		response, err := MakeRequestRetryWithContext(ctx, request)
		if err != nil {
			e <- err
		}
		if response != nil {
			r <- response
		}
	}()

	return r, e
}
<|endoftext|>


<commit_before>package api_test

import (
	. "cf/api"
	"code.google.com/p/gogoprotobuf/proto"
	"fmt"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	"github.com/stretchr/testify/assert"
	"math/rand"
	"testing"
	"time"
)

func TestPriorityQueue(t *testing.T) {
	pq := NewSortedMessageQueue(10 * time.Millisecond)

	msg3 := logMessageWithTime(t, "message 3", int64(130))
	pq.PushMessage(msg3)
	msg2 := logMessageWithTime(t, "message 2", int64(120))
	pq.PushMessage(msg2)
	msg4 := logMessageWithTime(t, "message 4", int64(140))
	pq.PushMessage(msg4)
	msg1 := logMessageWithTime(t, "message 1", int64(110))
	pq.PushMessage(msg1)

	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg1))
	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg2))
	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg3))
	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg4))
}

func TestPopOnEmptyQueue(t *testing.T) {
	pq := NewSortedMessageQueue(10 * time.Millisecond)

	var msg *logmessage.Message
	msg = nil
	assert.Equal(t, pq.PopMessage(), msg)
}

func TestNextTimestamp(t *testing.T) {
	pq := NewSortedMessageQueue(5 * time.Second)

	assert.Equal(t, pq.NextTimestamp(), MAX_INT64)

	msg2 := logMessageWithTime(t, "message 2", int64(130))
	pq.PushMessage(msg2)
	timeNowWhenInsertingMessage1 := time.Now()

	time.Sleep(50 * time.Millisecond)

	msg1 := logMessageWithTime(t, "message 1", int64(100))
	pq.PushMessage(msg1)

	allowedDelta := (20 * time.Microsecond).Nanoseconds()

	timeWhenOutputtable := time.Now().Add(5 * time.Second).UnixNano()
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable < allowedDelta)
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable > -allowedDelta)

	pq.PopMessage()

	timeWhenOutputtable = timeNowWhenInsertingMessage1.Add(5 * time.Second).UnixNano()
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable < allowedDelta)
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable > -allowedDelta)
}

func TestStableSort(t *testing.T) {
	pq := NewSortedMessageQueue(10 * time.Millisecond)

	msg1 := logMessageWithTime(t, "message first", int64(109))
	pq.PushMessage(msg1)

	for i := 1; i < 1000; i++ {
		msg := logMessageWithTime(t, fmt.Sprintf("message %d", i), int64(110))
		pq.PushMessage(msg)
	}
	msg2 := logMessageWithTime(t, "message last", int64(111))
	pq.PushMessage(msg2)

	assert.Equal(t, getMsgString(pq.PopMessage()), "message first")

	for i := 1; i < 1000; i++ {
		assert.Equal(t, getMsgString(pq.PopMessage()), fmt.Sprintf("message %d", i))
	}

	assert.Equal(t, getMsgString(pq.PopMessage()), "message last")
}

func BenchmarkPushMessages(b *testing.B) {
	r := rand.New(rand.NewSource(99))
	pq := NewSortedMessageQueue(10 * time.Millisecond)
	for i := 0; i < b.N; i++ {
		msg := logMessageForBenchmark(b, fmt.Sprintf("message %d", i), r.Int63())
		pq.PushMessage(msg)
	}
}

func logMessageWithTime(t *testing.T, messageString string, timestamp int64) *logmessage.Message {
	data, err := proto.Marshal(generateMessage(messageString, timestamp))
	assert.NoError(t, err)
	message, err := logmessage.ParseMessage(data)
	assert.NoError(t, err)

	return message
}

func logMessageForBenchmark(b *testing.B, messageString string, timestamp int64) *logmessage.Message {
	data, _ := proto.Marshal(generateMessage(messageString, timestamp))
	message, _ := logmessage.ParseMessage(data)
	return message
}

func generateMessage(messageString string, timestamp int64) *logmessage.LogMessage {
	messageType := logmessage.LogMessage_OUT
	sourceName := "DEA"
	return &logmessage.LogMessage{
		Message:     []byte(messageString),
		AppId:       proto.String("my-app-guid"),
		MessageType: &messageType,
		SourceName:  &sourceName,
		Timestamp:   proto.Int64(timestamp),
	}
}

func getMsgString(message *logmessage.Message) string {
	return string(message.GetLogMessage().GetMessage())
}
<commit_msg>Remove benchmark tests<commit_after>package api_test

import (
	. "cf/api"
	"code.google.com/p/gogoprotobuf/proto"
	"fmt"
	"github.com/cloudfoundry/loggregatorlib/logmessage"
	"github.com/stretchr/testify/assert"
	"testing"
	"time"
)

func TestPriorityQueue(t *testing.T) {
	pq := NewSortedMessageQueue(10 * time.Millisecond)

	msg3 := logMessageWithTime(t, "message 3", int64(130))
	pq.PushMessage(msg3)
	msg2 := logMessageWithTime(t, "message 2", int64(120))
	pq.PushMessage(msg2)
	msg4 := logMessageWithTime(t, "message 4", int64(140))
	pq.PushMessage(msg4)
	msg1 := logMessageWithTime(t, "message 1", int64(110))
	pq.PushMessage(msg1)

	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg1))
	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg2))
	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg3))
	assert.Equal(t, getMsgString(pq.PopMessage()), getMsgString(msg4))
}

func TestPopOnEmptyQueue(t *testing.T) {
	pq := NewSortedMessageQueue(10 * time.Millisecond)

	var msg *logmessage.Message
	msg = nil
	assert.Equal(t, pq.PopMessage(), msg)
}

func TestNextTimestamp(t *testing.T) {
	pq := NewSortedMessageQueue(5 * time.Second)

	assert.Equal(t, pq.NextTimestamp(), MAX_INT64)

	msg2 := logMessageWithTime(t, "message 2", int64(130))
	pq.PushMessage(msg2)
	timeNowWhenInsertingMessage1 := time.Now()

	time.Sleep(50 * time.Millisecond)

	msg1 := logMessageWithTime(t, "message 1", int64(100))
	pq.PushMessage(msg1)

	allowedDelta := (20 * time.Microsecond).Nanoseconds()

	timeWhenOutputtable := time.Now().Add(5 * time.Second).UnixNano()
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable < allowedDelta)
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable > -allowedDelta)

	pq.PopMessage()

	timeWhenOutputtable = timeNowWhenInsertingMessage1.Add(5 * time.Second).UnixNano()
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable < allowedDelta)
	assert.True(t, pq.NextTimestamp()-timeWhenOutputtable > -allowedDelta)
}

func TestStableSort(t *testing.T) {
	pq := NewSortedMessageQueue(10 * time.Millisecond)

	msg1 := logMessageWithTime(t, "message first", int64(109))
	pq.PushMessage(msg1)

	for i := 1; i < 1000; i++ {
		msg := logMessageWithTime(t, fmt.Sprintf("message %d", i), int64(110))
		pq.PushMessage(msg)
	}
	msg2 := logMessageWithTime(t, "message last", int64(111))
	pq.PushMessage(msg2)

	assert.Equal(t, getMsgString(pq.PopMessage()), "message first")

	for i := 1; i < 1000; i++ {
		assert.Equal(t, getMsgString(pq.PopMessage()), fmt.Sprintf("message %d", i))
	}

	assert.Equal(t, getMsgString(pq.PopMessage()), "message last")
}

func logMessageWithTime(t *testing.T, messageString string, timestamp int64) *logmessage.Message {
	data, err := proto.Marshal(generateMessage(messageString, timestamp))
	assert.NoError(t, err)
	message, err := logmessage.ParseMessage(data)
	assert.NoError(t, err)

	return message
}

func generateMessage(messageString string, timestamp int64) *logmessage.LogMessage {
	messageType := logmessage.LogMessage_OUT
	sourceName := "DEA"
	return &logmessage.LogMessage{
		Message:     []byte(messageString),
		AppId:       proto.String("my-app-guid"),
		MessageType: &messageType,
		SourceName:  &sourceName,
		Timestamp:   proto.Int64(timestamp),
	}
}

func getMsgString(message *logmessage.Message) string {
	return string(message.GetLogMessage().GetMessage())
}
<|endoftext|>


<commit_before>package fmutils

import (
	"testing"
	"fmt"
)

func TestGetFileMd5(t *testing.T) {
	digest := GetFileMd5("./types.go")
	fmt.Printf("%v\n", digest)
}
<commit_msg>rm fmtutils test<commit_after><|endoftext|>


<commit_before>package genutil

import (
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"strings"
)

type impConfig struct {
	UseGcFallback bool // Whether to fall back to GcImport when presented with a package that imports "C"
}

type importer struct {
	Imports   map[string]*types.Package // All packages imported by Importer
	Fallbacks []string                  // List of imports that we had to fall back to GcImport for
	Config    impConfig                 // Configuration for the importer
}

func newImporter() *importer {
	return &importer{
		Imports: make(map[string]*types.Package),
	}
}

// Import implements the Importer type from go/types.
func (imp *importer) Import(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// types.Importer does not seem to be designed for recursive
	// parsing like we're doing here. Specifically, each nested import
	// will maintain its own imports map. This will lead to duplicate
	// imports and in turn packages, which will lead to funny errors
	// such as "cannot pass argument ip (variable of type net.IP) to
	// variable of type net.IP"
	//
	// To work around this, we keep a global imports map, allImports,
	// to which we add all nested imports, and which we use as the
	// cache, instead of imports.
	//
	// Since all nested imports will also use this importer, there
	// should be no way to end up with duplicate imports.

	// We first try to use GcImport directly. This has the downside of
	// using possibly out-of-date packages, but it has the upside of
	// not having to parse most of the Go standard library.

	imported := func(pkg *types.Package) {
		// We don't use imports, but per API we have to add the package.
		imports[pkg.Path()] = pkg
		imp.Imports[pkg.Path()] = pkg
	}

	buildPkg, buildErr := build.Import(path, ".", 0)
	// If we found no build dir, assume we're dealing with installed
	// but no source. If we found a build dir, only use GcImport if
	// it's in GOROOT. This way we always use up-to-date code for
	// normal packages but avoid parsing the standard library.
	if (buildErr == nil && buildPkg.Goroot) || buildErr != nil {
		pkg, err = types.GcImport(imp.Imports, path)
		if err == nil {
			imported(pkg)
			return pkg, nil
		}
	}

	// See if we already imported this package
	if pkg = imp.Imports[path]; pkg != nil && pkg.Complete() {
		return pkg, nil
	}

	// allImports failed, try to use go/build
	if buildErr != nil {
		if pkg := imp.fallback(path, err); pkg != nil {
			imported(pkg)
			return pkg, nil
		}
		return nil, fmt.Errorf("build.Import failed: %s", buildErr)
	}

	// TODO check if the .a file is up to date and use it instead
	fileSet := token.NewFileSet()

	isGoFile := func(d os.FileInfo) bool {
		allFiles := make([]string, 0, len(buildPkg.GoFiles)+len(buildPkg.CgoFiles))
		allFiles = append(allFiles, buildPkg.GoFiles...)
		allFiles = append(allFiles, buildPkg.CgoFiles...)

		for _, file := range allFiles {
			if file == d.Name() {
				return true
			}
		}
		return false
	}
	pkgs, err := parser.ParseDir(fileSet, buildPkg.Dir, isGoFile, 0)
	if err != nil {
		return nil, err
	}

	delete(pkgs, "documentation")
	var astPkg *ast.Package
	var name string
	for name, astPkg = range pkgs {
		// Use the first non-main package, or the only package we
		// found.
		//
		// NOTE(dh) I can't think of a reason why there should be
		// multiple packages in a single directory, but ParseDir
		// accommodates for that possibility.
		if len(pkgs) == 1 || name != "main" {
			break
		}
	}

	if astPkg == nil {
		return nil, fmt.Errorf("can't find import: %s", name)
	}

	var ff []*ast.File
	for _, f := range astPkg.Files {
		ff = append(ff, f)
	}

	context := types.Config{
		Import: imp.Import,
	}

	pkg, err = context.Check(name, fileSet, ff, nil)
	if err != nil {
		if pkg := imp.fallback(path, err); pkg != nil {
			imported(pkg)
			return pkg, nil
		}
		return pkg, err
	}

	imports[path] = pkg
	imp.Imports[path] = pkg
	return pkg, nil
}

func (imp *importer) fallback(path string, err error) *types.Package {
	// As a special case, if type checking failed due cgo, try
	// again by using GcImport. That way we can extract all
	// required type information, but we risk importing an
	// outdated version.
	if imp.Config.UseGcFallback && (strings.Contains(err.Error(), `cannot find package "C" in`) || strings.Contains(err.Error(), `can't find import: C`)) {
		gcPkg, gcErr := types.GcImport(imp.Imports, path)
		if gcErr == nil {
			imp.Fallbacks = append(imp.Fallbacks, path)
			return gcPkg
		}
	}
	return nil
}
<commit_msg>Update to the new go.tools/go/types API<commit_after>package genutil

import (
	"code.google.com/p/go.tools/go/gcimporter"
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/token"
	"os"
	"strings"
)

type impConfig struct {
	UseGcFallback bool // Whether to fall back to GcImport when presented with a package that imports "C"
}

type importer struct {
	Imports   map[string]*types.Package // All packages imported by Importer
	Fallbacks []string                  // List of imports that we had to fall back to GcImport for
	Config    impConfig                 // Configuration for the importer
}

func newImporter() *importer {
	return &importer{
		Imports: make(map[string]*types.Package),
	}
}

// Import implements the Importer type from go/types.
func (imp *importer) Import(imports map[string]*types.Package, path string) (pkg *types.Package, err error) {
	// types.Importer does not seem to be designed for recursive
	// parsing like we're doing here. Specifically, each nested import
	// will maintain its own imports map. This will lead to duplicate
	// imports and in turn packages, which will lead to funny errors
	// such as "cannot pass argument ip (variable of type net.IP) to
	// variable of type net.IP"
	//
	// To work around this, we keep a global imports map, allImports,
	// to which we add all nested imports, and which we use as the
	// cache, instead of imports.
	//
	// Since all nested imports will also use this importer, there
	// should be no way to end up with duplicate imports.

	// We first try to use GcImport directly. This has the downside of
	// using possibly out-of-date packages, but it has the upside of
	// not having to parse most of the Go standard library.

	imported := func(pkg *types.Package) {
		// We don't use imports, but per API we have to add the package.
		imports[pkg.Path()] = pkg
		imp.Imports[pkg.Path()] = pkg
	}

	buildPkg, buildErr := build.Import(path, ".", 0)
	// If we found no build dir, assume we're dealing with installed
	// but no source. If we found a build dir, only use GcImport if
	// it's in GOROOT. This way we always use up-to-date code for
	// normal packages but avoid parsing the standard library.
	if (buildErr == nil && buildPkg.Goroot) || buildErr != nil {
		pkg, err = gcimporter.Import(imp.Imports, path)
		if err == nil {
			imported(pkg)
			return pkg, nil
		}
	}

	// See if we already imported this package
	if pkg = imp.Imports[path]; pkg != nil && pkg.Complete() {
		return pkg, nil
	}

	// allImports failed, try to use go/build
	if buildErr != nil {
		if pkg := imp.fallback(path, err); pkg != nil {
			imported(pkg)
			return pkg, nil
		}
		return nil, fmt.Errorf("build.Import failed: %s", buildErr)
	}

	// TODO check if the .a file is up to date and use it instead
	fileSet := token.NewFileSet()

	isGoFile := func(d os.FileInfo) bool {
		allFiles := make([]string, 0, len(buildPkg.GoFiles)+len(buildPkg.CgoFiles))
		allFiles = append(allFiles, buildPkg.GoFiles...)
		allFiles = append(allFiles, buildPkg.CgoFiles...)

		for _, file := range allFiles {
			if file == d.Name() {
				return true
			}
		}
		return false
	}
	pkgs, err := parser.ParseDir(fileSet, buildPkg.Dir, isGoFile, 0)
	if err != nil {
		return nil, err
	}

	delete(pkgs, "documentation")
	var astPkg *ast.Package
	var name string
	for name, astPkg = range pkgs {
		// Use the first non-main package, or the only package we
		// found.
		//
		// NOTE(dh) I can't think of a reason why there should be
		// multiple packages in a single directory, but ParseDir
		// accommodates for that possibility.
		if len(pkgs) == 1 || name != "main" {
			break
		}
	}

	if astPkg == nil {
		return nil, fmt.Errorf("can't find import: %s", name)
	}

	var ff []*ast.File
	for _, f := range astPkg.Files {
		ff = append(ff, f)
	}

	context := types.Config{
		Import: imp.Import,
	}

	pkg, err = context.Check(name, fileSet, ff, nil)
	if err != nil {
		if pkg := imp.fallback(path, err); pkg != nil {
			imported(pkg)
			return pkg, nil
		}
		return pkg, err
	}

	imports[path] = pkg
	imp.Imports[path] = pkg
	return pkg, nil
}

func (imp *importer) fallback(path string, err error) *types.Package {
	// As a special case, if type checking failed due cgo, try
	// again by using GcImport. That way we can extract all
	// required type information, but we risk importing an
	// outdated version.
	if imp.Config.UseGcFallback && (strings.Contains(err.Error(), `cannot find package "C" in`) || strings.Contains(err.Error(), `can't find import: C`)) {
		gcPkg, gcErr := gcimporter.Import(imp.Imports, path)
		if gcErr == nil {
			imp.Fallbacks = append(imp.Fallbacks, path)
			return gcPkg
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	host	       = flag.String("host", "", "the db host")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select `long`,`access_count` from `url` where short=?", shorturl).Scan(&longurl, &access_count)

	// TODO: update accessed and access_count

	switch {
	case err == sql.ErrNoRows:
		log.Printf("%q not found", shorturl)
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		access_count++
		log.Printf("%q -> %q", shorturl, longurl)
		http.Redirect(w, r, longurl, http.StatusFound)
	}
}

func main() {
	flag.Parse()

	c, err := sql.Open("mysql", *user+":"+*pwd+"@("+*host+")/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	dbConn = c
	defer dbConn.Close()

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<commit_msg>Update TODO comments<commit_after>package main

import (
	"database/sql"
	"flag"
	"log"
	"net/http"

	_ "github.com/go-sql-driver/mysql"
	"github.com/gorilla/mux"
)

var (
	dbConn *sql.DB = nil
	db             = flag.String("db", "", "the db")
	host	       = flag.String("host", "", "the db host")
	user           = flag.String("user", "", "the db user")
	pwd            = flag.String("pwd", "", "the db password")
	port	       = flag.String("port", "http", "the port to listen on")
)

func Handler(w http.ResponseWriter, r *http.Request) {
	shorturl := mux.Vars(r)["shorturl"]

	// lookup shorturl
	var longurl string
	var access_count int
	err := dbConn.QueryRow("select `long`,`access_count` from `url` where short=?", shorturl).Scan(&longurl, &access_count)

	switch {
	case err == sql.ErrNoRows:
		log.Printf("%q not found", shorturl)
		http.NotFound(w, r)

	case err != nil:
		log.Fatal(err)

	default:
		access_count++
		log.Printf("%q -> %q", shorturl, longurl)
		http.Redirect(w, r, longurl, http.StatusFound)
	}
}

func main() {
	flag.Parse()

	c, err := sql.Open("mysql", *user+":"+*pwd+"@("+*host+")/"+*db)
	if err != nil {
		log.Fatal("Failed to open sql dbConnection")
	}
	dbConn = c
	defer dbConn.Close()

	r := mux.NewRouter()
	// TODO: dashboard with metrics
	// TODO: ui for adding short urls
	r.HandleFunc("/{shorturl}", Handler).Methods("GET")
	http.Handle("/", r)

	log.Fatal(http.ListenAndServe(":"+*port, nil))
}
<|endoftext|>


<commit_before>package lich_test

import (
	"fmt"
	"github.com/earthboundkid/lich"
	"testing"
)

func ExampleData() {
	data := lich.Data("Hello, World!")

	fmt.Println(data)
	//Output: 13<Hello, World!>
}

func ExampleArray() {
	array := lich.Array{lich.Data("apple"), lich.Data("banana"), lich.Data("orange")}

	fmt.Println(array)
	//Output: 26[5<apple>6<banana>6<orange>]
}

func ExampleArrayFromStrings() {
	array := lich.ArrayFromStrings("simple", "general", "human-sympathetic")

	fmt.Println(array)
	//Output: 40[6<simple>7<general>17<human-sympathetic>]

}

func ExampleDict() {
	d1 := lich.Dict{
		"selling points": lich.ArrayFromStrings("simple", "general", "human-sympathetic"),
		"greeting":       lich.Data([]byte{72, 101, 108, 108, 111}),
		"fruit":          lich.Array{lich.Data("apple"), lich.Data("banana"), lich.Data("orange")},
	}

	fmt.Println(d1)
	//Output: 119{5<fruit>26[5<apple>6<banana>6<orange>]8<greeting>5<Hello>14<selling points>40[6<simple>7<general>17<human-sympathetic>]}
}

func ExampleDictFromMap() {
	d2, _ := lich.DictFromMap(map[string]interface{}{
		"greeting": map[string]interface{}{
			"English":  []byte{72, 101, 108, 108, 111},
			"Japanese": "Konnichiwa",
		},
		"fruit":          []string{"apple", "banana", "orange"},
		"selling points": lich.ArrayFromStrings("simple", "general", "human-sympathetic"),
	})
	fmt.Println(d2)
	//Output: 158{5<fruit>26[5<apple>6<banana>6<orange>]8<greeting>43{7<English>5<Hello>8<Japanese>10<Konnichiwa>}14<selling points>40[6<simple>7<general>17<human-sympathetic>]}

}

func TestEmptyData(t *testing.T) {
	data := lich.Data("")
	str := data.String()
	if str != "0<>" {
		t.Fatal(data, str)
	}
}

func TestEmptyArray(t *testing.T) {
	array := lich.Array{}
	str := array.String()
	if str != "0[]" {
		t.Fatal(array, str)
	}
}

func TestEmptyDict(t *testing.T) {
	d := lich.Dict{}
	str := d.String()
	if str != "0{}" {
		t.Fatal(d, str)
	}
}

func TestInvalidMap(t *testing.T) {
	d2, err := lich.DictFromMap(map[string]interface{}{
		"greeting": map[string]interface{}{
			"English":  []byte{72, 101, 108, 108, 111},
			"Japanese": "Konnichiwa",
			"Dog":      9.0,
		},
		"fruit":          []string{"apple", "banana", "orange"},
		"selling points": lich.ArrayFromStrings("simple", "general", "human-sympathetic"),
	})
	if (err != lich.UnknownTypeError{"Dog", 9.0}) {
		t.Fatal(d2, err)
	}
}

func TestSimpleDataParsing(t *testing.T) {
	s := "5<hello>"
	element, err := lich.Parse(s)
	if err != nil || element != lich.Data("hello") {
		t.Fatalf("Parsed %q\nGot element:\t%#v, %s\nError:\t%#v", s, element, element, err)
	}

}

func TestSimpleArrayParsing(t *testing.T) {
	s := "26[5<apple>6<banana>6<orange>]"
	array := lich.Array{lich.Data("apple"), lich.Data("banana"), lich.Data("orange")}

	element, err := lich.Parse(s)
	if err != nil || element.String() != array.String() {
		t.Fatalf("Parsed %q\nGot element:\t%#v, %s\nError:\t%#v", s, element, element, err)
	}
}
<commit_msg>Remove parser tests<commit_after>package lich_test

import (
	"fmt"
	"testing"

	"github.com/earthboundkid/lich"
)

func ExampleData() {
	data := lich.Data("Hello, World!")

	fmt.Println(data)
	//Output: 13<Hello, World!>
}

func ExampleArray() {
	array := lich.Array{lich.Data("apple"), lich.Data("banana"), lich.Data("orange")}

	fmt.Println(array)
	//Output: 26[5<apple>6<banana>6<orange>]
}

func ExampleArrayFromStrings() {
	array := lich.ArrayFromStrings("simple", "general", "human-sympathetic")

	fmt.Println(array)
	//Output: 40[6<simple>7<general>17<human-sympathetic>]

}

func ExampleDict() {
	d1 := lich.Dict{
		"selling points": lich.ArrayFromStrings("simple", "general", "human-sympathetic"),
		"greeting":       lich.Data([]byte{72, 101, 108, 108, 111}),
		"fruit":          lich.Array{lich.Data("apple"), lich.Data("banana"), lich.Data("orange")},
	}

	fmt.Println(d1)
	//Output: 119{5<fruit>26[5<apple>6<banana>6<orange>]8<greeting>5<Hello>14<selling points>40[6<simple>7<general>17<human-sympathetic>]}
}

func ExampleDictFromMap() {
	d2, _ := lich.DictFromMap(map[string]interface{}{
		"greeting": map[string]interface{}{
			"English":  []byte{72, 101, 108, 108, 111},
			"Japanese": "Konnichiwa",
		},
		"fruit":          []string{"apple", "banana", "orange"},
		"selling points": lich.ArrayFromStrings("simple", "general", "human-sympathetic"),
	})
	fmt.Println(d2)
	//Output: 158{5<fruit>26[5<apple>6<banana>6<orange>]8<greeting>43{7<English>5<Hello>8<Japanese>10<Konnichiwa>}14<selling points>40[6<simple>7<general>17<human-sympathetic>]}

}

func TestEmptyData(t *testing.T) {
	data := lich.Data("")
	str := data.String()
	if str != "0<>" {
		t.Fatal(data, str)
	}
}

func TestEmptyArray(t *testing.T) {
	array := lich.Array{}
	str := array.String()
	if str != "0[]" {
		t.Fatal(array, str)
	}
}

func TestEmptyDict(t *testing.T) {
	d := lich.Dict{}
	str := d.String()
	if str != "0{}" {
		t.Fatal(d, str)
	}
}

func TestInvalidMap(t *testing.T) {
	d2, err := lich.DictFromMap(map[string]interface{}{
		"greeting": map[string]interface{}{
			"English":  []byte{72, 101, 108, 108, 111},
			"Japanese": "Konnichiwa",
			"Dog":      9.0,
		},
		"fruit":          []string{"apple", "banana", "orange"},
		"selling points": lich.ArrayFromStrings("simple", "general", "human-sympathetic"),
	})
	if (err != lich.UnknownTypeError{"Dog", 9.0}) {
		t.Fatal(d2, err)
	}
}
<|endoftext|>