<commit_before>// Copyright 2015-2016 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tcontainer

import (
	"runtime"
	"sync"
)

// BytePool holds a set of byte buffers that are recycled after use.
// Buffers up to a size of 1000 KB can be managed by this pool, larger allocations
// will fall back to standard methods (i.e. make).
// This pool is useful for applications that need to handle a lot of arbitrary
// sized buffers. As this pool is GC backed no "put back" call is required.
type BytePool struct {
	tiny   slabsList // 64 to 960 byte
	small  slabsList // 1KB to 9KB
	medium slabsList // 10KB to 90KB
	large  slabsList // 100KB to 1000KB
}

type chunk []byte
type slab chan chunk
type slabsList struct {
	slabs     []slab
	chunkSize int
	numChunks int
	guard     *sync.Mutex
}

const (
	// tinySlabUnit is the multiplicator for tiny slabs
	tinyChunkSize = 64
	// tinySlabCount is the number of tiny slabs
	tinySlabCount = (smallChunkSize / tinyChunkSize) - 1
	// tinySlabMax is the largest tiny slab available
	tinyChunkMax = tinyChunkSize * tinySlabCount

	// smallSlabUnit is the multiplicator for small slabs
	smallChunkSize = 1024
	// smallSlabCount is the number of small slabs
	smallSlabCount = (mediumChunkSize / smallChunkSize) - 1
	// smallSlabMax is the largest small slab available
	smallChunkMax = smallChunkSize * smallSlabCount

	// mediumSlabUnit is the multiplicator for medium slabs
	mediumChunkSize = 1024 * 10
	// mediumSlabCount is the number of medium slabs
	mediumSlabCount = (largeChunkSize / mediumChunkSize) - 1
	// mediumSlabMax is the largest medium slab available
	mediumChunkMax = mediumChunkSize * mediumSlabCount

	// largeSlabUnit is the multiplicator for large slabs
	largeChunkSize = 1024 * 100
	// largeSlabCount is the number of large slabs
	largeSlabCount = 10
	// largeSlabMax is the largest large slab available
	largeChunkMax = largeChunkSize * largeSlabCount
)

// NewBytePool creates a new bytepool. This will call NewBytePoolWithSize
// with 1000, 100, 100, 10.
func NewBytePool() BytePool {
	return NewBytePoolWithSize(1000, 100, 100, 10)
}

// NewBytePoolWithSize creates a new BytePool. You can define the maximum
// number of chunks ([]byte) cached per storage size. Chunks wills always be
// returned, even if the cache is empty. If there are more chunks "in flight"
// than allowed, excess chunks will be garbage collected.
// Storage sizes in bytes: tiny: 1-960, small: 1024-9216, medium: 10240-92160.
// large: 102400-1024000. Each sorage size holds up to 10 slabs.
func NewBytePoolWithSize(tinyChunkCount int, smallChunkCount int, mediumChunkCount int, largeChunkCount int) BytePool {
	return BytePool{
		tiny:   newSlabsList(tinySlabCount, tinyChunkCount, tinyChunkSize),
		small:  newSlabsList(smallSlabCount, smallChunkCount, smallChunkSize),
		medium: newSlabsList(mediumSlabCount, mediumChunkCount, mediumChunkSize),
		large:  newSlabsList(largeSlabCount, largeChunkCount, largeChunkSize),
	}
}

// Get returns a byte slice that can hold the given number of bytes.
// If possible this slice is coming from a pool. Slices are automatically
// returned to the pool. No additional action is necessary.
func (b *BytePool) Get(size int) []byte {
	if size == 0 {
		return []byte{}
	}

	slab := b.getSlab(size)
	if slab == nil {
		return make([]byte, size) // ### return, oversized ###
	}

	select {
	case buffer := <-slab:
		return buffer[:size] // ### return, cached ###

	default:
		return make([]byte, size) // ### return, empty pool ###
	}
}

func newSlabsList(numSlabs int, numChunks int, chunkSize int) slabsList {
	return slabsList{
		slabs:     make([]slab, numSlabs),
		chunkSize: chunkSize,
		numChunks: numChunks,
		guard:     new(sync.Mutex),
	}
}

func (b *BytePool) getSlab(size int) slab {
	switch {

	case size <= tinyChunkMax:
		return b.tiny.fetch(size, b)

	case size <= smallChunkMax:
		return b.small.fetch(size, b)

	case size <= mediumChunkMax:
		return b.medium.fetch(size, b)

	case size <= largeChunkMax:
		return b.large.fetch(size, b)

	default:
		return nil // ### return, too large ###
	}
}

func (s *slabsList) fetch(size int, b *BytePool) slab {
	slabIdx := size / (s.chunkSize + 1)
	chunks := s.slabs[slabIdx]

	if chunks == nil {
		// First initialization can be racey
		s.guard.Lock()
		defer s.guard.Unlock()

		if chunks = s.slabs[slabIdx]; chunks == nil {
			chunks = make(slab, s.numChunks)
			chunkSize := (slabIdx + 1) * s.chunkSize

			// Prepopulate slab
			for i := 0; i < s.numChunks; i++ {
				buffer := make([]byte, chunkSize)
				runtime.SetFinalizer(&buffer, b.put)
				chunks <- buffer
			}

			s.slabs[slabIdx] = chunks
		}
	}

	return chunks
}

func (b *BytePool) put(buffer *[]byte) {
	slab := b.getSlab(cap(*buffer))
	select {
	case slab <- *buffer:
		runtime.SetFinalizer(buffer, b.put)
	default:
		// discard, pool is full
	}
}
<commit_msg>changed default pool sizes fixed slab index calculation<commit_after>// Copyright 2015-2016 trivago GmbH
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tcontainer

import (
	"runtime"
	"sync"
)

// BytePool holds a set of byte buffers that are recycled after use.
// Buffers up to a size of 1000 KB can be managed by this pool, larger allocations
// will fall back to standard methods (i.e. make).
// This pool is useful for applications that need to handle a lot of arbitrary
// sized buffers. As this pool is GC backed no "put back" call is required.
type BytePool struct {
	tiny   slabsList // 64 to 960 byte
	small  slabsList // 1KB to 9KB
	medium slabsList // 10KB to 90KB
	large  slabsList // 100KB to 1000KB
}

type chunk []byte
type slab chan chunk
type slabsList struct {
	slabs     []slab
	chunkSize int
	numChunks int
	guard     *sync.Mutex
}

const (
	// tinySlabUnit is the multiplicator for tiny slabs
	tinyChunkSize = 64
	// tinySlabCount is the number of tiny slabs
	tinySlabCount = (smallChunkSize / tinyChunkSize) - 1
	// tinySlabMax is the largest tiny slab available
	tinyChunkMax = tinyChunkSize * tinySlabCount

	// smallSlabUnit is the multiplicator for small slabs
	smallChunkSize = 1024
	// smallSlabCount is the number of small slabs
	smallSlabCount = (mediumChunkSize / smallChunkSize) - 1
	// smallSlabMax is the largest small slab available
	smallChunkMax = smallChunkSize * smallSlabCount

	// mediumSlabUnit is the multiplicator for medium slabs
	mediumChunkSize = 1024 * 10
	// mediumSlabCount is the number of medium slabs
	mediumSlabCount = (largeChunkSize / mediumChunkSize) - 1
	// mediumSlabMax is the largest medium slab available
	mediumChunkMax = mediumChunkSize * mediumSlabCount

	// largeSlabUnit is the multiplicator for large slabs
	largeChunkSize = 1024 * 100
	// largeSlabCount is the number of large slabs
	largeSlabCount = 10
	// largeSlabMax is the largest large slab available
	largeChunkMax = largeChunkSize * largeSlabCount
)

// NewBytePool creates a new bytepool. This will call NewBytePoolWithSize
// with 10000, 1000, 100, 10.
func NewBytePool() BytePool {
	return NewBytePoolWithSize(10000, 1000, 100, 10)
}

// NewBytePoolWithSize creates a new BytePool. You can define the maximum
// number of chunks ([]byte) cached per storage size. Chunks wills always be
// returned, even if the cache is empty. Excess chunks will be garbage collected.
// Storage sizes in bytes: tiny: 1-960, small: 1024-9216, medium: 10240-92160.
// large: 102400-1024000. Each sorage size holds up to 10 slabs.
func NewBytePoolWithSize(tinyChunkCount int, smallChunkCount int, mediumChunkCount int, largeChunkCount int) BytePool {
	return BytePool{
		tiny:   newSlabsList(tinySlabCount, tinyChunkCount, tinyChunkSize),
		small:  newSlabsList(smallSlabCount, smallChunkCount, smallChunkSize),
		medium: newSlabsList(mediumSlabCount, mediumChunkCount, mediumChunkSize),
		large:  newSlabsList(largeSlabCount, largeChunkCount, largeChunkSize),
	}
}

// Get returns a byte slice that can hold the given number of bytes.
// If possible this slice is coming from a pool. Slices are automatically
// returned to the pool. No additional action is necessary.
func (b *BytePool) Get(size int) []byte {
	if size == 0 {
		return []byte{}
	}

	slab := b.getSlab(size)
	if slab == nil {
		return make([]byte, size) // ### return, oversized ###
	}

	select {
	case buffer := <-slab:
		return buffer[:size] // ### return, cached ###
	default:
		return make([]byte, size) // ### return, empty pool ###
	}
}

func newSlabsList(numSlabs int, numChunks int, chunkSize int) slabsList {
	return slabsList{
		slabs:     make([]slab, numSlabs),
		chunkSize: chunkSize,
		numChunks: numChunks,
		guard:     new(sync.Mutex),
	}
}

func (b *BytePool) getSlab(size int) slab {
	switch {

	case size <= tinyChunkMax:
		return b.tiny.fetch(size, b)

	case size <= smallChunkMax:
		return b.small.fetch(size, b)

	case size <= mediumChunkMax:
		return b.medium.fetch(size, b)

	case size <= largeChunkMax:
		return b.large.fetch(size, b)

	default:
		return nil // ### return, too large ###
	}
}

func (s *slabsList) fetch(size int, b *BytePool) slab {
	slabIdx := (size - 1) / s.chunkSize
	chunks := s.slabs[slabIdx]

	if chunks == nil {
		// First initialization can be racy
		s.guard.Lock()
		defer s.guard.Unlock()

		if chunks = s.slabs[slabIdx]; chunks == nil {
			chunks = make(slab, s.numChunks)
			chunkSize := (slabIdx + 1) * s.chunkSize

			// Prepopulate slab
			for i := 0; i < s.numChunks; i++ {
				buffer := make([]byte, chunkSize)
				runtime.SetFinalizer(&buffer, b.put)
				chunks <- buffer
			}

			s.slabs[slabIdx] = chunks
		}
	}

	return chunks
}

func (b *BytePool) put(buffer *[]byte) {
	slab := b.getSlab(cap(*buffer))
	select {
	case slab <- *buffer:
		runtime.SetFinalizer(buffer, b.put)
	default:
		// discard, pool is full
	}
}
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dataflow contains the Dataflow runner for submitting pipelines
// to Google Cloud Dataflow.
package dataflow

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"path"
	"sync/atomic"
	"time"

	"cloud.google.com/go/storage"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/hooks"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	"github.com/apache/beam/sdks/go/pkg/beam/options/gcpopts"
	"github.com/apache/beam/sdks/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/dataflow/dataflowlib"
	"github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
	"github.com/apache/beam/sdks/go/pkg/beam/x/hooks/perf"
	"github.com/golang/protobuf/proto"
)

// TODO(herohde) 5/16/2017: the Dataflow flags should match the other SDKs.

var (
	endpoint             = flag.String("dataflow_endpoint", "", "Dataflow endpoint (optional).")
	stagingLocation      = flag.String("staging_location", "", "GCS staging location (required).")
	image                = flag.String("worker_harness_container_image", "", "Worker harness container image (required).")
	labels               = flag.String("labels", "", "JSON-formatted map[string]string of job labels (optional).")
	serviceAccountEmail  = flag.String("service_account_email", "", "Service account email (optional).")
	numWorkers           = flag.Int64("num_workers", 0, "Number of workers (optional).")
	maxNumWorkers        = flag.Int64("max_num_workers", 0, "Maximum number of workers during scaling (optional).")
	autoscalingAlgorithm = flag.String("autoscaling_algorithm", "", "Autoscaling mode to use (optional).")
	zone                 = flag.String("zone", "", "GCP zone (optional)")
	region               = flag.String("region", "", "GCP Region (required)")
	network              = flag.String("network", "", "GCP network (optional)")
	subnetwork           = flag.String("subnetwork", "", "GCP subnetwork (optional)")
	noUsePublicIPs       = flag.Bool("no_use_public_ips", false, "Workers must not use public IP addresses (optional)")
	tempLocation         = flag.String("temp_location", "", "Temp location (optional)")
	machineType          = flag.String("worker_machine_type", "", "GCE machine type (optional)")
	minCPUPlatform       = flag.String("min_cpu_platform", "", "GCE minimum cpu platform (optional)")
	workerJar            = flag.String("dataflow_worker_jar", "", "Dataflow worker jar (optional)")

	executeAsync   = flag.Bool("execute_async", false, "Asynchronous execution. Submit the job and return immediately.")
	dryRun         = flag.Bool("dry_run", false, "Dry run. Just print the job, but don't submit it.")
	teardownPolicy = flag.String("teardown_policy", "", "Job teardown policy (internal only).")

	// SDK options
	cpuProfiling     = flag.String("cpu_profiling", "", "Job records CPU profiles to this GCS location (optional)")
	sessionRecording = flag.String("session_recording", "", "Job records session transcripts")
)

func init() {
	// Note that we also _ import harness/init to setup the remote execution hook.
	beam.RegisterRunner("dataflow", Execute)

	perf.RegisterProfCaptureHook("gcs_profile_writer", gcsRecorderHook)
}

var unique int32

// Execute runs the given pipeline on Google Cloud Dataflow. It uses the
// default application credentials to submit the job.
func Execute(ctx context.Context, p *beam.Pipeline) error {
	// (1) Gather job options

	project := *gcpopts.Project
	if project == "" {
		return errors.New("no Google Cloud project specified. Use --project=<project>")
	}
	if *stagingLocation == "" {
		return errors.New("no GCS staging location specified. Use --staging_location=gs://<bucket>/<path>")
	}
	if *region == "" {
		return errors.New("No Google Cloud region specified. Use --region=<region>. See https://cloud.google.com/dataflow/docs/concepts/regional-endpoints")
	}
	if *image == "" {
		*image = getContainerImage(ctx)
	}
	var jobLabels map[string]string
	if *labels != "" {
		if err := json.Unmarshal([]byte(*labels), &jobLabels); err != nil {
			return errors.Wrapf(err, "error reading --label flag as JSON")
		}
	}

	if *cpuProfiling != "" {
		perf.EnableProfCaptureHook("gcs_profile_writer", *cpuProfiling)
	}

	if *sessionRecording != "" {
		// TODO(wcn): BEAM-4017
		// It's a bit inconvenient for GCS because the whole object is written in
		// one pass, whereas the session logs are constantly appended. We wouldn't
		// want to hold all the logs in memory to flush at the end of the pipeline
		// as we'd blow out memory on the worker. The implementation of the
		// CaptureHook should create an internal buffer and write chunks out to GCS
		// once they get to an appropriate size (50M or so?)
	}
	if *autoscalingAlgorithm != "" {
		if *autoscalingAlgorithm != "NONE" && *autoscalingAlgorithm != "THROUGHPUT_BASED" {
			return errors.New("invalid autoscaling algorithm. Use --autoscaling_algorithm=(NONE|THROUGHPUT_BASED)")
		}
	}

	hooks.SerializeHooksToOptions()

	experiments := jobopts.GetExperiments()
	if *minCPUPlatform != "" {
		experiments = append(experiments, fmt.Sprintf("min_cpu_platform=%v", *minCPUPlatform))
	}

	opts := &dataflowlib.JobOptions{
		Name:                jobopts.GetJobName(),
		Experiments:         experiments,
		Options:             beam.PipelineOptions.Export(),
		Project:             project,
		Region:              *region,
		Zone:                *zone,
		Network:             *network,
		Subnetwork:          *subnetwork,
		NoUsePublicIPs:      *noUsePublicIPs,
		NumWorkers:          *numWorkers,
		MaxNumWorkers:       *maxNumWorkers,
		Algorithm:           *autoscalingAlgorithm,
		MachineType:         *machineType,
		Labels:              jobLabels,
		ServiceAccountEmail: *serviceAccountEmail,
		TempLocation:        *tempLocation,
		Worker:              *jobopts.WorkerBinary,
		WorkerJar:           *workerJar,
		TeardownPolicy:      *teardownPolicy,
	}
	if opts.TempLocation == "" {
		opts.TempLocation = gcsx.Join(*stagingLocation, "tmp")
	}

	// (1) Build and submit

	edges, _, err := p.Build()
	if err != nil {
		return err
	}
	model, err := graphx.Marshal(edges, &graphx.Options{Environment: graphx.CreateEnvironment(
		ctx, jobopts.GetEnvironmentUrn(ctx), getContainerImage)})
	if err != nil {
		return errors.WithContext(err, "generating model pipeline")
	}

	// NOTE(herohde) 10/8/2018: the last segment of the names must be "worker" and "dataflow-worker.jar".
	id := fmt.Sprintf("go-%v-%v", atomic.AddInt32(&unique, 1), time.Now().UnixNano())

	modelURL := gcsx.Join(*stagingLocation, id, "model")
	workerURL := gcsx.Join(*stagingLocation, id, "worker")
	jarURL := gcsx.Join(*stagingLocation, id, "dataflow-worker.jar")

	if *dryRun {
		log.Info(ctx, "Dry-run: not submitting job!")

		log.Info(ctx, proto.MarshalTextString(model))
		job, err := dataflowlib.Translate(model, opts, workerURL, jarURL, modelURL)
		if err != nil {
			return err
		}
		dataflowlib.PrintJob(ctx, job)
		return nil
	}

	_, err = dataflowlib.Execute(ctx, model, opts, workerURL, jarURL, modelURL, *endpoint, *executeAsync)
	return err
}
func gcsRecorderHook(opts []string) perf.CaptureHook {
	bucket, prefix, err := gcsx.ParseObject(opts[0])
	if err != nil {
		panic(fmt.Sprintf("Invalid hook configuration for gcsRecorderHook: %s", opts))
	}

	return func(ctx context.Context, spec string, r io.Reader) error {
		client, err := gcsx.NewClient(ctx, storage.ScopeReadWrite)
		if err != nil {
			return errors.WithContext(err, "establishing GCS client")
		}
		return gcsx.WriteObject(ctx, client, bucket, path.Join(prefix, spec), r)
	}
}

func getContainerImage(ctx context.Context) string {
	urn := jobopts.GetEnvironmentUrn(ctx)
	if urn == "" || urn == "beam:env:docker:v1" {
		return jobopts.GetEnvironmentConfig(ctx)
	}
	panic(fmt.Sprintf("Unsupported environment %v", urn))
}
<commit_msg>[BEAM-9220] Go Dataflow jobs to use runner v2 (#11207)<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dataflow contains the Dataflow runner for submitting pipelines
// to Google Cloud Dataflow.
package dataflow

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"path"
	"strings"
	"sync/atomic"
	"time"

	"cloud.google.com/go/storage"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/hooks"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	"github.com/apache/beam/sdks/go/pkg/beam/options/gcpopts"
	"github.com/apache/beam/sdks/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/dataflow/dataflowlib"
	"github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
	"github.com/apache/beam/sdks/go/pkg/beam/x/hooks/perf"
	"github.com/golang/protobuf/proto"
)

// TODO(herohde) 5/16/2017: the Dataflow flags should match the other SDKs.

var (
	endpoint             = flag.String("dataflow_endpoint", "", "Dataflow endpoint (optional).")
	stagingLocation      = flag.String("staging_location", "", "GCS staging location (required).")
	image                = flag.String("worker_harness_container_image", "", "Worker harness container image (required).")
	labels               = flag.String("labels", "", "JSON-formatted map[string]string of job labels (optional).")
	serviceAccountEmail  = flag.String("service_account_email", "", "Service account email (optional).")
	numWorkers           = flag.Int64("num_workers", 0, "Number of workers (optional).")
	maxNumWorkers        = flag.Int64("max_num_workers", 0, "Maximum number of workers during scaling (optional).")
	autoscalingAlgorithm = flag.String("autoscaling_algorithm", "", "Autoscaling mode to use (optional).")
	zone                 = flag.String("zone", "", "GCP zone (optional)")
	region               = flag.String("region", "", "GCP Region (required)")
	network              = flag.String("network", "", "GCP network (optional)")
	subnetwork           = flag.String("subnetwork", "", "GCP subnetwork (optional)")
	noUsePublicIPs       = flag.Bool("no_use_public_ips", false, "Workers must not use public IP addresses (optional)")
	tempLocation         = flag.String("temp_location", "", "Temp location (optional)")
	machineType          = flag.String("worker_machine_type", "", "GCE machine type (optional)")
	minCPUPlatform       = flag.String("min_cpu_platform", "", "GCE minimum cpu platform (optional)")
	workerJar            = flag.String("dataflow_worker_jar", "", "Dataflow worker jar (optional)")

	executeAsync   = flag.Bool("execute_async", false, "Asynchronous execution. Submit the job and return immediately.")
	dryRun         = flag.Bool("dry_run", false, "Dry run. Just print the job, but don't submit it.")
	teardownPolicy = flag.String("teardown_policy", "", "Job teardown policy (internal only).")

	// SDK options
	cpuProfiling     = flag.String("cpu_profiling", "", "Job records CPU profiles to this GCS location (optional)")
	sessionRecording = flag.String("session_recording", "", "Job records session transcripts")
)

func init() {
	// Note that we also _ import harness/init to setup the remote execution hook.
	beam.RegisterRunner("dataflow", Execute)

	perf.RegisterProfCaptureHook("gcs_profile_writer", gcsRecorderHook)
}

var unique int32

// Execute runs the given pipeline on Google Cloud Dataflow. It uses the
// default application credentials to submit the job.
func Execute(ctx context.Context, p *beam.Pipeline) error {
	// (1) Gather job options

	project := *gcpopts.Project
	if project == "" {
		return errors.New("no Google Cloud project specified. Use --project=<project>")
	}
	if *stagingLocation == "" {
		return errors.New("no GCS staging location specified. Use --staging_location=gs://<bucket>/<path>")
	}
	if *region == "" {
		return errors.New("No Google Cloud region specified. Use --region=<region>. See https://cloud.google.com/dataflow/docs/concepts/regional-endpoints")
	}
	if *image == "" {
		*image = getContainerImage(ctx)
	}
	var jobLabels map[string]string
	if *labels != "" {
		if err := json.Unmarshal([]byte(*labels), &jobLabels); err != nil {
			return errors.Wrapf(err, "error reading --label flag as JSON")
		}
	}

	if *cpuProfiling != "" {
		perf.EnableProfCaptureHook("gcs_profile_writer", *cpuProfiling)
	}

	if *sessionRecording != "" {
		// TODO(wcn): BEAM-4017
		// It's a bit inconvenient for GCS because the whole object is written in
		// one pass, whereas the session logs are constantly appended. We wouldn't
		// want to hold all the logs in memory to flush at the end of the pipeline
		// as we'd blow out memory on the worker. The implementation of the
		// CaptureHook should create an internal buffer and write chunks out to GCS
		// once they get to an appropriate size (50M or so?)
	}
	if *autoscalingAlgorithm != "" {
		if *autoscalingAlgorithm != "NONE" && *autoscalingAlgorithm != "THROUGHPUT_BASED" {
			return errors.New("invalid autoscaling algorithm. Use --autoscaling_algorithm=(NONE|THROUGHPUT_BASED)")
		}
	}

	hooks.SerializeHooksToOptions()

	experiments := jobopts.GetExperiments()
	// Always use runner v2, unless set already.
	var v2set bool
	for _, e := range experiments {
		if strings.Contains(e, "use_runner_v2") || strings.Contains(e, "use_unified_worker") {
			v2set = true
			break
		}
	}
	if !v2set {
		experiments = append(experiments, "use_unified_worker")
	}
	if *minCPUPlatform != "" {
		experiments = append(experiments, fmt.Sprintf("min_cpu_platform=%v", *minCPUPlatform))
	}

	opts := &dataflowlib.JobOptions{
		Name:                jobopts.GetJobName(),
		Experiments:         experiments,
		Options:             beam.PipelineOptions.Export(),
		Project:             project,
		Region:              *region,
		Zone:                *zone,
		Network:             *network,
		Subnetwork:          *subnetwork,
		NoUsePublicIPs:      *noUsePublicIPs,
		NumWorkers:          *numWorkers,
		MaxNumWorkers:       *maxNumWorkers,
		Algorithm:           *autoscalingAlgorithm,
		MachineType:         *machineType,
		Labels:              jobLabels,
		ServiceAccountEmail: *serviceAccountEmail,
		TempLocation:        *tempLocation,
		Worker:              *jobopts.WorkerBinary,
		WorkerJar:           *workerJar,
		TeardownPolicy:      *teardownPolicy,
	}
	if opts.TempLocation == "" {
		opts.TempLocation = gcsx.Join(*stagingLocation, "tmp")
	}

	// (1) Build and submit

	edges, _, err := p.Build()
	if err != nil {
		return err
	}
	model, err := graphx.Marshal(edges, &graphx.Options{Environment: graphx.CreateEnvironment(
		ctx, jobopts.GetEnvironmentUrn(ctx), getContainerImage)})
	if err != nil {
		return errors.WithContext(err, "generating model pipeline")
	}

	// NOTE(herohde) 10/8/2018: the last segment of the names must be "worker" and "dataflow-worker.jar".
	id := fmt.Sprintf("go-%v-%v", atomic.AddInt32(&unique, 1), time.Now().UnixNano())

	modelURL := gcsx.Join(*stagingLocation, id, "model")
	workerURL := gcsx.Join(*stagingLocation, id, "worker")
	jarURL := gcsx.Join(*stagingLocation, id, "dataflow-worker.jar")

	if *dryRun {
		log.Info(ctx, "Dry-run: not submitting job!")

		log.Info(ctx, proto.MarshalTextString(model))
		job, err := dataflowlib.Translate(model, opts, workerURL, jarURL, modelURL)
		if err != nil {
			return err
		}
		dataflowlib.PrintJob(ctx, job)
		return nil
	}

	_, err = dataflowlib.Execute(ctx, model, opts, workerURL, jarURL, modelURL, *endpoint, *executeAsync)
	return err
}
func gcsRecorderHook(opts []string) perf.CaptureHook {
	bucket, prefix, err := gcsx.ParseObject(opts[0])
	if err != nil {
		panic(fmt.Sprintf("Invalid hook configuration for gcsRecorderHook: %s", opts))
	}

	return func(ctx context.Context, spec string, r io.Reader) error {
		client, err := gcsx.NewClient(ctx, storage.ScopeReadWrite)
		if err != nil {
			return errors.WithContext(err, "establishing GCS client")
		}
		return gcsx.WriteObject(ctx, client, bucket, path.Join(prefix, spec), r)
	}
}

func getContainerImage(ctx context.Context) string {
	urn := jobopts.GetEnvironmentUrn(ctx)
	if urn == "" || urn == "beam:env:docker:v1" {
		return jobopts.GetEnvironmentConfig(ctx)
	}
	panic(fmt.Sprintf("Unsupported environment %v", urn))
}
<|endoftext|>


<commit_before>// Copyright 2016 EF CTX. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"regexp"
	"strconv"

	"github.com/cezarsa/form"
	"github.com/gorilla/mux"
	"github.com/tsuru/tsuru/api"
)

var tsuruServer = newFakeTsuruServer()

// fakeTsuruServer provides a non-thread-safe, partial implementation of the
// tsuru API.
type fakeTsuruServer struct {
	apps    []app
	envVars map[string][]envVar
	deploys map[string][]deploy
	server  *httptest.Server
	router  *mux.Router
}

func newFakeTsuruServer() *fakeTsuruServer {
	var s fakeTsuruServer
	s.buildRouter()
	s.server = httptest.NewServer(s.router)
	s.reset()
	return &s
}

func (s *fakeTsuruServer) buildRouter() {
	s.router = mux.NewRouter()
	s.router.HandleFunc("/apps", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.createApp(w, r)
		case "GET":
			s.listApps(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	s.router.HandleFunc("/apps/{appname}", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "PUT":
			s.updateApp(w, r)
		case "GET":
			s.getApp(w, r)
		case "DELETE":
			s.deleteApp(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	s.router.HandleFunc("/apps/{appname}/env", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.setEnvs(w, r)
		case "GET":
			s.getEnvs(w, r)
		case "DELETE":
			s.unsetEnvs(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	s.router.HandleFunc("/deploys", s.listDeploys)
	s.router.HandleFunc("/apps/{appname}/cname", s.addCName)
}

func (s *fakeTsuruServer) createApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	if opts.Name == "" || opts.Platform == "" {
		http.Error(w, "invalid params", http.StatusBadRequest)
		return
	}
	_, index := s.findApp(opts.Name)
	if index > -1 {
		http.Error(w, "app already exists", http.StatusConflict)
		return
	}
	repositoryURL := fmt.Sprintf("git@gandalf.example.com:%s.git", opts.Name)
	s.apps = append(s.apps, app{
		Name:        opts.Name,
		Platform:    opts.Platform,
		Description: opts.Description,
		TeamOwner:   opts.Team,
		Teams:       []string{opts.Team},
		Plan: struct {
			Name string `json:"name"`
		}{Name: opts.Plan},
		Pool:          opts.Pool,
		Owner:         "user@example.com",
		RepositoryURL: repositoryURL,
		Addr:          opts.Name + ".tsuru.example.com",
	})
	s.writeJSON(w, map[string]string{"repository_url": repositoryURL})
}

func (s *fakeTsuruServer) listApps(w http.ResponseWriter, r *http.Request) {
	nameRegexp, err := regexp.Compile(r.URL.Query().Get("name"))
	if err != nil {
		http.Error(w, "invalid name regexp", http.StatusBadRequest)
		return
	}
	var apps []app
	for _, a := range s.apps {
		if nameRegexp.MatchString(a.Name) {
			apps = append(apps, a)
		}
	}
	if len(apps) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	s.writeJSON(w, apps)
}

func (s *fakeTsuruServer) updateApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if opts.Team != "" {
		a.TeamOwner = opts.Team
	}
	if opts.Description != "" {
		a.Description = opts.Description
	}
	if opts.Plan != "" {
		a.Plan.Name = opts.Plan
	}
	if opts.Pool != "" {
		a.Plan.Name = opts.Pool
	}
	s.apps[index] = a
}

func (s *fakeTsuruServer) getApp(w http.ResponseWriter, r *http.Request) {
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, a)
}

func (s *fakeTsuruServer) deleteApp(w http.ResponseWriter, r *http.Request) {
	_, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.apps[index] = s.apps[len(s.apps)-1]
	s.apps = s.apps[:len(s.apps)-1]
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) setEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	var evars api.Envs
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	form.DecodeValues(&evars, r.Form)
	for _, e := range evars.Envs {
		envs = append(envs, envVar{
			Name:   e.Name,
			Value:  e.Value,
			Public: !evars.Private,
		})
	}
	s.envVars[appName] = envs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) getEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, envs)
}

func (s *fakeTsuruServer) unsetEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	envNames := r.URL.Query()["env"]
	var newEnvs []envVar
	for _, e := range envs {
		var exclude bool
		for _, envName := range envNames {
			if e.Name == envName && e.Public {
				exclude = true
				break
			}
		}
		if !exclude {
			newEnvs = append(newEnvs, e)
		}
	}
	s.envVars[appName] = newEnvs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) listDeploys(w http.ResponseWriter, r *http.Request) {
	appName := r.URL.Query().Get("app")
	limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
	if appName == "" {
		http.Error(w, "missing app name in querystring", http.StatusBadRequest)
		return
	}
	deployList, ok := s.deploys[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if len(deployList) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	if limit == 0 || limit > len(deployList) {
		limit = len(deployList)
	}
	var deploys []deploy
	for i := len(deployList) - 1; i >= len(deployList)-limit; i-- {
		deploys = append(deploys, deployList[i])
	}
	s.writeJSON(w, deploys)
}

func (s *fakeTsuruServer) addCName(w http.ResponseWriter, r *http.Request) {
	cName := r.FormValue("cname")
	if cName == "" {
		http.Error(w, "missing param", http.StatusBadRequest)
		return
	}
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	for _, name := range a.CName {
		if name == cName {
			http.Error(w, "duplicate cname", http.StatusConflict)
			return
		}
	}
	a.CName = append(a.CName, cName)
	s.apps[index] = a
}

func (s *fakeTsuruServer) writeJSON(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(data)
}

func (s *fakeTsuruServer) findApp(name string) (a app, index int) {
	index = -1
	for i := range s.apps {
		if s.apps[i].Name == name {
			a = s.apps[i]
			index = i
			break
		}
	}
	return a, index
}

func (s *fakeTsuruServer) stop() {
	s.server.Close()
}

func (s *fakeTsuruServer) url() string {
	return s.server.URL
}

func (s *fakeTsuruServer) reset() {
	s.apps = nil
	s.envVars = make(map[string][]envVar)
	s.deploys = make(map[string][]deploy)
}
<commit_msg>Properly handle version prefix in fake tsuru server<commit_after>// Copyright 2016 EF CTX. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"regexp"
	"strconv"

	"github.com/cezarsa/form"
	"github.com/gorilla/mux"
	"github.com/tsuru/tsuru/api"
)

var tsuruServer = newFakeTsuruServer()

// fakeTsuruServer provides a non-thread-safe, partial implementation of the
// tsuru API.
type fakeTsuruServer struct {
	apps    []app
	envVars map[string][]envVar
	deploys map[string][]deploy
	server  *httptest.Server
	router  *mux.Router
}

func newFakeTsuruServer() *fakeTsuruServer {
	var s fakeTsuruServer
	s.buildRouter()
	s.server = httptest.NewServer(s.router)
	s.reset()
	return &s
}

func (s *fakeTsuruServer) buildRouter() {
	s.router = mux.NewRouter()
	r := s.router.PathPrefix("/1.0").Subrouter()
	r.HandleFunc("/apps", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.createApp(w, r)
		case "GET":
			s.listApps(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/apps/{appname}", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "PUT":
			s.updateApp(w, r)
		case "GET":
			s.getApp(w, r)
		case "DELETE":
			s.deleteApp(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/apps/{appname}/env", func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			s.setEnvs(w, r)
		case "GET":
			s.getEnvs(w, r)
		case "DELETE":
			s.unsetEnvs(w, r)
		default:
			http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
		}
	})
	r.HandleFunc("/deploys", s.listDeploys)
	r.HandleFunc("/apps/{appname}/cname", s.addCName)
}

func (s *fakeTsuruServer) createApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	if opts.Name == "" || opts.Platform == "" {
		http.Error(w, "invalid params", http.StatusBadRequest)
		return
	}
	_, index := s.findApp(opts.Name)
	if index > -1 {
		http.Error(w, "app already exists", http.StatusConflict)
		return
	}
	repositoryURL := fmt.Sprintf("git@gandalf.example.com:%s.git", opts.Name)
	s.apps = append(s.apps, app{
		Name:        opts.Name,
		Platform:    opts.Platform,
		Description: opts.Description,
		TeamOwner:   opts.Team,
		Teams:       []string{opts.Team},
		Plan: struct {
			Name string `json:"name"`
		}{Name: opts.Plan},
		Pool:          opts.Pool,
		Owner:         "user@example.com",
		RepositoryURL: repositoryURL,
		Addr:          opts.Name + ".tsuru.example.com",
	})
	s.writeJSON(w, map[string]string{"repository_url": repositoryURL})
}

func (s *fakeTsuruServer) listApps(w http.ResponseWriter, r *http.Request) {
	nameRegexp, err := regexp.Compile(r.URL.Query().Get("name"))
	if err != nil {
		http.Error(w, "invalid name regexp", http.StatusBadRequest)
		return
	}
	var apps []app
	for _, a := range s.apps {
		if nameRegexp.MatchString(a.Name) {
			apps = append(apps, a)
		}
	}
	if len(apps) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	s.writeJSON(w, apps)
}

func (s *fakeTsuruServer) updateApp(w http.ResponseWriter, r *http.Request) {
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	var opts createAppOptions
	form.DecodeValues(&opts, r.Form)
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if opts.Team != "" {
		a.TeamOwner = opts.Team
	}
	if opts.Description != "" {
		a.Description = opts.Description
	}
	if opts.Plan != "" {
		a.Plan.Name = opts.Plan
	}
	if opts.Pool != "" {
		a.Plan.Name = opts.Pool
	}
	s.apps[index] = a
}

func (s *fakeTsuruServer) getApp(w http.ResponseWriter, r *http.Request) {
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, a)
}

func (s *fakeTsuruServer) deleteApp(w http.ResponseWriter, r *http.Request) {
	_, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.apps[index] = s.apps[len(s.apps)-1]
	s.apps = s.apps[:len(s.apps)-1]
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) setEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	var evars api.Envs
	err := r.ParseForm()
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	form.DecodeValues(&evars, r.Form)
	for _, e := range evars.Envs {
		envs = append(envs, envVar{
			Name:   e.Name,
			Value:  e.Value,
			Public: !evars.Private,
		})
	}
	s.envVars[appName] = envs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) getEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	s.writeJSON(w, envs)
}

func (s *fakeTsuruServer) unsetEnvs(w http.ResponseWriter, r *http.Request) {
	appName := mux.Vars(r)["appname"]
	envs, ok := s.envVars[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	envNames := r.URL.Query()["env"]
	var newEnvs []envVar
	for _, e := range envs {
		var exclude bool
		for _, envName := range envNames {
			if e.Name == envName && e.Public {
				exclude = true
				break
			}
		}
		if !exclude {
			newEnvs = append(newEnvs, e)
		}
	}
	s.envVars[appName] = newEnvs
	w.WriteHeader(http.StatusOK)
}

func (s *fakeTsuruServer) listDeploys(w http.ResponseWriter, r *http.Request) {
	appName := r.URL.Query().Get("app")
	limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
	if appName == "" {
		http.Error(w, "missing app name in querystring", http.StatusBadRequest)
		return
	}
	deployList, ok := s.deploys[appName]
	if !ok {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	if len(deployList) == 0 {
		w.WriteHeader(http.StatusNoContent)
		return
	}
	if limit == 0 || limit > len(deployList) {
		limit = len(deployList)
	}
	var deploys []deploy
	for i := len(deployList) - 1; i >= len(deployList)-limit; i-- {
		deploys = append(deploys, deployList[i])
	}
	s.writeJSON(w, deploys)
}

func (s *fakeTsuruServer) addCName(w http.ResponseWriter, r *http.Request) {
	cName := r.FormValue("cname")
	if cName == "" {
		http.Error(w, "missing param", http.StatusBadRequest)
		return
	}
	a, index := s.findApp(mux.Vars(r)["appname"])
	if index < 0 {
		http.Error(w, "app not found", http.StatusNotFound)
		return
	}
	for _, name := range a.CName {
		if name == cName {
			http.Error(w, "duplicate cname", http.StatusConflict)
			return
		}
	}
	a.CName = append(a.CName, cName)
	s.apps[index] = a
}

func (s *fakeTsuruServer) writeJSON(w http.ResponseWriter, data interface{}) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	json.NewEncoder(w).Encode(data)
}

func (s *fakeTsuruServer) findApp(name string) (a app, index int) {
	index = -1
	for i := range s.apps {
		if s.apps[i].Name == name {
			a = s.apps[i]
			index = i
			break
		}
	}
	return a, index
}

func (s *fakeTsuruServer) stop() {
	s.server.Close()
}

func (s *fakeTsuruServer) url() string {
	return s.server.URL
}

func (s *fakeTsuruServer) reset() {
	s.apps = nil
	s.envVars = make(map[string][]envVar)
	s.deploys = make(map[string][]deploy)
}
<|endoftext|>


<commit_before>package editor

import (
	"fmt"
	"io"
)

func help(w io.Writer) error {
	for k, v := range helpMap {
		_, err := w.Write([]byte(fmt.Sprintf("%s   %s\n", k, v)))
		if err != nil {
			return err
		}
	}
	return nil
}

var helpMap = map[string]string{
	"h": "left",
	"l": "right",
}
<commit_msg>Add '0' / '$' command to help<commit_after>package editor

import (
	"fmt"
	"io"
)

func help(w io.Writer) error {
	for k, v := range helpMap {
		_, err := w.Write([]byte(fmt.Sprintf("%s   %s\n", k, v)))
		if err != nil {
			return err
		}
	}
	return nil
}

var helpMap = map[string]string{
	"h": "left",
	"l": "right",
	"0": "to first character in the line",
	"$": "to the last character in the line",
}
<|endoftext|>


<commit_before>package test

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/sacloud/libsacloud-v2/sacloud"
	"github.com/sacloud/libsacloud-v2/sacloud/types"
	"github.com/stretchr/testify/require"
)

func TestInternetOpCRUD(t *testing.T) {
	Run(t, &CRUDTestCase{
		Parallel: true,

		SetupAPICaller: singletonAPICaller,
		Create: &CRUDTestFunc{
			Func: testInternetCreate,
			Expect: &CRUDTestExpect{
				ExpectValue:  createInternetExpected,
				IgnoreFields: ignoreInternetFields,
			},
		},
		Read: &CRUDTestFunc{
			Func: testInternetRead,
			Expect: &CRUDTestExpect{
				ExpectValue:  createInternetExpected,
				IgnoreFields: ignoreInternetFields,
			},
		},
		Update: &CRUDTestFunc{
			Func: testInternetUpdate,
			Expect: &CRUDTestExpect{
				ExpectValue:  updateInternetExpected,
				IgnoreFields: ignoreInternetFields,
			},
		},
		Delete: &CRUDTestDeleteFunc{
			Func: testInternetDelete,
		},
	})
}

var (
	ignoreInternetFields = []string{
		"ID",
		"IconID",
		"CreatedAt",
		"Switch",
	}
	createInternetParam = &sacloud.InternetCreateRequest{
		Name:           "libsacloud-v2-internet",
		Description:    "desc",
		Tags:           []string{"tag1", "tag2"},
		NetworkMaskLen: 24,
		BandWidthMbps:  100,
	}
	createInternetExpected = &sacloud.Internet{
		Name:           createInternetParam.Name,
		Description:    createInternetParam.Description,
		Tags:           createInternetParam.Tags,
		NetworkMaskLen: createInternetParam.NetworkMaskLen,
		BandWidthMbps:  createInternetParam.BandWidthMbps,
	}
	updateInternetParam = &sacloud.InternetUpdateRequest{
		Name:        "libsacloud-v2-internet-upd",
		Tags:        []string{"tag1-upd", "tag2-upd"},
		Description: "desc-upd",
	}
	updateInternetExpected = &sacloud.Internet{
		Name:           updateInternetParam.Name,
		Description:    updateInternetParam.Description,
		Tags:           updateInternetParam.Tags,
		NetworkMaskLen: createInternetParam.NetworkMaskLen,
		BandWidthMbps:  createInternetParam.BandWidthMbps,
	}
)

func testInternetCreate(testContext *CRUDTestContext, caller sacloud.APICaller) (interface{}, error) {
	client := sacloud.NewInternetOp(caller)
	return client.Create(context.Background(), testZone, createInternetParam)
}

func testInternetRead(testContext *CRUDTestContext, caller sacloud.APICaller) (interface{}, error) {
	return readInternet(testContext.ID, caller)
}

func testInternetUpdate(testContext *CRUDTestContext, caller sacloud.APICaller) (interface{}, error) {
	client := sacloud.NewInternetOp(caller)
	return client.Update(context.Background(), testZone, testContext.ID, updateInternetParam)
}

func testInternetDelete(testContext *CRUDTestContext, caller sacloud.APICaller) error {
	client := sacloud.NewInternetOp(caller)
	return client.Delete(context.Background(), testZone, testContext.ID)
}

func readInternet(id types.ID, caller sacloud.APICaller) (interface{}, error) {
	client := sacloud.NewInternetOp(caller)
	// TODO スイッチ+ルータ作成後しばらくは404が返ってくる問題にどう対応するか?
	max := 30
	for {
		if max == 0 {
			break
		}
		res, err := client.Read(context.Background(), testZone, id)
		if err != nil || sacloud.IsNotFoundError(err) {
			max--
			time.Sleep(3 * time.Second)
			continue
		}
		return res, err
	}
	return nil, fmt.Errorf("internet[%s] is not found", id)
}

func TestInternetOp_Subnets(t *testing.T) {

	client := sacloud.NewInternetOp(singletonAPICaller())
	ctx := context.Background()

	// prepare
	internet, err := client.Create(ctx, testZone, createInternetParam)
	require.NoError(t, err)

	v, err := readInternet(internet.ID, singletonAPICaller())
	require.NoError(t, err)
	internet = v.(*sacloud.Internet)

	swOp := sacloud.NewSwitchOp(singletonAPICaller())
	sw, err := swOp.Read(ctx, testZone, internet.Switch.ID)
	require.NoError(t, err)

	// add subnet
	subnet, err := client.AddSubnet(ctx, testZone, internet.ID, &sacloud.InternetAddSubnetRequest{
		NetworkMaskLen: 28,
		NextHop:        sw.Subnets[0].AssignedIPAddressMin,
	})
	require.NoError(t, err)
	require.Len(t, subnet.IPAddresses, 16)
	require.Equal(t, sw.Subnets[0].AssignedIPAddressMin, subnet.NextHop)

	// update
	updSubnet, err := client.UpdateSubnet(ctx, testZone, internet.ID, subnet.ID, &sacloud.InternetUpdateSubnetRequest{
		NextHop: sw.Subnets[0].AssignedIPAddressMax,
	})
	require.NoError(t, err)
	require.Len(t, updSubnet.IPAddresses, 16)
	require.Equal(t, sw.Subnets[0].AssignedIPAddressMax, updSubnet.NextHop)

	// delete
	err = client.DeleteSubnet(ctx, testZone, internet.ID, subnet.ID)
	require.NoError(t, err)

	// delete internet
	err = client.Delete(ctx, testZone, internet.ID)
	require.NoError(t, err)

}
<commit_msg>inprove test<commit_after>package test

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/sacloud/libsacloud-v2/sacloud"
	"github.com/sacloud/libsacloud-v2/sacloud/types"
	"github.com/stretchr/testify/require"
)

func TestInternetOpCRUD(t *testing.T) {
	Run(t, &CRUDTestCase{
		Parallel: true,

		SetupAPICaller: singletonAPICaller,
		Create: &CRUDTestFunc{
			Func: testInternetCreate,
			Expect: &CRUDTestExpect{
				ExpectValue:  createInternetExpected,
				IgnoreFields: ignoreInternetFields,
			},
		},
		Read: &CRUDTestFunc{
			Func: testInternetRead,
			Expect: &CRUDTestExpect{
				ExpectValue:  createInternetExpected,
				IgnoreFields: ignoreInternetFields,
			},
		},
		Update: &CRUDTestFunc{
			Func: testInternetUpdate,
			Expect: &CRUDTestExpect{
				ExpectValue:  updateInternetExpected,
				IgnoreFields: ignoreInternetFields,
			},
		},
		Delete: &CRUDTestDeleteFunc{
			Func: testInternetDelete,
		},
	})
}

var (
	ignoreInternetFields = []string{
		"ID",
		"IconID",
		"CreatedAt",
		"Switch",
	}
	createInternetParam = &sacloud.InternetCreateRequest{
		Name:           "libsacloud-v2-internet",
		Description:    "desc",
		Tags:           []string{"tag1", "tag2"},
		NetworkMaskLen: 24,
		BandWidthMbps:  100,
	}
	createInternetExpected = &sacloud.Internet{
		Name:           createInternetParam.Name,
		Description:    createInternetParam.Description,
		Tags:           createInternetParam.Tags,
		NetworkMaskLen: createInternetParam.NetworkMaskLen,
		BandWidthMbps:  createInternetParam.BandWidthMbps,
	}
	updateInternetParam = &sacloud.InternetUpdateRequest{
		Name:        "libsacloud-v2-internet-upd",
		Tags:        []string{"tag1-upd", "tag2-upd"},
		Description: "desc-upd",
	}
	updateInternetExpected = &sacloud.Internet{
		Name:           updateInternetParam.Name,
		Description:    updateInternetParam.Description,
		Tags:           updateInternetParam.Tags,
		NetworkMaskLen: createInternetParam.NetworkMaskLen,
		BandWidthMbps:  createInternetParam.BandWidthMbps,
	}
	vpcRouterDeleted = false
)

func testInternetCreate(testContext *CRUDTestContext, caller sacloud.APICaller) (interface{}, error) {
	client := sacloud.NewInternetOp(caller)
	return client.Create(context.Background(), testZone, createInternetParam)
}

func testInternetRead(testContext *CRUDTestContext, caller sacloud.APICaller) (interface{}, error) {
	return readInternet(testContext.ID, caller)
}

func testInternetUpdate(testContext *CRUDTestContext, caller sacloud.APICaller) (interface{}, error) {
	client := sacloud.NewInternetOp(caller)
	return client.Update(context.Background(), testZone, testContext.ID, updateInternetParam)
}

func testInternetDelete(testContext *CRUDTestContext, caller sacloud.APICaller) error {
	client := sacloud.NewInternetOp(caller)
	err := client.Delete(context.Background(), testZone, testContext.ID)
	if err == nil {
		vpcRouterDeleted = true
	}
	return err
}

func readInternet(id types.ID, caller sacloud.APICaller) (interface{}, error) {
	client := sacloud.NewInternetOp(caller)
	if vpcRouterDeleted {
		return client.Read(context.Background(), testZone, id)
	}
	// TODO スイッチ+ルータ作成後しばらくは404が返ってくる問題にどう対応するか?
	max := 30
	for {
		if max == 0 {
			break
		}
		res, err := client.Read(context.Background(), testZone, id)
		if err != nil || sacloud.IsNotFoundError(err) {
			max--
			time.Sleep(3 * time.Second)
			continue
		}
		return res, err
	}
	return nil, fmt.Errorf("internet[%s] is not found", id)
}

func TestInternetOp_Subnets(t *testing.T) {

	client := sacloud.NewInternetOp(singletonAPICaller())
	ctx := context.Background()

	// prepare
	internet, err := client.Create(ctx, testZone, createInternetParam)
	require.NoError(t, err)

	v, err := readInternet(internet.ID, singletonAPICaller())
	require.NoError(t, err)
	internet = v.(*sacloud.Internet)

	swOp := sacloud.NewSwitchOp(singletonAPICaller())
	sw, err := swOp.Read(ctx, testZone, internet.Switch.ID)
	require.NoError(t, err)

	// add subnet
	subnet, err := client.AddSubnet(ctx, testZone, internet.ID, &sacloud.InternetAddSubnetRequest{
		NetworkMaskLen: 28,
		NextHop:        sw.Subnets[0].AssignedIPAddressMin,
	})
	require.NoError(t, err)
	require.Len(t, subnet.IPAddresses, 16)
	require.Equal(t, sw.Subnets[0].AssignedIPAddressMin, subnet.NextHop)

	// update
	updSubnet, err := client.UpdateSubnet(ctx, testZone, internet.ID, subnet.ID, &sacloud.InternetUpdateSubnetRequest{
		NextHop: sw.Subnets[0].AssignedIPAddressMax,
	})
	require.NoError(t, err)
	require.Len(t, updSubnet.IPAddresses, 16)
	require.Equal(t, sw.Subnets[0].AssignedIPAddressMax, updSubnet.NextHop)

	// delete
	err = client.DeleteSubnet(ctx, testZone, internet.ID, subnet.ID)
	require.NoError(t, err)

	// delete internet
	err = client.Delete(ctx, testZone, internet.ID)
	require.NoError(t, err)

}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"math"
	"bytes"
	"strings"
	"io/ioutil"
	"encoding/gob"
	"sync"
	"log"
)

const (
	DEFAULT_DURATION = 24 * 60 * 60
	DEFAULT_DT = 60
)

type Storage struct {
	metrics map[string]*Metric
	split_metric_names map[string][]string
	duration int
	dt int
}

type Metric struct {
	sync.RWMutex
	array []float32
	dt int
	duration int
	latest_i int
	latest_ts_k int64 // == timestamp / dt
}

type StoredMetric struct {
	Array []float32
	Dt int
	Duration int
	Latest_i int
	Latest_ts_k int64
}

func NewStorage() *Storage {
	s := new(Storage)
	s.duration = DEFAULT_DURATION
	s.dt = DEFAULT_DT
	s.metrics = make(map[string]*Metric)
	s.split_metric_names = make(map[string][]string)
	return s
}

func NewMetric(duration int, dt int, starting_ts int64) *Metric {
	m := new(Metric)
	m.latest_i = 0
	m.duration = duration
	m.dt = dt
	m.array = make([]float32, duration / dt)
	m.latest_ts_k = starting_ts / int64(dt)
	return m
}

func (self *Storage) StoreMetric(metric_name string, value float64, ts int64) {
	metric, ok := self.metrics[metric_name]
	if !ok {
		metric = NewMetric(self.duration, self.dt, ts)
		self.metrics[metric_name] = metric
		self.split_metric_names[metric_name] = strings.Split(metric_name, ".")
	}
	metric.Store(float32(value), ts)
}

func (self *Storage) RemoveMetric(metric_name string) {
	delete(self.metrics, metric_name)
	delete(self.split_metric_names, metric_name)
}

func (self *Storage) MetricCount() int {
	return len(self.metrics)
}

func (self *Storage) SetStorageParams(duration_hours int, precision_seconds int) {
	if duration_hours <= 0 {
		log.Fatal("duration must be greater than zero")
	}
	if precision_seconds <= 0 {
		log.Fatal("precision must be greater than zero")
	}
	self.duration = duration_hours * 60 * 60
	self.dt = precision_seconds
}

func matchesPattern (s []string, pattern []string) bool {
	if len(s) != len(pattern) {
		return false
	}
	for i := range s {
		if pattern[i] != "*" && s[i] != pattern[i] {
			return false
		}
	}
	return true
}

func (self *Storage) SumByPeriodGroupingQuery(metric_group_patterns []string, periods []int64, now int64) [][]float64 {
	sums := make([][]float64, len(metric_group_patterns))
	split_patterns := make([][]string, len(metric_group_patterns))
	for i := range metric_group_patterns {
		sums[i] = make([]float64, len(periods))
		split_patterns[i] = strings.Split(metric_group_patterns[i], ".")
	}

	for k := range self.metrics {
		split_k := self.split_metric_names[k]
		for i := range split_patterns {
			if matchesPattern(split_k, split_patterns[i]) {
				this_metric_sum := self.metrics[k].GetSumsPerPeriodUntilNow(periods, now)
				for j := range periods {
					sums[i][j] += this_metric_sum[j]
				}
				break // assume metric can match only one pattern
			}
		}
	}
	return sums
}

func (self *Storage) SaveToFile(filename string) error {
	tempfile, err := ioutil.TempFile("", "")
	if err != nil {
		return err
	}
	temppath := tempfile.Name()

	enc := gob.NewEncoder(tempfile)
	err = enc.Encode(self.metrics)
	if err != nil {
		tempfile.Close()
		os.Remove(temppath)
		return err
	}
	tempfile.Close()
	err = os.Rename(temppath, filename)
	if err != nil {
		os.Remove(temppath)
		return err
	}
	return nil
}

func (self *Storage) LoadFromFile(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	dec := gob.NewDecoder(file)
	err = dec.Decode(&self.metrics)
	if err != nil {
		return err
	}
	for k := range self.metrics {
		self.split_metric_names[k] = strings.Split(k, ".")
	}
	return nil
}

func (self *Metric) Store(value float32, ts int64) {
	self.Lock()
	defer self.Unlock()
	dt_64 := int64(self.dt)
	ts_k := ts / dt_64
	/*log.Printf("(%f, %d) ts_k %d, latest_ts_k %d", value, ts, ts_k, self.latest_ts_k)*/
	if self.latest_ts_k > ts_k {
		// amend value in the past
		i := int64(self.latest_i) - (self.latest_ts_k - ts_k)
		if i < 0 {
			i += int64(len(self.array))
		}
		if i < 0 {
			// falls outside the storage period
			return
		}
		self.array[i] += value
		return
	}
	for self.latest_ts_k < ts_k {
		self.latest_i = (self.latest_i + 1) % len(self.array)
		self.array[self.latest_i] = 0.0
		self.latest_ts_k += 1
	}
	self.array[self.latest_i] += value
}

func (self *Metric) GetValueAt(ts int64) float64 {
	self.RLock()
	defer self.RUnlock()
	ts_k := ts / int64(self.dt)
	if ts_k <= self.latest_ts_k - int64(len(self.array)) || ts_k > self.latest_ts_k {
		return 0.0
	}
	d_ts_k := self.latest_ts_k - ts_k
	i := (self.latest_i - int(d_ts_k))
	if i < 0 {
		i += len(self.array)
	}
	/*log.Printf("ts %v, ts_k %v, self.latest_ts_k %v --> i %v", ts, ts_k, self.latest_ts_k, i)*/
	return float64(self.array[i])
}

func (self *Metric) GetSumBetween(ts1 int64, ts2 int64) float64 {
	self.RLock()
	defer self.RUnlock()
	ts1_k := ts1 / int64(self.dt)
	ts2_k := ts2 / int64(self.dt)
	if ts2_k <= self.latest_ts_k - int64(len(self.array)) || ts1_k > self.latest_ts_k {
		return 0.0
	}

	if ts1_k <= self.latest_ts_k - int64(len(self.array)) {
		ts1_k = self.latest_ts_k - int64(len(self.array)) + 1
	}
	if ts2_k > self.latest_ts_k {
		ts2_k = self.latest_ts_k
	}

	d_ts1_k := self.latest_ts_k - ts1_k
	i := (self.latest_i - int(d_ts1_k))
	if i < 0 {
		i += len(self.array)
	}

	sum := 0.0
	for ts1_k <= ts2_k {
		sum += float64(self.array[i])
		i = (i + 1) % len(self.array)
		ts1_k += 1
	}
	return sum
}

func (self *Metric) GetSumForLastNSeconds(seconds int64, now_ts int64) float64 {
	ts2 := now_ts + int64(self.dt)
	ts1 := now_ts - seconds
	return self.GetSumBetween(ts1, ts2)
}

func (self *Metric) GetSumsPerPeriodUntilNow(periods []int64, now int64) []float64 {
	self.RLock()
	defer self.RUnlock()
	dt_64 := int64(self.dt)
	now_k := now / dt_64
	period_starts_k := make([]int64, len(periods))
	period_sums := make([]float64, len(periods))
	min_k := now_k

	for i := range periods {
		period_starts_k[i] = int64(math.Ceil((float64(now) - float64(periods[i])) / float64(dt_64)))
		if period_starts_k[i] < min_k {
			min_k = period_starts_k[i]
		}
		period_sums[i] = 0.0
	}

	if now_k <= self.latest_ts_k - int64(len(self.array)) || min_k > self.latest_ts_k {
		/*log.Printf("min_k %d, latest_ts_k %d, exiting", min_k, self.latest_ts_k)*/
		return period_sums
	}

	if min_k <= self.latest_ts_k - int64(len(self.array)) {
		min_k = self.latest_ts_k - int64(len(self.array)) + 1
	}
	if now_k > self.latest_ts_k {
		now_k = self.latest_ts_k
	}

	d_min_k := self.latest_ts_k - min_k
	i := (self.latest_i - int(d_min_k))
	if i < 0 {
		i += len(self.array)
	}

	for min_k <= now_k {
		/*log.Printf("Now at %d s, index %d", min_k * dt_64, i)*/
		for j := range periods {
			if period_starts_k[j] <= min_k {
				/*log.Printf(" - period %d..now gets +%f increment", periods[j], self.array[i])*/
				period_sums[j] += float64(self.array[i])
			}
		}
		i = (i + 1) % len(self.array)
		min_k += 1
	}
	/*log.Printf("Now is %d s, index %d", now, int(now_k) % len(self.array))*/
	return period_sums
}

func (self *Metric) GobEncode() ([]byte, error) {
	var sm StoredMetric
	var buf bytes.Buffer
	sm.Array = self.array
	sm.Dt = self.dt
	sm.Duration = self.duration
	sm.Latest_i = self.latest_i
	sm.Latest_ts_k = self.latest_ts_k
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(&sm)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (self *Metric) GobDecode(_bytes []byte) error {
	var sm StoredMetric
	buf := bytes.NewBuffer(_bytes)
	dec := gob.NewDecoder(buf)
	err := dec.Decode(&sm)
	if err != nil {
		return err
	}
	self.array = sm.Array
	self.dt = sm.Dt
	self.duration = sm.Duration
	self.latest_i = sm.Latest_i
	self.latest_ts_k = sm.Latest_ts_k
	return nil
}
<commit_msg>Ensuring temp file is on the same device while saving<commit_after>package main

import (
	"os"
	"math"
	"bytes"
	"strings"
	"encoding/gob"
	"sync"
	"log"
)

const (
	DEFAULT_DURATION = 24 * 60 * 60
	DEFAULT_DT = 60
)

type Storage struct {
	metrics map[string]*Metric
	split_metric_names map[string][]string
	duration int
	dt int
}

type Metric struct {
	sync.RWMutex
	array []float32
	dt int
	duration int
	latest_i int
	latest_ts_k int64 // == timestamp / dt
}

type StoredMetric struct {
	Array []float32
	Dt int
	Duration int
	Latest_i int
	Latest_ts_k int64
}

func NewStorage() *Storage {
	s := new(Storage)
	s.duration = DEFAULT_DURATION
	s.dt = DEFAULT_DT
	s.metrics = make(map[string]*Metric)
	s.split_metric_names = make(map[string][]string)
	return s
}

func NewMetric(duration int, dt int, starting_ts int64) *Metric {
	m := new(Metric)
	m.latest_i = 0
	m.duration = duration
	m.dt = dt
	m.array = make([]float32, duration / dt)
	m.latest_ts_k = starting_ts / int64(dt)
	return m
}

func (self *Storage) StoreMetric(metric_name string, value float64, ts int64) {
	metric, ok := self.metrics[metric_name]
	if !ok {
		metric = NewMetric(self.duration, self.dt, ts)
		self.metrics[metric_name] = metric
		self.split_metric_names[metric_name] = strings.Split(metric_name, ".")
	}
	metric.Store(float32(value), ts)
}

func (self *Storage) RemoveMetric(metric_name string) {
	delete(self.metrics, metric_name)
	delete(self.split_metric_names, metric_name)
}

func (self *Storage) MetricCount() int {
	return len(self.metrics)
}

func (self *Storage) SetStorageParams(duration_hours int, precision_seconds int) {
	if duration_hours <= 0 {
		log.Fatal("duration must be greater than zero")
	}
	if precision_seconds <= 0 {
		log.Fatal("precision must be greater than zero")
	}
	self.duration = duration_hours * 60 * 60
	self.dt = precision_seconds
}

func matchesPattern (s []string, pattern []string) bool {
	if len(s) != len(pattern) {
		return false
	}
	for i := range s {
		if pattern[i] != "*" && s[i] != pattern[i] {
			return false
		}
	}
	return true
}

func (self *Storage) SumByPeriodGroupingQuery(metric_group_patterns []string, periods []int64, now int64) [][]float64 {
	sums := make([][]float64, len(metric_group_patterns))
	split_patterns := make([][]string, len(metric_group_patterns))
	for i := range metric_group_patterns {
		sums[i] = make([]float64, len(periods))
		split_patterns[i] = strings.Split(metric_group_patterns[i], ".")
	}

	for k := range self.metrics {
		split_k := self.split_metric_names[k]
		for i := range split_patterns {
			if matchesPattern(split_k, split_patterns[i]) {
				this_metric_sum := self.metrics[k].GetSumsPerPeriodUntilNow(periods, now)
				for j := range periods {
					sums[i][j] += this_metric_sum[j]
				}
				break // assume metric can match only one pattern
			}
		}
	}
	return sums
}

func (self *Storage) SaveToFile(filename string) error {
	temppath := filename + ".tmp"
	tempfile, err := os.Create(temppath)
	if err != nil {
		return err
	}

	enc := gob.NewEncoder(tempfile)
	err = enc.Encode(self.metrics)
	if err != nil {
		tempfile.Close()
		os.Remove(temppath)
		return err
	}
	tempfile.Close()
	err = os.Rename(temppath, filename)
	if err != nil {
		os.Remove(temppath)
		return err
	}
	return nil
}

func (self *Storage) LoadFromFile(filename string) error {
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	defer file.Close()
	dec := gob.NewDecoder(file)
	err = dec.Decode(&self.metrics)
	if err != nil {
		return err
	}
	for k := range self.metrics {
		self.split_metric_names[k] = strings.Split(k, ".")
	}
	return nil
}

func (self *Metric) Store(value float32, ts int64) {
	self.Lock()
	defer self.Unlock()
	dt_64 := int64(self.dt)
	ts_k := ts / dt_64
	/*log.Printf("(%f, %d) ts_k %d, latest_ts_k %d", value, ts, ts_k, self.latest_ts_k)*/
	if self.latest_ts_k > ts_k {
		// amend value in the past
		i := int64(self.latest_i) - (self.latest_ts_k - ts_k)
		if i < 0 {
			i += int64(len(self.array))
		}
		if i < 0 {
			// falls outside the storage period
			return
		}
		self.array[i] += value
		return
	}
	for self.latest_ts_k < ts_k {
		self.latest_i = (self.latest_i + 1) % len(self.array)
		self.array[self.latest_i] = 0.0
		self.latest_ts_k += 1
	}
	self.array[self.latest_i] += value
}

func (self *Metric) GetValueAt(ts int64) float64 {
	self.RLock()
	defer self.RUnlock()
	ts_k := ts / int64(self.dt)
	if ts_k <= self.latest_ts_k - int64(len(self.array)) || ts_k > self.latest_ts_k {
		return 0.0
	}
	d_ts_k := self.latest_ts_k - ts_k
	i := (self.latest_i - int(d_ts_k))
	if i < 0 {
		i += len(self.array)
	}
	/*log.Printf("ts %v, ts_k %v, self.latest_ts_k %v --> i %v", ts, ts_k, self.latest_ts_k, i)*/
	return float64(self.array[i])
}

func (self *Metric) GetSumBetween(ts1 int64, ts2 int64) float64 {
	self.RLock()
	defer self.RUnlock()
	ts1_k := ts1 / int64(self.dt)
	ts2_k := ts2 / int64(self.dt)
	if ts2_k <= self.latest_ts_k - int64(len(self.array)) || ts1_k > self.latest_ts_k {
		return 0.0
	}

	if ts1_k <= self.latest_ts_k - int64(len(self.array)) {
		ts1_k = self.latest_ts_k - int64(len(self.array)) + 1
	}
	if ts2_k > self.latest_ts_k {
		ts2_k = self.latest_ts_k
	}

	d_ts1_k := self.latest_ts_k - ts1_k
	i := (self.latest_i - int(d_ts1_k))
	if i < 0 {
		i += len(self.array)
	}

	sum := 0.0
	for ts1_k <= ts2_k {
		sum += float64(self.array[i])
		i = (i + 1) % len(self.array)
		ts1_k += 1
	}
	return sum
}

func (self *Metric) GetSumForLastNSeconds(seconds int64, now_ts int64) float64 {
	ts2 := now_ts + int64(self.dt)
	ts1 := now_ts - seconds
	return self.GetSumBetween(ts1, ts2)
}

func (self *Metric) GetSumsPerPeriodUntilNow(periods []int64, now int64) []float64 {
	self.RLock()
	defer self.RUnlock()
	dt_64 := int64(self.dt)
	now_k := now / dt_64
	period_starts_k := make([]int64, len(periods))
	period_sums := make([]float64, len(periods))
	min_k := now_k

	for i := range periods {
		period_starts_k[i] = int64(math.Ceil((float64(now) - float64(periods[i])) / float64(dt_64)))
		if period_starts_k[i] < min_k {
			min_k = period_starts_k[i]
		}
		period_sums[i] = 0.0
	}

	if now_k <= self.latest_ts_k - int64(len(self.array)) || min_k > self.latest_ts_k {
		/*log.Printf("min_k %d, latest_ts_k %d, exiting", min_k, self.latest_ts_k)*/
		return period_sums
	}

	if min_k <= self.latest_ts_k - int64(len(self.array)) {
		min_k = self.latest_ts_k - int64(len(self.array)) + 1
	}
	if now_k > self.latest_ts_k {
		now_k = self.latest_ts_k
	}

	d_min_k := self.latest_ts_k - min_k
	i := (self.latest_i - int(d_min_k))
	if i < 0 {
		i += len(self.array)
	}

	for min_k <= now_k {
		/*log.Printf("Now at %d s, index %d", min_k * dt_64, i)*/
		for j := range periods {
			if period_starts_k[j] <= min_k {
				/*log.Printf(" - period %d..now gets +%f increment", periods[j], self.array[i])*/
				period_sums[j] += float64(self.array[i])
			}
		}
		i = (i + 1) % len(self.array)
		min_k += 1
	}
	/*log.Printf("Now is %d s, index %d", now, int(now_k) % len(self.array))*/
	return period_sums
}

func (self *Metric) GobEncode() ([]byte, error) {
	var sm StoredMetric
	var buf bytes.Buffer
	sm.Array = self.array
	sm.Dt = self.dt
	sm.Duration = self.duration
	sm.Latest_i = self.latest_i
	sm.Latest_ts_k = self.latest_ts_k
	enc := gob.NewEncoder(&buf)
	err := enc.Encode(&sm)
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

func (self *Metric) GobDecode(_bytes []byte) error {
	var sm StoredMetric
	buf := bytes.NewBuffer(_bytes)
	dec := gob.NewDecoder(buf)
	err := dec.Decode(&sm)
	if err != nil {
		return err
	}
	self.array = sm.Array
	self.dt = sm.Dt
	self.duration = sm.Duration
	self.latest_i = sm.Latest_i
	self.latest_ts_k = sm.Latest_ts_k
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cdrom

import (
	"context"
	"flag"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
)

type insert struct {
	*flags.DatastoreFlag
	*flags.VirtualMachineFlag

	device string
}

func init() {
	cli.Register("device.cdrom.insert", &insert{})
}

func (cmd *insert) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.DatastoreFlag, ctx = flags.NewDatastoreFlag(ctx)
	cmd.DatastoreFlag.Register(ctx, f)
	cmd.VirtualMachineFlag, ctx = flags.NewVirtualMachineFlag(ctx)
	cmd.VirtualMachineFlag.Register(ctx, f)

	f.StringVar(&cmd.device, "device", "", "CD-ROM device name")
}

func (cmd *insert) Process(ctx context.Context) error {
	if err := cmd.DatastoreFlag.Process(ctx); err != nil {
		return err
	}
	if err := cmd.VirtualMachineFlag.Process(ctx); err != nil {
		return err
	}
	return nil
}

func (cmd *insert) Usage() string {
	return "ISO"
}

func (cmd *insert) Description() string {
	return `Insert media on datastore into CD-ROM device.

If device is not specified, the first CD-ROM device is used.

Examples:
  govc device.cdrom.insert -vm vm-1 -device cdrom-3000 images/boot.iso`
}

func (cmd *insert) Run(ctx context.Context, f *flag.FlagSet) error {
	vm, err := cmd.VirtualMachine()
	if err != nil {
		return err
	}

	if vm == nil || f.NArg() != 1 {
		return flag.ErrHelp
	}

	devices, err := vm.Device(ctx)
	if err != nil {
		return err
	}

	c, err := devices.FindCdrom(cmd.device)
	if err != nil {
		return err
	}

	iso, err := cmd.DatastorePath(f.Arg(0))
	if err != nil {
		return nil
	}

	return vm.EditDevice(ctx, devices.InsertIso(c, iso))
}
<commit_msg>Report errors when cdrom.insert fails<commit_after>/*
Copyright (c) 2014-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cdrom

import (
	"context"
	"flag"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
)

type insert struct {
	*flags.DatastoreFlag
	*flags.VirtualMachineFlag

	device string
}

func init() {
	cli.Register("device.cdrom.insert", &insert{})
}

func (cmd *insert) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.DatastoreFlag, ctx = flags.NewDatastoreFlag(ctx)
	cmd.DatastoreFlag.Register(ctx, f)
	cmd.VirtualMachineFlag, ctx = flags.NewVirtualMachineFlag(ctx)
	cmd.VirtualMachineFlag.Register(ctx, f)

	f.StringVar(&cmd.device, "device", "", "CD-ROM device name")
}

func (cmd *insert) Process(ctx context.Context) error {
	if err := cmd.DatastoreFlag.Process(ctx); err != nil {
		return err
	}
	if err := cmd.VirtualMachineFlag.Process(ctx); err != nil {
		return err
	}
	return nil
}

func (cmd *insert) Usage() string {
	return "ISO"
}

func (cmd *insert) Description() string {
	return `Insert media on datastore into CD-ROM device.

If device is not specified, the first CD-ROM device is used.

Examples:
  govc device.cdrom.insert -vm vm-1 -device cdrom-3000 images/boot.iso`
}

func (cmd *insert) Run(ctx context.Context, f *flag.FlagSet) error {
	vm, err := cmd.VirtualMachine()
	if err != nil {
		return err
	}

	if vm == nil || f.NArg() != 1 {
		return flag.ErrHelp
	}

	devices, err := vm.Device(ctx)
	if err != nil {
		return err
	}

	c, err := devices.FindCdrom(cmd.device)
	if err != nil {
		return err
	}

	iso, err := cmd.DatastorePath(f.Arg(0))
	if err != nil {
		return err
	}

	return vm.EditDevice(ctx, devices.InsertIso(c, iso))
}
<|endoftext|>


<commit_before>package static

import (
	"os"
	"github.com/valyala/fasthttp"
)

func New(path string, IndexNames string) func(ctx *fasthttp.RequestCtx, next func(error)) {
	if gopath := os.Getenv("GOPATH"); gopath == "" {
		panic(`System doesn't set env GOPATH, please set the env variable`)
	}
	fs := &fasthttp.FS{
		Root:     os.Getenv("GOPATH") + path,
		Compress: true,
	}
	if IndexNames != "" {
		fs.IndexNames = []string{IndexNames}
	}
	staticHandler := fs.NewRequestHandler()
	return func(ctx *fasthttp.RequestCtx, next func(error)) {
		staticHandler(ctx)
	}
}
<commit_msg>static<commit_after>package static

import (
	"errors"
	"os"

	"github.com/valyala/fasthttp"
)

var DefaultFS = &FS{
	&fasthttp.FS{
		Compress:        true,
		IndexNames:      []string{"index.html"},
		Root:            ".",
		AcceptByteRange: true,
	},
	true,
}

type FS struct {
	// Fasthttp *fasthttp.FS
	*fasthttp.FS
	// When something error occours,
	// FallThrough == true, next(nil)
	// FallThrough == false, next(err)
	FallThrough bool
}

func New(fs *FS) func(ctx *fasthttp.RequestCtx, next func(error)) {
	staticHandler := fs.NewRequestHandler()
	return func(ctx *fasthttp.RequestCtx, next func(error)) {
		m := string(ctx.Method())
		if m != "GET" && m != "HEAD" {
			if fs.FallThrough {
				next(nil)
				return
			}
			// method not allowed
			next(errors.New("405"))
			return
		}

		fileInfo, err := os.Stat(fs.Root + string(ctx.Path()))
		// if err != nil && os.IsNotExist(err) {
		if err != nil {
			if fs.FallThrough {
				next(nil)
				return
			}
			next(err)
			return
		}

		if !fileInfo.IsDir() {
			staticHandler(ctx)
			return
		}

		errPath := filterPath(fs.Root+string(ctx.Path())+"/", fs.IndexNames, next)

		if errPath != nil {
			if fs.FallThrough {
				next(nil)
				return
			}
			next(errPath)
			return
		}
		staticHandler(ctx)

	}
}

func filterPath(path string, index []string, next func(error)) error {
	for _, v := range index {
		_, err := os.Stat(path + v)
		if err == nil {
			return nil
		}
	}
	return errors.New("Not found index page in directory: " + path)
}
<|endoftext|>


<commit_before>package turtle

// Version of the turtle library
const Version = "v0.1.0"

// Emojis maps a name to an Emoji
var Emojis = make(map[string]*Emoji)

// EmojisByChar maps a character to an Emoji
var EmojisByChar = make(map[string]*Emoji)

func init() {
	for _, e := range emojis {
		Emojis[e.Name] = e
		EmojisByChar[e.Char] = e
	}
}

// Search emojis by a name
func Search(s string) []*Emoji {
	return search(emojis, s)
}

// Keyword filters the emojis by a keyword
func Keyword(k string) []*Emoji {
	return keyword(emojis, k)
}

// Category filters the emojis by a category
func Category(c string) []*Emoji {
	return category(emojis, c)
}
<commit_msg>Add  new public Filter function for custom filters<commit_after>package turtle

// Version of the turtle library
const Version = "v0.1.0"

// Emojis maps a name to an Emoji
var Emojis = make(map[string]*Emoji)

// EmojisByChar maps a character to an Emoji
var EmojisByChar = make(map[string]*Emoji)

func init() {
	for _, e := range emojis {
		Emojis[e.Name] = e
		EmojisByChar[e.Char] = e
	}
}

// Search emojis by a name
func Search(s string) []*Emoji {
	return search(emojis, s)
}

// Keyword filters the emojis by a keyword
func Keyword(k string) []*Emoji {
	return keyword(emojis, k)
}

// Category filters the emojis by a category
func Category(c string) []*Emoji {
	return category(emojis, c)
}

// Filter the emojis based on the given comparison function
func Filter(f func(e *Emoji) bool) []*Emoji {
	return filter(emojis, f)
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/andygrunwald/go-trending"
	"log"
	"time"
)

// Tweet is a structure to store the tweet and the project name based on the tweet.
type Tweet struct {
	Tweet       string
	ProjectName string
}

// generateNewTweet is responsible to search a new project / repository
// and build a new tweet based on this.
// The generated tweet will be sent to tweetChan.
func generateNewTweet(tweetChan chan *Tweet, config *Configuration) {
	// TODO Make generation of trending project more intelligent.
	// Currently we do:
	// 		* Get all timeframes and get a random timeframe.
	//		  Get projects based on this timeframe and check if we can tweet them.
	//		  If this is not a success remove this timeframe from slice and continue.
	//
	// Steps to improve the area:
	//		* If no project was chosen, lets get trending languages
	//		  and chose one randomly and request the timeframes again
	//		  and repeat the "slice from removes trick there"
	var projectToTweet trending.Project
	trendingClient := NewTrendingClient()
	redisClient, err := NewRedisClient(&config.Redis)
	if err != nil {
		log.Fatal(err)
	}

	// Get timeframes and randomize them
	timeFrames := trendingClient.GetTimeFrames()
	ShuffleStringSlice(timeFrames)

	// First get the timeframes without any languages
	for _, timeFrame := range timeFrames {
		log.Printf("Getting projects for timeframe %s", timeFrame)
		getProject := trendingClient.GetRandomProjectGenerator(timeFrame, "")
		projectToTweet = findProjectWithRandomProjectGenerator(getProject, redisClient)

		// Check if we found a project.
		// If yes we can leave the loop and keep on rockin
		if isProjectEmpty(projectToTweet) == false {
			break
		}
	}

	// Check if we found a project. If yes tweet it.
	if isProjectEmpty(projectToTweet) == false {
		sendProject(tweetChan, projectToTweet)
		return
	}

	// If not, keep going and try to get some (trending) languages
	//
	// Yep, i know. This is nearly the same code as above.
	// And this is a code smell. But this is the first try.
	// And i want to get it working, before i will start refactoring.
	// You know? Make it work, make it fast, make it beautiful...
	languages := trendingClient.GetTrendingLanguages()
	ShuffleStringSlice(languages)
	ShuffleStringSlice(timeFrames)

	for _, language := range languages {
		for _, timeFrame := range timeFrames {
			log.Printf("Getting projects for timeframe %s and language %s", timeFrame, language)
			getProject := trendingClient.GetRandomProjectGenerator(timeFrame, language)
			projectToTweet = findProjectWithRandomProjectGenerator(getProject, redisClient)

			// Check if we found a project.
			// If yes we can leave the loop and keep on rockin
			if isProjectEmpty(projectToTweet) == false {
				break
			}
		}

		// If we found a project, break this loop again.
		if isProjectEmpty(projectToTweet) == false {
			break
		}
	}

	// Check if we found a project. If yes tweet it.
	if isProjectEmpty(projectToTweet) == false {
		sendProject(tweetChan, projectToTweet)
		return
	}
}

// sendProject puts the project we want to tweet into the tweet queue
// If the queue is ready to receive a new project, this will be tweeted
func sendProject(tweetChan chan *Tweet, p trending.Project) {
	tweet := &Tweet{
		Tweet:       buildTweet(p),
		ProjectName: p.Name,
	}
	tweetChan <- tweet
}

// isProjectEmpty checks if the incoming project is empty
func isProjectEmpty(p trending.Project) bool {
	if len(p.Name) > 0 {
		return false
	}

	return true
}

// findProjectWithRandomProjectGenerator retrieves a new project and
// checks if this was already tweeted.
func findProjectWithRandomProjectGenerator(getProject func() (trending.Project, error), redisClient *Redis) trending.Project {
	var projectToTweet trending.Project
	var project trending.Project
	var projectErr error

	for project, projectErr = getProject(); projectErr == nil; project, projectErr = getProject() {
		// Check if the project was already tweeted
		alreadyTweeted, err := redisClient.IsRepositoryAlreadyTweeted(project.Name)
		if err != nil {
			log.Println(err)
			continue
		}

		// If the project was already tweeted
		// we will skip this project and go to the next one
		if alreadyTweeted > 0 {
			continue
		}

		// This project wasn`t tweeted yet, so we will take over this job
		projectToTweet = project
		break
	}

	return projectToTweet
}

// buildTweet is responsible to build a 140 length string based on the project we found.
func buildTweet(p trending.Project) string {
	// TODO: We need to improve this.
	// 		* Get the real number of chars for a URL for t.co.
	//		* Get the length of the description and check if we need to trim this.
	// 		* Try to put more information into it (like Language)
	tweet := ""

	// Base length of a tweet
	tweetLen := 140

	// 25 letters for the url (+ 1 character for a whitespace)
	// TODO: Daily GET call to help/configuration to receive max length for URL
	// Only with this we are able to fill out 140 chars as max as possible.
	// @link https://dev.twitter.com/overview/t.co
	// @link https://dev.twitter.com/rest/reference/get/help/configuration
	//
	// Currently this value is hardcoded.
	// Anaconda (the twitter library we use) doesn`t support this yet.
	// There is a pull request waiting: https://github.com/ChimeraCoder/anaconda/pull/66
	// Today (2015-07-26) the values are
	//	"short_url_length": 22,
	//	"short_url_length_https": 23
	// We choose a few chars more to get some more time until Anaconda accepts the PR
	tweetLen -= 26

	// Check if the length of the project name is > 120 chars
	// We substract 3 chars, because we will add a suffix " - "
	if nameLen := len(p.Name); nameLen < (tweetLen - 3) {
		tweetLen -= len(p.Name)
		tweet += p.Name

		// Add name suffix " - "
		tweetLen -= 3
		tweet += " - "
	}

	// We only post descriptions if we got more than 20 charactes available
	if tweetLen > 20 {
		if len(p.Description) < tweetLen {
			tweet += p.Description
		} else {
			tweet += p.Description[0:(tweetLen - 1)]
		}
	}

	// Lets add the URL, but we don`t need to substract the chars
	// because we have done this before
	if p.URL != nil {
		tweet += " "
		tweet += p.URL.String()
	}

	return tweet
}

// markTweetAsAlreadyTweeted adds a projectName to the global blacklist of already tweeted projects.
// For this we use a Sorted Set where the score is the timestamp of the tweet.
func markTweetAsAlreadyTweeted(projectName string, config *Configuration) (int, error) {
	redisClient, err := NewRedisClient(&config.Redis)
	if err != nil {
		log.Fatal(err)
	}

	// Generate score in format YYYYMMDDHHiiss
	now := time.Now()
	score := now.Format("20060102150405")

	res, err := redisClient.AddRepositoryToTweetedList(projectName, score)
	if err != nil || res != 1 {
		log.Printf("Error during adding project %s to tweeted list: %s (%d)", projectName, err, res)
	}

	return res, err
}
<commit_msg>Moved timeframe loop to search a ne wtweet to seperate function<commit_after>package main

import (
	"github.com/andygrunwald/go-trending"
	"log"
	"time"
)

// Tweet is a structure to store the tweet and the project name based on the tweet.
type Tweet struct {
	Tweet       string
	ProjectName string
}

// generateNewTweet is responsible to search a new project / repository
// and build a new tweet based on this.
// The generated tweet will be sent to tweetChan.
func generateNewTweet(tweetChan chan *Tweet, config *Configuration) {
	// TODO Make generation of trending project more intelligent.
	// Currently we do:
	// 		* Get all timeframes and get a random timeframe.
	//		  Get projects based on this timeframe and check if we can tweet them.
	//		  If this is not a success remove this timeframe from slice and continue.
	//
	// Steps to improve the area:
	//		* If no project was chosen, lets get trending languages
	//		  and chose one randomly and request the timeframes again
	//		  and repeat the "slice from removes trick there"
	var projectToTweet trending.Project
	trendingClient := NewTrendingClient()
	redisClient, err := NewRedisClient(&config.Redis)
	if err != nil {
		log.Fatal(err)
	}

	// Get timeframes and randomize them
	timeFrames := trendingClient.GetTimeFrames()
	ShuffleStringSlice(timeFrames)

	// First get the timeframes without any languages
	projectToTweet = timeframeLoopToSearchAProject(timeFrames, "", trendingClient, redisClient)

	// Check if we found a project. If yes tweet it.
	if isProjectEmpty(projectToTweet) == false {
		sendProject(tweetChan, projectToTweet)
		return
	}

	// If not, keep going and try to get some (trending) languages
	//
	// Yep, i know. This is nearly the same code as above.
	// And this is a code smell. But this is the first try.
	// And i want to get it working, before i will start refactoring.
	// You know? Make it work, make it fast, make it beautiful...
	languages := trendingClient.GetTrendingLanguages()
	ShuffleStringSlice(languages)
	ShuffleStringSlice(timeFrames)

	for _, language := range languages {
		for _, timeFrame := range timeFrames {
			log.Printf("Getting projects for timeframe %s and language %s", timeFrame, language)
			getProject := trendingClient.GetRandomProjectGenerator(timeFrame, language)
			projectToTweet = findProjectWithRandomProjectGenerator(getProject, redisClient)

			// Check if we found a project.
			// If yes we can leave the loop and keep on rockin
			if isProjectEmpty(projectToTweet) == false {
				break
			}
		}

		// If we found a project, break this loop again.
		if isProjectEmpty(projectToTweet) == false {
			break
		}
	}

	// Check if we found a project. If yes tweet it.
	if isProjectEmpty(projectToTweet) == false {
		sendProject(tweetChan, projectToTweet)
	}
}

func timeframeLoopToSearchAProject(timeFrames []string, language string, trendingClient *Trend, redisClient *Redis) trending.Project {
	var projectToTweet trending.Project

	for _, timeFrame := range timeFrames {
		log.Printf("Getting projects for timeframe %s", timeFrame)
		getProject := trendingClient.GetRandomProjectGenerator(timeFrame, language)
		projectToTweet = findProjectWithRandomProjectGenerator(getProject, redisClient)

		// Check if we found a project.
		// If yes we can leave the loop and keep on rockin
		if isProjectEmpty(projectToTweet) == false {
			break
		}
	}

	return projectToTweet
}

// sendProject puts the project we want to tweet into the tweet queue
// If the queue is ready to receive a new project, this will be tweeted
func sendProject(tweetChan chan *Tweet, p trending.Project) {
	tweet := &Tweet{
		Tweet:       buildTweet(p),
		ProjectName: p.Name,
	}
	tweetChan <- tweet
}

// isProjectEmpty checks if the incoming project is empty
func isProjectEmpty(p trending.Project) bool {
	if len(p.Name) > 0 {
		return false
	}

	return true
}

// findProjectWithRandomProjectGenerator retrieves a new project and
// checks if this was already tweeted.
func findProjectWithRandomProjectGenerator(getProject func() (trending.Project, error), redisClient *Redis) trending.Project {
	var projectToTweet trending.Project
	var project trending.Project
	var projectErr error

	for project, projectErr = getProject(); projectErr == nil; project, projectErr = getProject() {
		// Check if the project was already tweeted
		alreadyTweeted, err := redisClient.IsRepositoryAlreadyTweeted(project.Name)
		if err != nil {
			log.Println(err)
			continue
		}

		// If the project was already tweeted
		// we will skip this project and go to the next one
		if alreadyTweeted > 0 {
			continue
		}

		// This project wasn`t tweeted yet, so we will take over this job
		projectToTweet = project
		break
	}

	return projectToTweet
}

// buildTweet is responsible to build a 140 length string based on the project we found.
func buildTweet(p trending.Project) string {
	// TODO: We need to improve this.
	// 		* Get the real number of chars for a URL for t.co.
	//		* Get the length of the description and check if we need to trim this.
	// 		* Try to put more information into it (like Language)
	tweet := ""

	// Base length of a tweet
	tweetLen := 140

	// 25 letters for the url (+ 1 character for a whitespace)
	// TODO: Daily GET call to help/configuration to receive max length for URL
	// Only with this we are able to fill out 140 chars as max as possible.
	// @link https://dev.twitter.com/overview/t.co
	// @link https://dev.twitter.com/rest/reference/get/help/configuration
	//
	// Currently this value is hardcoded.
	// Anaconda (the twitter library we use) doesn`t support this yet.
	// There is a pull request waiting: https://github.com/ChimeraCoder/anaconda/pull/66
	// Today (2015-07-26) the values are
	//	"short_url_length": 22,
	//	"short_url_length_https": 23
	// We choose a few chars more to get some more time until Anaconda accepts the PR
	tweetLen -= 26

	// Check if the length of the project name is > 120 chars
	// We substract 3 chars, because we will add a suffix " - "
	if nameLen := len(p.Name); nameLen < (tweetLen - 3) {
		tweetLen -= len(p.Name)
		tweet += p.Name

		// Add name suffix " - "
		tweetLen -= 3
		tweet += " - "
	}

	// We only post descriptions if we got more than 20 charactes available
	if tweetLen > 20 {
		if len(p.Description) < tweetLen {
			tweet += p.Description
		} else {
			tweet += p.Description[0:(tweetLen - 1)]
		}
	}

	// Lets add the URL, but we don`t need to substract the chars
	// because we have done this before
	if p.URL != nil {
		tweet += " "
		tweet += p.URL.String()
	}

	return tweet
}

// markTweetAsAlreadyTweeted adds a projectName to the global blacklist of already tweeted projects.
// For this we use a Sorted Set where the score is the timestamp of the tweet.
func markTweetAsAlreadyTweeted(projectName string, config *Configuration) (int, error) {
	redisClient, err := NewRedisClient(&config.Redis)
	if err != nil {
		log.Fatal(err)
	}

	// Generate score in format YYYYMMDDHHiiss
	now := time.Now()
	score := now.Format("20060102150405")

	res, err := redisClient.AddRepositoryToTweetedList(projectName, score)
	if err != nil || res != 1 {
		log.Printf("Error during adding project %s to tweeted list: %s (%d)", projectName, err, res)
	}

	return res, err
}
<|endoftext|>


<commit_before>// Package twilio implements the github.comitsabot/abot/interface/sms driver
// interface.
package twilio

import (
	"errors"
	"regexp"
	"strings"

	"github.com/itsabot/abot/shared/interface/sms"
	"github.com/itsabot/abot/shared/interface/sms/driver"
	"github.com/subosito/twilio"
)

type drv struct {
	fromKey string
	toKey   string
	msgKey  string
}

func (d *drv) Open(name string) (driver.Conn, error) {
	auth := strings.Split(name, ":")
	c := conn(*twilio.NewClient(auth[0], auth[1], nil))
	return &c, nil
}

func (d *drv) FromKey() string {
	return d.fromKey
}

func (d *drv) ToKey() string {
	return d.toKey
}

func (d *drv) MsgKey() string {
	return d.msgKey
}

func init() {
	sms.Register("twilio", &drv{
		fromKey: "From",
		toKey:   "To",
		msgKey:  "Body",
	})
}

type conn twilio.Client

// Send an SMS using a Twilio client to a specific phone number in the following
// valid international format ("+13105555555").
func (c *conn) Send(from, to, msg string) error {
	params := twilio.MessageParams{Body: msg}
	_, _, err := c.Messages.Send(from, to, params)
	return err
}

// Close the connection, but since Twilio connections are open as needed, there
// is nothing for us to close here. Return nil.
func (c *conn) Close() error {
	return nil
}

// phoneRegex determines whether a string is a phone number
var phoneRegex = regexp.MustCompile(`^\+?[1-9]\d{1,14}$`)

type phone string

// Valid determines the validity of a phone number according to Twilio's
// expectations for the number's formatting. If valid, error will be nil. If
// invalid, the returned error will contain the reason.
func (p phone) Valid() (valid bool, err error) {
	if len(p) < 10 || len(p) > 20 || !phoneRegex.MatchString(string(p)) {
		return false, errors.New("invalid phone number format: must have E.164 formatting")
	}
	if len(p) == 11 && p[0] != '1' {
		return false, errors.New("unsupported international number")
	}
	if len(p) == 12 && p[0] == '+' && p[1] != '1' {
		return false, errors.New("unsupported international number")
	}
	return true, nil
}
<commit_msg>Add TwilioResp XML struct<commit_after>// Package twilio implements the github.comitsabot/abot/interface/sms driver
// interface.
package twilio

import (
	"encoding/xml"
	"errors"
	"regexp"
	"strings"

	"github.com/itsabot/abot/shared/interface/sms"
	"github.com/itsabot/abot/shared/interface/sms/driver"
	"github.com/subosito/twilio"
)

type drv struct {
	fromKey string
	toKey   string
	msgKey  string
}

func (d *drv) Open(name string) (driver.Conn, error) {
	auth := strings.Split(name, ":")
	c := conn(*twilio.NewClient(auth[0], auth[1], nil))
	return &c, nil
}

func (d *drv) FromKey() string {
	return d.fromKey
}

func (d *drv) ToKey() string {
	return d.toKey
}

func (d *drv) MsgKey() string {
	return d.msgKey
}

func init() {
	sms.Register("twilio", &drv{
		fromKey: "From",
		toKey:   "To",
		msgKey:  "Body",
	})
}

type conn twilio.Client

// Send an SMS using a Twilio client to a specific phone number in the following
// valid international format ("+13105555555").
func (c *conn) Send(from, to, msg string) error {
	params := twilio.MessageParams{Body: msg}
	_, _, err := c.Messages.Send(from, to, params)
	return err
}

// Close the connection, but since Twilio connections are open as needed, there
// is nothing for us to close here. Return nil.
func (c *conn) Close() error {
	return nil
}

// phoneRegex determines whether a string is a phone number
var phoneRegex = regexp.MustCompile(`^\+?[1-9]\d{1,14}$`)

type phone string

// Valid determines the validity of a phone number according to Twilio's
// expectations for the number's formatting. If valid, error will be nil. If
// invalid, the returned error will contain the reason.
func (p phone) Valid() (valid bool, err error) {
	if len(p) < 10 || len(p) > 20 || !phoneRegex.MatchString(string(p)) {
		return false, errors.New("invalid phone number format: must have E.164 formatting")
	}
	if len(p) == 11 && p[0] != '1' {
		return false, errors.New("unsupported international number")
	}
	if len(p) == 12 && p[0] == '+' && p[1] != '1' {
		return false, errors.New("unsupported international number")
	}
	return true, nil
}

type TwilioResp struct {
	XMLName xml.Name `xml:"Response"`
	Message string
}
<|endoftext|>