<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package info

import (
	"testing"
	"time"
)

func TestStatsStartTime(t *testing.T) {
	N := 10
	stats := make([]*ContainerStats, 0, N)
	ct := time.Now()
	for i := 0; i < N; i++ {
		s := &ContainerStats{
			Timestamp: ct.Add(time.Duration(i) * time.Second),
		}
		stats = append(stats, s)
	}
	cinfo := &ContainerInfo{
		Name:  "/some/container",
		Stats: stats,
	}
	ref := ct.Add(time.Duration(N-1) * time.Second)
	end := cinfo.StatsEndTime()

	if !ref.Equal(end) {
		t.Errorf("end time is %v; should be %v", end, ref)
	}
}

func TestStatsEndTime(t *testing.T) {
	N := 10
	stats := make([]*ContainerStats, 0, N)
	ct := time.Now()
	for i := 0; i < N; i++ {
		s := &ContainerStats{
			Timestamp: ct.Add(time.Duration(i) * time.Second),
		}
		stats = append(stats, s)
	}
	cinfo := &ContainerInfo{
		Name:  "/some/container",
		Stats: stats,
	}
	ref := ct
	start := cinfo.StatsStartTime()

	if !ref.Equal(start) {
		t.Errorf("start time is %v; should be %v", start, ref)
	}
}

func TestPercentiles(t *testing.T) {
	N := 100
	data := make([]uint64, N)

	for i := 0; i < N; i++ {
		data[i] = uint64(i)
	}
	ps := []int{
		80,
		90,
		50,
	}
	ss := uint64Slice(data).Percentiles(ps...)
	for i, s := range ss {
		p := ps[i]
		d := uint64(N * p / 100)
		if d != s {
			t.Errorf("%v percentile data should be %v, but got %v", p, d, s)
		}
	}
}

func TestNewSampleNilStats(t *testing.T) {
	stats := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	stats.Cpu.Usage.PerCpu = []uint64{uint64(10)}
	stats.Cpu.Usage.Total = uint64(10)
	stats.Cpu.Usage.System = uint64(2)
	stats.Cpu.Usage.User = uint64(8)
	stats.Memory.Usage = uint64(200)

	sample, err := NewSample(nil, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}

	sample, err = NewSample(nil, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}

func TestAddSample(t *testing.T) {
	cpuPrevUsage := uint64(10)
	cpuCurrentUsage := uint64(15)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	sample, err := NewSample(prev, current)
	if err != nil {
		t.Errorf("should be able to generate a sample. but received error: %v", err)
	}
	if sample == nil {
		t.Fatalf("nil sample and nil error. unexpected result!")
	}

	if sample.Memory.Usage != memCurrentUsage {
		t.Errorf("wrong memory usage: %v. should be %v", sample.Memory.Usage, memCurrentUsage)
	}

	if sample.Cpu.Usage != cpuCurrentUsage-cpuPrevUsage {
		t.Errorf("wrong CPU usage: %v. should be %v", sample.Cpu.Usage, cpuCurrentUsage-cpuPrevUsage)
	}
}

func TestAddSampleIncompleteStats(t *testing.T) {
	cpuPrevUsage := uint64(10)
	cpuCurrentUsage := uint64(15)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	stats := &ContainerStats{
		Cpu:    prev.Cpu,
		Memory: nil,
	}
	sample, err := NewSample(stats, current)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
	sample, err = NewSample(prev, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}

	stats = &ContainerStats{
		Cpu:    nil,
		Memory: prev.Memory,
	}
	sample, err = NewSample(stats, current)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
	sample, err = NewSample(prev, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}

func TestAddSampleWrongOrder(t *testing.T) {
	cpuPrevUsage := uint64(10)
	cpuCurrentUsage := uint64(15)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	sample, err := NewSample(current, prev)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}

func TestAddSampleWrongCpuUsage(t *testing.T) {
	cpuPrevUsage := uint64(15)
	cpuCurrentUsage := uint64(10)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	sample, err := NewSample(prev, current)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}
<commit_msg>change var names<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package info

import (
	"testing"
	"time"
)

func TestStatsStartTime(t *testing.T) {
	N := 10
	stats := make([]*ContainerStats, 0, N)
	ct := time.Now()
	for i := 0; i < N; i++ {
		s := &ContainerStats{
			Timestamp: ct.Add(time.Duration(i) * time.Second),
		}
		stats = append(stats, s)
	}
	cinfo := &ContainerInfo{
		Name:  "/some/container",
		Stats: stats,
	}
	ref := ct.Add(time.Duration(N-1) * time.Second)
	end := cinfo.StatsEndTime()

	if !ref.Equal(end) {
		t.Errorf("end time is %v; should be %v", end, ref)
	}
}

func TestStatsEndTime(t *testing.T) {
	N := 10
	stats := make([]*ContainerStats, 0, N)
	ct := time.Now()
	for i := 0; i < N; i++ {
		s := &ContainerStats{
			Timestamp: ct.Add(time.Duration(i) * time.Second),
		}
		stats = append(stats, s)
	}
	cinfo := &ContainerInfo{
		Name:  "/some/container",
		Stats: stats,
	}
	ref := ct
	start := cinfo.StatsStartTime()

	if !ref.Equal(start) {
		t.Errorf("start time is %v; should be %v", start, ref)
	}
}

func TestPercentiles(t *testing.T) {
	N := 100
	data := make([]uint64, N)

	for i := 0; i < N; i++ {
		data[i] = uint64(i)
	}
	percentages := []int{
		80,
		90,
		50,
	}
	percentiles := uint64Slice(data).Percentiles(percentages...)
	for i, s := range percentiles {
		p := percentages[i]
		d := uint64(N * p / 100)
		if d != s {
			t.Errorf("%v percentile data should be %v, but got %v", p, d, s)
		}
	}
}

func TestNewSampleNilStats(t *testing.T) {
	stats := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	stats.Cpu.Usage.PerCpu = []uint64{uint64(10)}
	stats.Cpu.Usage.Total = uint64(10)
	stats.Cpu.Usage.System = uint64(2)
	stats.Cpu.Usage.User = uint64(8)
	stats.Memory.Usage = uint64(200)

	sample, err := NewSample(nil, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}

	sample, err = NewSample(nil, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}

func TestAddSample(t *testing.T) {
	cpuPrevUsage := uint64(10)
	cpuCurrentUsage := uint64(15)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	sample, err := NewSample(prev, current)
	if err != nil {
		t.Errorf("should be able to generate a sample. but received error: %v", err)
	}
	if sample == nil {
		t.Fatalf("nil sample and nil error. unexpected result!")
	}

	if sample.Memory.Usage != memCurrentUsage {
		t.Errorf("wrong memory usage: %v. should be %v", sample.Memory.Usage, memCurrentUsage)
	}

	if sample.Cpu.Usage != cpuCurrentUsage-cpuPrevUsage {
		t.Errorf("wrong CPU usage: %v. should be %v", sample.Cpu.Usage, cpuCurrentUsage-cpuPrevUsage)
	}
}

func TestAddSampleIncompleteStats(t *testing.T) {
	cpuPrevUsage := uint64(10)
	cpuCurrentUsage := uint64(15)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	stats := &ContainerStats{
		Cpu:    prev.Cpu,
		Memory: nil,
	}
	sample, err := NewSample(stats, current)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
	sample, err = NewSample(prev, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}

	stats = &ContainerStats{
		Cpu:    nil,
		Memory: prev.Memory,
	}
	sample, err = NewSample(stats, current)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
	sample, err = NewSample(prev, stats)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}

func TestAddSampleWrongOrder(t *testing.T) {
	cpuPrevUsage := uint64(10)
	cpuCurrentUsage := uint64(15)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	sample, err := NewSample(current, prev)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}

func TestAddSampleWrongCpuUsage(t *testing.T) {
	cpuPrevUsage := uint64(15)
	cpuCurrentUsage := uint64(10)
	memCurrentUsage := uint64(200)

	prev := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	prev.Cpu.Usage.PerCpu = []uint64{cpuPrevUsage}
	prev.Cpu.Usage.Total = cpuPrevUsage
	prev.Cpu.Usage.System = 0
	prev.Cpu.Usage.User = cpuPrevUsage
	prev.Timestamp = time.Now()

	current := &ContainerStats{
		Cpu:    &CpuStats{},
		Memory: &MemoryStats{},
	}
	current.Cpu.Usage.PerCpu = []uint64{cpuCurrentUsage}
	current.Cpu.Usage.Total = cpuCurrentUsage
	current.Cpu.Usage.System = 0
	current.Cpu.Usage.User = cpuCurrentUsage
	current.Memory.Usage = memCurrentUsage
	current.Timestamp = prev.Timestamp.Add(1 * time.Second)

	sample, err := NewSample(prev, current)
	if err == nil {
		t.Errorf("generated an unexpected sample: %+v", sample)
	}
}
<|endoftext|>


<commit_before>package route

import (
	"net/http"

	"github.com/gin-gonic/gin"
)

// GetCurrentUser return the current user
func GetCurrentUser(c *gin.Context) {
	c.JSON(http.StatusOK, gin.H{
		"message": "GetCurrentUser",
		"user":    "yamada",
	})
}
<commit_msg>Implement GetCurrentUser<commit_after>package route

import (
	"net/http"

	"github.com/gin-gonic/gin"
)

// GetCurrentUser return the current user
func GetCurrentUser(c *gin.Context) {
	user, exists := c.Get("currentUser")

	if exists {
		c.JSON(http.StatusOK, user)
	} else {
		c.JSON(http.StatusBadRequest, nil)
	}
}
<|endoftext|>


<commit_before>package bitsmanager

import (
	"bytes"
	"code.cloudfoundry.org/cli/cf/api/resources"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	. "code.cloudfoundry.org/cli/cf/i18n"
	"code.cloudfoundry.org/cli/cf/net"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/cloudfoundry/gofileutils/fileutils"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"runtime"
	"time"
)

const (
	DefaultAppUploadBitsTimeout = 15 * time.Minute
)

//go:generate counterfeiter . Repository
type Job struct {
	Metadata struct {
		GUID      string    `json:"guid"`
		CreatedAt time.Time `json:"created_at"`
		URL       string    `json:"url"`
	} `json:"metadata"`
	Entity struct {
		GUID         string `json:"guid"`
		Status       string `json:"status"`
		Error        string `json:"error"`
		ErrorDetails struct {
			Code        int    `json:"code"`
			Description string `json:"description"`
			ErrorCode   string `json:"error_code"`
		} `json:"error_details"`
	} `json:"entity"`
}
type ApplicationBitsRepository interface {
	GetApplicationSha1(appGUID string) (string, error)
	IsDiff(appGUID string, currentSha1 string) (bool, string, error)
	UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error)
	CopyBits(origAppGuid string, newAppGuid string) error
}

type CloudControllerApplicationBitsRepository struct {
	config  coreconfig.Reader
	gateway net.Gateway
}

func NewCloudControllerApplicationBitsRepository(config coreconfig.Reader, gateway net.Gateway) (repo CloudControllerApplicationBitsRepository) {
	repo.config = config
	repo.gateway = gateway
	return
}
func (repo CloudControllerApplicationBitsRepository) IsDiff(appGUID string, currentSha1 string) (bool, string, error) {
	sha1Found, err := repo.GetApplicationSha1(appGUID)
	if err != nil {
		return true, "", err
	}
	return currentSha1 != sha1Found, sha1Found, nil
}
func (repo CloudControllerApplicationBitsRepository) GetApplicationSha1(appGUID string) (string, error) {
	// we are oblige to do the request by itself because cli is reading the full response body
	// to dump the response into a possible logger.
	// we need to read just few bytes to create the sha1
	apiURL := fmt.Sprintf("/v2/apps/%s/download", appGUID)
	request, err := http.NewRequest("GET", repo.config.APIEndpoint()+apiURL, nil)
	if err != nil {
		return "", fmt.Errorf("%s: %s", T("Error building request"), err.Error())
	}
	request.Header.Set("Authorization", repo.config.AccessToken())
	request.Header.Set("accept", "application/json")
	request.Header.Set("Connection", "close")
	request.Header.Set("content-type", "application/json")
	request.Header.Set("User-Agent", "go-cli "+repo.config.CLIVersion()+" / "+runtime.GOOS)

	tr := &http.Transport{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: repo.config.IsSSLDisabled()},
	}
	client := &http.Client{
		Transport: tr,
		Timeout:   2 * time.Second,
	}

	resp, err := client.Do(request)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	sha1, err := GetSha1FromReader(resp.Body)
	if err != nil {
		return "", err
	}
	return sha1, nil
}
func (repo CloudControllerApplicationBitsRepository) CopyBits(origAppGuid string, newAppGuid string) error {
	apiURL := fmt.Sprintf("%s/v2/apps/%s/copy_bits", repo.config.APIEndpoint(), newAppGuid)
	data := bytes.NewReader([]byte(fmt.Sprintf(`{"source_app_guid":"%s"}`, origAppGuid)))
	req, err := repo.gateway.NewRequest("POST", apiURL, repo.config.AccessToken(), data)
	if err != nil {
		return err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return err
	}
	for {
		job, err := repo.getJob(job.Entity.GUID)
		if err != nil {
			return err
		}
		if job.Entity.Status == "finished" {
			return nil
		}
		if job.Entity.Status == "failed" {
			return fmt.Errorf(
				"Error %s, %s [code: %d]",
				job.Entity.ErrorDetails.ErrorCode,
				job.Entity.ErrorDetails.Description,
				job.Entity.ErrorDetails.Code,
			)
		}
		time.Sleep(2 * time.Second)
	}
	return nil
}
func (repo CloudControllerApplicationBitsRepository) getJob(jobGuid string) (Job, error) {
	apiURL := fmt.Sprintf("%s/v2/jobs/%s", repo.config.APIEndpoint(), jobGuid)
	req, err := repo.gateway.NewRequest("GET", apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return Job{}, err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return Job{}, err
	}
	return job, nil
}

/////
//Old way to send bits, now do it as a stream
func (repo CloudControllerApplicationBitsRepository) UploadBitsTmp(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error) {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	fileutils.TempFile("requests", func(requestFile *os.File, err error) {
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error creating tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		presentFiles := []resources.AppFileResource{}
		presentFilesJSON, _ := json.Marshal(presentFiles)

		boundary, err := repo.writeUploadBody(zipFile, fileSize, requestFile, presentFilesJSON)
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error writing to tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		var request *net.Request
		request, apiErr = repo.gateway.NewRequestForFile("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), requestFile)
		if apiErr != nil {
			return
		}

		contentType := fmt.Sprintf("multipart/form-data; boundary=%s", boundary)
		request.HTTPReq.Header.Set("Content-Type", contentType)

		response := &resources.Resource{}
		_, apiErr = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
		if apiErr != nil {
			return
		}
	})

	return
}
func (repo CloudControllerApplicationBitsRepository) writeUploadBody(zipFile io.ReadCloser, fileSize int64, body *os.File, presentResourcesJSON []byte) (boundary string, err error) {
	writer := multipart.NewWriter(body)
	defer writer.Close()

	boundary = writer.Boundary()

	part, err := writer.CreateFormField("resources")
	if err != nil {
		return
	}
	_, err = io.Copy(part, bytes.NewBuffer(presentResourcesJSON))
	if err != nil {
		return
	}

	if zipFile == nil {
		return
	}
	part, zipErr := createZipPartWriter(fileSize, writer)
	if zipErr != nil {
		return
	}

	_, zipErr = io.Copy(part, zipFile)
	if zipErr != nil {
		return
	}
	return
}

func createZipPartWriter(fileSize int64, writer *multipart.Writer) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
	h.Set("Content-Transfer-Encoding", "binary")
	return writer.CreatePart(h)
}

//Old way to send bits
/////
func (repo CloudControllerApplicationBitsRepository) UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) error {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	r, w := io.Pipe()
	mpw := multipart.NewWriter(w)
	go func() {
		var err error
		defer w.Close()
		part, err := mpw.CreateFormField("resources")
		if err != nil {
			panic(err)
		}
		_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
		if err != nil {
			panic(err)
		}
		h := make(textproto.MIMEHeader)
		h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
		h.Set("Content-Type", "application/zip")
		h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
		h.Set("Content-Transfer-Encoding", "binary")

		part, err = mpw.CreatePart(h)
		if err != nil {
			panic(err)
		}
		if _, err = io.Copy(part, zipFile); err != nil {
			panic(err)
		}
		mpw.Close()
	}()
	var request *net.Request
	request, err := repo.gateway.NewRequest("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return err
	}
	contentType := fmt.Sprintf("multipart/form-data; boundary=%s", mpw.Boundary())
	request.HTTPReq.Header.Set("Content-Type", contentType)
	request.HTTPReq.ContentLength = int64(repo.predictPart(int64(fileSize), mpw.Boundary()))
	request.HTTPReq.Body = r

	response := &resources.Resource{}
	_, err = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
	if err != nil {
		panic(err)
	}
	return nil
}

func (repo CloudControllerApplicationBitsRepository) predictPart(filesize int64, boundary string) int64 {
	buf := new(bytes.Buffer)
	mpw := multipart.NewWriter(buf)

	mpw.SetBoundary(boundary)
	part, err := mpw.CreateFormField("resources")
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
	if err != nil {
		panic(err)
	}
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", filesize))
	h.Set("Content-Transfer-Encoding", "binary")

	part, err = mpw.CreatePart(h)
	if err != nil {
		panic(err)
	}
	mpw.Close()
	b, _ := ioutil.ReadAll(buf)
	return int64(len(b)) + filesize
}
<commit_msg>close file when needed when sending bits<commit_after>package bitsmanager

import (
	"bytes"
	"code.cloudfoundry.org/cli/cf/api/resources"
	"code.cloudfoundry.org/cli/cf/configuration/coreconfig"
	. "code.cloudfoundry.org/cli/cf/i18n"
	"code.cloudfoundry.org/cli/cf/net"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"github.com/cloudfoundry/gofileutils/fileutils"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/textproto"
	"os"
	"runtime"
	"time"
)

const (
	DefaultAppUploadBitsTimeout = 15 * time.Minute
)

//go:generate counterfeiter . Repository
type Job struct {
	Metadata struct {
		GUID      string    `json:"guid"`
		CreatedAt time.Time `json:"created_at"`
		URL       string    `json:"url"`
	} `json:"metadata"`
	Entity struct {
		GUID         string `json:"guid"`
		Status       string `json:"status"`
		Error        string `json:"error"`
		ErrorDetails struct {
			Code        int    `json:"code"`
			Description string `json:"description"`
			ErrorCode   string `json:"error_code"`
		} `json:"error_details"`
	} `json:"entity"`
}
type ApplicationBitsRepository interface {
	GetApplicationSha1(appGUID string) (string, error)
	IsDiff(appGUID string, currentSha1 string) (bool, string, error)
	UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error)
	CopyBits(origAppGuid string, newAppGuid string) error
}

type CloudControllerApplicationBitsRepository struct {
	config  coreconfig.Reader
	gateway net.Gateway
}

func NewCloudControllerApplicationBitsRepository(config coreconfig.Reader, gateway net.Gateway) (repo CloudControllerApplicationBitsRepository) {
	repo.config = config
	repo.gateway = gateway
	return
}
func (repo CloudControllerApplicationBitsRepository) IsDiff(appGUID string, currentSha1 string) (bool, string, error) {
	sha1Found, err := repo.GetApplicationSha1(appGUID)
	if err != nil {
		return true, "", err
	}
	return currentSha1 != sha1Found, sha1Found, nil
}
func (repo CloudControllerApplicationBitsRepository) GetApplicationSha1(appGUID string) (string, error) {
	// we are oblige to do the request by itself because cli is reading the full response body
	// to dump the response into a possible logger.
	// we need to read just few bytes to create the sha1
	apiURL := fmt.Sprintf("/v2/apps/%s/download", appGUID)
	request, err := http.NewRequest("GET", repo.config.APIEndpoint()+apiURL, nil)
	if err != nil {
		return "", fmt.Errorf("%s: %s", T("Error building request"), err.Error())
	}
	request.Header.Set("Authorization", repo.config.AccessToken())
	request.Header.Set("accept", "application/json")
	request.Header.Set("Connection", "close")
	request.Header.Set("content-type", "application/json")
	request.Header.Set("User-Agent", "go-cli "+repo.config.CLIVersion()+" / "+runtime.GOOS)

	tr := &http.Transport{
		Proxy:           http.ProxyFromEnvironment,
		TLSClientConfig: &tls.Config{InsecureSkipVerify: repo.config.IsSSLDisabled()},
	}
	client := &http.Client{
		Transport: tr,
		Timeout:   2 * time.Second,
	}

	resp, err := client.Do(request)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()
	sha1, err := GetSha1FromReader(resp.Body)
	if err != nil {
		return "", err
	}
	return sha1, nil
}
func (repo CloudControllerApplicationBitsRepository) CopyBits(origAppGuid string, newAppGuid string) error {
	apiURL := fmt.Sprintf("%s/v2/apps/%s/copy_bits", repo.config.APIEndpoint(), newAppGuid)
	data := bytes.NewReader([]byte(fmt.Sprintf(`{"source_app_guid":"%s"}`, origAppGuid)))
	req, err := repo.gateway.NewRequest("POST", apiURL, repo.config.AccessToken(), data)
	if err != nil {
		return err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return err
	}
	for {
		job, err := repo.getJob(job.Entity.GUID)
		if err != nil {
			return err
		}
		if job.Entity.Status == "finished" {
			return nil
		}
		if job.Entity.Status == "failed" {
			return fmt.Errorf(
				"Error %s, %s [code: %d]",
				job.Entity.ErrorDetails.ErrorCode,
				job.Entity.ErrorDetails.Description,
				job.Entity.ErrorDetails.Code,
			)
		}
		time.Sleep(2 * time.Second)
	}
	return nil
}
func (repo CloudControllerApplicationBitsRepository) getJob(jobGuid string) (Job, error) {
	apiURL := fmt.Sprintf("%s/v2/jobs/%s", repo.config.APIEndpoint(), jobGuid)
	req, err := repo.gateway.NewRequest("GET", apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return Job{}, err
	}
	var job Job
	_, err = repo.gateway.PerformRequestForJSONResponse(req, &job)
	if err != nil {
		return Job{}, err
	}
	return job, nil
}

/////
//Old way to send bits, now do it as a stream
func (repo CloudControllerApplicationBitsRepository) UploadBitsTmp(appGUID string, zipFile io.ReadCloser, fileSize int64) (apiErr error) {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	fileutils.TempFile("requests", func(requestFile *os.File, err error) {
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error creating tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		presentFiles := []resources.AppFileResource{}
		presentFilesJSON, _ := json.Marshal(presentFiles)

		boundary, err := repo.writeUploadBody(zipFile, fileSize, requestFile, presentFilesJSON)
		if err != nil {
			apiErr = fmt.Errorf("%s: %s", T("Error writing to tmp file: {{.Err}}", map[string]interface{}{"Err": err}), err.Error())
			return
		}

		var request *net.Request
		request, apiErr = repo.gateway.NewRequestForFile("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), requestFile)
		if apiErr != nil {
			return
		}

		contentType := fmt.Sprintf("multipart/form-data; boundary=%s", boundary)
		request.HTTPReq.Header.Set("Content-Type", contentType)

		response := &resources.Resource{}
		_, apiErr = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
		if apiErr != nil {
			return
		}
	})

	return
}
func (repo CloudControllerApplicationBitsRepository) writeUploadBody(zipFile io.ReadCloser, fileSize int64, body *os.File, presentResourcesJSON []byte) (boundary string, err error) {
	writer := multipart.NewWriter(body)
	defer writer.Close()

	boundary = writer.Boundary()

	part, err := writer.CreateFormField("resources")
	if err != nil {
		return
	}
	_, err = io.Copy(part, bytes.NewBuffer(presentResourcesJSON))
	if err != nil {
		return
	}

	if zipFile == nil {
		return
	}
	part, zipErr := createZipPartWriter(fileSize, writer)
	if zipErr != nil {
		return
	}

	_, zipErr = io.Copy(part, zipFile)
	if zipErr != nil {
		return
	}
	return
}

func createZipPartWriter(fileSize int64, writer *multipart.Writer) (io.Writer, error) {
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
	h.Set("Content-Transfer-Encoding", "binary")
	return writer.CreatePart(h)
}

//Old way to send bits
/////

func (repo CloudControllerApplicationBitsRepository) UploadBits(appGUID string, zipFile io.ReadCloser, fileSize int64) error {
	apiURL := fmt.Sprintf("/v2/apps/%s/bits", appGUID)
	r, w := io.Pipe()
	mpw := multipart.NewWriter(w)
	go func() {
		var err error
		defer w.Close()
		part, err := mpw.CreateFormField("resources")
		if err != nil {
			mpw.Close()
			panic(err)
		}
		_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
		if err != nil {
			mpw.Close()
			panic(err)
		}
		h := make(textproto.MIMEHeader)
		h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
		h.Set("Content-Type", "application/zip")
		h.Set("Content-Length", fmt.Sprintf("%d", fileSize))
		h.Set("Content-Transfer-Encoding", "binary")

		part, err = mpw.CreatePart(h)
		if err != nil {
			mpw.Close()
			panic(err)
		}
		if _, err = io.Copy(part, zipFile); err != nil {
			mpw.Close()
			panic(err)
		}
		mpw.Close()
	}()
	var request *net.Request
	request, err := repo.gateway.NewRequest("PUT", repo.config.APIEndpoint()+apiURL, repo.config.AccessToken(), nil)
	if err != nil {
		return err
	}
	contentType := fmt.Sprintf("multipart/form-data; boundary=%s", mpw.Boundary())
	request.HTTPReq.Header.Set("Content-Type", contentType)
	request.HTTPReq.ContentLength = int64(repo.predictPart(int64(fileSize), mpw.Boundary()))
	request.HTTPReq.Body = r

	response := &resources.Resource{}
	_, err = repo.gateway.PerformPollingRequestForJSONResponse(repo.config.APIEndpoint(), request, response, DefaultAppUploadBitsTimeout)
	if err != nil {
		panic(err)
	}
	return nil
}

func (repo CloudControllerApplicationBitsRepository) predictPart(filesize int64, boundary string) int64 {
	buf := new(bytes.Buffer)
	mpw := multipart.NewWriter(buf)

	mpw.SetBoundary(boundary)
	part, err := mpw.CreateFormField("resources")
	if err != nil {
		mpw.Close()
		panic(err)
	}
	_, err = io.Copy(part, bytes.NewBuffer([]byte("[]")))
	if err != nil {
		mpw.Close()
		panic(err)
	}
	h := make(textproto.MIMEHeader)
	h.Set("Content-Disposition", `form-data; name="application"; filename="application.zip"`)
	h.Set("Content-Type", "application/zip")
	h.Set("Content-Length", fmt.Sprintf("%d", filesize))
	h.Set("Content-Transfer-Encoding", "binary")

	part, err = mpw.CreatePart(h)
	if err != nil {
		mpw.Close()
		panic(err)
	}
	mpw.Close()
	b, _ := ioutil.ReadAll(buf)
	return int64(len(b)) + filesize
}
<|endoftext|>


<commit_before>package ovhapi

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/mitchellh/mapstructure"
	"github.com/ovh/go-ovh/ovh"

	"github.com/ovh/venom"
	defaultctx "github.com/ovh/venom/context/default"
	"github.com/ovh/venom/executors"
)

// Name of executor
const Name = "ovhapi"

// New returns a new Executor
func New() venom.Executor {
	return &Executor{}
}

// Headers represents header HTTP for Request
type Headers map[string]string

// Executor struct. Json and yaml descriptor are used for json output
type Executor struct {
	Method   string  `json:"method" yaml:"method"`
	NoAuth   bool    `json:"no_auth" yaml:"noAuth"`
	Path     string  `json:"path" yaml:"path"`
	Body     string  `json:"body" yaml:"body"`
	BodyFile string  `json:"bodyfile" yaml:"bodyfile"`
	Headers  Headers `json:"headers" yaml:"headers"`
}

// Result represents a step result. Json and yaml descriptor are used for json output
type Result struct {
	Executor    Executor    `json:"executor,omitempty" yaml:"executor,omitempty"`
	TimeSeconds float64     `json:"timeseconds,omitempty" yaml:"timeseconds,omitempty"`
	TimeHuman   string      `json:"timehuman,omitempty" yaml:"timehuman,omitempty"`
	StatusCode  int         `json:"statuscode,omitempty" yaml:"statuscode,omitempty"`
	Body        string      `json:"body,omitempty" yaml:"body,omitempty"`
	BodyJSON    interface{} `json:"bodyjson,omitempty" yaml:"bodyjson,omitempty"`
	Err         string      `json:"err,omitempty" yaml:"err,omitempty"`
	Headers     Headers     `json:"headers" yaml:"headers"`
}

// ZeroValueResult return an empty implemtation of this executor result
func (Executor) ZeroValueResult() venom.ExecutorResult {
	r, _ := executors.Dump(Result{})
	return r
}

// GetDefaultAssertions return default assertions for this executor
// Optional
func (Executor) GetDefaultAssertions() venom.StepAssertions {
	return venom.StepAssertions{Assertions: []string{"result.statuscode ShouldEqual 200"}}
}

// Run execute TestStep
func (Executor) Run(testCaseContext venom.TestCaseContext, l venom.Logger, step venom.TestStep, workdir string) (venom.ExecutorResult, error) {
	// Get context
	ctx, ok := testCaseContext.(*defaultctx.DefaultTestCaseContext)
	if !ok {
		return nil, fmt.Errorf("ovhapi executor need a default context")
	}

	// transform step to Executor Instance
	var e Executor
	if err := mapstructure.Decode(step, &e); err != nil {
		return nil, err
	}

	// Get context
	var endpoint, applicationKey, applicationSecret, consumerKey string
	var err error
	if endpoint, err = ctx.GetString("endpoint"); err != nil {
		return nil, err
	}
	if !e.NoAuth {
		if applicationKey, err = ctx.GetString("applicationKey"); err != nil {
			return nil, err
		}
		if applicationSecret, err = ctx.GetString("applicationSecret"); err != nil {
			return nil, err
		}
		if consumerKey, err = ctx.GetString("consumerKey"); err != nil {
			return nil, err
		}
	}
	// set default values
	if e.Method == "" {
		e.Method = "GET"
	}

	// init result
	r := Result{Executor: e}

	start := time.Now()
	// prepare ovh api client
	client, err := ovh.NewClient(
		endpoint,
		applicationKey,
		applicationSecret,
		consumerKey,
	)
	if err != nil {
		return nil, err
	}

	if insecure, err := ctx.GetBool("insecureTLS"); err == nil && insecure {
		client.Client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	// get request body from file or from field
	requestBody, err := e.getRequestBody(workdir)
	if err != nil {
		return nil, err
	}

	req, err := client.NewRequest(e.Method, e.Path, requestBody, !e.NoAuth)
	if err != nil {
		return nil, err
	}

	var contextHeader map[string]string
	err = ctx.GetComplex("headers", &contextHeader)
	if err != nil && err != defaultctx.NotFound("headers") {
		l.Warnf("fail to read headers from context : '%s'", err)
	}
	for key, value := range contextHeader {
		req.Header.Set(key, value)
	}

	if e.Headers != nil {
		for key := range e.Headers {
			req.Header.Set(key, e.Headers[key])
		}
	}

	// do api call

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	r.Headers = Headers{}
	for k, v := range resp.Header {
		r.Headers[k] = strings.Join(v, ",")
	}

	res := new(interface{})
	if err = client.UnmarshalResponse(resp, res); err != nil {
		apiError, ok := err.(*ovh.APIError)
		if !ok {
			return nil, err
		}
		r.StatusCode = apiError.Code
		r.Err = apiError.Message
	} else {
		r.StatusCode = 200
	}

	elapsed := time.Since(start)
	r.TimeSeconds = elapsed.Seconds()
	r.TimeHuman = fmt.Sprintf("%s", elapsed)

	// Add response to result body
	if res != nil {
		r.BodyJSON = *res
		bb, err := json.Marshal(res)
		if err != nil {
			return nil, err
		}
		r.Body = string(bb)
	}

	return executors.Dump(r)
}

func (e Executor) getRequestBody(workdir string) (res interface{}, err error) {
	var bytes []byte
	if e.Body != "" {
		bytes = []byte(e.Body)
	} else if e.BodyFile != "" {
		path := filepath.Join(workdir, string(e.BodyFile))
		if _, err = os.Stat(path); !os.IsNotExist(err) {
			bytes, err = ioutil.ReadFile(path)
			if err != nil {
				return nil, err
			}
		}
	}
	if len(bytes) > 0 {
		res = new(interface{})
		err = json.Unmarshal(bytes, res)
		return
	}
	return nil, nil
}
<commit_msg>fix(ovh): headers not found error badly catched (#163)<commit_after>package ovhapi

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/mitchellh/mapstructure"
	"github.com/ovh/go-ovh/ovh"

	"github.com/ovh/venom"
	defaultctx "github.com/ovh/venom/context/default"
	"github.com/ovh/venom/executors"
)

// Name of executor
const Name = "ovhapi"

// New returns a new Executor
func New() venom.Executor {
	return &Executor{}
}

// Headers represents header HTTP for Request
type Headers map[string]string

// Executor struct. Json and yaml descriptor are used for json output
type Executor struct {
	Method   string  `json:"method" yaml:"method"`
	NoAuth   bool    `json:"no_auth" yaml:"noAuth"`
	Path     string  `json:"path" yaml:"path"`
	Body     string  `json:"body" yaml:"body"`
	BodyFile string  `json:"bodyfile" yaml:"bodyfile"`
	Headers  Headers `json:"headers" yaml:"headers"`
}

// Result represents a step result. Json and yaml descriptor are used for json output
type Result struct {
	Executor    Executor    `json:"executor,omitempty" yaml:"executor,omitempty"`
	TimeSeconds float64     `json:"timeseconds,omitempty" yaml:"timeseconds,omitempty"`
	TimeHuman   string      `json:"timehuman,omitempty" yaml:"timehuman,omitempty"`
	StatusCode  int         `json:"statuscode,omitempty" yaml:"statuscode,omitempty"`
	Body        string      `json:"body,omitempty" yaml:"body,omitempty"`
	BodyJSON    interface{} `json:"bodyjson,omitempty" yaml:"bodyjson,omitempty"`
	Err         string      `json:"err,omitempty" yaml:"err,omitempty"`
	Headers     Headers     `json:"headers" yaml:"headers"`
}

// ZeroValueResult return an empty implemtation of this executor result
func (Executor) ZeroValueResult() venom.ExecutorResult {
	r, _ := executors.Dump(Result{})
	return r
}

// GetDefaultAssertions return default assertions for this executor
// Optional
func (Executor) GetDefaultAssertions() venom.StepAssertions {
	return venom.StepAssertions{Assertions: []string{"result.statuscode ShouldEqual 200"}}
}

// Run execute TestStep
func (Executor) Run(testCaseContext venom.TestCaseContext, l venom.Logger, step venom.TestStep, workdir string) (venom.ExecutorResult, error) {
	// Get context
	ctx, ok := testCaseContext.(*defaultctx.DefaultTestCaseContext)
	if !ok {
		return nil, fmt.Errorf("ovhapi executor need a default context")
	}

	// transform step to Executor Instance
	var e Executor
	if err := mapstructure.Decode(step, &e); err != nil {
		return nil, err
	}

	// Get context
	var endpoint, applicationKey, applicationSecret, consumerKey string
	var err error
	if endpoint, err = ctx.GetString("endpoint"); err != nil {
		return nil, err
	}
	if !e.NoAuth {
		if applicationKey, err = ctx.GetString("applicationKey"); err != nil {
			return nil, err
		}
		if applicationSecret, err = ctx.GetString("applicationSecret"); err != nil {
			return nil, err
		}
		if consumerKey, err = ctx.GetString("consumerKey"); err != nil {
			return nil, err
		}
	}
	// set default values
	if e.Method == "" {
		e.Method = "GET"
	}

	// init result
	r := Result{Executor: e}

	start := time.Now()
	// prepare ovh api client
	client, err := ovh.NewClient(
		endpoint,
		applicationKey,
		applicationSecret,
		consumerKey,
	)
	if err != nil {
		return nil, err
	}

	if insecure, err := ctx.GetBool("insecureTLS"); err == nil && insecure {
		client.Client.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}

	// get request body from file or from field
	requestBody, err := e.getRequestBody(workdir)
	if err != nil {
		return nil, err
	}

	req, err := client.NewRequest(e.Method, e.Path, requestBody, !e.NoAuth)
	if err != nil {
		return nil, err
	}

	var contextHeader map[string]string
	err = ctx.GetComplex("headers", &contextHeader)
	if err != nil && err.Error() != defaultctx.NotFound("headers").Error() {
		l.Warnf("fail to read headers from context : '%s'", err)
	}
	for key, value := range contextHeader {
		req.Header.Set(key, value)
	}

	if e.Headers != nil {
		for key := range e.Headers {
			req.Header.Set(key, e.Headers[key])
		}
	}

	// do api call

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	r.Headers = Headers{}
	for k, v := range resp.Header {
		r.Headers[k] = strings.Join(v, ",")
	}

	res := new(interface{})
	if err = client.UnmarshalResponse(resp, res); err != nil {
		apiError, ok := err.(*ovh.APIError)
		if !ok {
			return nil, err
		}
		r.StatusCode = apiError.Code
		r.Err = apiError.Message
	} else {
		r.StatusCode = 200
	}

	elapsed := time.Since(start)
	r.TimeSeconds = elapsed.Seconds()
	r.TimeHuman = fmt.Sprintf("%s", elapsed)

	// Add response to result body
	if res != nil {
		r.BodyJSON = *res
		bb, err := json.Marshal(res)
		if err != nil {
			return nil, err
		}
		r.Body = string(bb)
	}

	return executors.Dump(r)
}

func (e Executor) getRequestBody(workdir string) (res interface{}, err error) {
	var bytes []byte
	if e.Body != "" {
		bytes = []byte(e.Body)
	} else if e.BodyFile != "" {
		path := filepath.Join(workdir, string(e.BodyFile))
		if _, err = os.Stat(path); !os.IsNotExist(err) {
			bytes, err = ioutil.ReadFile(path)
			if err != nil {
				return nil, err
			}
		}
	}
	if len(bytes) > 0 {
		res = new(interface{})
		err = json.Unmarshal(bytes, res)
		return
	}
	return nil, nil
}
<|endoftext|>


<commit_before>// +build linux darwin freebsd netbsd

package util

// ref. https://github.com/opscode/ohai/blob/master/lib/ohai/plugins/linux/filesystem.rb

import (
	"bufio"
	"fmt"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/Songmu/timeout"
	"github.com/mackerelio/golib/logging"
)

// DfStat is disk free statistics from df command.
// Field names are taken from column names of `df -P`
type DfStat struct {
	Name      string
	Blocks    uint64
	Used      uint64
	Available uint64
	Capacity  uint8
	Mounted   string
}

// `df -P` sample:
//  Filesystem     1024-blocks     Used Available Capacity Mounted on
//  /dev/sda1           19734388 16868164 1863772  91% /
//  tmpfs                 517224        0  517224   0% /lib/init/rw
//  udev                  512780       96  512684   1% /dev
//  tmpfs                 517224        4  517220   1% /dev/shm

var dfHeaderPattern = regexp.MustCompile(
	// 1024-blocks or 1k-blocks
	`^Filesystem\s+(?:1024|1[Kk])-block`,
)

var logger = logging.GetLogger("util.filesystem")

var dfOpt []string

func init() {
	switch runtime.GOOS {
	case "darwin":
		dfOpt = []string{"-Pkl"}
	case "freebsd":
		dfOpt = []string{"-Pkt", "noprocfs,devfs,fdescfs,nfs,nullfs,cd9660"}
	case "netbsd":
		dfOpt = []string{"-Pkl"}
	default:
		dfOpt = []string{"-P"}
	}
}

// CollectDfValues collects disk free statistics from df command
func CollectDfValues() ([]*DfStat, error) {
	cmd := exec.Command("df", dfOpt...)
	tio := &timeout.Timeout{
		Cmd:       cmd,
		Duration:  15 * time.Second,
		KillAfter: 5 * time.Second,
	}
	// Ignores exit status in case that df returns exit status 1
	// when the agent does not have permission to access file system info.
	_, stdout, _, err := tio.Run()

	if err != nil {
		logger.Warningf("'df %s' command exited with a non-zero status: '%s'", strings.Join(dfOpt, " "), err)
		return nil, nil
	}
	return parseDfLines(stdout), nil
}

func parseDfLines(out string) []*DfStat {
	lineScanner := bufio.NewScanner(strings.NewReader(out))
	var filesystems []*DfStat
	for lineScanner.Scan() {
		line := lineScanner.Text()
		if dfHeaderPattern.MatchString(line) {
			continue
		}
		dfstat, err := parseDfLine(line)
		if err != nil {
			logger.Warningf(err.Error())
			continue
		}
		// https://github.com/docker/docker/blob/v1.5.0/daemon/graphdriver/devmapper/deviceset.go#L981
		if strings.HasPrefix(dfstat.Name, "/dev/mapper/docker-") {
			continue
		}
		if strings.HasPrefix(dfstat.Name, "/dev/dm-") && strings.Contains(dfstat.Mounted, "devicemapper/mnt") {
			continue
		}
		filesystems = append(filesystems, dfstat)
	}
	return filesystems
}

var dfColumnsPattern = regexp.MustCompile(`^(.+?)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(.+)$`)

func parseDfLine(line string) (*DfStat, error) {
	matches := dfColumnsPattern.FindStringSubmatch(line)
	if matches == nil {
		return nil, fmt.Errorf("failed to parse line: [%s]", line)
	}
	name := matches[1]
	blocks, _ := strconv.ParseUint(matches[2], 0, 64)
	used, _ := strconv.ParseUint(matches[3], 0, 64)
	available, _ := strconv.ParseUint(matches[4], 0, 64)
	capacity, _ := strconv.ParseUint(matches[5], 0, 8)
	mounted := matches[6]

	return &DfStat{
		Name:      name,
		Blocks:    blocks,
		Used:      used,
		Available: available,
		Capacity:  uint8(capacity),
		Mounted:   mounted,
	}, nil
}
<commit_msg>add comments<commit_after>// +build linux darwin freebsd netbsd

package util

// ref. https://github.com/opscode/ohai/blob/master/lib/ohai/plugins/linux/filesystem.rb

import (
	"bufio"
	"fmt"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/Songmu/timeout"
	"github.com/mackerelio/golib/logging"
)

// DfStat is disk free statistics from df command.
// Field names are taken from column names of `df -P`
type DfStat struct {
	Name      string
	Blocks    uint64
	Used      uint64
	Available uint64
	Capacity  uint8
	Mounted   string
}

// `df -P` sample:
//  Filesystem     1024-blocks     Used Available Capacity Mounted on
//  /dev/sda1           19734388 16868164 1863772  91% /
//  tmpfs                 517224        0  517224   0% /lib/init/rw
//  udev                  512780       96  512684   1% /dev
//  tmpfs                 517224        4  517220   1% /dev/shm

var dfHeaderPattern = regexp.MustCompile(
	// 1024-blocks or 1k-blocks
	`^Filesystem\s+(?:1024|1[Kk])-block`,
)

var logger = logging.GetLogger("util.filesystem")

var dfOpt []string

func init() {
	switch runtime.GOOS {
	case "darwin":
		dfOpt = []string{"-Pkl"}
	case "freebsd":
		dfOpt = []string{"-Pkt", "noprocfs,devfs,fdescfs,nfs,nullfs,cd9660"}
	case "netbsd":
		dfOpt = []string{"-Pkl"}
	default:
		dfOpt = []string{"-P"}
	}
}

// CollectDfValues collects disk free statistics from df command
func CollectDfValues() ([]*DfStat, error) {
	cmd := exec.Command("df", dfOpt...)
	tio := &timeout.Timeout{
		Cmd:       cmd,
		Duration:  15 * time.Second,
		KillAfter: 5 * time.Second,
	}
	// Ignores exit status in case that df returns exit status 1
	// when the agent does not have permission to access file system info.
	_, stdout, _, err := tio.Run()

	if err != nil {
		logger.Warningf("'df %s' command exited with a non-zero status: '%s'", strings.Join(dfOpt, " "), err)
		return nil, nil
	}
	return parseDfLines(stdout), nil
}

func parseDfLines(out string) []*DfStat {
	lineScanner := bufio.NewScanner(strings.NewReader(out))
	var filesystems []*DfStat
	for lineScanner.Scan() {
		line := lineScanner.Text()
		if dfHeaderPattern.MatchString(line) {
			continue
		}
		dfstat, err := parseDfLine(line)
		if err != nil {
			logger.Warningf(err.Error())
			continue
		}
		// https://github.com/docker/docker/blob/v1.5.0/daemon/graphdriver/devmapper/deviceset.go#L981
		if strings.HasPrefix(dfstat.Name, "/dev/mapper/docker-") {
			continue
		}
		// https://debbugs.gnu.org/cgi/bugreport.cgi?bug=10363
		// http://git.savannah.gnu.org/gitweb/?p=coreutils.git;a=commit;h=1e18d8416f9ef43bf08982cabe54220587061a08
		// coreutils >= 8.15
		if strings.HasPrefix(dfstat.Name, "/dev/dm-") && strings.Contains(dfstat.Mounted, "devicemapper/mnt") {
			continue
		}
		filesystems = append(filesystems, dfstat)
	}
	return filesystems
}

var dfColumnsPattern = regexp.MustCompile(`^(.+?)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(.+)$`)

func parseDfLine(line string) (*DfStat, error) {
	matches := dfColumnsPattern.FindStringSubmatch(line)
	if matches == nil {
		return nil, fmt.Errorf("failed to parse line: [%s]", line)
	}
	name := matches[1]
	blocks, _ := strconv.ParseUint(matches[2], 0, 64)
	used, _ := strconv.ParseUint(matches[3], 0, 64)
	available, _ := strconv.ParseUint(matches[4], 0, 64)
	capacity, _ := strconv.ParseUint(matches[5], 0, 8)
	mounted := matches[6]

	return &DfStat{
		Name:      name,
		Blocks:    blocks,
		Used:      used,
		Available: available,
		Capacity:  uint8(capacity),
		Mounted:   mounted,
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"net/http"
	"encoding/json"
	"io/ioutil"
	"fmt"
	"sync"
	"time"
	"os"
	"net/http/httputil"
	"github.com/gin-gonic/gin"
)

const automl_index = 3

var wg sync.WaitGroup

// you can't use const for structs, so this is the closest thing we can get for this value
var default_api_response = APIResponse{
	Feedback: "Thank you for your response.",
	Feedback_type: "autoML",
	Optimal: true,
}

// TODO: This is a temporary replacement `http` that allows us to bypass SSL security checks
var client = &http.Client {
	Transport: &http.Transport {
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	},
}

func GetFeedbackHistoryUrl() (string) {
	lms_domain := GetLMSDomain()
	return fmt.Sprintf("%s/api/v1/feedback_histories/batch.json", lms_domain)
}

func GetLMSDomain() (string) {
	var maybe_domain = os.Getenv("lms_domain")

	var lms_domain = "https://www.quill.org"

	if len(maybe_domain) > 0 {
		lms_domain = maybe_domain
	}
	return lms_domain
}

func AssembleUrls() ([8]string) {
	lms_domain := GetLMSDomain()

	var (
		automl_api = fmt.Sprintf("%s/api/v1/comprehension/feedback/automl.json", lms_domain)
		plagiarism_api = fmt.Sprintf("%s/api/v1/comprehension/feedback/plagiarism.json", lms_domain)
		sentence_structure_regex_api = fmt.Sprintf("%s/api/v1/comprehension/feedback/regex/rules-based-1.json", lms_domain)
		post_topic_regex_api         = fmt.Sprintf("%s/api/v1/comprehension/feedback/regex/rules-based-2.json", lms_domain)
		typo_regex_api               = fmt.Sprintf("%s/api/v1/comprehension/feedback/regex/rules-based-3.json", lms_domain)
		spell_check_bing             = fmt.Sprintf("%s/api/v1/comprehension/feedback/spelling.json", lms_domain)

		grammar_check_api = "https://grammar-api.ue.r.appspot.com"
		opinion_check_api = "https://opinion-api.ue.r.appspot.com/"
	)

	var urls = [...]string{
		sentence_structure_regex_api,
		opinion_check_api,
		plagiarism_api,
		automl_api,
		post_topic_regex_api,
		grammar_check_api,
		spell_check_bing,
		typo_regex_api,
	}

	return urls
}

func Endpoint(context *gin.Context) {
	urls := AssembleUrls()

	requestDump, err := httputil.DumpRequest(context.Request, true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(string(requestDump))

	request_body, err := ioutil.ReadAll(context.Request.Body)
	if err != nil {
		//TODO make this response in the same format maybe?
		context.String(http.StatusInternalServerError, err.Error())
		return
	}

	results := map[int]InternalAPIResponse{}

	channel := make(chan InternalAPIResponse)

	for priority, url := range urls {
		go getAPIResponse(url, priority, request_body, channel)
	}

	var returnable_result APIResponse

	for response := range channel {
		results[response.Priority] = response
		return_index, finished := processResults(results, len(urls))

		if finished {
			// If processResults reports that it's "finished", but the APIResponse at
			// that index is flagged as an Error, we'll need to return a default response of
			// some sort
			// This should really only happen when there's no Semantic Feedback available,
			// even for success cases, which means that if everything is working correctly,
			// this only comes up during the initial Turking process
			if !results[return_index].Error {
				returnable_result = results[return_index].APIResponse
			} else {
				returnable_result = default_api_response
			}
			break
		}
	}

	// TODO make this a purely async task instead of coroutine that waits to finish
	wg.Add(1)

	var request_object APIRequest
	// TODO convert the 'feedback' bytes and combine with incoming_params bytes
	// instead of transforming from bytes to object, combining, and then converting back to bytes
	if err := json.NewDecoder(bytes.NewReader(request_body)).Decode(&request_object); err != nil {
		return
	}

	go recordFeedback(request_object, returnable_result, GetFeedbackHistoryUrl())

	context.Header("Access-Control-Allow-Origin", "*")
	context.Header("Content-Type", "application/json")
	context.JSON(200, returnable_result)

	wg.Wait()
}
// returns a typle of results index and that should be returned.
func processResults(results map[int]InternalAPIResponse, length int) (int, bool) {
	for i := 0; i < len(results); i++ {
		result, has_key := results[i]
		if !has_key {
			return 0, false
		}

		if !result.Error && !result.APIResponse.Optimal {
			return i, true
		}
	}

	all_correct := len(results) >= length

	return automl_index, all_correct
}

func getAPIResponse(url string, priority int, json_params [] byte, c chan InternalAPIResponse) {
	// response_json, err := http.Post(url, "application/json", bytes.NewReader(json_params))

	// TODO For now, just swallow any errors from this, but we'd want to report errors.
	// TODO: Replace "client" with "http" when we remove the segment above
	response_json, err := client.Post(url, "application/json",  bytes.NewReader(json_params))

	if err != nil {
		c <- InternalAPIResponse{Priority: priority, Error: true, APIResponse: APIResponse{Feedback: "There was an error hitting the API", Feedback_type: "API Error", Optimal: false}}
		return
	}

	var result APIResponse

	if err := json.NewDecoder(response_json.Body).Decode(&result); err != nil {
		// TODO might want to think about what this should be.
		c <- InternalAPIResponse{Priority: priority, Error: true, APIResponse: APIResponse{Feedback: "There was an JSON error" + err.Error(), Feedback_type: "API Error", Labels: url, Optimal: false}}
		return
	}

	c <- InternalAPIResponse{Priority: priority, Error: false, APIResponse: result}
}

func identifyUsedFeedbackIndex(feedbacks map[int]InternalAPIResponse) int {
	for i := 0; i < len(feedbacks); i++ {
		feedback := feedbacks[i]
		if !feedback.Error && !feedback.APIResponse.Optimal {
			return i
		}
	}
	// If none of the feedbacks are non-optimal, check to see if automl is feedback
	// is not optimal and not in error.  Because if it so, that's the feedback we'll use
	if !feedbacks[automl_index].Error && feedbacks[automl_index].APIResponse.Optimal {
		return automl_index
	}
	// We use -1 as the return value if we couldn't find an index since
	// it should correspond to no index
	return -1
}

func buildFeedbackHistory(request_object APIRequest, feedback APIResponse, used bool, time_received time.Time) FeedbackHistory {
	return FeedbackHistory{
		Feedback_session_uid: request_object.Session_id,
		Prompt_id: request_object.Prompt_id,
		Concept_uid: feedback.Concept_uid,
		Attempt: request_object.Attempt,
		Entry: request_object.Entry,
		Feedback_text: feedback.Feedback,
		Feedback_type: feedback.Feedback_type,
		Optimal: feedback.Optimal,
		Used: used,
		Time: time_received,
		Rule_uid: feedback.Rule_uid,
		Metadata: FeedbackHistoryMetadata{
			Highlight: feedback.Highlight,
			Labels: feedback.Labels,
			Response_id: feedback.Response_id,
		},
	}
}

func recordFeedback(incoming_params APIRequest, feedback APIResponse, feedback_history_url string) {
	defer wg.Done() // mark task as done in WaitGroup on return

	history := buildFeedbackHistory(incoming_params, feedback, true, time.Now())

	history_json, _ := json.Marshal(history)

	// TODO For now, just swallow any errors from this, but we'd want to report errors.
	// TODO: Replace "client" with "http" when we remove the segment above
	client.Post(feedback_history_url, "application/json",  bytes.NewBuffer(history_json))
}

type APIRequest struct {
	Entry string `json:"entry"`
	Prompt_text string `json:"prompt_text"`
	Prompt_id int `json:"prompt_id"`
	Session_id string `json:"session_id"`
	Attempt int `json:"attempt"`
}

type APIResponse struct {
	Concept_uid string `json:"concept_uid"`
	Feedback string `json:"feedback"`
	Feedback_type string `json:"feedback_type"`
	Optimal bool `json:"optimal"`
	Response_id string `json:"response_id"`
	Highlight []Highlight `json:"highlight"`
	Labels string `json:"labels,omitempty"`
	Rule_uid string `json:"rule_uid"`
}

type Highlight struct {
	Type string `json:"type"`
	Id int `json:"id,omitempty"`
	Text string `json:"text"`
	Category string `json:"category"`
	Character int `json:"character,omitempty"`
}

type InternalAPIResponse struct {
	Priority int
	APIResponse APIResponse
	Error bool
}

type FeedbackHistoryMetadata struct {
	Highlight []Highlight `json:"highlight"`
	Labels string `json:"labels,omitempty"`
	Response_id string `json:"response_id"`
}

type FeedbackHistory struct {
	Feedback_session_uid string `json:"feedback_session_uid"`
	Prompt_id int `json:"prompt_id"`
	Concept_uid string `json:"concept_uid"`
	Attempt int `json:"attempt"`
	Entry string `json:"entry"`
	Feedback_text string `json:"feedback_text"`
	Feedback_type string `json:"feedback_type"`
	Optimal bool `json:"optimal"`
	Used bool `json:"used"`
	Time time.Time `json:"time"`
	Metadata FeedbackHistoryMetadata `json:"metadata"`
	Rule_uid string `json:"rule_uid"`
}
<commit_msg>Use non-batch FeedbackHistory creation endpoint (#7782)<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"net/http"
	"encoding/json"
	"io/ioutil"
	"fmt"
	"sync"
	"time"
	"os"
	"net/http/httputil"
	"github.com/gin-gonic/gin"
)

const automl_index = 3

var wg sync.WaitGroup

// you can't use const for structs, so this is the closest thing we can get for this value
var default_api_response = APIResponse{
	Feedback: "Thank you for your response.",
	Feedback_type: "autoML",
	Optimal: true,
}

// TODO: This is a temporary replacement `http` that allows us to bypass SSL security checks
var client = &http.Client {
	Transport: &http.Transport {
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	},
}

func GetFeedbackHistoryUrl() (string) {
	lms_domain := GetLMSDomain()
	return fmt.Sprintf("%s/api/v1/feedback_histories.json", lms_domain)
}

func GetLMSDomain() (string) {
	var maybe_domain = os.Getenv("lms_domain")

	var lms_domain = "https://www.quill.org"

	if len(maybe_domain) > 0 {
		lms_domain = maybe_domain
	}
	return lms_domain
}

func AssembleUrls() ([8]string) {
	lms_domain := GetLMSDomain()

	var (
		automl_api = fmt.Sprintf("%s/api/v1/comprehension/feedback/automl.json", lms_domain)
		plagiarism_api = fmt.Sprintf("%s/api/v1/comprehension/feedback/plagiarism.json", lms_domain)
		sentence_structure_regex_api = fmt.Sprintf("%s/api/v1/comprehension/feedback/regex/rules-based-1.json", lms_domain)
		post_topic_regex_api         = fmt.Sprintf("%s/api/v1/comprehension/feedback/regex/rules-based-2.json", lms_domain)
		typo_regex_api               = fmt.Sprintf("%s/api/v1/comprehension/feedback/regex/rules-based-3.json", lms_domain)
		spell_check_bing             = fmt.Sprintf("%s/api/v1/comprehension/feedback/spelling.json", lms_domain)

		grammar_check_api = "https://grammar-api.ue.r.appspot.com"
		opinion_check_api = "https://opinion-api.ue.r.appspot.com/"
	)

	var urls = [...]string{
		sentence_structure_regex_api,
		opinion_check_api,
		plagiarism_api,
		automl_api,
		post_topic_regex_api,
		grammar_check_api,
		spell_check_bing,
		typo_regex_api,
	}

	return urls
}

func Endpoint(context *gin.Context) {
	urls := AssembleUrls()

	requestDump, err := httputil.DumpRequest(context.Request, true)
	if err != nil {
		fmt.Println(err)
	}
	fmt.Println(string(requestDump))

	request_body, err := ioutil.ReadAll(context.Request.Body)
	if err != nil {
		//TODO make this response in the same format maybe?
		context.String(http.StatusInternalServerError, err.Error())
		return
	}

	results := map[int]InternalAPIResponse{}

	channel := make(chan InternalAPIResponse)

	for priority, url := range urls {
		go getAPIResponse(url, priority, request_body, channel)
	}

	var returnable_result APIResponse

	for response := range channel {
		results[response.Priority] = response
		return_index, finished := processResults(results, len(urls))

		if finished {
			// If processResults reports that it's "finished", but the APIResponse at
			// that index is flagged as an Error, we'll need to return a default response of
			// some sort
			// This should really only happen when there's no Semantic Feedback available,
			// even for success cases, which means that if everything is working correctly,
			// this only comes up during the initial Turking process
			if !results[return_index].Error {
				returnable_result = results[return_index].APIResponse
			} else {
				returnable_result = default_api_response
			}
			break
		}
	}

	// TODO make this a purely async task instead of coroutine that waits to finish
	wg.Add(1)

	var request_object APIRequest
	// TODO convert the 'feedback' bytes and combine with incoming_params bytes
	// instead of transforming from bytes to object, combining, and then converting back to bytes
	if err := json.NewDecoder(bytes.NewReader(request_body)).Decode(&request_object); err != nil {
		return
	}

	go recordFeedback(request_object, returnable_result, GetFeedbackHistoryUrl())

	context.Header("Access-Control-Allow-Origin", "*")
	context.Header("Content-Type", "application/json")
	context.JSON(200, returnable_result)

	wg.Wait()
}
// returns a typle of results index and that should be returned.
func processResults(results map[int]InternalAPIResponse, length int) (int, bool) {
	for i := 0; i < len(results); i++ {
		result, has_key := results[i]
		if !has_key {
			return 0, false
		}

		if !result.Error && !result.APIResponse.Optimal {
			return i, true
		}
	}

	all_correct := len(results) >= length

	return automl_index, all_correct
}

func getAPIResponse(url string, priority int, json_params [] byte, c chan InternalAPIResponse) {
	// response_json, err := http.Post(url, "application/json", bytes.NewReader(json_params))

	// TODO For now, just swallow any errors from this, but we'd want to report errors.
	// TODO: Replace "client" with "http" when we remove the segment above
	response_json, err := client.Post(url, "application/json",  bytes.NewReader(json_params))

	if err != nil {
		c <- InternalAPIResponse{Priority: priority, Error: true, APIResponse: APIResponse{Feedback: "There was an error hitting the API", Feedback_type: "API Error", Optimal: false}}
		return
	}

	var result APIResponse

	if err := json.NewDecoder(response_json.Body).Decode(&result); err != nil {
		// TODO might want to think about what this should be.
		c <- InternalAPIResponse{Priority: priority, Error: true, APIResponse: APIResponse{Feedback: "There was an JSON error" + err.Error(), Feedback_type: "API Error", Labels: url, Optimal: false}}
		return
	}

	c <- InternalAPIResponse{Priority: priority, Error: false, APIResponse: result}
}

func identifyUsedFeedbackIndex(feedbacks map[int]InternalAPIResponse) int {
	for i := 0; i < len(feedbacks); i++ {
		feedback := feedbacks[i]
		if !feedback.Error && !feedback.APIResponse.Optimal {
			return i
		}
	}
	// If none of the feedbacks are non-optimal, check to see if automl is feedback
	// is not optimal and not in error.  Because if it so, that's the feedback we'll use
	if !feedbacks[automl_index].Error && feedbacks[automl_index].APIResponse.Optimal {
		return automl_index
	}
	// We use -1 as the return value if we couldn't find an index since
	// it should correspond to no index
	return -1
}

func buildFeedbackHistory(request_object APIRequest, feedback APIResponse, used bool, time_received time.Time) FeedbackHistory {
	return FeedbackHistory{
		Feedback_session_uid: request_object.Session_id,
		Prompt_id: request_object.Prompt_id,
		Concept_uid: feedback.Concept_uid,
		Attempt: request_object.Attempt,
		Entry: request_object.Entry,
		Feedback_text: feedback.Feedback,
		Feedback_type: feedback.Feedback_type,
		Optimal: feedback.Optimal,
		Used: used,
		Time: time_received,
		Rule_uid: feedback.Rule_uid,
		Metadata: FeedbackHistoryMetadata{
			Highlight: feedback.Highlight,
			Labels: feedback.Labels,
			Response_id: feedback.Response_id,
		},
	}
}

func recordFeedback(incoming_params APIRequest, feedback APIResponse, feedback_history_url string) {
	defer wg.Done() // mark task as done in WaitGroup on return

	history := buildFeedbackHistory(incoming_params, feedback, true, time.Now())

	history_json, _ := json.Marshal(history)

	// TODO For now, just swallow any errors from this, but we'd want to report errors.
	// TODO: Replace "client" with "http" when we remove the segment above
	client.Post(feedback_history_url, "application/json",  bytes.NewBuffer(history_json))
}

type APIRequest struct {
	Entry string `json:"entry"`
	Prompt_text string `json:"prompt_text"`
	Prompt_id int `json:"prompt_id"`
	Session_id string `json:"session_id"`
	Attempt int `json:"attempt"`
}

type APIResponse struct {
	Concept_uid string `json:"concept_uid"`
	Feedback string `json:"feedback"`
	Feedback_type string `json:"feedback_type"`
	Optimal bool `json:"optimal"`
	Response_id string `json:"response_id"`
	Highlight []Highlight `json:"highlight"`
	Labels string `json:"labels,omitempty"`
	Rule_uid string `json:"rule_uid"`
}

type Highlight struct {
	Type string `json:"type"`
	Id int `json:"id,omitempty"`
	Text string `json:"text"`
	Category string `json:"category"`
	Character int `json:"character,omitempty"`
}

type InternalAPIResponse struct {
	Priority int
	APIResponse APIResponse
	Error bool
}

type FeedbackHistoryMetadata struct {
	Highlight []Highlight `json:"highlight"`
	Labels string `json:"labels,omitempty"`
	Response_id string `json:"response_id"`
}

type FeedbackHistory struct {
	Feedback_session_uid string `json:"feedback_session_uid"`
	Prompt_id int `json:"prompt_id"`
	Concept_uid string `json:"concept_uid"`
	Attempt int `json:"attempt"`
	Entry string `json:"entry"`
	Feedback_text string `json:"feedback_text"`
	Feedback_type string `json:"feedback_type"`
	Optimal bool `json:"optimal"`
	Used bool `json:"used"`
	Time time.Time `json:"time"`
	Metadata FeedbackHistoryMetadata `json:"metadata"`
	Rule_uid string `json:"rule_uid"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/kyokomi/emoji")

func main() {
	fmt.Println("Hello Wolrd Emoji!")

	emoji.Println("@{:beer:} Beer!!!")

	pizzaMessage := emoji.Sprint("I like @{:pizza:}!!")
	fmt.Println(pizzaMessage)

	dessert := emoji.Sprintf("%s @{:custard:} @{:womens:}.", "This is")
	fmt.Println(dessert)
}

<commit_msg>example: fix v0.0.2<commit_after>package main

import (
	"fmt"

	"github.com/kyokomi/emoji"
)

func main() {
	fmt.Println("Hello Wolrd Emoji!")

	emoji.Println(":beer: Beer!!!")

	pizzaMessage := emoji.Sprint("I like a :pizza: and :sushi:!!")
	fmt.Println(pizzaMessage)
}
<|endoftext|>


<commit_before>package smaz

import (
	"bufio"
	"bytes"
	"os"
	"testing"
)

var antirezTestStrings = []string{"",
	"This is a small string",
	"foobar",
	"the end",
	"not-a-g00d-Exampl333",
	"Smaz is a simple compression library",
	"Nothing is more difficult, and therefore more precious, than to be able to decide",
	"this is an example of what works very well with smaz",
	"1000 numbers 2000 will 10 20 30 compress very little",
	"and now a few italian sentences:",
	"Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura",
	"Mi illumino di immenso",
	"L'autore di questa libreria vive in Sicilia",
	"try it against urls",
	"http://google.com",
	"http://programming.reddit.com",
	"http://github.com/antirez/smaz/tree/master",
	"/media/hdb1/music/Alben/The Bla",
}

func TestCorrectness(t *testing.T) {
	// Set up our slice of test strings.
	inputs := make([][]byte, 0)
	for _, s := range antirezTestStrings {
		inputs = append(inputs, []byte(s))
	}
	// An array with every possible byte value in it.
	allBytes := make([]byte, 256)
	for i := 0; i < 256; i++ {
		allBytes[i] = byte(i)
	}
	inputs = append(inputs, allBytes)
	// A long array of all 0s (the longest continuous string that can be represented is 256; any longer than
	// this and the compressor will need to split it into chunks)
	allZeroes := make([]byte, 300)
	for i := 0; i < 300; i++ {
		allZeroes[i] = byte(0)
	}
	inputs = append(inputs, allZeroes)

	for _, input := range inputs {
		compressed := Compress(input)
		decompressed, err := Decompress(compressed)
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(input, decompressed) {
			t.Fatal("want %q after decompression; got %q\n", input, decompressed)
		}

		if len(input) > 1 && len(input) < 50 {
			compressionLevel := 100 - ((100.0 * len(compressed)) / len(input))
			if compressionLevel < 0 {
				t.Logf("%q enlarged by %d%%\n", input, -compressionLevel)
			} else {
				t.Logf("%q compressed by %d%%\n", input, compressionLevel)
			}
		}
	}
}

func loadTestData(t testing.TB) [][]byte {
	f, err := os.Open("./testdata/pg5200.txt")
	if err != nil {
		t.Fatal(err)
	}

	var lines [][]byte
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		lines = append(lines, []byte(scanner.Text())) // Note that .Bytes() would require us to manually copy
	}
	return lines
}

func BenchmarkCompression(b *testing.B) {
	b.StopTimer()
	inputs := loadTestData(b)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		for _, input := range inputs {
			Compress(input)
		}
	}
}

func BenchmarkDecompression(b *testing.B) {
	b.StopTimer()
	inputs := loadTestData(b)
	compressedStrings := make([][]byte, len(inputs))
	for i, input := range inputs {
		compressedStrings[i] = Compress(input)
	}
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		for _, compressed := range compressedStrings {
			Decompress(compressed)
		}
	}
}
<commit_msg>Show MB/s in the benchmarks<commit_after>package smaz

import (
	"bufio"
	"bytes"
	"os"
	"testing"
)

var antirezTestStrings = []string{"",
	"This is a small string",
	"foobar",
	"the end",
	"not-a-g00d-Exampl333",
	"Smaz is a simple compression library",
	"Nothing is more difficult, and therefore more precious, than to be able to decide",
	"this is an example of what works very well with smaz",
	"1000 numbers 2000 will 10 20 30 compress very little",
	"and now a few italian sentences:",
	"Nel mezzo del cammin di nostra vita, mi ritrovai in una selva oscura",
	"Mi illumino di immenso",
	"L'autore di questa libreria vive in Sicilia",
	"try it against urls",
	"http://google.com",
	"http://programming.reddit.com",
	"http://github.com/antirez/smaz/tree/master",
	"/media/hdb1/music/Alben/The Bla",
}

func TestCorrectness(t *testing.T) {
	// Set up our slice of test strings.
	inputs := make([][]byte, 0)
	for _, s := range antirezTestStrings {
		inputs = append(inputs, []byte(s))
	}
	// An array with every possible byte value in it.
	allBytes := make([]byte, 256)
	for i := 0; i < 256; i++ {
		allBytes[i] = byte(i)
	}
	inputs = append(inputs, allBytes)
	// A long array of all 0s (the longest continuous string that can be represented is 256; any longer than
	// this and the compressor will need to split it into chunks)
	allZeroes := make([]byte, 300)
	for i := 0; i < 300; i++ {
		allZeroes[i] = byte(0)
	}
	inputs = append(inputs, allZeroes)

	for _, input := range inputs {
		compressed := Compress(input)
		decompressed, err := Decompress(compressed)
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(input, decompressed) {
			t.Fatal("want %q after decompression; got %q\n", input, decompressed)
		}

		if len(input) > 1 && len(input) < 50 {
			compressionLevel := 100 - ((100.0 * len(compressed)) / len(input))
			if compressionLevel < 0 {
				t.Logf("%q enlarged by %d%%\n", input, -compressionLevel)
			} else {
				t.Logf("%q compressed by %d%%\n", input, compressionLevel)
			}
		}
	}
}

func loadTestData(t testing.TB) ([][]byte, int64) {
	f, err := os.Open("./testdata/pg5200.txt")
	if err != nil {
		t.Fatal(err)
	}

	var lines [][]byte
	var n int64
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		input := []byte(scanner.Text()) // Note that .Bytes() would require us to manually copy
		lines = append(lines, input)
		n += int64(len(input))
	}
	return lines, n
}

func BenchmarkCompression(b *testing.B) {
	b.StopTimer()
	inputs, n := loadTestData(b)
	b.SetBytes(n)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		for _, input := range inputs {
			Compress(input)
		}
	}
}

func BenchmarkDecompression(b *testing.B) {
	b.StopTimer()
	inputs, _ := loadTestData(b)
	compressedStrings := make([][]byte, len(inputs))
	var n int64
	for i, input := range inputs {
		compressed := Compress(input)
		compressedStrings[i] = compressed
		n += int64(len(compressed))
	}
	b.SetBytes(n)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		for _, compressed := range compressedStrings {
			Decompress(compressed)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ziutek/gogammu"
	"github.com/ziutek/mymysql/autorc"
	_ "github.com/ziutek/mymysql/native"
	"io"
	"log"
	"strings"
	"time"
)

type SMSd struct {
	sm *gammu.StateMachine
	db *autorc.Conn

	end, newMsg chan event
	wait        bool

	sqlNumToId string

	stmtOutboxGet, stmtRecipGet, stmtRecipSent, stmtInboxPut,
	stmtRecipReport, stmtOutboxDel, stmtNumToId autorc.Stmt

	filter *Filter
}

func NewSMSd(db *autorc.Conn, numId, filter string) (*SMSd, error) {
	var err error
	smsd := new(SMSd)
	smsd.sm, err = gammu.NewStateMachine("")
	if err != nil {
		return nil, err
	}
	if filter != "" {
		smsd.filter, err = NewFilter(filter)
		if err != nil {
			return nil, err
		}
	}
	smsd.db = db
	smsd.db.Register(setNames)
	smsd.db.Register(createOutbox)
	smsd.db.Register(createRecipients)
	smsd.db.Register(createInbox)
	smsd.db.Register(setLocPrefix)
	smsd.sqlNumToId = numId
	smsd.end = make(chan event)
	smsd.newMsg = make(chan event)
	return smsd, nil
}

// Selects messages from Outbox that have any recipient without sent flag set
const outboxGet = `SELECT
	o.id, o.src, o.report, o.body
FROM
	` + outboxTable + ` o
WHERE
	EXISTS (SELECT * FROM ` + recipientsTable + ` p WHERE p.msgId=o.id && !p.sent)
`

// Selects all recipients without sent flag set for givem msgId
const recipientsGet = `SELECT
	id, number
FROM
	` + recipientsTable + `
WHERE
	!sent && msgId=?
`

const recipientsSent = "UPDATE " + recipientsTable + " SET sent=? WHERE id=?"

// Send messages from Outbox
func (smsd *SMSd) sendMessages() (gammuErr bool) {
	if !prepareOnce(smsd.db, &smsd.stmtOutboxGet, outboxGet) {
		return
	}
	if !prepareOnce(smsd.db, &smsd.stmtRecipGet, recipientsGet) {
		return
	}
	if !prepareOnce(smsd.db, &smsd.stmtRecipSent, recipientsSent) {
		return
	}
	msgs, res, err := smsd.stmtOutboxGet.Exec()
	if err != nil {
		log.Println("Can't get a messages from Outbox:", err)
		return
	}
	colMid := res.Map("id")
	colReport := res.Map("report")
	colBody := res.Map("body")
	for _, msg := range msgs {
		mid := msg.Uint(colMid)
		report := msg.Bool(colReport)
		body := msg.Str(colBody)

		recipients, res, err := smsd.stmtRecipGet.Exec(mid)
		if err != nil {
			log.Printf("Can't get a phone number for msg #%d: %s", mid, err)
			return
		}
		colPid := res.Map("id")
		colNum := res.Map("number")
		for _, p := range recipients {
			pid := p.Uint(colPid)
			num := p.Str(colNum)
			if !checkNumber(num) {
				continue
			}
			if isGammuError(smsd.sm.SendLongSMS(num, body, report)) {
				// Phone error or bad values
				gammuErr = true
			}
			_, _, err = smsd.stmtRecipSent.Exec(time.Now(), pid)
			if err != nil {
				log.Printf(
					"Can't mark a msg/recip #%d/#%d as sent: %s",
					mid, pid, err,
				)
				return
			}
		}
	}
	return
}

const inboxPut = `INSERT
	` + inboxTable + `
SET
	time=?,
	number=?,
	srcId=?,
	body=?
`

const recipReport = `UPDATE
	` + recipientsTable + `
SET
	report=?
WHERE
	!report && (number=? || concat(@localPrefix, number)=?)
ORDER BY
	abs(timediff(?, sent))
LIMIT 1`

type Msg struct {
	Time   time.Time
	Number string
	SrcId  uint
	Body   string
}

func (smsd *SMSd) recvMessages() (gammuErr bool) {
	if !prepareOnce(smsd.db, &smsd.stmtInboxPut, inboxPut) {
		return
	}
	if !prepareOnce(smsd.db, &smsd.stmtRecipReport, recipReport) {
		return
	}
	if smsd.sqlNumToId != "" {
		if !prepareOnce(smsd.db, &smsd.stmtNumToId, smsd.sqlNumToId) {
			return
		}
	}

	var msg Msg
	smsd.stmtInboxPut.Bind(&msg)

	for {
		sms, err := smsd.sm.GetSMS()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Printf("Can't get message from phone: %s", err)
			return true
		}
		if sms.Report {
			// Find a message and sender in Outbox and mark it
			m := strings.TrimSpace(sms.Body)
			if strings.ToLower(m) == "delivered" {
				_, _, err = smsd.stmtRecipReport.Exec(
					sms.SMSCTime, sms.Number, sms.Number, sms.Time,
				)
				if err != nil {
					log.Printf(
						"Can't mark recipient %s as reported: %s",
						sms.Number, err,
					)
					return
				}
			}
		} else {
			// Save a message in Inbox
			msg.Time = sms.Time
			msg.Number = sms.Number
			msg.SrcId = 0
			msg.Body = sms.Body
			//log.Printf("Odebrano: %+v", msg)
			if smsd.stmtNumToId.Raw != nil {
				id, _, err := smsd.stmtNumToId.ExecFirst(msg.Number)
				if err != nil {
					log.Printf(
						"Can't get srcId for number %s: %s",
						sms.Number, err,
					)
					return
				}
				if id != nil {
					msg.SrcId, err = id.UintErr(0)
					if err != nil {
						log.Printf("Bad srcId '%v': %s", id[0], err)
						return
					}
				}
			}
			if f := smsd.filter; f != nil {
				accept, err := f.Filter(&msg)
				if err != nil {
					log.Printf("Filter error: %s", err)
				} else if !accept {
					// Drop this message
					continue
				}
			}
			_, _, err = smsd.stmtInboxPut.Exec() // using msg
			if err != nil {
				log.Printf(
					"Can't insert message from %s into Inbox: %s",
					sms.Number, err,
				)
				return
			}
		}
	}
	return
}

const outboxDel = `DELETE FROM
	o
USING
	SMSd_Outbox o
WHERE
	o.del && !EXISTS(
		SELECT
			*
		FROM
			SMSd_Recipients r
		WHERE
			r.msgId = o.id && (!r.sent || o.report && !r.report)
	)
`

func (smsd *SMSd) delMessages() {
	if !prepareOnce(smsd.db, &smsd.stmtOutboxDel, outboxDel) {
		return
	}
	_, _, err := smsd.stmtOutboxDel.Exec()
	if err != nil {
		log.Println("Can't delete messages:", err)
		return
	}
}

func (smsd *SMSd) sendRecvDel(send bool) {
	var gammuErr bool
	if !smsd.sm.IsConnected() {
		if isGammuError(smsd.sm.Connect()) {
			return
		}
	}
	if send {
		gammuErr = smsd.sendMessages()
	}
	gammuErr = smsd.recvMessages() || gammuErr
	if send {
		smsd.delMessages()
	}
	if gammuErr && smsd.sm.IsConnected() {
		smsd.sm.Disconnect()
	}
}

func (smsd *SMSd) loop() {
	send := true
	for {
		smsd.sendRecvDel(send)
		// Wait for some event or timeout
		select {
		case <-smsd.end:
			return
		case <-smsd.newMsg:
			send = true
		case <-time.After(17 * time.Second):
			// send and del two times less frequently than recv
			send = !send
		}
	}
}

func (smsd *SMSd) Start() {
	go smsd.loop()
}

func (smsd *SMSd) Stop() {
	smsd.end <- event{}
}

func (smsd *SMSd) NewMsg() {
	select {
	case smsd.newMsg <- event{}:
	default:
	}
}
<commit_msg>Add buffering to newMsg channel<commit_after>package main

import (
	"github.com/ziutek/gogammu"
	"github.com/ziutek/mymysql/autorc"
	_ "github.com/ziutek/mymysql/native"
	"io"
	"log"
	"strings"
	"time"
)

type SMSd struct {
	sm *gammu.StateMachine
	db *autorc.Conn

	end, newMsg chan event
	wait        bool

	sqlNumToId string

	stmtOutboxGet, stmtRecipGet, stmtRecipSent, stmtInboxPut,
	stmtRecipReport, stmtOutboxDel, stmtNumToId autorc.Stmt

	filter *Filter
}

func NewSMSd(db *autorc.Conn, numId, filter string) (*SMSd, error) {
	var err error
	smsd := new(SMSd)
	smsd.sm, err = gammu.NewStateMachine("")
	if err != nil {
		return nil, err
	}
	if filter != "" {
		smsd.filter, err = NewFilter(filter)
		if err != nil {
			return nil, err
		}
	}
	smsd.db = db
	smsd.db.Register(setNames)
	smsd.db.Register(createOutbox)
	smsd.db.Register(createRecipients)
	smsd.db.Register(createInbox)
	smsd.db.Register(setLocPrefix)
	smsd.sqlNumToId = numId
	smsd.end = make(chan event)
	smsd.newMsg = make(chan event, 1)
	return smsd, nil
}

// Selects messages from Outbox that have any recipient without sent flag set
const outboxGet = `SELECT
	o.id, o.src, o.report, o.body
FROM
	` + outboxTable + ` o
WHERE
	EXISTS (SELECT * FROM ` + recipientsTable + ` p WHERE p.msgId=o.id && !p.sent)
`

// Selects all recipients without sent flag set for givem msgId
const recipientsGet = `SELECT
	id, number
FROM
	` + recipientsTable + `
WHERE
	!sent && msgId=?
`

const recipientsSent = "UPDATE " + recipientsTable + " SET sent=? WHERE id=?"

// Send messages from Outbox
func (smsd *SMSd) sendMessages() (gammuErr bool) {
	if !prepareOnce(smsd.db, &smsd.stmtOutboxGet, outboxGet) {
		return
	}
	if !prepareOnce(smsd.db, &smsd.stmtRecipGet, recipientsGet) {
		return
	}
	if !prepareOnce(smsd.db, &smsd.stmtRecipSent, recipientsSent) {
		return
	}
	msgs, res, err := smsd.stmtOutboxGet.Exec()
	if err != nil {
		log.Println("Can't get a messages from Outbox:", err)
		return
	}
	colMid := res.Map("id")
	colReport := res.Map("report")
	colBody := res.Map("body")
	for _, msg := range msgs {
		mid := msg.Uint(colMid)
		report := msg.Bool(colReport)
		body := msg.Str(colBody)

		recipients, res, err := smsd.stmtRecipGet.Exec(mid)
		if err != nil {
			log.Printf("Can't get a phone number for msg #%d: %s", mid, err)
			return
		}
		colPid := res.Map("id")
		colNum := res.Map("number")
		for _, p := range recipients {
			pid := p.Uint(colPid)
			num := p.Str(colNum)
			if !checkNumber(num) {
				continue
			}
			if isGammuError(smsd.sm.SendLongSMS(num, body, report)) {
				// Phone error or bad values
				gammuErr = true
			}
			_, _, err = smsd.stmtRecipSent.Exec(time.Now(), pid)
			if err != nil {
				log.Printf(
					"Can't mark a msg/recip #%d/#%d as sent: %s",
					mid, pid, err,
				)
				return
			}
		}
	}
	return
}

const inboxPut = `INSERT
	` + inboxTable + `
SET
	time=?,
	number=?,
	srcId=?,
	body=?
`

const recipReport = `UPDATE
	` + recipientsTable + `
SET
	report=?
WHERE
	!report && (number=? || concat(@localPrefix, number)=?)
ORDER BY
	abs(timediff(?, sent))
LIMIT 1`

type Msg struct {
	Time   time.Time
	Number string
	SrcId  uint
	Body   string
}

func (smsd *SMSd) recvMessages() (gammuErr bool) {
	if !prepareOnce(smsd.db, &smsd.stmtInboxPut, inboxPut) {
		return
	}
	if !prepareOnce(smsd.db, &smsd.stmtRecipReport, recipReport) {
		return
	}
	if smsd.sqlNumToId != "" {
		if !prepareOnce(smsd.db, &smsd.stmtNumToId, smsd.sqlNumToId) {
			return
		}
	}

	var msg Msg
	smsd.stmtInboxPut.Bind(&msg)

	for {
		sms, err := smsd.sm.GetSMS()
		if err != nil {
			if err == io.EOF {
				break
			}
			log.Printf("Can't get message from phone: %s", err)
			return true
		}
		if sms.Report {
			// Find a message and sender in Outbox and mark it
			m := strings.TrimSpace(sms.Body)
			if strings.ToLower(m) == "delivered" {
				_, _, err = smsd.stmtRecipReport.Exec(
					sms.SMSCTime, sms.Number, sms.Number, sms.Time,
				)
				if err != nil {
					log.Printf(
						"Can't mark recipient %s as reported: %s",
						sms.Number, err,
					)
					return
				}
			}
		} else {
			// Save a message in Inbox
			msg.Time = sms.Time
			msg.Number = sms.Number
			msg.SrcId = 0
			msg.Body = sms.Body
			//log.Printf("Odebrano: %+v", msg)
			if smsd.stmtNumToId.Raw != nil {
				id, _, err := smsd.stmtNumToId.ExecFirst(msg.Number)
				if err != nil {
					log.Printf(
						"Can't get srcId for number %s: %s",
						sms.Number, err,
					)
					return
				}
				if id != nil {
					msg.SrcId, err = id.UintErr(0)
					if err != nil {
						log.Printf("Bad srcId '%v': %s", id[0], err)
						return
					}
				}
			}
			if f := smsd.filter; f != nil {
				accept, err := f.Filter(&msg)
				if err != nil {
					log.Printf("Filter error: %s", err)
				} else if !accept {
					// Drop this message
					continue
				}
			}
			_, _, err = smsd.stmtInboxPut.Exec() // using msg
			if err != nil {
				log.Printf(
					"Can't insert message from %s into Inbox: %s",
					sms.Number, err,
				)
				return
			}
		}
	}
	return
}

const outboxDel = `DELETE FROM
	o
USING
	SMSd_Outbox o
WHERE
	o.del && !EXISTS(
		SELECT
			*
		FROM
			SMSd_Recipients r
		WHERE
			r.msgId = o.id && (!r.sent || o.report && !r.report)
	)
`

func (smsd *SMSd) delMessages() {
	if !prepareOnce(smsd.db, &smsd.stmtOutboxDel, outboxDel) {
		return
	}
	_, _, err := smsd.stmtOutboxDel.Exec()
	if err != nil {
		log.Println("Can't delete messages:", err)
		return
	}
}

func (smsd *SMSd) sendRecvDel(send bool) {
	var gammuErr bool
	if !smsd.sm.IsConnected() {
		if isGammuError(smsd.sm.Connect()) {
			return
		}
	}
	if send {
		gammuErr = smsd.sendMessages()
	}
	gammuErr = smsd.recvMessages() || gammuErr
	if send {
		smsd.delMessages()
	}
	if gammuErr && smsd.sm.IsConnected() {
		smsd.sm.Disconnect()
	}
}

func (smsd *SMSd) loop() {
	send := true
	for {
		smsd.sendRecvDel(send)
		// Wait for some event or timeout
		select {
		case <-smsd.end:
			return
		case <-smsd.newMsg:
			send = true
		case <-time.After(7 * time.Second):
			// send and del two times less frequently than recv
			send = !send
		}
	}
}

func (smsd *SMSd) Start() {
	go smsd.loop()
}

func (smsd *SMSd) Stop() {
	smsd.end <- event{}
}

func (smsd *SMSd) NewMsg() {
	select {
	case smsd.newMsg <- event{}:
	default:
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager

import (
	"context"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"time"

	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/endpoints/metrics"
	"k8s.io/klog/v2"
)

func determineIgnoreNonSemanticUpdatesEnabled() bool {
	if ignoreNonSemanticUpdatesString, exists := os.LookupEnv("KUBE_APISERVER_IGNORE_NON_SEMANTIC_UPDATES"); exists {
		if ret, err := strconv.ParseBool(ignoreNonSemanticUpdatesString); err == nil {
			return ret
		} else {
			klog.Errorf("failed to parse envar KUBE_APISERVER_IGNORE_NON_SEMANTIC_UPDATES: %v", err)
		}
	}

	// enabled by default
	return true
}

var (
	ignoreNonSemanticUpdatesEnabled = determineIgnoreNonSemanticUpdatesEnabled()
)

var ignoreTimestampEqualities = func() conversion.Equalities {
	var eqs = equality.Semantic.Copy()

	err := eqs.AddFunc(
		func(a, b metav1.ManagedFieldsEntry) bool {
			// Two objects' managed fields are equivalent if, ignoring timestamp,
			//	the objects are deeply equal.
			a.Time = nil
			b.Time = nil
			return reflect.DeepEqual(a, b)
		},
	)

	if err != nil {
		panic(err)
	}

	return eqs
}()

// IgnoreManagedFieldsTimestampsTransformer reverts timestamp updates
// if the non-managed parts of the object are equivalent
func IgnoreManagedFieldsTimestampsTransformer(
	_ context.Context,
	newObj runtime.Object,
	oldObj runtime.Object,
) (res runtime.Object, err error) {
	if !ignoreNonSemanticUpdatesEnabled {
		return newObj, nil
	}

	outcome := "unequal_objects_fast"
	start := time.Now()
	err = nil
	res = nil

	defer func() {
		if err != nil {
			outcome = "error"
		}

		metrics.RecordTimestampComparisonLatency(outcome, time.Since(start))
	}()

	// If managedFields modulo timestamps are unchanged
	//		and
	//	rest of object is unchanged
	//		then
	//	revert any changes to timestamps in managed fields
	//		(to prevent spurious ResourceVersion bump)
	//
	// Procecure:
	// Do a quicker check to see if just managed fields modulo timestamps are
	//	unchanged. If so, then do the full, slower check.
	//
	// In most cases which actually update the object, the managed fields modulo
	//	timestamp check will fail, and we will be able to return early.
	//
	// In other cases, the managed fields may be exactly the same,
	// 	except for timestamp, but the objects are the different. This is the
	//	slow path which checks the full object.
	oldAccessor, err := meta.Accessor(oldObj)
	if err != nil {
		return nil, fmt.Errorf("failed to acquire accessor for oldObj: %v", err)
	}

	accessor, err := meta.Accessor(newObj)
	if err != nil {
		return nil, fmt.Errorf("failed to acquire accessor for newObj: %v", err)
	}

	oldManagedFields := oldAccessor.GetManagedFields()
	newManagedFields := accessor.GetManagedFields()

	if len(oldManagedFields) != len(newManagedFields) {
		// Return early if any managed fields entry was added/removed.
		// We want to retain user expectation that even if they write to a field
		// whose value did not change, they will still result as the field
		// manager at the end.
		return newObj, nil
	} else if len(newManagedFields) == 0 {
		// This transformation only makes sense when managedFields are
		// non-empty
		return newObj, nil
	}

	// This transformation only makes sense if the managed fields has at least one
	// changed timestamp; and are otherwise equal. Return early if there are no
	// changed timestamps.
	allTimesUnchanged := true
	for i, e := range newManagedFields {
		if !e.Time.Equal(oldManagedFields[i].Time) {
			allTimesUnchanged = false
			break
		}
	}

	if allTimesUnchanged {
		return newObj, nil
	}

	// This condition ensures the managed fields are always compared first. If
	//	this check fails, the if statement will short circuit. If the check
	// 	succeeds the slow path is taken which compares entire objects.
	if !ignoreTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(oldManagedFields, newManagedFields) {
		return newObj, nil
	}

	if ignoreTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(newObj, oldObj) {
		// Remove any changed timestamps, so that timestamp is not the only
		// change seen by etcd.
		//
		// newManagedFields is known to be exactly pairwise equal to
		// oldManagedFields except for timestamps.
		//
		// Simply replace possibly changed new timestamps with their old values.
		for idx := 0; idx < len(oldManagedFields); idx++ {
			newManagedFields[idx].Time = oldManagedFields[idx].Time
		}

		accessor.SetManagedFields(newManagedFields)
		outcome = "equal_objects"
		return newObj, nil
	}

	outcome = "unequal_objects_slow"
	return newObj, nil
}
<commit_msg>use more apt name for flag<commit_after>/*
Copyright 2021 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fieldmanager

import (
	"context"
	"fmt"
	"os"
	"reflect"
	"strconv"
	"time"

	"k8s.io/apimachinery/pkg/api/equality"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/conversion"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/endpoints/metrics"
	"k8s.io/klog/v2"
)

func determineAvoidNoopTimestampUpdatesEnabled() bool {
	if avoidNoopTimestampUpdatesString, exists := os.LookupEnv("KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES"); exists {
		if ret, err := strconv.ParseBool(avoidNoopTimestampUpdatesString); err == nil {
			return ret
		} else {
			klog.Errorf("failed to parse envar KUBE_APISERVER_AVOID_NOOP_SSA_TIMESTAMP_UPDATES: %v", err)
		}
	}

	// enabled by default
	return true
}

var (
	avoidNoopTimestampUpdatesEnabled = determineAvoidNoopTimestampUpdatesEnabled()
)

var avoidTimestampEqualities = func() conversion.Equalities {
	var eqs = equality.Semantic.Copy()

	err := eqs.AddFunc(
		func(a, b metav1.ManagedFieldsEntry) bool {
			// Two objects' managed fields are equivalent if, ignoring timestamp,
			//	the objects are deeply equal.
			a.Time = nil
			b.Time = nil
			return reflect.DeepEqual(a, b)
		},
	)

	if err != nil {
		panic(err)
	}

	return eqs
}()

// IgnoreManagedFieldsTimestampsTransformer reverts timestamp updates
// if the non-managed parts of the object are equivalent
func IgnoreManagedFieldsTimestampsTransformer(
	_ context.Context,
	newObj runtime.Object,
	oldObj runtime.Object,
) (res runtime.Object, err error) {
	if !avoidNoopTimestampUpdatesEnabled {
		return newObj, nil
	}

	outcome := "unequal_objects_fast"
	start := time.Now()
	err = nil
	res = nil

	defer func() {
		if err != nil {
			outcome = "error"
		}

		metrics.RecordTimestampComparisonLatency(outcome, time.Since(start))
	}()

	// If managedFields modulo timestamps are unchanged
	//		and
	//	rest of object is unchanged
	//		then
	//	revert any changes to timestamps in managed fields
	//		(to prevent spurious ResourceVersion bump)
	//
	// Procecure:
	// Do a quicker check to see if just managed fields modulo timestamps are
	//	unchanged. If so, then do the full, slower check.
	//
	// In most cases which actually update the object, the managed fields modulo
	//	timestamp check will fail, and we will be able to return early.
	//
	// In other cases, the managed fields may be exactly the same,
	// 	except for timestamp, but the objects are the different. This is the
	//	slow path which checks the full object.
	oldAccessor, err := meta.Accessor(oldObj)
	if err != nil {
		return nil, fmt.Errorf("failed to acquire accessor for oldObj: %v", err)
	}

	accessor, err := meta.Accessor(newObj)
	if err != nil {
		return nil, fmt.Errorf("failed to acquire accessor for newObj: %v", err)
	}

	oldManagedFields := oldAccessor.GetManagedFields()
	newManagedFields := accessor.GetManagedFields()

	if len(oldManagedFields) != len(newManagedFields) {
		// Return early if any managed fields entry was added/removed.
		// We want to retain user expectation that even if they write to a field
		// whose value did not change, they will still result as the field
		// manager at the end.
		return newObj, nil
	} else if len(newManagedFields) == 0 {
		// This transformation only makes sense when managedFields are
		// non-empty
		return newObj, nil
	}

	// This transformation only makes sense if the managed fields has at least one
	// changed timestamp; and are otherwise equal. Return early if there are no
	// changed timestamps.
	allTimesUnchanged := true
	for i, e := range newManagedFields {
		if !e.Time.Equal(oldManagedFields[i].Time) {
			allTimesUnchanged = false
			break
		}
	}

	if allTimesUnchanged {
		return newObj, nil
	}

	// This condition ensures the managed fields are always compared first. If
	//	this check fails, the if statement will short circuit. If the check
	// 	succeeds the slow path is taken which compares entire objects.
	if !avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(oldManagedFields, newManagedFields) {
		return newObj, nil
	}

	if avoidTimestampEqualities.DeepEqualWithNilDifferentFromEmpty(newObj, oldObj) {
		// Remove any changed timestamps, so that timestamp is not the only
		// change seen by etcd.
		//
		// newManagedFields is known to be exactly pairwise equal to
		// oldManagedFields except for timestamps.
		//
		// Simply replace possibly changed new timestamps with their old values.
		for idx := 0; idx < len(oldManagedFields); idx++ {
			newManagedFields[idx].Time = oldManagedFields[idx].Time
		}

		accessor.SetManagedFields(newManagedFields)
		outcome = "equal_objects"
		return newObj, nil
	}

	outcome = "unequal_objects_slow"
	return newObj, nil
}
<|endoftext|>