<commit_before>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package worker

import (
	"fmt"
	"strings"

	log "github.com/golang/glog"

	"github.com/caicloud/cyclone/cmd/worker/options"
	"github.com/caicloud/cyclone/pkg/api"
	"github.com/caicloud/cyclone/pkg/docker"
	"github.com/caicloud/cyclone/pkg/scm"
	_ "github.com/caicloud/cyclone/pkg/scm/provider"
	"github.com/caicloud/cyclone/pkg/worker/cycloneserver"
	_ "github.com/caicloud/cyclone/pkg/worker/integrate/provider"
	_ "github.com/caicloud/cyclone/pkg/worker/scm/provider"
	"github.com/caicloud/cyclone/pkg/worker/stage"
)

// Worker ...
type Worker struct {
	Client  cycloneserver.CycloneServerClient
	Options *options.WorkerOptions
}

// NewWorker returns a new APIServer with config
func NewWorker(opts *options.WorkerOptions) *Worker {
	client := cycloneserver.NewClient(opts.CycloneServer)

	return &Worker{
		Client:  client,
		Options: opts,
	}
}

// Run starts the worker
func (worker *Worker) Run() error {
	log.Infof("worker start with options: %v", worker.Options)

	// Get event for cyclone server
	eventID := worker.Options.EventID
	event, err := worker.Client.GetEvent(eventID)
	if err != nil {
		log.Errorf("fail to get event %s as %s", eventID, err.Error())
		return err
	}

	// Handle event
	log.Infof("start to handle event %s", eventID)
	worker.HandleEvent(event)

	return nil
}

// Steps:
// 1. Git clone code
// 2. New Docker manager
// 3. Run pipeline stages
// 4. Create the tag in SCM
func (worker *Worker) HandleEvent(event *api.Event) {
	project := event.Project
	pipeline := event.Pipeline
	performParams := event.PipelineRecord.PerformParams
	performStages := performParams.Stages
	stageSet := convertPerformStageSet(performStages)
	build := pipeline.Build

	opts := worker.Options
	registryLocation := opts.RegistryLocation
	registryUsername := opts.RegistryUsername
	registryPassword := opts.RegistryPassword

	dockerManager, err := docker.NewDockerManager(opts.DockerHost, registryLocation, registryUsername, registryPassword)
	if err != nil {
		log.Error(err.Error())
		return
	}

	// Init StageStatus
	if event.PipelineRecord.StageStatus == nil {
		event.PipelineRecord.StageStatus = &api.StageStatus{}
	}

	// TODO(robin) Seperate unit test and package stage.
	stageManager := stage.NewStageManager(dockerManager, worker.Client, project.Registry, performParams)
	stageManager.SetRecordInfo(project.Name, pipeline.Name, event.ID)
	stageManager.SetEvent(event)

	// Execute the code checkout stage.
	err = stageManager.ExecCodeCheckout(project.SCM.Token, build.Stages.CodeCheckout)
	if err != nil {
		log.Error(err.Error())
		return
	}

	// Execute the package stage if necessary.
	if _, ok := stageSet[api.PackageStageName]; ok {
		err = stageManager.ExecPackage(build.BuilderImage, build.BuildInfo, build.Stages.UnitTest, build.Stages.Package)
		if err != nil {
			log.Error(err.Error())
		}

		if !build.Stages.Package.AllowFailure {
			return
		}
	}

	// Execute the package stage if necessary.
	if _, ok := stageSet[api.CodeScanStageName]; ok {
		err = stageManager.ExecCodeScan(build.Stages.CodeScan)
		if err != nil {
			log.Error(err.Error())
		}

		if !build.Stages.CodeScan.AllowFailure {
			return
		}
	}

	// The built images from image build stage.
	var builtImages []string

	// Execute the image build stage if necessary.
	if _, ok := stageSet[api.ImageBuildStageName]; ok {
		builtImages, err = stageManager.ExecImageBuild(build.Stages.ImageBuild)
		if err != nil {
			log.Error(err.Error())
			return
		}
	}

	// Execute the integration test stage if necessary.
	if _, ok := stageSet[api.IntegrationTestStageName]; ok {
		err = stageManager.ExecIntegrationTest(builtImages, build.Stages.IntegrationTest)
		if err != nil {
			log.Error(err.Error())
		}

		if !build.Stages.IntegrationTest.AllowFailure {
			return
		}
	}

	// Execute the image release stage if necessary.
	if _, ok := stageSet[api.ImageReleaseStageName]; ok {
		err = stageManager.ExecImageRelease(builtImages, build.Stages.ImageRelease)
		if err != nil {
			log.Error(err.Error())
		}

		if !build.Stages.ImageRelease.AllowFailure {
			return
		}
	}

	if event.PipelineRecord.PerformParams.CreateSCMTag {
		codesource := event.Pipeline.Build.Stages.CodeCheckout.MainRepo
		err = scm.NewTagFromLatest(codesource, project.SCM, event.PipelineRecord.Name, event.PipelineRecord.PerformParams.Description)
		if err != nil {
			log.Errorf("new tag from latest fail : %v", err)
			event.PipelineRecord.Status = api.Failed
			scmType := event.Pipeline.Build.Stages.CodeCheckout.MainRepo.Type
			if (scmType == api.Gitlab && strings.Contains(err.Error(), "403")) ||
				(scmType == api.Github && strings.Contains(err.Error(), "404")) {
				event.PipelineRecord.ErrorMessage = "Create SCM tag fails, please check your account permissions."
			} else {
				event.PipelineRecord.ErrorMessage = fmt.Sprintf("Create SCM tag fails : %v", err)
			}

			err = worker.Client.SendEvent(event)
			if err != nil {
				log.Errorf("set event result err: %v", err)
				return
			}
			return
		}
	}

	log.Info("success: ")

	// update event.PipelineRecord.Status from running to success
	event.PipelineRecord.Status = api.Success

	// Sent event for cyclone server
	err = worker.Client.SendEvent(event)
	if err != nil {
		log.Errorf("set event result err: %v", err)
		return
	}
	log.Infof("send event %+v to server", event)
}

func convertPerformStageSet(stages []api.PipelineStageName) map[api.PipelineStageName]struct{} {
	stageSet := make(map[api.PipelineStageName]struct{})
	for _, stage := range stages {
		stageSet[stage] = struct{}{}
	}

	return stageSet
}
<commit_msg>fix: pipeline is stuck when package step execution successfully (#643)<commit_after>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package worker

import (
	"fmt"
	"strings"

	log "github.com/golang/glog"

	"github.com/caicloud/cyclone/cmd/worker/options"
	"github.com/caicloud/cyclone/pkg/api"
	"github.com/caicloud/cyclone/pkg/docker"
	"github.com/caicloud/cyclone/pkg/scm"
	_ "github.com/caicloud/cyclone/pkg/scm/provider"
	"github.com/caicloud/cyclone/pkg/worker/cycloneserver"
	_ "github.com/caicloud/cyclone/pkg/worker/integrate/provider"
	_ "github.com/caicloud/cyclone/pkg/worker/scm/provider"
	"github.com/caicloud/cyclone/pkg/worker/stage"
)

// Worker ...
type Worker struct {
	Client  cycloneserver.CycloneServerClient
	Options *options.WorkerOptions
}

// NewWorker returns a new APIServer with config
func NewWorker(opts *options.WorkerOptions) *Worker {
	client := cycloneserver.NewClient(opts.CycloneServer)

	return &Worker{
		Client:  client,
		Options: opts,
	}
}

// Run starts the worker
func (worker *Worker) Run() error {
	log.Infof("worker start with options: %v", worker.Options)

	// Get event for cyclone server
	eventID := worker.Options.EventID
	event, err := worker.Client.GetEvent(eventID)
	if err != nil {
		log.Errorf("fail to get event %s as %s", eventID, err.Error())
		return err
	}

	// Handle event
	log.Infof("start to handle event %s", eventID)
	worker.HandleEvent(event)

	return nil
}

// Steps:
// 1. Git clone code
// 2. New Docker manager
// 3. Run pipeline stages
// 4. Create the tag in SCM
func (worker *Worker) HandleEvent(event *api.Event) {
	project := event.Project
	pipeline := event.Pipeline
	performParams := event.PipelineRecord.PerformParams
	performStages := performParams.Stages
	stageSet := convertPerformStageSet(performStages)
	log.Infof("stage set :%v", stageSet)
	build := pipeline.Build

	opts := worker.Options
	registryLocation := opts.RegistryLocation
	registryUsername := opts.RegistryUsername
	registryPassword := opts.RegistryPassword

	dockerManager, err := docker.NewDockerManager(opts.DockerHost, registryLocation, registryUsername, registryPassword)
	if err != nil {
		log.Error(err.Error())
		return
	}

	// Init StageStatus
	if event.PipelineRecord.StageStatus == nil {
		event.PipelineRecord.StageStatus = &api.StageStatus{}
	}

	// TODO(robin) Seperate unit test and package stage.
	stageManager := stage.NewStageManager(dockerManager, worker.Client, project.Registry, performParams)
	stageManager.SetRecordInfo(project.Name, pipeline.Name, event.ID)
	stageManager.SetEvent(event)

	// Execute the code checkout stage.
	err = stageManager.ExecCodeCheckout(project.SCM.Token, build.Stages.CodeCheckout)
	if err != nil {
		log.Error(err.Error())
		return
	}

	// Execute the package stage if necessary.
	if _, ok := stageSet[api.PackageStageName]; ok {
		err = stageManager.ExecPackage(build.BuilderImage, build.BuildInfo, build.Stages.UnitTest, build.Stages.Package)
		if err != nil {
			log.Error(err.Error())
			if !build.Stages.Package.AllowFailure {
				return
			}
		}

	}

	// Execute the package stage if necessary.
	if _, ok := stageSet[api.CodeScanStageName]; ok {
		err = stageManager.ExecCodeScan(build.Stages.CodeScan)
		if err != nil {
			log.Error(err.Error())
			if !build.Stages.CodeScan.AllowFailure {
				return
			}
		}

	}

	// The built images from image build stage.
	var builtImages []string

	// Execute the image build stage if necessary.
	if _, ok := stageSet[api.ImageBuildStageName]; ok {
		builtImages, err = stageManager.ExecImageBuild(build.Stages.ImageBuild)
		if err != nil {
			log.Error(err.Error())
			return
		}
	}

	// Execute the integration test stage if necessary.
	if _, ok := stageSet[api.IntegrationTestStageName]; ok {
		err = stageManager.ExecIntegrationTest(builtImages, build.Stages.IntegrationTest)
		if err != nil {
			log.Error(err.Error())
			if !build.Stages.IntegrationTest.AllowFailure {
				return
			}
		}

	}

	// Execute the image release stage if necessary.
	if _, ok := stageSet[api.ImageReleaseStageName]; ok {
		err = stageManager.ExecImageRelease(builtImages, build.Stages.ImageRelease)
		if err != nil {
			log.Error(err.Error())
			if !build.Stages.ImageRelease.AllowFailure {
				return
			}
		}

	}

	if event.PipelineRecord.PerformParams.CreateSCMTag {
		codesource := event.Pipeline.Build.Stages.CodeCheckout.MainRepo
		err = scm.NewTagFromLatest(codesource, project.SCM, event.PipelineRecord.Name, event.PipelineRecord.PerformParams.Description)
		if err != nil {
			log.Errorf("new tag from latest fail : %v", err)
			event.PipelineRecord.Status = api.Failed
			scmType := event.Pipeline.Build.Stages.CodeCheckout.MainRepo.Type
			if (scmType == api.Gitlab && strings.Contains(err.Error(), "403")) ||
				(scmType == api.Github && strings.Contains(err.Error(), "404")) {
				event.PipelineRecord.ErrorMessage = "Create SCM tag fails, please check your account permissions."
			} else {
				event.PipelineRecord.ErrorMessage = fmt.Sprintf("Create SCM tag fails : %v", err)
			}

			err = worker.Client.SendEvent(event)
			if err != nil {
				log.Errorf("set event result err: %v", err)
				return
			}
			return
		}
	}

	log.Info("success: ")

	// update event.PipelineRecord.Status from running to success
	event.PipelineRecord.Status = api.Success

	// Sent event for cyclone server
	err = worker.Client.SendEvent(event)
	if err != nil {
		log.Errorf("set event result err: %v", err)
		return
	}
	log.Infof("send event %+v to server", event)
}

func convertPerformStageSet(stages []api.PipelineStageName) map[api.PipelineStageName]struct{} {
	stageSet := make(map[api.PipelineStageName]struct{})
	for _, stage := range stages {
		stageSet[stage] = struct{}{}
	}

	return stageSet
}
<|endoftext|>


<commit_before>// +build windows

/*
 * Copyright 2016 ThoughtWorks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package agent

import (
	"strconv"
	"syscall"
	"unsafe"
	"os"
)

func UsableSpace() int64 {
	wd, err := os.Getwd()
	if err != nil {
		LogInfo("Cannot find working directory, error: %v", err)
		return -1
	}
	_, free, err := diskSpace(wd)
	if err != nil {
		LogInfo("Unknown diskspace, error: %v", err)
		return -1
	}
	return free
}

func UsableSpaceString() string {
	return strconv.FormatInt(UsableSpace(), 10)
}

func diskSpace(path string) (total, free int64, err error) {
	var (
		freeBytes int64
	 	totalBytes int64
		availBytes int64
	)

	h := syscall.MustLoadDLL("kernel32.dll")
	c := h.MustFindProc("GetDiskFreeSpaceExW")

	_, _, err = c.Call(
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&freeBytes)),
		uintptr(unsafe.Pointer(&totalBytes)),
		uintptr(unsafe.Pointer(&availBytes)))
	total = totalBytes
	free = freeBytes
	return
}
<commit_msg>This fixed #26  Err is never nil when calling native Windows functions through DLL<commit_after>// +build windows

/*
 * Copyright 2016 ThoughtWorks, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package agent

import (
	"strconv"
	"syscall"
	"unsafe"
	"os"
)

func UsableSpace() int64 {
	wd, err := os.Getwd()
	if err != nil {
		LogInfo("Cannot find working directory, error: %v", err)
		return -1
	}
	_, free, err := diskSpace(wd)
	if err != nil {
		LogInfo("Unknown diskspace, error: %v", err)
		return -1
	}
	return free
}

func UsableSpaceString() string {
	return strconv.FormatInt(UsableSpace(), 10)
}

func diskSpace(path string) (total, free int64, err error) {
	var (
		freeBytes int64
	 	totalBytes int64
		availBytes int64
	)

	h := syscall.MustLoadDLL("kernel32.dll")
	c := h.MustFindProc("GetDiskFreeSpaceExW")

	// According to MSDN, err will never be nil, r1 = 0 mean call failed
	r1, _, err = c.Call(
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&freeBytes)),
		uintptr(unsafe.Pointer(&totalBytes)),
		uintptr(unsafe.Pointer(&availBytes)))
	if r1 != 0 {
		err = nil
	}
	total = totalBytes
	free = freeBytes
	return
}
<|endoftext|>


<commit_before>package main

import (
  "fmt"
  "strings"
  "github.com/Jeffail/gabs"
)

func (p Person) true_delete(trait_name string) {
  current_json := p.Json

  path := create_path("person.traits", trait_name)
  if current_json.ExistsP(path) {
    // fmt.Println("we here")
  } else {
    fmt.Println("Nothing to delete")
    return
  }

  err := current_json.DeleteP(path)
  if err == nil {
    fmt.Println("It has been done.")
  }

  path_elements := strings.Split(path, ".")

  path = strings.Join(path_elements[:len(path_elements) - 1], ".")
  element_to_delete := path_elements[len(path_elements) - 1]
  // fmt.Println("To Delete:", element_to_delete)

  my_data := current_json.Path(path)

  element_depth := obtain_array_count(my_data.String())

  children, err := my_data.Children()
  check_err(err)
  for i := 0; i < element_depth - 1; i++ {
    for _, child := range children {
      children, err = child.Children()
      check_err(err)
    }
  }

  jsonObj := gabs.New()
  jsonObj.Array("foo", "array")

  for _, child := range children {
    potential_target := child.String()

    target_split := strings.Split(potential_target, ":")
    if clean_text(target_split[0]) == element_to_delete {

    } else {
      jsonObj.ArrayAppend(child.Data(), "foo", "array")
    }
  }

  new_arr := jsonObj.Path("foo.array")
  // fmt.Println(new_arr)

  current_val := current_json.Path(path)
  check_err(err)
  for i := 0; i < element_depth - 1; i++ {
    current_val = current_val.Index(0)
  }

  fmt.Println("->", current_val)
  kinder, err := current_val.Children()

  for _, kind := range kinder {
    check_err(err)
    fmt.Println(kind)
  }

  _, err = current_val.SetIndex(new_arr, 0)
  check_err(err)

  // fmt.Println(my_data)
}

// This may have some bad consequences down the line, warrents further investigation
func (p Person) clean_json() Person {
  current_json_text := p.Json.String()
  if current_json_text == "{}" { return p }
  replacer := strings.NewReplacer(
    "{},", "",
    "{}", "",
    "[],", "",
    "[]", "")
  temp := replacer.Replace(current_json_text)
  new_json, err := gabs.ParseJSON([]byte(temp))
  check_err(err)
  p.Json = new_json
  return p
}

func clean_text(text string) string {
  temp := strings.Replace(text, "{", "", -1)
  if temp[0] == '"' {
    temp = temp[1:]
  }
  if temp[len(temp) - 1] == '"' {
    temp = temp[:len(temp) - 1]
  }
  return temp
}

func obtain_array_count(text string) int {
  count := 0
  for _, elem := range text {
    if elem == '[' {
      count++
    } else {
      break
    }
  }
  return count
}
<commit_msg>Turns out we can manipulate objects low level... i dumbed that one<commit_after>package main

import (
  "fmt"
  "strings"
  "github.com/Jeffail/gabs"
)

func (p Person) true_delete(trait_name string) {
  current_json := p.Json

  path := create_path("person.traits", trait_name)
  if current_json.ExistsP(path) {
    // fmt.Println("we here")
  } else {
    fmt.Println("Nothing to delete")
    return
  }

  err := current_json.DeleteP(path)
  if err == nil {
    fmt.Println("It has been done.")
  }

  path_elements := strings.Split(path, ".")

  path = strings.Join(path_elements[:len(path_elements) - 1], ".")
  element_to_delete := path_elements[len(path_elements) - 1]
  // fmt.Println("To Delete:", element_to_delete)

  my_data := current_json.Path(path)

  element_depth := obtain_array_count(my_data.String())

  children, err := my_data.Children()
  check_err(err)
  for i := 0; i < element_depth - 1; i++ {
    for _, child := range children {
      children, err = child.Children()
      check_err(err)
    }
  }

  jsonObj := gabs.New()
  jsonObj.Array("foo", "array")

  for _, child := range children {
    potential_target := child.String()

    target_split := strings.Split(potential_target, ":")
    if clean_text(target_split[0]) == element_to_delete {

    } else {
      jsonObj.ArrayAppend(child.Data(), "foo", "array")
    }
  }

  new_arr := jsonObj.Path("foo.array")
  // fmt.Println(new_arr)

  current_val := current_json.Path(path)
  check_err(err)
  for i := 0; i < element_depth - 1; i++ {
    current_val = current_val.Index(0)
  }

  fmt.Println("->", current_val)
  kinder, err := current_val.Children()

  kind := gabs.New()
  for _, kind = range kinder {
    check_err(err)
    fmt.Println(kind)
  }
  value := kind.DeleteP("rat")
  check_err(value)

  _, err = current_val.SetIndex(new_arr, 0)
  check_err(err)

  // fmt.Println(my_data)
}

// This may have some bad consequences down the line, warrents further investigation
func (p Person) clean_json() Person {
  current_json_text := p.Json.String()
  if current_json_text == "{}" { return p }
  replacer := strings.NewReplacer(
    "{},", "",
    "{}", "",
    "[],", "",
    "[]", "")
  temp := replacer.Replace(current_json_text)
  new_json, err := gabs.ParseJSON([]byte(temp))
  check_err(err)
  p.Json = new_json
  return p
}

func clean_text(text string) string {
  temp := strings.Replace(text, "{", "", -1)
  if temp[0] == '"' {
    temp = temp[1:]
  }
  if temp[len(temp) - 1] == '"' {
    temp = temp[:len(temp) - 1]
  }
  return temp
}

func obtain_array_count(text string) int {
  count := 0
  for _, elem := range text {
    if elem == '[' {
      count++
    } else {
      break
    }
  }
  return count
}
<|endoftext|>


<commit_before>// run

// +build !nacl

// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var tests = `
# These are test cases for the linker analysis that detects chains of
# nosplit functions that would cause a stack overflow.
#
# Lines beginning with # are comments.
#
# Each test case describes a sequence of functions, one per line.
# Each function definition is the function name, then the frame size,
# then optionally the keyword 'nosplit', then the body of the function.
# The body is assembly code, with some shorthands.
# The shorthand 'call x' stands for CALL x(SB).
# The shorthand 'callind' stands for 'CALL R0', where R0 is a register.
# Each test case must define a function named main, and it must be first.
# That is, a line beginning "main " indicates the start of a new test case.
# Within a stanza, ; can be used instead of \n to separate lines.
#
# After the function definition, the test case ends with an optional
# REJECT line, specifying the architectures on which the case should
# be rejected. "REJECT" without any architectures means reject on all architectures.
# The linker should accept the test case on systems not explicitly rejected.
#
# 64-bit systems do not attempt to execute test cases with frame sizes
# that are only 32-bit aligned.

# Ordinary function should work
main 0

# Large frame marked nosplit is always wrong.
main 10000 nosplit
REJECT

# Calling a large frame is okay.
main 0 call big
big 10000

# But not if the frame is nosplit.
main 0 call big
big 10000 nosplit
REJECT

# Recursion is okay.
main 0 call main

# Recursive nosplit runs out of space.
main 0 nosplit call main
REJECT

# Chains of ordinary functions okay.
main 0 call f1
f1 80 call f2
f2 80

# Chains of nosplit must fit in the stack limit, 128 bytes.
main 0 call f1
f1 80 nosplit call f2
f2 80 nosplit
REJECT

# Larger chains.
main 0 call f1
f1 16 call f2
f2 16 call f3
f3 16 call f4
f4 16 call f5
f5 16 call f6
f6 16 call f7
f7 16 call f8
f8 16 call end
end 1000

main 0 call f1
f1 16 nosplit call f2
f2 16 nosplit call f3
f3 16 nosplit call f4
f4 16 nosplit call f5
f5 16 nosplit call f6
f6 16 nosplit call f7
f7 16 nosplit call f8
f8 16 nosplit call end
end 1000
REJECT

# Test cases near the 128-byte limit.

# Ordinary stack split frame is always okay.
main 112
main 116
main 120
main 124
main 128
main 132
main 136

# A nosplit leaf can use the whole 128-CallSize bytes available on entry.
main 112 nosplit
main 116 nosplit
main 120 nosplit
main 124 nosplit
main 128 nosplit; REJECT
main 132 nosplit; REJECT
main 136 nosplit; REJECT

# Calling a nosplit function from a nosplit function requires
# having room for the saved caller PC and the called frame.
# Because ARM doesn't save LR in the leaf, it gets an extra 4 bytes.
main 112 nosplit call f; f 0 nosplit
main 116 nosplit call f; f 0 nosplit; REJECT amd64
main 120 nosplit call f; f 0 nosplit; REJECT amd64
main 124 nosplit call f; f 0 nosplit; REJECT amd64 386
main 128 nosplit call f; f 0 nosplit; REJECT
main 132 nosplit call f; f 0 nosplit; REJECT
main 136 nosplit call f; f 0 nosplit; REJECT

# Calling a splitting function from a nosplit function requires
# having room for the saved caller PC of the call but also the
# saved caller PC for the call to morestack. Again the ARM works
# in less space.
main 104 nosplit call f; f 0 call f
main 108 nosplit call f; f 0 call f
main 112 nosplit call f; f 0 call f; REJECT amd64
main 116 nosplit call f; f 0 call f; REJECT amd64
main 120 nosplit call f; f 0 call f; REJECT amd64 386
main 124 nosplit call f; f 0 call f; REJECT amd64 386
main 128 nosplit call f; f 0 call f; REJECT
main 132 nosplit call f; f 0 call f; REJECT
main 136 nosplit call f; f 0 call f; REJECT

# Indirect calls are assumed to be splitting functions.
main 104 nosplit callind
main 108 nosplit callind
main 112 nosplit callind; REJECT amd64
main 116 nosplit callind; REJECT amd64
main 120 nosplit callind; REJECT amd64 386
main 124 nosplit callind; REJECT amd64 386
main 128 nosplit callind; REJECT
main 132 nosplit callind; REJECT
main 136 nosplit callind; REJECT

# Issue 7623
main 0 call f; f 112
main 0 call f; f 116
main 0 call f; f 120
main 0 call f; f 124
main 0 call f; f 128
main 0 call f; f 132
main 0 call f; f 136
`

var (
	commentRE = regexp.MustCompile(`(?m)^#.*`)
	rejectRE  = regexp.MustCompile(`(?s)\A(.+?)((\n|; *)REJECT(.*))?\z`)
	lineRE    = regexp.MustCompile(`(\w+) (\d+)( nosplit)?(.*)`)
	callRE    = regexp.MustCompile(`\bcall (\w+)\b`)
	callindRE = regexp.MustCompile(`\bcallind\b`)
)

func main() {
	goarch := os.Getenv("GOARCH")
	if goarch == "" {
		goarch = runtime.GOARCH
	}

	dir, err := ioutil.TempDir("", "go-test-nosplit")
	if err != nil {
		bug()
		fmt.Printf("creating temp dir: %v\n", err)
		return
	}
	defer os.RemoveAll(dir)
	ioutil.WriteFile(filepath.Join(dir, "main.go"), []byte("package main\nfunc main()\n"), 0666)

	tests = strings.Replace(tests, "\t", " ", -1)
	tests = commentRE.ReplaceAllString(tests, "")

	nok := 0
	nfail := 0
TestCases:
	for len(tests) > 0 {
		var stanza string
		i := strings.Index(tests, "\nmain ")
		if i < 0 {
			stanza, tests = tests, ""
		} else {
			stanza, tests = tests[:i], tests[i+1:]
		}

		m := rejectRE.FindStringSubmatch(stanza)
		if m == nil {
			bug()
			fmt.Printf("invalid stanza:\n\t%s\n", indent(stanza))
			continue
		}
		lines := strings.TrimSpace(m[1])
		reject := false
		if m[2] != "" {
			if strings.TrimSpace(m[4]) == "" {
				reject = true
			} else {
				for _, rej := range strings.Fields(m[4]) {
					if rej == goarch {
						reject = true
					}
				}
			}
		}
		if lines == "" && !reject {
			continue
		}

		var buf bytes.Buffer
		if goarch == "arm" {
			fmt.Fprintf(&buf, "#define CALL BL\n#define REGISTER (R0)\n")
		} else {
			fmt.Fprintf(&buf, "#define REGISTER AX\n")
		}

		for _, line := range strings.Split(lines, "\n") {
			line = strings.TrimSpace(line)
			if line == "" {
				continue
			}
			for _, subline := range strings.Split(line, ";") {
				subline = strings.TrimSpace(subline)
				if subline == "" {
					continue
				}
				m := lineRE.FindStringSubmatch(subline)
				if m == nil {
					bug()
					fmt.Printf("invalid function line: %s\n", subline)
					continue TestCases
				}
				name := m[1]
				size, _ := strconv.Atoi(m[2])
				if goarch == "amd64" && size%8 == 4 {
					continue TestCases
				}
				nosplit := m[3]
				body := m[4]

				if nosplit != "" {
					nosplit = ",7"
				} else {
					nosplit = ",0"
				}
				body = callRE.ReplaceAllString(body, "CALL ·$1(SB);")
				body = callindRE.ReplaceAllString(body, "CALL REGISTER;")

				fmt.Fprintf(&buf, "TEXT ·%s(SB)%s,$%d-0\n\t%s\n\tRET\n\n", name, nosplit, size, body)
			}
		}

		ioutil.WriteFile(filepath.Join(dir, "asm.s"), buf.Bytes(), 0666)

		cmd := exec.Command("go", "build")
		cmd.Dir = dir
		output, err := cmd.CombinedOutput()
		if err == nil {
			nok++
			if reject {
				bug()
				fmt.Printf("accepted incorrectly:\n\t%s\n", indent(strings.TrimSpace(stanza)))
			}
		} else {
			nfail++
			if !reject {
				bug()
				fmt.Printf("rejected incorrectly:\n\t%s\n", indent(strings.TrimSpace(stanza)))
				fmt.Printf("\n\tlinker output:\n\t%s\n", indent(string(output)))
			}
		}
	}

	if !bugged && (nok == 0 || nfail == 0) {
		bug()
		fmt.Printf("not enough test cases run\n")
	}
}

func indent(s string) string {
	return strings.Replace(s, "\n", "\n\t", -1)
}

var bugged = false

func bug() {
	if !bugged {
		bugged = true
		fmt.Printf("BUG\n")
	}
}
<commit_msg>[dev.power64] test/nosplit.go: add power64 support<commit_after>// run

// +build !nacl

// Copyright 2014 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var tests = `
# These are test cases for the linker analysis that detects chains of
# nosplit functions that would cause a stack overflow.
#
# Lines beginning with # are comments.
#
# Each test case describes a sequence of functions, one per line.
# Each function definition is the function name, then the frame size,
# then optionally the keyword 'nosplit', then the body of the function.
# The body is assembly code, with some shorthands.
# The shorthand 'call x' stands for CALL x(SB).
# The shorthand 'callind' stands for 'CALL R0', where R0 is a register.
# Each test case must define a function named main, and it must be first.
# That is, a line beginning "main " indicates the start of a new test case.
# Within a stanza, ; can be used instead of \n to separate lines.
#
# After the function definition, the test case ends with an optional
# REJECT line, specifying the architectures on which the case should
# be rejected. "REJECT" without any architectures means reject on all architectures.
# The linker should accept the test case on systems not explicitly rejected.
#
# 64-bit systems do not attempt to execute test cases with frame sizes
# that are only 32-bit aligned.

# Ordinary function should work
main 0

# Large frame marked nosplit is always wrong.
main 10000 nosplit
REJECT

# Calling a large frame is okay.
main 0 call big
big 10000

# But not if the frame is nosplit.
main 0 call big
big 10000 nosplit
REJECT

# Recursion is okay.
main 0 call main

# Recursive nosplit runs out of space.
main 0 nosplit call main
REJECT

# Chains of ordinary functions okay.
main 0 call f1
f1 80 call f2
f2 80

# Chains of nosplit must fit in the stack limit, 128 bytes.
main 0 call f1
f1 80 nosplit call f2
f2 80 nosplit
REJECT

# Larger chains.
main 0 call f1
f1 16 call f2
f2 16 call f3
f3 16 call f4
f4 16 call f5
f5 16 call f6
f6 16 call f7
f7 16 call f8
f8 16 call end
end 1000

main 0 call f1
f1 16 nosplit call f2
f2 16 nosplit call f3
f3 16 nosplit call f4
f4 16 nosplit call f5
f5 16 nosplit call f6
f6 16 nosplit call f7
f7 16 nosplit call f8
f8 16 nosplit call end
end 1000
REJECT

# Test cases near the 128-byte limit.

# Ordinary stack split frame is always okay.
main 112
main 116
main 120
main 124
main 128
main 132
main 136

# A nosplit leaf can use the whole 128-CallSize bytes available on entry.
main 112 nosplit
main 116 nosplit
main 120 nosplit
main 124 nosplit
main 128 nosplit; REJECT
main 132 nosplit; REJECT
main 136 nosplit; REJECT

# Calling a nosplit function from a nosplit function requires
# having room for the saved caller PC and the called frame.
# Because ARM doesn't save LR in the leaf, it gets an extra 4 bytes.
main 112 nosplit call f; f 0 nosplit
main 116 nosplit call f; f 0 nosplit; REJECT amd64
main 120 nosplit call f; f 0 nosplit; REJECT amd64
main 124 nosplit call f; f 0 nosplit; REJECT amd64 386
main 128 nosplit call f; f 0 nosplit; REJECT
main 132 nosplit call f; f 0 nosplit; REJECT
main 136 nosplit call f; f 0 nosplit; REJECT

# Calling a splitting function from a nosplit function requires
# having room for the saved caller PC of the call but also the
# saved caller PC for the call to morestack. Again the ARM works
# in less space.
main 104 nosplit call f; f 0 call f
main 108 nosplit call f; f 0 call f
main 112 nosplit call f; f 0 call f; REJECT amd64
main 116 nosplit call f; f 0 call f; REJECT amd64
main 120 nosplit call f; f 0 call f; REJECT amd64 386
main 124 nosplit call f; f 0 call f; REJECT amd64 386
main 128 nosplit call f; f 0 call f; REJECT
main 132 nosplit call f; f 0 call f; REJECT
main 136 nosplit call f; f 0 call f; REJECT

# Indirect calls are assumed to be splitting functions.
main 104 nosplit callind
main 108 nosplit callind
main 112 nosplit callind; REJECT amd64
main 116 nosplit callind; REJECT amd64
main 120 nosplit callind; REJECT amd64 386
main 124 nosplit callind; REJECT amd64 386
main 128 nosplit callind; REJECT
main 132 nosplit callind; REJECT
main 136 nosplit callind; REJECT

# Issue 7623
main 0 call f; f 112
main 0 call f; f 116
main 0 call f; f 120
main 0 call f; f 124
main 0 call f; f 128
main 0 call f; f 132
main 0 call f; f 136
`

var (
	commentRE = regexp.MustCompile(`(?m)^#.*`)
	rejectRE  = regexp.MustCompile(`(?s)\A(.+?)((\n|; *)REJECT(.*))?\z`)
	lineRE    = regexp.MustCompile(`(\w+) (\d+)( nosplit)?(.*)`)
	callRE    = regexp.MustCompile(`\bcall (\w+)\b`)
	callindRE = regexp.MustCompile(`\bcallind\b`)
)

func main() {
	goarch := os.Getenv("GOARCH")
	if goarch == "" {
		goarch = runtime.GOARCH
	}

	dir, err := ioutil.TempDir("", "go-test-nosplit")
	if err != nil {
		bug()
		fmt.Printf("creating temp dir: %v\n", err)
		return
	}
	defer os.RemoveAll(dir)
	ioutil.WriteFile(filepath.Join(dir, "main.go"), []byte("package main\nfunc main()\n"), 0666)

	tests = strings.Replace(tests, "\t", " ", -1)
	tests = commentRE.ReplaceAllString(tests, "")

	nok := 0
	nfail := 0
TestCases:
	for len(tests) > 0 {
		var stanza string
		i := strings.Index(tests, "\nmain ")
		if i < 0 {
			stanza, tests = tests, ""
		} else {
			stanza, tests = tests[:i], tests[i+1:]
		}

		m := rejectRE.FindStringSubmatch(stanza)
		if m == nil {
			bug()
			fmt.Printf("invalid stanza:\n\t%s\n", indent(stanza))
			continue
		}
		lines := strings.TrimSpace(m[1])
		reject := false
		if m[2] != "" {
			if strings.TrimSpace(m[4]) == "" {
				reject = true
			} else {
				for _, rej := range strings.Fields(m[4]) {
					if rej == goarch {
						reject = true
					}
				}
			}
		}
		if lines == "" && !reject {
			continue
		}

		var buf bytes.Buffer
		ptrSize := 4
		switch goarch {
		case "power64", "power64le":
			ptrSize = 8
			fmt.Fprintf(&buf, "#define CALL BL\n#define REGISTER (R0)\n#define RET RETURN\n")
		case "arm":
			fmt.Fprintf(&buf, "#define CALL BL\n#define REGISTER (R0)\n")
		case "amd64":
			ptrSize = 8
			fmt.Fprintf(&buf, "#define REGISTER AX\n")
		default:
			fmt.Fprintf(&buf, "#define REGISTER AX\n")
		}

		for _, line := range strings.Split(lines, "\n") {
			line = strings.TrimSpace(line)
			if line == "" {
				continue
			}
			for _, subline := range strings.Split(line, ";") {
				subline = strings.TrimSpace(subline)
				if subline == "" {
					continue
				}
				m := lineRE.FindStringSubmatch(subline)
				if m == nil {
					bug()
					fmt.Printf("invalid function line: %s\n", subline)
					continue TestCases
				}
				name := m[1]
				size, _ := strconv.Atoi(m[2])
				if size%ptrSize == 4 {
					continue TestCases
				}
				nosplit := m[3]
				body := m[4]

				if nosplit != "" {
					nosplit = ",7"
				} else {
					nosplit = ",0"
				}
				body = callRE.ReplaceAllString(body, "CALL ·$1(SB);")
				body = callindRE.ReplaceAllString(body, "CALL REGISTER;")

				fmt.Fprintf(&buf, "TEXT ·%s(SB)%s,$%d-0\n\t%s\n\tRET\n\n", name, nosplit, size, body)
			}
		}

		ioutil.WriteFile(filepath.Join(dir, "asm.s"), buf.Bytes(), 0666)

		cmd := exec.Command("go", "build")
		cmd.Dir = dir
		output, err := cmd.CombinedOutput()
		if err == nil {
			nok++
			if reject {
				bug()
				fmt.Printf("accepted incorrectly:\n\t%s\n", indent(strings.TrimSpace(stanza)))
			}
		} else {
			nfail++
			if !reject {
				bug()
				fmt.Printf("rejected incorrectly:\n\t%s\n", indent(strings.TrimSpace(stanza)))
				fmt.Printf("\n\tlinker output:\n\t%s\n", indent(string(output)))
			}
		}
	}

	if !bugged && (nok == 0 || nfail == 0) {
		bug()
		fmt.Printf("not enough test cases run\n")
	}
}

func indent(s string) string {
	return strings.Replace(s, "\n", "\n\t", -1)
}

var bugged = false

func bug() {
	if !bugged {
		bugged = true
		fmt.Printf("BUG\n")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// request contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package test

import (
	"context"
	"fmt"
	"net/http"
	"strings"

	"github.com/knative/pkg/test/logging"
	"github.com/knative/pkg/test/spoof"
	"go.opencensus.io/trace"
)

// MatchesAny is a NOP matcher. This is useful for polling until a 200 is returned.
func MatchesAny(_ *spoof.Response) (bool, error) {
	return true, nil
}

// Retrying modifies a ResponseChecker to retry certain response codes.
func Retrying(rc spoof.ResponseChecker, codes ...int) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, code := range codes {
			if resp.StatusCode == code {
				// Returning (false, nil) causes SpoofingClient.Poll to retry.
				// sc.logger.Infof("Retrying for code %v", resp.StatusCode)
				return false, nil
			}
		}

		// If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped.
		return rc(resp)
	}
}

// MatchesBody checks that the *first* response body matches the "expected" body, otherwise failing.
func MatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (true, err) causes SpoofingClient.Poll to fail.
			return true, fmt.Errorf("body mismatch: got %q, want %q", string(resp.Body), expected)
		}

		return true, nil
	}
}

// EventuallyMatchesBody checks that the response body *eventually* matches the expected body.
// TODO(#1178): Delete me. We don't want to need this; we should be waiting for an appropriate Status instead.
func EventuallyMatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (false, nil) causes SpoofingClient.Poll to retry.
			return false, nil
		}

		return true, nil
	}
}

// WaitForEndpointState will poll an endpoint until inState indicates the state is achieved.
// If resolvableDomain is false, it will use kubeClientset to look up the ingress and spoof
// the domain in the request headers, otherwise it will make the request directly to domain.
// desc will be used to name the metric that is emitted to track how long it took for the
// domain to get into the state checked by inState.  Commas in `desc` must be escaped.
func WaitForEndpointState(kubeClient *KubeClient, logger *logging.BaseLogger, domain string, inState spoof.ResponseChecker, desc string, resolvable bool) error {
	metricName := fmt.Sprintf("WaitForEndpointState/%s", desc)
	_, span := trace.StartSpan(context.Background(), metricName)
	defer span.End()

	client, err := NewSpoofingClient(kubeClient, logger, domain, resolvable)
	if err != nil {
		return err
	}

	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", domain), nil)
	if err != nil {
		return err
	}

	_, err = client.Poll(req, inState)
	return err
}
<commit_msg>Update WaitForEndpointState to return response (#51)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// request contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package test

import (
	"context"
	"fmt"
	"net/http"
	"strings"

	"github.com/knative/pkg/test/logging"
	"github.com/knative/pkg/test/spoof"
	"go.opencensus.io/trace"
)

// MatchesAny is a NOP matcher. This is useful for polling until a 200 is returned.
func MatchesAny(_ *spoof.Response) (bool, error) {
	return true, nil
}

// Retrying modifies a ResponseChecker to retry certain response codes.
func Retrying(rc spoof.ResponseChecker, codes ...int) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, code := range codes {
			if resp.StatusCode == code {
				// Returning (false, nil) causes SpoofingClient.Poll to retry.
				// sc.logger.Infof("Retrying for code %v", resp.StatusCode)
				return false, nil
			}
		}

		// If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped.
		return rc(resp)
	}
}

// MatchesBody checks that the *first* response body matches the "expected" body, otherwise failing.
func MatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (true, err) causes SpoofingClient.Poll to fail.
			return true, fmt.Errorf("body mismatch: got %q, want %q", string(resp.Body), expected)
		}

		return true, nil
	}
}

// EventuallyMatchesBody checks that the response body *eventually* matches the expected body.
// TODO(#1178): Delete me. We don't want to need this; we should be waiting for an appropriate Status instead.
func EventuallyMatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (false, nil) causes SpoofingClient.Poll to retry.
			return false, nil
		}

		return true, nil
	}
}

// WaitForEndpointState will poll an endpoint until inState indicates the state is achieved.
// If resolvableDomain is false, it will use kubeClientset to look up the ingress and spoof
// the domain in the request headers, otherwise it will make the request directly to domain.
// desc will be used to name the metric that is emitted to track how long it took for the
// domain to get into the state checked by inState.  Commas in `desc` must be escaped.
func WaitForEndpointState(kubeClient *KubeClient, logger *logging.BaseLogger, domain string, inState spoof.ResponseChecker, desc string, resolvable bool) (*spoof.Response, error) {
	metricName := fmt.Sprintf("WaitForEndpointState/%s", desc)
	_, span := trace.StartSpan(context.Background(), metricName)
	defer span.End()

	client, err := NewSpoofingClient(kubeClient, logger, domain, resolvable)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", domain), nil)
	if err != nil {
		return nil, err
	}

	return client.Poll(req, inState)
}
<|endoftext|>


<commit_before>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// make sure that even if a file imports runtime,
// it cannot get at the low-level runtime definitions
// known to the compiler.  for normal packages
// the compiler doesn't even record the lower case
// functions in its symbol table, but some functions
// in runtime are hard-coded into the compiler.

package main

import "runtime"

func main() {
	runtime.printbool(true);	// ERROR "cannot refer|undefined identifier"
}
<commit_msg>New gccgo error message; match both compilers with one string.<commit_after>// errchk $G $D/$F.go

// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// make sure that even if a file imports runtime,
// it cannot get at the low-level runtime definitions
// known to the compiler.  for normal packages
// the compiler doesn't even record the lower case
// functions in its symbol table, but some functions
// in runtime are hard-coded into the compiler.

package main

import "runtime"

func main() {
	runtime.printbool(true);	// ERROR "unexported"
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"transifex"
	"transifex/cli"
	"transifex/config"
)

var sourceLang string
var rootDir string
var transifexApi transifex.TransifexAPI
var existingResources = make(map[string]bool)

func main() {
	transifexCLI := cli.NewCLI()
	transifexApi = transifex.NewTransifexAPI(transifexCLI.ProjectSlug(), transifexCLI.Username(), transifexCLI.Password())
	rootDir = transifexCLI.RootDir()
	// transifexApi.Debug = true
	var err error
	if err = transifexApi.ValidateConfiguration(); err != nil {
		log.Fatalf(err.Error())
	}

	if sourceLang, err = transifexApi.SourceLanguage(); err != nil {
		log.Fatalf("Error loading the transifext project data.")
	}

	files, readFilesErr := config.ReadConfig(transifexCLI.ConfigFile(), rootDir, sourceLang, transifexApi)

	if readFilesErr != nil {
		fmt.Println(rootDir)
		log.Fatalf("Error reading reading language files: \n\n%s", readFilesErr)
	}

	readExistingResources()

	doneChannel := make(chan string, len(files))
	defer close(doneChannel)

	for _, file := range files {
		go upload(doneChannel, file)
	}

	for done := 0; done < len(files); {
		<-doneChannel

		done++
	}
}

func upload(doneChannel chan string, file config.LocalizationFile) {
	uploadFile(file)
	addTranslations(file)
	doneChannel <- file.Slug
}

func readBody(resp http.Response) []byte {
	bytes, readErr := ioutil.ReadAll(resp.Body)
	if readErr != nil {
		log.Fatalf("Failed to read response %s\n", readErr)
	}
	return bytes
}

func loadContent(lang string, file config.LocalizationFile) string {
	filename := file.Translations[sourceLang]
	content, fileErr := ioutil.ReadFile(rootDir + filename)
	if fileErr != nil {
		log.Fatalf("Unable to load file: %s", fileErr)
	}
	switch file.I18nType {
	case transifex.KeyValueJson:
		var data map[string]string
		jsonErr := json.Unmarshal(content, &data)
		if jsonErr != nil {
			log.Fatalf("%s is identifies as %s in the configuration file but is not valid json: %s", filename, file.I18nType, jsonErr)
		}
		for key, value := range data {
			if key == "" {
				delete(data, key)
			}
			if value == "" {
				data[key] = " "
			}
			content, jsonErr = json.Marshal(data)
			if jsonErr != nil {
				panic("An error occurred when encoding json after updating json so that transifex can use it")
			}
		}
	}
	return string(content)
}

func uploadFile(file config.LocalizationFile) {
	slug := file.Slug
	filename := file.Translations[sourceLang]
	content := loadContent(sourceLang, file)
	req := transifex.UploadResourceRequest{file.BaseResource, string(content), "true"}

	if _, has := existingResources[slug]; !has {
		fmt.Printf("Creating new resource: '%s' '%s'\n", filename, slug)
		err := transifexApi.CreateResource(req)
		if err != nil {
			log.Fatalf("Error encountered sending the request to transifex: \n%s'n", err)
		}

		fmt.Printf("Finished Adding '%s'\n", slug)
	} else {
		if err := transifexApi.UpdateResourceContent(slug, string(content)); err != nil {
			log.Fatalf("Error updating content")
		}

		fmt.Printf("Finished Updating '%s'\n", slug)
	}
}

func readExistingResources() {
	resources, err := transifexApi.ListResources()
	if err != nil {
		log.Fatalf("Unable to load resources: %s", err)
	}
	for _, res := range resources {
		existingResources[res.Slug] = true
	}
}

func addTranslations(file config.LocalizationFile) {
	for lang, _ := range file.Translations {
		if lang != sourceLang {
			content := loadContent(lang, file)

			transifexApi.UploadTranslationFile(file.Slug, lang, content)
		}
	}
}
<commit_msg>by default do not upload translations<commit_after>package main

import (
	"flag"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"transifex"
	"transifex/cli"
	"transifex/config"
)

var sourceLang string
var rootDir string
var transifexApi transifex.TransifexAPI
var existingResources = make(map[string]bool)
var uploadTranslations = flag.Bool("translations", false, "If true the translations will also be uploaded.  WARNING all changes on transifex for the uploaded languages will be lost");

func main() {
	transifexCLI := cli.NewCLI()
	transifexApi = transifex.NewTransifexAPI(transifexCLI.ProjectSlug(), transifexCLI.Username(), transifexCLI.Password())
	rootDir = transifexCLI.RootDir()
	// transifexApi.Debug = true
	var err error
	if err = transifexApi.ValidateConfiguration(); err != nil {
		log.Fatalf(err.Error())
	}

	if sourceLang, err = transifexApi.SourceLanguage(); err != nil {
		log.Fatalf("Error loading the transifext project data.")
	}

	files, readFilesErr := config.ReadConfig(transifexCLI.ConfigFile(), rootDir, sourceLang, transifexApi)

	if readFilesErr != nil {
		fmt.Println(rootDir)
		log.Fatalf("Error reading reading language files: \n\n%s", readFilesErr)
	}

	readExistingResources()

	doneChannel := make(chan string, len(files))
	defer close(doneChannel)

	for _, file := range files {
		go upload(doneChannel, file)
	}

	for done := 0; done < len(files); {
		<-doneChannel

		done++
	}
}

func upload(doneChannel chan string, file config.LocalizationFile) {
	uploadFile(file)
	if ! *uploadTranslations {
		addTranslations(file)
	}
	doneChannel <- file.Slug
}

func readBody(resp http.Response) []byte {
	bytes, readErr := ioutil.ReadAll(resp.Body)
	if readErr != nil {
		log.Fatalf("Failed to read response %s\n", readErr)
	}
	return bytes
}

func loadContent(lang string, file config.LocalizationFile) string {
	filename := file.Translations[sourceLang]
	content, fileErr := ioutil.ReadFile(rootDir + filename)
	if fileErr != nil {
		log.Fatalf("Unable to load file: %s", fileErr)
	}
	switch file.I18nType {
	case transifex.KeyValueJson:
		var data map[string]string
		jsonErr := json.Unmarshal(content, &data)
		if jsonErr != nil {
			log.Fatalf("%s is identifies as %s in the configuration file but is not valid json: %s", filename, file.I18nType, jsonErr)
		}
		for key, value := range data {
			if key == "" {
				delete(data, key)
			}
			if value == "" {
				data[key] = " "
			}
			content, jsonErr = json.Marshal(data)
			if jsonErr != nil {
				panic("An error occurred when encoding json after updating json so that transifex can use it")
			}
		}
	}
	return string(content)
}

func uploadFile(file config.LocalizationFile) {
	slug := file.Slug
	filename := file.Translations[sourceLang]
	content := loadContent(sourceLang, file)
	req := transifex.UploadResourceRequest{file.BaseResource, string(content), "true"}

	if _, has := existingResources[slug]; !has {
		fmt.Printf("Creating new resource: '%s' '%s'\n", filename, slug)
		err := transifexApi.CreateResource(req)
		if err != nil {
			log.Fatalf("Error encountered sending the request to transifex: \n%s'n", err)
		}

		fmt.Printf("Finished Adding '%s'\n", slug)
	} else {
		if err := transifexApi.UpdateResourceContent(slug, string(content)); err != nil {
			log.Fatalf("Error updating content")
		}

		fmt.Printf("Finished Updating '%s'\n", slug)
	}
}

func readExistingResources() {
	resources, err := transifexApi.ListResources()
	if err != nil {
		log.Fatalf("Unable to load resources: %s", err)
	}
	for _, res := range resources {
		existingResources[res.Slug] = true
	}
}

func addTranslations(file config.LocalizationFile) {
	for lang, _ := range file.Translations {
		if lang != sourceLang {
			content := loadContent(lang, file)

			transifexApi.UploadTranslationFile(file.Slug, lang, content)
		}
	}
}
<|endoftext|>


<commit_before>package dht

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/libp2p/go-libp2p-core/helpers"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"

	"github.com/libp2p/go-libp2p-kad-dht/metrics"
	pb "github.com/libp2p/go-libp2p-kad-dht/pb"
	msmux "github.com/multiformats/go-multistream"

	ggio "github.com/gogo/protobuf/io"

	"github.com/libp2p/go-msgio"
	"go.opencensus.io/stats"
	"go.opencensus.io/tag"
)

var dhtReadMessageTimeout = time.Minute
var dhtStreamIdleTimeout = 1 * time.Minute
var ErrReadTimeout = fmt.Errorf("timed out reading response")

// The Protobuf writer performs multiple small writes when writing a message.
// We need to buffer those writes, to make sure that we're not sending a new
// packet for every single write.
type bufferedDelimitedWriter struct {
	*bufio.Writer
	ggio.WriteCloser
}

var writerPool = sync.Pool{
	New: func() interface{} {
		w := bufio.NewWriter(nil)
		return &bufferedDelimitedWriter{
			Writer:      w,
			WriteCloser: ggio.NewDelimitedWriter(w),
		}
	},
}

func writeMsg(w io.Writer, mes *pb.Message) error {
	bw := writerPool.Get().(*bufferedDelimitedWriter)
	bw.Reset(w)
	err := bw.WriteMsg(mes)
	if err == nil {
		err = bw.Flush()
	}
	bw.Reset(nil)
	writerPool.Put(bw)
	return err
}

func (w *bufferedDelimitedWriter) Flush() error {
	return w.Writer.Flush()
}

// handleNewStream implements the network.StreamHandler
func (dht *IpfsDHT) handleNewStream(s network.Stream) {
	defer s.Reset()
	if dht.handleNewMessage(s) {
		// Gracefully close the stream for writes.
		s.Close()
	}
}

// Returns true on orderly completion of writes (so we can Close the stream).
func (dht *IpfsDHT) handleNewMessage(s network.Stream) bool {
	ctx := dht.ctx
	r := msgio.NewVarintReaderSize(s, network.MessageSizeMax)

	mPeer := s.Conn().RemotePeer()

	timer := time.AfterFunc(dhtStreamIdleTimeout, func() { s.Reset() })
	defer timer.Stop()

	for {
		if dht.getMode() != modeServer {
			logger.Errorf("ignoring incoming dht message while not in server mode")
			return false
		}

		var req pb.Message
		msgbytes, err := r.ReadMsg()
		msgLen := len(msgbytes)
		if err != nil {
			r.ReleaseMsg(msgbytes)
			if err == io.EOF {
				return true
			}
			// This string test is necessary because there isn't a single stream reset error
			// instance	in use.
			if err.Error() != "stream reset" {
				logger.Debugf("error reading message: %#v", err)
			}
			if msgLen > 0 {
				stats.RecordWithTags(ctx,
					[]tag.Mutator{tag.Upsert(metrics.KeyMessageType, "UNKNOWN")},
					metrics.ReceivedMessages.M(1),
					metrics.ReceivedMessageErrors.M(1),
					metrics.ReceivedBytes.M(int64(msgLen)),
				)
			}
			return false
		}
		err = req.Unmarshal(msgbytes)
		r.ReleaseMsg(msgbytes)
		if err != nil {
			logger.Debugf("error unmarshalling message: %#v", err)
			stats.RecordWithTags(ctx,
				[]tag.Mutator{tag.Upsert(metrics.KeyMessageType, "UNKNOWN")},
				metrics.ReceivedMessages.M(1),
				metrics.ReceivedMessageErrors.M(1),
				metrics.ReceivedBytes.M(int64(msgLen)),
			)
			return false
		}

		timer.Reset(dhtStreamIdleTimeout)

		startTime := time.Now()
		ctx, _ := tag.New(ctx,
			tag.Upsert(metrics.KeyMessageType, req.GetType().String()),
		)

		stats.Record(ctx,
			metrics.ReceivedMessages.M(1),
			metrics.ReceivedBytes.M(int64(msgLen)),
		)

		handler := dht.handlerForMsgType(req.GetType())
		if handler == nil {
			stats.Record(ctx, metrics.ReceivedMessageErrors.M(1))
			logger.Warningf("can't handle received message of type %v", req.GetType())
			return false
		}

		resp, err := handler(ctx, mPeer, &req)
		if err != nil {
			stats.Record(ctx, metrics.ReceivedMessageErrors.M(1))
			logger.Debugf("error handling message: %v", err)
			return false
		}

		if resp == nil {
			continue
		}

		// send out response msg
		err = writeMsg(s, resp)
		if err != nil {
			stats.Record(ctx, metrics.ReceivedMessageErrors.M(1))
			logger.Debugf("error writing response: %v", err)
			return false
		}

		elapsedTime := time.Since(startTime)
		latencyMillis := float64(elapsedTime) / float64(time.Millisecond)
		stats.Record(ctx, metrics.InboundRequestLatency.M(latencyMillis))
	}
}

// sendRequest sends out a request, but also makes sure to
// measure the RTT for latency measurements.
func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	ctx, _ = tag.New(ctx, metrics.UpsertMessageType(pmes))

	ms, err := dht.messageSenderForPeer(ctx, p)
	if err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentRequests.M(1),
			metrics.SentRequestErrors.M(1),
		)
		return nil, err
	}

	start := time.Now()

	rpmes, err := ms.SendRequest(ctx, pmes)
	if err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentRequests.M(1),
			metrics.SentRequestErrors.M(1),
		)
		return nil, err
	}

	stats.Record(ctx,
		metrics.SentRequests.M(1),
		metrics.SentBytes.M(int64(pmes.Size())),
		metrics.OutboundRequestLatency.M(float64(time.Since(start))/float64(time.Millisecond)),
	)
	dht.peerstore.RecordLatency(p, time.Since(start))
	logger.Event(ctx, "dhtReceivedMessage", dht.self, p, rpmes)
	return rpmes, nil
}

// sendMessage sends out a message
func (dht *IpfsDHT) sendMessage(ctx context.Context, p peer.ID, pmes *pb.Message) error {
	ctx, _ = tag.New(ctx, metrics.UpsertMessageType(pmes))

	ms, err := dht.messageSenderForPeer(ctx, p)
	if err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentMessages.M(1),
			metrics.SentMessageErrors.M(1),
		)
		return err
	}

	if err := ms.SendMessage(ctx, pmes); err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentMessages.M(1),
			metrics.SentMessageErrors.M(1),
		)
		return err
	}

	stats.Record(ctx,
		metrics.SentMessages.M(1),
		metrics.SentBytes.M(int64(pmes.Size())),
	)

	logger.Event(ctx, "dhtSentMessage", dht.self, p, pmes)
	return nil
}

func (dht *IpfsDHT) messageSenderForPeer(ctx context.Context, p peer.ID) (*messageSender, error) {
	dht.smlk.Lock()
	ms, ok := dht.strmap[p]
	if ok {
		dht.smlk.Unlock()
		return ms, nil
	}
	ms = &messageSender{p: p, dht: dht, lk: newCtxMutex()}
	dht.strmap[p] = ms
	dht.smlk.Unlock()

	if err := ms.prepOrInvalidate(ctx); err != nil {
		dht.smlk.Lock()
		defer dht.smlk.Unlock()

		if msCur, ok := dht.strmap[p]; ok {
			// Changed. Use the new one, old one is invalid and
			// not in the map so we can just throw it away.
			if ms != msCur {
				return msCur, nil
			}
			// Not changed, remove the now invalid stream from the
			// map.
			delete(dht.strmap, p)
		}
		// Invalid but not in map. Must have been removed by a disconnect.
		return nil, err
	}
	// All ready to go.
	return ms, nil
}

type messageSender struct {
	s   network.Stream
	r   msgio.ReadCloser
	lk  ctxMutex
	p   peer.ID
	dht *IpfsDHT

	invalid   bool
	singleMes int
}

// invalidate is called before this messageSender is removed from the strmap.
// It prevents the messageSender from being reused/reinitialized and then
// forgotten (leaving the stream open).
func (ms *messageSender) invalidate() {
	ms.invalid = true
	if ms.s != nil {
		ms.s.Reset()
		ms.s = nil
	}
}

func (ms *messageSender) prepOrInvalidate(ctx context.Context) error {
	if err := ms.lk.Lock(ctx); err != nil {
		return err
	}
	defer ms.lk.Unlock()

	if err := ms.prep(ctx); err != nil {
		ms.invalidate()
		return err
	}
	return nil
}

func (ms *messageSender) prep(ctx context.Context) error {
	if ms.invalid {
		return fmt.Errorf("message sender has been invalidated")
	}
	if ms.s != nil {
		return nil
	}

	// We only want to speak to peers using our primary protocols. We do not want to query any peer that only speaks
	// one of the secondary "server" protocols that we happen to support (e.g. older nodes that we can respond to for
	// backwards compatibility reasons).
	nstr, err := ms.dht.host.NewStream(ctx, ms.p, ms.dht.protocols...)
	if err != nil {
		return err
	}

	ms.r = msgio.NewVarintReaderSize(nstr, network.MessageSizeMax)
	ms.s = nstr

	return nil
}

// streamReuseTries is the number of times we will try to reuse a stream to a
// given peer before giving up and reverting to the old one-message-per-stream
// behaviour.
const streamReuseTries = 3

func (ms *messageSender) SendMessage(ctx context.Context, pmes *pb.Message) error {
	if err := ms.lk.Lock(ctx); err != nil {
		return err
	}
	defer ms.lk.Unlock()

	retry := false
	for {
		if err := ms.prep(ctx); err != nil {
			return err
		}

		if err := ms.writeMsg(pmes); err != nil {
			ms.s.Reset()
			ms.s = nil

			if retry {
				logger.Info("error writing message, bailing: ", err)
				return err
			}
			logger.Info("error writing message, trying again: ", err)
			retry = true
			continue
		}

		logger.Event(ctx, "dhtSentMessage", ms.dht.self, ms.p, pmes)

		if ms.singleMes > streamReuseTries {
			go helpers.FullClose(ms.s)
			ms.s = nil
		} else if retry {
			ms.singleMes++
		}

		return nil
	}
}

func (ms *messageSender) SendRequest(ctx context.Context, pmes *pb.Message) (*pb.Message, error) {
	if err := ms.lk.Lock(ctx); err != nil {
		return nil, err
	}
	defer ms.lk.Unlock()

	retry := false
	for {
		if err := ms.prep(ctx); err != nil {
			return nil, err
		}

		if err := ms.writeMsg(pmes); err != nil {
			ms.s.Reset()
			ms.s = nil

			if retry {
				logger.Info("error writing message, bailing: ", err)
				return nil, err
			}
			logger.Info("error writing message, trying again: ", err)
			retry = true
			continue
		}

		mes := new(pb.Message)
		if err := ms.ctxReadMsg(ctx, mes); err != nil {
			ms.s.Reset()
			ms.s = nil

			if retry {
				logger.Info("error reading message, bailing: ", err)
				return nil, err
			}
			logger.Info("error reading message, trying again: ", err)
			retry = true
			continue
		}

		logger.Event(ctx, "dhtSentMessage", ms.dht.self, ms.p, pmes)

		if ms.singleMes > streamReuseTries {
			go helpers.FullClose(ms.s)
			ms.s = nil
		} else if retry {
			ms.singleMes++
		}

		return mes, nil
	}
}

func (ms *messageSender) writeMsg(pmes *pb.Message) error {
	return writeMsg(ms.s, pmes)
}

func (ms *messageSender) ctxReadMsg(ctx context.Context, mes *pb.Message) error {
	errc := make(chan error, 1)
	go func(r msgio.ReadCloser) {
		bytes, err := r.ReadMsg()
		defer r.ReleaseMsg(bytes)
		if err != nil {
			errc <- err
			return
		}
		errc <- mes.Unmarshal(bytes)
	}(ms.r)

	t := time.NewTimer(dhtReadMessageTimeout)
	defer t.Stop()

	select {
	case err := <-errc:
		return err
	case <-ctx.Done():
		return ctx.Err()
	case <-t.C:
		return ErrReadTimeout
	}
}
<commit_msg>feat: reduce request read timeout (#527)<commit_after>package dht

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/libp2p/go-libp2p-core/helpers"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"

	"github.com/libp2p/go-libp2p-kad-dht/metrics"
	pb "github.com/libp2p/go-libp2p-kad-dht/pb"
	msmux "github.com/multiformats/go-multistream"

	ggio "github.com/gogo/protobuf/io"

	"github.com/libp2p/go-msgio"
	"go.opencensus.io/stats"
	"go.opencensus.io/tag"
)

var dhtReadMessageTimeout = 10 * time.Second
var dhtStreamIdleTimeout = 1 * time.Minute
var ErrReadTimeout = fmt.Errorf("timed out reading response")

// The Protobuf writer performs multiple small writes when writing a message.
// We need to buffer those writes, to make sure that we're not sending a new
// packet for every single write.
type bufferedDelimitedWriter struct {
	*bufio.Writer
	ggio.WriteCloser
}

var writerPool = sync.Pool{
	New: func() interface{} {
		w := bufio.NewWriter(nil)
		return &bufferedDelimitedWriter{
			Writer:      w,
			WriteCloser: ggio.NewDelimitedWriter(w),
		}
	},
}

func writeMsg(w io.Writer, mes *pb.Message) error {
	bw := writerPool.Get().(*bufferedDelimitedWriter)
	bw.Reset(w)
	err := bw.WriteMsg(mes)
	if err == nil {
		err = bw.Flush()
	}
	bw.Reset(nil)
	writerPool.Put(bw)
	return err
}

func (w *bufferedDelimitedWriter) Flush() error {
	return w.Writer.Flush()
}

// handleNewStream implements the network.StreamHandler
func (dht *IpfsDHT) handleNewStream(s network.Stream) {
	defer s.Reset()
	if dht.handleNewMessage(s) {
		// Gracefully close the stream for writes.
		s.Close()
	}
}

// Returns true on orderly completion of writes (so we can Close the stream).
func (dht *IpfsDHT) handleNewMessage(s network.Stream) bool {
	ctx := dht.ctx
	r := msgio.NewVarintReaderSize(s, network.MessageSizeMax)

	mPeer := s.Conn().RemotePeer()

	timer := time.AfterFunc(dhtStreamIdleTimeout, func() { s.Reset() })
	defer timer.Stop()

	for {
		if dht.getMode() != modeServer {
			logger.Errorf("ignoring incoming dht message while not in server mode")
			return false
		}

		var req pb.Message
		msgbytes, err := r.ReadMsg()
		msgLen := len(msgbytes)
		if err != nil {
			r.ReleaseMsg(msgbytes)
			if err == io.EOF {
				return true
			}
			// This string test is necessary because there isn't a single stream reset error
			// instance	in use.
			if err.Error() != "stream reset" {
				logger.Debugf("error reading message: %#v", err)
			}
			if msgLen > 0 {
				stats.RecordWithTags(ctx,
					[]tag.Mutator{tag.Upsert(metrics.KeyMessageType, "UNKNOWN")},
					metrics.ReceivedMessages.M(1),
					metrics.ReceivedMessageErrors.M(1),
					metrics.ReceivedBytes.M(int64(msgLen)),
				)
			}
			return false
		}
		err = req.Unmarshal(msgbytes)
		r.ReleaseMsg(msgbytes)
		if err != nil {
			logger.Debugf("error unmarshalling message: %#v", err)
			stats.RecordWithTags(ctx,
				[]tag.Mutator{tag.Upsert(metrics.KeyMessageType, "UNKNOWN")},
				metrics.ReceivedMessages.M(1),
				metrics.ReceivedMessageErrors.M(1),
				metrics.ReceivedBytes.M(int64(msgLen)),
			)
			return false
		}

		timer.Reset(dhtStreamIdleTimeout)

		startTime := time.Now()
		ctx, _ := tag.New(ctx,
			tag.Upsert(metrics.KeyMessageType, req.GetType().String()),
		)

		stats.Record(ctx,
			metrics.ReceivedMessages.M(1),
			metrics.ReceivedBytes.M(int64(msgLen)),
		)

		handler := dht.handlerForMsgType(req.GetType())
		if handler == nil {
			stats.Record(ctx, metrics.ReceivedMessageErrors.M(1))
			logger.Warningf("can't handle received message of type %v", req.GetType())
			return false
		}

		resp, err := handler(ctx, mPeer, &req)
		if err != nil {
			stats.Record(ctx, metrics.ReceivedMessageErrors.M(1))
			logger.Debugf("error handling message: %v", err)
			return false
		}

		if resp == nil {
			continue
		}

		// send out response msg
		err = writeMsg(s, resp)
		if err != nil {
			stats.Record(ctx, metrics.ReceivedMessageErrors.M(1))
			logger.Debugf("error writing response: %v", err)
			return false
		}

		elapsedTime := time.Since(startTime)
		latencyMillis := float64(elapsedTime) / float64(time.Millisecond)
		stats.Record(ctx, metrics.InboundRequestLatency.M(latencyMillis))
	}
}

// sendRequest sends out a request, but also makes sure to
// measure the RTT for latency measurements.
func (dht *IpfsDHT) sendRequest(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	ctx, _ = tag.New(ctx, metrics.UpsertMessageType(pmes))

	ms, err := dht.messageSenderForPeer(ctx, p)
	if err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentRequests.M(1),
			metrics.SentRequestErrors.M(1),
		)
		return nil, err
	}

	start := time.Now()

	rpmes, err := ms.SendRequest(ctx, pmes)
	if err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentRequests.M(1),
			metrics.SentRequestErrors.M(1),
		)
		return nil, err
	}

	stats.Record(ctx,
		metrics.SentRequests.M(1),
		metrics.SentBytes.M(int64(pmes.Size())),
		metrics.OutboundRequestLatency.M(float64(time.Since(start))/float64(time.Millisecond)),
	)
	dht.peerstore.RecordLatency(p, time.Since(start))
	logger.Event(ctx, "dhtReceivedMessage", dht.self, p, rpmes)
	return rpmes, nil
}

// sendMessage sends out a message
func (dht *IpfsDHT) sendMessage(ctx context.Context, p peer.ID, pmes *pb.Message) error {
	ctx, _ = tag.New(ctx, metrics.UpsertMessageType(pmes))

	ms, err := dht.messageSenderForPeer(ctx, p)
	if err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentMessages.M(1),
			metrics.SentMessageErrors.M(1),
		)
		return err
	}

	if err := ms.SendMessage(ctx, pmes); err != nil {
		if err == msmux.ErrNotSupported {
			dht.peerStoppedDHT(ctx, p)
		}
		stats.Record(ctx,
			metrics.SentMessages.M(1),
			metrics.SentMessageErrors.M(1),
		)
		return err
	}

	stats.Record(ctx,
		metrics.SentMessages.M(1),
		metrics.SentBytes.M(int64(pmes.Size())),
	)

	logger.Event(ctx, "dhtSentMessage", dht.self, p, pmes)
	return nil
}

func (dht *IpfsDHT) messageSenderForPeer(ctx context.Context, p peer.ID) (*messageSender, error) {
	dht.smlk.Lock()
	ms, ok := dht.strmap[p]
	if ok {
		dht.smlk.Unlock()
		return ms, nil
	}
	ms = &messageSender{p: p, dht: dht, lk: newCtxMutex()}
	dht.strmap[p] = ms
	dht.smlk.Unlock()

	if err := ms.prepOrInvalidate(ctx); err != nil {
		dht.smlk.Lock()
		defer dht.smlk.Unlock()

		if msCur, ok := dht.strmap[p]; ok {
			// Changed. Use the new one, old one is invalid and
			// not in the map so we can just throw it away.
			if ms != msCur {
				return msCur, nil
			}
			// Not changed, remove the now invalid stream from the
			// map.
			delete(dht.strmap, p)
		}
		// Invalid but not in map. Must have been removed by a disconnect.
		return nil, err
	}
	// All ready to go.
	return ms, nil
}

type messageSender struct {
	s   network.Stream
	r   msgio.ReadCloser
	lk  ctxMutex
	p   peer.ID
	dht *IpfsDHT

	invalid   bool
	singleMes int
}

// invalidate is called before this messageSender is removed from the strmap.
// It prevents the messageSender from being reused/reinitialized and then
// forgotten (leaving the stream open).
func (ms *messageSender) invalidate() {
	ms.invalid = true
	if ms.s != nil {
		ms.s.Reset()
		ms.s = nil
	}
}

func (ms *messageSender) prepOrInvalidate(ctx context.Context) error {
	if err := ms.lk.Lock(ctx); err != nil {
		return err
	}
	defer ms.lk.Unlock()

	if err := ms.prep(ctx); err != nil {
		ms.invalidate()
		return err
	}
	return nil
}

func (ms *messageSender) prep(ctx context.Context) error {
	if ms.invalid {
		return fmt.Errorf("message sender has been invalidated")
	}
	if ms.s != nil {
		return nil
	}

	// We only want to speak to peers using our primary protocols. We do not want to query any peer that only speaks
	// one of the secondary "server" protocols that we happen to support (e.g. older nodes that we can respond to for
	// backwards compatibility reasons).
	nstr, err := ms.dht.host.NewStream(ctx, ms.p, ms.dht.protocols...)
	if err != nil {
		return err
	}

	ms.r = msgio.NewVarintReaderSize(nstr, network.MessageSizeMax)
	ms.s = nstr

	return nil
}

// streamReuseTries is the number of times we will try to reuse a stream to a
// given peer before giving up and reverting to the old one-message-per-stream
// behaviour.
const streamReuseTries = 3

func (ms *messageSender) SendMessage(ctx context.Context, pmes *pb.Message) error {
	if err := ms.lk.Lock(ctx); err != nil {
		return err
	}
	defer ms.lk.Unlock()

	retry := false
	for {
		if err := ms.prep(ctx); err != nil {
			return err
		}

		if err := ms.writeMsg(pmes); err != nil {
			ms.s.Reset()
			ms.s = nil

			if retry {
				logger.Info("error writing message, bailing: ", err)
				return err
			}
			logger.Info("error writing message, trying again: ", err)
			retry = true
			continue
		}

		logger.Event(ctx, "dhtSentMessage", ms.dht.self, ms.p, pmes)

		if ms.singleMes > streamReuseTries {
			go helpers.FullClose(ms.s)
			ms.s = nil
		} else if retry {
			ms.singleMes++
		}

		return nil
	}
}

func (ms *messageSender) SendRequest(ctx context.Context, pmes *pb.Message) (*pb.Message, error) {
	if err := ms.lk.Lock(ctx); err != nil {
		return nil, err
	}
	defer ms.lk.Unlock()

	retry := false
	for {
		if err := ms.prep(ctx); err != nil {
			return nil, err
		}

		if err := ms.writeMsg(pmes); err != nil {
			ms.s.Reset()
			ms.s = nil

			if retry {
				logger.Info("error writing message, bailing: ", err)
				return nil, err
			}
			logger.Info("error writing message, trying again: ", err)
			retry = true
			continue
		}

		mes := new(pb.Message)
		if err := ms.ctxReadMsg(ctx, mes); err != nil {
			ms.s.Reset()
			ms.s = nil

			if retry {
				logger.Info("error reading message, bailing: ", err)
				return nil, err
			}
			logger.Info("error reading message, trying again: ", err)
			retry = true
			continue
		}

		logger.Event(ctx, "dhtSentMessage", ms.dht.self, ms.p, pmes)

		if ms.singleMes > streamReuseTries {
			go helpers.FullClose(ms.s)
			ms.s = nil
		} else if retry {
			ms.singleMes++
		}

		return mes, nil
	}
}

func (ms *messageSender) writeMsg(pmes *pb.Message) error {
	return writeMsg(ms.s, pmes)
}

func (ms *messageSender) ctxReadMsg(ctx context.Context, mes *pb.Message) error {
	errc := make(chan error, 1)
	go func(r msgio.ReadCloser) {
		bytes, err := r.ReadMsg()
		defer r.ReleaseMsg(bytes)
		if err != nil {
			errc <- err
			return
		}
		errc <- mes.Unmarshal(bytes)
	}(ms.r)

	t := time.NewTimer(dhtReadMessageTimeout)
	defer t.Stop()

	select {
	case err := <-errc:
		return err
	case <-ctx.Done():
		return ctx.Err()
	case <-t.C:
		return ErrReadTimeout
	}
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package qan

import (
	"fmt"
	"log"
	"runtime/debug"
	"strconv"
	"sync"
	"time"

	"github.com/percona/percona-agent/data"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/ticker"
)

const (
	MIN_SLOWLOG_ROTATION_SIZE = 4096
	MAX_SLOWLOG_ROTATION_SIZE = 1073741824
)

// A Worker gets queries, aggregates them, and returns a Result. Workers are ran
// by Analyzers. When ran, MySQL is presumed to be configured and ready.
type Worker interface {
	Setup(*Interval) error
	Run() (*Result, error)
	Stop() error
	Cleanup() error
	Status() map[string]string
}

// An Analyzer runs a Worker at each Interval. Analyzers are responsible for
// MySQL: configuring, restarts, etc. The Worker is only ran when MySQL is
// configured and ready. Analyzers are also responsible for making Reports from
// the Results returned by Workers. The Worker determines the type of Analyzer:
// slowlog or perfschema. Analyzers are ran by the QAN Manager.
type Analyzer interface {
	Start() error
	Stop() error
	Status() map[string]string
	String() string
}

// An AnalyzerFactory makes an Analyzer, real or mock.
type AnalyzerFactory interface {
	Make(config Config, name string, mysqlConn mysql.Connector, restartChan <-chan bool, tickChan chan time.Time) Analyzer
}

// --------------------------------------------------------------------------

type RealAnalyzer struct {
	logger      *pct.Logger
	config      Config
	iter        IntervalIter
	mysqlConn   mysql.Connector
	restartChan <-chan bool
	worker      Worker
	clock       ticker.Manager
	spool       data.Spooler
	// --
	name                string
	mysqlConfiguredChan chan bool
	workerDoneChan      chan *Interval
	status              *pct.Status
	runSync             *pct.SyncChan
	configureMySQLSync  *pct.SyncChan
	running             bool
	mux                 *sync.RWMutex
}

func NewRealAnalyzer(logger *pct.Logger, config Config, iter IntervalIter, mysqlConn mysql.Connector, restartChan <-chan bool, worker Worker, clock ticker.Manager, spool data.Spooler) *RealAnalyzer {
	name := logger.Service()
	a := &RealAnalyzer{
		logger:      logger,
		config:      config,
		iter:        iter,
		mysqlConn:   mysqlConn,
		restartChan: restartChan,
		worker:      worker,
		clock:       clock,
		spool:       spool,
		// --
		name:                name,
		mysqlConfiguredChan: make(chan bool, 1),
		workerDoneChan:      make(chan *Interval, 1),
		status:              pct.NewStatus([]string{name, name + "-last-interval", name + "-next-interval"}),
		runSync:             pct.NewSyncChan(),
		configureMySQLSync:  pct.NewSyncChan(),
		mux:                 &sync.RWMutex{},
	}
	return a
}

func (a *RealAnalyzer) String() string {
	return a.name
}

func (a *RealAnalyzer) Start() error {
	a.logger.Debug("Start:call")
	defer a.logger.Debug("Start:return")
	a.mux.Lock()
	defer a.mux.Unlock()
	if a.running {
		return nil
	}
	go a.run()
	a.running = true
	return nil
}

func (a *RealAnalyzer) Stop() error {
	a.logger.Debug("Stop:call")
	defer a.logger.Debug("Stop:return")
	a.mux.Lock()
	defer a.mux.Unlock()
	if !a.running {
		return nil
	}
	a.runSync.Stop()
	a.runSync.Wait()
	a.running = false
	return nil
}

func (a *RealAnalyzer) Status() map[string]string {
	a.mux.RLock()
	defer a.mux.RUnlock()
	if a.running {
		a.status.Update(a.name+"-next-interval", fmt.Sprintf("%.1fs", a.clock.ETA(a.iter.TickChan())))
	} else {
		a.status.Update(a.name+"-next-interval", "")
	}
	return a.status.Merge(a.worker.Status())
}

// --------------------------------------------------------------------------

// Returns a pointer to the config in use by this analizer, useful for testing purposes
func (a *RealAnalyzer) GetConfig() *Config {
	return &a.config
}

// Will try to detect if DB has slowlog rotation enabled and takeover the task
func (a *RealAnalyzer) TakeOverPSRotation() error {
	// Take over slowlog rotation if its set (in Percona Server)
	var maxLogSize int64 = 0
	psMaxLogSize, err := strconv.ParseUint(a.mysqlConn.GetGlobalVarString("max_slowlog_size"), 10, 32)
	if err != nil {
		return err
	}
	maxLogSize = int64(psMaxLogSize)
	if maxLogSize >= MIN_SLOWLOG_ROTATION_SIZE {
		a.logger.Info("Taking over Percona Server slowlog rotation, max_slowlog_size:", maxLogSize)
		a.config.MaxSlowLogSize = maxLogSize

		// Using Set makes testing easier
		disablePSrotation := []mysql.Query{
			mysql.Query{Set: "SET GLOBAL max_slowlog_size = 0"},
		}

		if err := a.mysqlConn.Set(disablePSrotation); err != nil {
			return err
		}
	}
	return nil
}

func (a *RealAnalyzer) configureMySQL(config []mysql.Query, tryLimit int) {
	a.logger.Debug("configureMySQL:call")
	defer func() {
		if err := recover(); err != nil {
			a.logger.Error(a.name+":configureMySQL crashed: ", err)
		}
		a.logger.Debug("configureMySQL:return")
	}()

	try := 0
	for (tryLimit == 0) || (try < tryLimit) {
		try++

		select {
		case <-a.configureMySQLSync.StopChan:
			a.logger.Debug("configureMySQL:stop")
			a.configureMySQLSync.Done()
			return
		default:
		}

		// Connect handles backoff internally.
		a.logger.Debug("configureMySQL:connecting")
		if err := a.mysqlConn.Connect(1); err != nil {
			a.logger.Warn("Cannot connect to MySQL:", err)
			continue
		}
		defer a.mysqlConn.Close()

		a.logger.Debug("configureMySQL:configuring")
		// Try to take over PS slowlog rotation
		if err := a.TakeOverPSRotation(); err != nil {
			a.logger.Warn("Cannot takeover slowlog rotation from Percona Server:", err)
		}
		if err := a.mysqlConn.Set(config); err != nil {
			a.mysqlConn.Close()
			a.logger.Warn("Cannot configure MySQL:", err)
			continue
		}

		a.logger.Debug("configureMySQL:configured")
		select {
		case a.mysqlConfiguredChan <- true:
			return // success
		case <-a.configureMySQLSync.StopChan:
			a.logger.Debug("configureMySQL:stop")
			a.configureMySQLSync.Done()
			return
		}
	}
}

func (a *RealAnalyzer) run() {
	a.logger.Debug("run:call")
	defer a.logger.Debug("run:return")

	mysqlConfigured := false
	go a.configureMySQL(a.config.Start, 0) // try forever

	defer func() {
		a.logger.Info("Stopping")

		a.status.Update(a.name, "Stopping worker")
		a.worker.Stop()

		a.status.Update(a.name, "Stopping interval iter")
		a.iter.Stop()

		if !mysqlConfigured {
			a.status.Update(a.name, "Stopping MySQL config")
			a.configureMySQLSync.Stop()
			a.configureMySQLSync.Wait()
		}

		a.status.Update(a.name, "Stopping QAN on MySQL")
		a.configureMySQL(a.config.Stop, 1) // try once

		if err := recover(); err != nil {
			a.logger.Error("QAN crashed: ", err)
			a.status.Update(a.name, "Crashed")
		} else {
			a.status.Update(a.name, "Stopped")
			a.logger.Info("Stopped")
		}

		a.runSync.Done()
	}()

	workerRunning := false
	lastTs := time.Time{}
	currentInterval := &Interval{}
	for {
		a.logger.Debug("run:idle")
		if mysqlConfigured {
			if workerRunning {
				a.status.Update(a.name, "Running")
			} else {
				a.status.Update(a.name, "Idle")
			}
		} else {
			a.status.Update(a.name, "Trying to configure MySQL")
		}

		select {
		case interval := <-a.iter.IntervalChan():
			if !mysqlConfigured {
				a.logger.Debug(fmt.Sprintf("run:interval:%d:skip (mysql not configured)", interval.Number))
				continue
			}

			if workerRunning {
				a.logger.Warn(fmt.Sprintf("Skipping interval '%s' because interval '%s' is still being parsed",
					interval, currentInterval))
				continue
			}

			a.status.Update(a.name, fmt.Sprintf("Starting interval '%s'", interval))
			a.logger.Debug(fmt.Sprintf("run:interval:%s", interval))
			currentInterval = interval

			// Run the worker, timing it, make a report from its results, spool
			// the report. When done the interval is returned on workerDoneChan.
			go a.runWorker(interval)
			workerRunning = true
		case interval := <-a.workerDoneChan:
			a.logger.Debug("run:worker:done")
			a.status.Update(a.name, fmt.Sprintf("Cleaning up after interval '%s'", interval))
			workerRunning = false

			if interval.StartTime.After(lastTs) {
				t0 := interval.StartTime.Format("2006-01-02 15:04:05")
				if a.config.CollectFrom == "slowlog" {
					t1 := interval.StopTime.Format("15:04:05 MST")
					a.status.Update(a.name+"-last-interval", fmt.Sprintf("%s to %s", t0, t1))
				} else {
					a.status.Update(a.name+"-last-interval", fmt.Sprintf("%s", t0))
				}
				lastTs = interval.StartTime
			}
		case mysqlConfigured = <-a.mysqlConfiguredChan:
			a.logger.Debug("run:mysql:configured")
			// Start the IntervalIter once MySQL has been configured.
			// This avoids no data or partial data, e.g. slow log verbosity
			// not set yet.
			a.iter.Start()

			// If the next interval is more than 1 minute in the future,
			// simulate a clock tick now to start the iter early. For example,
			// if the interval is 5m and it's currently 01:00, the next interval
			// starts in 4m and stops in 9m, so data won't be reported for about
			// 10m. Instead, tick now so start interval=01:00 and end interval
			// =05:00 and data is reported in about 6m.
			tickChan := a.iter.TickChan()
			t := a.clock.ETA(tickChan)
			if t > 60 {
				began := ticker.Began(a.config.Interval, uint(time.Now().UTC().Unix()))
				a.logger.Info("First interval began at", began)
				tickChan <- began
			} else {
				a.logger.Info(fmt.Sprintf("First interval begins in %.1f seconds", t))
			}
		case <-a.restartChan:
			a.logger.Debug("run:mysql:restart")
			// If MySQL is not configured, then configureMySQL() should already
			// be running, trying to configure it. Else, we need to run
			// configureMySQL again.
			if mysqlConfigured {
				mysqlConfigured = false
				a.iter.Stop()
				go a.configureMySQL(a.config.Start, 0) // try forever
			}
		case <-a.runSync.StopChan:
			a.logger.Debug("run:stop")
			return
		}
	}
}

func (a *RealAnalyzer) runWorker(interval *Interval) {
	a.logger.Debug(fmt.Sprintf("runWorker:call:%d", interval.Number))
	defer func() {
		if err := recover(); err != nil {
			errMsg := fmt.Sprintf(a.name+"-worker crashed: '%s': %s", interval, err)
			log.Println(errMsg)
			debug.PrintStack()
			a.logger.Error(errMsg)
		}
		a.workerDoneChan <- interval
		a.logger.Debug(fmt.Sprintf("runWorker:return:%d", interval.Number))
	}()

	// Let worker do whatever it needs before it starts processing
	// the interval. This mostly makes testing easier.
	if err := a.worker.Setup(interval); err != nil {
		a.logger.Warn(err)
		return
	}

	// Let worker do whatever it needs after processing the interval.
	// This mostly maske testing easier.
	defer func() {
		if err := a.worker.Cleanup(); err != nil {
			a.logger.Warn(err)
		}
	}()

	// Run the worker to process the interval.
	t0 := time.Now()
	result, err := a.worker.Run()
	t1 := time.Now()
	if err != nil {
		a.logger.Error(err)
		return
	}
	if result == nil {
		if a.config.CollectFrom == "slowlog" {
			// This shouldn't happen. If it does, the slow log worker has a bug
			// because it should have returned an error above.
			a.logger.Error("Nil result", interval)
		}
		return
	}
	result.RunTime = t1.Sub(t0).Seconds()

	// Translate the results into a report and spool.
	// NOTE: "qan" here is correct; do not use a.name.
	report := MakeReport(a.config, interval, result)
	if err := a.spool.Write("qan", report); err != nil {
		a.logger.Warn("Lost report:", err)
	}
}
<commit_msg>Analizer cancels qan if Percona Server slowlog rotation takeover fails on several consecutive attempts<commit_after>/*
   Copyright (c) 2014-2015, Percona LLC and/or its affiliates. All rights reserved.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published by
   the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>
*/

package qan

import (
	"fmt"
	"log"
	"runtime/debug"
	"strconv"
	"sync"
	"time"

	"github.com/percona/percona-agent/data"
	"github.com/percona/percona-agent/mysql"
	"github.com/percona/percona-agent/pct"
	"github.com/percona/percona-agent/ticker"
)

const (
	MIN_SLOWLOG_ROTATION_SIZE = 4096
	MAX_SLOWLOG_ROTATION_SIZE = 1073741824
)

// A Worker gets queries, aggregates them, and returns a Result. Workers are ran
// by Analyzers. When ran, MySQL is presumed to be configured and ready.
type Worker interface {
	Setup(*Interval) error
	Run() (*Result, error)
	Stop() error
	Cleanup() error
	Status() map[string]string
}

// An Analyzer runs a Worker at each Interval. Analyzers are responsible for
// MySQL: configuring, restarts, etc. The Worker is only ran when MySQL is
// configured and ready. Analyzers are also responsible for making Reports from
// the Results returned by Workers. The Worker determines the type of Analyzer:
// slowlog or perfschema. Analyzers are ran by the QAN Manager.
type Analyzer interface {
	Start() error
	Stop() error
	Status() map[string]string
	String() string
}

// An AnalyzerFactory makes an Analyzer, real or mock.
type AnalyzerFactory interface {
	Make(config Config, name string, mysqlConn mysql.Connector, restartChan <-chan bool, tickChan chan time.Time) Analyzer
}

// --------------------------------------------------------------------------

type RealAnalyzer struct {
	logger      *pct.Logger
	config      Config
	iter        IntervalIter
	mysqlConn   mysql.Connector
	restartChan <-chan bool
	worker      Worker
	clock       ticker.Manager
	spool       data.Spooler
	// --
	name                string
	mysqlConfiguredChan chan bool
	workerDoneChan      chan *Interval
	status              *pct.Status
	runSync             *pct.SyncChan
	configureMySQLSync  *pct.SyncChan
	running             bool
	mux                 *sync.RWMutex
}

func NewRealAnalyzer(logger *pct.Logger, config Config, iter IntervalIter, mysqlConn mysql.Connector, restartChan <-chan bool, worker Worker, clock ticker.Manager, spool data.Spooler) *RealAnalyzer {
	name := logger.Service()
	a := &RealAnalyzer{
		logger:      logger,
		config:      config,
		iter:        iter,
		mysqlConn:   mysqlConn,
		restartChan: restartChan,
		worker:      worker,
		clock:       clock,
		spool:       spool,
		// --
		name:                name,
		mysqlConfiguredChan: make(chan bool, 1),
		workerDoneChan:      make(chan *Interval, 1),
		status:              pct.NewStatus([]string{name, name + "-last-interval", name + "-next-interval"}),
		runSync:             pct.NewSyncChan(),
		configureMySQLSync:  pct.NewSyncChan(),
		mux:                 &sync.RWMutex{},
	}
	return a
}

func (a *RealAnalyzer) String() string {
	return a.name
}

func (a *RealAnalyzer) Start() error {
	a.logger.Debug("Start:call")
	defer a.logger.Debug("Start:return")
	a.mux.Lock()
	defer a.mux.Unlock()
	if a.running {
		return nil
	}
	go a.run()
	a.running = true
	return nil
}

func (a *RealAnalyzer) Stop() error {
	a.logger.Debug("Stop:call")
	defer a.logger.Debug("Stop:return")
	a.mux.Lock()
	defer a.mux.Unlock()
	if !a.running {
		return nil
	}
	a.runSync.Stop()
	a.runSync.Wait()
	a.running = false
	return nil
}

func (a *RealAnalyzer) Status() map[string]string {
	a.mux.RLock()
	defer a.mux.RUnlock()
	if a.running {
		a.status.Update(a.name+"-next-interval", fmt.Sprintf("%.1fs", a.clock.ETA(a.iter.TickChan())))
	} else {
		a.status.Update(a.name+"-next-interval", "")
	}
	return a.status.Merge(a.worker.Status())
}

// --------------------------------------------------------------------------

// Returns a pointer to the config in use by this analizer, useful for testing purposes
func (a *RealAnalyzer) GetConfig() *Config {
	return &a.config
}

// Will try to detect if DB has slowlog rotation enabled and takeover the task (normally only Percona Server)
func (a *RealAnalyzer) TakeOverPSRotation() error {
	var maxLogSize int64 = 0

	maxSlowlogSize := a.mysqlConn.GetGlobalVarString("max_slowlog_size")
	if maxSlowlogSize == "" {
		return nil
	}
	psMaxLogSize, err := strconv.ParseUint(maxSlowlogSize, 10, 64)
	if err != nil {
		return err
	}
	maxLogSize = int64(psMaxLogSize)
	if maxLogSize >= MIN_SLOWLOG_ROTATION_SIZE {
		a.logger.Info("Taking over Percona Server slowlog rotation, max_slowlog_size:", maxLogSize)
		a.config.MaxSlowLogSize = maxLogSize

		// Using Set makes testing easier
		disablePSrotation := []mysql.Query{
			mysql.Query{Set: "SET GLOBAL max_slowlog_size = 0"},
		}

		if err := a.mysqlConn.Set(disablePSrotation); err != nil {
			return err
		}
	}
	return nil
}

func (a *RealAnalyzer) configureMySQL(config []mysql.Query, tryLimit int) {
	a.logger.Debug("configureMySQL:call")
	defer func() {
		if err := recover(); err != nil {
			a.logger.Error(a.name+":configureMySQL crashed: ", err)
		}
		a.logger.Debug("configureMySQL:return")
	}()

	try := 0
	for (tryLimit == 0) || (try < tryLimit) {
		try++

		select {
		case <-a.configureMySQLSync.StopChan:
			a.logger.Debug("configureMySQL:stop")
			a.configureMySQLSync.Done()
			return
		default:
		}

		// Connect handles backoff internally.
		a.logger.Debug("configureMySQL:connecting")
		if err := a.mysqlConn.Connect(1); err != nil {
			a.logger.Warn("Cannot connect to MySQL:", err)
			continue
		}
		defer a.mysqlConn.Close()

		a.logger.Debug("configureMySQL:configuring")
		// Try to take over PS slowlog rotation
		if err := a.TakeOverPSRotation(); err != nil {
			a.logger.Warn("Cannot takeover slowlog rotation from Percona Server:", err)
			continue
		}
		if err := a.mysqlConn.Set(config); err != nil {
			a.mysqlConn.Close()
			a.logger.Warn("Cannot configure MySQL:", err)
			continue
		}

		a.logger.Debug("configureMySQL:configured")
		select {
		case a.mysqlConfiguredChan <- true:
			return // success
		case <-a.configureMySQLSync.StopChan:
			a.logger.Debug("configureMySQL:stop")
			a.configureMySQLSync.Done()
			return
		}
	}
}

func (a *RealAnalyzer) run() {
	a.logger.Debug("run:call")
	defer a.logger.Debug("run:return")

	mysqlConfigured := false
	go a.configureMySQL(a.config.Start, 0) // try forever

	defer func() {
		a.logger.Info("Stopping")

		a.status.Update(a.name, "Stopping worker")
		a.worker.Stop()

		a.status.Update(a.name, "Stopping interval iter")
		a.iter.Stop()

		if !mysqlConfigured {
			a.status.Update(a.name, "Stopping MySQL config")
			a.configureMySQLSync.Stop()
			a.configureMySQLSync.Wait()
		}

		a.status.Update(a.name, "Stopping QAN on MySQL")
		a.configureMySQL(a.config.Stop, 1) // try once

		if err := recover(); err != nil {
			a.logger.Error("QAN crashed: ", err)
			a.status.Update(a.name, "Crashed")
		} else {
			a.status.Update(a.name, "Stopped")
			a.logger.Info("Stopped")
		}

		a.runSync.Done()
	}()

	workerRunning := false
	lastTs := time.Time{}
	currentInterval := &Interval{}
	for {
		a.logger.Debug("run:idle")
		if mysqlConfigured {
			if workerRunning {
				a.status.Update(a.name, "Running")
			} else {
				a.status.Update(a.name, "Idle")
			}
		} else {
			a.status.Update(a.name, "Trying to configure MySQL")
		}

		select {
		case interval := <-a.iter.IntervalChan():
			if !mysqlConfigured {
				a.logger.Debug(fmt.Sprintf("run:interval:%d:skip (mysql not configured)", interval.Number))
				continue
			}

			if workerRunning {
				a.logger.Warn(fmt.Sprintf("Skipping interval '%s' because interval '%s' is still being parsed",
					interval, currentInterval))
				continue
			}

			a.status.Update(a.name, fmt.Sprintf("Starting interval '%s'", interval))
			a.logger.Debug(fmt.Sprintf("run:interval:%s", interval))
			currentInterval = interval

			// Run the worker, timing it, make a report from its results, spool
			// the report. When done the interval is returned on workerDoneChan.
			go a.runWorker(interval)
			workerRunning = true
		case interval := <-a.workerDoneChan:
			a.logger.Debug("run:worker:done")
			a.status.Update(a.name, fmt.Sprintf("Cleaning up after interval '%s'", interval))
			workerRunning = false

			if interval.StartTime.After(lastTs) {
				t0 := interval.StartTime.Format("2006-01-02 15:04:05")
				if a.config.CollectFrom == "slowlog" {
					t1 := interval.StopTime.Format("15:04:05 MST")
					a.status.Update(a.name+"-last-interval", fmt.Sprintf("%s to %s", t0, t1))
				} else {
					a.status.Update(a.name+"-last-interval", fmt.Sprintf("%s", t0))
				}
				lastTs = interval.StartTime
			}
		case mysqlConfigured = <-a.mysqlConfiguredChan:
			a.logger.Debug("run:mysql:configured")
			// Start the IntervalIter once MySQL has been configured.
			// This avoids no data or partial data, e.g. slow log verbosity
			// not set yet.
			a.iter.Start()

			// If the next interval is more than 1 minute in the future,
			// simulate a clock tick now to start the iter early. For example,
			// if the interval is 5m and it's currently 01:00, the next interval
			// starts in 4m and stops in 9m, so data won't be reported for about
			// 10m. Instead, tick now so start interval=01:00 and end interval
			// =05:00 and data is reported in about 6m.
			tickChan := a.iter.TickChan()
			t := a.clock.ETA(tickChan)
			if t > 60 {
				began := ticker.Began(a.config.Interval, uint(time.Now().UTC().Unix()))
				a.logger.Info("First interval began at", began)
				tickChan <- began
			} else {
				a.logger.Info(fmt.Sprintf("First interval begins in %.1f seconds", t))
			}
		case <-a.restartChan:
			a.logger.Debug("run:mysql:restart")
			// If MySQL is not configured, then configureMySQL() should already
			// be running, trying to configure it. Else, we need to run
			// configureMySQL again.
			if mysqlConfigured {
				mysqlConfigured = false
				a.iter.Stop()
				go a.configureMySQL(a.config.Start, 0) // try forever
			}
		case <-a.runSync.StopChan:
			a.logger.Debug("run:stop")
			return
		}
	}
}

func (a *RealAnalyzer) runWorker(interval *Interval) {
	a.logger.Debug(fmt.Sprintf("runWorker:call:%d", interval.Number))
	defer func() {
		if err := recover(); err != nil {
			errMsg := fmt.Sprintf(a.name+"-worker crashed: '%s': %s", interval, err)
			log.Println(errMsg)
			debug.PrintStack()
			a.logger.Error(errMsg)
		}
		a.workerDoneChan <- interval
		a.logger.Debug(fmt.Sprintf("runWorker:return:%d", interval.Number))
	}()

	// Let worker do whatever it needs before it starts processing
	// the interval. This mostly makes testing easier.
	if err := a.worker.Setup(interval); err != nil {
		a.logger.Warn(err)
		return
	}

	// Let worker do whatever it needs after processing the interval.
	// This mostly maske testing easier.
	defer func() {
		if err := a.worker.Cleanup(); err != nil {
			a.logger.Warn(err)
		}
	}()

	// Run the worker to process the interval.
	t0 := time.Now()
	result, err := a.worker.Run()
	t1 := time.Now()
	if err != nil {
		a.logger.Error(err)
		return
	}
	if result == nil {
		if a.config.CollectFrom == "slowlog" {
			// This shouldn't happen. If it does, the slow log worker has a bug
			// because it should have returned an error above.
			a.logger.Error("Nil result", interval)
		}
		return
	}
	result.RunTime = t1.Sub(t0).Seconds()

	// Translate the results into a report and spool.
	// NOTE: "qan" here is correct; do not use a.name.
	report := MakeReport(a.config, interval, result)
	if err := a.spool.Write("qan", report); err != nil {
		a.logger.Warn("Lost report:", err)
	}
}
<|endoftext|>