<commit_before>package ebs

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
	"strconv"
	"text/template"
	"time"
)

type stepCreateAMI struct {
	Tags []ec2.Tag
}

type amiNameData struct {
	CreateTime string
}

func (s *stepCreateAMI) Run(state map[string]interface{}) multistep.StepAction {
	config := state["config"].(config)
	ec2conn := state["ec2"].(*ec2.EC2)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)

	// Parse the name of the AMI
	amiNameBuf := new(bytes.Buffer)
	tData := amiNameData{
		strconv.FormatInt(time.Now().UTC().Unix(), 10),
	}

	t := template.Must(template.New("ami").Parse(config.AMIName))
	t.Execute(amiNameBuf, tData)
	amiName := amiNameBuf.String()

	// Create the image
	ui.Say(fmt.Sprintf("Creating the AMI: %s", amiName))
	createOpts := &ec2.CreateImage{
		InstanceId: instance.InstanceId,
		Name:       amiName,
	}

	createResp, err := ec2conn.CreateImage(createOpts)
	if err != nil {
		err := fmt.Errorf("Error creating AMI: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Set the AMI ID in the state
	ui.Say(fmt.Sprintf("AMI: %s", createResp.ImageId))
	amis := make(map[string]string)
	amis[ec2conn.Region.Name] = createResp.ImageId
	state["amis"] = amis

	// Wait for the image to become ready
	ui.Say("Waiting for AMI to become ready...")
	if err := awscommon.WaitForAMI(ec2conn, createResp.ImageId); err != nil {
		err := fmt.Errorf("Error waiting for AMI: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Add tags to AMI
	if s.Tags != nil {
		ui.Say(fmt.Sprintf("Add tags to AMI (%s)...", createResp.ImageId))
		amiId := []string{createResp.ImageId}
		_, err := ec2conn.CreateTags(amiId, s.Tags)
		if err != nil {
			err := fmt.Errorf("Error adding tags to AMI (%s): %s", createResp.ImageId, err)
			state["error"] = err
			ui.Error(err.Error())
			return multistep.ActionHalt
		}
	}

	return multistep.ActionContinue
}

func (s *stepCreateAMI) Cleanup(map[string]interface{}) {
	// No cleanup...
}
<commit_msg>Adds support for adding tags to the AMI<commit_after>package ebs

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/multistep"
	awscommon "github.com/mitchellh/packer/builder/amazon/common"
	"github.com/mitchellh/packer/packer"
	"strconv"
	"text/template"
	"time"
)

type stepCreateAMI struct {
	Tags []ec2.Tag
}

type amiNameData struct {
	CreateTime string
}

func (s *stepCreateAMI) Run(state map[string]interface{}) multistep.StepAction {
	config := state["config"].(config)
	ec2conn := state["ec2"].(*ec2.EC2)
	instance := state["instance"].(*ec2.Instance)
	ui := state["ui"].(packer.Ui)

	// Parse the name of the AMI
	amiNameBuf := new(bytes.Buffer)
	tData := amiNameData{
		strconv.FormatInt(time.Now().UTC().Unix(), 10),
	}

	t := template.Must(template.New("ami").Parse(config.AMIName))
	t.Execute(amiNameBuf, tData)
	amiName := amiNameBuf.String()

	// Create the image
	ui.Say(fmt.Sprintf("Creating the AMI: %s", amiName))
	createOpts := &ec2.CreateImage{
		InstanceId: instance.InstanceId,
		Name:       amiName,
	}

	createResp, err := ec2conn.CreateImage(createOpts)
	if err != nil {
		err := fmt.Errorf("Error creating AMI: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Set the AMI ID in the state
	ui.Say(fmt.Sprintf("AMI: %s", createResp.ImageId))
	amis := make(map[string]string)
	amis[ec2conn.Region.Name] = createResp.ImageId
	state["amis"] = amis

	// Wait for the image to become ready
	ui.Say("Waiting for AMI to become ready...")
	if err := awscommon.WaitForAMI(ec2conn, createResp.ImageId); err != nil {
		err := fmt.Errorf("Error waiting for AMI: %s", err)
		state["error"] = err
		ui.Error(err.Error())
		return multistep.ActionHalt
	}

	// Add tags to AMI
	if s.Tags != nil {
		ui.Say(fmt.Sprintf("Adding tags to AMI (%s)...", createResp.ImageId))
		amiId := []string{createResp.ImageId}
		_, err := ec2conn.CreateTags(amiId, s.Tags)
		if err != nil {
			err := fmt.Errorf("Error adding tags to AMI (%s): %s", createResp.ImageId, err)
			state["error"] = err
			ui.Error(err.Error())
			return multistep.ActionHalt
		}
	}

	return multistep.ActionContinue
}

func (s *stepCreateAMI) Cleanup(map[string]interface{}) {
	// No cleanup...
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	. "github.com/logrusorgru/aurora"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"
)

var infile string
var subfile string
var addsubs bool
var toplevel string
var webvtt bool
var jasonfile string
var cmdtemplate string
var completed string
var batch string

// Variant struct for HLS variants
type Variant struct {
	Name      string `json:"name"`
	Aspect    string `json:"aspect"`
	Rate      string `json:"framerate"`
	Vbr       string `json:"vbitrate"`
	Abr       string `json:"abitrate"`
	Bandwidth string
}

// Create variant's destination directory
func (v *Variant) mkDest() string {
	dest := fmt.Sprintf("%s/%s", toplevel, v.Name)
	os.MkdirAll(dest, 0755)
	return dest
}

// generates a string of inputs for the ffmpeg cmd.
func (v *Variant) mkInputs() string {
	inputs := fmt.Sprintf(" -i %s", infile)
	if addsubs && !(webvtt) {
		inputs = fmt.Sprintf(" -i %s -i %s  ", infile, subfile)
	}
	return inputs
}

// This Variant method assembles the ffmpeg command
func (v *Variant) mkCmd(cmdtemplate string) string {
	data, err := ioutil.ReadFile(cmdtemplate)
	chk(err, "Error reading template file")
	inputs := v.mkInputs()
	r := strings.NewReplacer("INPUTS", inputs, "ASPECT", v.Aspect,
		"VBITRATE", v.Vbr, "FRAMERATE", v.Rate, "ABITRATE", v.Abr,
		"TOPLEVEL", toplevel, "NAME", v.Name, "\n", " ")
	cmd := fmt.Sprintf("%s\n", r.Replace(string(data)))
	return cmd
}

// Read actual bitrate from first segment to set bandwidth in master.m3u8
func (v *Variant) readRate() {
	cmd := fmt.Sprintf("ffprobe -i %s/%s/index0.ts", toplevel, v.Name)
	data := chkExec(cmd)
	two := strings.Split(data, "bitrate: ")[1]
	rate := strings.Split(two, " kb/s")[0]
	v.Bandwidth = fmt.Sprintf("%v000", rate)
}

// Start transcoding the variant
func (v *Variant) start() {
	v.mkDest()
	fmt.Printf(" . variants: %s %s \r", Cyan(completed), v.Aspect)
	completed += fmt.Sprintf("%s ", v.Aspect)
	cmd := v.mkCmd(cmdtemplate)
	chkExec(cmd)
	v.readRate()
	fmt.Printf(" %s variants: %s  \r", Cyan("."), Cyan(completed))
}

// #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=7483000,RESOLUTION=1920:1080,
// hd1920/index.m3u8
func (v *Variant) mkStanza() string {
	stanza := fmt.Sprintf("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=%v,RESOLUTION=%v", v.Bandwidth, v.Aspect)
	if addsubs {
		stanza = fmt.Sprintf("%s,SUBTITLES=\"webvtt\"", stanza)
	}
	return stanza
}

// Executes external commands and checks for runtime errors
func chkExec(cmd string) string {
	parts := strings.Fields(cmd)
	data, err := exec.Command(parts[0], parts[1:]...).CombinedOutput()
	chk(err, fmt.Sprintf("Error running \n %s \n %v", cmd, string(data)))
	return string(data)
}

// probes for Closed Captions in video file.
func hasCaptions() bool {
	cmd := fmt.Sprintf("ffprobe -i %s", infile)
	data := chkExec(cmd)
	if strings.Contains(data, "Captions") {
		return true
	}
	return false
}

// Captions are segmented along with the first variant and then moved to toplevel/subs
func mvCaptions(vardir string) {
	srcdir := fmt.Sprintf("%s/%s", toplevel, vardir)
	destdir := fmt.Sprintf("%s/subs", toplevel)
	os.MkdirAll(destdir, 0755)
	files, err := ioutil.ReadDir(srcdir)
	chk(err, "Error moving Captions")
	for _, f := range files {
		if strings.Contains(f.Name(), "vtt") {
			os.Rename(fmt.Sprintf("%s/%s", srcdir, f.Name()), fmt.Sprintf("%s/%s", destdir, f.Name()))
		}
	}
}

// return a subtitle stanza for use in the  master.m3u8
func mkSubStanza() string {
	return "#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID=\"webvtt\",NAME=\"English\",DEFAULT=YES,AUTOSELECT=YES,FORCED=NO,LANGUAGE=\"en\",URI=\"subs/vtt_index.m3u8\"\n"
}

// Read json file for variants
func dataToVariants() []Variant {
	var variants []Variant
	data, err := ioutil.ReadFile(jasonfile)
	chk(err, "Error reading JSON file")
	json.Unmarshal(data, &variants)
	return variants
}

// Set the toplevel dir for variants by splitting video file name at the "."
func mkTopLevel() {
	if toplevel == "" {
		toplevel = strings.Split(infile, `.`)[0]
	}
	os.MkdirAll(toplevel, 0755)
}

func extractCaptions() string {
	fmt.Printf(" . %s", Cyan("extracting captions \r"))
	srtfile := fmt.Sprintf("%s/%s.ssa", toplevel, toplevel)
	cmd := fmt.Sprintf("ffmpeg -y -f lavfi -fix_sub_duration -i movie=%s[out0+subcc] -r 30 %s", infile, srtfile)
	chkExec(cmd)
	fmt.Printf(" %s 608 captions : %s \r", Cyan("."), Cyan(infile))

	return srtfile
}

func mkSubfile() {
	addsubs = false
	if !(webvtt) {
		if (subfile == "") && (hasCaptions()) {
			subfile = extractCaptions()
		}
		if subfile != "" {
			addsubs = true
		}
	}
}

// Generic catchall error checking
func chk(err error, mesg string) {
	if err != nil {
		fmt.Printf("%s\n", mesg)
		//panic(err)
	}
}

// Make all variants and write master.m3u8
func mkAll(variants []Variant) {
	mkTopLevel()
	fmt.Println(Cyan(" ."), "video file:", Cyan(infile), "\n", Cyan("."), "toplevel dir:", Cyan(toplevel))
	mkSubfile()
	var m3u8Master = fmt.Sprintf("%s/master.m3u8", toplevel)
	fp, err := os.Create(m3u8Master)
	chk(err, "in mkAll")
	defer fp.Close()
	w := bufio.NewWriter(fp)
	w.WriteString("#EXTM3U\n")
	fmt.Println("\n", Cyan("."), "ass file:", Cyan(subfile))
	for _, v := range variants {
		v.start()
		if addsubs && !(webvtt) {
			mvCaptions(v.Name)
			w.WriteString(mkSubStanza())
			webvtt = true
		}
		w.WriteString(fmt.Sprintf("%s\n", v.mkStanza()))
		w.WriteString(fmt.Sprintf("%s/index.m3u8\n", v.Name))
	}
	w.Flush()
}

func main() {
	flag.StringVar(&infile, "i", "", "Video file to segment (either -i or -b is required)")
	flag.StringVar(&subfile, "s", "", "subtitle file to segment (optional)")
	flag.StringVar(&toplevel, "d", "", "override top level directory for hls files (optional)")
	flag.StringVar(&jasonfile, "j", `./hls.json`, "JSON file of variants (optional)")
	flag.StringVar(&cmdtemplate, "t", `./cmd.template`, "command template file (optional)")
	flag.StringVar(&batch, "b", "", "batch mode, list multiple input files (either -i or -b is required)")

	flag.Parse()
	variants := dataToVariants()

	if batch != "" {
		batch = strings.Replace(batch, " ", ",", -1)
		splitbatch := strings.Split(batch, ",")
		for i, b := range splitbatch {
			t := time.Now()
			fmt.Println("\n", Cyan(i+1), "of", len(splitbatch))
			fmt.Println(Cyan(" ."), "started:", Cyan(t.Format(time.Stamp)))
			webvtt = false
			subfile = ""
			infile = b
			completed = ""
			toplevel = ""
			mkTopLevel()
			variants := dataToVariants()
			mkAll(variants)
		}
	} else {
		if infile != "" {
			mkAll(variants)
		} else {
			flag.PrintDefaults()
		}

	}
	fmt.Println("\n\n")

}
<commit_msg>Added proper codec strings<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	. "github.com/logrusorgru/aurora"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"
)

var infile string
var subfile string
var addsubs bool
var toplevel string
var webvtt bool
var jasonfile string
var cmdtemplate string
var completed string
var batch string
var x264level = "3.0"
var x264profile = "high"
var mastercodec ="avc1.64001E,mp4a.40.2"

// Variant struct for HLS variants
type Variant struct {
	Name      string `json:"name"`
	Aspect    string `json:"aspect"`
	Rate      string `json:"framerate"`
	Vbr       string `json:"vbitrate"`
	Abr       string `json:"abitrate"`
	Bandwidth string
}

// Create variant's destination directory
func (v *Variant) mkDest() string {
	dest := fmt.Sprintf("%s/%s", toplevel, v.Name)
	os.MkdirAll(dest, 0755)
	return dest
}

func (v *Variant) mkInputs() string {
	inputs := fmt.Sprintf(" -i %s", infile)
	if addsubs && !(webvtt) {
		inputs = fmt.Sprintf(" -i %s -i %s  ", infile, subfile)
	}
	return inputs
}

// This Variant method assembles the ffmpeg command
func (v *Variant) mkCmd(cmdtemplate string) string {
	data, err := ioutil.ReadFile(cmdtemplate)
	chk(err, "Error reading template file")
	inputs := v.mkInputs()
	r := strings.NewReplacer("INPUTS", inputs, "ASPECT", v.Aspect,
		"VBITRATE", v.Vbr,"X264LEVEL",x264level,"X264PROFILE",x264profile,
		"FRAMERATE", v.Rate, "ABITRATE", v.Abr,
		"TOPLEVEL", toplevel, "NAME", v.Name, "\n", " ")
	cmd := fmt.Sprintf("%s\n", r.Replace(string(data)))
	//fmt.Printf(cmd)
	return cmd
}

// Read actual bitrate from first segment to set bandwidth in master.m3u8
func (v *Variant) readRate() {
	cmd := fmt.Sprintf("ffprobe -i %s/%s/index0.ts", toplevel, v.Name)
	data := chkExec(cmd)
	two := strings.Split(data, "bitrate: ")[1]
	rate := strings.Split(two, " kb/s")[0]
	v.Bandwidth = fmt.Sprintf("%v000", rate)
}

// Start transcoding the variant
func (v *Variant) start() {
	v.mkDest()
	fmt.Printf(" . variants: %s %s \r", Cyan(completed), v.Aspect)
	completed += fmt.Sprintf("%s ", v.Aspect)
	cmd := v.mkCmd(cmdtemplate)
	chkExec(cmd)
	v.readRate()
	fmt.Printf(" %s variants: %s  \r", Cyan("."), Cyan(completed))
}

// #EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=7483000,RESOLUTION=1920:1080,
// hd1920/index.m3u8
func (v *Variant) mkStanza() string {
	stanza := fmt.Sprintf("#EXT-X-STREAM-INF:PROGRAM-ID=1,BANDWIDTH=%v,RESOLUTION=%v,CODECS=\"%s\"", v.Bandwidth, v.Aspect,mastercodec)
	if addsubs {
		stanza = fmt.Sprintf("%s,SUBTITLES=\"webvtt\"", stanza)
	}
	return stanza
}

func chkExec(cmd string) string {
	// Executes external commands and checks for runtime errors
	parts := strings.Fields(cmd)
	data, err := exec.Command(parts[0], parts[1:]...).CombinedOutput()
	chk(err, fmt.Sprintf("Error running \n %s \n %v", cmd, string(data)))
	return string(data)
}

// probes for Closed Captions in video file.
func hasCaptions() bool {
	cmd := fmt.Sprintf("ffprobe -i %s", infile)
	data := chkExec(cmd)
	if strings.Contains(data, "Captions") {
		return true
	}
	return false
}

// Captions are segmented along with the first variant and then moved to toplevel/subs
func mvCaptions(vardir string) {
	srcdir := fmt.Sprintf("%s/%s", toplevel, vardir)
	destdir := fmt.Sprintf("%s/subs", toplevel)
	os.MkdirAll(destdir, 0755)
	files, err := ioutil.ReadDir(srcdir)
	chk(err, "Error moving Captions")
	for _, f := range files {
		if strings.Contains(f.Name(), "vtt") {
			os.Rename(fmt.Sprintf("%s/%s", srcdir, f.Name()), fmt.Sprintf("%s/%s", destdir, f.Name()))
		}
	}
}

// return a subtitle stanza for use in the  master.m3u8
func mkSubStanza() string {
	return "#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID=\"webvtt\",NAME=\"English\",DEFAULT=YES,AUTOSELECT=YES,FORCED=NO,LANGUAGE=\"en\",URI=\"subs/vtt_index.m3u8\"\n"
}

// Read json file for variants
func dataToVariants() []Variant {
	var variants []Variant
	data, err := ioutil.ReadFile(jasonfile)
	chk(err, "Error reading JSON file")
	json.Unmarshal(data, &variants)
	return variants
}

// Set the toplevel dir for variants by splitting video file name at the "."
func mkTopLevel() {
	if toplevel == "" {
		toplevel = strings.Split(infile, `.`)[0]
	}
	os.MkdirAll(toplevel, 0755)
}

//Extract 608 captions to an srt file.
func extractCaptions() string {
	fmt.Printf(" . %s", Cyan("extracting captions \r"))
	srtfile := fmt.Sprintf("%s/%s.srt", toplevel, toplevel)
	cmd := fmt.Sprintf("ffmpeg -y -f lavfi -fix_sub_duration -i movie=%s[out0+subcc] -r 30  -scodec subrip %s", infile, srtfile)
	chkExec(cmd)
	fmt.Printf(" %s 608 captions : %s \r", Cyan("."), Cyan(infile))

	return srtfile
}

// Extract captions to segment,
// unless a subtitle file is passed in with "-s"
func mkSubfile() {
	addsubs = false
	if !(webvtt) {
		if (subfile == "") && (hasCaptions()) {
			subfile = extractCaptions()
		}
		if subfile != "" {
			addsubs = true
		}
	}
}

// Generic catchall error checking
func chk(err error, mesg string) {
	if err != nil {
		fmt.Printf("%s\n", mesg)
		//panic(err)
	}
}

// Make all variants and write master.m3u8
func mkAll(variants []Variant) {
	mkTopLevel()
	fmt.Println(Cyan(" ."), "video file:", Cyan(infile), "\n", Cyan("."), "toplevel dir:", Cyan(toplevel))
	mkSubfile()
	var m3u8Master = fmt.Sprintf("%s/master.m3u8", toplevel)
	fp, err := os.Create(m3u8Master)
	chk(err, "in mkAll")
	defer fp.Close()
	w := bufio.NewWriter(fp)
	w.WriteString("#EXTM3U\n")
	fmt.Println("\n", Cyan("."), "srt file:", Cyan(subfile))
	for _, v := range variants {
		v.start()
		if addsubs && !(webvtt) {
			mvCaptions(v.Name)
			w.WriteString(mkSubStanza())
			webvtt = true
		}
		w.WriteString(fmt.Sprintf("%s\n", v.mkStanza()))
		w.WriteString(fmt.Sprintf("%s/index.m3u8\n", v.Name))
	}
	w.Flush()
}

func main() {
	flag.StringVar(&infile, "i", "", "Video file to segment (either -i or -b is required)")
	flag.StringVar(&subfile, "s", "", "subtitle file to segment (optional)")
	flag.StringVar(&toplevel, "d", "", "override top level directory for hls files (optional)")
	flag.StringVar(&jasonfile, "j", `./hls.json`, "JSON file of variants (optional)")
	flag.StringVar(&cmdtemplate, "t", `./cmd.template`, "command template file (optional)")
	flag.StringVar(&batch, "b", "", "batch mode, list multiple input files (either -i or -b is required)")

	flag.Parse()
	variants := dataToVariants()

	if batch != "" {
		batch = strings.Replace(batch, " ", ",", -1)
		splitbatch := strings.Split(batch, ",")
		for i, b := range splitbatch {
			t := time.Now()
			fmt.Println("\n", Cyan(i+1), "of", len(splitbatch))
			fmt.Println(Cyan(" ."), "started:", Cyan(t.Format(time.Stamp)))
			webvtt = false
			subfile = ""
			infile = b
			completed = ""
			toplevel = ""
			mkTopLevel()
			variants := dataToVariants()
			mkAll(variants)
		}
	} else {
		if infile != "" {
			mkAll(variants)
		} else {
			flag.PrintDefaults()
		}
	}
	fmt.Println("\n\n")

}
<|endoftext|>


<commit_before>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import "fmt"

// FileBlame return the Blame object of file
func (repo *Repository) FileBlame(revision, path, file string) ([]byte, error) {
	return NewCommand("blame", "--root", file).RunInDirBytes(path)
}

// LineBlame returns the latest commit at the given line
func (repo *Repository) LineBlame(revision, path, file string, line uint) (*Commit, error) {
	res, err := NewCommand("blame", fmt.Sprintf("-L %d,%d", line, line), "-p", revision, file).RunInDir(path)
	if err != nil {
		return nil, err
	}
	if len(res) < 40 {
		return nil, fmt.Errorf("invalid result of blame: %s", res)
	}
	return repo.GetCommit(string(res[:40]))
}
<commit_msg>Fix blame problem for older git versions (#118)<commit_after>// Copyright 2017 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import "fmt"

// FileBlame return the Blame object of file
func (repo *Repository) FileBlame(revision, path, file string) ([]byte, error) {
	return NewCommand("blame", "--root", "--", file).RunInDirBytes(path)
}

// LineBlame returns the latest commit at the given line
func (repo *Repository) LineBlame(revision, path, file string, line uint) (*Commit, error) {
	res, err := NewCommand("blame", fmt.Sprintf("-L %d,%d", line, line), "-p", revision, "--", file).RunInDir(path)
	if err != nil {
		return nil, err
	}
	if len(res) < 40 {
		return nil, fmt.Errorf("invalid result of blame: %s", res)
	}
	return repo.GetCommit(string(res[:40]))
}
<|endoftext|>


<commit_before>// Copyright 2018 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"context"
	"errors"
	"fmt"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/prometheus/alertmanager/api/v2/client/silence"
	"github.com/prometheus/alertmanager/api/v2/models"
	"github.com/prometheus/alertmanager/cli/format"
	"github.com/prometheus/alertmanager/pkg/labels"
)

type silenceQueryCmd struct {
	expired  bool
	quiet    bool
	matchers []string
	within   time.Duration
}

const querySilenceHelp = `Query Alertmanager silences.

Amtool has a simplified prometheus query syntax, but contains robust support for
bash variable expansions. The non-option section of arguments constructs a list
of "Matcher Groups" that will be used to filter your query. The following
examples will attempt to show this behaviour in action:

amtool silence query alertname=foo node=bar

	This query will match all silences with the alertname=foo and node=bar label
	value pairs set.

amtool silence query foo node=bar

	If alertname is omitted and the first argument does not contain a '=' or a
	'=~' then it will be assumed to be the value of the alertname pair.

amtool silence query 'alertname=~foo.*'

	As well as direct equality, regex matching is also supported. The '=~' syntax
	(similar to prometheus) is used to represent a regex match. Regex matching
	can be used in combination with a direct match.

In addition to filtering by silence labels, one can also query for silences
that are due to expire soon with the "--within" parameter. In the event that
you want to preemptively act upon expiring silences by either fixing them or
extending them. For example:

amtool silence query --within 8h

returns all the silences due to expire within the next 8 hours. This syntax can
also be combined with the label based filtering above for more flexibility.

The "--expired" parameter returns only expired silences. Used in combination
with "--within=TIME", amtool returns the silences that expired within the
preceding duration.

amtool silence query --within 2h --expired

returns all silences that expired within the preceding 2 hours.
`

func configureSilenceQueryCmd(cc *kingpin.CmdClause) {
	var (
		c        = &silenceQueryCmd{}
		queryCmd = cc.Command("query", querySilenceHelp).Default()
	)

	queryCmd.Flag("expired", "Show expired silences instead of active").BoolVar(&c.expired)
	queryCmd.Flag("quiet", "Only show silence ids").Short('q').BoolVar(&c.quiet)
	queryCmd.Arg("matcher-groups", "Query filter").StringsVar(&c.matchers)
	queryCmd.Flag("within", "Show silences that will expire or have expired within a duration").DurationVar(&c.within)
	queryCmd.Action(execWithTimeout(c.query))
}

func (c *silenceQueryCmd) query(ctx context.Context, _ *kingpin.ParseContext) error {
	if len(c.matchers) > 0 {
		// If the parser fails then we likely don't have a (=|=~|!=|!~) so lets
		// assume that the user wants alertname=<arg> and prepend `alertname=`
		// to the front.
		_, err := labels.ParseMatcher(c.matchers[0])
		if err != nil {
			c.matchers[0] = fmt.Sprintf("alertname=%s", c.matchers[0])
		}
	}

	silenceParams := silence.NewGetSilencesParams().WithContext(ctx).WithFilter(c.matchers)

	amclient := NewAlertmanagerClient(alertmanagerURL)

	getOk, err := amclient.Silence.GetSilences(silenceParams)
	if err != nil {
		return err
	}

	displaySilences := []models.GettableSilence{}
	for _, silence := range getOk.Payload {
		// skip expired silences if --expired is not set
		if !c.expired && time.Time(*silence.EndsAt).Before(time.Now()) {
			continue
		}
		// skip active silences if --expired is set
		if c.expired && time.Time(*silence.EndsAt).After(time.Now()) {
			continue
		}
		// skip active silences expiring after "--within"
		if !c.expired && int64(c.within) > 0 && time.Time(*silence.EndsAt).After(time.Now().UTC().Add(c.within)) {
			continue
		}
		// skip silences that expired before "--within"
		if c.expired && int64(c.within) > 0 && time.Time(*silence.EndsAt).Before(time.Now().UTC().Add(-c.within)) {
			continue
		}

		displaySilences = append(displaySilences, *silence)
	}

	if c.quiet {
		for _, silence := range displaySilences {
			fmt.Println(*silence.ID)
		}
	} else {
		formatter, found := format.Formatters[output]
		if !found {
			return errors.New("unknown output formatter")
		}
		if err := formatter.FormatSilences(displaySilences); err != nil {
			return fmt.Errorf("error formatting silences: %v", err)
		}
	}
	return nil
}
<commit_msg>Allow filtering silences by createdBy author (#2718)<commit_after>// Copyright 2018 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"context"
	"errors"
	"fmt"
	"time"

	kingpin "gopkg.in/alecthomas/kingpin.v2"

	"github.com/prometheus/alertmanager/api/v2/client/silence"
	"github.com/prometheus/alertmanager/api/v2/models"
	"github.com/prometheus/alertmanager/cli/format"
	"github.com/prometheus/alertmanager/pkg/labels"
)

type silenceQueryCmd struct {
	expired   bool
	quiet     bool
	createdBy string
	matchers  []string
	within    time.Duration
}

const querySilenceHelp = `Query Alertmanager silences.

Amtool has a simplified prometheus query syntax, but contains robust support for
bash variable expansions. The non-option section of arguments constructs a list
of "Matcher Groups" that will be used to filter your query. The following
examples will attempt to show this behaviour in action:

amtool silence query alertname=foo node=bar

	This query will match all silences with the alertname=foo and node=bar label
	value pairs set.

amtool silence query foo node=bar

	If alertname is omitted and the first argument does not contain a '=' or a
	'=~' then it will be assumed to be the value of the alertname pair.

amtool silence query 'alertname=~foo.*'

	As well as direct equality, regex matching is also supported. The '=~' syntax
	(similar to prometheus) is used to represent a regex match. Regex matching
	can be used in combination with a direct match.

In addition to filtering by silence labels, one can also query for silences
that are due to expire soon with the "--within" parameter. In the event that
you want to preemptively act upon expiring silences by either fixing them or
extending them. For example:

amtool silence query --within 8h

returns all the silences due to expire within the next 8 hours. This syntax can
also be combined with the label based filtering above for more flexibility.

The "--expired" parameter returns only expired silences. Used in combination
with "--within=TIME", amtool returns the silences that expired within the
preceding duration.

amtool silence query --within 2h --expired

returns all silences that expired within the preceding 2 hours.
`

func configureSilenceQueryCmd(cc *kingpin.CmdClause) {
	var (
		c        = &silenceQueryCmd{}
		queryCmd = cc.Command("query", querySilenceHelp).Default()
	)

	queryCmd.Flag("expired", "Show expired silences instead of active").BoolVar(&c.expired)
	queryCmd.Flag("quiet", "Only show silence ids").Short('q').BoolVar(&c.quiet)
	queryCmd.Flag("created-by", "Show silences that belong to this creator").StringVar(&c.createdBy)
	queryCmd.Arg("matcher-groups", "Query filter").StringsVar(&c.matchers)
	queryCmd.Flag("within", "Show silences that will expire or have expired within a duration").DurationVar(&c.within)
	queryCmd.Action(execWithTimeout(c.query))
}

func (c *silenceQueryCmd) query(ctx context.Context, _ *kingpin.ParseContext) error {
	if len(c.matchers) > 0 {
		// If the parser fails then we likely don't have a (=|=~|!=|!~) so lets
		// assume that the user wants alertname=<arg> and prepend `alertname=`
		// to the front.
		_, err := labels.ParseMatcher(c.matchers[0])
		if err != nil {
			c.matchers[0] = fmt.Sprintf("alertname=%s", c.matchers[0])
		}
	}

	silenceParams := silence.NewGetSilencesParams().WithContext(ctx).WithFilter(c.matchers)

	amclient := NewAlertmanagerClient(alertmanagerURL)

	getOk, err := amclient.Silence.GetSilences(silenceParams)
	if err != nil {
		return err
	}

	displaySilences := []models.GettableSilence{}
	for _, silence := range getOk.Payload {
		// skip expired silences if --expired is not set
		if !c.expired && time.Time(*silence.EndsAt).Before(time.Now()) {
			continue
		}
		// skip active silences if --expired is set
		if c.expired && time.Time(*silence.EndsAt).After(time.Now()) {
			continue
		}
		// skip active silences expiring after "--within"
		if !c.expired && int64(c.within) > 0 && time.Time(*silence.EndsAt).After(time.Now().UTC().Add(c.within)) {
			continue
		}
		// skip silences that expired before "--within"
		if c.expired && int64(c.within) > 0 && time.Time(*silence.EndsAt).Before(time.Now().UTC().Add(-c.within)) {
			continue
		}
		// Skip silences if the author doesn't match.
		if c.createdBy != "" && *silence.CreatedBy != c.createdBy {
			continue
		}

		displaySilences = append(displaySilences, *silence)
	}

	if c.quiet {
		for _, silence := range displaySilences {
			fmt.Println(*silence.ID)
		}
	} else {
		formatter, found := format.Formatters[output]
		if !found {
			return errors.New("unknown output formatter")
		}
		if err := formatter.FormatSilences(displaySilences); err != nil {
			return fmt.Errorf("error formatting silences: %v", err)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package gateway

import (
	"net/http"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/golib/hack"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/msgs/:appid/:topic/:ver?group=xx&batch=10&wait=5s&reset=<newest|oldest>&ack=1&q=<dead|retry>
func (this *subServer) subHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		rawTopic   string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64         = -1
		limit      int           // max messages to include in the message set
		wait       time.Duration // max time to wait if insufficient data is available
		delayedAck bool          // explicit application level acknowledgement
		tagFilters []MsgTag      = nil
		err        error
	)

	if Options.EnableClientStats {
		this.gw.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	limit, err = getHttpQueryInt(&query, "batch", 1)
	if err != nil {
		writeBadRequest(w, "illegal limit")
		return
	}
	if limit > Options.MaxSubBatchSize && Options.MaxSubBatchSize > 0 {
		limit = Options.MaxSubBatchSize
	}

	wait, err = time.ParseDuration(query.Get("wait"))
	if err != nil || wait < MinSubWait {
		wait = MinSubWait
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	// fetch the client ack partition and offset
	delayedAck = query.Get("ack") == "1"
	if delayedAck {
		// consumers use explicit acknowledges in order to signal a message as processed successfully
		// if consumers fail to ACK, the message hangs and server will refuse to move ahead

		// get the partitionN and offsetN from client header
		// client will ack with partition=-1, offset=-1:
		// 1. handshake phase
		// 2. when 204 No Content
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int
			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} offset:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), offset)

				writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} partition:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), partition)

				writeBadRequest(w, "ack with bad partition")
				return
			}
		} else if len(partition+offset) != 0 {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s partition:%s offset:%s UA:%s} partial ack",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"))

			writeBadRequest(w, "partial ack not allowed")
			return
		}
	}

	shadow = query.Get("q")

	log.Debug("sub[%s] %s(%s): {app:%s q:%s topic:%s ver:%s group:%s batch:%d ack:%s partition:%s offset:%s UA:%s}",
		myAppid, r.RemoteAddr, realIp, hisAppid, shadow, topic, ver,
		group, limit, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"))

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} invalid shadow name",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} cluster not found",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"))

		writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset, Options.PermitStandbySub)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// commit the acked offset
	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		// what if shutdown kateway now?
		// the commit will be ok, and when pumpMessages, the conn will get http.StatusNoContent
		if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
			Topic:     rawTopic,
			Partition: int32(partitionN),
			Offset:    offsetN,
		}); err != nil {
			log.Error("sub commit[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:1 partition:%s offset:%s UA:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"), err)

			writeBadRequest(w, err.Error())
			return
		} else {
			log.Debug("sub land %s(%s): {G:%s, T:%s, P:%s, O:%s}",
				r.RemoteAddr, realIp, group, rawTopic, partition, offset)
		}
	}

	tag := r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		tagFilters = parseMessageTag(tag)
	}

	err = this.pumpMessages(w, r, fetcher, limit, wait, myAppid, hisAppid, topic, ver, group, delayedAck, tagFilters)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:%s partition:%s offset:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"), err)

		writeServerError(w, err.Error())

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver, group, err)
		}
	}
}

func (this *subServer) pumpMessages(w http.ResponseWriter, r *http.Request,
	fetcher store.Fetcher, limit int, wait time.Duration, myAppid, hisAppid, topic, ver,
	group string, delayedAck bool, tagFilters []MsgTag) error {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	var metaBuf []byte = nil
	n := 0
	idleTimeout := Options.SubTimeout
	realIp := getHttpRemoteIp(r)
	chunkedEver := false
	for {
		select {
		case <-clientGoneCh:
			// FIXME access log will not be able to record this behavior
			return ErrClientGone

		case <-this.gw.shutdownCh:
			// don't call me again
			w.Header().Set("Connection", "close")

			if !chunkedEver {
				w.WriteHeader(http.StatusNoContent)
				w.Write([]byte{})
			}

			return nil

		case err := <-fetcher.Errors():
			// e,g. consume a non-existent topic
			// e,g. conn with broker is broken
			return err

		case <-this.gw.timer.After(idleTimeout):
			if chunkedEver {
				// response already sent in chunk
				log.Debug("chunked sub idle timeout %s {A:%s/G:%s->A:%s T:%s V:%s}",
					idleTimeout, myAppid, group, hisAppid, topic, ver)
				return nil
			}

			w.WriteHeader(http.StatusNoContent)
			w.Write([]byte{}) // without this, client cant get response
			return nil

		case msg := <-fetcher.Messages():
			partition := strconv.FormatInt(int64(msg.Partition), 10)

			if limit == 1 {
				w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
				w.Header().Set(HttpHeaderPartition, partition)
				w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))
			}

			var (
				tags    []MsgTag
				bodyIdx int
				err     error
			)
			if IsTaggedMessage(msg.Value) {
				// TagMarkStart + tag + TagMarkEnd + body
				tags, bodyIdx, err = ExtractMessageTag(msg.Value)
				if limit == 1 && err == nil {
					// needn't check 'index out of range' here
					w.Header().Set(HttpHeaderMsgTag, hack.String(msg.Value[1:bodyIdx-1]))
				} else {
					// not a valid tagged message, treat it as non-tagged message
				}
			}

			if len(tags) > 0 {
				// TODO compare with tagFilters
			}

			if limit == 1 {
				// non-batch mode, just the message itself without meta
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					// when remote close silently, the write still ok
					return err
				}
			} else {
				// batch mode, write MessageSet
				// MessageSet => [Partition(int32) Offset(int64) MessageSize(int32) Message] BigEndian
				if metaBuf == nil {
					// initialize the reuseable buffer
					metaBuf = make([]byte, 8)

					// remove the middleware added header
					w.Header().Del("Content-Type")
				}

				if err = writeI32(w, metaBuf, msg.Partition); err != nil {
					return err
				}
				if err = writeI64(w, metaBuf, msg.Offset); err != nil {
					return err
				}
				if err = writeI32(w, metaBuf, int32(len(msg.Value[bodyIdx:]))); err != nil {
					return err
				}
				// TODO add tag?
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					return err
				}
			}

			if !delayedAck {
				log.Debug("sub commit offset %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)

				if err = fetcher.CommitUpto(msg); err != nil {
					return err
				}
			} else {
				log.Debug("sub take off %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)
			}

			this.subMetrics.ConsumeOk(myAppid, topic, ver)
			this.subMetrics.ConsumedOk(hisAppid, topic, ver)

			n++
			if n >= limit {
				return nil
			}

			// http chunked: len in hex
			// curl CURLOPT_HTTP_TRANSFER_DECODING will auto unchunk
			w.(http.Flusher).Flush()

			chunkedEver = true
			idleTimeout = wait // user specified wait only works after recv 1st message in batch
		}
	}
}
<commit_msg>code tidy up<commit_after>package gateway

import (
	"net/http"
	"strconv"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	"github.com/funkygao/gafka/sla"
	"github.com/funkygao/golib/hack"
	log "github.com/funkygao/log4go"
	"github.com/julienschmidt/httprouter"
)

// GET /v1/msgs/:appid/:topic/:ver?group=xx&batch=10&wait=5s&reset=<newest|oldest>&ack=1&q=<dead|retry>
func (this *subServer) subHandler(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	var (
		topic      string
		ver        string
		myAppid    string
		hisAppid   string
		reset      string
		group      string
		shadow     string
		rawTopic   string
		partition  string
		partitionN int = -1
		offset     string
		offsetN    int64         = -1
		limit      int           // max messages to include in the message set
		wait       time.Duration // max time to wait if insufficient data is available
		delayedAck bool          // explicit application level acknowledgement
		tagFilters []MsgTag      = nil
		err        error
	)

	if Options.EnableClientStats {
		this.gw.clientStates.RegisterSubClient(r)
	}

	query := r.URL.Query()
	group = query.Get("group")
	reset = query.Get("reset")
	if !manager.Default.ValidateGroupName(r.Header, group) {
		writeBadRequest(w, "illegal group")
		return
	}

	limit, err = getHttpQueryInt(&query, "batch", 1)
	if err != nil {
		writeBadRequest(w, "illegal limit")
		return
	}
	if limit > Options.MaxSubBatchSize && Options.MaxSubBatchSize > 0 {
		limit = Options.MaxSubBatchSize
	}

	wait, err = time.ParseDuration(query.Get("wait"))
	if err != nil || wait < MinSubWait {
		wait = MinSubWait
	}

	ver = params.ByName(UrlParamVersion)
	topic = params.ByName(UrlParamTopic)
	hisAppid = params.ByName(UrlParamAppid)
	myAppid = r.Header.Get(HttpHeaderAppid)
	realIp := getHttpRemoteIp(r)

	// auth
	if err = manager.Default.AuthSub(myAppid, r.Header.Get(HttpHeaderSubkey),
		hisAppid, topic, group); err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeAuthFailure(w, err)
		return
	}

	// fetch the client ack partition and offset
	delayedAck = query.Get("ack") == "1"
	if delayedAck {
		// consumers use explicit acknowledges in order to signal a message as processed successfully
		// if consumers fail to ACK, the message hangs and server will refuse to move ahead

		// get the partitionN and offsetN from client header
		// client will ack with partition=-1, offset=-1:
		// 1. handshake phase
		// 2. when 204 No Content
		partition = r.Header.Get(HttpHeaderPartition)
		offset = r.Header.Get(HttpHeaderOffset)
		if partition != "" && offset != "" {
			// convert partition and offset to int
			offsetN, err = strconv.ParseInt(offset, 10, 64)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} offset:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), offset)

				writeBadRequest(w, "ack with bad offset")
				return
			}
			partitionN, err = strconv.Atoi(partition)
			if err != nil {
				log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} partition:%s",
					myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
					group, r.Header.Get("User-Agent"), partition)

				writeBadRequest(w, "ack with bad partition")
				return
			}
		} else if len(partition+offset) != 0 {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s partition:%s offset:%s UA:%s} partial ack",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"))

			writeBadRequest(w, "partial ack not allowed")
			return
		}
	}

	shadow = query.Get("q")

	log.Debug("sub[%s] %s(%s): {app:%s q:%s topic:%s ver:%s group:%s batch:%d ack:%s partition:%s offset:%s UA:%s}",
		myAppid, r.RemoteAddr, realIp, hisAppid, shadow, topic, ver,
		group, limit, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"))

	// calculate raw topic according to shadow
	if shadow != "" {
		if !sla.ValidateShadowName(shadow) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} invalid shadow name",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "invalid shadow name")
			return
		}

		if !manager.Default.IsShadowedTopic(hisAppid, topic, ver, myAppid, group) {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s q:%s UA:%s} not a shadowed topic",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, shadow, r.Header.Get("User-Agent"))

			writeBadRequest(w, "register shadow first")
			return
		}

		rawTopic = manager.Default.ShadowTopic(shadow, myAppid, hisAppid, topic, ver, group)
	} else {
		rawTopic = manager.Default.KafkaTopic(hisAppid, topic, ver)
	}

	cluster, found := manager.Default.LookupCluster(hisAppid)
	if !found {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} cluster not found",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"))

		writeBadRequest(w, "invalid appid")
		return
	}

	fetcher, err := store.DefaultSubStore.Fetch(cluster, rawTopic,
		myAppid+"."+group, r.RemoteAddr, reset, Options.PermitStandbySub)
	if err != nil {
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, r.Header.Get("User-Agent"), err)

		writeBadRequest(w, err.Error())
		return
	}

	// commit the acked offset
	if delayedAck && partitionN >= 0 && offsetN >= 0 {
		// what if shutdown kateway now?
		// the commit will be ok, and when pumpMessages, the conn will get http.StatusNoContent
		if err = fetcher.CommitUpto(&sarama.ConsumerMessage{
			Topic:     rawTopic,
			Partition: int32(partitionN),
			Offset:    offsetN,
		}); err != nil {
			log.Error("sub commit[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:1 partition:%s offset:%s UA:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
				group, partition, offset, r.Header.Get("User-Agent"), err)

			writeBadRequest(w, err.Error())
			return
		} else {
			log.Debug("sub land %s(%s): {G:%s, T:%s, P:%s, O:%s}",
				r.RemoteAddr, realIp, group, rawTopic, partition, offset)
		}
	}

	tag := r.Header.Get(HttpHeaderMsgTag)
	if tag != "" {
		tagFilters = parseMessageTag(tag)
	}

	err = this.pumpMessages(w, r, fetcher, limit, wait, myAppid, hisAppid, topic, ver, group, delayedAck, tagFilters)
	if err != nil {
		// e,g. broken pipe, io timeout, client gone
		log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s ack:%s partition:%s offset:%s UA:%s} %v",
			myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver,
			group, query.Get("ack"), partition, offset, r.Header.Get("User-Agent"), err)

		writeServerError(w, err.Error())

		if err = fetcher.Close(); err != nil {
			log.Error("sub[%s] %s(%s): {app:%s topic:%s ver:%s group:%s} %v",
				myAppid, r.RemoteAddr, realIp, hisAppid, topic, ver, group, err)
		}
	}
}

func (this *subServer) pumpMessages(w http.ResponseWriter, r *http.Request,
	fetcher store.Fetcher, limit int, wait time.Duration, myAppid, hisAppid, topic, ver,
	group string, delayedAck bool, tagFilters []MsgTag) error {
	clientGoneCh := w.(http.CloseNotifier).CloseNotify()

	var (
		metaBuf     []byte = nil
		n                  = 0
		idleTimeout        = Options.SubTimeout
		realIp             = getHttpRemoteIp(r)
		chunkedEver        = false
	)
	for {
		select {
		case <-clientGoneCh:
			// FIXME access log will not be able to record this behavior
			return ErrClientGone

		case <-this.gw.shutdownCh:
			// don't call me again
			w.Header().Set("Connection", "close")

			if !chunkedEver {
				w.WriteHeader(http.StatusNoContent)
				w.Write([]byte{})
			}

			return nil

		case err := <-fetcher.Errors():
			// e,g. consume a non-existent topic
			// e,g. conn with broker is broken
			return err

		case <-this.gw.timer.After(idleTimeout):
			if chunkedEver {
				// response already sent in chunk
				log.Debug("chunked sub idle timeout %s {A:%s/G:%s->A:%s T:%s V:%s}",
					idleTimeout, myAppid, group, hisAppid, topic, ver)
				return nil
			}

			w.WriteHeader(http.StatusNoContent)
			w.Write([]byte{}) // without this, client cant get response
			return nil

		case msg := <-fetcher.Messages():
			partition := strconv.FormatInt(int64(msg.Partition), 10)

			if limit == 1 {
				w.Header().Set(HttpHeaderMsgKey, string(msg.Key))
				w.Header().Set(HttpHeaderPartition, partition)
				w.Header().Set(HttpHeaderOffset, strconv.FormatInt(msg.Offset, 10))
			}

			var (
				tags    []MsgTag
				bodyIdx int
				err     error
			)
			if IsTaggedMessage(msg.Value) {
				// TagMarkStart + tag + TagMarkEnd + body
				tags, bodyIdx, err = ExtractMessageTag(msg.Value)
				if limit == 1 && err == nil {
					// needn't check 'index out of range' here
					w.Header().Set(HttpHeaderMsgTag, hack.String(msg.Value[1:bodyIdx-1]))
				} else {
					// not a valid tagged message, treat it as non-tagged message
				}
			}

			if len(tags) > 0 {
				// TODO compare with tagFilters
			}

			if limit == 1 {
				// non-batch mode, just the message itself without meta
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					// when remote close silently, the write still ok
					return err
				}
			} else {
				// batch mode, write MessageSet
				// MessageSet => [Partition(int32) Offset(int64) MessageSize(int32) Message] BigEndian
				if metaBuf == nil {
					// initialize the reuseable buffer
					metaBuf = make([]byte, 8)

					// remove the middleware added header
					w.Header().Del("Content-Type")
				}

				if err = writeI32(w, metaBuf, msg.Partition); err != nil {
					return err
				}
				if err = writeI64(w, metaBuf, msg.Offset); err != nil {
					return err
				}
				if err = writeI32(w, metaBuf, int32(len(msg.Value[bodyIdx:]))); err != nil {
					return err
				}
				// TODO add tag?
				if _, err = w.Write(msg.Value[bodyIdx:]); err != nil {
					return err
				}
			}

			if !delayedAck {
				log.Debug("sub commit offset %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)

				if err = fetcher.CommitUpto(msg); err != nil {
					return err
				}
			} else {
				log.Debug("sub take off %s(%s): {G:%s, T:%s, P:%d, O:%d}",
					r.RemoteAddr, realIp, group, msg.Topic, msg.Partition, msg.Offset)
			}

			this.subMetrics.ConsumeOk(myAppid, topic, ver)
			this.subMetrics.ConsumedOk(hisAppid, topic, ver)

			n++
			if n >= limit {
				return nil
			}

			// http chunked: len in hex
			// curl CURLOPT_HTTP_TRANSFER_DECODING will auto unchunk
			w.(http.Flusher).Flush()

			chunkedEver = true
			idleTimeout = wait // user specified wait only works after recv 1st message in batch
		}
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"regexp"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsElasticSearchDomain() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElasticSearchDomainCreate,
		Read:   resourceAwsElasticSearchDomainRead,
		Update: resourceAwsElasticSearchDomainUpdate,
		Delete: resourceAwsElasticSearchDomainDelete,

		Schema: map[string]*schema.Schema{
			"access_policies": &schema.Schema{
				Type:      schema.TypeString,
				StateFunc: normalizeJson,
				Optional:  true,
			},
			"advanced_options": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				Computed: true,
			},
			"domain_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
					value := v.(string)
					if !regexp.MustCompile(`^[a-z][0-9a-z\-]{2,27}$`).MatchString(value) {
						errors = append(errors, fmt.Errorf(
							"%q must start with a lowercase alphabet and be at least 3 and no more than 28 characters long. Valid characters are a-z (lowercase letters), 0-9, and - (hyphen).", k))
					}
					return
				},
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"domain_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"endpoint": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"ebs_options": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"ebs_enabled": &schema.Schema{
							Type:     schema.TypeBool,
							Required: true,
						},
						"iops": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
						},
						"volume_size": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
						},
						"volume_type": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"cluster_config": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"dedicated_master_count": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
						},
						"dedicated_master_enabled": &schema.Schema{
							Type:     schema.TypeBool,
							Optional: true,
							Default:  false,
						},
						"dedicated_master_type": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
						"instance_count": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
							Default:  1,
						},
						"instance_type": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
							Default:  "m3.medium.elasticsearch",
						},
						"zone_awareness_enabled": &schema.Schema{
							Type:     schema.TypeBool,
							Optional: true,
						},
					},
				},
			},
			"snapshot_options": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"automated_snapshot_start_hour": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsElasticSearchDomainCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	input := elasticsearch.CreateElasticsearchDomainInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	}

	if v, ok := d.GetOk("access_policies"); ok {
		input.AccessPolicies = aws.String(v.(string))
	}

	if v, ok := d.GetOk("advanced_options"); ok {
		input.AdvancedOptions = stringMapToPointers(v.(map[string]interface{}))
	}

	if v, ok := d.GetOk("ebs_options"); ok {
		options := v.([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single ebs_options block is expected")
		} else if len(options) == 1 {
			if options[0] == nil {
				return fmt.Errorf("At least one field is expected inside ebs_options")
			}

			s := options[0].(map[string]interface{})
			input.EBSOptions = expandESEBSOptions(s)
		}
	}

	if v, ok := d.GetOk("cluster_config"); ok {
		config := v.([]interface{})

		if len(config) > 1 {
			return fmt.Errorf("Only a single cluster_config block is expected")
		} else if len(config) == 1 {
			if config[0] == nil {
				return fmt.Errorf("At least one field is expected inside cluster_config")
			}
			m := config[0].(map[string]interface{})
			input.ElasticsearchClusterConfig = expandESClusterConfig(m)
		}
	}

	if v, ok := d.GetOk("snapshot_options"); ok {
		options := v.([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single snapshot_options block is expected")
		} else if len(options) == 1 {
			if options[0] == nil {
				return fmt.Errorf("At least one field is expected inside snapshot_options")
			}

			o := options[0].(map[string]interface{})

			snapshotOptions := elasticsearch.SnapshotOptions{
				AutomatedSnapshotStartHour: aws.Int64(int64(o["automated_snapshot_start_hour"].(int))),
			}

			input.SnapshotOptions = &snapshotOptions
		}
	}

	log.Printf("[DEBUG] Creating ElasticSearch domain: %s", input)
	out, err := conn.CreateElasticsearchDomain(&input)
	if err != nil {
		return err
	}

	d.SetId(*out.DomainStatus.ARN)

	log.Printf("[DEBUG] Waiting for ElasticSearch domain %q to be created", d.Id())
	err = resource.Retry(30*time.Minute, func() *resource.RetryError {
		out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
			DomainName: aws.String(d.Get("domain_name").(string)),
		})
		if err != nil {
			return resource.NonRetryableError(err)
		}

		if !*out.DomainStatus.Processing && out.DomainStatus.Endpoint != nil {
			return nil
		}

		return resource.RetryableError(
			fmt.Errorf("%q: Timeout while waiting for the domain to be created", d.Id()))
	})
	if err != nil {
		return err
	}

	tags := tagsFromMapElasticsearchService(d.Get("tags").(map[string]interface{}))

	if err := setTagsElasticsearchService(conn, d, *out.DomainStatus.ARN); err != nil {
		return err
	}

	d.Set("tags", tagsToMapElasticsearchService(tags))
	d.SetPartial("tags")
	d.Partial(false)

	log.Printf("[DEBUG] ElasticSearch domain %q created", d.Id())

	return resourceAwsElasticSearchDomainRead(d, meta)
}

func resourceAwsElasticSearchDomainRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	})
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Received ElasticSearch domain: %s", out)

	ds := out.DomainStatus

	if ds.AccessPolicies != nil && *ds.AccessPolicies != "" {
		d.Set("access_policies", normalizeJson(*ds.AccessPolicies))
	}
	err = d.Set("advanced_options", pointersMapToStringList(ds.AdvancedOptions))
	if err != nil {
		return err
	}
	d.Set("domain_id", *ds.DomainId)
	d.Set("domain_name", *ds.DomainName)
	if ds.Endpoint != nil {
		d.Set("endpoint", *ds.Endpoint)
	}

	err = d.Set("ebs_options", flattenESEBSOptions(ds.EBSOptions))
	if err != nil {
		return err
	}
	err = d.Set("cluster_config", flattenESClusterConfig(ds.ElasticsearchClusterConfig))
	if err != nil {
		return err
	}
	if ds.SnapshotOptions != nil {
		d.Set("snapshot_options", map[string]interface{}{
			"automated_snapshot_start_hour": *ds.SnapshotOptions.AutomatedSnapshotStartHour,
		})
	}

	d.Set("arn", *ds.ARN)

	listOut, err := conn.ListTags(&elasticsearch.ListTagsInput{
		ARN: ds.ARN,
	})

	if err != nil {
		return err
	}
	var est []*elasticsearch.Tag
	if len(listOut.TagList) > 0 {
		est = listOut.TagList
	}

	d.Set("tags", tagsToMapElasticsearchService(est))

	return nil
}

func resourceAwsElasticSearchDomainUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	d.Partial(true)

	if err := setTagsElasticsearchService(conn, d, d.Id()); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	input := elasticsearch.UpdateElasticsearchDomainConfigInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	}

	if d.HasChange("access_policies") {
		input.AccessPolicies = aws.String(d.Get("access_policies").(string))
	}

	if d.HasChange("advanced_options") {
		input.AdvancedOptions = stringMapToPointers(d.Get("advanced_options").(map[string]interface{}))
	}

	if d.HasChange("ebs_options") {
		options := d.Get("ebs_options").([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single ebs_options block is expected")
		} else if len(options) == 1 {
			s := options[0].(map[string]interface{})
			input.EBSOptions = expandESEBSOptions(s)
		}
	}

	if d.HasChange("cluster_config") {
		config := d.Get("cluster_config").([]interface{})

		if len(config) > 1 {
			return fmt.Errorf("Only a single cluster_config block is expected")
		} else if len(config) == 1 {
			m := config[0].(map[string]interface{})
			input.ElasticsearchClusterConfig = expandESClusterConfig(m)
		}
	}

	if d.HasChange("snapshot_options") {
		options := d.Get("snapshot_options").([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single snapshot_options block is expected")
		} else if len(options) == 1 {
			o := options[0].(map[string]interface{})

			snapshotOptions := elasticsearch.SnapshotOptions{
				AutomatedSnapshotStartHour: aws.Int64(int64(o["automated_snapshot_start_hour"].(int))),
			}

			input.SnapshotOptions = &snapshotOptions
		}
	}

	_, err := conn.UpdateElasticsearchDomainConfig(&input)
	if err != nil {
		return err
	}

	err = resource.Retry(50*time.Minute, func() *resource.RetryError {
		out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
			DomainName: aws.String(d.Get("domain_name").(string)),
		})
		if err != nil {
			return resource.NonRetryableError(err)
		}

		if *out.DomainStatus.Processing == false {
			return nil
		}

		return resource.RetryableError(
			fmt.Errorf("%q: Timeout while waiting for changes to be processed", d.Id()))
	})
	if err != nil {
		return err
	}

	d.Partial(false)

	return resourceAwsElasticSearchDomainRead(d, meta)
}

func resourceAwsElasticSearchDomainDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	log.Printf("[DEBUG] Deleting ElasticSearch domain: %q", d.Get("domain_name").(string))
	_, err := conn.DeleteElasticsearchDomain(&elasticsearch.DeleteElasticsearchDomainInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	})
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Waiting for ElasticSearch domain %q to be deleted", d.Get("domain_name").(string))
	err = resource.Retry(15*time.Minute, func() *resource.RetryError {
		out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
			DomainName: aws.String(d.Get("domain_name").(string)),
		})

		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if !ok {
				return resource.NonRetryableError(err)
			}

			if awsErr.Code() == "ResourceNotFoundException" {
				return nil
			}

			return resource.NonRetryableError(err)
		}

		if !*out.DomainStatus.Processing {
			return nil
		}

		return resource.RetryableError(
			fmt.Errorf("%q: Timeout while waiting for the domain to be deleted", d.Id()))
	})

	d.SetId("")

	return err
}
<commit_msg>provider/aws: Bump ElasticSearch domain delete time to match create time. Should help test pass<commit_after>package aws

import (
	"fmt"
	"log"
	"regexp"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	elasticsearch "github.com/aws/aws-sdk-go/service/elasticsearchservice"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsElasticSearchDomain() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsElasticSearchDomainCreate,
		Read:   resourceAwsElasticSearchDomainRead,
		Update: resourceAwsElasticSearchDomainUpdate,
		Delete: resourceAwsElasticSearchDomainDelete,

		Schema: map[string]*schema.Schema{
			"access_policies": &schema.Schema{
				Type:      schema.TypeString,
				StateFunc: normalizeJson,
				Optional:  true,
			},
			"advanced_options": &schema.Schema{
				Type:     schema.TypeMap,
				Optional: true,
				Computed: true,
			},
			"domain_name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
					value := v.(string)
					if !regexp.MustCompile(`^[a-z][0-9a-z\-]{2,27}$`).MatchString(value) {
						errors = append(errors, fmt.Errorf(
							"%q must start with a lowercase alphabet and be at least 3 and no more than 28 characters long. Valid characters are a-z (lowercase letters), 0-9, and - (hyphen).", k))
					}
					return
				},
			},
			"arn": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"domain_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"endpoint": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"ebs_options": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"ebs_enabled": &schema.Schema{
							Type:     schema.TypeBool,
							Required: true,
						},
						"iops": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
						},
						"volume_size": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
						},
						"volume_type": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
					},
				},
			},
			"cluster_config": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"dedicated_master_count": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
						},
						"dedicated_master_enabled": &schema.Schema{
							Type:     schema.TypeBool,
							Optional: true,
							Default:  false,
						},
						"dedicated_master_type": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
						},
						"instance_count": &schema.Schema{
							Type:     schema.TypeInt,
							Optional: true,
							Default:  1,
						},
						"instance_type": &schema.Schema{
							Type:     schema.TypeString,
							Optional: true,
							Default:  "m3.medium.elasticsearch",
						},
						"zone_awareness_enabled": &schema.Schema{
							Type:     schema.TypeBool,
							Optional: true,
						},
					},
				},
			},
			"snapshot_options": &schema.Schema{
				Type:     schema.TypeList,
				Optional: true,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"automated_snapshot_start_hour": &schema.Schema{
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsElasticSearchDomainCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	input := elasticsearch.CreateElasticsearchDomainInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	}

	if v, ok := d.GetOk("access_policies"); ok {
		input.AccessPolicies = aws.String(v.(string))
	}

	if v, ok := d.GetOk("advanced_options"); ok {
		input.AdvancedOptions = stringMapToPointers(v.(map[string]interface{}))
	}

	if v, ok := d.GetOk("ebs_options"); ok {
		options := v.([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single ebs_options block is expected")
		} else if len(options) == 1 {
			if options[0] == nil {
				return fmt.Errorf("At least one field is expected inside ebs_options")
			}

			s := options[0].(map[string]interface{})
			input.EBSOptions = expandESEBSOptions(s)
		}
	}

	if v, ok := d.GetOk("cluster_config"); ok {
		config := v.([]interface{})

		if len(config) > 1 {
			return fmt.Errorf("Only a single cluster_config block is expected")
		} else if len(config) == 1 {
			if config[0] == nil {
				return fmt.Errorf("At least one field is expected inside cluster_config")
			}
			m := config[0].(map[string]interface{})
			input.ElasticsearchClusterConfig = expandESClusterConfig(m)
		}
	}

	if v, ok := d.GetOk("snapshot_options"); ok {
		options := v.([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single snapshot_options block is expected")
		} else if len(options) == 1 {
			if options[0] == nil {
				return fmt.Errorf("At least one field is expected inside snapshot_options")
			}

			o := options[0].(map[string]interface{})

			snapshotOptions := elasticsearch.SnapshotOptions{
				AutomatedSnapshotStartHour: aws.Int64(int64(o["automated_snapshot_start_hour"].(int))),
			}

			input.SnapshotOptions = &snapshotOptions
		}
	}

	log.Printf("[DEBUG] Creating ElasticSearch domain: %s", input)
	out, err := conn.CreateElasticsearchDomain(&input)
	if err != nil {
		return err
	}

	d.SetId(*out.DomainStatus.ARN)

	log.Printf("[DEBUG] Waiting for ElasticSearch domain %q to be created", d.Id())
	err = resource.Retry(30*time.Minute, func() *resource.RetryError {
		out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
			DomainName: aws.String(d.Get("domain_name").(string)),
		})
		if err != nil {
			return resource.NonRetryableError(err)
		}

		if !*out.DomainStatus.Processing && out.DomainStatus.Endpoint != nil {
			return nil
		}

		return resource.RetryableError(
			fmt.Errorf("%q: Timeout while waiting for the domain to be created", d.Id()))
	})
	if err != nil {
		return err
	}

	tags := tagsFromMapElasticsearchService(d.Get("tags").(map[string]interface{}))

	if err := setTagsElasticsearchService(conn, d, *out.DomainStatus.ARN); err != nil {
		return err
	}

	d.Set("tags", tagsToMapElasticsearchService(tags))
	d.SetPartial("tags")
	d.Partial(false)

	log.Printf("[DEBUG] ElasticSearch domain %q created", d.Id())

	return resourceAwsElasticSearchDomainRead(d, meta)
}

func resourceAwsElasticSearchDomainRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	})
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Received ElasticSearch domain: %s", out)

	ds := out.DomainStatus

	if ds.AccessPolicies != nil && *ds.AccessPolicies != "" {
		d.Set("access_policies", normalizeJson(*ds.AccessPolicies))
	}
	err = d.Set("advanced_options", pointersMapToStringList(ds.AdvancedOptions))
	if err != nil {
		return err
	}
	d.Set("domain_id", *ds.DomainId)
	d.Set("domain_name", *ds.DomainName)
	if ds.Endpoint != nil {
		d.Set("endpoint", *ds.Endpoint)
	}

	err = d.Set("ebs_options", flattenESEBSOptions(ds.EBSOptions))
	if err != nil {
		return err
	}
	err = d.Set("cluster_config", flattenESClusterConfig(ds.ElasticsearchClusterConfig))
	if err != nil {
		return err
	}
	if ds.SnapshotOptions != nil {
		d.Set("snapshot_options", map[string]interface{}{
			"automated_snapshot_start_hour": *ds.SnapshotOptions.AutomatedSnapshotStartHour,
		})
	}

	d.Set("arn", *ds.ARN)

	listOut, err := conn.ListTags(&elasticsearch.ListTagsInput{
		ARN: ds.ARN,
	})

	if err != nil {
		return err
	}
	var est []*elasticsearch.Tag
	if len(listOut.TagList) > 0 {
		est = listOut.TagList
	}

	d.Set("tags", tagsToMapElasticsearchService(est))

	return nil
}

func resourceAwsElasticSearchDomainUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	d.Partial(true)

	if err := setTagsElasticsearchService(conn, d, d.Id()); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	input := elasticsearch.UpdateElasticsearchDomainConfigInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	}

	if d.HasChange("access_policies") {
		input.AccessPolicies = aws.String(d.Get("access_policies").(string))
	}

	if d.HasChange("advanced_options") {
		input.AdvancedOptions = stringMapToPointers(d.Get("advanced_options").(map[string]interface{}))
	}

	if d.HasChange("ebs_options") {
		options := d.Get("ebs_options").([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single ebs_options block is expected")
		} else if len(options) == 1 {
			s := options[0].(map[string]interface{})
			input.EBSOptions = expandESEBSOptions(s)
		}
	}

	if d.HasChange("cluster_config") {
		config := d.Get("cluster_config").([]interface{})

		if len(config) > 1 {
			return fmt.Errorf("Only a single cluster_config block is expected")
		} else if len(config) == 1 {
			m := config[0].(map[string]interface{})
			input.ElasticsearchClusterConfig = expandESClusterConfig(m)
		}
	}

	if d.HasChange("snapshot_options") {
		options := d.Get("snapshot_options").([]interface{})

		if len(options) > 1 {
			return fmt.Errorf("Only a single snapshot_options block is expected")
		} else if len(options) == 1 {
			o := options[0].(map[string]interface{})

			snapshotOptions := elasticsearch.SnapshotOptions{
				AutomatedSnapshotStartHour: aws.Int64(int64(o["automated_snapshot_start_hour"].(int))),
			}

			input.SnapshotOptions = &snapshotOptions
		}
	}

	_, err := conn.UpdateElasticsearchDomainConfig(&input)
	if err != nil {
		return err
	}

	err = resource.Retry(50*time.Minute, func() *resource.RetryError {
		out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
			DomainName: aws.String(d.Get("domain_name").(string)),
		})
		if err != nil {
			return resource.NonRetryableError(err)
		}

		if *out.DomainStatus.Processing == false {
			return nil
		}

		return resource.RetryableError(
			fmt.Errorf("%q: Timeout while waiting for changes to be processed", d.Id()))
	})
	if err != nil {
		return err
	}

	d.Partial(false)

	return resourceAwsElasticSearchDomainRead(d, meta)
}

func resourceAwsElasticSearchDomainDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).esconn

	log.Printf("[DEBUG] Deleting ElasticSearch domain: %q", d.Get("domain_name").(string))
	_, err := conn.DeleteElasticsearchDomain(&elasticsearch.DeleteElasticsearchDomainInput{
		DomainName: aws.String(d.Get("domain_name").(string)),
	})
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Waiting for ElasticSearch domain %q to be deleted", d.Get("domain_name").(string))
	err = resource.Retry(30*time.Minute, func() *resource.RetryError {
		out, err := conn.DescribeElasticsearchDomain(&elasticsearch.DescribeElasticsearchDomainInput{
			DomainName: aws.String(d.Get("domain_name").(string)),
		})

		if err != nil {
			awsErr, ok := err.(awserr.Error)
			if !ok {
				return resource.NonRetryableError(err)
			}

			if awsErr.Code() == "ResourceNotFoundException" {
				return nil
			}

			return resource.NonRetryableError(err)
		}

		if !*out.DomainStatus.Processing {
			return nil
		}

		return resource.RetryableError(
			fmt.Errorf("%q: Timeout while waiting for the domain to be deleted", d.Id()))
	})

	d.SetId("")

	return err
}
<|endoftext|>


<commit_before>package dockerfile // import "github.com/docker/docker/builder/dockerfile"

import (
	"os"
	"runtime"
	"testing"

	"github.com/docker/docker/builder/remotecontext"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/reexec"
	"github.com/moby/buildkit/frontend/dockerfile/instructions"
	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
	"gotest.tools/skip"
)

type dispatchTestCase struct {
	name, expectedError string
	cmd                 instructions.Command
	files               map[string]string
}

func init() {
	reexec.Init()
}

func initDispatchTestCases() []dispatchTestCase {
	dispatchTestCases := []dispatchTestCase{
		{
			name: "ADD multiple files to file",
			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file1.txt",
				"file2.txt",
				"test",
			}},
			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
		},
		{
			name: "Wildcard ADD multiple files to file",
			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file*.txt",
				"test",
			}},
			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
		},
		{
			name: "COPY multiple files to file",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file1.txt",
				"file2.txt",
				"test",
			}},
			expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
		},
		{
			name: "ADD multiple files to file with whitespace",
			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
				"test file1.txt",
				"test file2.txt",
				"test",
			}},
			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
		},
		{
			name: "COPY multiple files to file with whitespace",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"test file1.txt",
				"test file2.txt",
				"test",
			}},
			expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
		},
		{
			name: "COPY wildcard no files",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file*.txt",
				"/tmp/",
			}},
			expectedError: "COPY failed: no source files were specified",
			files:         nil,
		},
		{
			name: "COPY url",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"https://index.docker.io/robots.txt",
				"/",
			}},
			expectedError: "source can't be a URL for COPY",
			files:         nil,
		}}

	return dispatchTestCases
}

func TestDispatch(t *testing.T) {
	if runtime.GOOS != "windows" {
		skip.If(t, os.Getuid() != 0, "skipping test that requires root")
	}
	testCases := initDispatchTestCases()

	for _, testCase := range testCases {
		executeTestCase(t, testCase)
	}
}

func executeTestCase(t *testing.T, testCase dispatchTestCase) {
	contextDir, cleanup := createTestTempDir(t, "", "builder-dockerfile-test")
	defer cleanup()

	for filename, content := range testCase.files {
		createTestTempFile(t, contextDir, filename, content, 0777)
	}

	tarStream, err := archive.Tar(contextDir, archive.Uncompressed)

	if err != nil {
		t.Fatalf("Error when creating tar stream: %s", err)
	}

	defer func() {
		if err = tarStream.Close(); err != nil {
			t.Fatalf("Error when closing tar stream: %s", err)
		}
	}()

	context, err := remotecontext.FromArchive(tarStream)

	if err != nil {
		t.Fatalf("Error when creating tar context: %s", err)
	}

	defer func() {
		if err = context.Close(); err != nil {
			t.Fatalf("Error when closing tar context: %s", err)
		}
	}()

	b := newBuilderWithMockBackend()
	sb := newDispatchRequest(b, '`', context, NewBuildArgs(make(map[string]*string)), newStagesBuildResults())
	err = dispatch(sb, testCase.cmd)
	assert.Check(t, is.ErrorContains(err, testCase.expectedError))
}
<commit_msg>TestDispatch: refactor to use subtests again, and fix linting (structcheck)<commit_after>package dockerfile // import "github.com/docker/docker/builder/dockerfile"

import (
	"os"
	"runtime"
	"testing"

	"github.com/docker/docker/builder/remotecontext"
	"github.com/docker/docker/pkg/archive"
	"github.com/docker/docker/pkg/reexec"
	"github.com/moby/buildkit/frontend/dockerfile/instructions"
	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
	"gotest.tools/skip"
)

type dispatchTestCase struct {
	name, expectedError string
	cmd                 instructions.Command
	files               map[string]string
}

func init() {
	reexec.Init()
}

func TestDispatch(t *testing.T) {
	if runtime.GOOS != "windows" {
		skip.If(t, os.Getuid() != 0, "skipping test that requires root")
	}
	testCases := []dispatchTestCase{
		{
			name: "ADD multiple files to file",
			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file1.txt",
				"file2.txt",
				"test",
			}},
			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
		},
		{
			name: "Wildcard ADD multiple files to file",
			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file*.txt",
				"test",
			}},
			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
		},
		{
			name: "COPY multiple files to file",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file1.txt",
				"file2.txt",
				"test",
			}},
			expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"file1.txt": "test1", "file2.txt": "test2"},
		},
		{
			name: "ADD multiple files to file with whitespace",
			cmd: &instructions.AddCommand{SourcesAndDest: instructions.SourcesAndDest{
				"test file1.txt",
				"test file2.txt",
				"test",
			}},
			expectedError: "When using ADD with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
		},
		{
			name: "COPY multiple files to file with whitespace",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"test file1.txt",
				"test file2.txt",
				"test",
			}},
			expectedError: "When using COPY with more than one source file, the destination must be a directory and end with a /",
			files:         map[string]string{"test file1.txt": "test1", "test file2.txt": "test2"},
		},
		{
			name: "COPY wildcard no files",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"file*.txt",
				"/tmp/",
			}},
			expectedError: "COPY failed: no source files were specified",
			files:         nil,
		},
		{
			name: "COPY url",
			cmd: &instructions.CopyCommand{SourcesAndDest: instructions.SourcesAndDest{
				"https://index.docker.io/robots.txt",
				"/",
			}},
			expectedError: "source can't be a URL for COPY",
			files:         nil,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			contextDir, cleanup := createTestTempDir(t, "", "builder-dockerfile-test")
			defer cleanup()

			for filename, content := range tc.files {
				createTestTempFile(t, contextDir, filename, content, 0777)
			}

			tarStream, err := archive.Tar(contextDir, archive.Uncompressed)

			if err != nil {
				t.Fatalf("Error when creating tar stream: %s", err)
			}

			defer func() {
				if err = tarStream.Close(); err != nil {
					t.Fatalf("Error when closing tar stream: %s", err)
				}
			}()

			context, err := remotecontext.FromArchive(tarStream)

			if err != nil {
				t.Fatalf("Error when creating tar context: %s", err)
			}

			defer func() {
				if err = context.Close(); err != nil {
					t.Fatalf("Error when closing tar context: %s", err)
				}
			}()

			b := newBuilderWithMockBackend()
			sb := newDispatchRequest(b, '`', context, NewBuildArgs(make(map[string]*string)), newStagesBuildResults())
			err = dispatch(sb, tc.cmd)
			assert.Check(t, is.ErrorContains(err, tc.expectedError))
		})
	}
}
<|endoftext|>


<commit_before>package mysql_nodejs

import (
	. "github.com/gocircuit/circuit/gocircuit.org/render"
)

func RenderRun() string {
	return RenderHtml("Run the app on the cluster", Render(runBody, nil))
}

const runBody = `

Run the app on the cluster

        `
<commit_msg>tut done<commit_after>package mysql_nodejs

import (
	. "github.com/gocircuit/circuit/gocircuit.org/render"
)

func RenderRun() string {
	return RenderHtml("Run the app on the cluster", Render(runBody, nil))
}

const runBody = `

Run the app on the cluster

Here we get to run the circuit program that we wrote in the previous section.
Log into any one of the EC2 instances that are part of your circuit cluster.

First, build and install the circuit app, which can be found within the circuit repo:

	$ go install github.com/gocircuit/circuit/tutorial/nodejs-using-mysql/start-app

This will place the resulting executable in $GOPATH/bin.

And finally we can execute the circuit app, instructing it to connect (as a client)
to the circuit server running on the host we are currently on:

	$ $GOPATH/bin/start-app -addr $(cat /var/circuit/address)

If successful, the app will print out the addresses of the MySQL server and
the Node.js service and will exit.

You should also be able to see the Node.js process element using the circuit command-line tool:

	$ circuit ls -d $(cat /var/circuit/address) -l /...

This concludes the tutorial.

        `
<|endoftext|>


<commit_before>package backend

import (
	"errors"
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"

	"github.com/Dataman-Cloud/swan/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/types"

	"github.com/Sirupsen/logrus"
)

// ScaleApplication is used to scale application instances.
func (b *Backend) ScaleApplication(applicationId string, instances int) error {
	app, err := b.store.FetchApplication(applicationId)
	if err != nil {
		return err
	}

	if app.Status != "RUNNING" {
		return errors.New("Operation Not Allowed")
	}

	// Update application status to SCALING
	if err := b.store.UpdateApplication(applicationId, "status", "SCALING"); err != nil {
		logrus.Errorf("Updating application status to SCALING failed: %s", err.Error())
		return err
	}

	versions, err := b.store.ListApplicationVersions(applicationId)
	if err != nil {
		return err
	}

	sort.Strings(versions)

	newestVersion := versions[len(versions)-1]
	version, err := b.store.FetchApplicationVersion(applicationId, newestVersion)
	if err != nil {
		return err
	}

	if app.Instances > instances {
		tasks, err := b.store.ListApplicationTasks(app.ID)
		if err != nil {
			return err
		}

		for _, task := range tasks {
			taskIndex, err := strconv.Atoi(strings.Split(task.Name, ".")[0])
			if err != nil {
				return err
			}

			if taskIndex+1 > instances {
				b.sched.HealthCheckManager.StopCheck(task.Name)

				if err := b.store.DeleteCheck(task.Name); err != nil {
					logrus.Errorf("Remove health check for %s failed: %s", task.Name, err.Error())
					return err
				}

				if _, err := b.sched.KillTask(task); err == nil {
					b.store.DeleteApplicationTask(app.ID, task.ID)
				}

				// reduce application tasks count
				if err := b.store.ReduceApplicationInstances(app.ID); err != nil {
					logrus.Errorf("Updating application %s instances count failed: %s", app.ID, err.Error())
					return err
				}

				logrus.Infof("Remove health check for task %s", task.Name)

				if err := b.store.DeleteApplicationTask(app.ID, task.Name); err != nil {
					logrus.Errorf("Delete task %s failed: %s", task.Name, err.Error())
				}

			}
		}
	}

	if app.Instances < instances {
		for i := 0; i < instances-app.Instances; i++ {
			b.sched.Status = "busy"

			resources := b.sched.BuildResources(version.Cpus, version.Mem, version.Disk)
			offers, err := b.sched.RequestOffers(resources)
			if err != nil {
				logrus.Errorf("Request offers failed: %s for rescheduling", err.Error())
			}

			var choosedOffer *mesos.Offer
			for _, offer := range offers {
				cpus, mem, disk := b.sched.OfferedResources(offer)
				if cpus >= version.Cpus && mem >= version.Mem && disk >= version.Disk {
					choosedOffer = offer
					break
				}
			}

			name := fmt.Sprintf("%d.%s.%s.%s", app.Instances+i, app.ID, app.UserId, app.ClusterId)

			task, err := b.sched.BuildTask(choosedOffer, version, name)
			if err != nil {
				logrus.Errorf("Build task failed: %s", err.Error())
				return err
			}

			var taskInfos []*mesos.TaskInfo
			taskInfo := b.sched.BuildTaskInfo(choosedOffer, resources, task)
			taskInfos = append(taskInfos, taskInfo)

			resp, err := b.sched.LaunchTasks(choosedOffer, taskInfos)
			if err != nil {
				logrus.Errorf("Launchs task failed: %s", err.Error())
				return err
			}

			if resp != nil && resp.StatusCode != http.StatusAccepted {
				return fmt.Errorf("status code %d received", resp.StatusCode)
			}

			if err := b.store.RegisterTask(task); err != nil {
				return err
			}

			if len(version.HealthChecks) != 0 {
				if err := b.store.RegisterCheck(task,
					*taskInfo.Container.Docker.PortMappings[0].HostPort,
					app.ID); err != nil {
				}
				for _, healthCheck := range task.HealthChecks {
					check := types.Check{
						ID:       task.Name,
						Address:  *task.AgentHostname,
						Port:     int(*taskInfo.Container.Docker.PortMappings[0].HostPort),
						TaskID:   task.Name,
						AppID:    app.ID,
						Protocol: healthCheck.Protocol,
						Interval: int(healthCheck.IntervalSeconds),
						Timeout:  int(healthCheck.TimeoutSeconds),
					}
					if healthCheck.Command != nil {
						check.Command = healthCheck.Command
					}

					if healthCheck.Path != nil {
						check.Path = *healthCheck.Path
					}

					if healthCheck.MaxConsecutiveFailures != nil {
						check.MaxFailures = *healthCheck.MaxConsecutiveFailures
					}

					b.sched.HealthCheckManager.Add(&check)
				}
			}

			// Increase application task count
			if err := b.store.IncreaseApplicationInstances(version.ID); err != nil {
				logrus.Errorf("Updating application %s instance count failed: %s", version.ID, err.Error())
				return err
			}

			b.sched.Status = "idle"
		}
	}

	// Update application status to RUNNING
	if err := b.store.UpdateApplication(version.ID, "status", "RUNNING"); err != nil {
		logrus.Errorf("Updating application %s status to RUNNING failed: %s", version.ID, err.Error())
		return err
	}

	return nil
}
<commit_msg>fixed scale as goroutine<commit_after>package backend

import (
	"errors"
	"fmt"
	"net/http"
	"sort"
	"strconv"
	"strings"

	"github.com/Dataman-Cloud/swan/mesosproto/mesos"
	"github.com/Dataman-Cloud/swan/types"

	"github.com/Sirupsen/logrus"
)

// ScaleApplication is used to scale application instances.
func (b *Backend) ScaleApplication(applicationId string, instances int) error {
	app, err := b.store.FetchApplication(applicationId)
	if err != nil {
		return err
	}

	if app.Status != "RUNNING" {
		return errors.New("Operation Not Allowed")
	}

	// Update application status to SCALING
	if err := b.store.UpdateApplication(applicationId, "status", "SCALING"); err != nil {
		logrus.Errorf("Updating application status to SCALING failed: %s", err.Error())
		return err
	}

	versions, err := b.store.ListApplicationVersions(applicationId)
	if err != nil {
		return err
	}

	sort.Strings(versions)

	newestVersion := versions[len(versions)-1]
	version, err := b.store.FetchApplicationVersion(applicationId, newestVersion)
	if err != nil {
		return err
	}

	go func() error {
		if app.Instances > instances {
			tasks, err := b.store.ListApplicationTasks(app.ID)
			if err != nil {
				return err
			}

			for _, task := range tasks {
				taskIndex, err := strconv.Atoi(strings.Split(task.Name, ".")[0])
				if err != nil {
					return err
				}

				if taskIndex+1 > instances {
					b.sched.HealthCheckManager.StopCheck(task.Name)

					if err := b.store.DeleteCheck(task.Name); err != nil {
						logrus.Errorf("Remove health check for %s failed: %s", task.Name, err.Error())
						return err
					}

					if _, err := b.sched.KillTask(task); err == nil {
						b.store.DeleteApplicationTask(app.ID, task.ID)
					}

					// reduce application tasks count
					if err := b.store.ReduceApplicationInstances(app.ID); err != nil {
						logrus.Errorf("Updating application %s instances count failed: %s", app.ID, err.Error())
						return err
					}

					logrus.Infof("Remove health check for task %s", task.Name)

					if err := b.store.DeleteApplicationTask(app.ID, task.Name); err != nil {
						logrus.Errorf("Delete task %s failed: %s", task.Name, err.Error())
					}

				}
			}
		}

		if app.Instances < instances {
			for i := 0; i < instances-app.Instances; i++ {
				b.sched.Status = "busy"

				resources := b.sched.BuildResources(version.Cpus, version.Mem, version.Disk)
				offers, err := b.sched.RequestOffers(resources)
				if err != nil {
					logrus.Errorf("Request offers failed: %s for rescheduling", err.Error())
				}

				var choosedOffer *mesos.Offer
				for _, offer := range offers {
					cpus, mem, disk := b.sched.OfferedResources(offer)
					if cpus >= version.Cpus && mem >= version.Mem && disk >= version.Disk {
						choosedOffer = offer
						break
					}
				}

				name := fmt.Sprintf("%d.%s.%s.%s", app.Instances+i, app.ID, app.UserId, app.ClusterId)

				task, err := b.sched.BuildTask(choosedOffer, version, name)
				if err != nil {
					logrus.Errorf("Build task failed: %s", err.Error())
					return err
				}

				var taskInfos []*mesos.TaskInfo
				taskInfo := b.sched.BuildTaskInfo(choosedOffer, resources, task)
				taskInfos = append(taskInfos, taskInfo)

				resp, err := b.sched.LaunchTasks(choosedOffer, taskInfos)
				if err != nil {
					logrus.Errorf("Launchs task failed: %s", err.Error())
					return err
				}

				if resp != nil && resp.StatusCode != http.StatusAccepted {
					return fmt.Errorf("status code %d received", resp.StatusCode)
				}

				if err := b.store.RegisterTask(task); err != nil {
					return err
				}

				if len(version.HealthChecks) != 0 {
					if err := b.store.RegisterCheck(task,
						*taskInfo.Container.Docker.PortMappings[0].HostPort,
						app.ID); err != nil {
					}
					for _, healthCheck := range task.HealthChecks {
						check := types.Check{
							ID:       task.Name,
							Address:  *task.AgentHostname,
							Port:     int(*taskInfo.Container.Docker.PortMappings[0].HostPort),
							TaskID:   task.Name,
							AppID:    app.ID,
							Protocol: healthCheck.Protocol,
							Interval: int(healthCheck.IntervalSeconds),
							Timeout:  int(healthCheck.TimeoutSeconds),
						}
						if healthCheck.Command != nil {
							check.Command = healthCheck.Command
						}

						if healthCheck.Path != nil {
							check.Path = *healthCheck.Path
						}

						if healthCheck.MaxConsecutiveFailures != nil {
							check.MaxFailures = *healthCheck.MaxConsecutiveFailures
						}

						b.sched.HealthCheckManager.Add(&check)
					}
				}

				// Increase application task count
				if err := b.store.IncreaseApplicationInstances(version.ID); err != nil {
					logrus.Errorf("Updating application %s instance count failed: %s", version.ID, err.Error())
					return err
				}

				b.sched.Status = "idle"
			}
		}

		// Update application status to RUNNING
		if err := b.store.UpdateApplication(version.ID, "status", "RUNNING"); err != nil {
			logrus.Errorf("Updating application %s status to RUNNING failed: %s", version.ID, err.Error())
			return err
		}

		return nil
	}()

	return nil
}
<|endoftext|>