<commit_before>package aws

import (
	"context"
	"encoding/base64"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/mq"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func resourceAwsMqConfiguration() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsMqConfigurationCreate,
		Read:   resourceAwsMqConfigurationRead,
		Update: resourceAwsMqConfigurationUpdate,
		Delete: resourceAwsMqConfigurationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		CustomizeDiff: func(_ context.Context, diff *schema.ResourceDiff, v interface{}) error {
			if diff.HasChange("description") {
				return diff.SetNewComputed("latest_revision")
			}
			if diff.HasChange("data") {
				o, n := diff.GetChange("data")
				os := o.(string)
				ns := n.(string)
				if !suppressXMLEquivalentConfig("data", os, ns, nil) {
					return diff.SetNewComputed("latest_revision")
				}
			}
			return nil
		},

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"data": {
				Type:             schema.TypeString,
				Required:         true,
				DiffSuppressFunc: suppressXMLEquivalentConfig,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"engine_type": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: validation.StringInSlice([]string{
					mq.EngineTypeActivemq,
				}, true),
			},
			"engine_version": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"latest_revision": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsMqConfigurationCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).mqconn

	input := mq.CreateConfigurationRequest{
		EngineType:    aws.String(d.Get("engine_type").(string)),
		EngineVersion: aws.String(d.Get("engine_version").(string)),
		Name:          aws.String(d.Get("name").(string)),
	}

	if v, ok := d.GetOk("tags"); ok {
		input.Tags = keyvaluetags.New(v.(map[string]interface{})).IgnoreAws().MqTags()
	}

	log.Printf("[INFO] Creating MQ Configuration: %s", input)
	out, err := conn.CreateConfiguration(&input)
	if err != nil {
		return err
	}

	d.SetId(*out.Id)
	d.Set("arn", out.Arn)

	return resourceAwsMqConfigurationUpdate(d, meta)
}

func resourceAwsMqConfigurationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).mqconn
	ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig

	log.Printf("[INFO] Reading MQ Configuration %s", d.Id())
	out, err := conn.DescribeConfiguration(&mq.DescribeConfigurationInput{
		ConfigurationId: aws.String(d.Id()),
	})
	if err != nil {
		if isAWSErr(err, mq.ErrCodeNotFoundException, "") {
			log.Printf("[WARN] MQ Configuration %q not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("arn", out.Arn)
	d.Set("description", out.LatestRevision.Description)
	d.Set("engine_type", out.EngineType)
	d.Set("engine_version", out.EngineVersion)
	d.Set("name", out.Name)
	d.Set("latest_revision", out.LatestRevision.Revision)

	rOut, err := conn.DescribeConfigurationRevision(&mq.DescribeConfigurationRevisionInput{
		ConfigurationId:       aws.String(d.Id()),
		ConfigurationRevision: aws.String(fmt.Sprintf("%d", *out.LatestRevision.Revision)),
	})
	if err != nil {
		return err
	}

	b, err := base64.StdEncoding.DecodeString(*rOut.Data)
	if err != nil {
		return err
	}

	d.Set("data", string(b))

	if err := d.Set("tags", keyvaluetags.MqKeyValueTags(out.Tags).IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}

func resourceAwsMqConfigurationUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).mqconn

	rawData := d.Get("data").(string)
	data := base64.StdEncoding.EncodeToString([]byte(rawData))

	input := mq.UpdateConfigurationRequest{
		ConfigurationId: aws.String(d.Id()),
		Data:            aws.String(data),
	}
	if v, ok := d.GetOk("description"); ok {
		input.Description = aws.String(v.(string))
	}

	log.Printf("[INFO] Updating MQ Configuration %s: %s", d.Id(), input)
	_, err := conn.UpdateConfiguration(&input)
	if err != nil {
		return err
	}

	if d.HasChange("tags") {
		o, n := d.GetChange("tags")

		if err := keyvaluetags.MqUpdateTags(conn, d.Get("arn").(string), o, n); err != nil {
			return fmt.Errorf("error updating MQ Broker (%s) tags: %s", d.Get("arn").(string), err)
		}
	}

	return resourceAwsMqConfigurationRead(d, meta)
}

func resourceAwsMqConfigurationDelete(d *schema.ResourceData, meta interface{}) error {
	// TODO: Delete is not available in the API

	return nil
}

func suppressXMLEquivalentConfig(k, old, new string, d *schema.ResourceData) bool {
	os, err := canonicalXML(old)
	if err != nil {
		log.Printf("[ERR] Error getting cannonicalXML from state (%s): %s", k, err)
		return false
	}
	ns, err := canonicalXML(new)
	if err != nil {
		log.Printf("[ERR] Error getting cannonicalXML from config (%s): %s", k, err)
		return false
	}

	return os == ns
}
<commit_msg>resource/aws_mq_configuration: Don't update the configuration if only tags change. (#14850)<commit_after>package aws

import (
	"context"
	"encoding/base64"
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/mq"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation"
	"github.com/terraform-providers/terraform-provider-aws/aws/internal/keyvaluetags"
)

func resourceAwsMqConfiguration() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsMqConfigurationCreate,
		Read:   resourceAwsMqConfigurationRead,
		Update: resourceAwsMqConfigurationUpdate,
		Delete: resourceAwsMqConfigurationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},
		CustomizeDiff: func(_ context.Context, diff *schema.ResourceDiff, v interface{}) error {
			if diff.HasChange("description") {
				return diff.SetNewComputed("latest_revision")
			}
			if diff.HasChange("data") {
				o, n := diff.GetChange("data")
				os := o.(string)
				ns := n.(string)
				if !suppressXMLEquivalentConfig("data", os, ns, nil) {
					return diff.SetNewComputed("latest_revision")
				}
			}
			return nil
		},

		Schema: map[string]*schema.Schema{
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"data": {
				Type:             schema.TypeString,
				Required:         true,
				DiffSuppressFunc: suppressXMLEquivalentConfig,
			},
			"description": {
				Type:     schema.TypeString,
				Optional: true,
			},
			"engine_type": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
				ValidateFunc: validation.StringInSlice([]string{
					mq.EngineTypeActivemq,
				}, true),
			},
			"engine_version": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"name": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"latest_revision": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"tags": tagsSchema(),
		},
	}
}

func resourceAwsMqConfigurationCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).mqconn

	input := mq.CreateConfigurationRequest{
		EngineType:    aws.String(d.Get("engine_type").(string)),
		EngineVersion: aws.String(d.Get("engine_version").(string)),
		Name:          aws.String(d.Get("name").(string)),
	}

	if v, ok := d.GetOk("tags"); ok {
		input.Tags = keyvaluetags.New(v.(map[string]interface{})).IgnoreAws().MqTags()
	}

	log.Printf("[INFO] Creating MQ Configuration: %s", input)
	out, err := conn.CreateConfiguration(&input)
	if err != nil {
		return err
	}

	d.SetId(*out.Id)
	d.Set("arn", out.Arn)

	return resourceAwsMqConfigurationUpdate(d, meta)
}

func resourceAwsMqConfigurationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).mqconn
	ignoreTagsConfig := meta.(*AWSClient).IgnoreTagsConfig

	log.Printf("[INFO] Reading MQ Configuration %s", d.Id())
	out, err := conn.DescribeConfiguration(&mq.DescribeConfigurationInput{
		ConfigurationId: aws.String(d.Id()),
	})
	if err != nil {
		if isAWSErr(err, mq.ErrCodeNotFoundException, "") {
			log.Printf("[WARN] MQ Configuration %q not found, removing from state", d.Id())
			d.SetId("")
			return nil
		}
		return err
	}

	d.Set("arn", out.Arn)
	d.Set("description", out.LatestRevision.Description)
	d.Set("engine_type", out.EngineType)
	d.Set("engine_version", out.EngineVersion)
	d.Set("name", out.Name)
	d.Set("latest_revision", out.LatestRevision.Revision)

	rOut, err := conn.DescribeConfigurationRevision(&mq.DescribeConfigurationRevisionInput{
		ConfigurationId:       aws.String(d.Id()),
		ConfigurationRevision: aws.String(fmt.Sprintf("%d", *out.LatestRevision.Revision)),
	})
	if err != nil {
		return err
	}

	b, err := base64.StdEncoding.DecodeString(*rOut.Data)
	if err != nil {
		return err
	}

	d.Set("data", string(b))

	if err := d.Set("tags", keyvaluetags.MqKeyValueTags(out.Tags).IgnoreAws().IgnoreConfig(ignoreTagsConfig).Map()); err != nil {
		return fmt.Errorf("error setting tags: %s", err)
	}

	return nil
}

func resourceAwsMqConfigurationUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).mqconn

	if d.HasChanges("data", "description") {
		rawData := d.Get("data").(string)
		data := base64.StdEncoding.EncodeToString([]byte(rawData))

		input := mq.UpdateConfigurationRequest{
			ConfigurationId: aws.String(d.Id()),
			Data:            aws.String(data),
		}
		if v, ok := d.GetOk("description"); ok {
			input.Description = aws.String(v.(string))
		}

		log.Printf("[INFO] Updating MQ Configuration %s: %s", d.Id(), input)
		_, err := conn.UpdateConfiguration(&input)
		if err != nil {
			return err
		}
	}

	if d.HasChange("tags") {
		o, n := d.GetChange("tags")

		if err := keyvaluetags.MqUpdateTags(conn, d.Get("arn").(string), o, n); err != nil {
			return fmt.Errorf("error updating MQ Broker (%s) tags: %s", d.Get("arn").(string), err)
		}
	}

	return resourceAwsMqConfigurationRead(d, meta)
}

func resourceAwsMqConfigurationDelete(d *schema.ResourceData, meta interface{}) error {
	// TODO: Delete is not available in the API

	return nil
}

func suppressXMLEquivalentConfig(k, old, new string, d *schema.ResourceData) bool {
	os, err := canonicalXML(old)
	if err != nil {
		log.Printf("[ERR] Error getting cannonicalXML from state (%s): %s", k, err)
		return false
	}
	ns, err := canonicalXML(new)
	if err != nil {
		log.Printf("[ERR] Error getting cannonicalXML from config (%s): %s", k, err)
		return false
	}

	return os == ns
}
<|endoftext|>


<commit_before>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/prometheus/procfs/internal/util"
	"strconv"
	"strings"
)

// Cgroup models one line from /proc/[pid]/cgroup. Each Cgroup struct describes the the placement of a PID inside a
// specific control hierarchy. The kernel has two cgroup APIs, v1 and v2. v1 has one hierarchy per available resource
// controller, while v2 has one unified hierarchy shared by all controllers. Regardless of v1 or v2, all hierarchies
// contain all running processes, so the question answerable with a Cgroup struct is 'where is this process in
// this hierarchy' (where==what path). By prefixing this path with the mount point of this hierarchy, you can locate
// the relevant pseudo-files needed to read/set the data for this PID in this hierarchy
//
// Also see http://man7.org/linux/man-pages/man7/cgroups.7.html
type Cgroup struct {
	// HierarchyId for cgroups V2 is always 0. For cgroups v1 this is a unique
	// ID number that can be matched to a hierarchy ID found in /proc/cgroups
	HierarchyId int
	// Controllers using this hierarchy of processes. Controllers are also known as subsystems.
	Controllers []string
	// Path of this control group, relative to the mount point of the various controllers
	Path string
}

// parseCgroupString parses each line of the /proc/[pid]/cgroup file
func parseCgroupString(cgroupStr string) (*Cgroup, error) {
	var err error

	fields := strings.Split(cgroupStr, ":")
	if len(fields) != 3 {
		return nil, fmt.Errorf("incorrect number of fields (%d) in cgroup string: %s", len(fields), cgroupStr)
	}

	cgroup := &Cgroup{
		Path:        fields[2],
		Controllers: nil,
	}
	cgroup.HierarchyId, err = strconv.Atoi(fields[0])
	if err != nil {
		return nil, fmt.Errorf("failed to parse hierarchy ID")
	}
	if fields[1] != "" {
		ssNames := strings.Split(fields[1], ",")
		cgroup.Controllers = append(cgroup.Controllers, ssNames...)
	}
	return cgroup, nil
}

// parseCgroups reads each line of the /proc/[pid]/cgroup file
func parseCgroups(data []byte) ([]*Cgroup, error) {
	cgroups := []*Cgroup{}
	scanner := bufio.NewScanner(bytes.NewReader(data))
	for scanner.Scan() {
		mountString := scanner.Text()
		parsedMounts, err := parseCgroupString(mountString)
		if err != nil {
			return nil, err
		}
		cgroups = append(cgroups, parsedMounts)
	}

	err := scanner.Err()
	return cgroups, err
}

// GetCgroups returns a Cgroup struct for all process control hierarchies running on this system
func GetCgroups(pid int) ([]*Cgroup, error) {
	data, err := util.ReadFileNoStat(fmt.Sprintf("/proc/%d/cgroup", pid))
	if err != nil {
		return nil, err
	}
	return parseCgroups(data)
}
<commit_msg>Improve cgroup documentation<commit_after>// Copyright 2019 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package procfs

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/prometheus/procfs/internal/util"
	"strconv"
	"strings"
)

// Cgroup models one line from /proc/[pid]/cgroup. Each Cgroup struct describes the the placement of a PID inside a
// specific control hierarchy. The kernel has two cgroup APIs, v1 and v2. v1 has one hierarchy per available resource
// controller, while v2 has one unified hierarchy shared by all controllers. Regardless of v1 or v2, all hierarchies
// contain all running processes, so the question answerable with a Cgroup struct is 'where is this process in
// this hierarchy' (where==what path on the specific cgroupfs). By prefixing this path with the mount point of
// *this specific* hierarchy, you can locate the relevant pseudo-files needed to read/set the data for this PID
// in this hierarchy
//
// Also see http://man7.org/linux/man-pages/man7/cgroups.7.html
type Cgroup struct {
	// HierarchyId that can be matched to a named hierarchy using /proc/cgroups. Cgroups V2 only has one
	// hierarchy, so HierarchyId is always 0. For cgroups v1 this is a unique ID number
	HierarchyId int
	// Controllers using this hierarchy of processes. Controllers are also known as subsystems. For
	// Cgroups V2 this may be empty, as all active controllers use the same hierarchy
	Controllers []string
	// Path of this control group, relative to the mount point of the cgroupfs representing this specific
	// hierarchy
	Path string
}

// parseCgroupString parses each line of the /proc/[pid]/cgroup file
// Line format is hierarchyID:[controller1,controller2]:path
func parseCgroupString(cgroupStr string) (*Cgroup, error) {
	var err error

	fields := strings.Split(cgroupStr, ":")
	if len(fields) != 3 {
		return nil, fmt.Errorf("incorrect number of fields (%d) in cgroup string: %s", len(fields), cgroupStr)
	}

	cgroup := &Cgroup{
		Path:        fields[2],
		Controllers: nil,
	}
	cgroup.HierarchyId, err = strconv.Atoi(fields[0])
	if err != nil {
		return nil, fmt.Errorf("failed to parse hierarchy ID")
	}
	if fields[1] != "" {
		ssNames := strings.Split(fields[1], ",")
		cgroup.Controllers = append(cgroup.Controllers, ssNames...)
	}
	return cgroup, nil
}

// parseCgroups reads each line of the /proc/[pid]/cgroup file
func parseCgroups(data []byte) ([]*Cgroup, error) {
	cgroups := []*Cgroup{}
	scanner := bufio.NewScanner(bytes.NewReader(data))
	for scanner.Scan() {
		mountString := scanner.Text()
		parsedMounts, err := parseCgroupString(mountString)
		if err != nil {
			return nil, err
		}
		cgroups = append(cgroups, parsedMounts)
	}

	err := scanner.Err()
	return cgroups, err
}

// GetCgroups returns a Cgroup struct for all process control hierarchies running on this system
func GetCgroups(pid int) ([]*Cgroup, error) {
	data, err := util.ReadFileNoStat(fmt.Sprintf("/proc/%d/cgroup", pid))
	if err != nil {
		return nil, err
	}
	return parseCgroups(data)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	skippedPaths   = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go|site/themes/docsy`)
	rootdir        *string
	boilerplatedir *string
)

func init() {
	cwd, _ := os.Getwd()
	boilerplatedir = flag.String("boilerplate-dir", cwd, "Boilerplate directory for boilerplate files")
	cwd = cwd + "/../../"
	rootdir = flag.String("rootdir", filepath.Dir(cwd), "Root directory to examine")
}

func main() {
	flag.Parse()
	refs := getRefs(*boilerplatedir)
	if len(refs) == 0 {
		log.Fatal("no references in ", *boilerplatedir)
	}
	files := getFileList(*rootdir, refs)
	for _, file := range files {
		if !filePasses(file, refs[getFileExtension(file)]) {
			fmt.Println(file)
		}
	}

}

func getRefs(dir string) map[string][]byte {
	refs := make(map[string][]byte)
	files, _ := filepath.Glob(dir + "/*.txt")
	for _, filename := range files {
		extension := strings.ToLower(strings.Split(filename, ".")[1])
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Fatal(err)
		}
		re := regexp.MustCompile(`\r`)
		refs[extension] = re.ReplaceAll(data, nil)
	}
	return refs
}

func filePasses(filename string, ref []byte) bool {
	var re *regexp.Regexp
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	re = regexp.MustCompile(`\r`)
	data = re.ReplaceAll(data, nil)

	extension := getFileExtension(filename)

	// remove build tags from the top of Go files
	if extension == "go" {
		re = regexp.MustCompile(`(?m)^(// \+build.*\n)+\n`)
		data = re.ReplaceAll(data, nil)
	}

	// remove shebang from the top of shell files
	if extension == "sh" {
		re = regexp.MustCompile(`(?m)^(#!.*\n)\n*`)
		data = re.ReplaceAll(data, nil)
	}

	// if our test file is smaller than the reference it surely fails!
	if len(data) < len(ref) {
		return false
	}

	data = data[:len(ref)]

	// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
	re = regexp.MustCompile(`Copyright YEAR`)
	if re.Match(data) {
		return false
	}

	// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
	re = regexp.MustCompile(`Copyright \d{4}`)
	data = re.ReplaceAll(data, []byte(`Copyright YEAR`))

	return bytes.Equal(data, ref)
}

// get the file extensin or the filename if the file has no extension
func getFileExtension(filename string) string {
	splitted := strings.Split(filepath.Base(filename), ".")
	return strings.ToLower(splitted[len(splitted)-1])
}

func getFileList(rootDir string, extensions map[string][]byte) []string {
	var outFiles []string
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() && !skippedPaths.MatchString(filepath.Dir(path)) {
			if extensions[strings.ToLower(getFileExtension(path))] != nil {
				outFiles = append(outFiles, path)
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}
	return outFiles
}
<commit_msg>Add comments to boilerplate.go<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var (
	skippedPaths   = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go|site/themes/docsy`)
	rootdir        *string
	boilerplatedir *string
)

func init() {
	cwd, _ := os.Getwd()
	boilerplatedir = flag.String("boilerplate-dir", cwd, "Boilerplate directory for boilerplate files")
	cwd = cwd + "/../../"
	rootdir = flag.String("rootdir", filepath.Dir(cwd), "Root directory to examine")
}

func main() {
	flag.Parse()
	refs := getRefs(*boilerplatedir)
	if len(refs) == 0 {
		log.Fatal("no references in ", *boilerplatedir)
	}
	files := getFileList(*rootdir, refs)
	for _, file := range files {
		if !filePasses(file, refs[getFileExtension(file)]) {
			fmt.Println(file)
		}
	}

}

/*
This function is to populate the refs variable with the
different boilerplate/template for different extension.
*/
func getRefs(dir string) map[string][]byte {
	refs := make(map[string][]byte)
	files, _ := filepath.Glob(dir + "/*.txt")
	for _, filename := range files {
		extension := strings.ToLower(strings.Split(filename, ".")[1])
		data, err := ioutil.ReadFile(filename)
		if err != nil {
			log.Fatal(err)
		}
		re := regexp.MustCompile(`\r`)
		refs[extension] = re.ReplaceAll(data, nil)
	}
	return refs
}

/*
Function to check whether the processed file
is valid.
Returning false means that the file does not the
proper boilerplate template
*/
func filePasses(filename string, ref []byte) bool {
	var re *regexp.Regexp
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatal(err)
	}
	re = regexp.MustCompile(`\r`)
	data = re.ReplaceAll(data, nil)

	extension := getFileExtension(filename)

	// remove build tags from the top of Go files
	if extension == "go" {
		re = regexp.MustCompile(`(?m)^(// \+build.*\n)+\n`)
		data = re.ReplaceAll(data, nil)
	}

	// remove shebang from the top of shell files
	if extension == "sh" {
		re = regexp.MustCompile(`(?m)^(#!.*\n)\n*`)
		data = re.ReplaceAll(data, nil)
	}

	// if our test file is smaller than the reference it surely fails!
	if len(data) < len(ref) {
		return false
	}

	data = data[:len(ref)]

	// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
	re = regexp.MustCompile(`Copyright YEAR`)
	if re.Match(data) {
		return false
	}

	// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
	re = regexp.MustCompile(`Copyright \d{4}`)
	data = re.ReplaceAll(data, []byte(`Copyright YEAR`))

	return bytes.Equal(data, ref)
}

/**
Function to get the file extensin or the filename if the file has no extension
*/
func getFileExtension(filename string) string {
	splitted := strings.Split(filepath.Base(filename), ".")
	return strings.ToLower(splitted[len(splitted)-1])
}

/**
Function to get all the files from the directory that heeds to be checked.
*/
func getFileList(rootDir string, extensions map[string][]byte) []string {
	var outFiles []string
	err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() && !skippedPaths.MatchString(filepath.Dir(path)) {
			if extensions[strings.ToLower(getFileExtension(path))] != nil {
				outFiles = append(outFiles, path)
			}
		}
		return nil
	})
	if err != nil {
		log.Fatal(err)
	}
	return outFiles
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/sirupsen/logrus"

	"github.com/ivan1993spb/snake-server/connections"
)

const URLRouteDeleteGameByID = "/games/{id}"

const MethodDeleteGame = http.MethodDelete

type responseDeleteGameHandler struct {
	ID int `json:"id"`
}

type deleteGameHandler struct {
	logger       logrus.FieldLogger
	groupManager *connections.ConnectionGroupManager
}

type ErrDeleteGameHandler string

func (e ErrDeleteGameHandler) Error() string {
	return "delete game handler error: " + string(e)
}

func NewDeleteGameHandler(logger logrus.FieldLogger, groupManager *connections.ConnectionGroupManager) http.Handler {
	return &deleteGameHandler{
		logger:       logger,
		groupManager: groupManager,
	}
}

func (h *deleteGameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)

	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}

	h.logger.Infoln("group id to delete:", id)

	group, err := h.groupManager.Get(id)
	if err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))

		switch err {
		case connections.ErrNotFoundGroup:
			http.NotFound(w, r)
		default:
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		}
		return
	}

	if !group.IsEmpty() {
		h.logger.Warn(ErrDeleteGameHandler("try to delete not empty group"))
		h.logger.Warnf("there is %d opened connections in group %d", group.GetCount(), id)
		http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
		return
	}

	if err := h.groupManager.Delete(group); err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))

		switch err {
		case connections.ErrDeleteNotFoundGroup:
			http.NotFound(w, r)
		case connections.ErrDeleteNotEmptyGroup:
			http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
		default:
			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		}
		return
	}

	h.logger.Info("stop group")
	group.Stop()

	h.logger.Infoln("group deleted:", id)

	w.Header().Set("Content-Type", "application/json; charset=utf-8")

	err = json.NewEncoder(w).Encode(responseDeleteGameHandler{
		ID: id,
	})
	if err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
}
<commit_msg>Create JSON error response in deleteGameHandler<commit_after>package handlers

import (
	"encoding/json"
	"net/http"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/sirupsen/logrus"

	"github.com/ivan1993spb/snake-server/connections"
)

const URLRouteDeleteGameByID = "/games/{id}"

const MethodDeleteGame = http.MethodDelete

type responseDeleteGameHandler struct {
	ID int `json:"id"`
}

type responseDeleteGameHandlerError struct {
	Code int    `json:"code"`
	Text string `json:"text"`
	ID   int    `json:"id"`
}

type deleteGameHandler struct {
	logger       logrus.FieldLogger
	groupManager *connections.ConnectionGroupManager
}

type ErrDeleteGameHandler string

func (e ErrDeleteGameHandler) Error() string {
	return "delete game handler error: " + string(e)
}

func NewDeleteGameHandler(logger logrus.FieldLogger, groupManager *connections.ConnectionGroupManager) http.Handler {
	return &deleteGameHandler{
		logger:       logger,
		groupManager: groupManager,
	}
}

func (h *deleteGameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)

	id, err := strconv.Atoi(vars["id"])
	if err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))
		h.writeResponseJSON(w, http.StatusBadRequest, &responseDeleteGameHandlerError{
			Code: http.StatusBadRequest,
			Text: "invalid game id",
			ID:   id,
		})
		return
	}

	h.logger.Infoln("group id to delete:", id)

	group, err := h.groupManager.Get(id)
	if err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))

		switch err {
		case connections.ErrNotFoundGroup:
			h.writeResponseJSON(w, http.StatusNotFound, &responseDeleteGameHandlerError{
				Code: http.StatusNotFound,
				Text: "game not found",
				ID:   id,
			})
		default:
			h.writeResponseJSON(w, http.StatusInternalServerError, &responseDeleteGameHandlerError{
				Code: http.StatusInternalServerError,
				Text: "unknown error",
				ID:   id,
			})
		}
		return
	}

	if !group.IsEmpty() {
		h.logger.Warn(ErrDeleteGameHandler("try to delete not empty group"))
		h.logger.Warnf("there is %d opened connections in group %d", group.GetCount(), id)
		h.writeResponseJSON(w, http.StatusServiceUnavailable, &responseDeleteGameHandlerError{
			Code: http.StatusServiceUnavailable,
			Text: "cannot delete not empty game",
			ID:   id,
		})
		return
	}

	if err := h.groupManager.Delete(group); err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))

		switch err {
		case connections.ErrDeleteNotFoundGroup:
			h.writeResponseJSON(w, http.StatusNotFound, &responseDeleteGameHandlerError{
				Code: http.StatusNotFound,
				Text: "game not found",
				ID:   id,
			})
		case connections.ErrDeleteNotEmptyGroup:
			h.writeResponseJSON(w, http.StatusServiceUnavailable, &responseDeleteGameHandlerError{
				Code: http.StatusServiceUnavailable,
				Text: "cannot delete not empty game",
				ID:   id,
			})
		default:
			h.writeResponseJSON(w, http.StatusInternalServerError, &responseDeleteGameHandlerError{
				Code: http.StatusInternalServerError,
				Text: "unknown error",
				ID:   id,
			})
		}
		return
	}

	h.logger.Info("stop group")
	group.Stop()

	h.logger.Infoln("group deleted:", id)

	h.writeResponseJSON(w, http.StatusOK, responseDeleteGameHandler{
		ID: id,
	})
}

func (h *deleteGameHandler) writeResponseJSON(w http.ResponseWriter, statusCode int, response interface{}) {
	w.WriteHeader(statusCode)
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("X-Content-Type-Options", "nosniff")

	if err := json.NewEncoder(w).Encode(response); err != nil {
		h.logger.Error(ErrDeleteGameHandler(err.Error()))
	}
}
<|endoftext|>


<commit_before>package seccomp

import (
	"fmt"
)

const (
	F_OK        = 0x0
	R_OK        = 0x4
	W_OK        = 0x2
	X_OK        = 0x1
	EFF_ONLY_OK = 0x08
)

func render_access(pid int, args RegisterArgs) (string, error) {

	flags := map[uint]string{
		R_OK:        "R_OK",
		W_OK:        "W_OK",
		X_OK:        "X_OK",
		EFF_ONLY_OK: "EFF_ONLY_OK",
	}

	mode := args[1]
	path, err := readStringArg(pid, uintptr(args[0]))

	if (err != nil) {
		return "", err
	}

	found := false
	var flagstr string

	if mode == F_OK {
		flagstr = "F_OK"
	} else {

		for flag := range flags {
			if (mode & uint64(flag)) == mode {
				if found == true {
					flagstr += "|"
				}
				flagstr += flags[flag]
				found = true
			}
		}

	}
	callrep := fmt.Sprintf("access(\"%s\", %s)", path, flagstr)

	return fmt.Sprintf("==============================================\nseccomp hit on sandbox pid %v (%v) syscall %v (%v): \n\n%s\nI ==============================================\n\n", pid, getProcessCmdLine(pid), "access", 1, callrep), nil
}
<commit_msg>Function for rendering an invocation of access(2) in the Oz seccomp tracer.<commit_after>package seccomp

import (
	"fmt"
	"syscall"
)

const (
	F_OK        = 0x0
	R_OK        = 0x4
	W_OK        = 0x2
	X_OK        = 0x1
	EFF_ONLY_OK = 0x08
)

func render_access(pid int, args RegisterArgs) (string, error) {

	flags := map[uint]string{
		R_OK:        "R_OK",
		W_OK:        "W_OK",
		X_OK:        "X_OK",
		EFF_ONLY_OK: "EFF_ONLY_OK",
	}

	mode := args[1]
	path, err := readStringArg(pid, uintptr(args[0]))

	if err != nil {
		return "", err
	}

	flagstr := ""
	if mode == F_OK {
		flagstr = "F_OK"
	} else {
		flagstr = renderFlags(flags, uint(mode))
	}
	callrep := fmt.Sprintf("access(\"%s\", %s)", path, flagstr)

	return fmt.Sprintf("==============================================\nseccomp hit on sandbox pid %v (%v) syscall %v (%v): \n\n%s\nI ==============================================\n\n", pid, getProcessCmdLine(pid), "access", syscall.SYS_ACCESS, callrep), nil
}
<|endoftext|>


<commit_before>package autonat

import (
	"time"
)

// config holds configurable options for the autonat subsystem.
type config struct {
	getAddressFunc  GetAddrs
	bootDelay       time.Duration
	retryInterval   time.Duration
	refreshInterval time.Duration
	requestTimeout  time.Duration
}

var defaults = func(c *config) error {
	c.bootDelay = 15 * time.Second
	c.retryInterval = 90 * time.Second
	c.refreshInterval = 15 * time.Minute
	c.requestTimeout = 30 * time.Second

	return nil
}

// WithAddresses allows overriding which Addresses the AutoNAT client beliieves
// are "its own". Useful for testing, or for more exotic port-forwarding
// scenarios where the host may be listening on different ports than it wants
// to externally advertise or verify connectability on.
func WithAddresses(addrFunc GetAddrs) Option {
	return func(c *config) error {
		c.getAddressFunc = addrFunc
		return nil
	}
}

// WithSchedule configures how agressively probes will be made to verify the
// address of the host. retryInterval indicates how often probes should be made
// when the host lacks confident about its address, while refresh interval
// is the schedule of periodic probes when the host believes it knows its
// steady-state reachability.
func WithSchedule(retryInterval, refreshInterval time.Duration) Option {
	return func(c *config) error {
		c.retryInterval = retryInterval
		c.refreshInterval = refreshInterval
		return nil
	}
}

// WithoutStartupDelay removes the initial delay the NAT subsystem typically
// uses as a buffer for ensuring that connectivity and guesses as to the hosts
// local interfaces have settled down during startup.
func WithoutStartupDelay() Option {
	return func(c *config) error {
		c.bootDelay = 1
		return nil
	}
}
<commit_msg>remove a foot-gun<commit_after>package autonat

import (
	"errors"
	"time"
)

// config holds configurable options for the autonat subsystem.
type config struct {
	getAddressFunc  GetAddrs
	bootDelay       time.Duration
	retryInterval   time.Duration
	refreshInterval time.Duration
	requestTimeout  time.Duration
}

var defaults = func(c *config) error {
	c.bootDelay = 15 * time.Second
	c.retryInterval = 90 * time.Second
	c.refreshInterval = 15 * time.Minute
	c.requestTimeout = 30 * time.Second

	return nil
}

// WithAddresses allows overriding which Addresses the AutoNAT client beliieves
// are "its own". Useful for testing, or for more exotic port-forwarding
// scenarios where the host may be listening on different ports than it wants
// to externally advertise or verify connectability on.
func WithAddresses(addrFunc GetAddrs) Option {
	return func(c *config) error {
		if addrFunc == nil {
			return errors.New("invalid address function supplied")
		}
		c.getAddressFunc = addrFunc
		return nil
	}
}

// WithSchedule configures how agressively probes will be made to verify the
// address of the host. retryInterval indicates how often probes should be made
// when the host lacks confident about its address, while refresh interval
// is the schedule of periodic probes when the host believes it knows its
// steady-state reachability.
func WithSchedule(retryInterval, refreshInterval time.Duration) Option {
	return func(c *config) error {
		c.retryInterval = retryInterval
		c.refreshInterval = refreshInterval
		return nil
	}
}

// WithoutStartupDelay removes the initial delay the NAT subsystem typically
// uses as a buffer for ensuring that connectivity and guesses as to the hosts
// local interfaces have settled down during startup.
func WithoutStartupDelay() Option {
	return func(c *config) error {
		c.bootDelay = 1
		return nil
	}
}
<|endoftext|>


<commit_before>package raftgorums

import (
	"container/list"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/go-kit/kit/metrics"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
	pb "github.com/relab/raft/raftgorums/raftpb"
)

// RequestVote implements gorums.RaftServer.
func (r *Raft) RequestVote(ctx context.Context, req *pb.RequestVoteRequest) (*pb.RequestVoteResponse, error) {
	return r.HandleRequestVoteRequest(req), nil
}

// AppendEntries implements gorums.RaftServer.
func (r *Raft) AppendEntries(ctx context.Context, req *pb.AppendEntriesRequest) (*pb.AppendEntriesResponse, error) {
	return r.HandleAppendEntriesRequest(req), nil
}

// InstallSnapshot implements gorums.RaftServer.
func (r *Raft) InstallSnapshot(ctx context.Context, snapshot *commonpb.Snapshot) (*pb.InstallSnapshotResponse, error) {
	return r.HandleInstallSnapshotRequest(snapshot), nil
}

// CatchMeUp implements gorums.RaftServer.
func (r *Raft) CatchMeUp(ctx context.Context, req *pb.CatchMeUpRequest) (res *pb.Empty, err error) {
	res = &pb.Empty{}
	r.match[r.mem.getNodeID(req.FollowerID)] <- req.NextIndex
	return
}

// HandleRequestVoteRequest must be called when receiving a RequestVoteRequest,
// the return value must be delivered to the requester.
func (r *Raft) HandleRequestVoteRequest(req *pb.RequestVoteRequest) *pb.RequestVoteResponse {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.rvreq)
		defer timer.ObserveDuration()
	}

	var voteGranted bool
	defer func() {
		r.logger.WithFields(logrus.Fields{
			"currentterm": r.currentTerm,
			"requestterm": req.Term,
			"prevote":     req.PreVote,
			"candidateid": req.CandidateID,
			"votegranted": voteGranted,
		}).Infoln("Got vote request")
	}()

	// #RV1 Reply false if term < currentTerm.
	if req.Term < r.currentTerm {
		return &pb.RequestVoteResponse{Term: r.currentTerm}
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if req.Term > r.currentTerm && !req.PreVote {
		r.becomeFollower(req.Term)
	}

	voted := r.votedFor != None

	if req.PreVote && (r.heardFromLeader || (voted && req.Term == r.currentTerm)) {
		// We don't grant pre-votes if we have recently heard from a
		// leader or already voted in the pre-term.
		return &pb.RequestVoteResponse{Term: r.currentTerm}
	}

	lastIndex := r.storage.NextIndex() - 1
	lastLogTerm := r.logTerm(lastIndex)

	// We can grant a vote in the same term, as long as it's to the same
	// candidate. This is useful if the response was lost, and the candidate
	// sends another request.
	alreadyVotedForCandidate := r.votedFor == req.CandidateID

	// If the logs have last entries with different terms, the log with the
	// later term is more up-to-date.
	laterTerm := req.LastLogTerm > lastLogTerm

	// If the logs end with the same term, whichever log is longer is more
	// up-to-date.
	longEnough := req.LastLogTerm == lastLogTerm && req.LastLogIndex >= lastIndex

	// We can only grant a vote if: we have not voted yet, we vote for the
	// same candidate again, or this is a pre-vote.
	canGrantVote := !voted || alreadyVotedForCandidate || req.PreVote

	// #RV2 If votedFor is null or candidateId, and candidate's log is at
	// least as up-to-date as receiver's log, grant vote.
	voteGranted = canGrantVote && (laterTerm || longEnough)

	if voteGranted {
		if req.PreVote {
			return &pb.RequestVoteResponse{VoteGranted: true, Term: req.Term}
		}

		r.votedFor = req.CandidateID
		r.storage.Set(raft.KeyVotedFor, req.CandidateID)

		// #F2 If election timeout elapses without receiving
		// AppendEntries RPC from current leader or granting a vote to
		// candidate: convert to candidate. Here we are granting a vote
		// to a candidate so we reset the election timeout.
		r.resetElection = true
		r.resetBaseline = true

		return &pb.RequestVoteResponse{VoteGranted: true, Term: r.currentTerm}
	}

	// #RV2 The candidate's log was not up-to-date
	return &pb.RequestVoteResponse{Term: r.currentTerm}
}

// HandleAppendEntriesRequest must be called when receiving a
// AppendEntriesRequest, the return value must be delivered to the requester.
func (r *Raft) HandleAppendEntriesRequest(req *pb.AppendEntriesRequest) *pb.AppendEntriesResponse {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.aereq)
		defer timer.ObserveDuration()
	}

	reqLogger := r.logger.WithFields(logrus.Fields{
		"currentterm":  r.currentTerm,
		"requestterm":  req.Term,
		"leaderid":     req.LeaderID,
		"prevlogindex": req.PrevLogIndex,
		"prevlogterm":  req.PrevLogTerm,
		"commitindex":  req.CommitIndex,
		"lenentries":   len(req.Entries),
	})
	reqLogger.Infoln("Got AppendEntries")

	logLen := r.storage.NextIndex() - 1

	res := &pb.AppendEntriesResponse{
		Term:       r.currentTerm,
		MatchIndex: logLen,
	}

	// #AE1 Reply false if term < currentTerm.
	if req.Term < r.currentTerm {
		return res
	}

	prevTerm := r.logTerm(req.PrevLogIndex)

	// An AppendEntries request is always successful for the first index. A
	// leader can only be elected leader if its log matches that of a
	// majority and our log is guaranteed to be at least 0 in length.
	firstIndex := req.PrevLogIndex == 0

	// The index preceding the entries we are going to replicate must be in our log.
	gotPrevIndex := req.PrevLogIndex <= logLen
	// The term must match to satisfy the log matching property.
	sameTerm := req.PrevLogTerm == prevTerm

	// If the previous entry is in our log, then our log matches the leaders
	// up till and including the previous entry. And we can safely replicate
	// next new entries.
	gotPrevEntry := gotPrevIndex && sameTerm

	success := firstIndex || gotPrevEntry

	// #A2 If RPC request or response contains term T > currentTerm: set
	// currentTerm = T, convert to follower. Transition to follower upon
	// receiving an AppendEntries call.
	if req.Term > r.currentTerm || r.state != Follower {
		r.becomeFollower(req.Term)
		res.Term = req.Term
	}

	if r.metricsEnabled {
		rmetrics.leader.Set(float64(req.LeaderID))
	}

	// We acknowledge this server as the leader as it's has the highest term
	// we have seen, and there can only be one leader per term.
	r.leader = req.LeaderID
	r.heardFromLeader = true
	r.seenLeader = true

	// Don't timeout during catch up.
	if uint64(len(req.Entries)) > r.burst {
		r.resetElection = true
	}

	if !success {
		r.cureqout <- &catchUpReq{
			leaderID: req.LeaderID,
			// TODO term: req.Term, ?
			matchIndex: res.MatchIndex,
		}

		return res
	}

	var toSave []*commonpb.Entry
	index := req.PrevLogIndex

	for _, entry := range req.Entries {
		// Increment first so we start at previous index + 1.
		index++

		// If the terms don't match, our logs conflict at this index. On
		// the first conflict this will truncate the log to the lowest
		// common matching index. After that it will fill the log with
		// the new entries from the leader. This is because entry.Term
		// will always conflict with term 0, which will be returned for
		// indexes outside our log.
		if entry.Term != r.logTerm(index) {
			logLen = r.storage.NextIndex() - 1
			for logLen > index-1 {
				// If we are overwriting the latest
				// configuration, rollback to the committed one.
				if logLen == r.mem.getIndex() {
					r.mem.rollback()
				}
				r.storage.RemoveEntries(logLen, logLen)
				logLen = r.storage.NextIndex() - 1
			}
			toSave = append(toSave, entry)
		}
	}

	if len(toSave) > 0 {
		r.storage.StoreEntries(toSave)
	}
	logLen = r.storage.NextIndex() - 1

	for _, entry := range toSave {
		if entry.EntryType == commonpb.EntryReconf {
			var reconf commonpb.ReconfRequest
			err := reconf.Unmarshal(entry.Data)

			if err != nil {
				panic("could not unmarshal reconf")
			}

			r.mem.setPending(&reconf)
			r.mem.set(entry.Index)
		}
	}

	old := r.commitIndex
	// Commit index can not exceed the length of our log.
	r.commitIndex = min(req.CommitIndex, logLen)

	if r.metricsEnabled {
		rmetrics.commitIndex.Set(float64(r.commitIndex))
	}

	if r.commitIndex > old {
		r.logger.WithFields(logrus.Fields{
			"oldcommitindex": old,
			"commitindex":    r.commitIndex,
		}).Infoln("Set commit index")

		r.newCommit(old)
	}

	reqLogger.WithFields(logrus.Fields{
		"lensaved": len(toSave),
		"lenlog":   logLen,
		"success":  success,
	}).Infoln("Saved entries to stable storage")

	res.Success = true
	return res
}

func (r *Raft) HandleInstallSnapshotRequest(snapshot *commonpb.Snapshot) (res *pb.InstallSnapshotResponse) {
	r.mu.Lock()
	defer r.mu.Unlock()

	res = &pb.InstallSnapshotResponse{
		Term: r.currentTerm,
	}

	return
}

// HandleRequestVoteResponse must be invoked when receiving a
// RequestVoteResponse.
func (r *Raft) HandleRequestVoteResponse(response *pb.RequestVoteResponse) {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.rvres)
		defer timer.ObserveDuration()
	}

	r.logger.WithFields(logrus.Fields{
		"currentterm":  r.currentTerm,
		"responseterm": response.Term,
		"votegranted":  response.VoteGranted,
	}).Infoln("Got vote response")

	term := r.currentTerm

	if r.preElection {
		term++
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if response.Term > term {
		r.becomeFollower(response.Term)

		return
	}

	// Ignore late response
	if response.Term < term {
		return
	}

	// Cont. from startElection(). We have now received a response from Gorums.

	// #C5 If votes received from majority of server: become leader.
	// Make sure we have not stepped down while waiting for replies.
	if r.state == Candidate && response.VoteGranted {
		if r.preElection {
			r.preElection = false
			select {
			case r.startElectionNow <- struct{}{}:
			case <-r.stop:
			}
			return
		}

		// We have received at least a quorum of votes.
		// We are the leader for this term. See Raft Paper Figure 2 -> Rules for Servers -> Leaders.

		if r.metricsEnabled {
			rmetrics.leader.Set(float64(r.id))
		}

		r.logger.WithFields(logrus.Fields{
			"currentterm": r.currentTerm,
		}).Infoln("Elected leader")

		logLen := r.storage.NextIndex() - 1

		r.state = Leader
		r.leader = r.id
		r.seenLeader = true
		r.heardFromLeader = true
		r.nextIndex = logLen + 1
		r.pending = list.New()
		r.pendingReads = nil
		r.mem.setStable(false)

		// Empty queue.
	EMPTYCH:
		for {
			select {
			case <-r.queue:
				// TODO Respond with NotLeader.
			default:
				// Paper §8: We add a no-op, so that the leader
				// commits an entry from its own term. This
				// ensures that the leader knows which entries
				// are committed.
				promise, _ := raft.NewPromiseEntry(&commonpb.Entry{
					EntryType: commonpb.EntryInternal,
					Term:      r.currentTerm,
					Data:      raft.NOOP,
				})
				r.queue <- promise
				break EMPTYCH
			}
		}

		select {
		case r.heartbeatNow <- struct{}{}:
		default:
		}

		return
	}

	r.preElection = true

	// #C7 If election timeout elapses: start new election.
	// This will happened if we don't receive enough replies in time. Or we lose the election but don't see a higher term number.
}

// HandleAppendEntriesResponse must be invoked when receiving an
// AppendEntriesResponse.
func (r *Raft) HandleAppendEntriesResponse(response *pb.AppendEntriesQFResponse, maxIndex uint64) {
	r.mu.Lock()
	defer func() {
		r.mu.Unlock()
		r.advanceCommitIndex()
	}()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.aeres)
		defer timer.ObserveDuration()
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	// If we didn't get a response from a majority (excluding self) step down.
	if response.Term > r.currentTerm || response.Replies < uint64((len(r.mem.get().NodeIDs())+1)/2) {
		// Become follower.
		select {
		case r.toggle <- struct{}{}:
			r.logger.Warnln("Leader stepping down")
		case <-r.stop:
		}

		return
	}

	// Heartbeat to a majority.
	r.resetElection = true

	// Ignore late response
	if response.Term < r.currentTerm {
		return
	}

	if response.Success {
		r.matchIndex = maxIndex
		r.nextIndex = r.matchIndex + 1
		r.logger.WithFields(logrus.Fields{
			"matchindex": r.matchIndex,
			"nextindex":  r.nextIndex,
		}).Warnln("Setting matchindex")

		return
	}

	// If AppendEntries was not successful lower match index.
	r.nextIndex = max(1, min(r.nextIndex-r.burst, r.matchIndex+1))
}

func (r *Raft) HandleInstallSnapshotResponse(res *pb.InstallSnapshotResponse) bool {
	r.mu.Lock()
	defer r.mu.Unlock()

	if res.Term > r.currentTerm {
		r.becomeFollower(res.Term)

		return false
	}

	return true
}
<commit_msg>raftgorums/incoming.go: Fix followers timing out<commit_after>package raftgorums

import (
	"container/list"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/go-kit/kit/metrics"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
	pb "github.com/relab/raft/raftgorums/raftpb"
)

// RequestVote implements gorums.RaftServer.
func (r *Raft) RequestVote(ctx context.Context, req *pb.RequestVoteRequest) (*pb.RequestVoteResponse, error) {
	return r.HandleRequestVoteRequest(req), nil
}

// AppendEntries implements gorums.RaftServer.
func (r *Raft) AppendEntries(ctx context.Context, req *pb.AppendEntriesRequest) (*pb.AppendEntriesResponse, error) {
	return r.HandleAppendEntriesRequest(req), nil
}

// InstallSnapshot implements gorums.RaftServer.
func (r *Raft) InstallSnapshot(ctx context.Context, snapshot *commonpb.Snapshot) (*pb.InstallSnapshotResponse, error) {
	return r.HandleInstallSnapshotRequest(snapshot), nil
}

// CatchMeUp implements gorums.RaftServer.
func (r *Raft) CatchMeUp(ctx context.Context, req *pb.CatchMeUpRequest) (res *pb.Empty, err error) {
	res = &pb.Empty{}
	r.match[r.mem.getNodeID(req.FollowerID)] <- req.NextIndex
	return
}

// HandleRequestVoteRequest must be called when receiving a RequestVoteRequest,
// the return value must be delivered to the requester.
func (r *Raft) HandleRequestVoteRequest(req *pb.RequestVoteRequest) *pb.RequestVoteResponse {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.rvreq)
		defer timer.ObserveDuration()
	}

	var voteGranted bool
	defer func() {
		r.logger.WithFields(logrus.Fields{
			"currentterm": r.currentTerm,
			"requestterm": req.Term,
			"prevote":     req.PreVote,
			"candidateid": req.CandidateID,
			"votegranted": voteGranted,
		}).Infoln("Got vote request")
	}()

	// #RV1 Reply false if term < currentTerm.
	if req.Term < r.currentTerm {
		return &pb.RequestVoteResponse{Term: r.currentTerm}
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if req.Term > r.currentTerm && !req.PreVote {
		r.becomeFollower(req.Term)
	}

	voted := r.votedFor != None

	if req.PreVote && (r.heardFromLeader || (voted && req.Term == r.currentTerm)) {
		// We don't grant pre-votes if we have recently heard from a
		// leader or already voted in the pre-term.
		return &pb.RequestVoteResponse{Term: r.currentTerm}
	}

	lastIndex := r.storage.NextIndex() - 1
	lastLogTerm := r.logTerm(lastIndex)

	// We can grant a vote in the same term, as long as it's to the same
	// candidate. This is useful if the response was lost, and the candidate
	// sends another request.
	alreadyVotedForCandidate := r.votedFor == req.CandidateID

	// If the logs have last entries with different terms, the log with the
	// later term is more up-to-date.
	laterTerm := req.LastLogTerm > lastLogTerm

	// If the logs end with the same term, whichever log is longer is more
	// up-to-date.
	longEnough := req.LastLogTerm == lastLogTerm && req.LastLogIndex >= lastIndex

	// We can only grant a vote if: we have not voted yet, we vote for the
	// same candidate again, or this is a pre-vote.
	canGrantVote := !voted || alreadyVotedForCandidate || req.PreVote

	// #RV2 If votedFor is null or candidateId, and candidate's log is at
	// least as up-to-date as receiver's log, grant vote.
	voteGranted = canGrantVote && (laterTerm || longEnough)

	if voteGranted {
		if req.PreVote {
			return &pb.RequestVoteResponse{VoteGranted: true, Term: req.Term}
		}

		r.votedFor = req.CandidateID
		r.storage.Set(raft.KeyVotedFor, req.CandidateID)

		// #F2 If election timeout elapses without receiving
		// AppendEntries RPC from current leader or granting a vote to
		// candidate: convert to candidate. Here we are granting a vote
		// to a candidate so we reset the election timeout.
		r.resetElection = true
		r.resetBaseline = true

		return &pb.RequestVoteResponse{VoteGranted: true, Term: r.currentTerm}
	}

	// #RV2 The candidate's log was not up-to-date
	return &pb.RequestVoteResponse{Term: r.currentTerm}
}

// HandleAppendEntriesRequest must be called when receiving a
// AppendEntriesRequest, the return value must be delivered to the requester.
func (r *Raft) HandleAppendEntriesRequest(req *pb.AppendEntriesRequest) *pb.AppendEntriesResponse {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.aereq)
		defer timer.ObserveDuration()
	}

	reqLogger := r.logger.WithFields(logrus.Fields{
		"currentterm":  r.currentTerm,
		"requestterm":  req.Term,
		"leaderid":     req.LeaderID,
		"prevlogindex": req.PrevLogIndex,
		"prevlogterm":  req.PrevLogTerm,
		"commitindex":  req.CommitIndex,
		"lenentries":   len(req.Entries),
	})
	reqLogger.Infoln("Got AppendEntries")

	logLen := r.storage.NextIndex() - 1

	res := &pb.AppendEntriesResponse{
		Term:       r.currentTerm,
		MatchIndex: logLen,
	}

	// #AE1 Reply false if term < currentTerm.
	if req.Term < r.currentTerm {
		return res
	}

	prevTerm := r.logTerm(req.PrevLogIndex)

	// An AppendEntries request is always successful for the first index. A
	// leader can only be elected leader if its log matches that of a
	// majority and our log is guaranteed to be at least 0 in length.
	firstIndex := req.PrevLogIndex == 0

	// The index preceding the entries we are going to replicate must be in our log.
	gotPrevIndex := req.PrevLogIndex <= logLen
	// The term must match to satisfy the log matching property.
	sameTerm := req.PrevLogTerm == prevTerm

	// If the previous entry is in our log, then our log matches the leaders
	// up till and including the previous entry. And we can safely replicate
	// next new entries.
	gotPrevEntry := gotPrevIndex && sameTerm

	success := firstIndex || gotPrevEntry

	// #A2 If RPC request or response contains term T > currentTerm: set
	// currentTerm = T, convert to follower. Transition to follower upon
	// receiving an AppendEntries call.
	if req.Term > r.currentTerm || r.state != Follower {
		r.becomeFollower(req.Term)
		res.Term = req.Term
	}

	if r.metricsEnabled {
		rmetrics.leader.Set(float64(req.LeaderID))
	}

	// We acknowledge this server as the leader as it's has the highest term
	// we have seen, and there can only be one leader per term.
	r.leader = req.LeaderID
	r.heardFromLeader = true
	r.seenLeader = true
	// TODO Revisit heartbeat mechanism.
	r.resetElection = true

	if !success {
		r.cureqout <- &catchUpReq{
			leaderID: req.LeaderID,
			// TODO term: req.Term, ?
			matchIndex: res.MatchIndex,
		}

		return res
	}

	var toSave []*commonpb.Entry
	index := req.PrevLogIndex

	for _, entry := range req.Entries {
		// Increment first so we start at previous index + 1.
		index++

		// If the terms don't match, our logs conflict at this index. On
		// the first conflict this will truncate the log to the lowest
		// common matching index. After that it will fill the log with
		// the new entries from the leader. This is because entry.Term
		// will always conflict with term 0, which will be returned for
		// indexes outside our log.
		if entry.Term != r.logTerm(index) {
			logLen = r.storage.NextIndex() - 1
			for logLen > index-1 {
				// If we are overwriting the latest
				// configuration, rollback to the committed one.
				if logLen == r.mem.getIndex() {
					r.mem.rollback()
				}
				r.storage.RemoveEntries(logLen, logLen)
				logLen = r.storage.NextIndex() - 1
			}
			toSave = append(toSave, entry)
		}
	}

	if len(toSave) > 0 {
		r.storage.StoreEntries(toSave)
	}
	logLen = r.storage.NextIndex() - 1

	for _, entry := range toSave {
		if entry.EntryType == commonpb.EntryReconf {
			var reconf commonpb.ReconfRequest
			err := reconf.Unmarshal(entry.Data)

			if err != nil {
				panic("could not unmarshal reconf")
			}

			r.mem.setPending(&reconf)
			r.mem.set(entry.Index)
		}
	}

	old := r.commitIndex
	// Commit index can not exceed the length of our log.
	r.commitIndex = min(req.CommitIndex, logLen)

	if r.metricsEnabled {
		rmetrics.commitIndex.Set(float64(r.commitIndex))
	}

	if r.commitIndex > old {
		r.logger.WithFields(logrus.Fields{
			"oldcommitindex": old,
			"commitindex":    r.commitIndex,
		}).Infoln("Set commit index")

		r.newCommit(old)
	}

	reqLogger.WithFields(logrus.Fields{
		"lensaved": len(toSave),
		"lenlog":   logLen,
		"success":  success,
	}).Infoln("Saved entries to stable storage")

	res.Success = true
	return res
}

func (r *Raft) HandleInstallSnapshotRequest(snapshot *commonpb.Snapshot) (res *pb.InstallSnapshotResponse) {
	r.mu.Lock()
	defer r.mu.Unlock()

	res = &pb.InstallSnapshotResponse{
		Term: r.currentTerm,
	}

	return
}

// HandleRequestVoteResponse must be invoked when receiving a
// RequestVoteResponse.
func (r *Raft) HandleRequestVoteResponse(response *pb.RequestVoteResponse) {
	r.mu.Lock()
	defer r.mu.Unlock()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.rvres)
		defer timer.ObserveDuration()
	}

	r.logger.WithFields(logrus.Fields{
		"currentterm":  r.currentTerm,
		"responseterm": response.Term,
		"votegranted":  response.VoteGranted,
	}).Infoln("Got vote response")

	term := r.currentTerm

	if r.preElection {
		term++
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	if response.Term > term {
		r.becomeFollower(response.Term)

		return
	}

	// Ignore late response
	if response.Term < term {
		return
	}

	// Cont. from startElection(). We have now received a response from Gorums.

	// #C5 If votes received from majority of server: become leader.
	// Make sure we have not stepped down while waiting for replies.
	if r.state == Candidate && response.VoteGranted {
		if r.preElection {
			r.preElection = false
			select {
			case r.startElectionNow <- struct{}{}:
			case <-r.stop:
			}
			return
		}

		// We have received at least a quorum of votes.
		// We are the leader for this term. See Raft Paper Figure 2 -> Rules for Servers -> Leaders.

		if r.metricsEnabled {
			rmetrics.leader.Set(float64(r.id))
		}

		r.logger.WithFields(logrus.Fields{
			"currentterm": r.currentTerm,
		}).Infoln("Elected leader")

		logLen := r.storage.NextIndex() - 1

		r.state = Leader
		r.leader = r.id
		r.seenLeader = true
		r.heardFromLeader = true
		r.nextIndex = logLen + 1
		r.pending = list.New()
		r.pendingReads = nil
		r.mem.setStable(false)

		// Empty queue.
	EMPTYCH:
		for {
			select {
			case <-r.queue:
				// TODO Respond with NotLeader.
			default:
				// Paper §8: We add a no-op, so that the leader
				// commits an entry from its own term. This
				// ensures that the leader knows which entries
				// are committed.
				promise, _ := raft.NewPromiseEntry(&commonpb.Entry{
					EntryType: commonpb.EntryInternal,
					Term:      r.currentTerm,
					Data:      raft.NOOP,
				})
				r.queue <- promise
				break EMPTYCH
			}
		}

		select {
		case r.heartbeatNow <- struct{}{}:
		default:
		}

		return
	}

	r.preElection = true

	// #C7 If election timeout elapses: start new election.
	// This will happened if we don't receive enough replies in time. Or we lose the election but don't see a higher term number.
}

// HandleAppendEntriesResponse must be invoked when receiving an
// AppendEntriesResponse.
func (r *Raft) HandleAppendEntriesResponse(response *pb.AppendEntriesQFResponse, maxIndex uint64) {
	r.mu.Lock()
	defer func() {
		r.mu.Unlock()
		r.advanceCommitIndex()
	}()
	if r.metricsEnabled {
		timer := metrics.NewTimer(rmetrics.aeres)
		defer timer.ObserveDuration()
	}

	// #A2 If RPC request or response contains term T > currentTerm: set currentTerm = T, convert to follower.
	// If we didn't get a response from a majority (excluding self) step down.
	if response.Term > r.currentTerm || response.Replies < uint64((len(r.mem.get().NodeIDs())+1)/2) {
		// Become follower.
		select {
		case r.toggle <- struct{}{}:
			r.logger.Warnln("Leader stepping down")
		case <-r.stop:
		}

		return
	}

	// Heartbeat to a majority.
	r.resetElection = true

	// Ignore late response
	if response.Term < r.currentTerm {
		return
	}

	if response.Success {
		r.matchIndex = maxIndex
		r.nextIndex = r.matchIndex + 1
		r.logger.WithFields(logrus.Fields{
			"matchindex": r.matchIndex,
			"nextindex":  r.nextIndex,
		}).Warnln("Setting matchindex")

		return
	}

	// If AppendEntries was not successful lower match index.
	r.nextIndex = max(1, min(r.nextIndex-r.burst, r.matchIndex+1))
}

func (r *Raft) HandleInstallSnapshotResponse(res *pb.InstallSnapshotResponse) bool {
	r.mu.Lock()
	defer r.mu.Unlock()

	if res.Term > r.currentTerm {
		r.becomeFollower(res.Term)

		return false
	}

	return true
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dashboard

// This file handles operations on the CL entity kind.

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"
	"appengine/user"
)

func init() {
	http.HandleFunc("/assign", handleAssign)
	http.HandleFunc("/update-cl", handleUpdateCL)
}

const codereviewBase = "http://codereview.appspot.com"
const gobotBase = "http://research.swtch.com/gobot_codereview"

var clRegexp = regexp.MustCompile(`\d+`)

// CL represents a code review.
type CL struct {
	Number string // e.g. "5903061"
	Closed bool
	Owner  string // email address

	Created, Modified time.Time

	Description []byte `datastore:",noindex"`
	FirstLine   string `datastore:",noindex"`
	LGTMs       []string
	NotLGTMs    []string

	// Mail information.
	Subject       string   `datastore:",noindex"`
	Recipients    []string `datastore:",noindex"`
	LastMessageID string   `datastore:",noindex"`

	// These are person IDs (e.g. "rsc"); they may be empty
	Author   string
	Reviewer string
}

// DisplayOwner returns the CL's owner, either as their email address
// or the person ID if it's a reviewer. It is for display only.
func (cl *CL) DisplayOwner() string {
	if p, ok := emailToPerson[cl.Owner]; ok {
		return p
	}
	return cl.Owner
}

func (cl *CL) FirstLineHTML() template.HTML {
	s := template.HTMLEscapeString(cl.FirstLine)
	// Embolden the package name.
	if i := strings.Index(s, ":"); i >= 0 {
		s = "" + s[:i] + "" + s[i:]
	}
	return template.HTML(s)
}

func formatEmails(e []string) template.HTML {
	x := make([]string, len(e))
	for i, s := range e {
		s = template.HTMLEscapeString(s)
		if !strings.Contains(s, "@") {
			s = "" + s + ""
		}
		s = `<span class="email">` + s + ""
		x[i] = s
	}
	return template.HTML(strings.Join(x, ", "))
}

func (cl *CL) LGTMHTML() template.HTML {
	return formatEmails(cl.LGTMs)
}

func (cl *CL) NotLGTMHTML() template.HTML {
	return formatEmails(cl.NotLGTMs)
}

func (cl *CL) ModifiedAgo() string {
	// Just the first non-zero unit.
	units := [...]struct {
		suffix string
		unit   time.Duration
	}{
		{"d", 24 * time.Hour},
		{"h", time.Hour},
		{"m", time.Minute},
		{"s", time.Second},
	}
	d := time.Now().Sub(cl.Modified)
	for _, u := range units {
		if d > u.unit {
			return fmt.Sprintf("%d%s", d/u.unit, u.suffix)
		}
	}
	return "just now"
}

func handleAssign(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	if r.Method != "POST" {
		http.Error(w, "Bad method "+r.Method, 400)
		return
	}

	u := user.Current(c)
	if _, ok := emailToPerson[u.Email]; !ok {
		http.Error(w, "Not allowed", http.StatusUnauthorized)
		return
	}

	n, rev := r.FormValue("cl"), r.FormValue("r")
	if !clRegexp.MatchString(n) {
		c.Errorf("Bad CL %q", n)
		http.Error(w, "Bad CL", 400)
		return
	}
	if _, ok := preferredEmail[rev]; !ok && rev != "" {
		c.Errorf("Unknown reviewer %q", rev)
		http.Error(w, "Unknown reviewer", 400)
		return
	}

	key := datastore.NewKey(c, "CL", n, 0, nil)

	if rev != "" {
		// Make sure the reviewer is listed in Rietveld as a reviewer.
		url := codereviewBase + "/" + n + "/fields"
		resp, err := urlfetch.Client(c).Get(url + "?field=reviewers")
		if err != nil {
			c.Errorf("Retrieving CL reviewer list failed: %v", err)
			http.Error(w, err.Error(), 500)
			return
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			c.Errorf("Retrieving CL reviewer list failed: got HTTP response %d", resp.StatusCode)
			http.Error(w, "Failed contacting Rietveld", 500)
			return
		}

		var apiResp struct {
			Reviewers []string `json:"reviewers"`
		}
		if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
			// probably can't be retried
			msg := fmt.Sprintf("Malformed JSON from %v: %v", url, err)
			c.Errorf("%s", msg)
			http.Error(w, msg, 500)
			return
		}
		found := false
		for _, r := range apiResp.Reviewers {
			if emailToPerson[r] == rev {
				found = true
				break
			}
		}
		if !found {
			c.Infof("Adding %v as a reviewer of CL %v", rev, n)

			url := fmt.Sprintf("%s?cl=%s&r=%s", gobotBase, n, rev)
			resp, err := urlfetch.Client(c).Get(url)
			if err != nil {
				c.Errorf("Gobot GET failed: %v", err)
				http.Error(w, err.Error(), 500)
				return
			}
			defer resp.Body.Close()
			if resp.StatusCode != 200 {
				c.Errorf("Gobot GET failed: got HTTP response %d", resp.StatusCode)
				http.Error(w, "Failed contacting Gobot", 500)
				return
			}

			c.Infof("Gobot said %q", resp.Status)
		}
	}

	// Update our own record.
	err := datastore.RunInTransaction(c, func(c appengine.Context) error {
		cl := new(CL)
		err := datastore.Get(c, key, cl)
		if err != nil {
			return err
		}
		cl.Reviewer = rev
		_, err = datastore.Put(c, key, cl)
		return err
	}, nil)
	if err != nil {
		msg := fmt.Sprintf("Assignment failed: %v", err)
		c.Errorf("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	c.Infof("Assigned CL %v to %v", n, rev)
}

func UpdateCLLater(c appengine.Context, n string, delay time.Duration) {
	t := taskqueue.NewPOSTTask("/update-cl", url.Values{
		"cl": []string{n},
	})
	t.Delay = delay
	if _, err := taskqueue.Add(c, t, "update-cl"); err != nil {
		c.Errorf("Failed adding task: %v", err)
	}
}

func handleUpdateCL(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	n := r.FormValue("cl")
	if !clRegexp.MatchString(n) {
		c.Errorf("Bad CL %q", n)
		http.Error(w, "Bad CL", 400)
		return
	}

	if err := updateCL(c, n); err != nil {
		c.Errorf("Failed updating CL %v: %v", n, err)
		http.Error(w, "Failed update", 500)
		return
	}

	io.WriteString(w, "OK")
}

// updateCL updates a single CL. If a retryable failure occurs, an error is returned.
func updateCL(c appengine.Context, n string) error {
	c.Debugf("Updating CL %v", n)
	key := datastore.NewKey(c, "CL", n, 0, nil)

	url := codereviewBase + "/api/" + n + "?messages=true"
	resp, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	raw, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Failed reading HTTP body: %v", err)
	}

	// Special case for abandoned CLs.
	if resp.StatusCode == 404 && bytes.Contains(raw, []byte("No issue exists with that id")) {
		// Don't bother checking for errors. The CL might never have been saved, for instance.
		datastore.Delete(c, key)
		c.Infof("Deleted abandoned CL %v", n)
		return nil
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("Update: got HTTP response %d", resp.StatusCode)
	}

	var apiResp struct {
		Description string   `json:"description"`
		Reviewers   []string `json:"reviewers"`
		Created     string   `json:"created"`
		OwnerEmail  string   `json:"owner_email"`
		Modified    string   `json:"modified"`
		Closed      bool     `json:"closed"`
		Subject     string   `json:"subject"`
		Messages    []struct {
			Text       string   `json:"text"`
			Sender     string   `json:"sender"`
			Recipients []string `json:"recipients"`
			Approval   bool     `json:"approval"`
		} `json:"messages"`
	}
	if err := json.Unmarshal(raw, &apiResp); err != nil {
		// probably can't be retried
		c.Errorf("Malformed JSON from %v: %v", url, err)
		return nil
	}
	//c.Infof("RAW: %+v", apiResp)

	cl := &CL{
		Number:      n,
		Closed:      apiResp.Closed,
		Owner:       apiResp.OwnerEmail,
		Description: []byte(apiResp.Description),
		FirstLine:   apiResp.Description,
		Subject:     apiResp.Subject,
		Author:      emailToPerson[apiResp.OwnerEmail],
	}
	cl.Created, err = time.Parse("2006-01-02 15:04:05.000000", apiResp.Created)
	if err != nil {
		c.Errorf("Bad creation time %q: %v", apiResp.Created, err)
	}
	cl.Modified, err = time.Parse("2006-01-02 15:04:05.000000", apiResp.Modified)
	if err != nil {
		c.Errorf("Bad modification time %q: %v", apiResp.Modified, err)
	}
	if i := strings.Index(cl.FirstLine, "\n"); i >= 0 {
		cl.FirstLine = cl.FirstLine[:i]
	}
	// Treat zero reviewers as a signal that the CL is completed.
	// This could be after the CL has been submitted, but before the CL author has synced,
	// but it could also be a CL manually edited to remove reviewers.
	if len(apiResp.Reviewers) == 0 {
		cl.Closed = true
	}

	lgtm := make(map[string]bool)
	notLGTM := make(map[string]bool)
	rcpt := make(map[string]bool)
	for _, msg := range apiResp.Messages {
		s, rev := msg.Sender, false
		if p, ok := emailToPerson[s]; ok {
			s, rev = p, true
		}

		// CLs submitted by someone other than the CL owner do not immediately
		// transition to "closed". Let's simulate the intention by treating
		// messages starting with "*** Submitted as " from a reviewer as a
		// signal that the CL is now closed.
		if rev && strings.HasPrefix(msg.Text, "*** Submitted as ") {
			cl.Closed = true
		}

		if msg.Approval {
			lgtm[s] = true
			delete(notLGTM, s) // "LGTM" overrules previous "NOT LGTM"
		}
		if strings.Contains(msg.Text, "NOT LGTM") {
			notLGTM[s] = true
			delete(lgtm, s) // "NOT LGTM" overrules previous "LGTM"
		}

		for _, r := range msg.Recipients {
			rcpt[r] = true
		}
	}
	for l := range lgtm {
		cl.LGTMs = append(cl.LGTMs, l)
	}
	for l := range notLGTM {
		cl.NotLGTMs = append(cl.NotLGTMs, l)
	}
	for r := range rcpt {
		cl.Recipients = append(cl.Recipients, r)
	}
	sort.Strings(cl.LGTMs)
	sort.Strings(cl.NotLGTMs)
	sort.Strings(cl.Recipients)

	err = datastore.RunInTransaction(c, func(c appengine.Context) error {
		ocl := new(CL)
		err := datastore.Get(c, key, ocl)
		if err != nil && err != datastore.ErrNoSuchEntity {
			return err
		} else if err == nil {
			// LastMessageID and Reviewer need preserving.
			cl.LastMessageID = ocl.LastMessageID
			cl.Reviewer = ocl.Reviewer
		}
		_, err = datastore.Put(c, key, cl)
		return err
	}, nil)
	if err != nil {
		return err
	}
	c.Infof("Updated CL %v", n)
	return nil
}
<commit_msg>misc/dashboard/codereview: pass user information to gobot when assigning reviewer.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dashboard

// This file handles operations on the CL entity kind.

import (
	"bytes"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"
	"appengine/user"
)

func init() {
	http.HandleFunc("/assign", handleAssign)
	http.HandleFunc("/update-cl", handleUpdateCL)
}

const codereviewBase = "http://codereview.appspot.com"
const gobotBase = "http://research.swtch.com/gobot_codereview"

var clRegexp = regexp.MustCompile(`\d+`)

// CL represents a code review.
type CL struct {
	Number string // e.g. "5903061"
	Closed bool
	Owner  string // email address

	Created, Modified time.Time

	Description []byte `datastore:",noindex"`
	FirstLine   string `datastore:",noindex"`
	LGTMs       []string
	NotLGTMs    []string

	// Mail information.
	Subject       string   `datastore:",noindex"`
	Recipients    []string `datastore:",noindex"`
	LastMessageID string   `datastore:",noindex"`

	// These are person IDs (e.g. "rsc"); they may be empty
	Author   string
	Reviewer string
}

// DisplayOwner returns the CL's owner, either as their email address
// or the person ID if it's a reviewer. It is for display only.
func (cl *CL) DisplayOwner() string {
	if p, ok := emailToPerson[cl.Owner]; ok {
		return p
	}
	return cl.Owner
}

func (cl *CL) FirstLineHTML() template.HTML {
	s := template.HTMLEscapeString(cl.FirstLine)
	// Embolden the package name.
	if i := strings.Index(s, ":"); i >= 0 {
		s = "" + s[:i] + "" + s[i:]
	}
	return template.HTML(s)
}

func formatEmails(e []string) template.HTML {
	x := make([]string, len(e))
	for i, s := range e {
		s = template.HTMLEscapeString(s)
		if !strings.Contains(s, "@") {
			s = "" + s + ""
		}
		s = `<span class="email">` + s + ""
		x[i] = s
	}
	return template.HTML(strings.Join(x, ", "))
}

func (cl *CL) LGTMHTML() template.HTML {
	return formatEmails(cl.LGTMs)
}

func (cl *CL) NotLGTMHTML() template.HTML {
	return formatEmails(cl.NotLGTMs)
}

func (cl *CL) ModifiedAgo() string {
	// Just the first non-zero unit.
	units := [...]struct {
		suffix string
		unit   time.Duration
	}{
		{"d", 24 * time.Hour},
		{"h", time.Hour},
		{"m", time.Minute},
		{"s", time.Second},
	}
	d := time.Now().Sub(cl.Modified)
	for _, u := range units {
		if d > u.unit {
			return fmt.Sprintf("%d%s", d/u.unit, u.suffix)
		}
	}
	return "just now"
}

func handleAssign(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	if r.Method != "POST" {
		http.Error(w, "Bad method "+r.Method, 400)
		return
	}

	u := user.Current(c)
	person, ok := emailToPerson[u.Email]
	if !ok {
		http.Error(w, "Not allowed", http.StatusUnauthorized)
		return
	}

	n, rev := r.FormValue("cl"), r.FormValue("r")
	if !clRegexp.MatchString(n) {
		c.Errorf("Bad CL %q", n)
		http.Error(w, "Bad CL", 400)
		return
	}
	if _, ok := preferredEmail[rev]; !ok && rev != "" {
		c.Errorf("Unknown reviewer %q", rev)
		http.Error(w, "Unknown reviewer", 400)
		return
	}

	key := datastore.NewKey(c, "CL", n, 0, nil)

	if rev != "" {
		// Make sure the reviewer is listed in Rietveld as a reviewer.
		url := codereviewBase + "/" + n + "/fields"
		resp, err := urlfetch.Client(c).Get(url + "?field=reviewers")
		if err != nil {
			c.Errorf("Retrieving CL reviewer list failed: %v", err)
			http.Error(w, err.Error(), 500)
			return
		}
		defer resp.Body.Close()
		if resp.StatusCode != 200 {
			c.Errorf("Retrieving CL reviewer list failed: got HTTP response %d", resp.StatusCode)
			http.Error(w, "Failed contacting Rietveld", 500)
			return
		}

		var apiResp struct {
			Reviewers []string `json:"reviewers"`
		}
		if err := json.NewDecoder(resp.Body).Decode(&apiResp); err != nil {
			// probably can't be retried
			msg := fmt.Sprintf("Malformed JSON from %v: %v", url, err)
			c.Errorf("%s", msg)
			http.Error(w, msg, 500)
			return
		}
		found := false
		for _, r := range apiResp.Reviewers {
			if emailToPerson[r] == rev {
				found = true
				break
			}
		}
		if !found {
			c.Infof("Adding %v as a reviewer of CL %v", rev, n)

			url := fmt.Sprintf("%s?cl=%s&r=%s&obo=%s", gobotBase, n, rev, person)
			resp, err := urlfetch.Client(c).Get(url)
			if err != nil {
				c.Errorf("Gobot GET failed: %v", err)
				http.Error(w, err.Error(), 500)
				return
			}
			defer resp.Body.Close()
			if resp.StatusCode != 200 {
				c.Errorf("Gobot GET failed: got HTTP response %d", resp.StatusCode)
				http.Error(w, "Failed contacting Gobot", 500)
				return
			}

			c.Infof("Gobot said %q", resp.Status)
		}
	}

	// Update our own record.
	err := datastore.RunInTransaction(c, func(c appengine.Context) error {
		cl := new(CL)
		err := datastore.Get(c, key, cl)
		if err != nil {
			return err
		}
		cl.Reviewer = rev
		_, err = datastore.Put(c, key, cl)
		return err
	}, nil)
	if err != nil {
		msg := fmt.Sprintf("Assignment failed: %v", err)
		c.Errorf("%s", msg)
		http.Error(w, msg, 500)
		return
	}
	c.Infof("Assigned CL %v to %v", n, rev)
}

func UpdateCLLater(c appengine.Context, n string, delay time.Duration) {
	t := taskqueue.NewPOSTTask("/update-cl", url.Values{
		"cl": []string{n},
	})
	t.Delay = delay
	if _, err := taskqueue.Add(c, t, "update-cl"); err != nil {
		c.Errorf("Failed adding task: %v", err)
	}
}

func handleUpdateCL(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	n := r.FormValue("cl")
	if !clRegexp.MatchString(n) {
		c.Errorf("Bad CL %q", n)
		http.Error(w, "Bad CL", 400)
		return
	}

	if err := updateCL(c, n); err != nil {
		c.Errorf("Failed updating CL %v: %v", n, err)
		http.Error(w, "Failed update", 500)
		return
	}

	io.WriteString(w, "OK")
}

// updateCL updates a single CL. If a retryable failure occurs, an error is returned.
func updateCL(c appengine.Context, n string) error {
	c.Debugf("Updating CL %v", n)
	key := datastore.NewKey(c, "CL", n, 0, nil)

	url := codereviewBase + "/api/" + n + "?messages=true"
	resp, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	raw, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("Failed reading HTTP body: %v", err)
	}

	// Special case for abandoned CLs.
	if resp.StatusCode == 404 && bytes.Contains(raw, []byte("No issue exists with that id")) {
		// Don't bother checking for errors. The CL might never have been saved, for instance.
		datastore.Delete(c, key)
		c.Infof("Deleted abandoned CL %v", n)
		return nil
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("Update: got HTTP response %d", resp.StatusCode)
	}

	var apiResp struct {
		Description string   `json:"description"`
		Reviewers   []string `json:"reviewers"`
		Created     string   `json:"created"`
		OwnerEmail  string   `json:"owner_email"`
		Modified    string   `json:"modified"`
		Closed      bool     `json:"closed"`
		Subject     string   `json:"subject"`
		Messages    []struct {
			Text       string   `json:"text"`
			Sender     string   `json:"sender"`
			Recipients []string `json:"recipients"`
			Approval   bool     `json:"approval"`
		} `json:"messages"`
	}
	if err := json.Unmarshal(raw, &apiResp); err != nil {
		// probably can't be retried
		c.Errorf("Malformed JSON from %v: %v", url, err)
		return nil
	}
	//c.Infof("RAW: %+v", apiResp)

	cl := &CL{
		Number:      n,
		Closed:      apiResp.Closed,
		Owner:       apiResp.OwnerEmail,
		Description: []byte(apiResp.Description),
		FirstLine:   apiResp.Description,
		Subject:     apiResp.Subject,
		Author:      emailToPerson[apiResp.OwnerEmail],
	}
	cl.Created, err = time.Parse("2006-01-02 15:04:05.000000", apiResp.Created)
	if err != nil {
		c.Errorf("Bad creation time %q: %v", apiResp.Created, err)
	}
	cl.Modified, err = time.Parse("2006-01-02 15:04:05.000000", apiResp.Modified)
	if err != nil {
		c.Errorf("Bad modification time %q: %v", apiResp.Modified, err)
	}
	if i := strings.Index(cl.FirstLine, "\n"); i >= 0 {
		cl.FirstLine = cl.FirstLine[:i]
	}
	// Treat zero reviewers as a signal that the CL is completed.
	// This could be after the CL has been submitted, but before the CL author has synced,
	// but it could also be a CL manually edited to remove reviewers.
	if len(apiResp.Reviewers) == 0 {
		cl.Closed = true
	}

	lgtm := make(map[string]bool)
	notLGTM := make(map[string]bool)
	rcpt := make(map[string]bool)
	for _, msg := range apiResp.Messages {
		s, rev := msg.Sender, false
		if p, ok := emailToPerson[s]; ok {
			s, rev = p, true
		}

		// CLs submitted by someone other than the CL owner do not immediately
		// transition to "closed". Let's simulate the intention by treating
		// messages starting with "*** Submitted as " from a reviewer as a
		// signal that the CL is now closed.
		if rev && strings.HasPrefix(msg.Text, "*** Submitted as ") {
			cl.Closed = true
		}

		if msg.Approval {
			lgtm[s] = true
			delete(notLGTM, s) // "LGTM" overrules previous "NOT LGTM"
		}
		if strings.Contains(msg.Text, "NOT LGTM") {
			notLGTM[s] = true
			delete(lgtm, s) // "NOT LGTM" overrules previous "LGTM"
		}

		for _, r := range msg.Recipients {
			rcpt[r] = true
		}
	}
	for l := range lgtm {
		cl.LGTMs = append(cl.LGTMs, l)
	}
	for l := range notLGTM {
		cl.NotLGTMs = append(cl.NotLGTMs, l)
	}
	for r := range rcpt {
		cl.Recipients = append(cl.Recipients, r)
	}
	sort.Strings(cl.LGTMs)
	sort.Strings(cl.NotLGTMs)
	sort.Strings(cl.Recipients)

	err = datastore.RunInTransaction(c, func(c appengine.Context) error {
		ocl := new(CL)
		err := datastore.Get(c, key, ocl)
		if err != nil && err != datastore.ErrNoSuchEntity {
			return err
		} else if err == nil {
			// LastMessageID and Reviewer need preserving.
			cl.LastMessageID = ocl.LastMessageID
			cl.Reviewer = ocl.Reviewer
		}
		_, err = datastore.Put(c, key, cl)
		return err
	}, nil)
	if err != nil {
		return err
	}
	c.Infof("Updated CL %v", n)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/user"
)

type RootData struct {
	Posts   *[]Entry
	IsAdmin bool
	Page    int64
	Prev    int64
	Next    int64
}

const perPage = 30

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	if r.Request.URL.Path == "/page/0" {
		http.Redirect(w, r.Request, "/", 301)
	}

	c := appengine.NewContext(r.Request)
	pg, err := strconv.ParseInt(r.Param("page"), 10, 64)
	if err != nil {
		log.Infof(c, "Error parsing: %+v", err)
		pg = 0
	}

	entries, err := Pagination(c, perPage, int(pg*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{
		Posts:   entries,
		IsAdmin: user.IsAdmin(c),
		Page:    pg,
		Next:    pg + 1,
		Prev:    pg - 1,
	}

	// If there are no posts left, don't show next button.
	if len(*entries) == 0 {
		data.Next = -1
	}

	// Get next page's posts so we don't show the next page if there is none.
	nxt_entr, err := Pagination(c, perPage, int((pg+1)*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if len(*nxt_entr) == 0 {
		data.Next = -1
	}

	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("about", nil)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func RedirectHomeHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "/", 302)
}

func CleanWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	q := datastore.NewQuery("Entry")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for _, p := range *entries {
		// TODO: Figure out how to unset all public.
		if &p.Draft == nil {
			p.Draft = false
		}
		if len(p.Title) == 0 {
			p.Title = fmt.Sprintf("Untitled #%d", p.Id)
		}
		p.Save(c)
	}
}

type SiteMapData struct {
	Posts  *[]Entry
	Newest time.Time
}

// http://www.sitemaps.org/protocol.html
func SitemapHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &SiteMapData{
		Posts:  entries,
		Newest: (*entries)[0].Modified,
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	w.Render("sitemap", data)
}

func queueWork(c context.Context, uri string) error {
	r := &taskqueue.RetryOptions{
		RetryLimit: 1,
	}

	t := taskqueue.NewPOSTTask(uri, url.Values{})
	t.RetryOptions = r
	_, err := taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing %s: %v", err.Error())
		return err
	}

	return nil
}

// This queues lots of work every fifteen minutes.
func WorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	err := queueWork(c, "/archive/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Download all the links.
	err = queueWork(c, "/link/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	//// Update the stats
	//err = queueWork(c, "/stats/work")
	//if err != nil {
	//	http.Error(w, err.Error(), 500)
	//	return
	//}

	// Update the longform data.
	err = queueWork(c, "/longform/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Clean the database
	err = queueWork(c, "/clean/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the Search Index
	err = queueWork(c, "/search/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	fmt.Fprint(w, "success.\n")
}

// This queues lots of work every twelve hours.
func LongWorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	err := queueWork(c, "/link/long-work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	fmt.Fprint(w, "success.\n")
}
<commit_msg>enable stats<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/user"
)

type RootData struct {
	Posts   *[]Entry
	IsAdmin bool
	Page    int64
	Prev    int64
	Next    int64
}

const perPage = 30

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	if r.Request.URL.Path == "/page/0" {
		http.Redirect(w, r.Request, "/", 301)
	}

	c := appengine.NewContext(r.Request)
	pg, err := strconv.ParseInt(r.Param("page"), 10, 64)
	if err != nil {
		log.Infof(c, "Error parsing: %+v", err)
		pg = 0
	}

	entries, err := Pagination(c, perPage, int(pg*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{
		Posts:   entries,
		IsAdmin: user.IsAdmin(c),
		Page:    pg,
		Next:    pg + 1,
		Prev:    pg - 1,
	}

	// If there are no posts left, don't show next button.
	if len(*entries) == 0 {
		data.Next = -1
	}

	// Get next page's posts so we don't show the next page if there is none.
	nxt_entr, err := Pagination(c, perPage, int((pg+1)*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if len(*nxt_entr) == 0 {
		data.Next = -1
	}

	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("about", nil)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func RedirectHomeHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "/", 302)
}

func CleanWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	q := datastore.NewQuery("Entry")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for _, p := range *entries {
		// TODO: Figure out how to unset all public.
		if &p.Draft == nil {
			p.Draft = false
		}
		if len(p.Title) == 0 {
			p.Title = fmt.Sprintf("Untitled #%d", p.Id)
		}
		p.Save(c)
	}
}

type SiteMapData struct {
	Posts  *[]Entry
	Newest time.Time
}

// http://www.sitemaps.org/protocol.html
func SitemapHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &SiteMapData{
		Posts:  entries,
		Newest: (*entries)[0].Modified,
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	w.Render("sitemap", data)
}

func queueWork(c context.Context, uri string) error {
	r := &taskqueue.RetryOptions{
		RetryLimit: 1,
	}

	t := taskqueue.NewPOSTTask(uri, url.Values{})
	t.RetryOptions = r
	_, err := taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing %s: %v", err.Error())
		return err
	}

	return nil
}

// This queues lots of work every fifteen minutes.
func WorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	err := queueWork(c, "/archive/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Download all the links.
	err = queueWork(c, "/link/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the stats
	err = queueWork(c, "/stats/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the longform data.
	err = queueWork(c, "/longform/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Clean the database
	err = queueWork(c, "/clean/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the Search Index
	err = queueWork(c, "/search/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	fmt.Fprint(w, "success.\n")
}

// This queues lots of work every twelve hours.
func LongWorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	err := queueWork(c, "/link/long-work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	fmt.Fprint(w, "success.\n")
}
<|endoftext|>


<commit_before>package ros

import (
	"fmt"
	"net/http"

	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/util"
)

//https://help.aliyun.com/document_detail/28916.html?spm=5176.doc49066.6.588.d7Ntjs
type Resource struct {
	Id           string
	Name         string
	Type         string
	Status       string
	StatusReason string
	Updated      string
	PhysicalId   string
}

func (client *Client) DescribeResources(stackId, stackName string) ([]*Resource, error) {
	response := make([]*Resource, 0)
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/stacks/%s/%s/resources", stackName, stackId), nil, nil, &response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28917.html?spm=5176.doc28916.6.589.BUPJqx
func (client *Client) DescribeResource(stackId, stackName, resourceName string) (*Resource, error) {
	response := &Resource{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/stacks/%s/%s/resources/%s", stackName, stackId, resourceName), nil, nil, &response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28918.html?spm=5176.doc28917.6.590.smknll
type SupportStatus string

const (
	SUPPORT_STATUS_UNKNOWN     = "UNKNOWN"
	SUPPORT_STATUS_SUPPORTED   = "SUPPORTED"
	SUPPORT_STATUS_DEPRECATED  = "DEPRECATED"
	SUPPORT_STATUS_UNSUPPORTED = "UNSUPPORTED"
	SUPPORT_STATUS_HIDDEN      = "HIDDEN"
)

type DescribeResoureTypesRequest struct {
	SupportStatus SupportStatus
}

type DescribeResoureTypesResponse struct {
	common.Response
	ResourceTypes []string
}

func (client *Client) DescribeResoureTypes(supportStatus SupportStatus) (*DescribeResoureTypesResponse, error) {
	query := util.ConvertToQueryValues(&DescribeResoureTypesRequest{
		SupportStatus: supportStatus,
	})

	response := &DescribeResoureTypesResponse{}
	err := client.Invoke("", http.MethodGet, "/resource_types", query, nil, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28919.html?spm=5176.doc28918.6.591.7QkDYC
type DescribeResoureTypeResponse struct {
	common.Response
	ResourceType  string
	Attributes    interface{}
	SupportStatus interface{}
	Properties    interface{}
}

func (client *Client) DescribeResoureType(typeName string) (*DescribeResoureTypeResponse, error) {
	response := &DescribeResoureTypeResponse{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/resource_types/%s", typeName), nil, nil, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28920.html?spm=5176.doc28919.6.592.IiEwar
type DescribeResoureTypeTemplateResponse struct {
	common.Response
	ROSTemplateFormatVersion string
	Parameters               interface{}
	Outputs                  interface{}
	Resources                interface{}
}

func (client *Client) DescribeResoureTypeTemplate(typeName string) (*DescribeResoureTypeTemplateResponse, error) {
	response := &DescribeResoureTypeTemplateResponse{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/resource_types/%s/template", typeName), nil, nil, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}
<commit_msg>ros DescribeResource support region id<commit_after>package ros

import (
	"fmt"
	"net/http"

	"github.com/denverdino/aliyungo/common"
	"github.com/denverdino/aliyungo/util"
)

//https://help.aliyun.com/document_detail/28916.html?spm=5176.doc49066.6.588.d7Ntjs
type Resource struct {
	Id           string
	Name         string
	Type         string
	Status       string
	StatusReason string
	Updated      string
	PhysicalId   string
}

func (client *Client) DescribeResources(stackId, stackName string) ([]*Resource, error) {
	response := make([]*Resource, 0)
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/stacks/%s/%s/resources", stackName, stackId), nil, nil, &response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28917.html?spm=5176.doc28916.6.589.BUPJqx
func (client *Client) DescribeResource(stackId, stackName, resourceName string) (*Resource, error) {
	response := &Resource{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/stacks/%s/%s/resources/%s", stackName, stackId, resourceName), nil, nil, &response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28917.html?spm=5176.doc28916.6.589.BUPJqx
func (client *Client) DescribeResourceByRegion(regionId common.Region, stackId, stackName, resourceName string) (*Resource, error) {
	response := &Resource{}
	err := client.Invoke(regionId, http.MethodGet, fmt.Sprintf("/stacks/%s/%s/resources/%s", stackName, stackId, resourceName), nil, nil, &response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28918.html?spm=5176.doc28917.6.590.smknll
type SupportStatus string

const (
	SUPPORT_STATUS_UNKNOWN     = "UNKNOWN"
	SUPPORT_STATUS_SUPPORTED   = "SUPPORTED"
	SUPPORT_STATUS_DEPRECATED  = "DEPRECATED"
	SUPPORT_STATUS_UNSUPPORTED = "UNSUPPORTED"
	SUPPORT_STATUS_HIDDEN      = "HIDDEN"
)

type DescribeResoureTypesRequest struct {
	SupportStatus SupportStatus
}

type DescribeResoureTypesResponse struct {
	common.Response
	ResourceTypes []string
}

func (client *Client) DescribeResoureTypes(supportStatus SupportStatus) (*DescribeResoureTypesResponse, error) {
	query := util.ConvertToQueryValues(&DescribeResoureTypesRequest{
		SupportStatus: supportStatus,
	})

	response := &DescribeResoureTypesResponse{}
	err := client.Invoke("", http.MethodGet, "/resource_types", query, nil, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28919.html?spm=5176.doc28918.6.591.7QkDYC
type DescribeResoureTypeResponse struct {
	common.Response
	ResourceType  string
	Attributes    interface{}
	SupportStatus interface{}
	Properties    interface{}
}

func (client *Client) DescribeResoureType(typeName string) (*DescribeResoureTypeResponse, error) {
	response := &DescribeResoureTypeResponse{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/resource_types/%s", typeName), nil, nil, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}

//https://help.aliyun.com/document_detail/28920.html?spm=5176.doc28919.6.592.IiEwar
type DescribeResoureTypeTemplateResponse struct {
	common.Response
	ROSTemplateFormatVersion string
	Parameters               interface{}
	Outputs                  interface{}
	Resources                interface{}
}

func (client *Client) DescribeResoureTypeTemplate(typeName string) (*DescribeResoureTypeTemplateResponse, error) {
	response := &DescribeResoureTypeTemplateResponse{}
	err := client.Invoke("", http.MethodGet, fmt.Sprintf("/resource_types/%s/template", typeName), nil, nil, response)
	if err != nil {
		return nil, err
	}

	return response, nil
}
<|endoftext|>