<commit_before>package main

import (
	"bytes"
	"database/sql"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const dbFilename = "shiritoriwords.db"
const jmdictFileName = "jmdict.xml"
const kanjidictFileName = "kanjidic2.xml"
const maxPts = 15  // Max grade of 10 + Max JLPT of 5
const maxLimit = 4 // Arbitrary limit
const ptsFactor = maxPts / maxLimit
const maxJLPT = 6

type kmap map[string]int

var db *sql.DB
var insertcount = 0
var failcount = 0
var kanjiExp = regexp.MustCompile(`\p{Han}+`)
var endsInNExp = regexp.MustCompile(`(ん|ン)$`)

func main() {
	log.Printf("Loading kanji dictionary...")
	dict, err := getKanjiDict()
	if err != nil {
		log.Printf("error: %v", err)
		return
	}
	log.Printf("Loading kanji points map...")
	kptsmap, err := getKanjiPtsMap()
	if err != nil {
		log.Printf("error: %v", err)
		return
	}
	log.Printf("Creating kanji database...")
	err = createKanjiDb(dict, kptsmap)
	if err != nil {
		log.Printf("error: %v", err)
		return
	}
}

func createKanjiDb(dict *jmdict, kptsmap kmap) error {
	var err error
	db, err = sql.Open("sqlite3", dbFilename)
	if err != nil {
		return err
	}
	dropTable("words")
	err = createTable("words (kanji TEXT PRIMARY KEY, kana TEXT, points INT)")
	if err != nil {
		return err
	}
	// Prepare the statement and use a transaction for massive speed increase.
	stmt, err := db.Prepare("INSERT INTO words (kanji, kana, points) VALUES (:KJ, :KN, :SC)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	// Optimize the database insertion
	execDb("PRAGMA synchronous = OFF")
	execDb("PRAGMA journal_mode = MEMORY")
	execDb("BEGIN")
	defer execDb("COMMIT")
	for _, e := range dict.Entry {
		if isNoun(e) {
			saveWord(stmt, e, kptsmap)
		}
	}
	log.Printf("Inserted %d record(s)", insertcount)
	log.Printf("Merged %d record(s)", failcount)
	return nil
}

func isNoun(e entry) bool {
	for _, s := range e.Sense {
		for _, p := range s.Pos {
			if p == "n" {
				return true
			}
		}
	}
	return false
}

func saveWord(stmt *sql.Stmt, e entry, kptsmap kmap) {
	var pts int
	kana, endsInN := getKana(e)
	// Get all of the entry kanji variants
	for _, kanji := range getKanjis(e) {
		if endsInN {
			// Automatic zero for ending in 'ん'
			pts = 0
		} else {
			pts = getKanjiWordPts(kanji, kptsmap)
		}
		_, err := stmt.Exec(sql.Named("KJ", &kanji), sql.Named("KN", &kana), sql.Named("SC", &pts))
		if err != nil {
			mergeRecords(kanji, kana, pts)
			failcount++
			// return err
		} else {
			insertcount++
		}
	}
}

func mergeRecords(kanji string, kana string, pts int) {
	var existingKana string
	var existingPts int
	found := queryDb(fmt.Sprintf("SELECT kana, points FROM words WHERE kanji = '%s'", kanji),
		&existingKana, &existingPts)
	if !found {
		log.Printf("DBERROR: Could not find record for %s", kanji)
		return
	}
	// Merge kana and the word pts.
	newKana := mergeStrings(existingKana, kana)
	newPts := mergePts(existingPts, pts)
	err := execDb("UPDATE words SET kana = ?, points = ? WHERE kanji = ?", &newKana, &newPts, &kanji)
	if err != nil {
		log.Printf("DBERROR updating %s", kanji)
	}
}

func mergeStrings(first string, second string) string {
	mergedmap := make(map[string]bool)
	addStrings(mergedmap, first)
	addStrings(mergedmap, second)
	mergedString := ""
	for str := range mergedmap {
		if len(mergedString) > 0 {
			mergedString += ","
		}
		mergedString += str
	}
	return mergedString
}

func addStrings(dest map[string]bool, strs string) {
	for _, str := range strings.Split(strs, ",") {
		dest[str] = true
	}
}

func mergePts(first int, second int) int {
	if first > second {
		return first
	}
	return second
}

func getKanjiWordPts(kanji string, kptsmap kmap) int {
	// Words entirely of hiragana or katakana are worth 1 point.
	pts := 1
	if kanjiExp.MatchString(kanji) {
		for _, k := range kanji {
			kpts := kptsmap[string(k)]
			if kpts > pts {
				// The word pts is equal to the highest kanji pts in the word.
				pts = kpts
			}
		}
	}
	return pts
}

/*
The kanji element, or in its absence, the reading element, is
the defining component of each entry.
The overwhelming majority of entries will have a single kanji
element associated with a word in Japanese. Where there are
multiple kanji elements within an entry, they will be orthographical
variants of the same word, either using variations in okurigana, or
alternative and equivalent kanji. Common "mis-spellings" may be
included, provided they are associated with appropriate information
fields. Synonyms are not included; they may be indicated in the
cross-reference field associated with the sense element.
*/
func getKanjis(e entry) []string {
	kanjis := make([]string, 0)
	for _, k := range e.Kele {
		if len(k.Keb) > 0 {
			kanjis = append(kanjis, k.Keb)
		}
	}
	return kanjis
}

func getKana(e entry) (string, bool) {
	kanas := ""
	endsInN := true
	for _, k := range e.Rele {
		if !endsInNExp.MatchString(k.Reb) {
			// If at least one variation does not end in 'ん', then it's valid.
			endsInN = false
		}
		if len(kanas) > 0 {
			kanas += ","
		}
		kanas += k.Reb
	}
	return kanas, endsInN
}

func getKanjiDict() (*jmdict, error) {
	data, err := loadXMLFile(jmdictFileName)
	if err != nil {
		return nil, err
	}
	d := xml.NewDecoder(bytes.NewReader(data))
	// Map the entities to standard XML, or else a parsing error occurs.
	d.Entity = jmentitymap
	dict := jmdict{}
	err = d.Decode(&dict)
	if err != nil {
		return nil, err
	}
	return &dict, nil
}

func getKanjiPtsMap() (kmap, error) {
	data, err := loadXMLFile(kanjidictFileName)
	if err != nil {
		log.Printf("error: %v", err)
		return nil, err
	}
	kanji := kanjidic{}
	err = xml.Unmarshal(data, &kanji)
	if err != nil {
		log.Printf("error: %v", err)
		return nil, err
	}
	kptsmap := make(kmap)
	for _, ch := range kanji.Character {
		kptsmap[ch.Literal] = getCharacterPts(ch)
	}
	return kptsmap, nil
}

func getCharacterPts(ch character) int {
	pts := ch.Misc.Grade
	if ch.Misc.JLPT > 0 {
		// JLPT is in reverse order. Higher level is lower number.
		pts += maxJLPT - ch.Misc.JLPT
	}
	// Kanji Pts: RawPts / PtsFactor
	return pts / ptsFactor
}

func loadXMLFile(filename string) ([]byte, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func dropTable(tableDef string) {
	execDb(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableDef))
}

func createTable(tableDef string) error {
	return execDb(fmt.Sprintf("CREATE TABLE %s", tableDef))
}

func createIndex(indexDef string) error {
	return execDb(fmt.Sprintf("CREATE INDEX %s", indexDef))
}

func execDb(stmt string, args ...interface{}) error {
	statement, err := db.Prepare(stmt)
	if err != nil {
		return err
	}
	_, err = statement.Exec(args...)
	return err
}

func queryDb(stmt string, args ...interface{}) bool {
	rows, err := db.Query(stmt)
	defer closeRows(rows)
	if err != nil {
		log.Printf("DBERROR: Querying %s: %v", stmt, err)
		return false
	}
	if rows.Next() {
		if args != nil {
			rows.Scan(args...)
		}
		return true
	}
	return false
}

func closeRows(rows *sql.Rows) {
	if nil != rows {
		rows.Close()
	}
}
<commit_msg>Fix bug that was not committing words to database.<commit_after>package main

import (
	"bytes"
	"database/sql"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const dbFilename = "torigemu.db"

const jmdictFileName = "jmdict.xml"
const kanjidictFileName = "kanjidic2.xml"
const maxPts = 15  // Max grade of 10 + Max JLPT of 5
const maxLimit = 4 // Arbitrary limit
const ptsFactor = maxPts / maxLimit
const maxJLPT = 6

type kmap map[string]int

var db *sql.DB
var insertcount = 0
var failcount = 0
var kanjiExp = regexp.MustCompile(`\p{Han}+`)
var endsInNExp = regexp.MustCompile(`(ん|ン)$`)

func main() {
	log.Printf("Loading kanji dictionary...")
	dict, err := getKanjiDict()
	if err != nil {
		log.Printf("error: %v", err)
		return
	}
	log.Printf("Loading kanji points map...")
	kptsmap, err := getKanjiPtsMap()
	if err != nil {
		log.Printf("error: %v", err)
		return
	}
	log.Printf("Creating kanji database...")
	err = createKanjiDb(dict, kptsmap)
	if err != nil {
		log.Printf("error: %v", err)
		return
	}
}

func createKanjiDb(dict *jmdict, kptsmap kmap) error {
	var err error
	db, err = sql.Open("sqlite3", dbFilename)
	if err != nil {
		return err
	}
	dropTable("words")
	err = createTable("words (kanji TEXT PRIMARY KEY, kana TEXT, points INT)")
	if err != nil {
		return err
	}
	// Prepare the statement and use a transaction for massive speed increase.
	stmt, err := db.Prepare("INSERT INTO words (kanji, kana, points) VALUES (:KJ, :KN, :SC)")
	if err != nil {
		return err
	}
	defer stmt.Close()
	// Optimize the database insertion
	execDb("BEGIN")
	for _, e := range dict.Entry {
		if isNoun(e) {
			saveWord(stmt, e, kptsmap)
		}
	}
	execDb("COMMIT")
	log.Printf("Inserted %d record(s)", insertcount)
	log.Printf("Merged %d record(s)", failcount)
	return nil
}

func isNoun(e entry) bool {
	for _, s := range e.Sense {
		for _, p := range s.Pos {
			if p == "n" {
				return true
			}
		}
	}
	return false
}

func saveWord(stmt *sql.Stmt, e entry, kptsmap kmap) {
	var pts int
	kana, endsInN := getKana(e)
	// Get all of the entry kanji variants
	for _, kanji := range getKanjis(e) {
		if endsInN {
			// Automatic zero for ending in 'ん'
			pts = 0
		} else {
			pts = getKanjiWordPts(kanji, kptsmap)
		}
		_, err := stmt.Exec(sql.Named("KJ", &kanji), sql.Named("KN", &kana), sql.Named("SC", &pts))
		if err != nil {
			mergeRecords(kanji, kana, pts)
			failcount++
			// return err
		} else {
			insertcount++
		}
	}
}

func mergeRecords(kanji string, kana string, pts int) {
	var existingKana string
	var existingPts int
	found := queryDb(fmt.Sprintf("SELECT kana, points FROM words WHERE kanji = '%s'", kanji),
		&existingKana, &existingPts)
	if !found {
		log.Printf("DBERROR: Could not find record for %s", kanji)
		return
	}
	// Merge kana and the word pts.
	newKana := mergeStrings(existingKana, kana)
	newPts := mergePts(existingPts, pts)
	err := execDb("UPDATE words SET kana = ?, points = ? WHERE kanji = ?", &newKana, &newPts, &kanji)
	if err != nil {
		log.Printf("DBERROR updating %s", kanji)
	}
}

func mergeStrings(first string, second string) string {
	mergedmap := make(map[string]bool)
	addStrings(mergedmap, first)
	addStrings(mergedmap, second)
	mergedString := ""
	for str := range mergedmap {
		if len(mergedString) > 0 {
			mergedString += ","
		}
		mergedString += str
	}
	return mergedString
}

func addStrings(dest map[string]bool, strs string) {
	for _, str := range strings.Split(strs, ",") {
		dest[str] = true
	}
}

func mergePts(first int, second int) int {
	if first > second {
		return first
	}
	return second
}

func getKanjiWordPts(kanji string, kptsmap kmap) int {
	// Words entirely of hiragana or katakana are worth 1 point.
	pts := 1
	if kanjiExp.MatchString(kanji) {
		for _, k := range kanji {
			kpts := kptsmap[string(k)]
			if kpts > pts {
				// The word pts is equal to the highest kanji pts in the word.
				pts = kpts
			}
		}
	}
	return pts
}

/*
The kanji element, or in its absence, the reading element, is
the defining component of each entry.
The overwhelming majority of entries will have a single kanji
element associated with a word in Japanese. Where there are
multiple kanji elements within an entry, they will be orthographical
variants of the same word, either using variations in okurigana, or
alternative and equivalent kanji. Common "mis-spellings" may be
included, provided they are associated with appropriate information
fields. Synonyms are not included; they may be indicated in the
cross-reference field associated with the sense element.
*/
func getKanjis(e entry) []string {
	kanjis := make([]string, 0)
	for _, k := range e.Kele {
		if len(k.Keb) > 0 {
			kanjis = append(kanjis, k.Keb)
		}
	}
	return kanjis
}

func getKana(e entry) (string, bool) {
	kanas := ""
	endsInN := true
	for _, k := range e.Rele {
		if !endsInNExp.MatchString(k.Reb) {
			// If at least one variation does not end in 'ん', then it's valid.
			endsInN = false
		}
		if len(kanas) > 0 {
			kanas += ","
		}
		kanas += k.Reb
	}
	return kanas, endsInN
}

func getKanjiDict() (*jmdict, error) {
	data, err := loadXMLFile(jmdictFileName)
	if err != nil {
		return nil, err
	}
	d := xml.NewDecoder(bytes.NewReader(data))
	// Map the entities to standard XML, or else a parsing error occurs.
	d.Entity = jmentitymap
	dict := jmdict{}
	err = d.Decode(&dict)
	if err != nil {
		return nil, err
	}
	return &dict, nil
}

func getKanjiPtsMap() (kmap, error) {
	data, err := loadXMLFile(kanjidictFileName)
	if err != nil {
		log.Printf("error: %v", err)
		return nil, err
	}
	kanji := kanjidic{}
	err = xml.Unmarshal(data, &kanji)
	if err != nil {
		log.Printf("error: %v", err)
		return nil, err
	}
	kptsmap := make(kmap)
	for _, ch := range kanji.Character {
		kptsmap[ch.Literal] = getCharacterPts(ch)
	}
	return kptsmap, nil
}

func getCharacterPts(ch character) int {
	pts := ch.Misc.Grade
	if ch.Misc.JLPT > 0 {
		// JLPT is in reverse order. Higher level is lower number.
		pts += maxJLPT - ch.Misc.JLPT
	}
	// Kanji Pts: RawPts / PtsFactor
	return pts / ptsFactor
}

func loadXMLFile(filename string) ([]byte, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	data, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func dropTable(tableDef string) {
	execDb(fmt.Sprintf("DROP TABLE IF EXISTS %s", tableDef))
}

func createTable(tableDef string) error {
	return execDb(fmt.Sprintf("CREATE TABLE %s", tableDef))
}

func createIndex(indexDef string) error {
	return execDb(fmt.Sprintf("CREATE INDEX %s", indexDef))
}

func execDb(stmt string, args ...interface{}) error {
	statement, err := db.Prepare(stmt)
	if err != nil {
		return err
	}
	_, err = statement.Exec(args...)
	return err
}

func queryDb(stmt string, args ...interface{}) bool {
	rows, err := db.Query(stmt)
	defer closeRows(rows)
	if err != nil {
		log.Printf("DBERROR: Querying %s: %v", stmt, err)
		return false
	}
	if rows.Next() {
		if args != nil {
			rows.Scan(args...)
		}
		return true
	}
	return false
}

func closeRows(rows *sql.Rows) {
	if nil != rows {
		rows.Close()
	}
}
<|endoftext|>


<commit_before>package elb

import (
	"encoding/xml"
	"fmt"
	"log"
	"net/url"
	"strconv"
	"time"

	"github.com/dynport/gocloud/aws"
)

const (
	API_VERSION = "2012-06-01"
)

type Client struct {
	*aws.Client
}

func NewFromEnv() *Client {
	return &Client{
		aws.NewFromEnv(),
	}
}

type DescribeLoadBalancersResponse struct {
	XMLName       xml.Name        `xml:"DescribeLoadBalancersResponse"`
	LoadBalancers []*LoadBalancer `xml:"DescribeLoadBalancersResult>LoadBalancerDescriptions>member"`
}

type LoadBalancer struct {
	LoadBalancerName          string    `"xml:"LoadBalancerName"`
	CreatedTime               time.Time `xml:"CreatedTime"`
	VPCId                     string    `xml:"VPCId"`
	CanonicalHostedZoneName   string    `xml:"CanonicalHostedZoneName"`
	CanonicalHostedZoneNameID string    `xml:"CanonicalHostedZoneNameID"`
	Scheme                    string    `xml:"Scheme"`
	DNSName                   string    `xml:"DNSName"`
	BackendServerDescriptions string    `xml:"BackendServerDescriptions"`

	HealthCheckInterval           int    `xml:"HealthCheck>Interval"`
	HealthCheckTarget             string `xml:"HealthCheck>Target"`
	HealthCheckHealthyThreshold   int    `xml:"HealthCheck>HealthyThreshold"`
	HealthCheckTimeout            int    `xml:"HealthCheck>Timeout"`
	HealthCheckUnhealthyThreshold int    `xml:"HealthCheck>UnhealthyThreshold"`

	SourceSecurityGroupOwnerAlias string `xml:"SourceSecurityGroup>OwnerAlias"`
	SourceSecurityGroupGroupName  string `xml:"SourceSecurityGroup>GroupName"`

	Listeners         []*Listener `xml:"ListenerDescriptions>member>Listener"`
	AvailabilityZones []string    `xml:"AvailabilityZones>member"`
	Instances         []string    `xml:"Instances>member>InstanceId"`
	Subnets           []string    `xml:"Subnets>member"`
}

type Listener struct {
	Protocol         string `xml:"Protocol"`
	LoadBalancerPort int    `xml:"LoadBalancerPort"`
	InstanceProtocol string `xml:"InstanceProtocol"`
	InstancePort     int    `xml:"InstancePort"`
}

type InstanceState struct {
	Description string `xml:"Description"`
	InstanceId  string `xml:"InstanceId"`
	State       string `xml:"State"`
	ReasonCode  string `xml:"ReasonCode"`
}

type DescribeInstanceHealthResponse struct {
	XMLName        xml.Name         `xml:"DescribeInstanceHealthResponse"`
	InstanceStates []*InstanceState `xml:"DescribeInstanceHealthResult>InstanceStates>member"`
}

func queryForAction(action string) string {
	return "Version=" + API_VERSION + "&Action=" + action
}

type RegisterInstancesWithLoadBalancerResponse struct {
	RequestId string `xml:"ResponseMetadata>RequestId"`
}

func (client *Client) Endpoint() string {
	prefix := "https://elasticloadbalancing"
	if client.Client.Region != "" {
		prefix += "." + client.Client.Region
	}
	return prefix + ".amazonaws.com"
}

func (client *Client) DescribeInstanceHealth(name string) (states []*InstanceState, e error) {
	query := queryForAction("DescribeInstanceHealth") + "&LoadBalancerName=" + name
	raw, e := client.DoSignedRequest("GET", client.Endpoint(), query, nil)
	if e != nil {
		return nil, e
	}
	rsp := &DescribeInstanceHealthResponse{}
	e = xml.Unmarshal(raw.Content, rsp)
	if e != nil {
		return states, e
	}
	return rsp.InstanceStates, nil
}

func (client *Client) DeregisterInstancesWithLoadBalancer(loadBalancerName string, instances []string) error {
	return client.updateLoadBalancerCall("DeregisterInstancesFromLoadBalancer", loadBalancerName, instances)
}

func (client *Client) RegisterInstancesWithLoadBalancer(loadBalancerName string, instances []string) error {
	log.Print("registering %#v with %s", instances, loadBalancerName)
	return client.updateLoadBalancerCall("RegisterInstancesWithLoadBalancer", loadBalancerName, instances)
}

func (client *Client) updateLoadBalancerCall(action string, loadBalancerName string, instances []string) error {
	query := queryForAction(action) + "&LoadBalancerName=" + loadBalancerName
	for i, id := range instances {
		query += fmt.Sprintf("&Instances.member.%d.InstanceId=%s", i+1, id)
	}
	log.Printf("sending request %s", query)
	raw, e := client.DoSignedRequest("POST", client.Endpoint(), query, nil)
	if e != nil {
		return e
	}
	log.Printf("status: %s", raw.StatusCode)
	log.Println(string(raw.Content))
	return nil
}

type DescribeLoadBalancersOptions struct {
	LoadBalancerNames []string
	Marker            string
	PageSize          int
}

func (client *Client) DescribeLoadBalancers(a ...func(*DescribeLoadBalancersOptions)) (lbs []*LoadBalancer, e error) {
	o := &DescribeLoadBalancersOptions{}
	for _, f := range a {
		f(o)
	}
	v := url.Values{
		"Version": {API_VERSION},
		"Action":  {"DescribeLoadBalancers"},
	}
	for i, n := range o.LoadBalancerNames {
		v.Set("LoadBalancerNames.member."+strconv.Itoa(i+1), n)
	}
	if o.Marker != "" {
		v.Set("Marker", o.Marker)
	}
	if o.PageSize > 0 {
		v.Set("PageSize", strconv.Itoa(o.PageSize))
	}
	raw, e := client.DoSignedRequest("GET", client.Endpoint(), v.Encode(), nil)
	if e != nil {
		return lbs, e
	}
	rsp := &DescribeLoadBalancersResponse{}
	e = xml.Unmarshal(raw.Content, rsp)
	if e != nil {
		return lbs, e
	}
	return rsp.LoadBalancers, e
}
<commit_msg>add opt for selecting elb names<commit_after>package elb

import (
	"encoding/xml"
	"fmt"
	"log"
	"net/url"
	"strconv"
	"time"

	"github.com/dynport/gocloud/aws"
)

const (
	API_VERSION = "2012-06-01"
)

type Client struct {
	*aws.Client
}

func NewFromEnv() *Client {
	return &Client{
		aws.NewFromEnv(),
	}
}

type DescribeLoadBalancersResponse struct {
	XMLName       xml.Name        `xml:"DescribeLoadBalancersResponse"`
	LoadBalancers []*LoadBalancer `xml:"DescribeLoadBalancersResult>LoadBalancerDescriptions>member"`
}

type LoadBalancer struct {
	LoadBalancerName          string    `"xml:"LoadBalancerName"`
	CreatedTime               time.Time `xml:"CreatedTime"`
	VPCId                     string    `xml:"VPCId"`
	CanonicalHostedZoneName   string    `xml:"CanonicalHostedZoneName"`
	CanonicalHostedZoneNameID string    `xml:"CanonicalHostedZoneNameID"`
	Scheme                    string    `xml:"Scheme"`
	DNSName                   string    `xml:"DNSName"`
	BackendServerDescriptions string    `xml:"BackendServerDescriptions"`

	HealthCheckInterval           int    `xml:"HealthCheck>Interval"`
	HealthCheckTarget             string `xml:"HealthCheck>Target"`
	HealthCheckHealthyThreshold   int    `xml:"HealthCheck>HealthyThreshold"`
	HealthCheckTimeout            int    `xml:"HealthCheck>Timeout"`
	HealthCheckUnhealthyThreshold int    `xml:"HealthCheck>UnhealthyThreshold"`

	SourceSecurityGroupOwnerAlias string `xml:"SourceSecurityGroup>OwnerAlias"`
	SourceSecurityGroupGroupName  string `xml:"SourceSecurityGroup>GroupName"`

	Listeners         []*Listener `xml:"ListenerDescriptions>member>Listener"`
	AvailabilityZones []string    `xml:"AvailabilityZones>member"`
	Instances         []string    `xml:"Instances>member>InstanceId"`
	Subnets           []string    `xml:"Subnets>member"`
}

type Listener struct {
	Protocol         string `xml:"Protocol"`
	LoadBalancerPort int    `xml:"LoadBalancerPort"`
	InstanceProtocol string `xml:"InstanceProtocol"`
	InstancePort     int    `xml:"InstancePort"`
}

type InstanceState struct {
	Description string `xml:"Description"`
	InstanceId  string `xml:"InstanceId"`
	State       string `xml:"State"`
	ReasonCode  string `xml:"ReasonCode"`
}

type DescribeInstanceHealthResponse struct {
	XMLName        xml.Name         `xml:"DescribeInstanceHealthResponse"`
	InstanceStates []*InstanceState `xml:"DescribeInstanceHealthResult>InstanceStates>member"`
}

func queryForAction(action string) string {
	return "Version=" + API_VERSION + "&Action=" + action
}

type RegisterInstancesWithLoadBalancerResponse struct {
	RequestId string `xml:"ResponseMetadata>RequestId"`
}

func (client *Client) Endpoint() string {
	prefix := "https://elasticloadbalancing"
	if client.Client.Region != "" {
		prefix += "." + client.Client.Region
	}
	return prefix + ".amazonaws.com"
}

func (client *Client) DescribeInstanceHealth(name string) (states []*InstanceState, e error) {
	query := queryForAction("DescribeInstanceHealth") + "&LoadBalancerName=" + name
	raw, e := client.DoSignedRequest("GET", client.Endpoint(), query, nil)
	if e != nil {
		return nil, e
	}
	rsp := &DescribeInstanceHealthResponse{}
	e = xml.Unmarshal(raw.Content, rsp)
	if e != nil {
		return states, e
	}
	return rsp.InstanceStates, nil
}

func (client *Client) DeregisterInstancesWithLoadBalancer(loadBalancerName string, instances []string) error {
	return client.updateLoadBalancerCall("DeregisterInstancesFromLoadBalancer", loadBalancerName, instances)
}

func (client *Client) RegisterInstancesWithLoadBalancer(loadBalancerName string, instances []string) error {
	log.Print("registering %#v with %s", instances, loadBalancerName)
	return client.updateLoadBalancerCall("RegisterInstancesWithLoadBalancer", loadBalancerName, instances)
}

func (client *Client) updateLoadBalancerCall(action string, loadBalancerName string, instances []string) error {
	query := queryForAction(action) + "&LoadBalancerName=" + loadBalancerName
	for i, id := range instances {
		query += fmt.Sprintf("&Instances.member.%d.InstanceId=%s", i+1, id)
	}
	log.Printf("sending request %s", query)
	raw, e := client.DoSignedRequest("POST", client.Endpoint(), query, nil)
	if e != nil {
		return e
	}
	log.Printf("status: %s", raw.StatusCode)
	log.Println(string(raw.Content))
	return nil
}

func OptLoadBalancerName(s string) func(o *DescribeLoadBalancersOptions) {
	return func(o *DescribeLoadBalancersOptions) { o.LoadBalancerNames = []string{s} }
}

type DescribeLoadBalancersOptions struct {
	LoadBalancerNames []string
	Marker            string
	PageSize          int
}

func (client *Client) DescribeLoadBalancers(a ...func(*DescribeLoadBalancersOptions)) (lbs []*LoadBalancer, e error) {
	o := &DescribeLoadBalancersOptions{}
	for _, f := range a {
		f(o)
	}
	v := url.Values{
		"Version": {API_VERSION},
		"Action":  {"DescribeLoadBalancers"},
	}
	for i, n := range o.LoadBalancerNames {
		v.Set("LoadBalancerNames.member."+strconv.Itoa(i+1), n)
	}
	if o.Marker != "" {
		v.Set("Marker", o.Marker)
	}
	if o.PageSize > 0 {
		v.Set("PageSize", strconv.Itoa(o.PageSize))
	}
	raw, e := client.DoSignedRequest("GET", client.Endpoint(), v.Encode(), nil)
	if e != nil {
		return lbs, e
	}
	rsp := &DescribeLoadBalancersResponse{}
	e = xml.Unmarshal(raw.Content, rsp)
	if e != nil {
		return lbs, e
	}
	return rsp.LoadBalancers, e
}
<|endoftext|>


<commit_before>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fauxfile

import (
	"fmt"
	"sort"
	"testing"
	"os"
)

func ExpectCwd(t *testing.T, expected string, mf *MockFilesystem) {
	if mf.cwd.path() != expected {
		t.Fatalf("Expected cwd of '%v', got '%v'", expected, mf.cwd)
	}
}

func ExpectDir(t *testing.T, path string, mf *MockFilesystem) {
	fi, err := mf.resolve(path)
	if err != nil {
		t.Fatalf("Expected path of '%s' to be present", path)
	}
	if !fi.IsDir() {
		t.Fatalf("Expected '%v' to be directory", path)
	}
}

func ExpectFile(t *testing.T, path string, mf *MockFilesystem) *MockFileInfo {
	fi, err := mf.resolve(path)
	if err != nil {
		t.Fatalf("Expected file at '%s' to be present", path)
	}
	return fi
}

func ExpectEqual(t *testing.T, expected string, actual string) {
	if expected != actual {
		t.Fatalf("Expected '%v', got '%v'", expected, actual)
	}
}

func TestChdir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("/foo", 0755)
	ExpectCwd(t, "/", mf)
	mf.Chdir("foo")
	ExpectCwd(t, "/foo", mf)
}

func TestMkdir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("/foo", 0755)
	mf.Mkdir("/bar", 0777)
	mf.Chdir("bar")
	mf.Mkdir("baz", 0777)
	ExpectDir(t, "/foo", mf)
	ExpectDir(t, "/bar", mf)
	ExpectDir(t, "/bar/baz", mf)
}

func TestMkdirAll(t *testing.T) {
	mf := NewMockFilesystem()
	err := mf.MkdirAll("/foo/bar/baz", 0755)
	if err != nil {
		t.Fatalf("Problem creating directories: %v", err)
	}
	ExpectDir(t, "/foo", mf)
	ExpectDir(t, "/foo/bar", mf)
	ExpectDir(t, "/foo/bar/baz", mf)
}

func TestCreate(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Create("foo.txt")
	fi := ExpectFile(t, "/foo.txt", mf)
	if fi.Mode().Perm() != 0666 {
		t.Fatalf("New file perm %v, expected 0666", fi.Mode().Perm())
	}
}

func TestCreateSubdirectoryPath(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Chdir("foo")
	_, err := mf.Create("foo.txt")
	if err != nil {
		t.Fatalf("Create should not throw error")
	}
	fi := ExpectFile(t, "/foo/foo.txt", mf)
	ExpectEqual(t, "/foo/foo.txt", fi.path())
	ExpectEqual(t, "foo.txt", fi.name)
}

func TestLongFormCreate(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	_, err := mf.Create("foo/foo.txt")
	if err != nil {
		t.Fatalf("Create should not throw error")
	}
	fi := ExpectFile(t, "/foo/foo.txt", mf)
	ExpectEqual(t, "/foo/foo.txt", fi.path())
	ExpectEqual(t, "foo.txt", fi.name)
}

func TestOpen(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Create("foo.txt")
	f, err := mf.Open("foo.txt")
	if err != nil {
		t.Fatalf("Error: %v", err)
	}
	ExpectEqual(t, "foo.txt", f.(*MockFile).Name())
	ExpectEqual(t, "foo.txt", f.(*MockFile).path)
}

func TestRemove(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Create("foo/foo.txt")
	err := mf.Remove("foo")
	if err == nil {
		t.Fatalf("Should not be able to remove non-empty directories")
	}
	err = mf.Remove("foo/foo.txt")
	if err != nil {
		t.Fatalf("Should be able to remove file")
	}
	_, err = mf.Open("foo/foo.txt")
	if err == nil {
		t.Fatalf("Remove did not remove file")
	}
	err = mf.Remove("foo")
	if err != nil {
		t.Fatalf("Should be able to remove empty directory")
	}
	_, err = mf.Open("foo")
	if err == nil {
		t.Fatalf("Remove did not remove directory")
	}
}

func TestRemoveAll(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Create("foo/foo.txt")
	mf.Create("foo/bar.txt")
	mf.RemoveAll("foo")
	_, err := mf.Open("/foo/foo.txt")
	if err == nil {
		t.Fatalf("RemoveAll should remove children")
	}
	_, err = mf.Open("/foo/bar.txt")
	if err == nil {
		t.Fatalf("RemoveAll should remove children")
	}
	_, err = mf.Open("/foo")
	if err == nil {
		t.Fatalf("RemoveAll should remove target")
	}
}

func TestReaddir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Mkdir("foo/a", 0755)
	mf.Create("foo/b.txt")
	mf.Create("foo/c.txt")
	f, _ := mf.Open("foo")
	fi, err := f.Readdir(-1)
	if err != nil {
		t.Fatalf("Readdir should not throw error.")
	}
	files := map[string]string{}
	for _, i := range fi {
		files[i.Name()] = i.(*MockFileInfo).path()
	}
	ExpectEqual(t, "/foo/a", files["a"])
	ExpectEqual(t, "/foo/b.txt", files["b.txt"])
	ExpectEqual(t, "/foo/c.txt", files["c.txt"])
	ExpectEqual(t, "3", fmt.Sprintf("%v", len(files)))
}

func TestReaddirnames(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Mkdir("foo/a", 0755)
	mf.Create("foo/b.txt")
	mf.Create("foo/c.txt")
	f, _ := mf.Open("foo")
	names, err := f.Readdirnames(-1)
	if err != nil {
		t.Fatalf("Readdir should not throw error.")
	}
	sort.Strings(names)
	ExpectEqual(t, "a", names[0])
	ExpectEqual(t, "b.txt", names[1])
	ExpectEqual(t, "c.txt", names[2])
}

func TestFileChdir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.MkdirAll("/foo/bar/baz", 0755)
	mf.Create("/foo/bar/baz/foo.txt")
	f, _ := mf.Open("/foo/bar/baz/foo.txt")
	ExpectCwd(t, "/", mf)
	f.Chdir()
	ExpectCwd(t, "/foo/bar/baz", mf)
}

func TestFileChmod(t *testing.T) {
	var (
		f File
		fi os.FileInfo
		err error
	)
	mf := NewMockFilesystem()
	mf.Create("foo.txt")
	if f, err = mf.Open("foo.txt"); err != nil {
		t.Fatalf("File expected: %v", err)
	}
	fi = ExpectFile(t, "foo.txt", mf)
	if perm := fi.Mode().Perm(); perm != 0666 {
		t.Fatalf("New file perm %v, expected 0666", perm)
	}
	if err = f.Chmod(0755); err != nil {
		t.Fatalf("Chmod should not return error: %v", err)
	}
	if perm := fi.Mode().Perm(); perm != 0755 {
		t.Fatalf("Perm %v, expected 0755", perm)
	}
}

func TestStat(t *testing.T) {
	var (
		mf *MockFilesystem
		f File
		fi os.FileInfo
		err error
	)
	mf = NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Create("/foo/foo.txt")
	if f, err = mf.Open("/foo/foo.txt"); err != nil {
		t.Fatalf("File should exist: %v", err)
	}
	if fi, err = f.Stat(); err != nil {
		t.Fatalf("Stat should not throw error: %v", err)
	}
	if fi.Name() != "foo.txt" {
		t.Fatalf("Stat should return accurate file object.")
	}
}

func TestFileInfoPath(t *testing.T) {
	mf := NewMockFilesystem()
	mf.MkdirAll("/foo/bar/baz", 0755)
	fp := "/foo/bar/baz/foo.txt"
	mf.Create(fp)
	fi := ExpectFile(t, fp, mf)
	if fi.path() != fp {
		t.Fatalf("Expected path of %v, got %v", fp, fi.path())
	}
}
<commit_msg>Runs gofmt<commit_after>// Copyright 2012 Arne Roomann-Kurrik
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fauxfile

import (
	"fmt"
	"os"
	"sort"
	"testing"
)

func ExpectCwd(t *testing.T, expected string, mf *MockFilesystem) {
	if mf.cwd.path() != expected {
		t.Fatalf("Expected cwd of '%v', got '%v'", expected, mf.cwd)
	}
}

func ExpectDir(t *testing.T, path string, mf *MockFilesystem) {
	fi, err := mf.resolve(path)
	if err != nil {
		t.Fatalf("Expected path of '%s' to be present", path)
	}
	if !fi.IsDir() {
		t.Fatalf("Expected '%v' to be directory", path)
	}
}

func ExpectFile(t *testing.T, path string, mf *MockFilesystem) *MockFileInfo {
	fi, err := mf.resolve(path)
	if err != nil {
		t.Fatalf("Expected file at '%s' to be present", path)
	}
	return fi
}

func ExpectEqual(t *testing.T, expected string, actual string) {
	if expected != actual {
		t.Fatalf("Expected '%v', got '%v'", expected, actual)
	}
}

func TestChdir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("/foo", 0755)
	ExpectCwd(t, "/", mf)
	mf.Chdir("foo")
	ExpectCwd(t, "/foo", mf)
}

func TestMkdir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("/foo", 0755)
	mf.Mkdir("/bar", 0777)
	mf.Chdir("bar")
	mf.Mkdir("baz", 0777)
	ExpectDir(t, "/foo", mf)
	ExpectDir(t, "/bar", mf)
	ExpectDir(t, "/bar/baz", mf)
}

func TestMkdirAll(t *testing.T) {
	mf := NewMockFilesystem()
	err := mf.MkdirAll("/foo/bar/baz", 0755)
	if err != nil {
		t.Fatalf("Problem creating directories: %v", err)
	}
	ExpectDir(t, "/foo", mf)
	ExpectDir(t, "/foo/bar", mf)
	ExpectDir(t, "/foo/bar/baz", mf)
}

func TestCreate(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Create("foo.txt")
	fi := ExpectFile(t, "/foo.txt", mf)
	if fi.Mode().Perm() != 0666 {
		t.Fatalf("New file perm %v, expected 0666", fi.Mode().Perm())
	}
}

func TestCreateSubdirectoryPath(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Chdir("foo")
	_, err := mf.Create("foo.txt")
	if err != nil {
		t.Fatalf("Create should not throw error")
	}
	fi := ExpectFile(t, "/foo/foo.txt", mf)
	ExpectEqual(t, "/foo/foo.txt", fi.path())
	ExpectEqual(t, "foo.txt", fi.name)
}

func TestLongFormCreate(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	_, err := mf.Create("foo/foo.txt")
	if err != nil {
		t.Fatalf("Create should not throw error")
	}
	fi := ExpectFile(t, "/foo/foo.txt", mf)
	ExpectEqual(t, "/foo/foo.txt", fi.path())
	ExpectEqual(t, "foo.txt", fi.name)
}

func TestOpen(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Create("foo.txt")
	f, err := mf.Open("foo.txt")
	if err != nil {
		t.Fatalf("Error: %v", err)
	}
	ExpectEqual(t, "foo.txt", f.(*MockFile).Name())
	ExpectEqual(t, "foo.txt", f.(*MockFile).path)
}

func TestRemove(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Create("foo/foo.txt")
	err := mf.Remove("foo")
	if err == nil {
		t.Fatalf("Should not be able to remove non-empty directories")
	}
	err = mf.Remove("foo/foo.txt")
	if err != nil {
		t.Fatalf("Should be able to remove file")
	}
	_, err = mf.Open("foo/foo.txt")
	if err == nil {
		t.Fatalf("Remove did not remove file")
	}
	err = mf.Remove("foo")
	if err != nil {
		t.Fatalf("Should be able to remove empty directory")
	}
	_, err = mf.Open("foo")
	if err == nil {
		t.Fatalf("Remove did not remove directory")
	}
}

func TestRemoveAll(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Create("foo/foo.txt")
	mf.Create("foo/bar.txt")
	mf.RemoveAll("foo")
	_, err := mf.Open("/foo/foo.txt")
	if err == nil {
		t.Fatalf("RemoveAll should remove children")
	}
	_, err = mf.Open("/foo/bar.txt")
	if err == nil {
		t.Fatalf("RemoveAll should remove children")
	}
	_, err = mf.Open("/foo")
	if err == nil {
		t.Fatalf("RemoveAll should remove target")
	}
}

func TestReaddir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Mkdir("foo/a", 0755)
	mf.Create("foo/b.txt")
	mf.Create("foo/c.txt")
	f, _ := mf.Open("foo")
	fi, err := f.Readdir(-1)
	if err != nil {
		t.Fatalf("Readdir should not throw error.")
	}
	files := map[string]string{}
	for _, i := range fi {
		files[i.Name()] = i.(*MockFileInfo).path()
	}
	ExpectEqual(t, "/foo/a", files["a"])
	ExpectEqual(t, "/foo/b.txt", files["b.txt"])
	ExpectEqual(t, "/foo/c.txt", files["c.txt"])
	ExpectEqual(t, "3", fmt.Sprintf("%v", len(files)))
}

func TestReaddirnames(t *testing.T) {
	mf := NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Mkdir("foo/a", 0755)
	mf.Create("foo/b.txt")
	mf.Create("foo/c.txt")
	f, _ := mf.Open("foo")
	names, err := f.Readdirnames(-1)
	if err != nil {
		t.Fatalf("Readdir should not throw error.")
	}
	sort.Strings(names)
	ExpectEqual(t, "a", names[0])
	ExpectEqual(t, "b.txt", names[1])
	ExpectEqual(t, "c.txt", names[2])
}

func TestFileChdir(t *testing.T) {
	mf := NewMockFilesystem()
	mf.MkdirAll("/foo/bar/baz", 0755)
	mf.Create("/foo/bar/baz/foo.txt")
	f, _ := mf.Open("/foo/bar/baz/foo.txt")
	ExpectCwd(t, "/", mf)
	f.Chdir()
	ExpectCwd(t, "/foo/bar/baz", mf)
}

func TestFileChmod(t *testing.T) {
	var (
		f   File
		fi  os.FileInfo
		err error
	)
	mf := NewMockFilesystem()
	mf.Create("foo.txt")
	if f, err = mf.Open("foo.txt"); err != nil {
		t.Fatalf("File expected: %v", err)
	}
	fi = ExpectFile(t, "foo.txt", mf)
	if perm := fi.Mode().Perm(); perm != 0666 {
		t.Fatalf("New file perm %v, expected 0666", perm)
	}
	if err = f.Chmod(0755); err != nil {
		t.Fatalf("Chmod should not return error: %v", err)
	}
	if perm := fi.Mode().Perm(); perm != 0755 {
		t.Fatalf("Perm %v, expected 0755", perm)
	}
}

func TestStat(t *testing.T) {
	var (
		mf  *MockFilesystem
		f   File
		fi  os.FileInfo
		err error
	)
	mf = NewMockFilesystem()
	mf.Mkdir("foo", 0755)
	mf.Create("/foo/foo.txt")
	if f, err = mf.Open("/foo/foo.txt"); err != nil {
		t.Fatalf("File should exist: %v", err)
	}
	if fi, err = f.Stat(); err != nil {
		t.Fatalf("Stat should not throw error: %v", err)
	}
	if fi.Name() != "foo.txt" {
		t.Fatalf("Stat should return accurate file object.")
	}
}

func TestFileInfoPath(t *testing.T) {
	mf := NewMockFilesystem()
	mf.MkdirAll("/foo/bar/baz", 0755)
	fp := "/foo/bar/baz/foo.txt"
	mf.Create(fp)
	fi := ExpectFile(t, fp, mf)
	if fi.path() != fp {
		t.Fatalf("Expected path of %v, got %v", fp, fi.path())
	}
}
<|endoftext|>


<commit_before>package pod

import (
	"encoding/json"
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	"github.com/caicloud/cyclone/pkg/apis/cyclone/v1alpha1"
	"github.com/caicloud/cyclone/pkg/k8s/clientset"
	"github.com/caicloud/cyclone/pkg/meta"
	"github.com/caicloud/cyclone/pkg/workflow/common"
	"github.com/caicloud/cyclone/pkg/workflow/controller"
	"github.com/caicloud/cyclone/pkg/workflow/workflowrun"
)

// Operator ...
type Operator struct {
	client        clientset.Interface
	clusterClient kubernetes.Interface
	workflowRun   string
	stage         string
	metaNamespace string
	pod           *corev1.Pod
}

// NewOperator ...
func NewOperator(clusterClient kubernetes.Interface, client clientset.Interface, pod *corev1.Pod) (*Operator, error) {
	annotations := pod.Annotations
	wfr, ok := annotations[meta.AnnotationWorkflowRunName]
	if !ok {
		return nil, fmt.Errorf("invalid workflow pod, without annotation %s", meta.AnnotationWorkflowRunName)
	}
	stage, ok := annotations[meta.AnnotationStageName]
	if !ok {
		return nil, fmt.Errorf("invalid workflow pod, without annotation %s", meta.AnnotationStageName)
	}
	metaNamespace, ok := annotations[meta.AnnotationMetaNamespace]
	if !ok {
		return nil, fmt.Errorf("invalid workflow pod, without annotation %s", meta.AnnotationMetaNamespace)
	}

	return &Operator{
		clusterClient: clusterClient,
		client:        client,
		workflowRun:   wfr,
		stage:         stage,
		metaNamespace: metaNamespace,
		pod:           pod,
	}, nil
}

// OnDelete handles the situation when a stage pod gotten delete. It updates
// corresponding WorkflowRun's status.
func (p *Operator) OnDelete() error {
	origin, err := p.client.CycloneV1alpha1().WorkflowRuns(p.metaNamespace).Get(p.workflowRun, metav1.GetOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			log.WithField("name", p.workflowRun).Error("Get WorkflowRun error: ", err)
			return err
		}
		return nil
	}

	wfr := origin.DeepCopy()
	operator, err := workflowrun.NewOperator(p.clusterClient, p.client, wfr, origin.Namespace)
	if err != nil {
		return err
	}
	status, ok := wfr.Status.Stages[p.stage]
	if !ok || status.Status.Phase == v1alpha1.StatusRunning {
		operator.UpdateStageStatus(p.stage, &v1alpha1.Status{
			Phase:              v1alpha1.StatusFailed,
			LastTransitionTime: metav1.Time{Time: time.Now()},
			Reason:             "PodDeleted",
		})
	}

	return operator.Update()
}

// OnUpdated ...
func (p *Operator) OnUpdated() error {
	origin, err := p.client.CycloneV1alpha1().WorkflowRuns(p.metaNamespace).Get(p.workflowRun, metav1.GetOptions{})
	if err != nil {
		if errors.IsNotFound(err) {
			log.WithField("wfr", p.workflowRun).WithField("ns", p.metaNamespace).Warn("wfr not found")
			// Delete the pod if WorkflowRun not exists any more, there is possible that the pod been deleted elsewhere on WorkflowRun deletion,
			// so if we delete pod failed here due to not found, just ignore it.
			if err := p.clusterClient.CoreV1().Pods(p.pod.Namespace).Delete(p.pod.Name, &metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
				log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Warn("Delete orphan pod error: ", err)
			} else {
				log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Info("Orphan pod deleted")
			}
			return nil
		}
		log.WithField("name", p.workflowRun).Error("Get WorkflowRun error: ", err)
		return err
	}

	// If the WorkflowRun has already been in terminated state, skip it.
	if origin.Status.Overall.Phase == v1alpha1.StatusSucceeded ||
		origin.Status.Overall.Phase == v1alpha1.StatusFailed ||
		origin.Status.Overall.Phase == v1alpha1.StatusCancelled {
		return nil
	}

	wfr := origin.DeepCopy()
	wfrOperator, err := workflowrun.NewOperator(p.clusterClient, p.client, wfr, origin.Namespace)
	if err != nil {
		return err
	}

	status, ok := wfr.Status.Stages[p.stage]

	switch p.pod.Status.Phase {
	case corev1.PodFailed:
		if !ok || status.Status.Phase != v1alpha1.StatusFailed {
			log.WithField("wfr", wfr.Name).
				WithField("stg", p.stage).
				WithField("status", v1alpha1.StatusFailed).
				Info("To update stage status")
			wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
				Phase:              v1alpha1.StatusFailed,
				LastTransitionTime: metav1.Time{Time: time.Now()},
				Reason:             "PodFailed",
			})
		}
	case corev1.PodSucceeded:
		if !ok || status.Status.Phase != v1alpha1.StatusSucceeded {
			log.WithField("wfr", wfr.Name).
				WithField("stage", p.stage).
				WithField("status", v1alpha1.StatusSucceeded).
				Info("To update stage status")
			wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
				Phase:              v1alpha1.StatusSucceeded,
				LastTransitionTime: metav1.Time{Time: time.Now()},
				Reason:             "PodSucceed",
			})
		}
	default:
		p.DetermineStatus(wfrOperator)
	}

	// Sync stage execution results from pod annotation to WorkflowRun status
	if p.pod.Annotations != nil {
		v, ok := p.pod.Annotations[meta.AnnotationStageResult]
		if ok && v != "" {
			var keyValues []v1alpha1.KeyValue
			if err := json.Unmarshal([]byte(v), &keyValues); err != nil {
				log.WithField("stg", p.stage).Warning("Unmarshal key-value results error: ", err)
			} else {
				wfrOperator.UpdateStageOutputs(p.stage, keyValues)
			}
		}
	}

	return wfrOperator.Update()
}

// DetermineStatus determines status of a stage and update WorkflowRun status accordingly.
// Because coordinator container is the last container running in the pod (it performs collect
// logs, artifacts, notify resource resolver to push resource), when the coordinator container
// have been finished (no matter Succeed or Failed), we need to update stage status, and take
// necessary actions to stop the pod.
func (p *Operator) DetermineStatus(wfrOperator workflowrun.Operator) {
	// If there are containers that haven't report status, no need to judge pod status.
	if len(p.pod.Status.ContainerStatuses) != len(p.pod.Spec.Containers) {
		return
	}

	// Check coordinator container's status, if it's terminated, we regard the pod completed.
	var terminatedCoordinatorState *corev1.ContainerStateTerminated
	var coordinatorReady = false
	for _, containerStatus := range p.pod.Status.ContainerStatuses {
		if containerStatus.Name == common.CoordinatorSidecarName {
			if containerStatus.State.Terminated == nil {
				log.WithField("container", containerStatus.Name).Debug("Coordinator not terminated")
				return
			}

			terminatedCoordinatorState = containerStatus.State.Terminated
			coordinatorReady = containerStatus.Ready
			// There is only one coordinator container in each pod.
			break
		}
	}

	// Now the workload containers and coordinator container have all been finished. We then:
	// - Update the stage status in WorkflowRun based on coordinator's exit code and readiness condition.
	if terminatedCoordinatorState.ExitCode == 0 && coordinatorReady {
		log.WithField("wfr", wfrOperator.GetWorkflowRun().Name).
			WithField("stg", p.stage).
			WithField("status", v1alpha1.StatusSucceeded).
			Info("To update stage status")
		wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
			Phase:              v1alpha1.StatusSucceeded,
			LastTransitionTime: metav1.Time{Time: time.Now()},
			Reason:             "CoordinatorCompleted",
			Message:            "Coordinator completed",
		})
	} else {
		log.WithField("wfr", wfrOperator.GetWorkflowRun().Name).
			WithField("stg", p.stage).
			WithField("status", v1alpha1.StatusFailed).
			Info("To update stage status")
		wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
			Phase:              v1alpha1.StatusFailed,
			LastTransitionTime: metav1.Time{Time: time.Now()},
			Reason:             terminatedCoordinatorState.Reason,
			Message:            terminatedCoordinatorState.Message,
		})
	}

	// The workload and coordinator containers have all been finished, but maybe some others are still Running,
	// Delete the pod and release cpu/memory resources if gc delay seconds is 0.
	if controller.Config.GC.DelaySeconds == 0 {
		if err := p.clusterClient.CoreV1().Pods(p.pod.Namespace).Delete(p.pod.Name, &metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
			log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Warn("Delete pod error: ", err)
		} else {
			log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Info("Pod deleted")
		}
	}
}
<commit_msg>fix: workflow run status check logic (#1220)<commit_after>package pod

import (
	"encoding/json"
	"fmt"
	"time"

	log "github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	"github.com/caicloud/cyclone/pkg/apis/cyclone/v1alpha1"
	"github.com/caicloud/cyclone/pkg/k8s/clientset"
	"github.com/caicloud/cyclone/pkg/meta"
	"github.com/caicloud/cyclone/pkg/workflow/common"
	"github.com/caicloud/cyclone/pkg/workflow/controller"
	"github.com/caicloud/cyclone/pkg/workflow/workflowrun"
)

// Operator ...
type Operator struct {
	client        clientset.Interface
	clusterClient kubernetes.Interface
	workflowRun   string
	stage         string
	metaNamespace string
	pod           *corev1.Pod
}

// NewOperator ...
func NewOperator(clusterClient kubernetes.Interface, client clientset.Interface, pod *corev1.Pod) (*Operator, error) {
	annotations := pod.Annotations
	wfr, ok := annotations[meta.AnnotationWorkflowRunName]
	if !ok {
		return nil, fmt.Errorf("invalid workflow pod, without annotation %s", meta.AnnotationWorkflowRunName)
	}
	stage, ok := annotations[meta.AnnotationStageName]
	if !ok {
		return nil, fmt.Errorf("invalid workflow pod, without annotation %s", meta.AnnotationStageName)
	}
	metaNamespace, ok := annotations[meta.AnnotationMetaNamespace]
	if !ok {
		return nil, fmt.Errorf("invalid workflow pod, without annotation %s", meta.AnnotationMetaNamespace)
	}

	return &Operator{
		clusterClient: clusterClient,
		client:        client,
		workflowRun:   wfr,
		stage:         stage,
		metaNamespace: metaNamespace,
		pod:           pod,
	}, nil
}

// OnDelete handles the situation when a stage pod gotten delete. It updates
// corresponding WorkflowRun's status.
func (p *Operator) OnDelete() error {
	origin, err := p.client.CycloneV1alpha1().WorkflowRuns(p.metaNamespace).Get(p.workflowRun, metav1.GetOptions{})
	if err != nil {
		if !errors.IsNotFound(err) {
			log.WithField("name", p.workflowRun).Error("Get WorkflowRun error: ", err)
			return err
		}
		return nil
	}

	wfr := origin.DeepCopy()
	operator, err := workflowrun.NewOperator(p.clusterClient, p.client, wfr, origin.Namespace)
	if err != nil {
		return err
	}
	status, ok := wfr.Status.Stages[p.stage]
	if !ok || status.Status.Phase == v1alpha1.StatusRunning {
		operator.UpdateStageStatus(p.stage, &v1alpha1.Status{
			Phase:              v1alpha1.StatusFailed,
			LastTransitionTime: metav1.Time{Time: time.Now()},
			Reason:             "PodDeleted",
		})
	}

	return operator.Update()
}

// OnUpdated ...
func (p *Operator) OnUpdated() error {
	origin, err := p.client.CycloneV1alpha1().WorkflowRuns(p.metaNamespace).Get(p.workflowRun, metav1.GetOptions{})
	if err != nil {
		if errors.IsNotFound(err) {
			log.WithField("wfr", p.workflowRun).WithField("ns", p.metaNamespace).Warn("wfr not found")
			// Delete the pod if WorkflowRun not exists any more, there is possible that the pod been deleted elsewhere on WorkflowRun deletion,
			// so if we delete pod failed here due to not found, just ignore it.
			if err := p.clusterClient.CoreV1().Pods(p.pod.Namespace).Delete(p.pod.Name, &metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
				log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Warn("Delete orphan pod error: ", err)
			} else {
				log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Info("Orphan pod deleted")
			}
			return nil
		}
		log.WithField("name", p.workflowRun).Error("Get WorkflowRun error: ", err)
		return err
	}

	// If the WorkflowRun has already been in terminated state, skip it.
	if origin.Status.Overall.Phase == v1alpha1.StatusSucceeded ||
		origin.Status.Overall.Phase == v1alpha1.StatusFailed ||
		origin.Status.Overall.Phase == v1alpha1.StatusCancelled {
		return nil
	}

	wfr := origin.DeepCopy()
	wfrOperator, err := workflowrun.NewOperator(p.clusterClient, p.client, wfr, origin.Namespace)
	if err != nil {
		return err
	}

	status, ok := wfr.Status.Stages[p.stage]

	switch p.pod.Status.Phase {
	case corev1.PodFailed:
		if !ok || status.Status.Phase != v1alpha1.StatusFailed {
			log.WithField("wfr", wfr.Name).
				WithField("stg", p.stage).
				WithField("status", v1alpha1.StatusFailed).
				Info("To update stage status")
			wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
				Phase:              v1alpha1.StatusFailed,
				LastTransitionTime: metav1.Time{Time: time.Now()},
				Reason:             "PodFailed",
			})
		}
	case corev1.PodSucceeded:
		if !ok || status.Status.Phase != v1alpha1.StatusSucceeded {
			log.WithField("wfr", wfr.Name).
				WithField("stage", p.stage).
				WithField("status", v1alpha1.StatusSucceeded).
				Info("To update stage status")
			wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
				Phase:              v1alpha1.StatusSucceeded,
				LastTransitionTime: metav1.Time{Time: time.Now()},
				Reason:             "PodSucceed",
			})
		}
	default:
		p.DetermineStatus(wfrOperator)
	}

	// Sync stage execution results from pod annotation to WorkflowRun status
	if p.pod.Annotations != nil {
		v, ok := p.pod.Annotations[meta.AnnotationStageResult]
		if ok && v != "" {
			var keyValues []v1alpha1.KeyValue
			if err := json.Unmarshal([]byte(v), &keyValues); err != nil {
				log.WithField("stg", p.stage).Warning("Unmarshal key-value results error: ", err)
			} else {
				wfrOperator.UpdateStageOutputs(p.stage, keyValues)
			}
		}
	}

	return wfrOperator.Update()
}

// DetermineStatus determines status of a stage and update WorkflowRun status accordingly.
// Because coordinator container is the last container running in the pod (it performs collect
// logs, artifacts, notify resource resolver to push resource), when the coordinator container
// have been finished (no matter Succeed or Failed), we need to update stage status, and take
// necessary actions to stop the pod.
func (p *Operator) DetermineStatus(wfrOperator workflowrun.Operator) {
	// If there are containers that haven't report status, no need to judge pod status.
	if len(p.pod.Status.ContainerStatuses) != len(p.pod.Spec.Containers) {
		return
	}

	// Check coordinator container's status, if it's terminated, we regard the pod completed.
	var terminatedCoordinatorState *corev1.ContainerStateTerminated
	for _, containerStatus := range p.pod.Status.ContainerStatuses {
		if containerStatus.Name == common.CoordinatorSidecarName {
			if containerStatus.State.Terminated == nil {
				log.WithField("container", containerStatus.Name).Debug("Coordinator not terminated")
				return
			}

			terminatedCoordinatorState = containerStatus.State.Terminated
			// There is only one coordinator container in each pod.
			break
		}
	}

	// Now the workload containers and coordinator container have all been finished. We then:
	// - Update the stage status in WorkflowRun based on coordinator's exit code and started time.
	if terminatedCoordinatorState.ExitCode == 0 && !terminatedCoordinatorState.StartedAt.IsZero() {
		log.WithField("wfr", wfrOperator.GetWorkflowRun().Name).
			WithField("stg", p.stage).
			WithField("status", v1alpha1.StatusSucceeded).
			Info("To update stage status")
		wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
			Phase:              v1alpha1.StatusSucceeded,
			LastTransitionTime: metav1.Time{Time: time.Now()},
			Reason:             "CoordinatorCompleted",
			Message:            "Coordinator completed",
		})
	} else {
		log.WithField("wfr", wfrOperator.GetWorkflowRun().Name).
			WithField("stg", p.stage).
			WithField("status", v1alpha1.StatusFailed).
			Info("To update stage status")
		wfrOperator.UpdateStageStatus(p.stage, &v1alpha1.Status{
			Phase:              v1alpha1.StatusFailed,
			LastTransitionTime: metav1.Time{Time: time.Now()},
			Reason:             terminatedCoordinatorState.Reason,
			Message:            terminatedCoordinatorState.Message,
		})
	}

	// The workload and coordinator containers have all been finished, but maybe some others are still Running,
	// Delete the pod and release cpu/memory resources if gc delay seconds is 0.
	if controller.Config.GC.DelaySeconds == 0 {
		if err := p.clusterClient.CoreV1().Pods(p.pod.Namespace).Delete(p.pod.Name, &metav1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
			log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Warn("Delete pod error: ", err)
		} else {
			log.WithField("ns", p.pod.Namespace).WithField("pod", p.pod.Name).Info("Pod deleted")
		}
	}
}
<|endoftext|>


<commit_before>package weed_server

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/util/mem"
	"io"
	"io/fs"
	"mime/multipart"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"

	"github.com/gorilla/mux"
)

var serverStats *stats.ServerStats
var startTime = time.Now()

func init() {
	serverStats = stats.NewServerStats()
	go serverStats.Start()
}

// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
func bodyAllowedForStatus(status int) bool {
	switch {
	case status >= 100 && status <= 199:
		return false
	case status == http.StatusNoContent:
		return false
	case status == http.StatusNotModified:
		return false
	}
	return true
}

func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
	if !bodyAllowedForStatus(httpStatus) {
		return
	}

	var bytes []byte
	if obj != nil {
		if r.FormValue("pretty") != "" {
			bytes, err = json.MarshalIndent(obj, "", "  ")
		} else {
			bytes, err = json.Marshal(obj)
		}
	}
	if err != nil {
		return
	}

	if httpStatus >= 400 {
		glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s",
			r.Method, r.URL.String(), httpStatus, string(bytes))
	}

	callback := r.FormValue("callback")
	if callback == "" {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		_, err = w.Write(bytes)
	} else {
		w.Header().Set("Content-Type", "application/javascript")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}

	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
	if err := writeJson(w, r, httpStatus, obj); err != nil {
		glog.V(0).Infof("error writing JSON status %d: %v", httpStatus, err)
		glog.V(1).Infof("JSON content: %+v", obj)
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
	m := make(map[string]interface{})
	m["error"] = err.Error()
	writeJsonQuiet(w, r, httpStatus, m)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}

func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) {
	m := make(map[string]interface{})
	if r.Method != "POST" {
		writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!"))
		return
	}

	debug("parsing upload file...")
	bytesBuffer := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(bytesBuffer)
	pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
	if pe != nil {
		writeJsonError(w, r, http.StatusBadRequest, pe)
		return
	}

	debug("assigning file id for", pu.FileName)
	r.ParseForm()
	count := uint64(1)
	if r.FormValue("count") != "" {
		count, pe = strconv.ParseUint(r.FormValue("count"), 10, 32)
		if pe != nil {
			writeJsonError(w, r, http.StatusBadRequest, pe)
			return
		}
	}
	ar := &operation.VolumeAssignRequest{
		Count:       count,
		DataCenter:  r.FormValue("dataCenter"),
		Rack:        r.FormValue("rack"),
		Replication: r.FormValue("replication"),
		Collection:  r.FormValue("collection"),
		Ttl:         r.FormValue("ttl"),
		DiskType:    r.FormValue("disk"),
	}
	assignResult, ae := operation.Assign(masterFn, grpcDialOption, ar)
	if ae != nil {
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		return
	}

	url := "http://" + assignResult.Url + "/" + assignResult.Fid
	if pu.ModifiedTime != 0 {
		url = url + "?ts=" + strconv.FormatUint(pu.ModifiedTime, 10)
	}

	debug("upload file to store", url)
	uploadOption := &operation.UploadOption{
		UploadUrl:         url,
		Filename:          pu.FileName,
		Cipher:            false,
		IsInputCompressed: pu.IsGzipped,
		MimeType:          pu.MimeType,
		PairMap:           pu.PairMap,
		Jwt:               assignResult.Auth,
	}
	uploadResult, err := operation.UploadData(pu.Data, uploadOption)
	if err != nil {
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	m["fileName"] = pu.FileName
	m["fid"] = assignResult.Fid
	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
	m["size"] = pu.OriginalDataSize
	m["eTag"] = uploadResult.ETag
	writeJsonQuiet(w, r, http.StatusCreated, m)
	return
}

func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
	switch strings.Count(path, "/") {
	case 3:
		parts := strings.Split(path, "/")
		vid, fid, filename = parts[1], parts[2], parts[3]
		ext = filepath.Ext(filename)
	case 2:
		parts := strings.Split(path, "/")
		vid, fid = parts[1], parts[2]
		dotIndex := strings.LastIndex(fid, ".")
		if dotIndex > 0 {
			ext = fid[dotIndex:]
			fid = fid[0:dotIndex]
		}
	default:
		sepIndex := strings.LastIndex(path, "/")
		commaIndex := strings.LastIndex(path[sepIndex:], ",")
		if commaIndex <= 0 {
			vid, isVolumeIdOnly = path[sepIndex+1:], true
			return
		}
		dotIndex := strings.LastIndex(path[sepIndex:], ".")
		vid = path[sepIndex+1 : commaIndex]
		fid = path[commaIndex+1:]
		ext = ""
		if dotIndex > 0 {
			fid = path[commaIndex+1 : dotIndex]
			ext = path[dotIndex:]
		}
	}
	return
}

func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	writeJsonQuiet(w, r, http.StatusOK, m)
}
func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Counters"] = serverStats
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Memory"] = stats.MemStat()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

var StaticFS fs.FS

func handleStaticResources(defaultMux *http.ServeMux) {
	defaultMux.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	defaultMux.Handle("/seaweedfsstatic/", http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func handleStaticResources2(r *mux.Router) {
	r.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func adjustPassthroughHeaders(w http.ResponseWriter, r *http.Request, filename string) {
	for header, values := range r.Header {
		if normalizedHeader, ok := xhttp.PassThroughHeaders[strings.ToLower(header)]; ok {
			w.Header()[normalizedHeader] = values
		}
	}
	adjustHeaderContentDisposition(w, r, filename)
}
func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
	if contentDisposition := w.Header().Get("Content-Disposition"); contentDisposition != "" {
		return
	}
	if filename != "" {
		filename = url.QueryEscape(filename)
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
}

func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
	rangeReq := r.Header.Get("Range")

	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if err := writeFn(w, 0, totalSize); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))

		w.WriteHeader(http.StatusPartialContent)
		err = writeFn(w, ra.start, ra.length)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if e = writeFn(part, ra.start, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	buf := mem.Allocate(128 * 1024)
	defer mem.Free(buf)
	if _, err := io.CopyBuffer(w, io.LimitReader(sendContent, sendSize), buf); err != nil {
		http.Error(w, "Internal Error", http.StatusInternalServerError)
		return
	}
}
<commit_msg>buffer for all range requests<commit_after>package weed_server

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"io"
	"io/fs"
	"mime/multipart"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/util"

	"github.com/gorilla/mux"
)

var serverStats *stats.ServerStats
var startTime = time.Now()

func init() {
	serverStats = stats.NewServerStats()
	go serverStats.Start()
}

// bodyAllowedForStatus is a copy of http.bodyAllowedForStatus non-exported function.
func bodyAllowedForStatus(status int) bool {
	switch {
	case status >= 100 && status <= 199:
		return false
	case status == http.StatusNoContent:
		return false
	case status == http.StatusNotModified:
		return false
	}
	return true
}

func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) {
	if !bodyAllowedForStatus(httpStatus) {
		return
	}

	var bytes []byte
	if obj != nil {
		if r.FormValue("pretty") != "" {
			bytes, err = json.MarshalIndent(obj, "", "  ")
		} else {
			bytes, err = json.Marshal(obj)
		}
	}
	if err != nil {
		return
	}

	if httpStatus >= 400 {
		glog.V(0).Infof("response method:%s URL:%s with httpStatus:%d and JSON:%s",
			r.Method, r.URL.String(), httpStatus, string(bytes))
	}

	callback := r.FormValue("callback")
	if callback == "" {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		_, err = w.Write(bytes)
	} else {
		w.Header().Set("Content-Type", "application/javascript")
		w.WriteHeader(httpStatus)
		if httpStatus == http.StatusNotModified {
			return
		}
		if _, err = w.Write([]uint8(callback)); err != nil {
			return
		}
		if _, err = w.Write([]uint8("(")); err != nil {
			return
		}
		fmt.Fprint(w, string(bytes))
		if _, err = w.Write([]uint8(")")); err != nil {
			return
		}
	}

	return
}

// wrapper for writeJson - just logs errors
func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) {
	if err := writeJson(w, r, httpStatus, obj); err != nil {
		glog.V(0).Infof("error writing JSON status %d: %v", httpStatus, err)
		glog.V(1).Infof("JSON content: %+v", obj)
	}
}
func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) {
	m := make(map[string]interface{})
	m["error"] = err.Error()
	writeJsonQuiet(w, r, httpStatus, m)
}

func debug(params ...interface{}) {
	glog.V(4).Infoln(params...)
}

func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterFn operation.GetMasterFn, grpcDialOption grpc.DialOption) {
	m := make(map[string]interface{})
	if r.Method != "POST" {
		writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!"))
		return
	}

	debug("parsing upload file...")
	bytesBuffer := bufPool.Get().(*bytes.Buffer)
	defer bufPool.Put(bytesBuffer)
	pu, pe := needle.ParseUpload(r, 256*1024*1024, bytesBuffer)
	if pe != nil {
		writeJsonError(w, r, http.StatusBadRequest, pe)
		return
	}

	debug("assigning file id for", pu.FileName)
	r.ParseForm()
	count := uint64(1)
	if r.FormValue("count") != "" {
		count, pe = strconv.ParseUint(r.FormValue("count"), 10, 32)
		if pe != nil {
			writeJsonError(w, r, http.StatusBadRequest, pe)
			return
		}
	}
	ar := &operation.VolumeAssignRequest{
		Count:       count,
		DataCenter:  r.FormValue("dataCenter"),
		Rack:        r.FormValue("rack"),
		Replication: r.FormValue("replication"),
		Collection:  r.FormValue("collection"),
		Ttl:         r.FormValue("ttl"),
		DiskType:    r.FormValue("disk"),
	}
	assignResult, ae := operation.Assign(masterFn, grpcDialOption, ar)
	if ae != nil {
		writeJsonError(w, r, http.StatusInternalServerError, ae)
		return
	}

	url := "http://" + assignResult.Url + "/" + assignResult.Fid
	if pu.ModifiedTime != 0 {
		url = url + "?ts=" + strconv.FormatUint(pu.ModifiedTime, 10)
	}

	debug("upload file to store", url)
	uploadOption := &operation.UploadOption{
		UploadUrl:         url,
		Filename:          pu.FileName,
		Cipher:            false,
		IsInputCompressed: pu.IsGzipped,
		MimeType:          pu.MimeType,
		PairMap:           pu.PairMap,
		Jwt:               assignResult.Auth,
	}
	uploadResult, err := operation.UploadData(pu.Data, uploadOption)
	if err != nil {
		writeJsonError(w, r, http.StatusInternalServerError, err)
		return
	}

	m["fileName"] = pu.FileName
	m["fid"] = assignResult.Fid
	m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid
	m["size"] = pu.OriginalDataSize
	m["eTag"] = uploadResult.ETag
	writeJsonQuiet(w, r, http.StatusCreated, m)
	return
}

func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
	switch strings.Count(path, "/") {
	case 3:
		parts := strings.Split(path, "/")
		vid, fid, filename = parts[1], parts[2], parts[3]
		ext = filepath.Ext(filename)
	case 2:
		parts := strings.Split(path, "/")
		vid, fid = parts[1], parts[2]
		dotIndex := strings.LastIndex(fid, ".")
		if dotIndex > 0 {
			ext = fid[dotIndex:]
			fid = fid[0:dotIndex]
		}
	default:
		sepIndex := strings.LastIndex(path, "/")
		commaIndex := strings.LastIndex(path[sepIndex:], ",")
		if commaIndex <= 0 {
			vid, isVolumeIdOnly = path[sepIndex+1:], true
			return
		}
		dotIndex := strings.LastIndex(path[sepIndex:], ".")
		vid = path[sepIndex+1 : commaIndex]
		fid = path[commaIndex+1:]
		ext = ""
		if dotIndex > 0 {
			fid = path[commaIndex+1 : dotIndex]
			ext = path[dotIndex:]
		}
	}
	return
}

func statsHealthHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	writeJsonQuiet(w, r, http.StatusOK, m)
}
func statsCounterHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Counters"] = serverStats
	writeJsonQuiet(w, r, http.StatusOK, m)
}

func statsMemoryHandler(w http.ResponseWriter, r *http.Request) {
	m := make(map[string]interface{})
	m["Version"] = util.Version()
	m["Memory"] = stats.MemStat()
	writeJsonQuiet(w, r, http.StatusOK, m)
}

var StaticFS fs.FS

func handleStaticResources(defaultMux *http.ServeMux) {
	defaultMux.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	defaultMux.Handle("/seaweedfsstatic/", http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func handleStaticResources2(r *mux.Router) {
	r.Handle("/favicon.ico", http.FileServer(http.FS(StaticFS)))
	r.PathPrefix("/seaweedfsstatic/").Handler(http.StripPrefix("/seaweedfsstatic", http.FileServer(http.FS(StaticFS))))
}

func adjustPassthroughHeaders(w http.ResponseWriter, r *http.Request, filename string) {
	for header, values := range r.Header {
		if normalizedHeader, ok := xhttp.PassThroughHeaders[strings.ToLower(header)]; ok {
			w.Header()[normalizedHeader] = values
		}
	}
	adjustHeaderContentDisposition(w, r, filename)
}
func adjustHeaderContentDisposition(w http.ResponseWriter, r *http.Request, filename string) {
	if contentDisposition := w.Header().Get("Content-Disposition"); contentDisposition != "" {
		return
	}
	if filename != "" {
		filename = url.QueryEscape(filename)
		contentDisposition := "inline"
		if r.FormValue("dl") != "" {
			if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl {
				contentDisposition = "attachment"
			}
		}
		w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`)
	}
}

func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64, mimeType string, writeFn func(writer io.Writer, offset int64, size int64) error) {
	rangeReq := r.Header.Get("Range")
	bufferedWriter := bufio.NewWriterSize(w, 128*1024)
	defer bufferedWriter.Flush()

	if rangeReq == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
		if err := writeFn(bufferedWriter, 0, totalSize); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	//the rest is dealing with partial content request
	//mostly copy from src/pkg/net/http/fs.go
	ranges, err := parseRange(rangeReq, totalSize)
	if err != nil {
		http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
		return
	}
	if sumRangesSize(ranges) > totalSize {
		// The total number of bytes in all the ranges
		// is larger than the size of the file by
		// itself, so this is probably an attack, or a
		// dumb client.  Ignore the range request.
		return
	}
	if len(ranges) == 0 {
		return
	}
	if len(ranges) == 1 {
		// RFC 2616, Section 14.16:
		// "When an HTTP message includes the content of a single
		// range (for example, a response to a request for a
		// single range, or to a request for a set of ranges
		// that overlap without any holes), this content is
		// transmitted with a Content-Range header, and a
		// Content-Length header showing the number of bytes
		// actually transferred.
		// ...
		// A response to a request for a single range MUST NOT
		// be sent using the multipart/byteranges media type."
		ra := ranges[0]
		w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10))
		w.Header().Set("Content-Range", ra.contentRange(totalSize))

		w.WriteHeader(http.StatusPartialContent)
		err = writeFn(bufferedWriter, ra.start, ra.length)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		return
	}

	// process multiple ranges
	for _, ra := range ranges {
		if ra.start > totalSize {
			http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable)
			return
		}
	}
	sendSize := rangesMIMESize(ranges, mimeType, totalSize)
	pr, pw := io.Pipe()
	mw := multipart.NewWriter(pw)
	w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary())
	sendContent := pr
	defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish.
	go func() {
		for _, ra := range ranges {
			part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize))
			if e != nil {
				pw.CloseWithError(e)
				return
			}
			if e = writeFn(part, ra.start, ra.length); e != nil {
				pw.CloseWithError(e)
				return
			}
		}
		mw.Close()
		pw.Close()
	}()
	if w.Header().Get("Content-Encoding") == "" {
		w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
	}
	w.WriteHeader(http.StatusPartialContent)
	if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil {
		http.Error(w, "Internal Error", http.StatusInternalServerError)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/fzzbt/radix/redis"
)

var key string = "foo:rand:000000000000"
var tests = make(map[string]func(string, *redis.Client, chan struct{}))

func init() {
    tests["set"] = setHandle
	tests["get"] = getHandle
}

func setHandle(data string, c *redis.Client, ch chan struct{}) {
    for _ = range ch {
        c.Set(key, data)
    }
}

func getHandle(data string, c *redis.Client, ch chan struct{}) {
    for _ = range ch {
        c.Set(key, data)
    }
}<commit_msg>bench: added ping<commit_after>package main

import (
    "github.com/fzzbt/radix/redis"
)

var key string = "foo:rand:000000000000"
var tests = make(map[string]func(string, *redis.Client, chan struct{}))

func init() {
    tests["ping"] = pingHandle
    tests["set"] = setHandle
	tests["get"] = getHandle
}

func pingHandle(data string, c *redis.Client, ch chan struct{}) {
    for _ = range ch {
        c.Ping()
    }
}

func setHandle(data string, c *redis.Client, ch chan struct{}) {
    for _ = range ch {
        c.Set(key, data)
    }
}

func getHandle(data string, c *redis.Client, ch chan struct{}) {
    for _ = range ch {
        c.Set(key, data)
    }
}<|endoftext|>


<commit_before>package editor

import (
	"fmt"
	"strings"

	"github.com/elpinal/coco3/screen"
)

type exCommand struct {
	name string
	fn   func(*commandline, []string) continuity
}

// exComands represents a table of Ex commands and corresponding functions.
// The order is important. Precede commands have higher precedence.
var exCommands = []exCommand{
	{"help", (*commandline).help},
	{"delete", (*commandline).delete},
	{"quit", (*commandline).quit},
	{"substitute", (*commandline).substitute},
}

type commandline struct {
	streamSet
	*editor

	basic *basic

	history [][]rune
	age     int
}

func newCommandline(s streamSet, e *editor) *commandline {
	return &commandline{
		streamSet: s,
		editor:    e,
		basic:     &basic{},
	}
}

func (e *commandline) Mode() mode {
	return modeCommandline
}

func (e *commandline) Position() int {
	return e.basic.pos + 1
}

func (e *commandline) Runes() []rune {
	return e.buf
}

func (e *commandline) Message() []rune {
	return append([]rune{':'}, e.basic.buf...)
}

func (e *commandline) Highlight() *screen.Hi {
	return nil
}

func (e *commandline) Run() (end continuity, next modeChanger, err error) {
	r, _, err := e.streamSet.in.ReadRune()
	if err != nil {
		return end, next, err
	}
	switch r {
	case CharCtrlM, CharCtrlJ:
	case CharEscape, CharCtrlC:
		next = norm()
		return end, next, err
	case CharBackspace, CharCtrlH:
		if len(e.basic.buf) == 0 {
			next = norm()
			return
		}
		e.basic.delete(e.basic.pos-1, e.basic.pos)
	case CharCtrlB:
		e.basic.move(0)
	case CharCtrlE:
		e.basic.move(len(e.basic.buf))
	case CharCtrlP:
		e.historyBack()
	case CharCtrlU:
		e.basic.delete(0, e.basic.pos)
	case CharCtrlW:
		// FIXME: It's redundant.
		ed := newEditor()
		ed.pos = e.basic.pos
		ed.buf = e.basic.buf
		pos := ed.pos
		ed.wordBackward()
		e.basic.delete(pos, ed.pos)
		return
	default:
		e.basic.insert([]rune{r}, e.basic.pos)
	}
	if r != CharCtrlM && r != CharCtrlJ {
		return
	}
	next = norm()
	var candidate exCommand
	s := string(e.basic.buf)
	if s == "" {
		return
	}
	args := strings.Split(s, " ")
	s = args[0]
	args = args[1:]
	defer func() {
		e.history = append(e.history, e.basic.buf)
	}()
	for _, cmd := range exCommands {
		if !strings.HasPrefix(cmd.name, s) {
			continue
		}
		if cmd.name == s {
			end = cmd.fn(e, args)
			return
		}
		if candidate.name == "" {
			candidate = cmd
		}
	}
	if candidate.name != "" {
		end = candidate.fn(e, args)
		return
	}
	err = fmt.Errorf("not a command: %q", s)
	return
}

func (e *commandline) historyBack() {
	l := len(e.history)
	if l-e.age == 0 {
		return
	}
	e.age++
	e.basic.buf = e.history[l-e.age]
}

func (e *commandline) historyForward() {
	l := len(e.history)
	if e.age == 0 {
		return
	}
	e.age--
	e.basic.buf = e.history[l-e.age]
}

func (e *commandline) quit(args []string) continuity {
	return exit
}

func (e *commandline) delete(args []string) (_ continuity) {
	e.editor.delete(0, len(e.editor.buf))
	return
}

func (e *commandline) help(args []string) continuity {
	e.buf = []rune("help")
	e.pos = 4
	return execute
}

func (e *commandline) substitute(args []string) (_ continuity) {
	if len(args) != 2 {
		return
	}
	pat := args[0]
	s0 := args[1]
	s := strings.Replace(string(e.buf), pat, s0, -1)
	e.buf = []rune(s)
	return
}
<commit_msg>Bind c_C-n<commit_after>package editor

import (
	"fmt"
	"strings"

	"github.com/elpinal/coco3/screen"
)

type exCommand struct {
	name string
	fn   func(*commandline, []string) continuity
}

// exComands represents a table of Ex commands and corresponding functions.
// The order is important. Precede commands have higher precedence.
var exCommands = []exCommand{
	{"help", (*commandline).help},
	{"delete", (*commandline).delete},
	{"quit", (*commandline).quit},
	{"substitute", (*commandline).substitute},
}

type commandline struct {
	streamSet
	*editor

	basic *basic

	history [][]rune
	age     int
}

func newCommandline(s streamSet, e *editor) *commandline {
	return &commandline{
		streamSet: s,
		editor:    e,
		basic:     &basic{},
	}
}

func (e *commandline) Mode() mode {
	return modeCommandline
}

func (e *commandline) Position() int {
	return e.basic.pos + 1
}

func (e *commandline) Runes() []rune {
	return e.buf
}

func (e *commandline) Message() []rune {
	return append([]rune{':'}, e.basic.buf...)
}

func (e *commandline) Highlight() *screen.Hi {
	return nil
}

func (e *commandline) Run() (end continuity, next modeChanger, err error) {
	r, _, err := e.streamSet.in.ReadRune()
	if err != nil {
		return end, next, err
	}
	switch r {
	case CharCtrlM, CharCtrlJ:
	case CharEscape, CharCtrlC:
		next = norm()
		return end, next, err
	case CharBackspace, CharCtrlH:
		if len(e.basic.buf) == 0 {
			next = norm()
			return
		}
		e.basic.delete(e.basic.pos-1, e.basic.pos)
	case CharCtrlB:
		e.basic.move(0)
	case CharCtrlE:
		e.basic.move(len(e.basic.buf))
	case CharCtrlN:
		e.historyForward()
	case CharCtrlP:
		e.historyBack()
	case CharCtrlU:
		e.basic.delete(0, e.basic.pos)
	case CharCtrlW:
		// FIXME: It's redundant.
		ed := newEditor()
		ed.pos = e.basic.pos
		ed.buf = e.basic.buf
		pos := ed.pos
		ed.wordBackward()
		e.basic.delete(pos, ed.pos)
		return
	default:
		e.basic.insert([]rune{r}, e.basic.pos)
	}
	if r != CharCtrlM && r != CharCtrlJ {
		return
	}
	next = norm()
	var candidate exCommand
	s := string(e.basic.buf)
	if s == "" {
		return
	}
	args := strings.Split(s, " ")
	s = args[0]
	args = args[1:]
	defer func() {
		e.history = append(e.history, e.basic.buf)
	}()
	for _, cmd := range exCommands {
		if !strings.HasPrefix(cmd.name, s) {
			continue
		}
		if cmd.name == s {
			end = cmd.fn(e, args)
			return
		}
		if candidate.name == "" {
			candidate = cmd
		}
	}
	if candidate.name != "" {
		end = candidate.fn(e, args)
		return
	}
	err = fmt.Errorf("not a command: %q", s)
	return
}

func (e *commandline) historyBack() {
	l := len(e.history)
	if l-e.age == 0 {
		return
	}
	e.age++
	e.basic.buf = e.history[l-e.age]
}

func (e *commandline) historyForward() {
	l := len(e.history)
	if e.age == 0 {
		return
	}
	e.age--
	e.basic.buf = e.history[l-e.age]
}

func (e *commandline) quit(args []string) continuity {
	return exit
}

func (e *commandline) delete(args []string) (_ continuity) {
	e.editor.delete(0, len(e.editor.buf))
	return
}

func (e *commandline) help(args []string) continuity {
	e.buf = []rune("help")
	e.pos = 4
	return execute
}

func (e *commandline) substitute(args []string) (_ continuity) {
	if len(args) != 2 {
		return
	}
	pat := args[0]
	s0 := args[1]
	s := strings.Replace(string(e.buf), pat, s0, -1)
	e.buf = []rune(s)
	return
}
<|endoftext|>


<commit_before>package editor

import (
	"fmt"
	"strings"

	"github.com/elpinal/coco3/screen"
)

type exCommand struct {
	name string
	fn   func(*commandline, []string) continuity
}

// exComands represents a table of Ex commands and corresponding functions.
// The order is important. Precede commands have higher precedence.
var exCommands = []exCommand{
	{"help", (*commandline).help},
	{"delete", (*commandline).delete},
	{"quit", (*commandline).quit},
}

type commandline struct {
	streamSet
	*editor

	basic *basic
}

func newCommandline(s streamSet, e *editor) *commandline {
	return &commandline{
		streamSet: s,
		editor:    e,
		basic:     &basic{},
	}
}

func (e *commandline) Mode() mode {
	return modeCommandline
}

func (e *commandline) Position() int {
	return e.basic.pos + 1
}

func (e *commandline) Runes() []rune {
	return e.buf
}

func (e *commandline) Message() []rune {
	return append([]rune{':'}, e.basic.buf...)
}

func (e *commandline) Highlight() *screen.Hi {
	return nil
}

func (e *commandline) Run() (end continuity, next modeChanger, err error) {
	r, _, err := e.streamSet.in.ReadRune()
	if err != nil {
		return end, next, err
	}
	switch r {
	case CharCtrlM, CharCtrlJ:
	case CharEscape, CharCtrlC:
		next = norm()
		return end, next, err
	case CharBackspace, CharCtrlH:
		if len(e.basic.buf) == 0 {
			next = norm()
			return
		}
		e.basic.delete(e.basic.pos-1, e.basic.pos)
	case CharCtrlB:
		e.basic.move(0)
	case CharCtrlE:
		e.basic.move(len(e.basic.buf))
	case CharCtrlU:
		e.basic.delete(0, e.basic.pos)
	case CharCtrlW:
		// FIXME: It's redundant.
		ed := newEditor()
		ed.pos = e.basic.pos
		ed.buf = e.basic.buf
		pos := ed.pos
		ed.wordBackward()
		e.basic.delete(pos, ed.pos)
		return
	default:
		e.basic.insert([]rune{r}, e.basic.pos)
	}
	if r != CharCtrlM && r != CharCtrlJ {
		return
	}
	next = norm()
	var candidate exCommand
	s := string(e.basic.buf)
	if s == "" {
		return
	}
	for _, cmd := range exCommands {
		if !strings.HasPrefix(cmd.name, s) {
			continue
		}
		if cmd.name == s {
			end = cmd.fn(e, nil)
			return
		}
		if candidate.name == "" {
			candidate = cmd
		}
	}
	if candidate.name != "" {
		end = candidate.fn(e, nil)
		return
	}
	err = fmt.Errorf("not a command: %q", s)
	return
}

func (e *commandline) quit(args []string) continuity {
	return exit
}

func (e *commandline) delete(args []string) (_ continuity) {
	e.editor.delete(0, len(e.editor.buf))
	return
}

func (e *commandline) help(args []string) continuity {
	e.buf = []rune("help")
	e.pos = 4
	return execute
}
<commit_msg>Add :substitute<commit_after>package editor

import (
	"fmt"
	"strings"

	"github.com/elpinal/coco3/screen"
)

type exCommand struct {
	name string
	fn   func(*commandline, []string) continuity
}

// exComands represents a table of Ex commands and corresponding functions.
// The order is important. Precede commands have higher precedence.
var exCommands = []exCommand{
	{"help", (*commandline).help},
	{"delete", (*commandline).delete},
	{"quit", (*commandline).quit},
}

type commandline struct {
	streamSet
	*editor

	basic *basic
}

func newCommandline(s streamSet, e *editor) *commandline {
	return &commandline{
		streamSet: s,
		editor:    e,
		basic:     &basic{},
	}
}

func (e *commandline) Mode() mode {
	return modeCommandline
}

func (e *commandline) Position() int {
	return e.basic.pos + 1
}

func (e *commandline) Runes() []rune {
	return e.buf
}

func (e *commandline) Message() []rune {
	return append([]rune{':'}, e.basic.buf...)
}

func (e *commandline) Highlight() *screen.Hi {
	return nil
}

func (e *commandline) Run() (end continuity, next modeChanger, err error) {
	r, _, err := e.streamSet.in.ReadRune()
	if err != nil {
		return end, next, err
	}
	switch r {
	case CharCtrlM, CharCtrlJ:
	case CharEscape, CharCtrlC:
		next = norm()
		return end, next, err
	case CharBackspace, CharCtrlH:
		if len(e.basic.buf) == 0 {
			next = norm()
			return
		}
		e.basic.delete(e.basic.pos-1, e.basic.pos)
	case CharCtrlB:
		e.basic.move(0)
	case CharCtrlE:
		e.basic.move(len(e.basic.buf))
	case CharCtrlU:
		e.basic.delete(0, e.basic.pos)
	case CharCtrlW:
		// FIXME: It's redundant.
		ed := newEditor()
		ed.pos = e.basic.pos
		ed.buf = e.basic.buf
		pos := ed.pos
		ed.wordBackward()
		e.basic.delete(pos, ed.pos)
		return
	default:
		e.basic.insert([]rune{r}, e.basic.pos)
	}
	if r != CharCtrlM && r != CharCtrlJ {
		return
	}
	next = norm()
	var candidate exCommand
	s := string(e.basic.buf)
	if s == "" {
		return
	}
	for _, cmd := range exCommands {
		if !strings.HasPrefix(cmd.name, s) {
			continue
		}
		if cmd.name == s {
			end = cmd.fn(e, nil)
			return
		}
		if candidate.name == "" {
			candidate = cmd
		}
	}
	if candidate.name != "" {
		end = candidate.fn(e, nil)
		return
	}
	err = fmt.Errorf("not a command: %q", s)
	return
}

func (e *commandline) quit(args []string) continuity {
	return exit
}

func (e *commandline) delete(args []string) (_ continuity) {
	e.editor.delete(0, len(e.editor.buf))
	return
}

func (e *commandline) help(args []string) continuity {
	e.buf = []rune("help")
	e.pos = 4
	return execute
}

func (e *commandline) substitute(args []string) (_ continuity) {
	return
}
<|endoftext|>