<commit_before>//
// Copyright (c) 2017 Jon Carlson.  All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
//

package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"github.com/joncrlsn/misc"
	"github.com/joncrlsn/pgutil"
	"sort"
	"strconv"
	"strings"
	"text/template"
)

var (
	columnSqlTemplate = initColumnSqlTemplate()
)

// Initializes the Sql template
func initColumnSqlTemplate() *template.Template {
	sql := `
SELECT table_schema
    , {{if eq $.DbSchema "*" }}table_schema || '.' || {{end}}table_name || '.' || column_name  AS compare_name
	, table_name
    , column_name
    , data_type
    , is_nullable
    , column_default
    , character_maximum_length
    , is_identity
    , identity_generation
	, substring(udt_name from 2) AS array_type
FROM information_schema.columns
WHERE is_updatable = 'YES'
{{if eq $.DbSchema "*" }}
AND table_schema NOT LIKE 'pg_%'
AND table_schema <> 'information_schema'
{{else}}
AND table_schema = '{{$.DbSchema}}'
{{end}}
ORDER BY compare_name ASC;
`
	t := template.New("ColumnSqlTmpl")
	template.Must(t.Parse(sql))
	return t
}

// ==================================
// Column Rows definition
// ==================================

// ColumnRows is a sortable slice of string maps
type ColumnRows []map[string]string

func (slice ColumnRows) Len() int {
	return len(slice)
}

func (slice ColumnRows) Less(i, j int) bool {
	return slice[i]["compare_name"] < slice[j]["compare_name"]
}

func (slice ColumnRows) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// ==================================
// ColumnSchema definition
// (implements Schema -- defined in pgdiff.go)
// ==================================

// ColumnSchema holds a slice of rows from one of the databases as well as
// a reference to the current row of data we're viewing.
type ColumnSchema struct {
	rows   ColumnRows
	rowNum int
	done   bool
}

// get returns the value from the current row for the given key
func (c *ColumnSchema) get(key string) string {
	if c.rowNum >= len(c.rows) {
		return ""
	}
	return c.rows[c.rowNum][key]
}

// NextRow increments the rowNum and tells you whether or not there are more
func (c *ColumnSchema) NextRow() bool {
	if c.rowNum >= len(c.rows)-1 {
		c.done = true
	}
	c.rowNum = c.rowNum + 1
	return !c.done
}

// Compare tells you, in one pass, whether or not the first row matches, is less than, or greater than the second row
func (c *ColumnSchema) Compare(obj interface{}) int {
	c2, ok := obj.(*ColumnSchema)
	if !ok {
		fmt.Println("Error!!!, Compare needs a ColumnSchema instance", c2)
	}

	val := misc.CompareStrings(c.get("compare_name"), c2.get("compare_name"))
	return val
}

// Add prints SQL to add the column
func (c *ColumnSchema) Add() {

	schema := dbInfo2.DbSchema
	if schema == "*" {
		schema = c.get("table_schema")
	}

	// Knowing the version of db2 would eliminate the need for this warning
	if c.get("is_identity") == "YES" {
		fmt.Println("-- WARNING: identity columns are not supported in PostgreSQL versions < 10.")
		fmt.Println("-- Attempting to create identity columns in earlier versions will probably result in errors.")
	}

	if c.get("data_type") == "character varying" {
		maxLength, valid := getMaxLength(c.get("character_maximum_length"))
		if !valid {
			fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s character varying", schema, c.get("table_name"), c.get("column_name"))
		} else {
			fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s character varying(%s)", schema, c.get("table_name"), c.get("column_name"), maxLength)
		}
	} else {
		dataType := c.get("data_type")
		//if c.get("data_type") == "ARRAY" {
			//fmt.Println("-- Note that adding of array data types are not yet generated properly.")
		//}
		if dataType == "ARRAY" {
			dataType = c.get("array_type")+"[]"
		}
		//fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s %s", schema, c.get("table_name"), c.get("column_name"), c.get("data_type"))
		fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s %s", schema, c.get("table_name"), c.get("column_name"), dataType)
	}

	if c.get("is_nullable") == "NO" {
		fmt.Printf(" NOT NULL")
	}
	if c.get("column_default") != "null" {
		fmt.Printf(" DEFAULT %s", c.get("column_default"))
	}
	// NOTE: there are more identity column sequence options according to the PostgreSQL
	// CREATE TABLE docs, but these do not appear to be available as of version 10.1
	if c.get("is_identity") == "YES" {
		fmt.Printf(" GENERATED %s AS IDENTITY", c.get("identity_generation"))
	}
	fmt.Printf(";\n")
}

// Drop prints SQL to drop the column
func (c *ColumnSchema) Drop() {
	// if dropping column
	fmt.Printf("ALTER TABLE %s.%s DROP COLUMN IF EXISTS %s;\n", c.get("table_schema"), c.get("table_name"), c.get("column_name"))
}

// Change handles the case where the table and column match, but the details do not
func (c *ColumnSchema) Change(obj interface{}) {
	c2, ok := obj.(*ColumnSchema)
	if !ok {
		fmt.Println("Error!!!, ColumnSchema.Change(obj) needs a ColumnSchema instance", c2)
	}

	// Adjust data type for array columns
	dataType1 := c.get("data_type")
	if dataType1 == "ARRAY" {
		dataType1 = c.get("array_type")+"[]"
	}
	dataType2 := c2.get("data_type")
	if dataType2 == "ARRAY" {
		dataType2 = c2.get("array_type")+"[]"
	}

	// Detect column type change (mostly varchar length, or number size increase)
	// (integer to/from bigint is OK)
	if dataType1 == dataType2 {
		if dataType1 == "character varying" {
			max1, max1Valid := getMaxLength(c.get("character_maximum_length"))
			max2, max2Valid := getMaxLength(c2.get("character_maximum_length"))
			if !max1Valid && !max2Valid {
				// Leave them alone, they both have undefined max lengths
			} else if (max1Valid || !max2Valid) && (max1 != c2.get("character_maximum_length")) {
				//if !max1Valid {
				//    fmt.Println("-- WARNING: varchar column has no maximum length.  Setting to 1024, which may result in data loss.")
				//}
				max1Int, err1 := strconv.Atoi(max1)
				check("converting string to int", err1)
				max2Int, err2 := strconv.Atoi(max2)
				check("converting string to int", err2)
				if max1Int < max2Int {
					fmt.Println("-- WARNING: The next statement will shorten a character varying column, which may result in data loss.")
				}
				fmt.Printf("-- max1Valid: %v  max2Valid: %v \n", max1Valid, max2Valid)
				fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s TYPE character varying(%s);\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), max1)
			}
		}
	}

	// Code and test a column change from integer to bigint
	if dataType1 != dataType2 {
		fmt.Printf("-- WARNING: This type change may not work well: (%s to %s).\n", dataType2, dataType1)
		if strings.HasPrefix(dataType1, "character") {
			max1, max1Valid := getMaxLength(c.get("character_maximum_length"))
			if !max1Valid {
				fmt.Println("-- WARNING: varchar column has no maximum length.  Setting to 1024")
			}
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s TYPE %s(%s);\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), dataType1, max1)
		} else {
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s TYPE %s;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), dataType1)
		}
	}

	// Detect column default change (or added, dropped)
	if c.get("column_default") == "null" {
		if c.get("column_default") != "null" {
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s DROP DEFAULT;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
		}
	} else if c.get("column_default") != c2.get("column_default") {
		fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s SET DEFAULT %s;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), c.get("column_default"))
	}

	// Detect identity column change
	// Save result to variable instead of printing because order for adding/removing
	// is_nullable affects identity columns
	var identitySql string
	if c.get("is_identity") != c2.get("is_identity") {
		// Knowing the version of db2 would eliminate the need for this warning
		fmt.Println("-- WARNING: identity columns are not supported in PostgreSQL versions < 10.")
		fmt.Println("-- Attempting to create identity columns in earlier versions will probably result in errors.")
		if c.get("is_identity") == "YES" {
			identitySql = fmt.Sprintf("ALTER TABLE \"%s\".\"%s\" ALTER COLUMN \"%s\" ADD GENERATED %s AS IDENTITY;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), c.get("identity_generation"))
		} else {
			identitySql = fmt.Sprintf("ALTER TABLE \"%s\".\"%s\" ALTER COLUMN \"%s\" DROP IDENTITY;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
		}
	}

	// Detect not-null and nullable change
	if c.get("is_nullable") != c2.get("is_nullable") {
		if c.get("is_nullable") == "YES" {
			if identitySql != "" {
				fmt.Printf(identitySql)
			}
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s DROP NOT NULL;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
		} else {
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s SET NOT NULL;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
			if identitySql != "" {
				fmt.Printf(identitySql)
			}
		}
	} else {
		if identitySql != "" {
			fmt.Printf(identitySql)
		}
	}
}

// ==================================
// Standalone Functions
// ==================================

// compareColumns outputs SQL to make the columns match between two databases or schemas
func compareColumns(conn1 *sql.DB, conn2 *sql.DB) {

	buf1 := new(bytes.Buffer)
	columnSqlTemplate.Execute(buf1, dbInfo1)

	buf2 := new(bytes.Buffer)
	columnSqlTemplate.Execute(buf2, dbInfo2)

	rowChan1, _ := pgutil.QueryStrings(conn1, buf1.String())
	rowChan2, _ := pgutil.QueryStrings(conn2, buf2.String())

	//rows1 := make([]map[string]string, 500)
	rows1 := make(ColumnRows, 0)
	for row := range rowChan1 {
		rows1 = append(rows1, row)
	}
	sort.Sort(rows1)

	//rows2 := make([]map[string]string, 500)
	rows2 := make(ColumnRows, 0)
	for row := range rowChan2 {
		rows2 = append(rows2, row)
	}
	sort.Sort(&rows2)

	// We have to explicitly type this as Schema here for some unknown reason
	var schema1 Schema = &ColumnSchema{rows: rows1, rowNum: -1}
	var schema2 Schema = &ColumnSchema{rows: rows2, rowNum: -1}

	// Compare the columns
	doDiff(schema1, schema2)
}

// getMaxLength returns the maximum length and whether or not it is valid
func getMaxLength(maxLength string) (string, bool) {

	if maxLength == "null" {
		// default to 1024
		return "1024", false
	}
	return maxLength, true
}

<commit_msg>Fix indentation for column.go<commit_after>//
// Copyright (c) 2017 Jon Carlson.  All rights reserved.
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.
//

package main

import (
	"bytes"
	"database/sql"
	"fmt"
	"github.com/joncrlsn/misc"
	"github.com/joncrlsn/pgutil"
	"sort"
	"strconv"
	"strings"
	"text/template"
)

var (
	columnSqlTemplate = initColumnSqlTemplate()
)

// Initializes the Sql template
func initColumnSqlTemplate() *template.Template {
	sql := `
SELECT table_schema
    , {{if eq $.DbSchema "*" }}table_schema || '.' || {{end}}table_name || '.' || column_name  AS compare_name
	, table_name
    , column_name
    , data_type
    , is_nullable
    , column_default
    , character_maximum_length
    , is_identity
    , identity_generation
    , substring(udt_name from 2) AS array_type
FROM information_schema.columns
WHERE is_updatable = 'YES'
{{if eq $.DbSchema "*" }}
AND table_schema NOT LIKE 'pg_%'
AND table_schema <> 'information_schema'
{{else}}
AND table_schema = '{{$.DbSchema}}'
{{end}}
ORDER BY compare_name ASC;
`
	t := template.New("ColumnSqlTmpl")
	template.Must(t.Parse(sql))
	return t
}

// ==================================
// Column Rows definition
// ==================================

// ColumnRows is a sortable slice of string maps
type ColumnRows []map[string]string

func (slice ColumnRows) Len() int {
	return len(slice)
}

func (slice ColumnRows) Less(i, j int) bool {
	return slice[i]["compare_name"] < slice[j]["compare_name"]
}

func (slice ColumnRows) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// ==================================
// ColumnSchema definition
// (implements Schema -- defined in pgdiff.go)
// ==================================

// ColumnSchema holds a slice of rows from one of the databases as well as
// a reference to the current row of data we're viewing.
type ColumnSchema struct {
	rows   ColumnRows
	rowNum int
	done   bool
}

// get returns the value from the current row for the given key
func (c *ColumnSchema) get(key string) string {
	if c.rowNum >= len(c.rows) {
		return ""
	}
	return c.rows[c.rowNum][key]
}

// NextRow increments the rowNum and tells you whether or not there are more
func (c *ColumnSchema) NextRow() bool {
	if c.rowNum >= len(c.rows)-1 {
		c.done = true
	}
	c.rowNum = c.rowNum + 1
	return !c.done
}

// Compare tells you, in one pass, whether or not the first row matches, is less than, or greater than the second row
func (c *ColumnSchema) Compare(obj interface{}) int {
	c2, ok := obj.(*ColumnSchema)
	if !ok {
		fmt.Println("Error!!!, Compare needs a ColumnSchema instance", c2)
	}

	val := misc.CompareStrings(c.get("compare_name"), c2.get("compare_name"))
	return val
}

// Add prints SQL to add the column
func (c *ColumnSchema) Add() {

	schema := dbInfo2.DbSchema
	if schema == "*" {
		schema = c.get("table_schema")
	}

	// Knowing the version of db2 would eliminate the need for this warning
	if c.get("is_identity") == "YES" {
		fmt.Println("-- WARNING: identity columns are not supported in PostgreSQL versions < 10.")
		fmt.Println("-- Attempting to create identity columns in earlier versions will probably result in errors.")
	}

	if c.get("data_type") == "character varying" {
		maxLength, valid := getMaxLength(c.get("character_maximum_length"))
		if !valid {
			fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s character varying", schema, c.get("table_name"), c.get("column_name"))
		} else {
			fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s character varying(%s)", schema, c.get("table_name"), c.get("column_name"), maxLength)
		}
	} else {
		dataType := c.get("data_type")
		//if c.get("data_type") == "ARRAY" {
			//fmt.Println("-- Note that adding of array data types are not yet generated properly.")
		//}
		if dataType == "ARRAY" {
			dataType = c.get("array_type")+"[]"
		}
		//fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s %s", schema, c.get("table_name"), c.get("column_name"), c.get("data_type"))
		fmt.Printf("ALTER TABLE %s.%s ADD COLUMN %s %s", schema, c.get("table_name"), c.get("column_name"), dataType)
	}

	if c.get("is_nullable") == "NO" {
		fmt.Printf(" NOT NULL")
	}
	if c.get("column_default") != "null" {
		fmt.Printf(" DEFAULT %s", c.get("column_default"))
	}
	// NOTE: there are more identity column sequence options according to the PostgreSQL
	// CREATE TABLE docs, but these do not appear to be available as of version 10.1
	if c.get("is_identity") == "YES" {
		fmt.Printf(" GENERATED %s AS IDENTITY", c.get("identity_generation"))
	}
	fmt.Printf(";\n")
}

// Drop prints SQL to drop the column
func (c *ColumnSchema) Drop() {
	// if dropping column
	fmt.Printf("ALTER TABLE %s.%s DROP COLUMN IF EXISTS %s;\n", c.get("table_schema"), c.get("table_name"), c.get("column_name"))
}

// Change handles the case where the table and column match, but the details do not
func (c *ColumnSchema) Change(obj interface{}) {
	c2, ok := obj.(*ColumnSchema)
	if !ok {
		fmt.Println("Error!!!, ColumnSchema.Change(obj) needs a ColumnSchema instance", c2)
	}

	// Adjust data type for array columns
	dataType1 := c.get("data_type")
	if dataType1 == "ARRAY" {
		dataType1 = c.get("array_type")+"[]"
	}
	dataType2 := c2.get("data_type")
	if dataType2 == "ARRAY" {
		dataType2 = c2.get("array_type")+"[]"
	}

	// Detect column type change (mostly varchar length, or number size increase)
	// (integer to/from bigint is OK)
	if dataType1 == dataType2 {
		if dataType1 == "character varying" {
			max1, max1Valid := getMaxLength(c.get("character_maximum_length"))
			max2, max2Valid := getMaxLength(c2.get("character_maximum_length"))
			if !max1Valid && !max2Valid {
				// Leave them alone, they both have undefined max lengths
			} else if (max1Valid || !max2Valid) && (max1 != c2.get("character_maximum_length")) {
				//if !max1Valid {
				//    fmt.Println("-- WARNING: varchar column has no maximum length.  Setting to 1024, which may result in data loss.")
				//}
				max1Int, err1 := strconv.Atoi(max1)
				check("converting string to int", err1)
				max2Int, err2 := strconv.Atoi(max2)
				check("converting string to int", err2)
				if max1Int < max2Int {
					fmt.Println("-- WARNING: The next statement will shorten a character varying column, which may result in data loss.")
				}
				fmt.Printf("-- max1Valid: %v  max2Valid: %v \n", max1Valid, max2Valid)
				fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s TYPE character varying(%s);\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), max1)
			}
		}
	}

	// Code and test a column change from integer to bigint
	if dataType1 != dataType2 {
		fmt.Printf("-- WARNING: This type change may not work well: (%s to %s).\n", dataType2, dataType1)
		if strings.HasPrefix(dataType1, "character") {
			max1, max1Valid := getMaxLength(c.get("character_maximum_length"))
			if !max1Valid {
				fmt.Println("-- WARNING: varchar column has no maximum length.  Setting to 1024")
			}
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s TYPE %s(%s);\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), dataType1, max1)
		} else {
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s TYPE %s;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), dataType1)
		}
	}

	// Detect column default change (or added, dropped)
	if c.get("column_default") == "null" {
		if c.get("column_default") != "null" {
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s DROP DEFAULT;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
		}
	} else if c.get("column_default") != c2.get("column_default") {
		fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s SET DEFAULT %s;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), c.get("column_default"))
	}

	// Detect identity column change
	// Save result to variable instead of printing because order for adding/removing
	// is_nullable affects identity columns
	var identitySql string
	if c.get("is_identity") != c2.get("is_identity") {
		// Knowing the version of db2 would eliminate the need for this warning
		fmt.Println("-- WARNING: identity columns are not supported in PostgreSQL versions < 10.")
		fmt.Println("-- Attempting to create identity columns in earlier versions will probably result in errors.")
		if c.get("is_identity") == "YES" {
			identitySql = fmt.Sprintf("ALTER TABLE \"%s\".\"%s\" ALTER COLUMN \"%s\" ADD GENERATED %s AS IDENTITY;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"), c.get("identity_generation"))
		} else {
			identitySql = fmt.Sprintf("ALTER TABLE \"%s\".\"%s\" ALTER COLUMN \"%s\" DROP IDENTITY;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
		}
	}

	// Detect not-null and nullable change
	if c.get("is_nullable") != c2.get("is_nullable") {
		if c.get("is_nullable") == "YES" {
			if identitySql != "" {
				fmt.Printf(identitySql)
			}
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s DROP NOT NULL;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
		} else {
			fmt.Printf("ALTER TABLE %s.%s ALTER COLUMN %s SET NOT NULL;\n", c2.get("table_schema"), c.get("table_name"), c.get("column_name"))
			if identitySql != "" {
				fmt.Printf(identitySql)
			}
		}
	} else {
		if identitySql != "" {
			fmt.Printf(identitySql)
		}
	}
}

// ==================================
// Standalone Functions
// ==================================

// compareColumns outputs SQL to make the columns match between two databases or schemas
func compareColumns(conn1 *sql.DB, conn2 *sql.DB) {

	buf1 := new(bytes.Buffer)
	columnSqlTemplate.Execute(buf1, dbInfo1)

	buf2 := new(bytes.Buffer)
	columnSqlTemplate.Execute(buf2, dbInfo2)

	rowChan1, _ := pgutil.QueryStrings(conn1, buf1.String())
	rowChan2, _ := pgutil.QueryStrings(conn2, buf2.String())

	//rows1 := make([]map[string]string, 500)
	rows1 := make(ColumnRows, 0)
	for row := range rowChan1 {
		rows1 = append(rows1, row)
	}
	sort.Sort(rows1)

	//rows2 := make([]map[string]string, 500)
	rows2 := make(ColumnRows, 0)
	for row := range rowChan2 {
		rows2 = append(rows2, row)
	}
	sort.Sort(&rows2)

	// We have to explicitly type this as Schema here for some unknown reason
	var schema1 Schema = &ColumnSchema{rows: rows1, rowNum: -1}
	var schema2 Schema = &ColumnSchema{rows: rows2, rowNum: -1}

	// Compare the columns
	doDiff(schema1, schema2)
}

// getMaxLength returns the maximum length and whether or not it is valid
func getMaxLength(maxLength string) (string, bool) {

	if maxLength == "null" {
		// default to 1024
		return "1024", false
	}
	return maxLength, true
}

<|endoftext|>


<commit_before>package command

import (
	//"bufio"
	"fmt"
	"log"
	"strings"
	//"text/scanner"
)

const (
	MOTD = iota
	USER = iota
	PASS = iota
	EXEC = iota
	ENAB = iota
	CONF = iota
)

type CmdClient interface {
	ConfigPath() string
	ConfigPathSet(path string)
	Sendln(msg string)
	SendlnNow(msg string)
	InputQuit()
	Output() chan<- string
}

type CmdFunc func(ctx ConfContext, node *CmdNode, line string, c CmdClient)

const (
	CMD_NONE = uint64(0)
	CMD_CONF = uint64(1 << 0)
)

type CmdNode struct {
	Path     string
	Desc     string
	MinLevel int
	Handler  CmdFunc
	Children []*CmdNode
	Options  uint64
}

type ConfNode struct {
	Path     string
	Value    []string
	Children []*ConfNode
}

func (n *ConfNode) ValueAdd(value string) error {
	for _, v := range n.Value {
		if v == value {
			return nil // already exists
		}
	}
	n.Value = append(n.Value, value) // append new
	return nil
}

func (n *ConfNode) ValueSet(value string) error {
	if len(n.Value) > 0 {
		n.Value[0] = value
	}
	n.Value = append(n.Value, value)
	return nil
}

func (n *ConfNode) Set(path, line string) (*ConfNode, error, bool) {

	expanded, err := cmdExpand(line, path)
	if err != nil {
		return nil, fmt.Errorf("ConfNode.Set error: %v", err), false
	}

	log.Printf("ConfNode.Set: line=[%v] path=[%v] expand=[%s]", line, path, expanded)

	labels := strings.Fields(expanded)
	size := len(labels)
	parent := n
	for i, label := range labels {
		child := parent.findChild(label)
		if child != nil {
			// found, search next
			parent = child
			continue
		}

		// not found

		for ; i < size-1; i++ {
			// intermmediate label
			label = labels[i]
			currPath := strings.Join(labels[:i+1], " ")
			newNode := &ConfNode{Path: currPath}
			parent.Children = append(parent.Children, newNode)
			parent = newNode
		}

		// last label
		label = labels[size-1]
		newNode := &ConfNode{Path: expanded}
		parent.Children = append(parent.Children, newNode)

		return newNode, nil, false
	}

	// existing node found

	return parent, nil, true
}

func (n *ConfNode) Get(path string) (*ConfNode, error) {

	labels := strings.Fields(path)
	parent := n
	for _, label := range labels {
		child := parent.findChild(label)
		if child != nil {
			// found, search next
			parent = child
			continue
		}

		// not found
		return nil, fmt.Errorf("ConfNode.Get not found")
	}

	// found

	return parent, nil
}

func (n *ConfNode) findChild(label string) *ConfNode {

	for _, c := range n.Children {
		last := LastToken(c.Path)
		if label == last {
			return c
		}
	}

	return nil
}

type ConfContext interface {
	CmdRoot() *CmdNode
	ConfRootCandidate() *ConfNode
	ConfRootActive() *ConfNode

	Hostname() string
}

/*
func firstToken(path string) string {
	// fixme with tokenizer
	return strings.Fields(path)[0]
}
*/

func LastToken(path string) string {
	// fixme with tokenizer
	f := strings.Fields(path)
	return f[len(f)-1]
}

func StripLastToken(path string) (string, string) {
	last := strings.LastIndexByte(path, ' ')
	return path[:last], path[last+1:]
}

func dumpChildren(node *CmdNode) string {
	str := ""
	for _, p := range node.Children {
		str += fmt.Sprintf(",%s", p.Path)
	}
	return str
}

func pushChild(node, child *CmdNode) {
	//log.Printf("pushChild: parent=[%s] child=[%s] before: [%v]", node.Path, child.Path, dumpChildren(node))
	node.Children = append(node.Children, child)
	//log.Printf("pushChild: parent=[%s] child=[%s] after: [%v]", node.Path, child.Path, dumpChildren(node))
}

func CmdInstall(root *CmdNode, opt uint64, path string, min int, cmd CmdFunc, desc string) {
	if err := cmdAdd(root, opt, path, min, cmd, desc); err != nil {
		log.Printf("cmdInstall: error %s", err)
	}
}

func cmdAdd(root *CmdNode, opt uint64, path string, min int, cmd CmdFunc, desc string) error {
	//log.Printf("cmdInstall: [%s]", path)

	labelList := strings.Fields(path)
	size := len(labelList)
	parent := root
	for i, label := range labelList {
		currPath := strings.Join(labelList[:i+1], " ")
		//log.Printf("cmdInstall: %d: curr=[%s] label=[%s]", i, currPath, label)
		child := findChild(parent, label)
		if child != nil {
			// found, search next
			//log.Printf("cmdInstall: found [%s]", currPath)
			parent = child
			continue
		}

		// not found

		//log.Printf("cmdInstall: new [%s]", currPath)

		for ; i < size-1; i++ {
			// intermmediate label
			label = labelList[i]
			currPath = strings.Join(labelList[:i+1], " ")
			//log.Printf("cmdInstall: %d: intermmediate curr=[%s] label=[%s]", i, currPath, label)
			newNode := &CmdNode{Path: currPath, MinLevel: EXEC, Options: opt}
			pushChild(parent, newNode)
			parent = newNode
		}

		// last label
		label = labelList[size-1]
		//log.Printf("cmdInstall: %d: leaf curr=[%s] label=[%s]", i, path, label)
		newNode := &CmdNode{Path: path, Desc: desc, MinLevel: min, Handler: cmd, Options: opt}
		pushChild(parent, newNode)

		// did this command create an unreachable location?
		if _, err := CmdFind(root, path, CONF); err != nil {
			return fmt.Errorf("root=[%s] cmd=[%s] created unreachable command node: %v", root.Path, path, err)
		}

		return nil
	}

	// command node found

	return fmt.Errorf("[%s] already exists", path)
}

func findChild(node *CmdNode, label string) *CmdNode {

	for _, c := range node.Children {
		last := LastToken(c.Path)
		//log.Printf("findChild: searching [%s] against (%s)[%s] under [%s]", label, last, c.Path, node.Path)
		if label == last {
			//log.Printf("findChild: found [%s] as [%s] under [%s]", label, c.Path, node.Path)
			return c
		}
	}

	//log.Printf("findChild: not found [%s] under [%s]", label, node.Path)

	return nil
}

func isConfigValueKeyword(str string) bool {
	return str[0] == '{' && str[len(str)-1] == '}'
}

func matchChildren(children []*CmdNode, label string) []*CmdNode {
	c := []*CmdNode{}

	for _, n := range children {
		last := LastToken(n.Path)
		if isConfigValueKeyword(last) {
			// these keywords match any label
			c = append(c, n)
			continue
		}
		if strings.HasPrefix(last, label) {
			c = append(c, n)
			continue
		}
	}

	return c
}

func CmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {

	/*
		var s scanner.Scanner
		s.Error = func(s *scanner.Scanner, msg string) {
			log.Printf("command scan error: %s [%s]", msg, path)
		}
		s.Init(strings.NewReader(path))
	*/

	tokens := strings.Fields(path)

	parent := root
	//for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
	for _, label := range tokens {
		//log.Printf("cmdFind: token: [%s]", s.TokenText())
		//label := s.TokenText()
		//log.Printf("cmdFind: token: [%s]", label)
		children := matchChildren(parent.Children, label)
		size := len(children)
		if size < 1 {
			return nil, fmt.Errorf("CmdFind: not found: [%s] under [%s]", label, parent.Path)
		}
		if size > 1 {
			return nil, fmt.Errorf("CmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
		}
		parent = children[0]
	}

	//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)

	return parent, nil
}

func cmdExpand(originalLine, commandFullPath string) (string, error) {
	lineFields := strings.Fields(originalLine)
	pathFields := strings.Fields(commandFullPath)

	lineLen := len(lineFields)
	pathLen := len(pathFields)

	if len(lineFields) != len(pathFields) {
		return "", fmt.Errorf("cmdExpand: length mismatch: line=%d path=%d", lineLen, pathLen)
	}

	for i, label := range pathFields {
		if label[0] == '{' {
			pathFields[i] = lineFields[i]
			continue
		}
	}

	return strings.Join(pathFields, " "), nil
}
<commit_msg>Clean-up.<commit_after>package command

import (
	//"bufio"
	"fmt"
	"log"
	"strings"
	//"text/scanner"
)

const (
	MOTD = iota
	USER = iota
	PASS = iota
	EXEC = iota
	ENAB = iota
	CONF = iota
)

type CmdClient interface {
	ConfigPath() string
	ConfigPathSet(path string)
	Sendln(msg string)
	SendlnNow(msg string)
	InputQuit()
	Output() chan<- string
}

type CmdFunc func(ctx ConfContext, node *CmdNode, line string, c CmdClient)

const (
	CMD_NONE = uint64(0)
	CMD_CONF = uint64(1 << 0)
)

type CmdNode struct {
	Path     string
	Desc     string
	MinLevel int
	Handler  CmdFunc
	Children []*CmdNode
	Options  uint64
}

type ConfNode struct {
	Path     string
	Value    []string
	Children []*ConfNode
}

func (n *ConfNode) ValueAdd(value string) error {
	for _, v := range n.Value {
		if v == value {
			return nil // already exists
		}
	}
	n.Value = append(n.Value, value) // append new
	return nil
}

func (n *ConfNode) ValueSet(value string) {
	n.Value = []string{value}
}

func (n *ConfNode) Set(path, line string) (*ConfNode, error, bool) {

	expanded, err := cmdExpand(line, path)
	if err != nil {
		return nil, fmt.Errorf("ConfNode.Set error: %v", err), false
	}

	log.Printf("ConfNode.Set: line=[%v] path=[%v] expand=[%s]", line, path, expanded)

	labels := strings.Fields(expanded)
	size := len(labels)
	parent := n
	for i, label := range labels {
		child := parent.findChild(label)
		if child != nil {
			// found, search next
			parent = child
			continue
		}

		// not found

		for ; i < size-1; i++ {
			// intermmediate label
			label = labels[i]
			currPath := strings.Join(labels[:i+1], " ")
			newNode := &ConfNode{Path: currPath}
			parent.Children = append(parent.Children, newNode)
			parent = newNode
		}

		// last label
		label = labels[size-1]
		newNode := &ConfNode{Path: expanded}
		parent.Children = append(parent.Children, newNode)

		return newNode, nil, false
	}

	// existing node found

	return parent, nil, true
}

func (n *ConfNode) Get(path string) (*ConfNode, error) {

	labels := strings.Fields(path)
	parent := n
	for _, label := range labels {
		child := parent.findChild(label)
		if child != nil {
			// found, search next
			parent = child
			continue
		}

		// not found
		return nil, fmt.Errorf("ConfNode.Get not found")
	}

	// found

	return parent, nil
}

func (n *ConfNode) findChild(label string) *ConfNode {

	for _, c := range n.Children {
		last := LastToken(c.Path)
		if label == last {
			return c
		}
	}

	return nil
}

type ConfContext interface {
	CmdRoot() *CmdNode
	ConfRootCandidate() *ConfNode
	ConfRootActive() *ConfNode

	Hostname() string
}

/*
func firstToken(path string) string {
	// fixme with tokenizer
	return strings.Fields(path)[0]
}
*/

func LastToken(path string) string {
	// fixme with tokenizer
	f := strings.Fields(path)
	return f[len(f)-1]
}

func StripLastToken(path string) (string, string) {
	last := strings.LastIndexByte(path, ' ')
	return path[:last], path[last+1:]
}

func dumpChildren(node *CmdNode) string {
	str := ""
	for _, p := range node.Children {
		str += fmt.Sprintf(",%s", p.Path)
	}
	return str
}

func pushChild(node, child *CmdNode) {
	//log.Printf("pushChild: parent=[%s] child=[%s] before: [%v]", node.Path, child.Path, dumpChildren(node))
	node.Children = append(node.Children, child)
	//log.Printf("pushChild: parent=[%s] child=[%s] after: [%v]", node.Path, child.Path, dumpChildren(node))
}

func CmdInstall(root *CmdNode, opt uint64, path string, min int, cmd CmdFunc, desc string) {
	if err := cmdAdd(root, opt, path, min, cmd, desc); err != nil {
		log.Printf("cmdInstall: error %s", err)
	}
}

func cmdAdd(root *CmdNode, opt uint64, path string, min int, cmd CmdFunc, desc string) error {
	//log.Printf("cmdInstall: [%s]", path)

	labelList := strings.Fields(path)
	size := len(labelList)
	parent := root
	for i, label := range labelList {
		currPath := strings.Join(labelList[:i+1], " ")
		//log.Printf("cmdInstall: %d: curr=[%s] label=[%s]", i, currPath, label)
		child := findChild(parent, label)
		if child != nil {
			// found, search next
			//log.Printf("cmdInstall: found [%s]", currPath)
			parent = child
			continue
		}

		// not found

		//log.Printf("cmdInstall: new [%s]", currPath)

		for ; i < size-1; i++ {
			// intermmediate label
			label = labelList[i]
			currPath = strings.Join(labelList[:i+1], " ")
			//log.Printf("cmdInstall: %d: intermmediate curr=[%s] label=[%s]", i, currPath, label)
			newNode := &CmdNode{Path: currPath, MinLevel: EXEC, Options: opt}
			pushChild(parent, newNode)
			parent = newNode
		}

		// last label
		label = labelList[size-1]
		//log.Printf("cmdInstall: %d: leaf curr=[%s] label=[%s]", i, path, label)
		newNode := &CmdNode{Path: path, Desc: desc, MinLevel: min, Handler: cmd, Options: opt}
		pushChild(parent, newNode)

		// did this command create an unreachable location?
		if _, err := CmdFind(root, path, CONF); err != nil {
			return fmt.Errorf("root=[%s] cmd=[%s] created unreachable command node: %v", root.Path, path, err)
		}

		return nil
	}

	// command node found

	return fmt.Errorf("[%s] already exists", path)
}

func findChild(node *CmdNode, label string) *CmdNode {

	for _, c := range node.Children {
		last := LastToken(c.Path)
		//log.Printf("findChild: searching [%s] against (%s)[%s] under [%s]", label, last, c.Path, node.Path)
		if label == last {
			//log.Printf("findChild: found [%s] as [%s] under [%s]", label, c.Path, node.Path)
			return c
		}
	}

	//log.Printf("findChild: not found [%s] under [%s]", label, node.Path)

	return nil
}

func isConfigValueKeyword(str string) bool {
	return str[0] == '{' && str[len(str)-1] == '}'
}

func matchChildren(children []*CmdNode, label string) []*CmdNode {
	c := []*CmdNode{}

	for _, n := range children {
		last := LastToken(n.Path)
		if isConfigValueKeyword(last) {
			// these keywords match any label
			c = append(c, n)
			continue
		}
		if strings.HasPrefix(last, label) {
			c = append(c, n)
			continue
		}
	}

	return c
}

func CmdFind(root *CmdNode, path string, level int) (*CmdNode, error) {

	/*
		var s scanner.Scanner
		s.Error = func(s *scanner.Scanner, msg string) {
			log.Printf("command scan error: %s [%s]", msg, path)
		}
		s.Init(strings.NewReader(path))
	*/

	tokens := strings.Fields(path)

	parent := root
	//for tok := s.Scan(); tok != scanner.EOF; tok = s.Scan() {
	for _, label := range tokens {
		//log.Printf("cmdFind: token: [%s]", s.TokenText())
		//label := s.TokenText()
		//log.Printf("cmdFind: token: [%s]", label)
		children := matchChildren(parent.Children, label)
		size := len(children)
		if size < 1 {
			return nil, fmt.Errorf("CmdFind: not found: [%s] under [%s]", label, parent.Path)
		}
		if size > 1 {
			return nil, fmt.Errorf("CmdFind: ambiguous: [%s] under [%s]", label, parent.Path)
		}
		parent = children[0]
	}

	//log.Printf("cmdFind: found [%s] as [%s]", path, parent.Path)

	return parent, nil
}

func cmdExpand(originalLine, commandFullPath string) (string, error) {
	lineFields := strings.Fields(originalLine)
	pathFields := strings.Fields(commandFullPath)

	lineLen := len(lineFields)
	pathLen := len(pathFields)

	if len(lineFields) != len(pathFields) {
		return "", fmt.Errorf("cmdExpand: length mismatch: line=%d path=%d", lineLen, pathLen)
	}

	for i, label := range pathFields {
		if label[0] == '{' {
			pathFields[i] = lineFields[i]
			continue
		}
	}

	return strings.Join(pathFields, " "), nil
}
<|endoftext|>


<commit_before>/**
 * @author Dmitry Vovk <dmitry.vovk@gmail.com>
 * @copyright 2014
 */
package conf

import (
	"log"
	"io/ioutil"
	"encoding/json"
	"errors"
	"regexp"
	"net"
	"strconv"
)

type Url struct {
	Source    string `json:"source"`
	Interface string `json:"interface"`
	Set       uint `json:"set"`
}

type UrlConfig map[string]Url

/**
 * Read and parse JSON config
 */
func ReadUrls(fileName *string) (UrlConfig, error) {
	log.Print("Reading config")
	file, err := ioutil.ReadFile(*fileName)
	if err != nil {
		log.Printf("Could not read config: %s", err)
		return UrlConfig{}, errors.New("Could not read config")
	}
	var config UrlConfig
	err = json.Unmarshal(file, &config)
	if err != nil {
		log.Printf("Could not parse config: %s", err)
		return UrlConfig{}, errors.New("Could not parse config")
	}
	log.Printf("Read %d records", len(config))
	if configValid(config) {
		return config, nil
	} else {
		return UrlConfig{}, errors.New("Config is not valid")
	}
}

/**
 * Check config for validity:
 * - urls match regexp pattern
 * - ip:port pairs are valid ip and port number
 * - interfaces exists
 */
func configValid(config UrlConfig) bool {
	var validPath = regexp.MustCompile(`^/[a-z0-9-]+$`)
	var ifaceNames map[string]bool
	interfaces, err := net.Interfaces()
	if err != nil {
		log.Printf("Could not get interfaces list: %s", err)
		return false
	}
	ifaceNames = make(map[string]bool, len(interfaces))
	for _, iface := range interfaces {
		ifaceNames[iface.Name] = true
	}
	for path, url := range config {
		if !validPath.MatchString(path) {
			log.Printf("Invalid path found: %s", path)
			return false
		}
		host, port, err := net.SplitHostPort(url.Source)
		if err != nil {
			log.Printf("Could not parse host:port source pair of path %s: %s", path, url.Source)
			return false
		}
		ipAddr := net.ParseIP(host)
		if ipAddr == nil {
			log.Printf("Invalid ip address in source %s: %s", path, host)
			return false
		}
		dPort, err := strconv.Atoi(port)
		if dPort == 0 || err != nil {
			log.Printf("Invalid port in source %s: %s", path, port)
			return false
		}
		if _, ok := ifaceNames[url.Interface]; !ok {
			log.Printf("Interface for source %s not found: %s", path, url.Interface)
			return false
		}
	}
	return true
}
<commit_msg>Move URL regexp into const<commit_after>/**
 * @author Dmitry Vovk <dmitry.vovk@gmail.com>
 * @copyright 2014
 */
package conf

import (
	"log"
	"io/ioutil"
	"encoding/json"
	"errors"
	"regexp"
	"net"
	"strconv"
)

type Url struct {
	Source    string `json:"source"`
	Interface string `json:"interface"`
	Set       uint `json:"set"`
}

type UrlConfig map[string]Url

const (
	VALID_PATH = `^/[a-z0-9-]+$`
)

/**
 * Read and parse JSON config
 */
func ReadUrls(fileName *string) (UrlConfig, error) {
	log.Print("Reading config")
	file, err := ioutil.ReadFile(*fileName)
	if err != nil {
		log.Printf("Could not read config: %s", err)
		return UrlConfig{}, errors.New("Could not read config")
	}
	var config UrlConfig
	err = json.Unmarshal(file, &config)
	if err != nil {
		log.Printf("Could not parse config: %s", err)
		return UrlConfig{}, errors.New("Could not parse config")
	}
	log.Printf("Read %d records", len(config))
	if configValid(config) {
		return config, nil
	} else {
		return UrlConfig{}, errors.New("Config is not valid")
	}
}

/**
 * Check config for validity:
 * - urls match regexp pattern
 * - ip:port pairs are valid ip and port number
 * - interfaces exists
 */
func configValid(config UrlConfig) bool {
	var validPath = regexp.MustCompile(VALID_PATH)
	var ifaceNames map[string]bool
	interfaces, err := net.Interfaces()
	if err != nil {
		log.Printf("Could not get interfaces list: %s", err)
		return false
	}
	ifaceNames = make(map[string]bool, len(interfaces))
	for _, iface := range interfaces {
		ifaceNames[iface.Name] = true
	}
	for path, url := range config {
		if !validPath.MatchString(path) {
			log.Printf("Invalid path found: %s", path)
			return false
		}
		host, port, err := net.SplitHostPort(url.Source)
		if err != nil {
			log.Printf("Could not parse host:port source pair of path %s: %s", path, url.Source)
			return false
		}
		ipAddr := net.ParseIP(host)
		if ipAddr == nil {
			log.Printf("Invalid ip address in source %s: %s", path, host)
			return false
		}
		dPort, err := strconv.Atoi(port)
		if dPort == 0 || err != nil {
			log.Printf("Invalid port in source %s: %s", path, port)
			return false
		}
		if _, ok := ifaceNames[url.Interface]; !ok {
			log.Printf("Interface for source %s not found: %s", path, url.Interface)
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/customdiff"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	appengine "google.golang.org/api/appengine/v1"
)

func resourceAppEngineApplication() *schema.Resource {
	return &schema.Resource{
		Create: resourceAppEngineApplicationCreate,
		Read:   resourceAppEngineApplicationRead,
		Update: resourceAppEngineApplicationUpdate,
		Delete: resourceAppEngineApplicationDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		CustomizeDiff: customdiff.All(
			appEngineApplicationLocationIDCustomizeDiff,
		),

		Schema: map[string]*schema.Schema{
			"project": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				ForceNew:     true,
				ValidateFunc: validateProjectID(),
			},
			"auth_domain": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"location_id": {
				Type:     schema.TypeString,
				Required: true,
				ValidateFunc: validation.StringInSlice([]string{
					"northamerica-northeast1",
					"us-central",
					"us-west2",
					"us-east1",
					"us-east4",
					"southamerica-east1",
					"europe-west",
					"europe-west2",
					"europe-west3",
					"asia-northeast1",
					"asia-south1",
					"australia-southeast1",
				}, false),
			},
			"serving_status": {
				Type:     schema.TypeString,
				Optional: true,
				ValidateFunc: validation.StringInSlice([]string{
					"UNSPECIFIED",
					"SERVING",
					"USER_DISABLED",
					"SYSTEM_DISABLED",
				}, false),
				Computed: true,
			},
			"feature_settings": {
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				MaxItems: 1,
				Elem:     appEngineApplicationFeatureSettingsResource(),
			},
			"name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"url_dispatch_rule": {
				Type:     schema.TypeList,
				Computed: true,
				Elem:     appEngineApplicationURLDispatchRuleResource(),
			},
			"code_bucket": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_hostname": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_bucket": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"gcr_domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func appEngineApplicationURLDispatchRuleResource() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			"domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"path": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"service": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func appEngineApplicationFeatureSettingsResource() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			"split_health_checks": {
				Type:     schema.TypeBool,
				Optional: true,
			},
		},
	}
}

func appEngineApplicationLocationIDCustomizeDiff(d *schema.ResourceDiff, meta interface{}) error {
	old, new := d.GetChange("location_id")
	if old != "" && old != new {
		return fmt.Errorf("Cannot change location_id once the resource is created.")
	}
	return nil
}

func resourceAppEngineApplicationCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}
	app, err := expandAppEngineApplication(d, project)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Creating App Engine App")
	op, err := config.clientAppEngine.Apps.Create(app).Do()
	if err != nil {
		return fmt.Errorf("Error creating App Engine application: %s", err.Error())
	}

	d.SetId(project)

	// Wait for the operation to complete
	waitErr := appEngineOperationWait(config.clientAppEngine, op, project, "App Engine app to create")
	if waitErr != nil {
		d.SetId("")
		return waitErr
	}
	log.Printf("[DEBUG] Created App Engine App")

	return resourceAppEngineApplicationRead(d, meta)
}

func resourceAppEngineApplicationRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()

	app, err := config.clientAppEngine.Apps.Get(pid).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("App Engine Application %q", pid))
	}
	d.Set("auth_domain", app.AuthDomain)
	d.Set("code_bucket", app.CodeBucket)
	d.Set("default_bucket", app.DefaultBucket)
	d.Set("default_hostname", app.DefaultHostname)
	d.Set("location_id", app.LocationId)
	d.Set("name", app.Name)
	d.Set("serving_status", app.ServingStatus)
	d.Set("project", pid)
	dispatchRules, err := flattenAppEngineApplicationDispatchRules(app.DispatchRules)
	if err != nil {
		return err
	}
	err = d.Set("url_dispatch_rule", dispatchRules)
	if err != nil {
		return fmt.Errorf("Error setting dispatch rules in state. This is a bug, please report it at https://github.com/terraform-providers/terraform-provider-google/issues. Error is:\n%s", err.Error())
	}
	featureSettings, err := flattenAppEngineApplicationFeatureSettings(app.FeatureSettings)
	if err != nil {
		return err
	}
	err = d.Set("feature_settings", featureSettings)
	if err != nil {
		return fmt.Errorf("Error setting feature settings in state. This is a bug, please report it at https://github.com/terraform-providers/terraform-provider-google/issues. Error is:\n%s", err.Error())
	}
	return nil
}

func resourceAppEngineApplicationUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()
	app, err := expandAppEngineApplication(d, pid)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Updating App Engine App")
	op, err := config.clientAppEngine.Apps.Patch(pid, app).UpdateMask("authDomain,servingStatus,featureSettings.splitHealthChecks").Do()
	if err != nil {
		return fmt.Errorf("Error updating App Engine application: %s", err.Error())
	}

	// Wait for the operation to complete
	waitErr := appEngineOperationWait(config.clientAppEngine, op, pid, "App Engine app to update")
	if waitErr != nil {
		return waitErr
	}
	log.Printf("[DEBUG] Updated App Engine App")

	return resourceAppEngineApplicationRead(d, meta)
}

func resourceAppEngineApplicationDelete(d *schema.ResourceData, meta interface{}) error {
	log.Println("[WARN] App Engine applications cannot be destroyed once created. The project must be deleted to delete the application.")
	return nil
}

func expandAppEngineApplication(d *schema.ResourceData, project string) (*appengine.Application, error) {
	result := &appengine.Application{
		AuthDomain:    d.Get("auth_domain").(string),
		LocationId:    d.Get("location_id").(string),
		Id:            project,
		GcrDomain:     d.Get("gcr_domain").(string),
		ServingStatus: d.Get("serving_status").(string),
	}
	featureSettings, err := expandAppEngineApplicationFeatureSettings(d)
	if err != nil {
		return nil, err
	}
	result.FeatureSettings = featureSettings
	return result, nil
}

func expandAppEngineApplicationFeatureSettings(d *schema.ResourceData) (*appengine.FeatureSettings, error) {
	blocks := d.Get("feature_settings").([]interface{})
	if len(blocks) < 1 {
		return nil, nil
	}
	return &appengine.FeatureSettings{
		SplitHealthChecks: d.Get("feature_settings.0.split_health_checks").(bool),
		// force send SplitHealthChecks, so if it's set to false it still gets disabled
		ForceSendFields: []string{"SplitHealthChecks"},
	}, nil
}

func flattenAppEngineApplicationFeatureSettings(settings *appengine.FeatureSettings) ([]map[string]interface{}, error) {
	if settings == nil {
		return []map[string]interface{}{}, nil
	}
	result := map[string]interface{}{
		"split_health_checks": settings.SplitHealthChecks,
	}
	return []map[string]interface{}{result}, nil
}

func flattenAppEngineApplicationDispatchRules(rules []*appengine.UrlDispatchRule) ([]map[string]interface{}, error) {
	results := make([]map[string]interface{}, 0, len(rules))
	for _, rule := range rules {
		results = append(results, map[string]interface{}{
			"domain":  rule.Domain,
			"path":    rule.Path,
			"service": rule.Service,
		})
	}
	return results, nil
}
<commit_msg>Remember to set gcr domain in AppEngine Read. (#1453)<commit_after>package google

import (
	"fmt"
	"log"

	"github.com/hashicorp/terraform/helper/customdiff"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
	appengine "google.golang.org/api/appengine/v1"
)

func resourceAppEngineApplication() *schema.Resource {
	return &schema.Resource{
		Create: resourceAppEngineApplicationCreate,
		Read:   resourceAppEngineApplicationRead,
		Update: resourceAppEngineApplicationUpdate,
		Delete: resourceAppEngineApplicationDelete,

		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		CustomizeDiff: customdiff.All(
			appEngineApplicationLocationIDCustomizeDiff,
		),

		Schema: map[string]*schema.Schema{
			"project": {
				Type:         schema.TypeString,
				Optional:     true,
				Computed:     true,
				ForceNew:     true,
				ValidateFunc: validateProjectID(),
			},
			"auth_domain": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
			},
			"location_id": {
				Type:     schema.TypeString,
				Required: true,
				ValidateFunc: validation.StringInSlice([]string{
					"northamerica-northeast1",
					"us-central",
					"us-west2",
					"us-east1",
					"us-east4",
					"southamerica-east1",
					"europe-west",
					"europe-west2",
					"europe-west3",
					"asia-northeast1",
					"asia-south1",
					"australia-southeast1",
				}, false),
			},
			"serving_status": {
				Type:     schema.TypeString,
				Optional: true,
				ValidateFunc: validation.StringInSlice([]string{
					"UNSPECIFIED",
					"SERVING",
					"USER_DISABLED",
					"SYSTEM_DISABLED",
				}, false),
				Computed: true,
			},
			"feature_settings": {
				Type:     schema.TypeList,
				Optional: true,
				Computed: true,
				MaxItems: 1,
				Elem:     appEngineApplicationFeatureSettingsResource(),
			},
			"name": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"url_dispatch_rule": {
				Type:     schema.TypeList,
				Computed: true,
				Elem:     appEngineApplicationURLDispatchRuleResource(),
			},
			"code_bucket": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_hostname": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"default_bucket": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"gcr_domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func appEngineApplicationURLDispatchRuleResource() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			"domain": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"path": {
				Type:     schema.TypeString,
				Computed: true,
			},
			"service": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func appEngineApplicationFeatureSettingsResource() *schema.Resource {
	return &schema.Resource{
		Schema: map[string]*schema.Schema{
			"split_health_checks": {
				Type:     schema.TypeBool,
				Optional: true,
			},
		},
	}
}

func appEngineApplicationLocationIDCustomizeDiff(d *schema.ResourceDiff, meta interface{}) error {
	old, new := d.GetChange("location_id")
	if old != "" && old != new {
		return fmt.Errorf("Cannot change location_id once the resource is created.")
	}
	return nil
}

func resourceAppEngineApplicationCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	project, err := getProject(d, config)
	if err != nil {
		return err
	}
	app, err := expandAppEngineApplication(d, project)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Creating App Engine App")
	op, err := config.clientAppEngine.Apps.Create(app).Do()
	if err != nil {
		return fmt.Errorf("Error creating App Engine application: %s", err.Error())
	}

	d.SetId(project)

	// Wait for the operation to complete
	waitErr := appEngineOperationWait(config.clientAppEngine, op, project, "App Engine app to create")
	if waitErr != nil {
		d.SetId("")
		return waitErr
	}
	log.Printf("[DEBUG] Created App Engine App")

	return resourceAppEngineApplicationRead(d, meta)
}

func resourceAppEngineApplicationRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()

	app, err := config.clientAppEngine.Apps.Get(pid).Do()
	if err != nil {
		return handleNotFoundError(err, d, fmt.Sprintf("App Engine Application %q", pid))
	}
	d.Set("auth_domain", app.AuthDomain)
	d.Set("code_bucket", app.CodeBucket)
	d.Set("default_bucket", app.DefaultBucket)
	d.Set("default_hostname", app.DefaultHostname)
	d.Set("location_id", app.LocationId)
	d.Set("name", app.Name)
	d.Set("serving_status", app.ServingStatus)
	d.Set("gcr_domain", app.GcrDomain)
	d.Set("project", pid)
	dispatchRules, err := flattenAppEngineApplicationDispatchRules(app.DispatchRules)
	if err != nil {
		return err
	}
	err = d.Set("url_dispatch_rule", dispatchRules)
	if err != nil {
		return fmt.Errorf("Error setting dispatch rules in state. This is a bug, please report it at https://github.com/terraform-providers/terraform-provider-google/issues. Error is:\n%s", err.Error())
	}
	featureSettings, err := flattenAppEngineApplicationFeatureSettings(app.FeatureSettings)
	if err != nil {
		return err
	}
	err = d.Set("feature_settings", featureSettings)
	if err != nil {
		return fmt.Errorf("Error setting feature settings in state. This is a bug, please report it at https://github.com/terraform-providers/terraform-provider-google/issues. Error is:\n%s", err.Error())
	}
	return nil
}

func resourceAppEngineApplicationUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)
	pid := d.Id()
	app, err := expandAppEngineApplication(d, pid)
	if err != nil {
		return err
	}
	log.Printf("[DEBUG] Updating App Engine App")
	op, err := config.clientAppEngine.Apps.Patch(pid, app).UpdateMask("authDomain,servingStatus,featureSettings.splitHealthChecks").Do()
	if err != nil {
		return fmt.Errorf("Error updating App Engine application: %s", err.Error())
	}

	// Wait for the operation to complete
	waitErr := appEngineOperationWait(config.clientAppEngine, op, pid, "App Engine app to update")
	if waitErr != nil {
		return waitErr
	}
	log.Printf("[DEBUG] Updated App Engine App")

	return resourceAppEngineApplicationRead(d, meta)
}

func resourceAppEngineApplicationDelete(d *schema.ResourceData, meta interface{}) error {
	log.Println("[WARN] App Engine applications cannot be destroyed once created. The project must be deleted to delete the application.")
	return nil
}

func expandAppEngineApplication(d *schema.ResourceData, project string) (*appengine.Application, error) {
	result := &appengine.Application{
		AuthDomain:    d.Get("auth_domain").(string),
		LocationId:    d.Get("location_id").(string),
		Id:            project,
		GcrDomain:     d.Get("gcr_domain").(string),
		ServingStatus: d.Get("serving_status").(string),
	}
	featureSettings, err := expandAppEngineApplicationFeatureSettings(d)
	if err != nil {
		return nil, err
	}
	result.FeatureSettings = featureSettings
	return result, nil
}

func expandAppEngineApplicationFeatureSettings(d *schema.ResourceData) (*appengine.FeatureSettings, error) {
	blocks := d.Get("feature_settings").([]interface{})
	if len(blocks) < 1 {
		return nil, nil
	}
	return &appengine.FeatureSettings{
		SplitHealthChecks: d.Get("feature_settings.0.split_health_checks").(bool),
		// force send SplitHealthChecks, so if it's set to false it still gets disabled
		ForceSendFields: []string{"SplitHealthChecks"},
	}, nil
}

func flattenAppEngineApplicationFeatureSettings(settings *appengine.FeatureSettings) ([]map[string]interface{}, error) {
	if settings == nil {
		return []map[string]interface{}{}, nil
	}
	result := map[string]interface{}{
		"split_health_checks": settings.SplitHealthChecks,
	}
	return []map[string]interface{}{result}, nil
}

func flattenAppEngineApplicationDispatchRules(rules []*appengine.UrlDispatchRule) ([]map[string]interface{}, error) {
	results := make([]map[string]interface{}, 0, len(rules))
	for _, rule := range rules {
		results = append(results, map[string]interface{}{
			"domain":  rule.Domain,
			"path":    rule.Path,
			"service": rule.Service,
		})
	}
	return results, nil
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package html provides functions for escaping and unescaping HTML text.
package html

import (
	"strings"
	"unicode/utf8"
)

// These replacements permit compatibility with old numeric entities that
// assumed Windows-1252 encoding.
// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference
var replacementTable = [...]rune{
	'\u20AC', // First entry is what 0x80 should be replaced with.
	'\u0081',
	'\u201A',
	'\u0192',
	'\u201E',
	'\u2026',
	'\u2020',
	'\u2021',
	'\u02C6',
	'\u2030',
	'\u0160',
	'\u2039',
	'\u0152',
	'\u008D',
	'\u017D',
	'\u008F',
	'\u0090',
	'\u2018',
	'\u2019',
	'\u201C',
	'\u201D',
	'\u2022',
	'\u2013',
	'\u2014',
	'\u02DC',
	'\u2122',
	'\u0161',
	'\u203A',
	'\u0153',
	'\u009D',
	'\u017E',
	'\u0178', // Last entry is 0x9F.
	// 0x00->'\uFFFD' is handled programmatically.
	// 0x0D->'\u000D' is a no-op.
}

// unescapeEntity reads an entity like "<" from b[src:] and writes the
// corresponding "<" to b[dst:], returning the incremented dst and src cursors.
// Precondition: b[src] == '&' && dst <= src.
func unescapeEntity(b []byte, dst, src int) (dst1, src1 int) {
	const attribute = false

	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference

	// i starts at 1 because we already know that s[0] == '&'.
	i, s := 1, b[src:]

	if len(s) <= 1 {
		b[dst] = b[src]
		return dst + 1, src + 1
	}

	if s[i] == '#' {
		if len(s) <= 3 { // We need to have at least "&#.".
			b[dst] = b[src]
			return dst + 1, src + 1
		}
		i++
		c := s[i]
		hex := false
		if c == 'x' || c == 'X' {
			hex = true
			i++
		}

		x := '\x00'
		for i < len(s) {
			c = s[i]
			i++
			if hex {
				if '0' <= c && c <= '9' {
					x = 16*x + rune(c) - '0'
					continue
				} else if 'a' <= c && c <= 'f' {
					x = 16*x + rune(c) - 'a' + 10
					continue
				} else if 'A' <= c && c <= 'F' {
					x = 16*x + rune(c) - 'A' + 10
					continue
				}
			} else if '0' <= c && c <= '9' {
				x = 10*x + rune(c) - '0'
				continue
			}
			if c != ';' {
				i--
			}
			break
		}

		if i <= 3 { // No characters matched.
			b[dst] = b[src]
			return dst + 1, src + 1
		}

		if 0x80 <= x && x <= 0x9F {
			// Replace characters from Windows-1252 with UTF-8 equivalents.
			x = replacementTable[x-0x80]
		} else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF {
			// Replace invalid characters with the replacement character.
			x = '\uFFFD'
		}

		return dst + utf8.EncodeRune(b[dst:], x), src + i
	}

	// Consume the maximum number of characters possible, with the
	// consumed characters matching one of the named references.

	for i < len(s) {
		c := s[i]
		i++
		// Lower-cased characters are more common in entities, so we check for them first.
		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
			continue
		}
		if c != ';' {
			i--
		}
		break
	}

	entityName := s[1:i]
	if len(entityName) == 0 {
		// No-op.
	} else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' {
		// No-op.
	} else if x := entity[string(entityName)]; x != 0 {
		return dst + utf8.EncodeRune(b[dst:], x), src + i
	} else if x := entity2[string(entityName)]; x[0] != 0 {
		dst1 := dst + utf8.EncodeRune(b[dst:], x[0])
		return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i
	} else if !attribute {
		maxLen := len(entityName) - 1
		if maxLen > longestEntityWithoutSemicolon {
			maxLen = longestEntityWithoutSemicolon
		}
		for j := maxLen; j > 1; j-- {
			if x := entity[string(entityName[:j])]; x != 0 {
				return dst + utf8.EncodeRune(b[dst:], x), src + j + 1
			}
		}
	}

	dst1, src1 = dst+i, src+i
	copy(b[dst:dst1], b[src:src1])
	return dst1, src1
}

var htmlEscaper = strings.NewReplacer(
	`&`, "&",
	`'`, "'", // "'" is shorter than "'" and apos was not in HTML until HTML5.
	`<`, "<",
	`>`, ">",
	`"`, """, // """ is shorter than """.
)

// EscapeString escapes special characters like "<" to become "<". It
// escapes only five such characters: <, >, &, ' and ".
// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
// always true.
func EscapeString(s string) string {
	return htmlEscaper.Replace(s)
}

// UnescapeString unescapes entities like "<" to become "<". It unescapes a
// larger range of entities than EscapeString escapes. For example, "á"
// unescapes to "á", as does "á" and "á".
// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
// always true.
func UnescapeString(s string) string {
	populateMapsOnce.Do(populateMaps)
	i := strings.IndexByte(s, '&')

	if i < 0 {
		return s
	}

	b := []byte(s)
	dst, src := unescapeEntity(b, i, i)
	for len(s[src:]) > 0 {
		if s[src] == '&' {
			i = 0
		} else {
			i = strings.IndexByte(s[src:], '&')
		}
		if i < 0 {
			dst += copy(b[dst:], s[src:])
			break
		}

		if i > 0 {
			copy(b[dst:], s[src:src+i])
		}
		dst, src = unescapeEntity(b, dst+i, src+i)
	}
	return string(b[:dst])
}
<commit_msg>html: update URL in comment<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package html provides functions for escaping and unescaping HTML text.
package html

import (
	"strings"
	"unicode/utf8"
)

// These replacements permit compatibility with old numeric entities that
// assumed Windows-1252 encoding.
// https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state
var replacementTable = [...]rune{
	'\u20AC', // First entry is what 0x80 should be replaced with.
	'\u0081',
	'\u201A',
	'\u0192',
	'\u201E',
	'\u2026',
	'\u2020',
	'\u2021',
	'\u02C6',
	'\u2030',
	'\u0160',
	'\u2039',
	'\u0152',
	'\u008D',
	'\u017D',
	'\u008F',
	'\u0090',
	'\u2018',
	'\u2019',
	'\u201C',
	'\u201D',
	'\u2022',
	'\u2013',
	'\u2014',
	'\u02DC',
	'\u2122',
	'\u0161',
	'\u203A',
	'\u0153',
	'\u009D',
	'\u017E',
	'\u0178', // Last entry is 0x9F.
	// 0x00->'\uFFFD' is handled programmatically.
	// 0x0D->'\u000D' is a no-op.
}

// unescapeEntity reads an entity like "<" from b[src:] and writes the
// corresponding "<" to b[dst:], returning the incremented dst and src cursors.
// Precondition: b[src] == '&' && dst <= src.
func unescapeEntity(b []byte, dst, src int) (dst1, src1 int) {
	const attribute = false

	// http://www.whatwg.org/specs/web-apps/current-work/multipage/tokenization.html#consume-a-character-reference

	// i starts at 1 because we already know that s[0] == '&'.
	i, s := 1, b[src:]

	if len(s) <= 1 {
		b[dst] = b[src]
		return dst + 1, src + 1
	}

	if s[i] == '#' {
		if len(s) <= 3 { // We need to have at least "&#.".
			b[dst] = b[src]
			return dst + 1, src + 1
		}
		i++
		c := s[i]
		hex := false
		if c == 'x' || c == 'X' {
			hex = true
			i++
		}

		x := '\x00'
		for i < len(s) {
			c = s[i]
			i++
			if hex {
				if '0' <= c && c <= '9' {
					x = 16*x + rune(c) - '0'
					continue
				} else if 'a' <= c && c <= 'f' {
					x = 16*x + rune(c) - 'a' + 10
					continue
				} else if 'A' <= c && c <= 'F' {
					x = 16*x + rune(c) - 'A' + 10
					continue
				}
			} else if '0' <= c && c <= '9' {
				x = 10*x + rune(c) - '0'
				continue
			}
			if c != ';' {
				i--
			}
			break
		}

		if i <= 3 { // No characters matched.
			b[dst] = b[src]
			return dst + 1, src + 1
		}

		if 0x80 <= x && x <= 0x9F {
			// Replace characters from Windows-1252 with UTF-8 equivalents.
			x = replacementTable[x-0x80]
		} else if x == 0 || (0xD800 <= x && x <= 0xDFFF) || x > 0x10FFFF {
			// Replace invalid characters with the replacement character.
			x = '\uFFFD'
		}

		return dst + utf8.EncodeRune(b[dst:], x), src + i
	}

	// Consume the maximum number of characters possible, with the
	// consumed characters matching one of the named references.

	for i < len(s) {
		c := s[i]
		i++
		// Lower-cased characters are more common in entities, so we check for them first.
		if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' {
			continue
		}
		if c != ';' {
			i--
		}
		break
	}

	entityName := s[1:i]
	if len(entityName) == 0 {
		// No-op.
	} else if attribute && entityName[len(entityName)-1] != ';' && len(s) > i && s[i] == '=' {
		// No-op.
	} else if x := entity[string(entityName)]; x != 0 {
		return dst + utf8.EncodeRune(b[dst:], x), src + i
	} else if x := entity2[string(entityName)]; x[0] != 0 {
		dst1 := dst + utf8.EncodeRune(b[dst:], x[0])
		return dst1 + utf8.EncodeRune(b[dst1:], x[1]), src + i
	} else if !attribute {
		maxLen := len(entityName) - 1
		if maxLen > longestEntityWithoutSemicolon {
			maxLen = longestEntityWithoutSemicolon
		}
		for j := maxLen; j > 1; j-- {
			if x := entity[string(entityName[:j])]; x != 0 {
				return dst + utf8.EncodeRune(b[dst:], x), src + j + 1
			}
		}
	}

	dst1, src1 = dst+i, src+i
	copy(b[dst:dst1], b[src:src1])
	return dst1, src1
}

var htmlEscaper = strings.NewReplacer(
	`&`, "&",
	`'`, "'", // "'" is shorter than "'" and apos was not in HTML until HTML5.
	`<`, "<",
	`>`, ">",
	`"`, """, // """ is shorter than """.
)

// EscapeString escapes special characters like "<" to become "<". It
// escapes only five such characters: <, >, &, ' and ".
// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
// always true.
func EscapeString(s string) string {
	return htmlEscaper.Replace(s)
}

// UnescapeString unescapes entities like "<" to become "<". It unescapes a
// larger range of entities than EscapeString escapes. For example, "á"
// unescapes to "á", as does "á" and "á".
// UnescapeString(EscapeString(s)) == s always holds, but the converse isn't
// always true.
func UnescapeString(s string) string {
	populateMapsOnce.Do(populateMaps)
	i := strings.IndexByte(s, '&')

	if i < 0 {
		return s
	}

	b := []byte(s)
	dst, src := unescapeEntity(b, i, i)
	for len(s[src:]) > 0 {
		if s[src] == '&' {
			i = 0
		} else {
			i = strings.IndexByte(s[src:], '&')
		}
		if i < 0 {
			dst += copy(b[dst:], s[src:])
			break
		}

		if i > 0 {
			copy(b[dst:], s[src:src+i])
		}
		dst, src = unescapeEntity(b, dst+i, src+i)
	}
	return string(b[:dst])
}
<|endoftext|>


<commit_before>package stripe

import (
	"net/url"
	"strconv"
)

type Customer struct {
	Id                string `json:"id"`
	Object            string `json:"object"`
  Livemode          bool   `json:"livemode"`
  Created           int64  `json:"created"`
  AccountBalance    int64  `json:"account_balance"`
  Currency          string `json:"currency"`
  DefaultCard       string `json:"default_card"`
  Delinquent        bool   `json:"delinquent"`
  Discount          Discount `json:"discount"`
  Email             string `json:"email"`
  //Subscription      Subscription `json:"subscription"`
}

// Delete deletes a customer.
//
// For more information: https://stripe.com/docs/api#delete_customer
func (c *Customer) Delete() (*DeleteResponse, error) {
  response := DeleteResponse{}
  err := delete("/customers/" + c.Id, nil, &response)
  return &response, err
}

// Update updates a customer.
//
// For more information: https://stripe.com/docs/api#update_customer
func (c *Customer) Update(params *CustomerParams) (*Customer, error) {
  values := url.Values{}
  parseCustomerParams(params, &values)
  err := post("/customers/" + c.Id, values, c)
  return c, err
}

// The CustomerClient is the receiver for most standard customer related endpoints.
type CustomerClient struct{}

// Create creates a customer.
//
// For more information: https://stripe.com/docs/api#create_customer
func (c *CustomerClient) Create(params *CustomerParams) (*Customer, error) {
  customer := Customer{}
  values := url.Values{}
  parseCustomerParams(params, &values)
  err := post("/customers", values, &customer)
  return &customer, err
}

// Retrieve loads a customer.
//
// For more information: https://stripe.com/docs/api#retrieve_customer
func (c *CustomerClient) Retrieve(id string) (*Customer, error) {
  customer := Customer{}
  err := get("/customers/" + id, nil, &customer)
  return &customer, err
}

// Update updates a customer.
//
// For more information: https://stripe.com/docs/api#update_customer
func (c *CustomerClient) Update(id string, params *CustomerParams) (*Customer, error) {
  customer := Customer{}
  values := url.Values{}
  parseCustomerParams(params, &values)
  err := post("/customers/" + id, values, &customer)
  return &customer, err
}

// Delete deletes a customer.
//
// For more information: https://stripe.com/docs/api#delete_customer
func (c *CustomerClient) Delete(id string) (*DeleteResponse, error) {
  response := DeleteResponse{}
  err := delete("/customers/" + id, nil, &response)
  return &response, err
}

// List lists the first 10 customers. It calls ListCount with 10 as the count
// and 0 as the offset, which are the defaults in the Stripe API.
//
// For more information: https://stripe.com/docs/api#list_customers
func (c *CustomerClient) List() ([]*Customer, error) {
	return c.ListCount(10, 0)
}

// ListCount lists `count` customers starting at `offset`.
//
// For more information: https://stripe.com/docs/api#list_customers
func (c *CustomerClient) ListCount(count, offset int) ([]*Customer, error) {
	type customers struct{ Data []*Customer }
	list := customers{}

	params := url.Values{
		"count"   : {strconv.Itoa(count)},
		"offset"  : {strconv.Itoa(offset)},
	}

	err := get("/customers", params, &list)
	return list.Data, err
}

// parseCustomerParams takes a pointer to a CustomerParams and a pointer to a url.Values,
// it iterates over everything in the CustomerParams struct and Adds what is there
// to the url.Values.
func parseCustomerParams(params *CustomerParams, values *url.Values) {

  // Use parseCardParams from cards.go to setup the card param
  if params.CardParams != nil {
    parseCardParams(params.CardParams, values, true)
  }

  if params.AccountBalance != 0 {
    values.Add("account_balance", strconv.Itoa(params.AccountBalance))
  }

  if params.Coupon != "" {
    values.Add("coupon", params.Coupon)
  }

  if params.Description != "" {
    values.Add("description", params.Description)
  }

  if params.Email != "" {
    values.Add("email", params.Email)
  }

  if params.Plan != "" {
    values.Add("plan", params.Plan)
  }

  if params.Quantity != 0 {
    values.Add("quantity", strconv.Itoa(params.Quantity))
  }

  if params.TrialEnd != 0 {
    values.Add("trial_end", strconv.Itoa(params.TrialEnd))
  }
}
<commit_msg>added Subscription to Customer<commit_after>package stripe

import (
	"net/url"
	"strconv"
)

type Customer struct {
	Id             string       `json:"id"`
	Object         string       `json:"object"`
	Livemode       bool         `json:"livemode"`
	Created        int64        `json:"created"`
	AccountBalance int64        `json:"account_balance"`
	Currency       string       `json:"currency"`
	DefaultCard    string       `json:"default_card"`
	Delinquent     bool         `json:"delinquent"`
	Discount       Discount     `json:"discount"`
	Email          string       `json:"email"`
	Subscription   Subscription `json:"subscription"`
}

// Delete deletes a customer.
//
// For more information: https://stripe.com/docs/api#delete_customer
func (c *Customer) Delete() (*DeleteResponse, error) {
	response := DeleteResponse{}
	err := delete("/customers/"+c.Id, nil, &response)
	return &response, err
}

// Update updates a customer.
//
// For more information: https://stripe.com/docs/api#update_customer
func (c *Customer) Update(params *CustomerParams) (*Customer, error) {
	values := url.Values{}
	parseCustomerParams(params, &values)
	err := post("/customers/"+c.Id, values, c)
	return c, err
}

// The CustomerClient is the receiver for most standard customer related endpoints.
type CustomerClient struct{}

// Create creates a customer.
//
// For more information: https://stripe.com/docs/api#create_customer
func (c *CustomerClient) Create(params *CustomerParams) (*Customer, error) {
	customer := Customer{}
	values := url.Values{}
	parseCustomerParams(params, &values)
	err := post("/customers", values, &customer)
	return &customer, err
}

// Retrieve loads a customer.
//
// For more information: https://stripe.com/docs/api#retrieve_customer
func (c *CustomerClient) Retrieve(id string) (*Customer, error) {
	customer := Customer{}
	err := get("/customers/"+id, nil, &customer)
	return &customer, err
}

// Update updates a customer.
//
// For more information: https://stripe.com/docs/api#update_customer
func (c *CustomerClient) Update(id string, params *CustomerParams) (*Customer, error) {
	customer := Customer{}
	values := url.Values{}
	parseCustomerParams(params, &values)
	err := post("/customers/"+id, values, &customer)
	return &customer, err
}

// Delete deletes a customer.
//
// For more information: https://stripe.com/docs/api#delete_customer
func (c *CustomerClient) Delete(id string) (*DeleteResponse, error) {
	response := DeleteResponse{}
	err := delete("/customers/"+id, nil, &response)
	return &response, err
}

// List lists the first 10 customers. It calls ListCount with 10 as the count
// and 0 as the offset, which are the defaults in the Stripe API.
//
// For more information: https://stripe.com/docs/api#list_customers
func (c *CustomerClient) List() ([]*Customer, error) {
	return c.ListCount(10, 0)
}

// ListCount lists `count` customers starting at `offset`.
//
// For more information: https://stripe.com/docs/api#list_customers
func (c *CustomerClient) ListCount(count, offset int) ([]*Customer, error) {
	type customers struct{ Data []*Customer }
	list := customers{}

	params := url.Values{
		"count":  {strconv.Itoa(count)},
		"offset": {strconv.Itoa(offset)},
	}

	err := get("/customers", params, &list)
	return list.Data, err
}

// parseCustomerParams takes a pointer to a CustomerParams and a pointer to a url.Values,
// it iterates over everything in the CustomerParams struct and Adds what is there
// to the url.Values.
func parseCustomerParams(params *CustomerParams, values *url.Values) {

	// Use parseCardParams from cards.go to setup the card param
	if params.CardParams != nil {
		parseCardParams(params.CardParams, values, true)
	}

	if params.AccountBalance != 0 {
		values.Add("account_balance", strconv.Itoa(params.AccountBalance))
	}

	if params.Coupon != "" {
		values.Add("coupon", params.Coupon)
	}

	if params.Description != "" {
		values.Add("description", params.Description)
	}

	if params.Email != "" {
		values.Add("email", params.Email)
	}

	if params.Plan != "" {
		values.Add("plan", params.Plan)
	}

	if params.Quantity != 0 {
		values.Add("quantity", strconv.Itoa(params.Quantity))
	}

	if params.TrialEnd != 0 {
		values.Add("trial_end", strconv.Itoa(params.TrialEnd))
	}
}
<|endoftext|>


<commit_before>package state

//
// DefinitionSelect postgres specific query for definitions
//
const DefinitionSelect = `
select td.definition_id                    as definitionid,
       td.adaptive_resource_allocation     as adaptiveresourceallocation,
       td.image                            as image,
       td.group_name                       as groupname,
       td.alias                            as alias,
       td.memory                           as memory,
       coalesce(td.command, '')            as command,
       coalesce(td.task_type, '')          as tasktype,
       env::TEXT                           as env,
       td.cpu                              as cpu,
       td.gpu                              as gpu,
       array_to_json('{""}'::TEXT[])::TEXT as tags,
       array_to_json('{}'::INT[])::TEXT    as ports
from (select * from task_def) td
`

//
// ListDefinitionsSQL postgres specific query for listing definitions
//
const ListDefinitionsSQL = DefinitionSelect + "\n%s %s limit $1 offset $2"

//
// GetDefinitionSQL postgres specific query for getting a single definition
//
const GetDefinitionSQL = DefinitionSelect + "\nwhere definition_id = $1"

//
// GetDefinitionByAliasSQL get definition by alias
//
const GetDefinitionByAliasSQL = DefinitionSelect + "\nwhere alias = $1"

const TaskResourcesSelectCommandSQL = `
SELECT cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_memory_used)) * 1.75 as int) as memory,
       cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_cpu_used)) * 1.25  as int)  as cpu
FROM (SELECT memory as max_memory_used, cpu as max_cpu_used
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '3 days'
           AND (exit_code = 137 or exit_reason = 'OOMKilled')
           AND engine = 'eks'
           AND definition_id = $1
           AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A
`

const TaskResourcesExecutorCountSQL = `
SELECT least(coalesce(cast((percentile_disc(0.99) within GROUP (ORDER BY A.executor_count)) as int), 25), 100) as executor_count
FROM (SELECT CASE
                 WHEN (exit_reason like '%Exception%')
                     THEN (spark_extension -> 'spark_submit_job_driver' -> 'num_executors')::int * 1.75
                 ELSE (spark_extension -> 'spark_submit_job_driver' -> 'num_executors')::int * 1
                 END as executor_count
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '24 hours'
           AND engine = 'eks-spark'
           AND definition_id = $1
           AND command_hash = $2
      LIMIT 30) A
`
const TaskResourcesDriverOOMSQL = `
SELECT (spark_extension -> 'driver_oom')::boolean AS driver_oom
FROM TASK
WHERE queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
  AND engine = 'eks-spark'
  AND definition_id = $1
  AND command_hash = $2
  AND exit_code = 137
  AND spark_extension ? 'driver_oom'
GROUP BY 1
`

const TaskIdempotenceKeyCheckSQL = `
SELECT run_id
FROM task
WHERE idempotence_key = $1 and (exit_code = 0 or exit_code is null) and queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
ORDER BY queued_at desc
LIMIT 1
`

const TaskResourcesExecutorOOMSQL = `
SELECT CASE WHEN A.c >= 1 THEN true::boolean ELSE false::boolean END
FROM (SELECT count(*) as c
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
           AND definition_id = $1
           AND command_hash = $2
		   AND engine = 'eks-spark'
           AND exit_code !=0
      LIMIT 30) A
`

const TaskResourcesExecutorNodeLifecycleSQL = `
SELECT CASE WHEN A.c >= 1 THEN 'ondemand' ELSE 'spot' END
FROM (SELECT count(*) as c
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '12 hour'
           AND definition_id = $1
           AND command_hash = $2
           AND exit_code !=0
      LIMIT 30) A
`

const TaskExecutionRuntimeCommandSQL = `
SELECT percentile_disc(0.95) within GROUP (ORDER BY A.minutes) as minutes
FROM (SELECT EXTRACT(epoch from finished_at - started_at) / 60 as minutes
      FROM TASK
      WHERE definition_id = $1
        AND exit_code = 0
        AND engine = 'eks'
        AND queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
        AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A
`

const ListFailingNodesSQL = `
SELECT instance_dns_name
FROM (
         SELECT instance_dns_name, count(*) as c
         FROM TASK
         WHERE (exit_code = 128 OR
                pod_events @> '[{"reason": "Failed"}]' OR
                pod_events @> '[{"reason": "FailedSync"}]' OR
                pod_events @> '[{"reason": "FailedCreatePodSandBox"}]' OR
                pod_events @> '[{"reason": "OutOfmemory"}]')
           AND engine = 'eks'
           AND queued_at >= NOW() - INTERVAL '1 HOURS'
           AND instance_dns_name like 'ip-%'
         GROUP BY 1
         order by 2 desc) AS all_nodes
WHERE c >= 5
`

const PodReAttemptRate = `
SELECT (multiple_attempts / (CASE WHEN single_attempts = 0 THEN 1 ELSE single_attempts END)) AS attempts
FROM (
      SELECT COUNT(CASE WHEN attempt_count <= 1 THEN 1 END) * 1.0 AS single_attempts,
             COUNT(CASE WHEN attempt_count > 1 THEN 1 END) * 1.0 AS multiple_attempts
      FROM task
      WHERE engine = 'eks' AND
            queued_at >= NOW() - INTERVAL '18 MINUTES' AND
            node_lifecycle = 'spot') A
`

//
// RunSelect postgres specific query for runs
//
const RunSelect = `
select t.run_id                          as runid,
       coalesce(t.definition_id, '')     as definitionid,
       coalesce(t.alias, '')             as alias,
       coalesce(t.image, '')             as image,
       coalesce(t.cluster_name, '')      as clustername,
       t.exit_code                       as exitcode,
       t.exit_reason                     as exitreason,
       coalesce(t.status, '')            as status,
       queued_at                         as queuedat,
       started_at                        as startedat,
       finished_at                       as finishedat,
       coalesce(t.instance_id, '')       as instanceid,
       coalesce(t.instance_dns_name, '') as instancednsname,
       coalesce(t.group_name, '')        as groupname,
       coalesce(t.task_type, '')         as tasktype,
       env::TEXT                         as env,
       command,
       memory,
       cpu,
       gpu,
       engine,
       ephemeral_storage                 as ephemeralstorage,
       node_lifecycle                    as nodelifecycle,
       pod_name                          as podname,
       namespace,
       max_cpu_used                      as maxcpuused,
       max_memory_used                   as maxmemoryused,
       pod_events::TEXT                  as podevents,
       command_hash                      as commandhash,
       cloudtrail_notifications::TEXT    as cloudtrailnotifications,
       coalesce(executable_id, '')       as executableid,
       coalesce(executable_type, '')     as executabletype,
       execution_request_custom::TEXT    as executionrequestcustom,
       cpu_limit                         as cpulimit,
       memory_limit                      as memorylimit,
       attempt_count                     as attemptcount,
       spawned_runs::TEXT                as spawnedruns,
       run_exceptions::TEXT              as runexceptions,
       active_deadline_seconds           as activedeadlineseconds,
       spark_extension::TEXT             as sparkextension,
       metrics_uri                       as metricsuri,
       description                       as description,
	   idempotence_key                   as idempotencekey
from task t
`

//
// ListRunsSQL postgres specific query for listing runs
//
const ListRunsSQL = RunSelect + "\n%s %s limit $1 offset $2"

//
// GetRunSQL postgres specific query for getting a single run
//
const GetRunSQL = RunSelect + "\nwhere run_id = $1"

const GetRunSQLByEMRJobId = RunSelect + "\nwhere spark_extension->>'emr_job_id' = $1"

//
// GetRunSQLForUpdate postgres specific query for getting a single run
// for update
//
const GetRunSQLForUpdate = GetRunSQL + " for update"

//
// GroupsSelect postgres specific query for getting existing definition
// group_names
//
const GroupsSelect = `
select distinct group_name from task_def
`

//
// TagsSelect postgres specific query for getting existing definition tags
//
const TagsSelect = `
select distinct text from tags
`

//
// ListGroupsSQL postgres specific query for listing definition group_names
//
const ListGroupsSQL = GroupsSelect + "\n%s order by group_name asc limit $1 offset $2"

//
// ListTagsSQL postgres specific query for listing definition tags
//
const ListTagsSQL = TagsSelect + "\n%s order by text asc limit $1 offset $2"

//
// WorkerSelect postgres specific query for workers
//
const WorkerSelect = `
  select
    worker_type        as workertype,
    count_per_instance as countperinstance,
    engine
  from worker
`

//
// ListWorkersSQL postgres specific query for listing workers
//
const ListWorkersSQL = WorkerSelect

const GetWorkerEngine = WorkerSelect + "\nwhere engine = $1"

//
// GetWorkerSQL postgres specific query for retrieving data for a specific
// worker type.
//
const GetWorkerSQL = WorkerSelect + "\nwhere worker_type = $1 and engine = $2"

//
// GetWorkerSQLForUpdate postgres specific query for retrieving data for a specific
// worker type; locks the row.
//
const GetWorkerSQLForUpdate = GetWorkerSQL + " for update"

// TemplateSelect selects a template
const TemplateSelect = `
SELECT
  template_id as templateid,
  template_name as templatename,
  version,
  schema,
  command_template as commandtemplate,
  adaptive_resource_allocation as adaptiveresourceallocation,
  image,
  memory,
  env::TEXT as env,
  privileged,
  cpu,
  gpu,
  defaults,
  coalesce(avatar_uri, '') as avataruri
FROM template
`

// ListTemplatesSQL postgres specific query for listing templates
const ListTemplatesSQL = TemplateSelect + "\n%s limit $1 offset $2"

// GetTemplateByIDSQL postgres specific query for getting a single template
const GetTemplateByIDSQL = TemplateSelect + "\nwhere template_id = $1"

// ListTemplatesLatestOnlySQL lists the latest version of each distinct
// template name.
const ListTemplatesLatestOnlySQL = `
  SELECT DISTINCT ON (template_name)
    template_id as templateid,
    template_name as templatename,
    version,
    schema,
    command_template as commandtemplate,
    adaptive_resource_allocation as adaptiveresourceallocation,
    image,
    memory,
    env::TEXT as env,
    privileged,
    cpu,
    gpu,
    defaults,
    coalesce(avatar_uri, '') as avataruri
  FROM template
  ORDER BY template_name, version DESC, template_id
  LIMIT $1 OFFSET $2
`

// GetTemplateLatestOnlySQL get the latest version of a specific template name.
const GetTemplateLatestOnlySQL = TemplateSelect + "\nWHERE template_name = $1 ORDER BY version DESC LIMIT 1;"
const GetTemplateByVersionSQL = TemplateSelect + "\nWHERE template_name = $1 AND version = $2 ORDER BY version DESC LIMIT 1;"
<commit_msg>optimize TaskIdempotenceKeyCheckSQL by using CTE<commit_after>package state

//
// DefinitionSelect postgres specific query for definitions
//
const DefinitionSelect = `
select td.definition_id                    as definitionid,
       td.adaptive_resource_allocation     as adaptiveresourceallocation,
       td.image                            as image,
       td.group_name                       as groupname,
       td.alias                            as alias,
       td.memory                           as memory,
       coalesce(td.command, '')            as command,
       coalesce(td.task_type, '')          as tasktype,
       env::TEXT                           as env,
       td.cpu                              as cpu,
       td.gpu                              as gpu,
       array_to_json('{""}'::TEXT[])::TEXT as tags,
       array_to_json('{}'::INT[])::TEXT    as ports
from (select * from task_def) td
`

//
// ListDefinitionsSQL postgres specific query for listing definitions
//
const ListDefinitionsSQL = DefinitionSelect + "\n%s %s limit $1 offset $2"

//
// GetDefinitionSQL postgres specific query for getting a single definition
//
const GetDefinitionSQL = DefinitionSelect + "\nwhere definition_id = $1"

//
// GetDefinitionByAliasSQL get definition by alias
//
const GetDefinitionByAliasSQL = DefinitionSelect + "\nwhere alias = $1"

const TaskResourcesSelectCommandSQL = `
SELECT cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_memory_used)) * 1.75 as int) as memory,
       cast((percentile_disc(0.99) within GROUP (ORDER BY A.max_cpu_used)) * 1.25  as int)  as cpu
FROM (SELECT memory as max_memory_used, cpu as max_cpu_used
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '3 days'
           AND (exit_code = 137 or exit_reason = 'OOMKilled')
           AND engine = 'eks'
           AND definition_id = $1
           AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A
`

const TaskResourcesExecutorCountSQL = `
SELECT least(coalesce(cast((percentile_disc(0.99) within GROUP (ORDER BY A.executor_count)) as int), 25), 100) as executor_count
FROM (SELECT CASE
                 WHEN (exit_reason like '%Exception%')
                     THEN (spark_extension -> 'spark_submit_job_driver' -> 'num_executors')::int * 1.75
                 ELSE (spark_extension -> 'spark_submit_job_driver' -> 'num_executors')::int * 1
                 END as executor_count
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '24 hours'
           AND engine = 'eks-spark'
           AND definition_id = $1
           AND command_hash = $2
      LIMIT 30) A
`
const TaskResourcesDriverOOMSQL = `
SELECT (spark_extension -> 'driver_oom')::boolean AS driver_oom
FROM TASK
WHERE queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
  AND engine = 'eks-spark'
  AND definition_id = $1
  AND command_hash = $2
  AND exit_code = 137
  AND spark_extension ? 'driver_oom'
GROUP BY 1
`

const TaskIdempotenceKeyCheckSQL = `
WITH runs as (
    SELECT run_id
    FROM task
    WHERE idempotence_key = $1
      and (exit_code = 0 or exit_code is null)
      and queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days')
SELECT run_id
FROM runs
LIMIT 1;
`

const TaskResourcesExecutorOOMSQL = `
SELECT CASE WHEN A.c >= 1 THEN true::boolean ELSE false::boolean END
FROM (SELECT count(*) as c
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
           AND definition_id = $1
           AND command_hash = $2
		   AND engine = 'eks-spark'
           AND exit_code !=0
      LIMIT 30) A
`

const TaskResourcesExecutorNodeLifecycleSQL = `
SELECT CASE WHEN A.c >= 1 THEN 'ondemand' ELSE 'spot' END
FROM (SELECT count(*) as c
      FROM TASK
      WHERE
           queued_at >= CURRENT_TIMESTAMP - INTERVAL '12 hour'
           AND definition_id = $1
           AND command_hash = $2
           AND exit_code !=0
      LIMIT 30) A
`

const TaskExecutionRuntimeCommandSQL = `
SELECT percentile_disc(0.95) within GROUP (ORDER BY A.minutes) as minutes
FROM (SELECT EXTRACT(epoch from finished_at - started_at) / 60 as minutes
      FROM TASK
      WHERE definition_id = $1
        AND exit_code = 0
        AND engine = 'eks'
        AND queued_at >= CURRENT_TIMESTAMP - INTERVAL '7 days'
        AND command_hash = (SELECT command_hash FROM task WHERE run_id = $2)
      LIMIT 30) A
`

const ListFailingNodesSQL = `
SELECT instance_dns_name
FROM (
         SELECT instance_dns_name, count(*) as c
         FROM TASK
         WHERE (exit_code = 128 OR
                pod_events @> '[{"reason": "Failed"}]' OR
                pod_events @> '[{"reason": "FailedSync"}]' OR
                pod_events @> '[{"reason": "FailedCreatePodSandBox"}]' OR
                pod_events @> '[{"reason": "OutOfmemory"}]')
           AND engine = 'eks'
           AND queued_at >= NOW() - INTERVAL '1 HOURS'
           AND instance_dns_name like 'ip-%'
         GROUP BY 1
         order by 2 desc) AS all_nodes
WHERE c >= 5
`

const PodReAttemptRate = `
SELECT (multiple_attempts / (CASE WHEN single_attempts = 0 THEN 1 ELSE single_attempts END)) AS attempts
FROM (
      SELECT COUNT(CASE WHEN attempt_count <= 1 THEN 1 END) * 1.0 AS single_attempts,
             COUNT(CASE WHEN attempt_count > 1 THEN 1 END) * 1.0 AS multiple_attempts
      FROM task
      WHERE engine = 'eks' AND
            queued_at >= NOW() - INTERVAL '18 MINUTES' AND
            node_lifecycle = 'spot') A
`

//
// RunSelect postgres specific query for runs
//
const RunSelect = `
select t.run_id                          as runid,
       coalesce(t.definition_id, '')     as definitionid,
       coalesce(t.alias, '')             as alias,
       coalesce(t.image, '')             as image,
       coalesce(t.cluster_name, '')      as clustername,
       t.exit_code                       as exitcode,
       t.exit_reason                     as exitreason,
       coalesce(t.status, '')            as status,
       queued_at                         as queuedat,
       started_at                        as startedat,
       finished_at                       as finishedat,
       coalesce(t.instance_id, '')       as instanceid,
       coalesce(t.instance_dns_name, '') as instancednsname,
       coalesce(t.group_name, '')        as groupname,
       coalesce(t.task_type, '')         as tasktype,
       env::TEXT                         as env,
       command,
       memory,
       cpu,
       gpu,
       engine,
       ephemeral_storage                 as ephemeralstorage,
       node_lifecycle                    as nodelifecycle,
       pod_name                          as podname,
       namespace,
       max_cpu_used                      as maxcpuused,
       max_memory_used                   as maxmemoryused,
       pod_events::TEXT                  as podevents,
       command_hash                      as commandhash,
       cloudtrail_notifications::TEXT    as cloudtrailnotifications,
       coalesce(executable_id, '')       as executableid,
       coalesce(executable_type, '')     as executabletype,
       execution_request_custom::TEXT    as executionrequestcustom,
       cpu_limit                         as cpulimit,
       memory_limit                      as memorylimit,
       attempt_count                     as attemptcount,
       spawned_runs::TEXT                as spawnedruns,
       run_exceptions::TEXT              as runexceptions,
       active_deadline_seconds           as activedeadlineseconds,
       spark_extension::TEXT             as sparkextension,
       metrics_uri                       as metricsuri,
       description                       as description,
	   idempotence_key                   as idempotencekey
from task t
`

//
// ListRunsSQL postgres specific query for listing runs
//
const ListRunsSQL = RunSelect + "\n%s %s limit $1 offset $2"

//
// GetRunSQL postgres specific query for getting a single run
//
const GetRunSQL = RunSelect + "\nwhere run_id = $1"

const GetRunSQLByEMRJobId = RunSelect + "\nwhere spark_extension->>'emr_job_id' = $1"

//
// GetRunSQLForUpdate postgres specific query for getting a single run
// for update
//
const GetRunSQLForUpdate = GetRunSQL + " for update"

//
// GroupsSelect postgres specific query for getting existing definition
// group_names
//
const GroupsSelect = `
select distinct group_name from task_def
`

//
// TagsSelect postgres specific query for getting existing definition tags
//
const TagsSelect = `
select distinct text from tags
`

//
// ListGroupsSQL postgres specific query for listing definition group_names
//
const ListGroupsSQL = GroupsSelect + "\n%s order by group_name asc limit $1 offset $2"

//
// ListTagsSQL postgres specific query for listing definition tags
//
const ListTagsSQL = TagsSelect + "\n%s order by text asc limit $1 offset $2"

//
// WorkerSelect postgres specific query for workers
//
const WorkerSelect = `
  select
    worker_type        as workertype,
    count_per_instance as countperinstance,
    engine
  from worker
`

//
// ListWorkersSQL postgres specific query for listing workers
//
const ListWorkersSQL = WorkerSelect

const GetWorkerEngine = WorkerSelect + "\nwhere engine = $1"

//
// GetWorkerSQL postgres specific query for retrieving data for a specific
// worker type.
//
const GetWorkerSQL = WorkerSelect + "\nwhere worker_type = $1 and engine = $2"

//
// GetWorkerSQLForUpdate postgres specific query for retrieving data for a specific
// worker type; locks the row.
//
const GetWorkerSQLForUpdate = GetWorkerSQL + " for update"

// TemplateSelect selects a template
const TemplateSelect = `
SELECT
  template_id as templateid,
  template_name as templatename,
  version,
  schema,
  command_template as commandtemplate,
  adaptive_resource_allocation as adaptiveresourceallocation,
  image,
  memory,
  env::TEXT as env,
  privileged,
  cpu,
  gpu,
  defaults,
  coalesce(avatar_uri, '') as avataruri
FROM template
`

// ListTemplatesSQL postgres specific query for listing templates
const ListTemplatesSQL = TemplateSelect + "\n%s limit $1 offset $2"

// GetTemplateByIDSQL postgres specific query for getting a single template
const GetTemplateByIDSQL = TemplateSelect + "\nwhere template_id = $1"

// ListTemplatesLatestOnlySQL lists the latest version of each distinct
// template name.
const ListTemplatesLatestOnlySQL = `
  SELECT DISTINCT ON (template_name)
    template_id as templateid,
    template_name as templatename,
    version,
    schema,
    command_template as commandtemplate,
    adaptive_resource_allocation as adaptiveresourceallocation,
    image,
    memory,
    env::TEXT as env,
    privileged,
    cpu,
    gpu,
    defaults,
    coalesce(avatar_uri, '') as avataruri
  FROM template
  ORDER BY template_name, version DESC, template_id
  LIMIT $1 OFFSET $2
`

// GetTemplateLatestOnlySQL get the latest version of a specific template name.
const GetTemplateLatestOnlySQL = TemplateSelect + "\nWHERE template_name = $1 ORDER BY version DESC LIMIT 1;"
const GetTemplateByVersionSQL = TemplateSelect + "\nWHERE template_name = $1 AND version = $2 ORDER BY version DESC LIMIT 1;"
<|endoftext|>