<commit_before>package sqlx

import (
	"database/sql"
	"encoding/json"
	"errors"
	"reflect"
	"strings"
	"time"
)

type DBI interface {
	NamedExec(query string, arg interface{}) (sql.Result, error)
	NamedQueryRow(query string, arg interface{}) *Row
	NamedQuery(query string, arg interface{}) (*Rows, error)
	QueryRowx(query string, args ...interface{}) *Row
	QueryRow(query string, args ...interface{}) *sql.Row
	Queryx(query string, args ...interface{}) (*Rows, error)
	ExecOne(query string, args ...interface{}) error
	Rebind(query string) string
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// Return the name of a Struct to tablename
func DefaultTableName(i interface{}) string {
	return strings.ToLower(reflect.TypeOf(reflect.Indirect(reflect.ValueOf(i)).Interface()).Name())
}

type Helper struct {
	DBI
}

type StructTable interface {
	TableName() string
	Validate() error
}

type SafeSelector map[string]interface{}

// expand expands the selector into a clause delimited by some space and a list of
// args to append into prepared statements
func expand(s map[string]interface{}, spacer string) (string, []interface{}) {
	args := []interface{}{}
	cnt := 0
	query := ""
	for key, value := range s {
		query += key
		query += "=?"
		if cnt != len(s)-1 {
			query += spacer
		}
		args = append(args, value)
		cnt += 1
	}
	return query, args
}

// extract takes in a struct object and extracts out the mapping
func extract(obj StructTable) (map[string]interface{}, error) {
	// Validate the schema.
	if err := obj.Validate(); err != nil {
		return nil, err
	}
	baseType := reflect.TypeOf(obj) // eg. Parameter
	items := map[string]interface{}{}
	for i := 0; i < baseType.NumField(); i++ {
		fieldName := baseType.Field(i).Name // eg. "Torsion"
		possiblyPtr := reflect.ValueOf(obj).FieldByName(fieldName)
		// possiblyPtr could also be a struct or pointer
		if possiblyPtr.Kind() == reflect.Struct {
			subMap, err := extract(possiblyPtr.Interface().(StructTable))
			if err != nil {
				return nil, err
			}
			for k, v := range subMap {
				items[k] = v
			}
			continue
		}
		if possiblyPtr.IsNil() {
			// pass
		} else {
			// we are not a nil pointer, then indirect would always work.
			fieldValue := reflect.Indirect(possiblyPtr)
			concreteValue := fieldValue.Interface()
			dbName, _ := parseTag(baseType.Field(i).Tag.Get("json"))
			// if tagOptions.Contains("nonzero") && isZeroValue(fieldValue) {
			// 	return nil, errors.New("Zero value found for tagged nonzero field:" + fieldName)
			// }
			switch item := concreteValue.(type) {
			default:
				items[dbName] = concreteValue
				// dbVals = append(dbVals, ":"+dbName)
			case time.Time:
				if item.IsZero() {
					items[dbName] = "NOW"
				} else {
					items[dbName] = concreteValue
				}
			}
		}
	}
	return items, nil
}

func LookupTag(obj StructTable, field string) string {
	b, ok := reflect.TypeOf(obj).FieldByName(field)
	if !ok {
		return ""
	}
	tagName, _ := parseTag(b.Tag.Get("json"))
	return tagName
}

/* START ripped from unexported std lib END */
type tagOptions string

func parseTag(tag string) (string, tagOptions) {
	if idx := strings.Index(tag, ","); idx != -1 {
		return tag[:idx], tagOptions(tag[idx+1:])
	}
	return tag, tagOptions("")
}

func (o tagOptions) Contains(optionName string) bool {
	if len(o) == 0 {
		return false
	}
	s := string(o)
	for s != "" {
		var next string
		i := strings.Index(s, ",")
		if i >= 0 {
			s, next = s[:i], s[i+1:]
		}
		if s == optionName {
			return true
		}
		s = next
	}
	return false
}

/* END ripped from unexported std lib END */

// MsiToStruct takes in a JSON serializable map[string]interface{} and converts
// it the actual object
func JsonToStruct(input map[string]interface{}, s StructTable) error {
	// YT: LOL
	b, err := json.Marshal(input)
	if err != nil {
		return err
	}
	return json.Unmarshal(b, s)
}

func MakeStructTable(input map[string]interface{}, obj StructTable) error {
	base := reflect.Indirect(reflect.ValueOf(obj))
	baseType := reflect.TypeOf(base.Interface())
	for k, v := range input {
		_, ok := baseType.FieldByName(k)
		if !ok {
			return errors.New("Bad input name: " + k)
		}
		fv := base.FieldByName(k)
		ptr := reflect.New(reflect.TypeOf(v))
		reflect.Indirect(ptr).Set(reflect.ValueOf(v))
		fv.Set(ptr)
	}
	return nil
}

// special insertion rules:
// 		if type is time.Time, and the value is a Zero Value, then CURRENT_TIMESTAMP will be inserted
//		if type is a Pointer, and its indirected value is nil, then it is omitted.
func (h *Helper) CreateObject(obj StructTable) error {
	msi, err := extract(obj)
	if err != nil {
		return err
	}
	dbKeys := []string{}
	dbVals := []interface{}{}
	for k, v := range msi {
		dbKeys = append(dbKeys, k)
		dbVals = append(dbVals, v)
	}
	query := "INSERT INTO " + obj.TableName()
	query += " ("
	for idx, key := range dbKeys {
		query += key
		if idx != len(dbKeys)-1 {
			query += ","
		}
	}
	query += ") VALUES ("
	for idx, _ := range dbKeys {
		query += "?"
		if idx != len(dbKeys)-1 {
			query += ","
		}
	}
	query += ")"
	query = h.Rebind(query)
	_, err = h.Exec(query, dbVals...)
	return err
}

// DeleteAll removes all rows in the table matching condition.
// If no matching row was deleted, then an error is returned.
func (h *Helper) DeleteAll(condition StructTable) error {
	tableName := condition.TableName()
	msi, err := extract(condition)
	if err != nil {
		return err
	}
	query := "DELETE FROM " + tableName
	query += " WHERE "
	where, args := expand(msi, " AND ")
	query += where
	query = h.Rebind(query)
	res, err := h.Exec(query, args...)
	if err != nil {
		return err
	}
	cnt, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if cnt == 0 {
		return errors.New("No row was deleted.")
	}
	return nil
}

func (h *Helper) buildQuery(condition StructTable, projection []string) (string, []interface{}, error) {
	tableName := condition.TableName()
	query := "SELECT "
	if len(projection) > 0 {
		for idx, p := range projection {
			query += p
			if idx != len(projection)-1 {
				query += ","
			}
		}
	} else {
		query += "*"
	}
	query += " FROM "
	query += tableName
	msi, err := extract(condition)
	if err != nil {
		return "", nil, err
	}
	args := []interface{}{}
	if len(msi) > 0 {
		query += " WHERE "
		var where string
		where, args = expand(msi, " AND ")
		query += where
	}
	return query, args, nil
}

// QueryOne returns a scanned object corresponding to the first row matching condition. For
// more complicated tasks such as pagination, etc. It's more sensible to build your own SQL.
// objPtr must be some pointer to a StructTable to receive the deserialized value. Projection
// should be json tags.
func (h *Helper) QueryOne(condition StructTable, objPtr StructTable, projection ...string) error {
	query, args, err := h.buildQuery(condition, projection)
	if err != nil {
		return err
	}
	query += " LIMIT 1"
	query = h.Rebind(query)
	return h.QueryRowx(query, args...).StructScan(objPtr)
}

// QueryRows returns a pointer to a sql.Rows object that can iterated over and scanned. Projection
// should be json tags.
func (h *Helper) QueryRows(condition StructTable, projection ...string) (*Rows, error) {
	query, args, err := h.buildQuery(condition, projection)
	if err != nil {
		return nil, err
	}
	query = h.Rebind(query)
	return h.Queryx(query, args...)
}

// UpdateAll updates rows matching condition with new values given by updates.
// If no matching row was updated, then an error is returned.
func (h *Helper) UpdateAll(update StructTable, condition StructTable) error {
	tableName := update.TableName()
	msi1, err := extract(update)
	if err != nil {
		return err
	}
	if len(msi1) == 0 {
		// nothing to update, all nil
		return nil
	}
	msi2, err := extract(condition)
	if err != nil {
		return err
	}
	query := "UPDATE " + tableName + " SET "
	expansion, args := expand(msi1, ",")
	query += expansion
	// all_args := append(args
	if len(msi2) > 0 {
		query += " WHERE "
		expansion2, args2 := expand(msi2, " AND ")
		query += expansion2
		args = append(args, args2...)
	}
	query = h.Rebind(query)
	res, err := h.Exec(query, args...)
	if err != nil {
		return err
	}
	cnt, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if cnt == 0 {
		return errors.New("No row was updated.")
	}
	return nil
}
<commit_msg>Make Extract and Expand public<commit_after>package sqlx

import (
	"database/sql"
	"encoding/json"
	"errors"
	"reflect"
	"strings"
	"time"
)

type DBI interface {
	NamedExec(query string, arg interface{}) (sql.Result, error)
	NamedQueryRow(query string, arg interface{}) *Row
	NamedQuery(query string, arg interface{}) (*Rows, error)
	QueryRowx(query string, args ...interface{}) *Row
	QueryRow(query string, args ...interface{}) *sql.Row
	Queryx(query string, args ...interface{}) (*Rows, error)
	ExecOne(query string, args ...interface{}) error
	Rebind(query string) string
	Exec(query string, args ...interface{}) (sql.Result, error)
}

// Return the name of a Struct to tablename
func DefaultTableName(i interface{}) string {
	return strings.ToLower(reflect.TypeOf(reflect.Indirect(reflect.ValueOf(i)).Interface()).Name())
}

type Helper struct {
	DBI
}

type StructTable interface {
	TableName() string
	Validate() error
}

type SafeSelector map[string]interface{}

// Expand expands the selector into a clause delimited by some space and a list of
// args to append into prepared statements
func Expand(s map[string]interface{}, spacer string) (string, []interface{}) {
	args := []interface{}{}
	cnt := 0
	query := ""
	for key, value := range s {
		query += key
		query += "=?"
		if cnt != len(s)-1 {
			query += spacer
		}
		args = append(args, value)
		cnt += 1
	}
	return query, args
}

// Extract takes in a struct object and extracts out the mapping
func Extract(obj StructTable) (map[string]interface{}, error) {
	// Validate the schema.
	if err := obj.Validate(); err != nil {
		return nil, err
	}
	baseType := reflect.TypeOf(obj) // eg. Parameter
	items := map[string]interface{}{}
	for i := 0; i < baseType.NumField(); i++ {
		fieldName := baseType.Field(i).Name // eg. "Torsion"
		possiblyPtr := reflect.ValueOf(obj).FieldByName(fieldName)
		// possiblyPtr could also be a struct or pointer
		if possiblyPtr.Kind() == reflect.Struct {
			subMap, err := Extract(possiblyPtr.Interface().(StructTable))
			if err != nil {
				return nil, err
			}
			for k, v := range subMap {
				items[k] = v
			}
			continue
		}
		if possiblyPtr.IsNil() {
			// pass
		} else {
			// we are not a nil pointer, then indirect would always work.
			fieldValue := reflect.Indirect(possiblyPtr)
			concreteValue := fieldValue.Interface()
			dbName, _ := parseTag(baseType.Field(i).Tag.Get("json"))
			// if tagOptions.Contains("nonzero") && isZeroValue(fieldValue) {
			// 	return nil, errors.New("Zero value found for tagged nonzero field:" + fieldName)
			// }
			switch item := concreteValue.(type) {
			default:
				items[dbName] = concreteValue
				// dbVals = append(dbVals, ":"+dbName)
			case time.Time:
				if item.IsZero() {
					items[dbName] = "NOW"
				} else {
					items[dbName] = concreteValue
				}
			}
		}
	}
	return items, nil
}

func LookupTag(obj StructTable, field string) string {
	b, ok := reflect.TypeOf(obj).FieldByName(field)
	if !ok {
		return ""
	}
	tagName, _ := parseTag(b.Tag.Get("json"))
	return tagName
}

/* START ripped from unexported std lib END */
type tagOptions string

func parseTag(tag string) (string, tagOptions) {
	if idx := strings.Index(tag, ","); idx != -1 {
		return tag[:idx], tagOptions(tag[idx+1:])
	}
	return tag, tagOptions("")
}

func (o tagOptions) Contains(optionName string) bool {
	if len(o) == 0 {
		return false
	}
	s := string(o)
	for s != "" {
		var next string
		i := strings.Index(s, ",")
		if i >= 0 {
			s, next = s[:i], s[i+1:]
		}
		if s == optionName {
			return true
		}
		s = next
	}
	return false
}

/* END ripped from unexported std lib END */

// MsiToStruct takes in a JSON serializable map[string]interface{} and converts
// it the actual object
func JsonToStruct(input map[string]interface{}, s StructTable) error {
	// YT: LOL
	b, err := json.Marshal(input)
	if err != nil {
		return err
	}
	return json.Unmarshal(b, s)
}

func MakeStructTable(input map[string]interface{}, obj StructTable) error {
	base := reflect.Indirect(reflect.ValueOf(obj))
	baseType := reflect.TypeOf(base.Interface())
	for k, v := range input {
		_, ok := baseType.FieldByName(k)
		if !ok {
			return errors.New("Bad input name: " + k)
		}
		fv := base.FieldByName(k)
		ptr := reflect.New(reflect.TypeOf(v))
		reflect.Indirect(ptr).Set(reflect.ValueOf(v))
		fv.Set(ptr)
	}
	return nil
}

// special insertion rules:
// 		if type is time.Time, and the value is a Zero Value, then CURRENT_TIMESTAMP will be inserted
//		if type is a Pointer, and its indirected value is nil, then it is omitted.
func (h *Helper) CreateObject(obj StructTable) error {
	msi, err := Extract(obj)
	if err != nil {
		return err
	}
	dbKeys := []string{}
	dbVals := []interface{}{}
	for k, v := range msi {
		dbKeys = append(dbKeys, k)
		dbVals = append(dbVals, v)
	}
	query := "INSERT INTO " + obj.TableName()
	query += " ("
	for idx, key := range dbKeys {
		query += key
		if idx != len(dbKeys)-1 {
			query += ","
		}
	}
	query += ") VALUES ("
	for idx, _ := range dbKeys {
		query += "?"
		if idx != len(dbKeys)-1 {
			query += ","
		}
	}
	query += ")"
	query = h.Rebind(query)
	_, err = h.Exec(query, dbVals...)
	return err
}

// DeleteAll removes all rows in the table matching condition.
// If no matching row was deleted, then an error is returned.
func (h *Helper) DeleteAll(condition StructTable) error {
	tableName := condition.TableName()
	msi, err := Extract(condition)
	if err != nil {
		return err
	}
	query := "DELETE FROM " + tableName
	query += " WHERE "
	where, args := Expand(msi, " AND ")
	query += where
	query = h.Rebind(query)
	res, err := h.Exec(query, args...)
	if err != nil {
		return err
	}
	cnt, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if cnt == 0 {
		return errors.New("No row was deleted.")
	}
	return nil
}

func (h *Helper) buildQuery(condition StructTable, projection []string) (string, []interface{}, error) {
	tableName := condition.TableName()
	query := "SELECT "
	if len(projection) > 0 {
		for idx, p := range projection {
			query += p
			if idx != len(projection)-1 {
				query += ","
			}
		}
	} else {
		query += "*"
	}
	query += " FROM "
	query += tableName
	msi, err := Extract(condition)
	if err != nil {
		return "", nil, err
	}
	args := []interface{}{}
	if len(msi) > 0 {
		query += " WHERE "
		var where string
		where, args = Expand(msi, " AND ")
		query += where
	}
	return query, args, nil
}

// QueryOne returns a scanned object corresponding to the first row matching condition. For
// more complicated tasks such as pagination, etc. It's more sensible to build your own SQL.
// objPtr must be some pointer to a StructTable to receive the deserialized value. Projection
// should be json tags.
func (h *Helper) QueryOne(condition StructTable, objPtr StructTable, projection ...string) error {
	query, args, err := h.buildQuery(condition, projection)
	if err != nil {
		return err
	}
	query += " LIMIT 1"
	query = h.Rebind(query)
	return h.QueryRowx(query, args...).StructScan(objPtr)
}

// QueryRows returns a pointer to a sql.Rows object that can iterated over and scanned. Projection
// should be json tags.
func (h *Helper) QueryRows(condition StructTable, projection ...string) (*Rows, error) {
	query, args, err := h.buildQuery(condition, projection)
	if err != nil {
		return nil, err
	}
	query = h.Rebind(query)
	return h.Queryx(query, args...)
}

// UpdateAll updates rows matching condition with new values given by updates.
// If no matching row was updated, then an error is returned.
func (h *Helper) UpdateAll(update StructTable, condition StructTable) error {
	tableName := update.TableName()
	msi1, err := Extract(update)
	if err != nil {
		return err
	}
	if len(msi1) == 0 {
		// nothing to update, all nil
		return nil
	}
	msi2, err := Extract(condition)
	if err != nil {
		return err
	}
	query := "UPDATE " + tableName + " SET "
	expansion, args := Expand(msi1, ",")
	query += expansion
	// all_args := append(args
	if len(msi2) > 0 {
		query += " WHERE "
		expansion2, args2 := Expand(msi2, " AND ")
		query += expansion2
		args = append(args, args2...)
	}
	query = h.Rebind(query)
	res, err := h.Exec(query, args...)
	if err != nil {
		return err
	}
	cnt, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if cnt == 0 {
		return errors.New("No row was updated.")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}

type HTTPRESP struct {
	URL string
	FD  io.Reader
}

const MaxOutstanding_URL = 80
const MaxOutstanding_RESP = 2
const DEBUG = 0

//Channels
var chan_urls = make(chan string, 10)         // buffered channel of strings
var chan_ioreaders = make(chan HTTPRESP, 10)  // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 10) // buffered channel of structs
//Semaphore Channels
var sem_URL = make(chan int, MaxOutstanding_URL)
var sem_RESP = make(chan int, MaxOutstanding_RESP)

func debugausgabe(msg string) {
	if DEBUG == 1 {

		fmt.Printf("\n%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url string) {
	debugausgabe("Fetcher starten")
	<-sem_URL     // Wait for active queue to drain.
	fetchURL(url) // May take a long time.
	sem_URL <- 1  // Done; enable next request to run.
}

func handleParser(a HTTPRESP) {
	debugausgabe("Parser starten")
	<-sem_RESP    // Wait for active queue to drain.
	parseHtml(a)  // May take a long time.
	sem_RESP <- 1 // Done; enable next request to run.
}

func starten() {
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_URL; i++ {
		sem_URL <- 1
	}
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_RESP; i++ {
		sem_RESP <- 1
	}
	for {
		work := <-chan_urls
		go handleFetcher(work) // Don't wait for handle to finish.
		work2 := <-chan_ioreaders
		go handleParser(work2)
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und gibt je eine Map mit Links und Wörtern zurück
func parseHtml(a HTTPRESP) {
	d := html.NewTokenizer(a.FD)
	var links map[string]int
	var words map[string]int
	links = make(map[string]int)
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		if tokenType == html.ErrorToken {
			//fmt.Printf("%v", links)
			//fmt.Printf("%v", words)
			//TODO CHANNEL nehmen für Datenaustausch
			chan_urlindexes <- URLINDEX{a.URL, words}
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			//Eine Map mit Links erstellen
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						//Link normalisieren
						url, err := url.Parse(element.Val)
						//debugausgabe(url.Scheme)
						if url.IsAbs() && err == nil && url.Scheme == "http" {
							//neuen Link zur Map hinzufügen und hochzählen
							links[element.Val] = links[element.Val] + 1
							if links[element.Val] == 1 {
								chan_urls <- element.Val //URL CHANNEL füllen
								debugausgabe(element.Val)
							}

						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data)
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// Gibt die komplette Seite zurück
func fetchURL(url string) {
	response, err := http.Get(url)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url, response.Body}
	return
}

func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data := <-chan_urlindexes
		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

func main() {

	start := time.Now()

	chan_urls <- "http://www.wikipedia.org"

	go save()
	starten()

	//TODO paralleles schreiben in db

	fmt.Println()
	fmt.Printf("Dauer : [%.2fs]\n", time.Since(start).Seconds())
}
<commit_msg>Buffer für URL Channel erhöht. Sonst sehr langsam, da sehr viele URLs in den Channel geschrieben werden und die Weiterverarbeitung ein lock hat sobald der URL Channel voll ist.<commit_after>package main

import (
	"bufio"
	"code.google.com/p/go.net/html" //Tokenizer für HTML
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"
	"time"
)

type URLINDEX struct {
	URL   string
	WORDS map[string]int
}

type HTTPRESP struct {
	URL string
	FD  io.Reader
}

const MaxOutstanding_URL = 100
const MaxOutstanding_RESP = 4
const DEBUG = 1

//Channels
var chan_urls = make(chan string, 100000)      // buffered channel of strings
var chan_ioreaders = make(chan HTTPRESP, 200)  // buffered channel of structs
var chan_urlindexes = make(chan URLINDEX, 100) // buffered channel of structs
//Semaphore Channels
var sem_URL = make(chan int, MaxOutstanding_URL)
var sem_RESP = make(chan int, MaxOutstanding_RESP)

func debugausgabe(msg string) {
	if DEBUG == 1 {

		fmt.Printf("\n%s DEBUG: %s\n", time.Now(), msg)
	}
}

func handleFetcher(url string) {
	//debugausgabe("Fetcher starten")
	<-sem_URL     // Wait for active queue to drain.
	fetchURL(url) // May take a long time.
	sem_URL <- 1  // Done; enable next request to run.
}

func handleParser(a HTTPRESP) {
	//debugausgabe("Parser starten")
	<-sem_RESP    // Wait for active queue to drain.
	parseHtml(a)  // May take a long time.
	sem_RESP <- 1 // Done; enable next request to run.
}

func starten() {
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_URL; i++ {
		sem_URL <- 1
	}
	//Initiales setzen der Ressourcen
	for i := 0; i < MaxOutstanding_RESP; i++ {
		sem_RESP <- 1
	}
	for {
		work := <-chan_urls
		go handleFetcher(work) // Don't wait for handle to finish.
		work2 := <-chan_ioreaders
		go handleParser(work2)
	}
}

// parseHTML bekommt eine komplette HTML Seite
// und gibt je eine Map mit Links und Wörtern zurück
func parseHtml(a HTTPRESP) {
	start := time.Now()
	d := html.NewTokenizer(a.FD)
	var links map[string]int
	var words map[string]int
	links = make(map[string]int)
	words = make(map[string]int)

	for {
		// token type
		tokenType := d.Next()
		if d.Err() != nil {
			fmt.Printf("TokenERR vorher: %v", d.Err())
		}

		if tokenType == html.ErrorToken {
			chan_urlindexes <- URLINDEX{a.URL, words}
			fmt.Printf("Parse-Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), a.URL)
			return
		}
		token := d.Token()
		switch tokenType {
		case html.StartTagToken: // <tag>
			//Eine Map mit Links erstellen
			if token.Data == "a" {
				for _, element := range token.Attr {
					if element.Key == "href" {
						//Link normalisieren
						url, err := url.Parse(element.Val)
						//debugausgabe(url.Scheme)
						if url.IsAbs() && err == nil && url.Scheme == "http" {
							//neuen Link zur Map hinzufügen und hochzählen
							links[element.Val] = links[element.Val] + 1
							if links[element.Val] == 1 {
								chan_urls <- element.Val //URL CHANNEL füllen
							}

						}
					}
				}
			}

		case html.TextToken: // text between start and end tag
			//Map mit Wörtern erstellen
			temp := strings.Fields(token.Data)
			for _, element := range temp {
				//TODO: einzelne Örter noch besser von Sonderzeichen trennen z.b. mit TRIM()
				words[element] = words[element] + 1
			}

			//fmt.Printf("%q\n", temp)
		case html.EndTagToken: // </tag>
		case html.SelfClosingTagToken: // <tag/>

		}
	}
}

// fetchURL Bekommt eine URL
// lädt die Seite herunter
// Gibt die komplette Seite zurück
func fetchURL(url string) {
	//start := time.Now()
	response, err := http.Get(url)
	if err != nil {
		fmt.Printf("Fehler: %s beim HTTP GET von: %s\n", err, url)
		return
	}
	//fmt.Printf("%T", response.Body)
	chan_ioreaders <- HTTPRESP{url, response.Body}
	//fmt.Printf("Dauer : [%.2fs]  URL: %s\n", time.Since(start).Seconds(), url)
	return
}

//save holt sich daten (structs vom Typ urlindexes) aus dem Channel chan_urlindexes
// und schreibt die Daten in eine Datei "output.txt"
func save() {

	//fo, err := os.OpenFile("output.txt", os.O_APPEND, 0777)
	fo, err := os.Create("output.txt")
	if err != nil {
		panic(err)
	}
	// close fo on exit and check for its returned error
	defer func() {
		if err := fo.Close(); err != nil {
			panic(err)
		}
	}()
	// make a write buffer
	w := bufio.NewWriter(fo)
	for {
		data := <-chan_urlindexes
		fmt.Fprintf(w, "\nURL: %s\nMAP:\n%v\n\n\n\n", data.URL, data.WORDS)
		w.Flush()
	}
}

func main() {

	chan_urls <- "http://www.ebay.com"

	go save()
	starten()

	//TODO paralleles schreiben in db
}
<|endoftext|>


<commit_before>package thermal

import (
	//"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/go.crypto/nacl/box"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"log"
)

// cs3a is an implementation of the NaCl based cipher set 3a
type cs3a struct {
	id             string
	fingerprintBin []byte
	fingerprintHex string
	publicKey      [32]byte
	privateKey     [32]byte
}

// initialize generates a key pair and sets up the cipher set
func (cs *cs3a) initialize() error {

	// generate the key pair
	publicKey, privateKey, err := box.GenerateKey(rand.Reader)
	if err != nil {
		log.Println("Error generating NaCl keypair in cs3a initialization")
		return err
	}

	// generate the fingerprint hash
	hash256 := sha256.New()
	hash256.Write(publicKey[:])
	fingerprintBin := hash256.Sum(nil)
	fingerprintHex := fmt.Sprintf("%x", fingerprintBin)

	// initialize the struct
	cs.id = "cs3a"
	cs.fingerprintBin = fingerprintBin
	cs.fingerprintHex = fingerprintHex
	cs.publicKey = *publicKey
	cs.privateKey = *privateKey

	return nil

}

func (cs *cs3a) String() string {
	return fmt.Sprintf("%s: %x", cs.id, cs.fingerprint)
}

func (cs *cs3a) csid() string {
	return cs.id
}

// fingerprint returns the csid and fingerprint for use in a 'parts' set
func (cs *cs3a) fingerprint() (string, string) {
	return cs.id, cs.fingerprintHex
}

func (cs *cs3a) encryptOpenPacket(packet []byte) (csdata []byte) {

	//------------------------------------------------------
	// Setup the keys
	//------------------------------------------------------
	// . existing switch key pair
	// . new line key pair?	// return?  store in local map?

	//------------------------------------------------------
	// Encrypt the inner packet
	//------------------------------------------------------
	// . 24 byte nonce of 0
	// nonce = make([]byte, 24)

	// . generate the shared secret
	// crypto_box(agreedKey, receiver.pubkey, sender.line.prvkey)

	// . use secretbox to encrypt the inner packet
	// encInnerPacket = crypto_secretbox(data, nonce, agreedKey)

	//------------------------------------------------------
	// Generate the hmac and assemble the outer packet
	//------------------------------------------------------
	// . <open-HMAC><sender-line-public-key><encrypted-inner-packet-data>
	// .		csdata.00-32  == auth - onetimeauth
	// .		csdata.33-64  == line_key - senders line level public key
	// .		csdata.rest   == inner_ciphertext (encrypted open packet)

	// . assemble part of the outer packet
	// outerPacketData = sender_linekey.public + encInnerPacket

	// . generate the macKey for use in generating the hmac
	// crypto_box(macKey, receiver.pubkey, sender.prvkey)

	// . generate the hmac
	// hmac = crypto_onetimeauth(encInnerPacket, macKey)
	// var hmac
	// crypt.poly1305.sum(hmac, encInnerPacket, macKey)

	// . assemble the rest of the outer packet BODY
	// csdata = hmac + outerPacketData
	//
	return csdata

}

func (cs *cs3a) decryptOpenPacket(csdata []byte) (packet []byte) {
	return packet
}

func (cs *cs3a) encryptLinePacket(packet []byte) (csdata []byte) {
	return csdata
}

func (cs *cs3a) decryptLinePacket(csdata []byte) (packet []byte) {
	return packet
}
<commit_msg>early shot at cs3a encryptOpenPacket<commit_after>package thermal

import (
	"code.google.com/p/go.crypto/nacl/box"
	"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/go.crypto/poly1305"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"log"
)

// cs3a is an implementation of the NaCl based cipher set 3a
type cs3a struct {
	id             string
	fingerprintBin []byte
	fingerprintHex string
	publicKey      [32]byte
	privateKey     [32]byte
}

// initialize generates a key pair and sets up the cipher set
func (cs *cs3a) initialize() error {

	// generate the key pair
	publicKey, privateKey, err := box.GenerateKey(rand.Reader)
	if err != nil {
		log.Println("Error generating NaCl keypair in cs3a initialization")
		return err
	}

	// generate the fingerprint hash
	hash256 := sha256.New()
	hash256.Write(publicKey[:])
	fingerprintBin := hash256.Sum(nil)
	fingerprintHex := fmt.Sprintf("%x", fingerprintBin)

	// initialize the struct
	cs.id = "cs3a"
	cs.fingerprintBin = fingerprintBin
	cs.fingerprintHex = fingerprintHex
	cs.publicKey = *publicKey
	cs.privateKey = *privateKey

	return nil

}

func (cs *cs3a) String() string {
	return fmt.Sprintf("%s: %x", cs.id, cs.fingerprint)
}

func (cs *cs3a) csid() string {
	return cs.id
}

// fingerprint returns the csid and fingerprint for use in a 'parts' set
func (cs *cs3a) fingerprint() (string, string) {
	return cs.id, cs.fingerprintHex
}

func (cs *cs3a) encryptOpenPacket(packet []byte, receiverPublicKey *[32]byte) (openPacketBody []byte, err error) {

	// switch key pair
	// cs.publicKey and cs.privateKey should already be populated

	// line key pair
	linePublicKey, linePrivateKey, err := box.GenerateKey(rand.Reader)
	if err != nil {
		log.Println("Error generating NaCl keypair for line")
		return openPacketBody, err
	}

	// todo - store the line key pair?  return it?

	// Encrypt the inner packet
	var nonce [24]byte
	var lineSharedKey [32]byte
	var encInnerPacket []byte

	box.Precompute(&lineSharedKey, receiverPublicKey, linePrivateKey)
	secretbox.Seal(encInnerPacket, packet, &nonce, &lineSharedKey)

	// Generate the mac and assemble the body for the outer packet
	// <mac><sender-line-public-key><encrypted-inner-packet-data>
	var macKey [32]byte
	var mac [16]byte
	var openPacketData []byte

	box.Precompute(&macKey, receiverPublicKey, &cs.privateKey)
	openPacketData = append(linePublicKey[:], encInnerPacket...)
	poly1305.Sum(&mac, openPacketData, &macKey)
	openPacketBody = append(mac[:], openPacketData...)

	return openPacketBody, nil

}

func (cs *cs3a) decryptOpenPacket(openPacketBody []byte) (packet []byte) {
	return packet
}

func (cs *cs3a) encryptLinePacket(packet []byte) (linePacketBody []byte) {
	return linePacketBody
}

func (cs *cs3a) decryptLinePacket(linePacketBody []byte) (packet []byte) {
	return packet
}
<|endoftext|>


<commit_before>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/tomasen/fcgi_client"
)

var (
	phpfpmUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "up"),
		"Whether scraping PHP-FPM's metrics was successful.",
		[]string{"socket_path"}, nil)

	phpfpmAcceptedConnections = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "accepted_connections_total"),
		"Number of request accepted by the pool.",
		[]string{"socket_path"}, nil)

	phpfpmStartTime = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "start_time_seconds"),
		"Unix time when FPM has started or reloaded.",
		[]string{"socket_path"}, nil)

	phpfpmGauges = map[string]*prometheus.Desc{
		"listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue"),
			"Number of request in the queue of pending connections.",
			[]string{"socket_path"}, nil),
		"max listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_listen_queue"),
			"Maximum number of requests in the queue of pending connections since FPM has started.",
			[]string{"socket_path"}, nil),
		"listen queue len": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue_length"),
			"The size of the socket queue of pending connections.",
			[]string{"socket_path"}, nil),
		"idle processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "idle_processes"),
			"Number of idle processes.",
			[]string{"socket_path"}, nil),
		"active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "active_processes"),
			"Number of active processes.",
			[]string{"socket_path"}, nil),
		"max active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_active_processes"),
			"Maximum number of active processes since FPM has started.",
			[]string{"socket_path"}, nil),
		"max children reached": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_children_reached"),
			"Number of times, the process limit has been reached.",
			[]string{"socket_path"}, nil),
		"slow requests": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "slow_requests"),
			"Enable php-fpm slow-log before you consider this. If this value is non-zero you may have slow php processes.",
			[]string{"socket_path"}, nil),
	}
)

// Converts the output of Dovecot's EXPORT command to metrics.
func CollectFromReader(reader io.Reader, socketPath string, ch chan<- prometheus.Metric) error {
	scanner := bufio.NewScanner(reader)
	re := regexp.MustCompile("^(.*): +(.*)$")

	// Scrape the interesting values:
	for scanner.Scan() {
		fields := re.FindStringSubmatch(scanner.Text())
		if fields == nil {
			return fmt.Errorf("Failed to parse %s", scanner.Text())
		}

		if gauge, ok := phpfpmGauges[fields[1]]; ok {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				gauge,
				prometheus.GaugeValue,
				f,
				socketPath)
		} else if fields[1] == "accepted conn" {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				phpfpmAcceptedConnections,
				prometheus.CounterValue,
				f,
				socketPath)
		} else if fields[1] == "start time" {
			location, err := time.LoadLocation("Local")
			if err != nil {
				return err
			}
			since, err := time.ParseInLocation("02/Jan/2006:15:04:05 -0700", fields[2], location)
			if err != nil {
				return err
			}
			f := float64(since.Unix())
			ch <- prometheus.MustNewConstMetric(
				phpfpmStartTime,
				prometheus.GaugeValue,
				f,
				socketPath)
		}
	}
	return nil
}

func CollectFromSocket(path string, scriptName string, ch chan<- prometheus.Metric) error {

	env := make(map[string]string)
	env["SCRIPT_FILENAME"] = scriptName
	env["SCRIPT_NAME"] = scriptName
	env["REQUEST_METHODGET"] = "GET"

	fcgi, err := fcgiclient.Dial("unix", path)
	if err != nil {
		return err
	}

	resp, err := fcgi.Get(env)
	if err != nil {
		return err
	}

	return CollectFromReader(resp.Body, path, ch)
}

type PhpfpmExporter struct {
	socketPaths []string
	scriptName  string
}

func NewPhpfpmExporter(socketPaths []string, scriptName string) (*PhpfpmExporter, error) {
	return &PhpfpmExporter{
		socketPaths: socketPaths,
		scriptName:  scriptName,
	}, nil
}

func (e *PhpfpmExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- phpfpmUpDesc
	ch <- phpfpmAcceptedConnections
	ch <- phpfpmStartTime
	for _, desc := range phpfpmGauges {
		ch <- desc
	}
}

func (e *PhpfpmExporter) Collect(ch chan<- prometheus.Metric) {
	for _, socketPath := range e.socketPaths {
		err := CollectFromSocket(socketPath, e.scriptName, ch)
		if err == nil {
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				1.0,
				socketPath)
		} else {
			log.Printf("Failed to scrape socket: %s", err)
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				0.0,
				socketPath)
		}
	}
}

func main() {
	var (
		listenAddress = flag.String("web.listen-address", ":9253", "Address to listen on for web interface and telemetry.")
		metricsPath   = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
		socketPaths   = flag.String("phpfpm.socket-paths", "", "Paths of the PHP-FPM sockets.")
		scriptName    = flag.String("phpfpm.scriptname", "/server-status-fpm.php", "Scriptname for fcgi socket communication.")
	)
	flag.Parse()

	exporter, err := NewPhpfpmExporter(strings.Split(*socketPaths, ","), *scriptName)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>PHP-FPM Exporter</title>

PHP-FPM Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<commit_msg>Rename variable and update description to default status url of phpfpm<commit_after>// Copyright 2017 Kumina, https://kumina.nl/
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/tomasen/fcgi_client"
)

var (
	phpfpmUpDesc = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "up"),
		"Whether scraping PHP-FPM's metrics was successful.",
		[]string{"socket_path"}, nil)

	phpfpmAcceptedConnections = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "accepted_connections_total"),
		"Number of request accepted by the pool.",
		[]string{"socket_path"}, nil)

	phpfpmStartTime = prometheus.NewDesc(
		prometheus.BuildFQName("php", "fpm", "start_time_seconds"),
		"Unix time when FPM has started or reloaded.",
		[]string{"socket_path"}, nil)

	phpfpmGauges = map[string]*prometheus.Desc{
		"listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue"),
			"Number of request in the queue of pending connections.",
			[]string{"socket_path"}, nil),
		"max listen queue": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_listen_queue"),
			"Maximum number of requests in the queue of pending connections since FPM has started.",
			[]string{"socket_path"}, nil),
		"listen queue len": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "listen_queue_length"),
			"The size of the socket queue of pending connections.",
			[]string{"socket_path"}, nil),
		"idle processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "idle_processes"),
			"Number of idle processes.",
			[]string{"socket_path"}, nil),
		"active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "active_processes"),
			"Number of active processes.",
			[]string{"socket_path"}, nil),
		"max active processes": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_active_processes"),
			"Maximum number of active processes since FPM has started.",
			[]string{"socket_path"}, nil),
		"max children reached": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "max_children_reached"),
			"Number of times, the process limit has been reached.",
			[]string{"socket_path"}, nil),
		"slow requests": prometheus.NewDesc(
			prometheus.BuildFQName("php", "fpm", "slow_requests"),
			"Enable php-fpm slow-log before you consider this. If this value is non-zero you may have slow php processes.",
			[]string{"socket_path"}, nil),
	}
)

// Converts the output of Dovecot's EXPORT command to metrics.
func CollectFromReader(reader io.Reader, socketPath string, ch chan<- prometheus.Metric) error {
	scanner := bufio.NewScanner(reader)
	re := regexp.MustCompile("^(.*): +(.*)$")

	// Scrape the interesting values:
	for scanner.Scan() {
		fields := re.FindStringSubmatch(scanner.Text())
		if fields == nil {
			return fmt.Errorf("Failed to parse %s", scanner.Text())
		}

		if gauge, ok := phpfpmGauges[fields[1]]; ok {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				gauge,
				prometheus.GaugeValue,
				f,
				socketPath)
		} else if fields[1] == "accepted conn" {
			f, err := strconv.ParseFloat(fields[2], 64)
			if err != nil {
				return err
			}
			ch <- prometheus.MustNewConstMetric(
				phpfpmAcceptedConnections,
				prometheus.CounterValue,
				f,
				socketPath)
		} else if fields[1] == "start time" {
			location, err := time.LoadLocation("Local")
			if err != nil {
				return err
			}
			since, err := time.ParseInLocation("02/Jan/2006:15:04:05 -0700", fields[2], location)
			if err != nil {
				return err
			}
			f := float64(since.Unix())
			ch <- prometheus.MustNewConstMetric(
				phpfpmStartTime,
				prometheus.GaugeValue,
				f,
				socketPath)
		}
	}
	return nil
}

func CollectFromSocket(path string, statusPath string, ch chan<- prometheus.Metric) error {

	env := make(map[string]string)
	env["SCRIPT_FILENAME"] = statusPath
	env["SCRIPT_NAME"] = statusPath
	env["REQUEST_METHODGET"] = "GET"

	fcgi, err := fcgiclient.Dial("unix", path)
	if err != nil {
		return err
	}

	resp, err := fcgi.Get(env)
	if err != nil {
		return err
	}

	return CollectFromReader(resp.Body, path, ch)
}

type PhpfpmExporter struct {
	socketPaths []string
	statusPath  string
}

func NewPhpfpmExporter(socketPaths []string, statusPath string) (*PhpfpmExporter, error) {
	return &PhpfpmExporter{
		socketPaths: socketPaths,
		statusPath:  statusPath,
	}, nil
}

func (e *PhpfpmExporter) Describe(ch chan<- *prometheus.Desc) {
	ch <- phpfpmUpDesc
	ch <- phpfpmAcceptedConnections
	ch <- phpfpmStartTime
	for _, desc := range phpfpmGauges {
		ch <- desc
	}
}

func (e *PhpfpmExporter) Collect(ch chan<- prometheus.Metric) {
	for _, socketPath := range e.socketPaths {
		err := CollectFromSocket(socketPath, e.statusPath, ch)
		if err == nil {
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				1.0,
				socketPath)
		} else {
			log.Printf("Failed to scrape socket: %s", err)
			ch <- prometheus.MustNewConstMetric(
				phpfpmUpDesc,
				prometheus.GaugeValue,
				0.0,
				socketPath)
		}
	}
}

func main() {
	var (
		listenAddress = flag.String("web.listen-address", ":9253", "Address to listen on for web interface and telemetry.")
		metricsPath   = flag.String("web.telemetry-path", "/metrics", "Path under which to expose metrics.")
		socketPaths   = flag.String("phpfpm.socket-paths", "", "Paths of the PHP-FPM sockets.")
		statusPath    = flag.String("phpfpm.statuspath", "/status", "Path which has been configured in PHP-FPM to show status page.")
	)
	flag.Parse()

	exporter, err := NewPhpfpmExporter(strings.Split(*socketPaths, ","), *statusPath)
	if err != nil {
		panic(err)
	}
	prometheus.MustRegister(exporter)

	http.Handle(*metricsPath, prometheus.Handler())
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`

			<title>PHP-FPM Exporter</title>

PHP-FPM Exporter

<a href='` + *metricsPath + `'>Metrics

			`))
	})
	log.Fatal(http.ListenAndServe(*listenAddress, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	. "github.com/Deleplace/programming-idioms/pig"

	"google.golang.org/appengine/log"
)

func implSave(w http.ResponseWriter, r *http.Request) error {
	idiomIDStr := r.FormValue("idiom_id")
	existingIDStr := r.FormValue("impl_id")
	username := r.FormValue("user_nickname")
	username = Truncate(username, 30)

	if !toggles["anonymousWrite"] {
		if username == "" {
			return PiError{"Username is mandatory. No anonymous edit.", http.StatusBadRequest}
		}
	}

	setNicknameCookie(w, username)

	if existingIDStr == "" {
		return newImplSave(w, r, username, idiomIDStr)
	}
	return existingImplSave(w, r, username, idiomIDStr, existingIDStr)
}

func newImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string) error {
	if err := togglesMissing(w, r, "implAddition"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_language"); err != nil {
		return err
	}

	c := r.Context()
	language := NormLang(r.FormValue("impl_language"))
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")
	editSummary := fmt.Sprintf("New %v implementation by user [%v]", language, username)

	imports = Truncate(imports, 200)
	code = TruncateBytes(NoCR(code), 500)
	comment = TruncateBytes(comment, 500)
	attributionURL = Truncate(attributionURL, 250)
	demoURL = Truncate(demoURL, 250)
	docURL = Truncate(docURL, 250)

	log.Infof(c, "[%v] is creating new %v impl for idiom %v", username, language, idiomIDStr)

	if !StringSliceContains(AllLanguages(), language) {
		return PiError{fmt.Sprintf("Sorry, [%v] is currently not a supported language. Supported languages are %v.", r.FormValue("impl_language"), AllNiceLangs), http.StatusBadRequest}
	}

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	implID, err := dao.nextImplID(c)
	if err != nil {
		return err
	}
	now := time.Now()
	newImpl := Impl{
		Id:                     implID,
		OrigId:                 implID,
		Author:                 username,
		CreationDate:           now,
		LastEditor:             username,
		LanguageName:           language,
		ImportsBlock:           imports,
		CodeBlock:              code,
		AuthorComment:          comment,
		OriginalAttributionURL: attributionURL,
		DemoURL:                demoURL,
		DocumentationURL:       docURL,
		Version:                1,
		VersionDate:            now,
	}

	if IsAdmin(r) {
		// 2016-10: only Admin may set an impl picture
		newImpl.PictureURL = r.FormValue("impl_picture_url")
	}

	idiom.Implementations = append(idiom.Implementations, newImpl)
	idiom.EditSummary = editSummary
	idiom.LastEditedImplID = implID

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, language), http.StatusFound)
	return nil
}

func existingImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string, existingImplIDStr string) error {
	if err := togglesMissing(w, r, "implEditing"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_version"); err != nil {
		return err
	}

	c := r.Context()
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")

	imports = Truncate(imports, 200)
	code = TruncateBytes(NoCR(code), 500)
	comment = TruncateBytes(comment, 500)
	attributionURL = Truncate(attributionURL, 250)
	demoURL = Truncate(demoURL, 250)
	docURL = Truncate(docURL, 250)

	log.Infof(c, "[%v] is updating impl %v of idiom %v", username, existingImplIDStr, idiomIDStr)

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find implementation " + existingImplIDStr + " for idiom " + idiomIDStr, http.StatusNotFound}
	}

	implID := String2Int(existingImplIDStr)
	if implID == -1 {
		return PiError{existingImplIDStr + " is not a valid implementation id.", http.StatusBadRequest}
	}

	_, impl, _ := idiom.FindImplInIdiom(implID)

	isAdmin := IsAdmin(r)
	if idiom.Protected && !isAdmin {
		return PiError{"Can't edit protected idiom " + idiomIDStr, http.StatusUnauthorized}
	}
	if impl.Protected && !isAdmin {
		return PiError{"Can't edit protected impl " + existingImplIDStr, http.StatusUnauthorized}
	}
	if isAdmin {
		wasProtected := impl.Protected
		impl.Protected = r.FormValue("impl_protected") != ""

		if wasProtected && !impl.Protected {
			log.Infof(c, "[%v] unprotects impl %v of idiom %v", username, existingImplIDStr, idiomIDStr)
		}
		if !wasProtected && impl.Protected {
			log.Infof(c, "[%v] protects impl %v of idiom %v", username, existingImplIDStr, idiomIDStr)
		}
	}

	if r.FormValue("impl_version") != strconv.Itoa(impl.Version) {
		return PiError{fmt.Sprintf("Implementation has been concurrently modified (editing version %v, current version is %v)", r.FormValue("impl_version"), impl.Version), http.StatusConflict}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	idiom.EditSummary = "[" + impl.LanguageName + "] " + r.FormValue("edit_summary")
	idiom.LastEditedImplID = implID
	impl.ImportsBlock = imports
	impl.CodeBlock = code
	impl.AuthorComment = comment
	impl.LastEditor = username
	impl.OriginalAttributionURL = attributionURL
	impl.DemoURL = demoURL
	impl.DocumentationURL = docURL
	impl.Version = impl.Version + 1
	impl.VersionDate = time.Now()

	if isAdmin {
		// 2016-10: only Admin may set an impl picture
		impl.PictureURL = r.FormValue("impl_picture_url")
	}

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, impl.LanguageName), http.StatusFound)
	return nil
}
<commit_msg>Nice language name (C++, C#) instead of normalized name (cpp, csharp) in history.<commit_after>package main

import (
	"fmt"
	"net/http"
	"strconv"
	"time"

	. "github.com/Deleplace/programming-idioms/pig"

	"google.golang.org/appengine/log"
)

func implSave(w http.ResponseWriter, r *http.Request) error {
	idiomIDStr := r.FormValue("idiom_id")
	existingIDStr := r.FormValue("impl_id")
	username := r.FormValue("user_nickname")
	username = Truncate(username, 30)

	if !toggles["anonymousWrite"] {
		if username == "" {
			return PiError{"Username is mandatory. No anonymous edit.", http.StatusBadRequest}
		}
	}

	setNicknameCookie(w, username)

	if existingIDStr == "" {
		return newImplSave(w, r, username, idiomIDStr)
	}
	return existingImplSave(w, r, username, idiomIDStr, existingIDStr)
}

func newImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string) error {
	if err := togglesMissing(w, r, "implAddition"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_language"); err != nil {
		return err
	}

	c := r.Context()
	language := NormLang(r.FormValue("impl_language"))
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")
	editSummary := fmt.Sprintf("New %s implementation by user [%s]", PrintNiceLang(language), username)

	imports = Truncate(imports, 200)
	code = TruncateBytes(NoCR(code), 500)
	comment = TruncateBytes(comment, 500)
	attributionURL = Truncate(attributionURL, 250)
	demoURL = Truncate(demoURL, 250)
	docURL = Truncate(docURL, 250)

	log.Infof(c, "[%s] is creating new %s impl for idiom %v", username, PrintNiceLang(language), idiomIDStr)

	if !StringSliceContains(AllLanguages(), language) {
		return PiError{fmt.Sprintf("Sorry, [%v] is currently not a supported language. Supported languages are %v.", r.FormValue("impl_language"), AllNiceLangs), http.StatusBadRequest}
	}

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find idiom " + idiomIDStr, http.StatusNotFound}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	implID, err := dao.nextImplID(c)
	if err != nil {
		return err
	}
	now := time.Now()
	newImpl := Impl{
		Id:                     implID,
		OrigId:                 implID,
		Author:                 username,
		CreationDate:           now,
		LastEditor:             username,
		LanguageName:           language,
		ImportsBlock:           imports,
		CodeBlock:              code,
		AuthorComment:          comment,
		OriginalAttributionURL: attributionURL,
		DemoURL:                demoURL,
		DocumentationURL:       docURL,
		Version:                1,
		VersionDate:            now,
	}

	if IsAdmin(r) {
		// 2016-10: only Admin may set an impl picture
		newImpl.PictureURL = r.FormValue("impl_picture_url")
	}

	idiom.Implementations = append(idiom.Implementations, newImpl)
	idiom.EditSummary = editSummary
	idiom.LastEditedImplID = implID

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, language), http.StatusFound)
	return nil
}

func existingImplSave(w http.ResponseWriter, r *http.Request, username string, idiomIDStr string, existingImplIDStr string) error {
	if err := togglesMissing(w, r, "implEditing"); err != nil {
		return err
	}
	if err := parametersMissing(w, r, "impl_version"); err != nil {
		return err
	}

	c := r.Context()
	imports := r.FormValue("impl_imports")
	code := r.FormValue("impl_code")
	comment := r.FormValue("impl_comment")
	attributionURL := r.FormValue("impl_attribution_url")
	demoURL := r.FormValue("impl_demo_url")
	docURL := r.FormValue("impl_doc_url")

	imports = Truncate(imports, 200)
	code = TruncateBytes(NoCR(code), 500)
	comment = TruncateBytes(comment, 500)
	attributionURL = Truncate(attributionURL, 250)
	demoURL = Truncate(demoURL, 250)
	docURL = Truncate(docURL, 250)

	log.Infof(c, "[%s] is updating impl %s of idiom %s", username, existingImplIDStr, idiomIDStr)

	idiomID := String2Int(idiomIDStr)
	if idiomID == -1 {
		return PiError{idiomIDStr + " is not a valid idiom id.", http.StatusBadRequest}
	}

	key, idiom, err := dao.getIdiom(c, idiomID)
	if err != nil {
		return PiError{"Could not find implementation " + existingImplIDStr + " for idiom " + idiomIDStr, http.StatusNotFound}
	}

	implID := String2Int(existingImplIDStr)
	if implID == -1 {
		return PiError{existingImplIDStr + " is not a valid implementation id.", http.StatusBadRequest}
	}

	_, impl, _ := idiom.FindImplInIdiom(implID)

	isAdmin := IsAdmin(r)
	if idiom.Protected && !isAdmin {
		return PiError{"Can't edit protected idiom " + idiomIDStr, http.StatusUnauthorized}
	}
	if impl.Protected && !isAdmin {
		return PiError{"Can't edit protected impl " + existingImplIDStr, http.StatusUnauthorized}
	}
	if isAdmin {
		wasProtected := impl.Protected
		impl.Protected = r.FormValue("impl_protected") != ""

		if wasProtected && !impl.Protected {
			log.Infof(c, "[%v] unprotects impl %v of idiom %v", username, existingImplIDStr, idiomIDStr)
		}
		if !wasProtected && impl.Protected {
			log.Infof(c, "[%v] protects impl %v of idiom %v", username, existingImplIDStr, idiomIDStr)
		}
	}

	if r.FormValue("impl_version") != strconv.Itoa(impl.Version) {
		return PiError{fmt.Sprintf("Implementation has been concurrently modified (editing version %v, current version is %v)", r.FormValue("impl_version"), impl.Version), http.StatusConflict}
	}

	if err := validateURLFormatOrEmpty(attributionURL); err != nil {
		return PiError{"Can't accept URL [" + attributionURL + "]", http.StatusBadRequest}
	}

	if err := validateURLFormatOrEmpty(demoURL); err != nil {
		return PiError{"Can't accept URL [" + demoURL + "]", http.StatusBadRequest}
	}

	idiom.EditSummary = "[" + PrintNiceLang(impl.LanguageName) + "] " + r.FormValue("edit_summary")
	idiom.LastEditedImplID = implID
	impl.ImportsBlock = imports
	impl.CodeBlock = code
	impl.AuthorComment = comment
	impl.LastEditor = username
	impl.OriginalAttributionURL = attributionURL
	impl.DemoURL = demoURL
	impl.DocumentationURL = docURL
	impl.Version = impl.Version + 1
	impl.VersionDate = time.Now()

	if isAdmin {
		// 2016-10: only Admin may set an impl picture
		impl.PictureURL = r.FormValue("impl_picture_url")
	}

	err = dao.saveExistingIdiom(c, key, idiom)
	if err != nil {
		return err
	}

	http.Redirect(w, r, NiceImplURL(idiom, implID, impl.LanguageName), http.StatusFound)
	return nil
}
<|endoftext|>


<commit_before>package doc

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/text"
	"github.com/octago/sflags/gen/gpflag"
	"github.com/spf13/cobra"
)

func Doc(ctx *context.Context) *cobra.Command {
	var flags struct {
		Output  string `desc:"output file, STDOUT by default"`
		Command string `desc:"print docs for command and its subcommands, example 'chkit doc --command \"create depl\"'"`
		MD      bool   `desc:"generate markdown docs"`
	}
	var cmd = &cobra.Command{
		Use:   "doc",
		Short: "Print full chkit help",
		Run: func(cmd *cobra.Command, args []string) {
			var currentCommand *cobra.Command
			if flags.Command == "" {
				currentCommand = cmd.Root()
			} else {
				currentCommand, _, _ = cmd.Parent().Find(strings.Fields(flags.Command))
			}
			var doc = &bytes.Buffer{}
			var stack = []*cobra.Command{currentCommand}
			for len(stack) > 0 {
				var head = func() *cobra.Command {
					var cmd = stack[len(stack)-1]
					stack = stack[:len(stack)-1]
					return cmd
				}()
				head.SetOutput(doc)
				stack = append(stack, head.Commands()...)
				if flags.MD {
					doc.WriteString(DocMD(head))
				} else {
					fmt.Fprintf(doc, "\n------------------------\nCommand : %s\n", func() string {
						if head.Parent() != nil && head.Parent().Use != "chkit" {
							return head.Parent().Use + " " + head.Use
						}
						return head.Use
					}())

					//doc.WriteString(getDoc(head))
					head.Usage()
				}
			}
			if flags.Output == "" {
				fmt.Println(doc)
			} else {
				if err := ioutil.WriteFile(flags.Output, doc.Bytes(), os.ModePerm); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
			}

		},
	}
	if err := gpflag.ParseTo(&flags, cmd.PersistentFlags()); err != nil {
		panic(err)
	}
	return cmd
}

func DocMD(cmd *cobra.Command) string {
	var doc = &bytes.Buffer{}
	fmt.Fprintf(doc, "\n##%s\n\n"+
		"###Aliases:\n  %s\n"+
		"###Usage  :\n %s\n"+
		"###Example:\n  %s\n"+
		"###Flags  :\n%s\n"+
		"###Subcommands :\n%s\n",
		func() string {
			if cmd.Parent() != nil && cmd.Parent().Use != "chkit" {
				return cmd.Parent().Use + " " + cmd.Use
			}
			return cmd.Use
		}(),
		strings.Join(cmd.Aliases, ", "),
		activekit.OrString(cmd.Long, cmd.Short),
		cmd.Example,
		cmd.LocalFlags().FlagUsages(),
		func() string {
			var d string
			for _, sub := range cmd.Commands() {
				d += fmt.Sprintf("+ %s : %s\n", sub.Use, sub.Short)
			}
			return text.Indent(d, 2)
		}())
	return doc.String()
}
<commit_msg>fix flag formatting<commit_after>package doc

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/containerum/chkit/pkg/context"
	"github.com/containerum/chkit/pkg/util/activekit"
	"github.com/containerum/chkit/pkg/util/text"
	"github.com/octago/sflags/gen/gpflag"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
)

func Doc(ctx *context.Context) *cobra.Command {
	var flags struct {
		Output  string `desc:"output file, STDOUT by default"`
		Command string `desc:"print docs for command and its subcommands, example 'chkit doc --command \"create depl\"'"`
		MD      bool   `desc:"generate markdown docs"`
	}
	var cmd = &cobra.Command{
		Use:   "doc",
		Short: "Print full chkit help",
		Run: func(cmd *cobra.Command, args []string) {
			var currentCommand *cobra.Command
			if flags.Command == "" {
				currentCommand = cmd.Root()
			} else {
				currentCommand, _, _ = cmd.Parent().Find(strings.Fields(flags.Command))
			}
			var doc = &bytes.Buffer{}
			var stack = []*cobra.Command{currentCommand}
			for len(stack) > 0 {
				var head = func() *cobra.Command {
					var cmd = stack[len(stack)-1]
					stack = stack[:len(stack)-1]
					return cmd
				}()
				head.SetOutput(doc)
				stack = append(stack, head.Commands()...)
				if flags.MD {
					doc.WriteString(DocMD(head))
				} else {
					fmt.Fprintf(doc, "\n------------------------\nCommand : %s\n", func() string {
						if head.Parent() != nil && head.Parent().Use != "chkit" {
							return head.Parent().Use + " " + head.Use
						}
						return head.Use
					}())

					//doc.WriteString(getDoc(head))
					head.Usage()
				}
			}
			if flags.Output == "" {
				fmt.Println(doc)
			} else {
				if err := ioutil.WriteFile(flags.Output, doc.Bytes(), os.ModePerm); err != nil {
					fmt.Println(err)
					os.Exit(1)
				}
			}

		},
	}
	if err := gpflag.ParseTo(&flags, cmd.PersistentFlags()); err != nil {
		panic(err)
	}
	return cmd
}

func DocMD(cmd *cobra.Command) string {
	var doc = &bytes.Buffer{}
	fmt.Fprintf(doc, "\n## %s\n\n"+
		"### Aliases:\n  %s\n"+
		"### Usage  :\n %s\n"+
		"### Example:\n  %s\n"+
		"### Flags  :\n%s\n"+
		"### Subcommands :\n%s\n",
		func() string {
			if cmd.Parent() != nil && cmd.Parent().Use != "chkit" {
				return cmd.Parent().Use + " " + cmd.Use
			}
			return cmd.Use
		}(),
		strings.Join(cmd.Aliases, ", "),
		activekit.OrString(cmd.Long, cmd.Short),
		cmd.Example,
		func() string {
			var d string
			cmd.LocalFlags().VisitAll(func(flag *pflag.Flag) {
				if !flag.Hidden {
					d += fmt.Sprintf("+ %s %s : %s\n", flag.Name, flag.Shorthand, flag.Usage)
				}
			})
			return text.Indent(d, 2)
		}(),
		func() string {
			var d string
			for _, sub := range cmd.Commands() {
				d += fmt.Sprintf("+ %s : %s\n", sub.Use, sub.Short)
			}
			return text.Indent(d, 2)
		}())
	return doc.String()
}
<|endoftext|>


<commit_before>package deploy

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/flant/dapp/pkg/dapp"
	"github.com/flant/dapp/pkg/lock"
	"github.com/flant/kubedog/pkg/kube"
	"github.com/flant/kubedog/pkg/tracker"
	"github.com/flant/kubedog/pkg/trackers/rollout"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/helm/pkg/releaseutil"
	"k8s.io/kubernetes/pkg/util/file"
)

type HelmChartOptions struct {
	Set     []string
	Values  []string
	DryRun  bool
	Debug   bool
	Timeout time.Duration
}

func DeployHelmChart(chartPath string, releaseName string, namespace string, opts HelmChartOptions) error {
	lockName := fmt.Sprintf("helm_release.%s", releaseName)
	return lock.WithLock(lockName, lock.LockOptions{Timeout: time.Second * 600}, func() error {
		if err := doDeployHelmChart(chartPath, releaseName, namespace, opts); err != nil {
			return err
		}

		return nil
	})
}

func doDeployHelmChart(chartPath string, releaseName string, namespace string, opts HelmChartOptions) error {
	releaseExist, err := isReleaseExist(releaseName)
	if err != nil {
		return fmt.Errorf("checking release failed: %s", err)
	}

	templates, err := parseTemplates(chartPath, releaseName)
	if err != nil {
		return fmt.Errorf("parsing templates failed: %s", err)
	}

	if err := removeOldJobs(templates, namespace); err != nil {
		return fmt.Errorf("removing old jobs failed: %s", err)
	}

	deployStartTime := time.Now()

	jobHooksWatcherDone, err := watchJobHooks(templates, releaseExist, deployStartTime, namespace, opts)
	if err != nil {
		return fmt.Errorf("watching job hooks failed: %s", err)
	}

	args := commonDeployCommandArgs(namespace, opts)
	if releaseExist {
		args = append([]string{"upgrade", releaseName, chartPath}, args...)
		fmt.Printf("# Upgrading helm release '%s'...\n", releaseName)
	} else {
		args = append([]string{"install", chartPath, "--name", releaseName}, args...)
		fmt.Printf("# Installing helm release '%s'...\n", releaseName)
	}

	stdout, stderr, err := HelmCmd(args...)
	if err != nil {
		if strings.HasSuffix(stderr, "has no deployed releases\n") {
			fmt.Printf("WARN: Helm release '%s' is in improper state: %s", releaseName, stderr)
			fmt.Printf("WARN: Helm release %s will be removed with `helm delete --purge` on the next run of `dapp kube deploy`", releaseName)
		}

		if err := createAutoPurgeTriggerFilePath(releaseName); err != nil {
			return err
		}

		return fmt.Errorf("%s\n%s", stdout, stderr)
	}

	if err := deleteAutoPurgeTriggerFilePath(releaseName); err != nil {
		return err
	}

	<-jobHooksWatcherDone

	fmt.Printf("%s\n%s\n", stdout, stderr)

	if err := watchDeployments(templates, deployStartTime, namespace, opts); err != nil {
		return fmt.Errorf("watching deployments failed: %s", err)
	}

	return nil
}

func watchDeployments(templates *ChartTemplates, deployStartTime time.Time, namespace string, opts HelmChartOptions) error {
	for _, template := range templates.Deployments() {
		fmt.Printf("# Run watch for deployment '%s'\n", template.Metadata.Name)

		err := rollout.TrackDeploymentTillReady(template.Metadata.Name, template.Namespace(namespace), kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(opts.Timeout), LogsFromTime: deployStartTime})
		if err != nil {
			return err
		}
	}

	return nil
}

func watchJobHooks(templates *ChartTemplates, releaseExist bool, deployStartTime time.Time, namespace string, opts HelmChartOptions) (chan bool, error) {
	jobHooksWatcherDone := make(chan bool)

	jobHooksToWatch, err := jobHooksToWatch(templates, releaseExist)
	if err != nil {
		return jobHooksWatcherDone, err
	}

	go func() {
		for _, template := range jobHooksToWatch {
			fmt.Printf("# Run watch for job '%s'\n", template.Metadata.Name)

			var jobNamespace string
			if template.Metadata.Namespace != "" {
				jobNamespace = template.Metadata.Namespace
			} else {
				jobNamespace = namespace
			}

			err := rollout.TrackJobTillDone(template.Metadata.Name, jobNamespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(opts.Timeout), LogsFromTime: deployStartTime})
			if err != nil {
				fmt.Fprintf(os.Stderr, "ERROR %s\n", err)
				break
			}
		}

		jobHooksWatcherDone <- true
	}()

	return jobHooksWatcherDone, nil
}

func commonDeployCommandArgs(namespace string, opts HelmChartOptions) []string {
	var args []string

	if namespace != "" {
		args = append(args, "--namespace", namespace)
	}

	for _, set := range opts.Set {
		args = append(args, "--set", set)
	}

	for _, values := range opts.Values {
		args = append(args, "--values", values)
	}

	if opts.DryRun {
		args = append(args, "--dry-run")
		args = append(args, "--debug")
	}

	if opts.Timeout != 0 {
		args = append(args, "--timeout", fmt.Sprintf("%v", opts.Timeout.Seconds()))
	}

	return args
}

func isReleaseExist(releaseName string) (bool, error) {
	var releaseStatus string
	var releaseExist bool

	helmStatusStdout, _, helmStatusErr := HelmCmd("status", releaseName)
	if helmStatusErr == nil {
		statusLinePrefix := "STATUS: "
		scanner := bufio.NewScanner(strings.NewReader(helmStatusStdout))
		for scanner.Scan() {
			line := scanner.Text()
			if strings.HasPrefix(line, statusLinePrefix) {
				releaseStatus = line[len(statusLinePrefix):]
				break
			}
		}
	}

	if helmStatusErr != nil {
		releaseExist = false
		if err := createAutoPurgeTriggerFilePath(releaseName); err != nil {
			return false, err
		}
	} else if releaseStatus != "" && (releaseStatus == "FAILED" || releaseStatus == "PENDING_INSTALL") {
		releaseExist = true

		if exist, err := file.FileExists(autoPurgeTriggerFilePath(releaseName)); err != nil {
			return false, err
		} else if exist {
			fmt.Printf("# Delete release '%s'\n", releaseName)
			if _, _, err := HelmCmd("delete", "--purge", releaseName); err != nil {
				return false, err
			}

			releaseExist = false
		}
	} else {
		if exist, err := file.FileExists(autoPurgeTriggerFilePath(releaseName)); err != nil {
			return false, err
		} else if exist {
			fmt.Printf("WARN: Will not purge helm release '%s': expected FAILED or PENDING_INSTALL release status, got %s\n", releaseName, releaseStatus)
		}

		releaseExist = true

		if err := deleteAutoPurgeTriggerFilePath(releaseName); err != nil {
			return false, err
		}
	}

	return releaseExist, nil
}

type ChartTemplates []*Template

func (templates *ChartTemplates) Jobs() []*Template {
	return templates.ByKind("Job")
}

func (templates *ChartTemplates) Deployments() []*Template {
	return templates.ByKind("Deployment")
}

func (templates *ChartTemplates) ByKind(kind string) []*Template {
	var resultTemplates []*Template

	for _, template := range []*Template(*templates) {
		if template.Kind == kind {
			resultTemplates = append(resultTemplates, template)
		}
	}

	return resultTemplates
}

type Template struct {
	Version  string `yaml:"apiVersion"`
	Kind     string `yaml:"kind,omitempty"`
	Metadata *struct {
		Name        string            `yaml:"name"`
		Namespace   string            `yaml:"namespace"`
		Annotations map[string]string `yaml:"annotations"`
		Uid         string            `yaml:"uid"`
	} `yaml:"metadata,omitempty"`
	Status string `yaml:"status,omitempty"`
}

func (t *Template) Namespace(namespace string) string {
	if t.Metadata.Namespace != "" {
		return t.Metadata.Namespace
	}

	return namespace
}

func parseTemplates(chartPath, releaseName string) (*ChartTemplates, error) {
	var templates []*Template

	stdout, stderr, err := HelmCmd("template", chartPath, "--name", releaseName)
	if err != nil {
		return nil, fmt.Errorf(stderr)
	}

	for _, doc := range releaseutil.SplitManifests(stdout) {
		var t Template
		err := yaml.Unmarshal([]byte(doc), &t)
		if err != nil {
			return nil, err
		}

		if t.Metadata != nil && t.Metadata.Name != "" {
			templates = append(templates, &t)
		}
	}

	chartTemplates := ChartTemplates(templates)
	return &chartTemplates, nil
}

func removeOldJobs(templates *ChartTemplates, namespace string) error {
	isJobExist := func(name string, namespace string) (bool, error) {
		options := v1.GetOptions{IncludeUninitialized: true}
		_, err := kube.Kubernetes.BatchV1().Jobs(namespace).Get(name, options)
		if err != nil {
			if apierrors.IsNotFound(err) {
				return false, nil
			}

			return true, err
		}

		return true, nil
	}

	for _, template := range templates.Jobs() {
		value, ok := template.Metadata.Annotations["dapp/recreate"]
		if ok && (value == "0" || value == "false") {
			continue
		}

		exist, err := isJobExist(template.Metadata.Name, template.Namespace(namespace))
		if err != nil {
			return err
		} else if !exist {
			continue
		}

		fmt.Printf("# Deleting hook job '%s' (dapp/recreate)...\n", template.Metadata.Name)

		deletePropagation := v1.DeletePropagationForeground
		deleteOptions := &v1.DeleteOptions{
			PropagationPolicy: &deletePropagation,
		}
		err = kube.Kubernetes.BatchV1().Jobs(template.Namespace(namespace)).Delete(template.Metadata.Name, deleteOptions)
		if err != nil {
			return err
		}

		for {
			exist, err := isJobExist(template.Metadata.Name, template.Namespace(namespace))
			if err != nil {
				return err
			} else if !exist {
				break
			}

			time.Sleep(500 * time.Millisecond)
		}
	}

	return nil
}

func jobHooksToWatch(templates *ChartTemplates, releaseExist bool) ([]*Template, error) {
	var jobHooksToWatch []*Template
	jobHooksByType := make(map[string][]*Template)

	for _, template := range templates.Jobs() {
		if anno, ok := template.Metadata.Annotations["helm.sh/hook"]; ok {
			for _, hookType := range strings.Split(anno, ",") {
				if _, ok := jobHooksByType[hookType]; !ok {
					jobHooksByType[hookType] = []*Template{}
				}
				jobHooksByType[hookType] = append(jobHooksByType[hookType], template)
			}
		}
	}

	for _, templates := range jobHooksByType {
		sort.Slice(templates, func(i, j int) bool {
			toWeight := func(t *Template) int {
				val, ok := t.Metadata.Annotations["helm.sh/hook-weight"]
				if !ok {
					return 0
				}

				i, err := strconv.Atoi(val)
				if err != nil {
					fmt.Printf("WARN: Incorrect hook-weight anno value '%v'\n", val)
					return 0
				}

				return i
			}

			return toWeight(templates[i]) < toWeight(templates[j])
		})
	}

	if releaseExist {
		for _, hookType := range []string{"pre-upgrade", "post-upgrade"} {
			if templates, ok := jobHooksByType[hookType]; ok {
				jobHooksToWatch = append(jobHooksToWatch, templates...)
			}
		}
	} else {
		for _, hookType := range []string{"pre-install", "post-install"} {
			if templates, ok := jobHooksByType[hookType]; ok {
				jobHooksToWatch = append(jobHooksToWatch, templates...)
			}
		}
	}

	return jobHooksToWatch, nil
}

func createAutoPurgeTriggerFilePath(releaseName string) error {
	filePath := autoPurgeTriggerFilePath(releaseName)
	dirPath := path.Dir(filePath)

	if fileExist, err := file.FileExists(filePath); err != nil {
		return err
	} else if !fileExist {
		if dirExist, err := file.FileExists(dirPath); err != nil {
			return err
		} else if !dirExist {
			if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
				return err
			}
		}

		if _, err := os.Create(filePath); err != nil {
			return err
		}
	}

	return nil
}

func deleteAutoPurgeTriggerFilePath(releaseName string) error {
	filePath := autoPurgeTriggerFilePath(releaseName)
	if fileExist, err := file.FileExists(filePath); err != nil {
		return err
	} else if fileExist {
		if err := os.Remove(filePath); err != nil {
			return err
		}
	}

	return nil
}

func autoPurgeTriggerFilePath(releaseName string) string {
	return filepath.Join(dapp.GetHomeDir(), "helm", releaseName, "auto_purge_failed_release_on_next_deploy")
}

func HelmCmd(args ...string) (stdout string, stderr string, err error) {
	binPath := "/usr/local/bin/helm"
	cmd := exec.Command(binPath, args...)
	cmd.Env = os.Environ()

	var stdoutBuf bytes.Buffer
	cmd.Stdout = &stdoutBuf
	var stderrBuf bytes.Buffer
	cmd.Stderr = &stderrBuf

	err = cmd.Run()
	stdout = strings.TrimSpace(stdoutBuf.String())
	stderr = strings.TrimSpace(stderrBuf.String())

	return
}
<commit_msg>[go deploy] deploy helm chart with default timeout file lock<commit_after>package deploy

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"

	"gopkg.in/yaml.v2"

	"github.com/flant/dapp/pkg/dapp"
	"github.com/flant/dapp/pkg/lock"
	"github.com/flant/kubedog/pkg/kube"
	"github.com/flant/kubedog/pkg/tracker"
	"github.com/flant/kubedog/pkg/trackers/rollout"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/helm/pkg/releaseutil"
	"k8s.io/kubernetes/pkg/util/file"
)

type HelmChartOptions struct {
	Set     []string
	Values  []string
	DryRun  bool
	Debug   bool
	Timeout time.Duration
}

func DeployHelmChart(chartPath string, releaseName string, namespace string, opts HelmChartOptions) error {
	lockName := fmt.Sprintf("helm_release.%s", releaseName)
	return lock.WithLock(lockName, lock.LockOptions{}, func() error {
		if err := doDeployHelmChart(chartPath, releaseName, namespace, opts); err != nil {
			return err
		}

		return nil
	})
}

func doDeployHelmChart(chartPath string, releaseName string, namespace string, opts HelmChartOptions) error {
	releaseExist, err := isReleaseExist(releaseName)
	if err != nil {
		return fmt.Errorf("checking release failed: %s", err)
	}

	templates, err := parseTemplates(chartPath, releaseName)
	if err != nil {
		return fmt.Errorf("parsing templates failed: %s", err)
	}

	if err := removeOldJobs(templates, namespace); err != nil {
		return fmt.Errorf("removing old jobs failed: %s", err)
	}

	deployStartTime := time.Now()

	jobHooksWatcherDone, err := watchJobHooks(templates, releaseExist, deployStartTime, namespace, opts)
	if err != nil {
		return fmt.Errorf("watching job hooks failed: %s", err)
	}

	args := commonDeployCommandArgs(namespace, opts)
	if releaseExist {
		args = append([]string{"upgrade", releaseName, chartPath}, args...)
		fmt.Printf("# Upgrading helm release '%s'...\n", releaseName)
	} else {
		args = append([]string{"install", chartPath, "--name", releaseName}, args...)
		fmt.Printf("# Installing helm release '%s'...\n", releaseName)
	}

	stdout, stderr, err := HelmCmd(args...)
	if err != nil {
		if strings.HasSuffix(stderr, "has no deployed releases\n") {
			fmt.Printf("WARN: Helm release '%s' is in improper state: %s", releaseName, stderr)
			fmt.Printf("WARN: Helm release %s will be removed with `helm delete --purge` on the next run of `dapp kube deploy`", releaseName)
		}

		if err := createAutoPurgeTriggerFilePath(releaseName); err != nil {
			return err
		}

		return fmt.Errorf("%s\n%s", stdout, stderr)
	}

	if err := deleteAutoPurgeTriggerFilePath(releaseName); err != nil {
		return err
	}

	<-jobHooksWatcherDone

	fmt.Printf("%s\n%s\n", stdout, stderr)

	if err := watchDeployments(templates, deployStartTime, namespace, opts); err != nil {
		return fmt.Errorf("watching deployments failed: %s", err)
	}

	return nil
}

func watchDeployments(templates *ChartTemplates, deployStartTime time.Time, namespace string, opts HelmChartOptions) error {
	for _, template := range templates.Deployments() {
		fmt.Printf("# Run watch for deployment '%s'\n", template.Metadata.Name)

		err := rollout.TrackDeploymentTillReady(template.Metadata.Name, template.Namespace(namespace), kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(opts.Timeout), LogsFromTime: deployStartTime})
		if err != nil {
			return err
		}
	}

	return nil
}

func watchJobHooks(templates *ChartTemplates, releaseExist bool, deployStartTime time.Time, namespace string, opts HelmChartOptions) (chan bool, error) {
	jobHooksWatcherDone := make(chan bool)

	jobHooksToWatch, err := jobHooksToWatch(templates, releaseExist)
	if err != nil {
		return jobHooksWatcherDone, err
	}

	go func() {
		for _, template := range jobHooksToWatch {
			fmt.Printf("# Run watch for job '%s'\n", template.Metadata.Name)

			var jobNamespace string
			if template.Metadata.Namespace != "" {
				jobNamespace = template.Metadata.Namespace
			} else {
				jobNamespace = namespace
			}

			err := rollout.TrackJobTillDone(template.Metadata.Name, jobNamespace, kube.Kubernetes, tracker.Options{Timeout: time.Second * time.Duration(opts.Timeout), LogsFromTime: deployStartTime})
			if err != nil {
				fmt.Fprintf(os.Stderr, "ERROR %s\n", err)
				break
			}
		}

		jobHooksWatcherDone <- true
	}()

	return jobHooksWatcherDone, nil
}

func commonDeployCommandArgs(namespace string, opts HelmChartOptions) []string {
	var args []string

	if namespace != "" {
		args = append(args, "--namespace", namespace)
	}

	for _, set := range opts.Set {
		args = append(args, "--set", set)
	}

	for _, values := range opts.Values {
		args = append(args, "--values", values)
	}

	if opts.DryRun {
		args = append(args, "--dry-run")
		args = append(args, "--debug")
	}

	if opts.Timeout != 0 {
		args = append(args, "--timeout", fmt.Sprintf("%v", opts.Timeout.Seconds()))
	}

	return args
}

func isReleaseExist(releaseName string) (bool, error) {
	var releaseStatus string
	var releaseExist bool

	helmStatusStdout, _, helmStatusErr := HelmCmd("status", releaseName)
	if helmStatusErr == nil {
		statusLinePrefix := "STATUS: "
		scanner := bufio.NewScanner(strings.NewReader(helmStatusStdout))
		for scanner.Scan() {
			line := scanner.Text()
			if strings.HasPrefix(line, statusLinePrefix) {
				releaseStatus = line[len(statusLinePrefix):]
				break
			}
		}
	}

	if helmStatusErr != nil {
		releaseExist = false
		if err := createAutoPurgeTriggerFilePath(releaseName); err != nil {
			return false, err
		}
	} else if releaseStatus != "" && (releaseStatus == "FAILED" || releaseStatus == "PENDING_INSTALL") {
		releaseExist = true

		if exist, err := file.FileExists(autoPurgeTriggerFilePath(releaseName)); err != nil {
			return false, err
		} else if exist {
			fmt.Printf("# Delete release '%s'\n", releaseName)
			if _, _, err := HelmCmd("delete", "--purge", releaseName); err != nil {
				return false, err
			}

			releaseExist = false
		}
	} else {
		if exist, err := file.FileExists(autoPurgeTriggerFilePath(releaseName)); err != nil {
			return false, err
		} else if exist {
			fmt.Printf("WARN: Will not purge helm release '%s': expected FAILED or PENDING_INSTALL release status, got %s\n", releaseName, releaseStatus)
		}

		releaseExist = true

		if err := deleteAutoPurgeTriggerFilePath(releaseName); err != nil {
			return false, err
		}
	}

	return releaseExist, nil
}

type ChartTemplates []*Template

func (templates *ChartTemplates) Jobs() []*Template {
	return templates.ByKind("Job")
}

func (templates *ChartTemplates) Deployments() []*Template {
	return templates.ByKind("Deployment")
}

func (templates *ChartTemplates) ByKind(kind string) []*Template {
	var resultTemplates []*Template

	for _, template := range []*Template(*templates) {
		if template.Kind == kind {
			resultTemplates = append(resultTemplates, template)
		}
	}

	return resultTemplates
}

type Template struct {
	Version  string `yaml:"apiVersion"`
	Kind     string `yaml:"kind,omitempty"`
	Metadata *struct {
		Name        string            `yaml:"name"`
		Namespace   string            `yaml:"namespace"`
		Annotations map[string]string `yaml:"annotations"`
		Uid         string            `yaml:"uid"`
	} `yaml:"metadata,omitempty"`
	Status string `yaml:"status,omitempty"`
}

func (t *Template) Namespace(namespace string) string {
	if t.Metadata.Namespace != "" {
		return t.Metadata.Namespace
	}

	return namespace
}

func parseTemplates(chartPath, releaseName string) (*ChartTemplates, error) {
	var templates []*Template

	stdout, stderr, err := HelmCmd("template", chartPath, "--name", releaseName)
	if err != nil {
		return nil, fmt.Errorf(stderr)
	}

	for _, doc := range releaseutil.SplitManifests(stdout) {
		var t Template
		err := yaml.Unmarshal([]byte(doc), &t)
		if err != nil {
			return nil, err
		}

		if t.Metadata != nil && t.Metadata.Name != "" {
			templates = append(templates, &t)
		}
	}

	chartTemplates := ChartTemplates(templates)
	return &chartTemplates, nil
}

func removeOldJobs(templates *ChartTemplates, namespace string) error {
	isJobExist := func(name string, namespace string) (bool, error) {
		options := v1.GetOptions{IncludeUninitialized: true}
		_, err := kube.Kubernetes.BatchV1().Jobs(namespace).Get(name, options)
		if err != nil {
			if apierrors.IsNotFound(err) {
				return false, nil
			}

			return true, err
		}

		return true, nil
	}

	for _, template := range templates.Jobs() {
		value, ok := template.Metadata.Annotations["dapp/recreate"]
		if ok && (value == "0" || value == "false") {
			continue
		}

		exist, err := isJobExist(template.Metadata.Name, template.Namespace(namespace))
		if err != nil {
			return err
		} else if !exist {
			continue
		}

		fmt.Printf("# Deleting hook job '%s' (dapp/recreate)...\n", template.Metadata.Name)

		deletePropagation := v1.DeletePropagationForeground
		deleteOptions := &v1.DeleteOptions{
			PropagationPolicy: &deletePropagation,
		}
		err = kube.Kubernetes.BatchV1().Jobs(template.Namespace(namespace)).Delete(template.Metadata.Name, deleteOptions)
		if err != nil {
			return err
		}

		for {
			exist, err := isJobExist(template.Metadata.Name, template.Namespace(namespace))
			if err != nil {
				return err
			} else if !exist {
				break
			}

			time.Sleep(500 * time.Millisecond)
		}
	}

	return nil
}

func jobHooksToWatch(templates *ChartTemplates, releaseExist bool) ([]*Template, error) {
	var jobHooksToWatch []*Template
	jobHooksByType := make(map[string][]*Template)

	for _, template := range templates.Jobs() {
		if anno, ok := template.Metadata.Annotations["helm.sh/hook"]; ok {
			for _, hookType := range strings.Split(anno, ",") {
				if _, ok := jobHooksByType[hookType]; !ok {
					jobHooksByType[hookType] = []*Template{}
				}
				jobHooksByType[hookType] = append(jobHooksByType[hookType], template)
			}
		}
	}

	for _, templates := range jobHooksByType {
		sort.Slice(templates, func(i, j int) bool {
			toWeight := func(t *Template) int {
				val, ok := t.Metadata.Annotations["helm.sh/hook-weight"]
				if !ok {
					return 0
				}

				i, err := strconv.Atoi(val)
				if err != nil {
					fmt.Printf("WARN: Incorrect hook-weight anno value '%v'\n", val)
					return 0
				}

				return i
			}

			return toWeight(templates[i]) < toWeight(templates[j])
		})
	}

	if releaseExist {
		for _, hookType := range []string{"pre-upgrade", "post-upgrade"} {
			if templates, ok := jobHooksByType[hookType]; ok {
				jobHooksToWatch = append(jobHooksToWatch, templates...)
			}
		}
	} else {
		for _, hookType := range []string{"pre-install", "post-install"} {
			if templates, ok := jobHooksByType[hookType]; ok {
				jobHooksToWatch = append(jobHooksToWatch, templates...)
			}
		}
	}

	return jobHooksToWatch, nil
}

func createAutoPurgeTriggerFilePath(releaseName string) error {
	filePath := autoPurgeTriggerFilePath(releaseName)
	dirPath := path.Dir(filePath)

	if fileExist, err := file.FileExists(filePath); err != nil {
		return err
	} else if !fileExist {
		if dirExist, err := file.FileExists(dirPath); err != nil {
			return err
		} else if !dirExist {
			if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
				return err
			}
		}

		if _, err := os.Create(filePath); err != nil {
			return err
		}
	}

	return nil
}

func deleteAutoPurgeTriggerFilePath(releaseName string) error {
	filePath := autoPurgeTriggerFilePath(releaseName)
	if fileExist, err := file.FileExists(filePath); err != nil {
		return err
	} else if fileExist {
		if err := os.Remove(filePath); err != nil {
			return err
		}
	}

	return nil
}

func autoPurgeTriggerFilePath(releaseName string) string {
	return filepath.Join(dapp.GetHomeDir(), "helm", releaseName, "auto_purge_failed_release_on_next_deploy")
}

func HelmCmd(args ...string) (stdout string, stderr string, err error) {
	binPath := "/usr/local/bin/helm"
	cmd := exec.Command(binPath, args...)
	cmd.Env = os.Environ()

	var stdoutBuf bytes.Buffer
	cmd.Stdout = &stdoutBuf
	var stderrBuf bytes.Buffer
	cmd.Stderr = &stderrBuf

	err = cmd.Run()
	stdout = strings.TrimSpace(stdoutBuf.String())
	stderr = strings.TrimSpace(stderrBuf.String())

	return
}
<|endoftext|>


<commit_before>package domain

import (
	"errors"
	"io/ioutil"
	"net/http"

	"github.com/vardius/go-api-boilerplate/pkg/http/response"
	"github.com/vardius/gorouter"
)

// ErrEmptyRequestBody is when an request has empty body.
var ErrEmptyRequestBody = errors.New("Empty request body")

// ErrInvalidURLParams is when an request has invalid or missing parameters.
var ErrInvalidURLParams = errors.New("Invalid request URL params")

type dispatcher struct {
	commandBus CommandBus
}

func (d *dispatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var e error

	if r.Body == nil {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   ErrEmptyRequestBody,
			Message: ErrEmptyRequestBody.Error,
		}))
		return
	}

	params, ok := gorouter.FromContext(r.Context())
	if !ok {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   ErrInvalidURLParams,
			Message: ErrInvalidURLParams.Error,
		}))
		return
	}

	defer r.Body.Close()
	body, e := ioutil.ReadAll(r.Body)
	if e != nil {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   e,
			Message: "Invalid request body",
		}))
		return
	}

	out := make(chan error)
	defer close(out)

	go func() {
		d.commandBus.Publish(
			r.Context(),
			params.Value("domain")+params.Value("command"),
			body,
			out,
		)
	}()

	if e = <-out; e != nil {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   e,
			Message: "Invalid request",
		}))
		return
	}

	w.WriteHeader(http.StatusCreated)

	return
}

// NewDispatcher creates handler for command bus
func NewDispatcher(cb CommandBus) http.Handler {
	return &dispatcher{cb}
}
<commit_msg>Update error message<commit_after>package domain

import (
	"errors"
	"io/ioutil"
	"net/http"

	"github.com/vardius/go-api-boilerplate/pkg/http/response"
	"github.com/vardius/gorouter"
)

// ErrEmptyRequestBody is when an request has empty body.
var ErrEmptyRequestBody = errors.New("Empty request body")

// ErrInvalidURLParams is when an request has invalid or missing parameters.
var ErrInvalidURLParams = errors.New("Invalid request URL params")

type dispatcher struct {
	commandBus CommandBus
}

func (d *dispatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var e error

	if r.Body == nil {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   ErrEmptyRequestBody,
			Message: ErrEmptyRequestBody.Error(),
		}))
		return
	}

	params, ok := gorouter.FromContext(r.Context())
	if !ok {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   ErrInvalidURLParams,
			Message: ErrInvalidURLParams.Error(),
		}))
		return
	}

	defer r.Body.Close()
	body, e := ioutil.ReadAll(r.Body)
	if e != nil {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   e,
			Message: "Invalid request body",
		}))
		return
	}

	out := make(chan error)
	defer close(out)

	go func() {
		d.commandBus.Publish(
			r.Context(),
			params.Value("domain")+params.Value("command"),
			body,
			out,
		)
	}()

	if e = <-out; e != nil {
		r.WithContext(response.WithPayload(r, response.HTTPError{
			Code:    http.StatusBadRequest,
			Error:   e,
			Message: "Invalid request",
		}))
		return
	}

	w.WriteHeader(http.StatusCreated)

	return
}

// NewDispatcher creates handler for command bus
func NewDispatcher(cb CommandBus) http.Handler {
	return &dispatcher{cb}
}
<|endoftext|>


<commit_before>package dqlite

import (
	"context"

	"github.com/canonical/go-dqlite/client"
	"github.com/sirupsen/logrus"
)

func (d *DQLite) Test(ctx context.Context) error {
	var ips []string
	peers, err := d.NodeStore.Get(ctx)
	if err != nil {
		return err
	}

	for _, peer := range peers {
		ips = append(ips, peer.Address)
	}

	logrus.Infof("Testing connection to peers %v", ips)
	return d.Join(ctx, nil)
}

func (d *DQLite) Join(ctx context.Context, nodes []client.NodeInfo) error {
	if len(nodes) > 0 {
		if err := d.NodeStore.Set(ctx, nodes); err != nil {
			return err
		}
	}

	client, err := client.FindLeader(ctx, d.NodeStore, d.clientOpts...)
	if err != nil {
		return err
	}
	defer client.Close()

	current, err := client.Cluster(ctx)
	if err != nil {
		return err
	}

	for _, testNode := range current {
		if testNode.Address == d.NodeInfo.Address {
			return nil
		}
	}

	logrus.Infof("Joining dqlite cluster as address=%s, id=%d")
	return client.Add(ctx, d.NodeInfo)
}
<commit_msg>Add logging parameters<commit_after>package dqlite

import (
	"context"

	"github.com/canonical/go-dqlite/client"
	"github.com/sirupsen/logrus"
)

func (d *DQLite) Test(ctx context.Context) error {
	var ips []string
	peers, err := d.NodeStore.Get(ctx)
	if err != nil {
		return err
	}

	for _, peer := range peers {
		ips = append(ips, peer.Address)
	}

	logrus.Infof("Testing connection to peers %v", ips)
	return d.Join(ctx, nil)
}

func (d *DQLite) Join(ctx context.Context, nodes []client.NodeInfo) error {
	if len(nodes) > 0 {
		if err := d.NodeStore.Set(ctx, nodes); err != nil {
			return err
		}
	}

	client, err := client.FindLeader(ctx, d.NodeStore, d.clientOpts...)
	if err != nil {
		return err
	}
	defer client.Close()

	current, err := client.Cluster(ctx)
	if err != nil {
		return err
	}

	for _, testNode := range current {
		if testNode.Address == d.NodeInfo.Address {
			return nil
		}
	}

	logrus.Infof("Joining dqlite cluster as address=%s, id=%d", d.NodeInfo.Address, d.NodeInfo.ID)
	return client.Add(ctx, d.NodeInfo)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"runtime/debug"
	"strconv"

	. "github.com/3d0c/gmf"
)

func fatal(err error) {
	debug.PrintStack()
	log.Fatal(err)
}

func assert(i interface{}, err error) interface{} {
	if err != nil {
		fatal(err)
	}

	return i
}

var i int = 0

func writeFile(b []byte) {
	name := "./tmp/" + strconv.Itoa(i) + ".jpg"

	fp, err := os.Create(name)
	if err != nil {
		fatal(err)
	}

	defer func() {
		if err := fp.Close(); err != nil {
			fatal(err)
		}
		i++
	}()

	if n, err := fp.Write(b); err != nil {
		fatal(err)
	} else {
		log.Println(n, "bytes written to", name)
	}
}

func main() {
	srcFileName := "tests-sample.mp4"

	os.Mkdir("./tmp", 0755)

	if len(os.Args) > 1 {
		srcFileName = os.Args[1]
	}

	inputCtx := assert(NewInputCtx(srcFileName)).(*FmtCtx)
	defer inputCtx.CloseInputAndRelease()

	srcVideoStream, err := inputCtx.GetBestStream(AVMEDIA_TYPE_VIDEO)
	if err != nil {
		log.Println("No video stream found in", srcFileName)
	}

	codec, err := FindEncoder("mjpeg")
	if err != nil {
		fatal(err)
	}

	cc := NewCodecCtx(codec)
	defer Release(cc)

	cc.SetPixFmt(AV_PIX_FMT_YUVJ420P)
	cc.SetWidth(srcVideoStream.CodecCtx().Width())
	cc.SetHeight(srcVideoStream.CodecCtx().Height())
	cc.SetTimeBase(srcVideoStream.CodecCtx().TimeBase().AVR())

	if codec.IsExperimental() {
		cc.SetStrictCompliance(FF_COMPLIANCE_EXPERIMENTAL)
	}

	if err := cc.Open(nil); err != nil {
		fatal(err)
	}

	for packet := range inputCtx.GetNewPackets() {
		if packet.StreamIndex() != srcVideoStream.Index() {
			// skip non video streams
			continue
		}
		ist := assert(inputCtx.GetStream(packet.StreamIndex())).(*Stream)

	decode:
		frame, err := packet.Frames(ist.CodecCtx())
		if err != nil {
			// Retry if EAGAIN
			if err.Error() == "Resource temporarily unavailable" {
				goto decode
			}
			log.Fatal(err)
		}

		p, err := frame.Encode(cc)
		if err != nil {
			Release(p)
			fatal(err)
		}

		writeFile(p.Data())

		Release(p)
		Release(packet)
	}
}
<commit_msg>Scale frames in video-to-mjpeg.go<commit_after>package main

import (
	"log"
	"os"
	"runtime/debug"
	"strconv"

	. "github.com/3d0c/gmf"
)

func fatal(err error) {
	debug.PrintStack()
	log.Fatal(err)
}

func assert(i interface{}, err error) interface{} {
	if err != nil {
		fatal(err)
	}

	return i
}

var i int = 0

func writeFile(b []byte) {
	name := "./tmp/" + strconv.Itoa(i) + ".jpg"

	fp, err := os.Create(name)
	if err != nil {
		fatal(err)
	}

	defer func() {
		if err := fp.Close(); err != nil {
			fatal(err)
		}
		i++
	}()

	if n, err := fp.Write(b); err != nil {
		fatal(err)
	} else {
		log.Println(n, "bytes written to", name)
	}
}

func main() {
	srcFileName := "tests-sample.mp4"

	os.Mkdir("./tmp", 0755)

	if len(os.Args) > 1 {
		srcFileName = os.Args[1]
	}

	inputCtx := assert(NewInputCtx(srcFileName)).(*FmtCtx)
	defer inputCtx.CloseInputAndRelease()

	srcVideoStream, err := inputCtx.GetBestStream(AVMEDIA_TYPE_VIDEO)
	if err != nil {
		log.Println("No video stream found in", srcFileName)
	}

	codec, err := FindEncoder("mjpeg")
	if err != nil {
		fatal(err)
	}

	cc := NewCodecCtx(codec)
	defer Release(cc)

	cc.SetPixFmt(AV_PIX_FMT_YUVJ420P)
	cc.SetWidth(srcVideoStream.CodecCtx().Width())
	cc.SetHeight(srcVideoStream.CodecCtx().Height())
	cc.SetTimeBase(srcVideoStream.CodecCtx().TimeBase().AVR())

	if codec.IsExperimental() {
		cc.SetStrictCompliance(FF_COMPLIANCE_EXPERIMENTAL)
	}

	if err := cc.Open(nil); err != nil {
		fatal(err)
	}

	swsCtx := NewSwsCtx(srcVideoStream.CodecCtx(), cc, SWS_BICUBIC)
	defer Release(swsCtx)

	dstFrame := NewFrame().
		SetWidth(srcVideoStream.CodecCtx().Width()).
		SetHeight(srcVideoStream.CodecCtx().Height()).
		SetFormat(AV_PIX_FMT_YUVJ420P) // see above
	defer Release(dstFrame)

	if err := dstFrame.ImgAlloc(); err != nil {
		fatal(err)
	}

	for packet := range inputCtx.GetNewPackets() {
		if packet.StreamIndex() != srcVideoStream.Index() {
			// skip non video streams
			continue
		}
		ist := assert(inputCtx.GetStream(packet.StreamIndex())).(*Stream)

	decode:
		frame, err := packet.Frames(ist.CodecCtx())
		if err != nil {
			// Retry if EAGAIN
			if err.Error() == "Resource temporarily unavailable" {
				goto decode
			}
			log.Fatal(err)
		}

		swsCtx.Scale(frame, dstFrame)

		p, err := dstFrame.Encode(cc)
		if err != nil {
			Release(p)
			fatal(err)
		}

		writeFile(p.Data())

		Release(p)
		Release(frame)
		Release(packet)
	}
}
<|endoftext|>