<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io
import os "os"
import syscall "syscall"

export type Read interface {
	Read(p *[]byte) (n int, err *os.Error);
}

export type Write interface {
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWrite interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWriteClose interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
	Close() *os.Error;
}

export func WriteString(w Write, s string) (n int, err *os.Error) {
	b := new([]byte, len(s)+1);
	if !syscall.StringToBytes(b, s) {
		return -1, os.EINVAL
	}
	// BUG return w.Write(b[0:len(s)])
	r, e := w.Write(b[0:len(s)]);
	return r, e
}
<commit_msg>Implement a "full reader" wrapper for io.Read, guaranteeing that either the buffer is full, zero bytes were read, or an error is returned.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package io

import (
	"os";
	"syscall";
)

export var ErrEOF = os.NewError("EOF")

export type Read interface {
	Read(p *[]byte) (n int, err *os.Error);
}

export type Write interface {
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWrite interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
}

export type ReadWriteClose interface {
	Read(p *[]byte) (n int, err *os.Error);
	Write(p *[]byte) (n int, err *os.Error);
	Close() *os.Error;
}

export func WriteString(w Write, s string) (n int, err *os.Error) {
	b := new([]byte, len(s)+1);
	if !syscall.StringToBytes(b, s) {
		return -1, os.EINVAL
	}
	// BUG return w.Write(b[0:len(s)])
	r, e := w.Write(b[0:len(s)]);
	return r, e
}

// Read until buffer is full,  EOF, or error
export func Readn(fd Read, buf *[]byte) (n int, err *os.Error) {
	n = 0;
	for n < len(buf) {
		nn, e := fd.Read(buf[n:len(buf)]);
		if nn > 0 {
			n += nn
		}
		if e != nil {
			return n, e
		}
		if nn <= 0 {
			return n, ErrEOF	// no error but insufficient data
		}
	}
	return n, nil
}

// Convert something that implements Read into something
// whose Reads are always Readn
type FullRead struct {
	fd	Read;
}

func (fd *FullRead) Read(p *[]byte) (n int, err *os.Error) {
	n, err = Readn(fd, p);
	return n, err
}

export func MakeFullReader(fd Read) Read {
	if fr, ok := fd.(*FullRead); ok {
		// already a FullRead
		return fd
	}
	return &FullRead{fd}
}
<|endoftext|>


<commit_before>package types

import (
	"fmt"
	"log"
	"sort"
	"strings"
)

const (
	SEPARATOR = ";"
)

var (
	Main_POS map[string]bool
	MDParams map[string]MDParam = map[string]MDParam{
		"Form":                            Form,
		"Form_Prop":                       Form_Prop,
		"POS":                             POS,
		"POS_Prop":                        POS_Prop,
		"Form_POS_Prop":                   Form_POS_Prop,
		"Form_Lemma_POS_Prop":             Form_Lemma_POS_Prop,
		"Form_POS":                        Form_POS,
		"Funcs_Main_POS_Both_Prop":        Funcs_Main_POS_Both_Prop,
		"Funcs_Main_POS_Both_Prop_WLemma": Funcs_Main_POS_Both_Prop_WLemma,
		"Funcs_Main_POS":                  Funcs_Main_POS,
		"Funcs_Main_POS_Prop":             Funcs_Main_POS_Prop,
	}

	AllParamFuncNames string
)

func init() {
	// HEB TB:
	Main_POS_Types := []string{"ADVERB", "BN", "BNT", "CD", "CDT", "JJ", "JJT", "NN", "NNP", "NNT", "RB", "VB"}
	// UD:
	// Main_POS_Types := []string{"ADJ", "ADV", "INTJ", "NOUN", "PROPN", "VERB"}
	log.Println("Using Main_POS_Types", Main_POS_Types)
	Main_POS = make(map[string]bool, len(Main_POS_Types))
	for _, pos := range Main_POS_Types {
		Main_POS[pos] = true
	}

	paramFuncStrs := make([]string, 0, len(MDParams))
	for k, _ := range MDParams {
		paramFuncStrs = append(paramFuncStrs, k)
	}
	sort.Strings(paramFuncStrs)
	AllParamFuncNames = strings.Join(paramFuncStrs, ", ")
}

// func Full(s Spellout) string {
// 	return s.AsString()
// }

type MDParam func(m *EMorpheme) string

func ProjectSpellout(s Spellout, f MDParam) string {
	strs := make([]string, len(s))
	for i, morph := range s {
		strs[i] = f(morph)
	}
	return strings.Join(strs, SEPARATOR)
}

func Form(m *EMorpheme) string {
	return m.Form
}

func Lemma(m *EMorpheme) string {
	return m.Lemma
}

func Form_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s", m.Form, m.FeatureStr)
}

func POS(m *EMorpheme) string {
	return m.CPOS
}

func POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
}

func Form_POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
}

func Lemma_POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s", m.Lemma, m.CPOS, m.FeatureStr)
}

func Form_Lemma_POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s_%s", m.Form, m.Lemma, m.CPOS, m.FeatureStr)
}

func Form_POS(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s", m.Form, m.CPOS)
}

func Funcs_Main_POS_No_Prop(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s_%s", m.CPOS)
	} else {
		return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
	}
}

func Funcs_Main_POS_Both_Prop(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
	} else {
		return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
	}
}

func Funcs_Main_POS_Both_Prop_WLemma(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s", m.Lemma, m.CPOS, m.FeatureStr)
	// if _, exists := Main_POS[m.CPOS]; exists {
	// 	return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
	// } else {
	// 	return fmt.Sprintf("%s_%s_%s", m.Lemma, m.CPOS, m.FeatureStr)
	// }
}

func Funcs_All_WLemma(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s_%s_%s", m.Form, m.Lemma, m.CPOS, m.POS, m.FeatureStr)
}

func Funcs_Lemma_Main_POS(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s", m.CPOS)
	} else {
		return fmt.Sprintf("%s_%s", m.Lemma, m.CPOS)
	}
}

func Funcs_Main_POS(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s", m.CPOS)
	} else {
		return fmt.Sprintf("%s_%s", m.Form, m.CPOS)
	}
}

func Funcs_Main_POS_Prop(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
	} else {
		return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
	}
}
<commit_msg>Set param open class as package variable<commit_after>package types

import (
	"fmt"
	"log"
	"sort"
	"strings"
)

const (
	SEPARATOR = ";"
)

var (
	Main_POS map[string]bool
	MDParams map[string]MDParam = map[string]MDParam{
		"Form":                            Form,
		"Form_Prop":                       Form_Prop,
		"POS":                             POS,
		"POS_Prop":                        POS_Prop,
		"Form_POS_Prop":                   Form_POS_Prop,
		"Form_Lemma_POS_Prop":             Form_Lemma_POS_Prop,
		"Form_POS":                        Form_POS,
		"Funcs_Main_POS_Both_Prop":        Funcs_Main_POS_Both_Prop,
		"Funcs_Main_POS_Both_Prop_WLemma": Funcs_Main_POS_Both_Prop_WLemma,
		"Funcs_Main_POS":                  Funcs_Main_POS,
		"Funcs_Main_POS_Prop":             Funcs_Main_POS_Prop,
	}
	PARAM_TYPE = "HEBTB" // HEBTB|UD

	AllParamFuncNames string
)

func InitOpenParamTypes(pType string) {
	var Main_POS_Types []string
	switch pType {
	case "HEBTB":
		Main_POS_Types = []string{"ADVERB", "BN", "BNT", "CD", "CDT", "JJ", "JJT", "NN", "NNP", "NNT", "RB", "VB"}
		break
	case "UD":
		Main_POS_Types = []string{"ADJ", "ADV", "INTJ", "NOUN", "PROPN", "VERB"}
		break
	default:
		panic(fmt.Sprintf("Unknown open class family %s", pType))
	}
	log.Println("Using Family", pType, "of Main_POS_Types [", Main_POS_Types, "]")
	Main_POS = make(map[string]bool, len(Main_POS_Types))
	for _, pos := range Main_POS_Types {
		Main_POS[pos] = true
	}
}

func init() {
	InitOpenParamTypes(PARAM_TYPE)

	paramFuncStrs := make([]string, 0, len(MDParams))
	for k, _ := range MDParams {
		paramFuncStrs = append(paramFuncStrs, k)
	}
	sort.Strings(paramFuncStrs)
	AllParamFuncNames = strings.Join(paramFuncStrs, ", ")
}

// func Full(s Spellout) string {
// 	return s.AsString()
// }

type MDParam func(m *EMorpheme) string

func ProjectSpellout(s Spellout, f MDParam) string {
	strs := make([]string, len(s))
	for i, morph := range s {
		strs[i] = f(morph)
	}
	return strings.Join(strs, SEPARATOR)
}

func Form(m *EMorpheme) string {
	return m.Form
}

func Lemma(m *EMorpheme) string {
	return m.Lemma
}

func Form_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s", m.Form, m.FeatureStr)
}

func POS(m *EMorpheme) string {
	return m.CPOS
}

func POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
}

func Form_POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
}

func Lemma_POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s", m.Lemma, m.CPOS, m.FeatureStr)
}

func Form_Lemma_POS_Prop(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s_%s", m.Form, m.Lemma, m.CPOS, m.FeatureStr)
}

func Form_POS(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s", m.Form, m.CPOS)
}

func Funcs_Main_POS_No_Prop(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s_%s", m.CPOS)
	} else {
		return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
	}
}

func Funcs_Main_POS_Both_Prop(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
	} else {
		return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
	}
}

func Funcs_Main_POS_Both_Prop_WLemma(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s", m.Lemma, m.CPOS, m.FeatureStr)
	// if _, exists := Main_POS[m.CPOS]; exists {
	// 	return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
	// } else {
	// 	return fmt.Sprintf("%s_%s_%s", m.Lemma, m.CPOS, m.FeatureStr)
	// }
}

func Funcs_All_WLemma(m *EMorpheme) string {
	return fmt.Sprintf("%s_%s_%s_%s_%s", m.Form, m.Lemma, m.CPOS, m.POS, m.FeatureStr)
}

func Funcs_Lemma_Main_POS(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s", m.CPOS)
	} else {
		return fmt.Sprintf("%s_%s", m.Lemma, m.CPOS)
	}
}

func Funcs_Main_POS(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s", m.CPOS)
	} else {
		return fmt.Sprintf("%s_%s", m.Form, m.CPOS)
	}
}

func Funcs_Main_POS_Prop(m *EMorpheme) string {
	if _, exists := Main_POS[m.CPOS]; exists {
		return fmt.Sprintf("%s_%s", m.CPOS, m.FeatureStr)
	} else {
		return fmt.Sprintf("%s_%s_%s", m.Form, m.CPOS, m.FeatureStr)
	}
}
<|endoftext|>


<commit_before>package main
import (
  "os"
  "log"
  "fmt"
  "net/http"
  "encoding/json"
  "time"
  "strconv"
  "strings"
)

const pollPeriod = time.Second * 10

var url string

var tick = time.NewTicker(pollPeriod).C
var metricsOut = make(chan []Metric)
var metricRequest = make(chan bool)

type Target struct {
  Target string `json:"target"`
  Datapoints []Datapoint `json:"datapoints"`
}
func (t Target) String() string {
    return fmt.Sprintf("target= %s, datapoints= %s", t.Target, t.Datapoints)
}

type Datapoint [2]*float64
func (g Datapoint) String() string {
  if g[0] == nil {
    return fmt.Sprintf("[null, %.0f]", *g[1])
  } else {
    return fmt.Sprintf("[%.1f, %.0f]", *g[0], *g[1])
  }
}

type Metric struct {
  Name string
  Value float64
}
func (m Metric) String() string {
  return m.Name + " " + strconv.FormatFloat(m.Value, 'f', -1, 64) + "\n"
}

var myClient = &http.Client{Timeout: 10 * time.Second}

func poller(channel chan []Metric) {
//  log.Println("Start polling metrics from Graphite")
  metrics := getMetrics(url)
//  log.Println("Got metrics from Graphite", metrics)
  channel <- metrics
//  log.Println("Poll finished")
}

func getMetrics(url string) []Metric {
  log.Println("Retrieving from: "+url)

  data := []Target{}
  getJson(url, &data)
  log.Println(data)
  m := []Metric{}
  for _, t := range data {
    metric := Metric{Name: strings.Replace(t.Target, ".", "_", -1), Value: getLastNonNullValue(t.Datapoints)}
    m = append(m, metric)
  }
  return m
}

func getLastNonNullValue(d []Datapoint) float64 {
  for i:= len(d)-1 ; i>=0 ; i-- {
    if d[i][0] != nil { return *d[i][0] }
  }
  return 0
}

func getJson(url string, target interface{}) error {
    r, err := myClient.Get(url)
    if err != nil {
        return err
    }
    return json.NewDecoder(r.Body).Decode(target)
}

func serveGraphite(w http.ResponseWriter, r *http.Request) {
  if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
  defer r.Body.Close()
//  log.Println("Got request")
  w.Header().Set("Content-Type", "text/plain; charset=utf-8")
  metricRequest <- true
  metrics := <- metricsOut
//  log.Println("Received to handler", metrics)
  c := 0
  for _, m := range metrics {
//    log.Println("writing metric to response #", c ," - " , m)
    fmt.Fprintf(w, m.String() )
    c++
  }
//  log.Println("Finished writing ", c, " metrics")
}

func storage() {
  var metrics = []Metric{}
  var metricsIn = make(chan []Metric)
  go poller(metricsIn)
  for {
    select {
    case <- tick:
      go poller(metricsIn)
      case metrics = <- metricsIn:
//      log.Println("Received metrics from poller", metrics)
    case <- metricRequest:
//      log.Println("Received request for metrics")
      metricsOut <- metrics
//      log.Println("Metrics sent to handler", metrics)
    }
  }
}

func main() {
  var graphiteBaseURL = os.Getenv("GRAPHITE_URL")
  if (graphiteBaseURL == "") { graphiteBaseURL = "http://localhost:8080/render" }
  var graphiteTargets = os.Getenv("TARGETS")
  if (graphiteTargets == "") { graphiteTargets = "*.*" }
  var graphiteParameters = "target="+graphiteTargets+"&from=-50s&format=json"
  // stats_counts.*
  url = graphiteBaseURL+"?"+graphiteParameters
  go storage()

  http.HandleFunc("/", serveGraphite)
  log.Fatal(http.ListenAndServe(":8081", nil))
}<commit_msg>Rework async channels<commit_after>package main
import (
  "os"
  "log"
  "fmt"
  "net/http"
  "encoding/json"
  "time"
  "strconv"
  "strings"
)

const pollPeriod = time.Second * 10
var graphiteBaseURL = os.Getenv("GRAPHITE_URL")
var graphiteTargets = os.Getenv("TARGETS")
var url string

var metricRequest = make(chan chan []Metric)

type Target struct {
  Target string `json:"target"`
  Datapoints []Datapoint `json:"datapoints"`
}
func (t Target) String() string {
    return fmt.Sprintf("target= %s, datapoints= %s", t.Target, t.Datapoints)
}

type Datapoint [2]*float64
func (g Datapoint) String() string {
  if g[0] == nil {
    return fmt.Sprintf("[null, %.0f]", *g[1])
  } else {
    return fmt.Sprintf("[%.1f, %.0f]", *g[0], *g[1])
  }
}

type Metric struct {
  Name string
  Value float64
  Labels map[string]string
}
func (m Metric) String() string {
  labelList := []string{}
  for k, v := range m.Labels {
    labelList = append(labelList, k+"=\""+v+"\"")
  }
  return m.Name + "{" + strings.Join(labelList, ",") + "}" + " " + strconv.FormatFloat(m.Value, 'f', -1, 64) + "\n"
}

var myClient = &http.Client{Timeout: 10 * time.Second}

func poller(channel chan []Metric) {
//  log.Println("Start polling metrics from Graphite")
  metrics := getMetrics(url)
//  log.Println("Poller: Got metrics from Graphite", metrics)
  channel <- metrics
//  log.Println("Poll finished")
}

func getMetrics(url string) []Metric {
  log.Println("Retrieving from: "+url)

  data := []Target{}
  getJson(url, &data)
  log.Println(data)
  m := []Metric{}
  for _, t := range data {
    metric := Metric{Name: strings.Replace(t.Target, ".", "_", -1), Value: getLastNonNullValue(t.Datapoints), Labels: getLabels()}
    m = append(m, metric)
  }
  return m
}

func getLabels() map[string]string {
  return map[string]string{"url": graphiteBaseURL}
}

func getLastNonNullValue(d []Datapoint) float64 {
  for i:= len(d)-1 ; i>=0 ; i-- {
    if d[i][0] != nil { return *d[i][0] }
  }
  return 0
}

func getJson(url string, target interface{}) error {
    r, err := myClient.Get(url)
    if err != nil {
        return err
    }
    return json.NewDecoder(r.Body).Decode(target)
}

func serveGraphite(w http.ResponseWriter, r *http.Request) {
  if r.Method != "GET" {
		http.Error(w, "Method not allowed", 405)
		return
	}
  defer r.Body.Close()
  log.Println("Server: Got request")
  w.Header().Set("Content-Type", "text/plain; charset=utf-8")
  var metricsOut = make(chan []Metric)
  defer close(metricsOut)
  metricRequest <- metricsOut
  metrics := <- metricsOut
  log.Println("Server: Received to handler", metrics)
  c := 0
  for _, m := range metrics {
//    log.Println("writing metric to response #", c ," - " , m)
    fmt.Fprintf(w, m.String() )
    c++
  }
//  log.Println("Finished writing ", c, " metrics")
}

func storage() {
  var metrics = []Metric{}
  var metricsIn = make(chan []Metric)
  var tick = time.NewTicker(pollPeriod).C
  go poller(metricsIn)
  for {
    select {
    case <- tick:
      go poller(metricsIn)
    case metrics = <- metricsIn:
//      log.Println("Received metrics from poller", metrics)
    case c := <- metricRequest:
//      log.Println("Received request for metrics")
      c <- metrics
//      log.Println("Metrics sent to handler", metrics)
    }
  }
}

func main() {
  if (graphiteBaseURL == "") { graphiteBaseURL = "http://localhost:8080/render" }
  if (graphiteTargets == "") { graphiteTargets = "*.*" }
  var graphiteParameters = "target="+graphiteTargets+"&from=-50s&format=json"
  // stats_counts.*
  url = graphiteBaseURL+"?"+graphiteParameters
  go storage()

  http.HandleFunc("/", serveGraphite)
  log.Fatal(http.ListenAndServe(":8081", nil))
}<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package leaderelection

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog"
)

const (
	defaultLeaseDuration = 15 * time.Second
	defaultRenewDeadline = 10 * time.Second
	defaultRetryPeriod   = 5 * time.Second
)

// leaderElection is a convenience wrapper around client-go's leader election library.
type leaderElection struct {
	runFunc func(ctx context.Context)

	// the lockName identifies the leader election config and should be shared across all members
	lockName string
	// the identity is the unique identity of the currently running member
	identity string
	// the namespace to store the lock resource
	namespace string
	// resourceLock defines the type of leaderelection that should be used
	// valid options are resourcelock.LeasesResourceLock, resourcelock.EndpointsResourceLock,
	// and resourcelock.ConfigMapsResourceLock
	resourceLock string

	leaseDuration time.Duration
	renewDeadline time.Duration
	retryPeriod   time.Duration

	clientset kubernetes.Interface
}

// NewLeaderElection returns the default & preferred leader election type
func NewLeaderElection(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return NewLeaderElectionWithLeases(clientset, lockName, runFunc)
}

// NewLeaderElectionWithLeases returns an implementation of leader election using Leases
func NewLeaderElectionWithLeases(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return &leaderElection{
		runFunc:       runFunc,
		lockName:      lockName,
		resourceLock:  resourcelock.LeasesResourceLock,
		leaseDuration: defaultLeaseDuration,
		renewDeadline: defaultRenewDeadline,
		retryPeriod:   defaultRetryPeriod,
		clientset:     clientset,
	}
}

// NewLeaderElectionWithEndpoints returns an implementation of leader election using Endpoints
func NewLeaderElectionWithEndpoints(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return &leaderElection{
		runFunc:       runFunc,
		lockName:      lockName,
		resourceLock:  resourcelock.EndpointsResourceLock,
		leaseDuration: defaultLeaseDuration,
		renewDeadline: defaultRenewDeadline,
		retryPeriod:   defaultRetryPeriod,
		clientset:     clientset,
	}
}

// NewLeaderElectionWithConfigMaps returns an implementation of leader election using ConfigMaps
func NewLeaderElectionWithConfigMaps(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return &leaderElection{
		runFunc:       runFunc,
		lockName:      lockName,
		resourceLock:  resourcelock.ConfigMapsResourceLock,
		leaseDuration: defaultLeaseDuration,
		renewDeadline: defaultRenewDeadline,
		retryPeriod:   defaultRetryPeriod,
		clientset:     clientset,
	}
}

func (l *leaderElection) WithIdentity(identity string) {
	l.identity = identity
}

func (l *leaderElection) WithNamespace(namespace string) {
	l.namespace = namespace
}

func (l *leaderElection) WithLeaseDuration(leaseDuration time.Duration) {
	l.leaseDuration = leaseDuration
}

func (l *leaderElection) WithRenewDeadline(renewDeadline time.Duration) {
	l.renewDeadline = renewDeadline
}

func (l *leaderElection) WithRetryPeriod(retryPeriod time.Duration) {
	l.retryPeriod = retryPeriod
}

func (l *leaderElection) Run() error {
	if l.identity == "" {
		id, err := defaultLeaderElectionIdentity()
		if err != nil {
			return fmt.Errorf("error getting the default leader identity: %v", err)
		}

		l.identity = id
	}

	if l.namespace == "" {
		l.namespace = inClusterNamespace()
	}

	broadcaster := record.NewBroadcaster()
	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: l.clientset.CoreV1().Events(l.namespace)})
	eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("%s/%s", l.lockName, string(l.identity))})

	rlConfig := resourcelock.ResourceLockConfig{
		Identity:      sanitizeName(l.identity),
		EventRecorder: eventRecorder,
	}

	lock, err := resourcelock.New(l.resourceLock, l.namespace, sanitizeName(l.lockName), l.clientset.CoreV1(), l.clientset.CoordinationV1(), rlConfig)
	if err != nil {
		return err
	}

	leaderConfig := leaderelection.LeaderElectionConfig{
		Lock:          lock,
		LeaseDuration: l.leaseDuration,
		RenewDeadline: l.renewDeadline,
		RetryPeriod:   l.retryPeriod,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: func(ctx context.Context) {
				klog.V(2).Info("became leader, starting")
				l.runFunc(ctx)
			},
			OnStoppedLeading: func() {
				klog.Fatal("stopped leading")
			},
			OnNewLeader: func(identity string) {
				klog.V(3).Infof("new leader detected, current leader: %s", identity)
			},
		},
	}

	leaderelection.RunOrDie(context.TODO(), leaderConfig)
	return nil // should never reach here
}

func defaultLeaderElectionIdentity() (string, error) {
	return os.Hostname()
}

// sanitizeName sanitizes the provided string so it can be consumed by leader election library
func sanitizeName(name string) string {
	re := regexp.MustCompile("[^a-zA-Z0-9-]")
	name = re.ReplaceAllString(name, "-")
	if name[len(name)-1] == '-' {
		// name must not end with '-'
		name = name + "X"
	}
	return name
}

// inClusterNamespace returns the namespace in which the pod is running in by checking
// the env var POD_NAMESPACE, then the file /var/run/secrets/kubernetes.io/serviceaccount/namespace.
// if neither returns a valid namespace, the "default" namespace is returned
func inClusterNamespace() string {
	if ns := os.Getenv("POD_NAMESPACE"); ns != "" {
		return ns
	}

	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}

	return "default"
}
<commit_msg>leaderelection add context<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package leaderelection

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"time"

	"k8s.io/api/core/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"k8s.io/klog"
)

const (
	defaultLeaseDuration = 15 * time.Second
	defaultRenewDeadline = 10 * time.Second
	defaultRetryPeriod   = 5 * time.Second
)

// leaderElection is a convenience wrapper around client-go's leader election library.
type leaderElection struct {
	runFunc func(ctx context.Context)

	// the lockName identifies the leader election config and should be shared across all members
	lockName string
	// the identity is the unique identity of the currently running member
	identity string
	// the namespace to store the lock resource
	namespace string
	// resourceLock defines the type of leaderelection that should be used
	// valid options are resourcelock.LeasesResourceLock, resourcelock.EndpointsResourceLock,
	// and resourcelock.ConfigMapsResourceLock
	resourceLock string

	leaseDuration time.Duration
	renewDeadline time.Duration
	retryPeriod   time.Duration

	ctx context.Context

	clientset kubernetes.Interface
}

// NewLeaderElection returns the default & preferred leader election type
func NewLeaderElection(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return NewLeaderElectionWithLeases(clientset, lockName, runFunc)
}

// NewLeaderElectionWithLeases returns an implementation of leader election using Leases
func NewLeaderElectionWithLeases(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return &leaderElection{
		runFunc:       runFunc,
		lockName:      lockName,
		resourceLock:  resourcelock.LeasesResourceLock,
		leaseDuration: defaultLeaseDuration,
		renewDeadline: defaultRenewDeadline,
		retryPeriod:   defaultRetryPeriod,
		clientset:     clientset,
	}
}

// NewLeaderElectionWithEndpoints returns an implementation of leader election using Endpoints
func NewLeaderElectionWithEndpoints(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return &leaderElection{
		runFunc:       runFunc,
		lockName:      lockName,
		resourceLock:  resourcelock.EndpointsResourceLock,
		leaseDuration: defaultLeaseDuration,
		renewDeadline: defaultRenewDeadline,
		retryPeriod:   defaultRetryPeriod,
		clientset:     clientset,
	}
}

// NewLeaderElectionWithConfigMaps returns an implementation of leader election using ConfigMaps
func NewLeaderElectionWithConfigMaps(clientset kubernetes.Interface, lockName string, runFunc func(ctx context.Context)) *leaderElection {
	return &leaderElection{
		runFunc:       runFunc,
		lockName:      lockName,
		resourceLock:  resourcelock.ConfigMapsResourceLock,
		leaseDuration: defaultLeaseDuration,
		renewDeadline: defaultRenewDeadline,
		retryPeriod:   defaultRetryPeriod,
		clientset:     clientset,
	}
}

func (l *leaderElection) WithIdentity(identity string) {
	l.identity = identity
}

func (l *leaderElection) WithNamespace(namespace string) {
	l.namespace = namespace
}

func (l *leaderElection) WithLeaseDuration(leaseDuration time.Duration) {
	l.leaseDuration = leaseDuration
}

func (l *leaderElection) WithRenewDeadline(renewDeadline time.Duration) {
	l.renewDeadline = renewDeadline
}

func (l *leaderElection) WithRetryPeriod(retryPeriod time.Duration) {
	l.retryPeriod = retryPeriod
}

// WithContext Add context
func (l *leaderElection) WithContext(ctx context.Context) {
	l.ctx = ctx
}

func (l *leaderElection) Run() error {
	if l.identity == "" {
		id, err := defaultLeaderElectionIdentity()
		if err != nil {
			return fmt.Errorf("error getting the default leader identity: %v", err)
		}

		l.identity = id
	}

	if l.namespace == "" {
		l.namespace = inClusterNamespace()
	}

	broadcaster := record.NewBroadcaster()
	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: l.clientset.CoreV1().Events(l.namespace)})
	eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: fmt.Sprintf("%s/%s", l.lockName, string(l.identity))})

	rlConfig := resourcelock.ResourceLockConfig{
		Identity:      sanitizeName(l.identity),
		EventRecorder: eventRecorder,
	}

	lock, err := resourcelock.New(l.resourceLock, l.namespace, sanitizeName(l.lockName), l.clientset.CoreV1(), l.clientset.CoordinationV1(), rlConfig)
	if err != nil {
		return err
	}

	leaderConfig := leaderelection.LeaderElectionConfig{
		Lock:          lock,
		LeaseDuration: l.leaseDuration,
		RenewDeadline: l.renewDeadline,
		RetryPeriod:   l.retryPeriod,
		Callbacks: leaderelection.LeaderCallbacks{
			OnStartedLeading: func(ctx context.Context) {
				klog.V(2).Info("became leader, starting")
				l.runFunc(ctx)
			},
			OnStoppedLeading: func() {
				klog.Fatal("stopped leading")
			},
			OnNewLeader: func(identity string) {
				klog.V(3).Infof("new leader detected, current leader: %s", identity)
			},
		},
	}

	ctx := l.ctx
	if ctx == nil {
		ctx = context.Background()
	}
	leaderelection.RunOrDie(ctx, leaderConfig)
	return nil // should never reach here
}

func defaultLeaderElectionIdentity() (string, error) {
	return os.Hostname()
}

// sanitizeName sanitizes the provided string so it can be consumed by leader election library
func sanitizeName(name string) string {
	re := regexp.MustCompile("[^a-zA-Z0-9-]")
	name = re.ReplaceAllString(name, "-")
	if name[len(name)-1] == '-' {
		// name must not end with '-'
		name = name + "X"
	}
	return name
}

// inClusterNamespace returns the namespace in which the pod is running in by checking
// the env var POD_NAMESPACE, then the file /var/run/secrets/kubernetes.io/serviceaccount/namespace.
// if neither returns a valid namespace, the "default" namespace is returned
func inClusterNamespace() string {
	if ns := os.Getenv("POD_NAMESPACE"); ns != "" {
		return ns
	}

	if data, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); err == nil {
		if ns := strings.TrimSpace(string(data)); len(ns) > 0 {
			return ns
		}
	}

	return "default"
}
<|endoftext|>


<commit_before>package rotator

import (
	"fmt"
	"os"
	"strconv"
	"sync"
)

const (
	defaultRotationSize = 1024 * 1024 * 10
	defaultMaxRotation  = 999
)

// SizeRotator is file writer which rotates files by size
type SizeRotator struct {
	path         string     // base file path
	totalSize    int64      // current file size
	file         *os.File   // current file
	mutex        sync.Mutex // lock
	RotationSize int64      // size threshold of the rotation
	MaxRotation  int        // maximum count of the rotation
}

// Write bytes to the file. If binaries exceeds rotation threshold,
// it will automatically rotate the file.
func (r *SizeRotator) Write(bytes []byte) (n int, err error) {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.file == nil {
		// Check file existence
		stat, _ := os.Lstat(r.path)
		if stat != nil {
			// Update initial size by file size
			r.totalSize = stat.Size()
		}
	}

	// Do rotate when size exceeded
	if r.totalSize+int64(len(bytes)) > r.RotationSize {
		r.file.Close()
		r.file = nil
		// Remove oldest file (in case it exists)
		dpath := r.path + "." + strconv.Itoa(r.MaxRotation)
		err := os.Remove(dpath)
		if err != nil && !os.IsNotExist(err) {
			return 0, fmt.Errorf("Unable to delete oldest file: %v", err)
		}

		// Rename existing files
		for i := r.MaxRotation - 1; i >= 0; i-- {
			opath := r.path
			if i != 0 {
				opath = opath + "." + strconv.Itoa(i)
			}
			npath := r.path + "." + strconv.Itoa(i+1)
			err := os.Rename(opath, npath)
			if err != nil && !os.IsNotExist(err) {
				return 0, fmt.Errorf("Unable to rename old file %v to %v: %v", opath, npath, err)
			}
		}
	}

	if r.file == nil {
		r.file, err = os.OpenFile(r.path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
		if err != nil {
			return 0, err
		}
		// Switch current date
		r.totalSize = 0
	}

	n, err = r.file.Write(bytes)
	r.totalSize += int64(n)
	return n, err
}

// WriteString writes strings to the file. If binaries exceeds rotation threshold,
// it will automatically rotate the file.
func (r *SizeRotator) WriteString(str string) (n int, err error) {
	return r.Write([]byte(str))
}

// Close the file
func (r *SizeRotator) Close() error {
	return r.file.Close()
}

// NewSizeRotator creates new writer of the file
func NewSizeRotator(path string) *SizeRotator {
	return &SizeRotator{
		path:         path,
		RotationSize: defaultRotationSize,
		MaxRotation:  defaultMaxRotation,
	}
}
<commit_msg>Code review fixes<commit_after>package rotator

import (
	"fmt"
	"os"
	"strconv"
	"sync"
)

const (
	defaultRotationSize = 1024 * 1024 * 10
	defaultMaxRotation  = 999
)

// SizeRotator is file writer which rotates files by size
type SizeRotator struct {
	path         string     // base file path
	totalSize    int64      // current file size
	file         *os.File   // current file
	mutex        sync.Mutex // lock
	RotationSize int64      // size threshold of the rotation
	MaxRotation  int        // maximum count of the rotation
}

// Write bytes to the file. If binaries exceeds rotation threshold,
// it will automatically rotate the file.
func (r *SizeRotator) Write(bytes []byte) (n int, err error) {

	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.file == nil {
		// Check file existence
		stat, _ := os.Lstat(r.path)
		if stat != nil {
			// Update initial size by file size
			r.totalSize = stat.Size()
		}
	}

	// Do rotate when size exceeded
	if r.totalSize+int64(len(bytes)) > r.RotationSize {
		if r.file != nil {
			r.file.Close()
			r.file = nil
		}
		// Remove oldest file (in case it exists)
		dpath := r.path + "." + strconv.Itoa(r.MaxRotation)
		err := os.Remove(dpath)
		if err != nil && !os.IsNotExist(err) {
			return 0, fmt.Errorf("Unable to delete oldest file: %v", err)
		}

		// Rename existing files
		for i := r.MaxRotation - 1; i >= 0; i-- {
			opath := r.path
			if i != 0 {
				opath = opath + "." + strconv.Itoa(i)
			}
			npath := r.path + "." + strconv.Itoa(i+1)
			err := os.Rename(opath, npath)
			if err != nil && !os.IsNotExist(err) {
				return 0, fmt.Errorf("Unable to rename old file %v to %v: %v", opath, npath, err)
			}
		}
	}

	if r.file == nil {
		r.file, err = os.OpenFile(r.path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
		if err != nil {
			return 0, err
		}
		// Switch current date
		r.totalSize = 0
	}

	n, err = r.file.Write(bytes)
	r.totalSize += int64(n)
	return n, err
}

// WriteString writes strings to the file. If binaries exceeds rotation threshold,
// it will automatically rotate the file.
func (r *SizeRotator) WriteString(str string) (n int, err error) {
	return r.Write([]byte(str))
}

// Close the file
func (r *SizeRotator) Close() error {
	return r.file.Close()
}

// NewSizeRotator creates new writer of the file
func NewSizeRotator(path string) *SizeRotator {
	return &SizeRotator{
		path:         path,
		RotationSize: defaultRotationSize,
		MaxRotation:  defaultMaxRotation,
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2016 Nicolas Lamirault <nicolas.lamirault@gmail.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

// Version represents the application version using SemVer
const Version string = "0.1.0"
<commit_msg>bump version<commit_after>// Copyright (C) 2016 Nicolas Lamirault <nicolas.lamirault@gmail.com>

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package version

// Version represents the application version using SemVer
const Version string = "0.2.0"
<|endoftext|>


<commit_before>package transactor

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/chop-dbhi/origins"
	"github.com/chop-dbhi/origins/dal"
	"github.com/chop-dbhi/origins/storage"
	"github.com/sirupsen/logrus"
)

const (
	macrosDomain = "origins.macros"
)

var (
	ErrCanceled       = errors.New("transactor: canceled")
	ErrNoID           = errors.New("transactor: could not create tx id")
	ErrCommitConflict = errors.New("transactor: commit conflict")
	ErrReceiveTimeout = errors.New("transactor: receive timeout")
	ErrNoDomain       = errors.New("transactor: no fact domain")
	ErrCouldNotRoute  = errors.New("transactor: could not route")
)

// txid increments a global transaction ID.
func txid(tx storage.Tx) (uint64, error) {
	return tx.Incr("origins", "tx")
}

// Options are used to supply default values as well as alter the behavior
// of a running transaction.
type Options struct {
	// Default domain for unbounded facts. If this is ommitted and a fact
	// does not specify a domain, an error will occur.
	DefaultDomain string

	// Duration of time wait to receive facts before timing out the transaction.
	ReceiveWait time.Duration

	// The router to use for the transaction.
	Router Router

	// Defines the buffer size of the channel that receives facts for processing.
	// Increasing this may increase throughput at the expense of memory.
	BufferSize int

	// If true, a zeroed fact time will be set to the transaction start time. This
	// is useful for facts that are considered "new in the world".
	SetDefaultTime bool
}

// DefaultOptions hold the default options for a transaction.
var DefaultOptions = Options{
	ReceiveWait: time.Minute,
	BufferSize:  1000,
}

// Transaction is the entrypoint for transacting facts.
type Transaction struct {
	// Unique ID for the transaction.
	ID uint64

	// The start and end time of transaction.
	StartTime time.Time
	EndTime   time.Time

	// The error that caused the transaction to fail is one occurs.
	Error error

	Engine storage.Engine

	options Options

	// Router for the transaction.
	router Router

	// Main channel received facts are received by.
	stream chan *origins.Fact

	// Channel that can be passed around for signaling the
	// transaction stream has been closed.
	done chan struct{}

	// Shared error channel for all goroutines to communicate when an
	// error occurs.
	errch chan error

	// Wait groups for main goroutine and pipelines.
	mainwg *sync.WaitGroup
	pipewg *sync.WaitGroup

	// Pipelines and channels.
	pipes map[Pipeline]chan<- *origins.Fact
}

// Info holds information about a transaction.
type Info struct {
	ID        uint64
	StartTime time.Time
	EndTime   time.Time
	Duration  time.Duration
	Pipelines int
	Domains   []string
	Bytes     int
	Count     int
}

// Stats returns the stats of the transaction which aggregates
// them from the pipelines.
func (tx *Transaction) Info() *Info {
	var (
		domains      []string
		bytes, count int

		stats *Stats
	)

	for pipe := range tx.pipes {
		stats = pipe.Stats()
		domains = append(domains, stats.Domains...)
		bytes += stats.Bytes
		count += stats.Count
	}

	return &Info{
		ID:        tx.ID,
		StartTime: tx.StartTime,
		EndTime:   tx.EndTime,
		Duration:  tx.EndTime.Sub(tx.StartTime),
		Pipelines: len(tx.pipes),
		Domains:   domains,
		Bytes:     bytes,
		Count:     count,
	}
}

// evaluate evaluates a fact against the log.
func (tx *Transaction) evaluate(f *origins.Fact) error {
	if f.Domain == "" {
		if tx.options.DefaultDomain == "" {
			return ErrNoDomain
		}

		f.Domain = tx.options.DefaultDomain
	}

	// Default to assertion.
	if f.Operation == origins.Noop {
		f.Operation = origins.Assertion
	}

	// Default to fact domain.
	if f.Entity.Domain == "" {
		f.Entity.Domain = f.Domain
	}

	if f.Attribute.Domain == "" {
		f.Attribute.Domain = f.Domain
	}

	// Set fact time to the transaction time if flagged to do so.
	if f.Time.IsZero() && tx.options.SetDefaultTime {
		f.Time = tx.StartTime
	}

	return nil
}

// macro takes a fact and resolves the macro.
func (tx *Transaction) macro(fact *origins.Fact) error {
	// Macro domain. Fact is about the domain itself.
	if fact.Entity.Domain == "origins.macros" {
		switch fact.Entity.Name {
		case "domain":
			fact.Entity.Name = fact.Domain
			fact.Domain = "origins.domains"
			fact.Entity.Domain = ""

		case "tx":
			fact.Domain = fmt.Sprintf("origins.tx.%s", fact.Domain)
			fact.Entity.Domain = ""
			fact.Entity.Name = tx.StartTime.String()

		default:
			return fmt.Errorf("transactor: unknown entity macro: %s", fact.Entity.Name)
		}
	}

	if fact.Value.Domain == "origins.macros" {
		switch fact.Value.Name {
		case "now":
			fact.Value.Domain = ""
			fact.Value.Name = tx.StartTime.String()

		default:
			return fmt.Errorf("transactor: unknown value macro: %s", fact.Value.Name)
		}
	}

	return nil
}

// route uses the router to get the pipeline for the fact.
// Each pipeline
func (tx *Transaction) route(fact *origins.Fact) error {
	var (
		ok   bool
		err  error
		pipe Pipeline
		ch   chan<- *origins.Fact
	)

	// Evaluate.
	if err = tx.evaluate(fact); err != nil {
		return err
	}

	// Convert.
	if err = tx.macro(fact); err != nil {
		return err
	}

	// Route.
	if pipe, err = tx.router.Route(fact); err != nil {
		logrus.Debugf("transactor: could not route: %s", err)
		return ErrCouldNotRoute
	}

	// Get or spawn a pipeline.
	if ch, ok = tx.pipes[pipe]; !ok {
		ch = tx.spawn(pipe)
		tx.pipes[pipe] = ch
	}

	// Send fact to the pipeline.
	ch <- fact

	return nil
}

func (tx *Transaction) run() {
	// Start the receiver. This blocks until the stream ends or is interrupted.
	tx.Error = tx.receive()

	// Signal the transaction is closed.
	close(tx.done)

	// Wait for the pipelines to finish there work.
	tx.pipewg.Wait()

	// Mark the end time of processing.
	tx.EndTime = time.Now().UTC()

	// Complete the transaction by committing or aborting.
	tx.complete()

	tx.mainwg.Done()
}

// receive is the coordinator for receiving and routing facts.
func (tx *Transaction) receive() error {
	var (
		err  error
		fact *origins.Fact
	)

	logrus.Debugf("transactor(%d): begin receiving facts", tx.ID)

	for {
		select {
		// An error occurred in a pipeline.
		case err = <-tx.errch:
			logrus.Debugf("transactor(%d): %s", tx.ID, err)
			close(tx.stream)
			return err

		// Receive facts from stream and route to pipeline.
		// If an error occurs while routing, stop processing.
		case fact = <-tx.stream:
			if fact == nil {
				logrus.Debugf("transactor(%d): end of stream", tx.ID)
				return nil
			}

			if err = tx.route(fact); err != nil {
				logrus.Debugf("transactor(%d): error routing fact", tx.ID)
				return err
			}

		// Transaction timeout.
		case <-time.After(tx.options.ReceiveWait):
			logrus.Debugf("transactor(%d): receive timeout", tx.ID)
			return ErrReceiveTimeout
		}
	}

	return nil
}

// complete commits or aborts the transaction depending if an error occurred
// during processing.
func (tx *Transaction) complete() {
	var err error

	// No error in the transaction, commit the transaction.
	if tx.Error == nil {
		if err = tx.commit(); err != nil {
			logrus.Errorf("transactor(%d): commit failed: %s", tx.ID, err)
		} else {
			logrus.Debugf("transactor(%d): commit succeeded", tx.ID)
		}
	}

	// Error occurred in the transaction or during the commit. Attempt to abort.
	// TODO: if the abort fails, how can the storage garbage be reclaimed?
	if tx.Error != nil || err != nil {
		if err = tx.abort(); err != nil {
			logrus.Errorf("transactor(%d): abort failed: %s", tx.ID, err)
		} else {
			logrus.Debugf("transactor(%d): abort succeeded", tx.ID)
		}
	}

	// Update transaction record.
	_, err = dal.SetTransaction(tx.Engine, &dal.Transaction{
		ID:        tx.ID,
		StartTime: tx.StartTime,
		EndTime:   tx.EndTime,
		Error:     tx.Error,
	})

	if err != nil {
		logrus.Errorf("transactor(%d): error writing transaction record: %s", tx.ID, err)
	}
}

// commit commits all the pipelines in a transaction.
func (tx *Transaction) commit() error {
	return tx.Engine.Multi(func(etx storage.Tx) error {
		for pipe, _ := range tx.pipes {
			if err := pipe.Commit(etx); err != nil {
				return err
			}

			logrus.Debugf("transactor(%d): committed pipeline %v", tx.ID, pipe)
		}

		return nil
	})
}

// abort aborts all of the pipelines in a transaction.
func (tx *Transaction) abort() error {
	return tx.Engine.Multi(func(etx storage.Tx) error {
		for pipe, _ := range tx.pipes {
			if err := pipe.Abort(etx); err != nil {
				return err
			}

			logrus.Debugf("transactor(%d): aborted pipeline %v", tx.ID, pipe)
		}

		return nil
	})
}

// spawn all values on the channel on the pipeline.
func (tx *Transaction) spawn(pipe Pipeline) chan<- *origins.Fact {
	ch := make(chan *origins.Fact)

	tx.pipes[pipe] = ch
	tx.pipewg.Add(1)

	go func() {
		defer func() {
			close(ch)
			tx.pipewg.Done()
		}()

		var (
			err  error
			fact *origins.Fact
		)

		// Initialize the pipeline. If an error occurs, send it to the transaction's error channel
		// which will trigger the cancellation procedure.
		if err = pipe.Init(tx); err != nil {
			tx.errch <- err
			return
		}

		logrus.Debugf("transactor(%d): initialized pipeline %T(%s)", tx.ID, pipe, pipe)

		// Reads facts from the channel until there are no more or break
		// if an error occurs from the pipeline.
		for {
			select {
			case <-tx.done:
				return

			case fact = <-ch:
				if err = pipe.Receive(fact); err != nil {
					tx.errch <- err
					return
				}
			}
		}
	}()

	return ch
}

// Write writes a fact to the transaction.
func (tx *Transaction) Write(fact *origins.Fact) error {
	tx.stream <- fact
	return nil
}

// Cancel cancels the transaction.
func (tx *Transaction) Cancel() error {
	tx.errch <- ErrCanceled
	tx.mainwg.Wait()
	return tx.Error
}

// Commit commits the transaction. All head of all affected logs will be
// atomically updated to make the transacted data visible to clients.
func (tx *Transaction) Commit() error {
	close(tx.stream)
	tx.mainwg.Wait()
	return tx.Error
}

// Consume reads data from a stream and writes it to the transaction.
func (tx *Transaction) Consume(pub origins.Publisher) error {
	var (
		ok   bool
		err  error
		fact *origins.Fact
	)

	// Subscribe to the publisher. It takes a channel to signal when
	// this consumer is done and returns a channel that produces facts.
	ch, errch := pub.Subscribe(tx.done)

	// Consume facts until the producer is closed. This may occur upstream
	// by the producer itself or the transaction is closed prematurely.
	for {
		select {
		case err = <-errch:
			return err

		case fact, ok = <-ch:
			// Publisher closed the channel.
			if !ok {
				return nil
			}

			tx.Write(fact)
		}
	}

	return nil
}

// New initializes and returns a transaction for passed storage engine. The options
// are used to change the behavior of the transaction itself.
func New(engine storage.Engine, options Options) (*Transaction, error) {
	var (
		id  uint64
		err error
	)

	now := time.Now().UTC()

	// Increment the transaction ID and store the record.
	err = engine.Multi(func(tx storage.Tx) error {
		if id, err = txid(tx); err != nil {
			return err
		}

		_, err = dal.SetTransaction(tx, &dal.Transaction{
			ID:        id,
			StartTime: now,
		})

		return err
	})

	if err != nil {
		logrus.Errorf("transactor: could not create transaction: %s", err)
		return nil, ErrNoID
	}

	if options.Router == nil {
		options.Router = NewDomainRouter()
	}

	if options.ReceiveWait == 0 {
		options.ReceiveWait = DefaultOptions.ReceiveWait
	}

	if options.BufferSize == 0 {
		options.BufferSize = DefaultOptions.BufferSize
	}

	tx := Transaction{
		ID:        id,
		StartTime: time.Now().UTC(),
		Engine:    engine,
		options:   options,
		pipes:     make(map[Pipeline]chan<- *origins.Fact),
		router:    options.Router,
		stream:    make(chan *origins.Fact, options.BufferSize),
		done:      make(chan struct{}),
		errch:     make(chan error),
		pipewg:    &sync.WaitGroup{},
		mainwg:    &sync.WaitGroup{},
	}

	tx.mainwg.Add(1)

	// Run transaction.
	go tx.run()

	return &tx, nil
}
<commit_msg>Remove redundant pipeline registration<commit_after>package transactor

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/chop-dbhi/origins"
	"github.com/chop-dbhi/origins/dal"
	"github.com/chop-dbhi/origins/storage"
	"github.com/sirupsen/logrus"
)

const (
	macrosDomain = "origins.macros"
)

var (
	ErrCanceled       = errors.New("transactor: canceled")
	ErrNoID           = errors.New("transactor: could not create tx id")
	ErrCommitConflict = errors.New("transactor: commit conflict")
	ErrReceiveTimeout = errors.New("transactor: receive timeout")
	ErrNoDomain       = errors.New("transactor: no fact domain")
	ErrCouldNotRoute  = errors.New("transactor: could not route")
)

// txid increments a global transaction ID.
func txid(tx storage.Tx) (uint64, error) {
	return tx.Incr("origins", "tx")
}

// Options are used to supply default values as well as alter the behavior
// of a running transaction.
type Options struct {
	// Default domain for unbounded facts. If this is ommitted and a fact
	// does not specify a domain, an error will occur.
	DefaultDomain string

	// Duration of time wait to receive facts before timing out the transaction.
	ReceiveWait time.Duration

	// The router to use for the transaction.
	Router Router

	// Defines the buffer size of the channel that receives facts for processing.
	// Increasing this may increase throughput at the expense of memory.
	BufferSize int

	// If true, a zeroed fact time will be set to the transaction start time. This
	// is useful for facts that are considered "new in the world".
	SetDefaultTime bool
}

// DefaultOptions hold the default options for a transaction.
var DefaultOptions = Options{
	ReceiveWait: time.Minute,
	BufferSize:  1000,
}

// Transaction is the entrypoint for transacting facts.
type Transaction struct {
	// Unique ID for the transaction.
	ID uint64

	// The start and end time of transaction.
	StartTime time.Time
	EndTime   time.Time

	// The error that caused the transaction to fail is one occurs.
	Error error

	Engine storage.Engine

	options Options

	// Router for the transaction.
	router Router

	// Main channel received facts are received by.
	stream chan *origins.Fact

	// Channel that can be passed around for signaling the
	// transaction stream has been closed.
	done chan struct{}

	// Shared error channel for all goroutines to communicate when an
	// error occurs.
	errch chan error

	// Wait groups for main goroutine and pipelines.
	mainwg *sync.WaitGroup
	pipewg *sync.WaitGroup

	// Pipelines and channels.
	pipes map[Pipeline]chan<- *origins.Fact
}

// Info holds information about a transaction.
type Info struct {
	ID        uint64
	StartTime time.Time
	EndTime   time.Time
	Duration  time.Duration
	Pipelines int
	Domains   []string
	Bytes     int
	Count     int
}

// Stats returns the stats of the transaction which aggregates
// them from the pipelines.
func (tx *Transaction) Info() *Info {
	var (
		domains      []string
		bytes, count int

		stats *Stats
	)

	for pipe := range tx.pipes {
		stats = pipe.Stats()
		domains = append(domains, stats.Domains...)
		bytes += stats.Bytes
		count += stats.Count
	}

	return &Info{
		ID:        tx.ID,
		StartTime: tx.StartTime,
		EndTime:   tx.EndTime,
		Duration:  tx.EndTime.Sub(tx.StartTime),
		Pipelines: len(tx.pipes),
		Domains:   domains,
		Bytes:     bytes,
		Count:     count,
	}
}

// evaluate evaluates a fact against the log.
func (tx *Transaction) evaluate(f *origins.Fact) error {
	if f.Domain == "" {
		if tx.options.DefaultDomain == "" {
			return ErrNoDomain
		}

		f.Domain = tx.options.DefaultDomain
	}

	// Default to assertion.
	if f.Operation == origins.Noop {
		f.Operation = origins.Assertion
	}

	// Default to fact domain.
	if f.Entity.Domain == "" {
		f.Entity.Domain = f.Domain
	}

	if f.Attribute.Domain == "" {
		f.Attribute.Domain = f.Domain
	}

	// Set fact time to the transaction time if flagged to do so.
	if f.Time.IsZero() && tx.options.SetDefaultTime {
		f.Time = tx.StartTime
	}

	return nil
}

// macro takes a fact and resolves the macro.
func (tx *Transaction) macro(fact *origins.Fact) error {
	// Macro domain. Fact is about the domain itself.
	if fact.Entity.Domain == "origins.macros" {
		switch fact.Entity.Name {
		case "domain":
			fact.Entity.Name = fact.Domain
			fact.Domain = "origins.domains"
			fact.Entity.Domain = ""

		case "tx":
			fact.Domain = fmt.Sprintf("origins.tx.%s", fact.Domain)
			fact.Entity.Domain = ""
			fact.Entity.Name = tx.StartTime.String()

		default:
			return fmt.Errorf("transactor: unknown entity macro: %s", fact.Entity.Name)
		}
	}

	if fact.Value.Domain == "origins.macros" {
		switch fact.Value.Name {
		case "now":
			fact.Value.Domain = ""
			fact.Value.Name = tx.StartTime.String()

		default:
			return fmt.Errorf("transactor: unknown value macro: %s", fact.Value.Name)
		}
	}

	return nil
}

// route uses the router to get the pipeline for the fact.
// Each pipeline
func (tx *Transaction) route(fact *origins.Fact) error {
	var (
		ok   bool
		err  error
		pipe Pipeline
		ch   chan<- *origins.Fact
	)

	// Evaluate.
	if err = tx.evaluate(fact); err != nil {
		return err
	}

	// Convert.
	if err = tx.macro(fact); err != nil {
		return err
	}

	// Route.
	if pipe, err = tx.router.Route(fact); err != nil {
		logrus.Debugf("transactor: could not route: %s", err)
		return ErrCouldNotRoute
	}

	// Get or spawn a pipeline.
	if ch, ok = tx.pipes[pipe]; !ok {
		ch = tx.spawn(pipe)
		tx.pipes[pipe] = ch
		tx.pipewg.Add(1)
	}

	// Send fact to the pipeline.
	ch <- fact

	return nil
}

func (tx *Transaction) run() {
	// Start the receiver. This blocks until the stream ends or is interrupted.
	tx.Error = tx.receive()

	// Signal the transaction is closed.
	close(tx.done)

	// Wait for the pipelines to finish there work.
	tx.pipewg.Wait()

	// Mark the end time of processing.
	tx.EndTime = time.Now().UTC()

	// Complete the transaction by committing or aborting.
	tx.complete()

	tx.mainwg.Done()
}

// receive is the coordinator for receiving and routing facts.
func (tx *Transaction) receive() error {
	var (
		err  error
		fact *origins.Fact
	)

	logrus.Debugf("transactor(%d): begin receiving facts", tx.ID)

	for {
		select {
		// An error occurred in a pipeline.
		case err = <-tx.errch:
			logrus.Debugf("transactor(%d): %s", tx.ID, err)
			close(tx.stream)
			return err

		// Receive facts from stream and route to pipeline.
		// If an error occurs while routing, stop processing.
		case fact = <-tx.stream:
			if fact == nil {
				logrus.Debugf("transactor(%d): end of stream", tx.ID)
				return nil
			}

			if err = tx.route(fact); err != nil {
				logrus.Debugf("transactor(%d): error routing fact", tx.ID)
				return err
			}

		// Transaction timeout.
		case <-time.After(tx.options.ReceiveWait):
			logrus.Debugf("transactor(%d): receive timeout", tx.ID)
			return ErrReceiveTimeout
		}
	}

	return nil
}

// complete commits or aborts the transaction depending if an error occurred
// during processing.
func (tx *Transaction) complete() {
	var err error

	// No error in the transaction, commit the transaction.
	if tx.Error == nil {
		if err = tx.commit(); err != nil {
			logrus.Errorf("transactor(%d): commit failed: %s", tx.ID, err)
		} else {
			logrus.Debugf("transactor(%d): commit succeeded", tx.ID)
		}
	}

	// Error occurred in the transaction or during the commit. Attempt to abort.
	// TODO: if the abort fails, how can the storage garbage be reclaimed?
	if tx.Error != nil || err != nil {
		if err = tx.abort(); err != nil {
			logrus.Errorf("transactor(%d): abort failed: %s", tx.ID, err)
		} else {
			logrus.Debugf("transactor(%d): abort succeeded", tx.ID)
		}
	}

	// Update transaction record.
	_, err = dal.SetTransaction(tx.Engine, &dal.Transaction{
		ID:        tx.ID,
		StartTime: tx.StartTime,
		EndTime:   tx.EndTime,
		Error:     tx.Error,
	})

	if err != nil {
		logrus.Errorf("transactor(%d): error writing transaction record: %s", tx.ID, err)
	}
}

// commit commits all the pipelines in a transaction.
func (tx *Transaction) commit() error {
	return tx.Engine.Multi(func(etx storage.Tx) error {
		for pipe, _ := range tx.pipes {
			if err := pipe.Commit(etx); err != nil {
				return err
			}

			logrus.Debugf("transactor(%d): committed pipeline %v", tx.ID, pipe)
		}

		return nil
	})
}

// abort aborts all of the pipelines in a transaction.
func (tx *Transaction) abort() error {
	return tx.Engine.Multi(func(etx storage.Tx) error {
		for pipe, _ := range tx.pipes {
			if err := pipe.Abort(etx); err != nil {
				return err
			}

			logrus.Debugf("transactor(%d): aborted pipeline %v", tx.ID, pipe)
		}

		return nil
	})
}

// spawn all values on the channel on the pipeline.
func (tx *Transaction) spawn(pipe Pipeline) chan<- *origins.Fact {
	ch := make(chan *origins.Fact)

	go func() {
		defer func() {
			close(ch)
			tx.pipewg.Done()
		}()

		var (
			err  error
			fact *origins.Fact
		)

		// Initialize the pipeline. If an error occurs, send it to the transaction's error channel
		// which will trigger the cancellation procedure.
		if err = pipe.Init(tx); err != nil {
			tx.errch <- err
			return
		}

		logrus.Debugf("transactor(%d): initialized pipeline %T(%s)", tx.ID, pipe, pipe)

		// Reads facts from the channel until there are no more or break
		// if an error occurs from the pipeline.
		for {
			select {
			case <-tx.done:
				return

			case fact = <-ch:
				if err = pipe.Receive(fact); err != nil {
					tx.errch <- err
					return
				}
			}
		}
	}()

	return ch
}

// Write writes a fact to the transaction.
func (tx *Transaction) Write(fact *origins.Fact) error {
	tx.stream <- fact
	return nil
}

// Cancel cancels the transaction.
func (tx *Transaction) Cancel() error {
	tx.errch <- ErrCanceled
	tx.mainwg.Wait()
	return tx.Error
}

// Commit commits the transaction. All head of all affected logs will be
// atomically updated to make the transacted data visible to clients.
func (tx *Transaction) Commit() error {
	close(tx.stream)
	tx.mainwg.Wait()
	return tx.Error
}

// Consume reads data from a stream and writes it to the transaction.
func (tx *Transaction) Consume(pub origins.Publisher) error {
	var (
		ok   bool
		err  error
		fact *origins.Fact
	)

	// Subscribe to the publisher. It takes a channel to signal when
	// this consumer is done and returns a channel that produces facts.
	ch, errch := pub.Subscribe(tx.done)

	// Consume facts until the producer is closed. This may occur upstream
	// by the producer itself or the transaction is closed prematurely.
	for {
		select {
		case err = <-errch:
			return err

		case fact, ok = <-ch:
			// Publisher closed the channel.
			if !ok {
				return nil
			}

			tx.Write(fact)
		}
	}

	return nil
}

// New initializes and returns a transaction for passed storage engine. The options
// are used to change the behavior of the transaction itself.
func New(engine storage.Engine, options Options) (*Transaction, error) {
	var (
		id  uint64
		err error
	)

	now := time.Now().UTC()

	// Increment the transaction ID and store the record.
	err = engine.Multi(func(tx storage.Tx) error {
		if id, err = txid(tx); err != nil {
			return err
		}

		_, err = dal.SetTransaction(tx, &dal.Transaction{
			ID:        id,
			StartTime: now,
		})

		return err
	})

	if err != nil {
		logrus.Errorf("transactor: could not create transaction: %s", err)
		return nil, ErrNoID
	}

	if options.Router == nil {
		options.Router = NewDomainRouter()
	}

	if options.ReceiveWait == 0 {
		options.ReceiveWait = DefaultOptions.ReceiveWait
	}

	if options.BufferSize == 0 {
		options.BufferSize = DefaultOptions.BufferSize
	}

	tx := Transaction{
		ID:        id,
		StartTime: time.Now().UTC(),
		Engine:    engine,
		options:   options,
		pipes:     make(map[Pipeline]chan<- *origins.Fact),
		router:    options.Router,
		stream:    make(chan *origins.Fact, options.BufferSize),
		done:      make(chan struct{}),
		errch:     make(chan error),
		pipewg:    &sync.WaitGroup{},
		mainwg:    &sync.WaitGroup{},
	}

	tx.mainwg.Add(1)

	// Run transaction.
	go tx.run()

	return &tx, nil
}
<|endoftext|>


<commit_before>package hcl2template

import (
	"fmt"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/gohcl"
	"github.com/hashicorp/hcl/v2/hclsyntax"
	packerregistry "github.com/hashicorp/packer/internal/registry"
	"github.com/hashicorp/packer/internal/registry/env"
	"github.com/zclconf/go-cty/cty"
)

const (
	buildFromLabel = "from"

	buildSourceLabel = "source"

	buildProvisionerLabel = "provisioner"

	buildErrorCleanupProvisionerLabel = "error-cleanup-provisioner"

	buildPostProcessorLabel = "post-processor"

	buildPostProcessorsLabel = "post-processors"

	buildHCPPackerRegistryLabel = "hcp_packer_registry"
)

var buildSchema = &hcl.BodySchema{
	Blocks: []hcl.BlockHeaderSchema{
		{Type: buildFromLabel, LabelNames: []string{"type"}},
		{Type: sourceLabel, LabelNames: []string{"reference"}},
		{Type: buildProvisionerLabel, LabelNames: []string{"type"}},
		{Type: buildErrorCleanupProvisionerLabel, LabelNames: []string{"type"}},
		{Type: buildPostProcessorLabel, LabelNames: []string{"type"}},
		{Type: buildPostProcessorsLabel, LabelNames: []string{}},
		{Type: buildHCPPackerRegistryLabel},
	},
}

var postProcessorsSchema = &hcl.BodySchema{
	Blocks: []hcl.BlockHeaderSchema{
		{Type: buildPostProcessorLabel, LabelNames: []string{"type"}},
	},
}

// BuildBlock references an HCL 'build' block and it content, for example :
//
//	build {
//		sources = [
//			...
//		]
//		provisioner "" { ... }
//		post-processor "" { ... }
//	}
type BuildBlock struct {
	// Name is a string representing the named build to show in the logs
	Name string

	// A description of what this build does, it could be used in a inspect
	// call for example.
	Description string

	// HCPPackerRegistry contains the configuration for publishing the image to the HCP Packer Registry.
	HCPPackerRegistry *HCPPackerRegistryBlock

	// Sources is the list of sources that we want to start in this build block.
	Sources []SourceUseBlock

	// ProvisionerBlocks references a list of HCL provisioner block that will
	// will be ran against the sources.
	ProvisionerBlocks []*ProvisionerBlock

	// ErrorCleanupProvisionerBlock references a special provisioner block that
	// will be ran only if the provision step fails.
	ErrorCleanupProvisionerBlock *ProvisionerBlock

	// PostProcessorLists references the lists of lists of HCL post-processors
	// block that will be run against the artifacts from the provisioning
	// steps.
	PostProcessorsLists [][]*PostProcessorBlock

	HCL2Ref HCL2Ref
}

type Builds []*BuildBlock

// decodeBuildConfig is called when a 'build' block has been detected. It will
// load the references to the contents of the build block.
func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildBlock, hcl.Diagnostics) {
	build := &BuildBlock{}
	body := block.Body

	var b struct {
		Name        string   `hcl:"name,optional"`
		Description string   `hcl:"description,optional"`
		FromSources []string `hcl:"sources,optional"`
		Config      hcl.Body `hcl:",remain"`
	}
	diags := gohcl.DecodeBody(body, cfg.EvalContext(LocalContext, nil), &b)
	if diags.HasErrors() {
		return nil, diags
	}

	build.Name = b.Name
	build.Description = b.Description

	// Expose build.name during parsing of pps and provisioners
	ectx := cfg.EvalContext(BuildContext, nil)
	ectx.Variables[buildAccessor] = cty.ObjectVal(map[string]cty.Value{
		"name": cty.StringVal(b.Name),
	})

	for _, buildFrom := range b.FromSources {
		ref := sourceRefFromString(buildFrom)

		if ref == NoSource ||
			!hclsyntax.ValidIdentifier(ref.Type) ||
			!hclsyntax.ValidIdentifier(ref.Name) {
			diags = append(diags, &hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "Invalid " + sourceLabel + " reference",
				Detail: "A " + sourceLabel + " type is made of three parts that are" +
					"split by a dot `.`; each part must start with a letter and " +
					"may contain only letters, digits, underscores, and dashes." +
					"A valid source reference looks like: `source.type.name`",
				Subject: block.DefRange.Ptr(),
			})
			continue
		}

		// source with no body
		build.Sources = append(build.Sources, SourceUseBlock{SourceRef: ref})
	}

	body = b.Config
	content, moreDiags := body.Content(buildSchema)
	diags = append(diags, moreDiags...)
	if diags.HasErrors() {
		return nil, diags
	}
	for _, block := range content.Blocks {
		switch block.Type {
		case buildHCPPackerRegistryLabel:
			if build.HCPPackerRegistry != nil {
				diags = append(diags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  fmt.Sprintf("Only one " + buildHCPPackerRegistryLabel + " is allowed"),
					Subject:  block.DefRange.Ptr(),
				})
				continue
			}
			hcpPackerRegistry, moreDiags := p.decodeHCPRegistry(block, cfg)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.HCPPackerRegistry = hcpPackerRegistry
		case sourceLabel:
			ref, moreDiags := p.decodeBuildSource(block)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.Sources = append(build.Sources, ref)
		case buildProvisionerLabel:
			p, moreDiags := p.decodeProvisioner(block, ectx)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.ProvisionerBlocks = append(build.ProvisionerBlocks, p)
		case buildErrorCleanupProvisionerLabel:
			if build.ErrorCleanupProvisionerBlock != nil {
				diags = append(diags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  fmt.Sprintf("Only one " + buildErrorCleanupProvisionerLabel + " is allowed"),
					Subject:  block.DefRange.Ptr(),
				})
				continue
			}
			p, moreDiags := p.decodeProvisioner(block, ectx)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.ErrorCleanupProvisionerBlock = p
		case buildPostProcessorLabel:
			pp, moreDiags := p.decodePostProcessor(block, ectx)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.PostProcessorsLists = append(build.PostProcessorsLists, []*PostProcessorBlock{pp})
		case buildPostProcessorsLabel:

			content, moreDiags := block.Body.Content(postProcessorsSchema)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}

			errored := false
			postProcessors := []*PostProcessorBlock{}
			for _, block := range content.Blocks {
				pp, moreDiags := p.decodePostProcessor(block, ectx)
				diags = append(diags, moreDiags...)
				if moreDiags.HasErrors() {
					errored = true
					break
				}
				postProcessors = append(postProcessors, pp)
			}
			if errored == false {
				build.PostProcessorsLists = append(build.PostProcessorsLists, postProcessors)
			}
		}
	}

	// Creates a bucket if either a hcp_packer_registry block is set or the HCP
	// Packer registry is enabled via environment variable
	if build.HCPPackerRegistry != nil || env.IsPAREnabled() {
		var err error
		cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{
			TemplateBaseDir: cfg.Basedir,
		})

		if err != nil {
			diags = append(diags, &hcl.Diagnostic{
				Summary:  "Unable to create a valid bucket object for HCP Packer Registry",
				Detail:   fmt.Sprintf("%s", err),
				Severity: hcl.DiagError,
			})

			return build, diags
		}

		cfg.bucket.LoadDefaultSettingsFromEnv()
		build.HCPPackerRegistry.WriteToBucketConfig(cfg.bucket)

		// If at this point the bucket.Slug is still empty,
		// last try is to use the build.Name if present
		if cfg.bucket.Slug == "" && build.Name != "" {
			cfg.bucket.Slug = build.Name
		}

		// If the description is empty, use the one from the build block
		if cfg.bucket.Description == "" {
			cfg.bucket.Description = build.Description
		}

		for _, source := range build.Sources {
			cfg.bucket.RegisterBuildForComponent(source.String())
		}

		// Known HCP Packer Image Datasource, whose id is the SourceImageId for some build.
		const hcpDatasourceType string = "hcp-packer-image"
		values := ectx.Variables[dataAccessor].AsValueMap()

		dsValues, ok := values[hcpDatasourceType]
		if !ok {
			return build, diags
		}

		for _, value := range dsValues.AsValueMap() {
			values := value.AsValueMap()
			imgID, itID := values["id"], values["iteration_id"]
			cfg.bucket.SourceImagesToParentIterations[imgID.AsString()] = itID.AsString()
		}
	}

	return build, diags
}
<commit_msg>hcl2template: only iterate on known HCP datasource (#11883)<commit_after>package hcl2template

import (
	"fmt"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/gohcl"
	"github.com/hashicorp/hcl/v2/hclsyntax"
	packerregistry "github.com/hashicorp/packer/internal/registry"
	"github.com/hashicorp/packer/internal/registry/env"
	"github.com/zclconf/go-cty/cty"
)

const (
	buildFromLabel = "from"

	buildSourceLabel = "source"

	buildProvisionerLabel = "provisioner"

	buildErrorCleanupProvisionerLabel = "error-cleanup-provisioner"

	buildPostProcessorLabel = "post-processor"

	buildPostProcessorsLabel = "post-processors"

	buildHCPPackerRegistryLabel = "hcp_packer_registry"
)

var buildSchema = &hcl.BodySchema{
	Blocks: []hcl.BlockHeaderSchema{
		{Type: buildFromLabel, LabelNames: []string{"type"}},
		{Type: sourceLabel, LabelNames: []string{"reference"}},
		{Type: buildProvisionerLabel, LabelNames: []string{"type"}},
		{Type: buildErrorCleanupProvisionerLabel, LabelNames: []string{"type"}},
		{Type: buildPostProcessorLabel, LabelNames: []string{"type"}},
		{Type: buildPostProcessorsLabel, LabelNames: []string{}},
		{Type: buildHCPPackerRegistryLabel},
	},
}

var postProcessorsSchema = &hcl.BodySchema{
	Blocks: []hcl.BlockHeaderSchema{
		{Type: buildPostProcessorLabel, LabelNames: []string{"type"}},
	},
}

// BuildBlock references an HCL 'build' block and it content, for example :
//
//	build {
//		sources = [
//			...
//		]
//		provisioner "" { ... }
//		post-processor "" { ... }
//	}
type BuildBlock struct {
	// Name is a string representing the named build to show in the logs
	Name string

	// A description of what this build does, it could be used in a inspect
	// call for example.
	Description string

	// HCPPackerRegistry contains the configuration for publishing the image to the HCP Packer Registry.
	HCPPackerRegistry *HCPPackerRegistryBlock

	// Sources is the list of sources that we want to start in this build block.
	Sources []SourceUseBlock

	// ProvisionerBlocks references a list of HCL provisioner block that will
	// will be ran against the sources.
	ProvisionerBlocks []*ProvisionerBlock

	// ErrorCleanupProvisionerBlock references a special provisioner block that
	// will be ran only if the provision step fails.
	ErrorCleanupProvisionerBlock *ProvisionerBlock

	// PostProcessorLists references the lists of lists of HCL post-processors
	// block that will be run against the artifacts from the provisioning
	// steps.
	PostProcessorsLists [][]*PostProcessorBlock

	HCL2Ref HCL2Ref
}

type Builds []*BuildBlock

// decodeBuildConfig is called when a 'build' block has been detected. It will
// load the references to the contents of the build block.
func (p *Parser) decodeBuildConfig(block *hcl.Block, cfg *PackerConfig) (*BuildBlock, hcl.Diagnostics) {
	build := &BuildBlock{}
	body := block.Body

	var b struct {
		Name        string   `hcl:"name,optional"`
		Description string   `hcl:"description,optional"`
		FromSources []string `hcl:"sources,optional"`
		Config      hcl.Body `hcl:",remain"`
	}
	diags := gohcl.DecodeBody(body, cfg.EvalContext(LocalContext, nil), &b)
	if diags.HasErrors() {
		return nil, diags
	}

	build.Name = b.Name
	build.Description = b.Description

	// Expose build.name during parsing of pps and provisioners
	ectx := cfg.EvalContext(BuildContext, nil)
	ectx.Variables[buildAccessor] = cty.ObjectVal(map[string]cty.Value{
		"name": cty.StringVal(b.Name),
	})

	for _, buildFrom := range b.FromSources {
		ref := sourceRefFromString(buildFrom)

		if ref == NoSource ||
			!hclsyntax.ValidIdentifier(ref.Type) ||
			!hclsyntax.ValidIdentifier(ref.Name) {
			diags = append(diags, &hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "Invalid " + sourceLabel + " reference",
				Detail: "A " + sourceLabel + " type is made of three parts that are" +
					"split by a dot `.`; each part must start with a letter and " +
					"may contain only letters, digits, underscores, and dashes." +
					"A valid source reference looks like: `source.type.name`",
				Subject: block.DefRange.Ptr(),
			})
			continue
		}

		// source with no body
		build.Sources = append(build.Sources, SourceUseBlock{SourceRef: ref})
	}

	body = b.Config
	content, moreDiags := body.Content(buildSchema)
	diags = append(diags, moreDiags...)
	if diags.HasErrors() {
		return nil, diags
	}
	for _, block := range content.Blocks {
		switch block.Type {
		case buildHCPPackerRegistryLabel:
			if build.HCPPackerRegistry != nil {
				diags = append(diags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  fmt.Sprintf("Only one " + buildHCPPackerRegistryLabel + " is allowed"),
					Subject:  block.DefRange.Ptr(),
				})
				continue
			}
			hcpPackerRegistry, moreDiags := p.decodeHCPRegistry(block, cfg)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.HCPPackerRegistry = hcpPackerRegistry
		case sourceLabel:
			ref, moreDiags := p.decodeBuildSource(block)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.Sources = append(build.Sources, ref)
		case buildProvisionerLabel:
			p, moreDiags := p.decodeProvisioner(block, ectx)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.ProvisionerBlocks = append(build.ProvisionerBlocks, p)
		case buildErrorCleanupProvisionerLabel:
			if build.ErrorCleanupProvisionerBlock != nil {
				diags = append(diags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  fmt.Sprintf("Only one " + buildErrorCleanupProvisionerLabel + " is allowed"),
					Subject:  block.DefRange.Ptr(),
				})
				continue
			}
			p, moreDiags := p.decodeProvisioner(block, ectx)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.ErrorCleanupProvisionerBlock = p
		case buildPostProcessorLabel:
			pp, moreDiags := p.decodePostProcessor(block, ectx)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			build.PostProcessorsLists = append(build.PostProcessorsLists, []*PostProcessorBlock{pp})
		case buildPostProcessorsLabel:

			content, moreDiags := block.Body.Content(postProcessorsSchema)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}

			errored := false
			postProcessors := []*PostProcessorBlock{}
			for _, block := range content.Blocks {
				pp, moreDiags := p.decodePostProcessor(block, ectx)
				diags = append(diags, moreDiags...)
				if moreDiags.HasErrors() {
					errored = true
					break
				}
				postProcessors = append(postProcessors, pp)
			}
			if errored == false {
				build.PostProcessorsLists = append(build.PostProcessorsLists, postProcessors)
			}
		}
	}

	// Creates a bucket if either a hcp_packer_registry block is set or the HCP
	// Packer registry is enabled via environment variable
	if build.HCPPackerRegistry != nil || env.IsPAREnabled() {
		var err error
		cfg.bucket, err = packerregistry.NewBucketWithIteration(packerregistry.IterationOptions{
			TemplateBaseDir: cfg.Basedir,
		})

		if err != nil {
			diags = append(diags, &hcl.Diagnostic{
				Summary:  "Unable to create a valid bucket object for HCP Packer Registry",
				Detail:   fmt.Sprintf("%s", err),
				Severity: hcl.DiagError,
			})

			return build, diags
		}

		cfg.bucket.LoadDefaultSettingsFromEnv()
		build.HCPPackerRegistry.WriteToBucketConfig(cfg.bucket)

		// If at this point the bucket.Slug is still empty,
		// last try is to use the build.Name if present
		if cfg.bucket.Slug == "" && build.Name != "" {
			cfg.bucket.Slug = build.Name
		}

		// If the description is empty, use the one from the build block
		if cfg.bucket.Description == "" {
			cfg.bucket.Description = build.Description
		}

		for _, source := range build.Sources {
			cfg.bucket.RegisterBuildForComponent(source.String())
		}

		// Known HCP Packer Image Datasource, whose id is the SourceImageId for some build.
		const hcpDatasourceType string = "hcp-packer-image"
		values := ectx.Variables[dataAccessor].AsValueMap()

		dsValues, ok := values[hcpDatasourceType]
		if !ok {
			return build, diags
		}

		for _, value := range dsValues.AsValueMap() {
			// Only try to check ancestry if we have fetched the
			// data from HCP for the data source.
			//
			// NOTE: this will happen especially during validate as
			// when we reach this point, we haven't fetched the
			// data from HCP.
			//
			// TODO: maybe move this HCP-related logic outside that
			// decode block so it's only executed during build?
			if !value.IsKnown() {
				continue
			}

			values := value.AsValueMap()
			imgID, itID := values["id"], values["iteration_id"]
			cfg.bucket.SourceImagesToParentIterations[imgID.AsString()] = itID.AsString()
		}
	}

	return build, diags
}
<|endoftext|>


<commit_before>package language

import (
	"encoding/json"
	"log"
	"math/rand"
	"regexp"
	"sort"
	"strings"

	"github.com/avabot/ava/Godeps/_workspace/src/github.com/mattbaird/elastigo/lib"
	"github.com/avabot/ava/shared/datatypes"
)

var regexArticle = regexp.MustCompile(`^(a|an|the|A|An|The)\s`)

type WordT struct {
	Word  string
	POS   string
	Index int
}

type ByIndex []WordT

func (a ByIndex) Len() int {
	return len(a)
}

func (a ByIndex) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a ByIndex) Less(i, j int) bool {
	return a[i].Index < a[j].Index
}

func (a ByIndex) StringSlice() []string {
	var tmp []string
	for _, el := range a {
		tmp = append(tmp, el.Word)
	}
	return tmp
}

// Summarize identifies keyword phrases in text. keywordSource is the
// ElasticSearch type in the form of index_type. For example, to identify
// keywords in a wine review, keywordSource would be "products_alcohol".
func Summarize(product *dt.Product, keywordSource string) (string, error) {
	// TODO catch negative connations in a clause, so the summary does not
	// include or emphasize them.
	shortSummary := buildShortSummary(product)
	if len(product.Reviews) == 0 {
		return shortSummary, nil
	}
	text := product.Reviews[0].Body
	ec := dt.NewSearchClient()
	q := map[string]interface{}{
		"query": map[string]interface{}{
			"match": map[string]string{"Name": text},
		},
		"size": 50,
	}
	res, err := ec.Search("keywords", keywordSource, nil, q)
	if err != nil {
		return shortSummary, err
	}
	keywords, err := extractKeywords(text, res.Hits.Hits)
	if err != nil {
		return shortSummary, err
	}
	log.Println("=== KEYWORDS ===")
	log.Println(keywords)
	keywords = combineKeywordsIntoRanges(keywords)
	log.Println("=== RANGES ===")
	log.Println(keywords)
	summary := buildSummary(product, keywords)
	log.Println("=== SUMMARY ===")
	log.Println(summary)
	return shortSummary + summary, nil
}

func buildSummary(product *dt.Product, keywords []WordT) string {
	var totalNounLen int
	var totalAdjLen int
	var summary string
	var nouns []string
	var adjs []string
	nounAndOr := "and"
	adjAndOr := "and"
	for _, wt := range keywords {
		if wt.POS == "n" {
			nouns = append(nouns, wt.Word)
			totalNounLen += len(wt.Word)
			if strings.Contains(wt.Word, "and") {
				nounAndOr = "as well as"
			}
		} else if wt.POS == "adj" {
			adjs = append(adjs, wt.Word)
			totalAdjLen += len(wt.Word)
			if strings.Contains(wt.Word, "and") {
				adjAndOr = "as well as"
			}
		}
	}
	var rem string
	var addition string
	if totalNounLen >= totalAdjLen {
		n := rand.Intn(2)
		switch n {
		case 0:
			summary = "hints of "
		case 1:
			summary = "notes of "
		}
		n = rand.Intn(10)
		switch n {
		case 0:
			summary = "It features " + summary
		case 1:
			summary = "It's characterized by " + summary
		case 2:
			summary = "It's known for " + summary
		case 3:
			summary = "It's known for its " + summary
		case 4:
			summary = "It's known for having " + summary
		case 5:
			summary = "It has " + summary
		case 6:
			summary = "You'll experience " + summary
		case 7:
			summary = "You'll sense " + summary
		case 8:
			summary = "You'll enjoy " + summary
		case 9:
			summary = "You'll love its " + summary
		}
		for i := 0; i <= len(nouns)-1; i++ {
			// TODO design more robust maxLength control
			if totalNounLen > 60 {
				rem, nouns = nouns[len(nouns)-1], nouns[:len(nouns)-1]
				totalNounLen -= len(rem)
			}
		}
		for _, noun := range nouns {
			ws := strings.Fields(noun)
			for _, w := range ws {
				if w == "is" {
					nounAndOr = "."
					break
				}
			}
		}
		addition = SliceToString(nouns, nounAndOr)
	} else {
		summary = "It's "
		for i := 0; i <= len(adjs)-1; i++ {
			if totalAdjLen > 60 {
				rem, adjs = adjs[len(adjs)-1], adjs[:len(adjs)-1]
				totalAdjLen -= len(rem)
			}
		}
		for _, adj := range adjs {
			ws := strings.Fields(adj)
			for _, w := range ws {
				if w == "is" {
					adjAndOr = "."
					break
				}
			}
		}
		addition = SliceToString(adjs, adjAndOr)
	}
	if len(addition) == 0 {
		return ""
	}
	summary += strings.TrimRight(addition, ".,;:!?'\"") + "."
	return summary
}

func combineKeywordsIntoRanges(keywords []WordT) []WordT {
	var ranges []WordT
	var buf *WordT
	for i := 0; i < len(keywords); i++ {
		buf, i = appendStops(buf, keywords, i, 1)
		if buf == nil {
			if i == len(keywords) {
				i--
			}
			buf = &keywords[i]
		}
		buf.Word = strings.TrimRight(buf.Word, ".,;:'\"?!")
		ranges = append(ranges, *buf)
		buf = nil
	}
	return ranges
}

func appendStops(buf *WordT, keywords []WordT, i, j int) (*WordT, int) {
	kw := keywords[i]
	// prevent joining initial stopwords
	if Contains(StopWords, kw.Word) {
		return buf, i
	}
	// if next word is also a keyword
	if i < len(keywords)-j && keywords[i+j].Index == kw.Index+j {
		if buf == nil {
			buf = &WordT{
				Index: kw.Index,
				POS:   keywords[i+j].POS,
				Word:  kw.Word + " " + keywords[i+j].Word,
			}
		} else {
			buf.POS = keywords[i+j].POS
			buf.Word += " " + keywords[i+j].Word
		}
		// separated by a period. don't continue
		lastLetter := keywords[i+j].Word[len(keywords[i+j].Word)-1]
		if lastLetter == '.' || lastLetter == ';' {
			return buf, i + 1
		}
		buf, i = appendStops(buf, keywords, i, j+1)
	}
	return buf, i + 1
}

func extractKeywords(text string, hits []elastigo.Hit) ([]WordT, error) {
	var keywords []WordT
	words := strings.Fields(text)
	for i := 0; i < len(words); i++ {
		word := words[i]
		if Contains(StopWords, word) {
			keywords = append(keywords, WordT{
				Word:  word,
				Index: i,
			})
			continue
		}
	}
	for _, hit := range hits {
		for i := 0; i < len(words); i++ {
			word := words[i]
			tmp := strings.TrimRight(word, ".,;:?!'\"")
			if tmp == hit.Id {
				var tmp WordT
				bytes, err := hit.Source.MarshalJSON()
				if err != nil {
					return keywords, err
				}
				if err = json.Unmarshal(bytes, &tmp); err != nil {
					return keywords, err
				}
				tmp.Word = word
				tmp.Index = i
				keywords = append(keywords, tmp)
			}
		}
	}
	sort.Sort(ByIndex(keywords))
	return keywords, nil
}

var StopWords = []string{
	"and",
	"but",
	"for",
	"a",
	"an",
	"the",
	"yet",
	"with",
	"so",
	"of",
	"by",
	"is",
	"before",
	"after",
	"above",
	"below",
	"over",
	"under",
	"some",
	"alongside",
}

func Contains(wordList []string, s string) bool {
	s = strings.TrimRight(strings.ToLower(s), ".,;:!?'\"")
	for _, word := range wordList {
		if s == word {
			return true
		}
	}
	return false
}

func buildShortSummary(p *dt.Product) string {
	log.Printf("%+v\n", *p)
	if len(p.Category) == 0 {
		return ""
	}
	tmp := "It's "
	n := rand.Intn(8)
	switch n {
	case 0:
		tmp += "a gorgeous "
	case 1:
		tmp += "a brilliant "
	case 2:
		tmp += "an amazing "
	case 3:
		tmp += "a spectacular "
	case 4:
		tmp += "a tasty "
	case 5:
		tmp += "a delicious "
	case 6, 7:
		tmp += "a "
	}
	return tmp + p.Category + ". "
}
<commit_msg>clean logging from language/summarize<commit_after>package language

import (
	"encoding/json"
	"math/rand"
	"regexp"
	"sort"
	"strings"

	"github.com/avabot/ava/Godeps/_workspace/src/github.com/mattbaird/elastigo/lib"
	"github.com/avabot/ava/shared/datatypes"
)

var regexArticle = regexp.MustCompile(`^(a|an|the|A|An|The)\s`)

type WordT struct {
	Word  string
	POS   string
	Index int
}

type ByIndex []WordT

func (a ByIndex) Len() int {
	return len(a)
}

func (a ByIndex) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a ByIndex) Less(i, j int) bool {
	return a[i].Index < a[j].Index
}

func (a ByIndex) StringSlice() []string {
	var tmp []string
	for _, el := range a {
		tmp = append(tmp, el.Word)
	}
	return tmp
}

// Summarize identifies keyword phrases in text. keywordSource is the
// ElasticSearch type in the form of index_type. For example, to identify
// keywords in a wine review, keywordSource would be "products_alcohol".
func Summarize(product *dt.Product, keywordSource string) (string, error) {
	// TODO catch negative connations in a clause, so the summary does not
	// include or emphasize them.
	shortSummary := buildShortSummary(product)
	if len(product.Reviews) == 0 {
		return shortSummary, nil
	}
	text := product.Reviews[0].Body
	ec := dt.NewSearchClient()
	q := map[string]interface{}{
		"query": map[string]interface{}{
			"match": map[string]string{"Name": text},
		},
		"size": 50,
	}
	res, err := ec.Search("keywords", keywordSource, nil, q)
	if err != nil {
		return shortSummary, err
	}
	keywords, err := extractKeywords(text, res.Hits.Hits)
	if err != nil {
		return shortSummary, err
	}
	keywords = combineKeywordsIntoRanges(keywords)
	summary := buildSummary(product, keywords)
	return shortSummary + summary, nil
}

func buildSummary(product *dt.Product, keywords []WordT) string {
	var totalNounLen int
	var totalAdjLen int
	var summary string
	var nouns []string
	var adjs []string
	nounAndOr := "and"
	adjAndOr := "and"
	for _, wt := range keywords {
		if wt.POS == "n" {
			nouns = append(nouns, wt.Word)
			totalNounLen += len(wt.Word)
			if strings.Contains(wt.Word, "and") {
				nounAndOr = "as well as"
			}
		} else if wt.POS == "adj" {
			adjs = append(adjs, wt.Word)
			totalAdjLen += len(wt.Word)
			if strings.Contains(wt.Word, "and") {
				adjAndOr = "as well as"
			}
		}
	}
	var rem string
	var addition string
	if totalNounLen >= totalAdjLen {
		n := rand.Intn(2)
		switch n {
		case 0:
			summary = "hints of "
		case 1:
			summary = "notes of "
		}
		n = rand.Intn(10)
		switch n {
		case 0:
			summary = "It features " + summary
		case 1:
			summary = "It's characterized by " + summary
		case 2:
			summary = "It's known for " + summary
		case 3:
			summary = "It's known for its " + summary
		case 4:
			summary = "It's known for having " + summary
		case 5:
			summary = "It has " + summary
		case 6:
			summary = "You'll experience " + summary
		case 7:
			summary = "You'll sense " + summary
		case 8:
			summary = "You'll enjoy " + summary
		case 9:
			summary = "You'll love its " + summary
		}
		for i := 0; i <= len(nouns)-1; i++ {
			// TODO design more robust maxLength control
			if totalNounLen > 60 {
				rem, nouns = nouns[len(nouns)-1], nouns[:len(nouns)-1]
				totalNounLen -= len(rem)
			}
		}
		for _, noun := range nouns {
			ws := strings.Fields(noun)
			for _, w := range ws {
				if w == "is" {
					nounAndOr = "."
					break
				}
			}
		}
		addition = SliceToString(nouns, nounAndOr)
	} else {
		summary = "It's "
		for i := 0; i <= len(adjs)-1; i++ {
			if totalAdjLen > 60 {
				rem, adjs = adjs[len(adjs)-1], adjs[:len(adjs)-1]
				totalAdjLen -= len(rem)
			}
		}
		for _, adj := range adjs {
			ws := strings.Fields(adj)
			for _, w := range ws {
				if w == "is" {
					adjAndOr = "."
					break
				}
			}
		}
		addition = SliceToString(adjs, adjAndOr)
	}
	if len(addition) == 0 {
		return ""
	}
	summary += strings.TrimRight(addition, ".,;:!?'\"") + "."
	return summary
}

func combineKeywordsIntoRanges(keywords []WordT) []WordT {
	var ranges []WordT
	var buf *WordT
	for i := 0; i < len(keywords); i++ {
		buf, i = appendStops(buf, keywords, i, 1)
		if buf == nil {
			if i == len(keywords) {
				i--
			}
			buf = &keywords[i]
		}
		buf.Word = strings.TrimRight(buf.Word, ".,;:'\"?!")
		ranges = append(ranges, *buf)
		buf = nil
	}
	return ranges
}

func appendStops(buf *WordT, keywords []WordT, i, j int) (*WordT, int) {
	kw := keywords[i]
	// prevent joining initial stopwords
	if Contains(StopWords, kw.Word) {
		return buf, i
	}
	// if next word is also a keyword
	if i < len(keywords)-j && keywords[i+j].Index == kw.Index+j {
		if buf == nil {
			buf = &WordT{
				Index: kw.Index,
				POS:   keywords[i+j].POS,
				Word:  kw.Word + " " + keywords[i+j].Word,
			}
		} else {
			buf.POS = keywords[i+j].POS
			buf.Word += " " + keywords[i+j].Word
		}
		// separated by a period. don't continue
		lastLetter := keywords[i+j].Word[len(keywords[i+j].Word)-1]
		if lastLetter == '.' || lastLetter == ';' {
			return buf, i + 1
		}
		buf, i = appendStops(buf, keywords, i, j+1)
	}
	return buf, i + 1
}

func extractKeywords(text string, hits []elastigo.Hit) ([]WordT, error) {
	var keywords []WordT
	words := strings.Fields(text)
	for i := 0; i < len(words); i++ {
		word := words[i]
		if Contains(StopWords, word) {
			keywords = append(keywords, WordT{
				Word:  word,
				Index: i,
			})
			continue
		}
	}
	for _, hit := range hits {
		for i := 0; i < len(words); i++ {
			word := words[i]
			tmp := strings.TrimRight(word, ".,;:?!'\"")
			if tmp == hit.Id {
				var tmp WordT
				bytes, err := hit.Source.MarshalJSON()
				if err != nil {
					return keywords, err
				}
				if err = json.Unmarshal(bytes, &tmp); err != nil {
					return keywords, err
				}
				tmp.Word = word
				tmp.Index = i
				keywords = append(keywords, tmp)
			}
		}
	}
	sort.Sort(ByIndex(keywords))
	return keywords, nil
}

var StopWords = []string{
	"and",
	"but",
	"for",
	"a",
	"an",
	"the",
	"yet",
	"with",
	"so",
	"of",
	"by",
	"is",
	"before",
	"after",
	"above",
	"below",
	"over",
	"under",
	"some",
	"alongside",
}

func Contains(wordList []string, s string) bool {
	s = strings.TrimRight(strings.ToLower(s), ".,;:!?'\"")
	for _, word := range wordList {
		if s == word {
			return true
		}
	}
	return false
}

func buildShortSummary(p *dt.Product) string {
	if len(p.Category) == 0 {
		return ""
	}
	tmp := "It's "
	n := rand.Intn(8)
	switch n {
	case 0:
		tmp += "a gorgeous "
	case 1:
		tmp += "a brilliant "
	case 2:
		tmp += "an amazing "
	case 3:
		tmp += "a spectacular "
	case 4:
		tmp += "a tasty "
	case 5:
		tmp += "a delicious "
	case 6, 7:
		tmp += "a "
	}
	return tmp + p.Category + ". "
}
<|endoftext|>


<commit_before>package socketio

import (
	"http"
	"os"
	"websocket"
)

var errWebsocketHandshake = os.NewError("websocket handshake error")

// The websocket transport.
type websocketTransport struct {
	rtimeout int64 // The period during which the client must send a message.
	wtimeout int64 // The period during which a write must succeed.
}

// Creates a new websocket transport with the given read and write timeouts.
func NewWebsocketTransport(rtimeout, wtimeout int64) Transport {
	return &websocketTransport{rtimeout, wtimeout}
}

// Returns the resource name.
func (t *websocketTransport) Resource() string {
	return "websocket"
}

// Creates a new socket that can be used with a connection.
func (t *websocketTransport) newSocket() socket {
	return &websocketSocket{t: t}
}

// websocketTransport implements the transport interface for websockets
type websocketSocket struct {
	t         *websocketTransport // the transport configuration
	ws        *websocket.Conn     // the websocket connection
	connected bool                // used internally to represent the connection state
	close     chan byte
}

// Transport returns the transport the socket is based on.
func (s *websocketSocket) Transport() Transport {
	return s.t
}

// String returns the verbose representation of the socket.
func (s *websocketSocket) String() string {
	return s.t.Resource()
}

// Accepts a http connection & request pair. It upgrades the connection and calls
// proceed if succesfull.
//
// TODO: Remove the ugly channels and timeouts. They should not be needed!
func (s *websocketSocket) accept(w http.ResponseWriter, req *http.Request, proceed func()) (err os.Error) {
	if s.connected {
		return ErrConnected
	}

	f := func(ws *websocket.Conn) {
		err = nil
		ws.SetReadTimeout(s.t.rtimeout)
		ws.SetWriteTimeout(s.t.wtimeout)
		s.connected = true
		s.ws = ws
		s.close = make(chan byte)
		defer close(s.close)

		proceed()

		// must block until closed
		<-s.close
	}

	err = errWebsocketHandshake
	if _, ok := req.Header["Sec-Websocket-Key1"]; ok {
		websocket.Handler(f).ServeHTTP(w, req)
	} else {
		websocket.Draft75Handler(f).ServeHTTP(w, req)
	}

	return
}

func (s *websocketSocket) Read(p []byte) (int, os.Error) {
	if !s.connected {
		return 0, ErrNotConnected
	}

	return s.ws.Read(p)
}

func (s *websocketSocket) Write(p []byte) (int, os.Error) {
	if !s.connected {
		return 0, ErrNotConnected
	}

	return s.ws.Write(p)
}

func (s *websocketSocket) Close() os.Error {
	if !s.connected {
		return ErrNotConnected
	}

	s.connected = false
	go func() {
		select {
		case s.close <- 1:
		default:
		}
	}()

	return s.ws.Close()
}
<commit_msg>websocket: Draft75Handler is obsolete<commit_after>package socketio

import (
	"http"
	"os"
	"websocket"
)

var errWebsocketHandshake = os.NewError("websocket handshake error")

// The websocket transport.
type websocketTransport struct {
	rtimeout int64 // The period during which the client must send a message.
	wtimeout int64 // The period during which a write must succeed.
}

// Creates a new websocket transport with the given read and write timeouts.
func NewWebsocketTransport(rtimeout, wtimeout int64) Transport {
	return &websocketTransport{rtimeout, wtimeout}
}

// Returns the resource name.
func (t *websocketTransport) Resource() string {
	return "websocket"
}

// Creates a new socket that can be used with a connection.
func (t *websocketTransport) newSocket() socket {
	return &websocketSocket{t: t}
}

// websocketTransport implements the transport interface for websockets
type websocketSocket struct {
	t         *websocketTransport // the transport configuration
	ws        *websocket.Conn     // the websocket connection
	connected bool                // used internally to represent the connection state
	close     chan byte
}

// Transport returns the transport the socket is based on.
func (s *websocketSocket) Transport() Transport {
	return s.t
}

// String returns the verbose representation of the socket.
func (s *websocketSocket) String() string {
	return s.t.Resource()
}

// Accepts a http connection & request pair. It upgrades the connection and calls
// proceed if succesfull.
//
// TODO: Remove the ugly channels and timeouts. They should not be needed!
func (s *websocketSocket) accept(w http.ResponseWriter, req *http.Request, proceed func()) (err os.Error) {
	if s.connected {
		return ErrConnected
	}

	f := func(ws *websocket.Conn) {
		err = nil
		ws.SetReadTimeout(s.t.rtimeout)
		ws.SetWriteTimeout(s.t.wtimeout)
		s.connected = true
		s.ws = ws
		s.close = make(chan byte)
		defer close(s.close)

		proceed()

		// must block until closed
		<-s.close
	}

	err = errWebsocketHandshake
	websocket.Handler(f).ServeHTTP(w, req)
	return
}

func (s *websocketSocket) Read(p []byte) (int, os.Error) {
	if !s.connected {
		return 0, ErrNotConnected
	}

	return s.ws.Read(p)
}

func (s *websocketSocket) Write(p []byte) (int, os.Error) {
	if !s.connected {
		return 0, ErrNotConnected
	}

	return s.ws.Write(p)
}

func (s *websocketSocket) Close() os.Error {
	if !s.connected {
		return ErrNotConnected
	}

	s.connected = false
	go func() {
		select {
		case s.close <- 1:
		default:
		}
	}()

	return s.ws.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/MG-RAST/Shock/conf"
	"github.com/MG-RAST/Shock/goweb"
)

func main() {
	goweb.ConfigureDefaultFormatters()
	goweb.MapFunc("/resources", ResourceDescription)
	goweb.MapFunc("/raw", RawDir)
	goweb.MapFunc("/assets", AssetsDir)
	goweb.MapRest("/node", new(NodeController))
	goweb.MapRest("/user", new(UserController))
	goweb.MapFunc("*", Site)
	fmt.Printf("Shock (port:%d, dataroot:%q, mongodb_host:%q, secretkey:%q)... starting\n", *conf.PORT, *conf.DATAROOT, *conf.MONGODB, *conf.SECRETKEY)
	goweb.ListenAndServe(":" + fmt.Sprintf("%d", *conf.PORT))
}
<commit_msg>site listening on :80, api on user defined, added logo to startup<commit_after>package main

import (
	"fmt"
	"github.com/MG-RAST/Shock/conf"
	"github.com/MG-RAST/Shock/goweb"
)

func main() {
	fmt.Printf("%s\n######### Conf #########\ndata-root:\t%s\nmongodb:\t%s\nsecretkey:\t%s\nsite-port:\t80\napi-port:\t%d\n\n####### Starting #######\n", logo, *conf.DATAROOT, *conf.MONGODB, *conf.SECRETKEY, *conf.PORT)

	c := make(chan int)
	goweb.ConfigureDefaultFormatters()
	// start site
	go func() {
		r := &goweb.RouteManager{}
		r.MapFunc("/raw", RawDir)
		r.MapFunc("/assets", AssetsDir)
		r.MapFunc("*", Site)
		c <- 1
		goweb.ListenAndServeRoutes(":80", r)
		c <- 1
	}()
	<-c
	fmt.Printf("site :80... running\n")

	// start api
	go func() {
		r := &goweb.RouteManager{}
		r.MapRest("/node", new(NodeController))
		r.MapRest("/user", new(UserController))
		r.MapFunc("*", ResourceDescription, goweb.GetMethod)
		c <- 1
		goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", *conf.PORT), r)
		c <- 1
	}()
	<-c
	fmt.Printf("api  :%d... running\n", *conf.PORT)
	fmt.Printf("\n######### Log  #########\n")
	<-c
}
<|endoftext|>