<commit_before>// Copyright 2017, TCN Inc.
// All rights reserved.

// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:

//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of TCN Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package generator

import (
	"fmt"
	"strings"
	"github.com/xwb1989/sqlparser"
	"github.com/Sirupsen/logrus"
)

type QueryArg struct {
	Name         string      // name in the map,
	Value        interface{} // generic value of the argument. If is a field, this will be empty
	IsFieldValue bool        // Whether this refers to a field passed in
	Field        TypeDesc    // if IsFieldValue is true, this will describe the Field
}

type SpannerHelper struct {
	RawQuery string
	Query string
	ParsedQuery sqlparser.Statement
	TableName string
	OptionArguments []string
	IsSelect bool
	IsUpdate bool
	IsInsert bool
	IsDelete bool
	QueryArgs []QueryArg
	InsertCols []string // the column names for insert queries
	Parent *Method
	ProtoFieldDescs map[string]TypeDesc
}

func (sh *SpannerHelper) String() string {
	if sh != nil {
		return fmt.Sprintf("SpannerHelper\n\tQuery: %s\n\tIsSelect: %t\n\tIsUpdate: %t\n\tIsInsert: %t\n\tIsDelete: %t\n\n",
				sh.Query, sh.IsSelect, sh.IsUpdate, sh.IsInsert, sh.IsDelete)
	}
	return "<nil>"
}

func NewSpannerHelper(p *Method) (*SpannerHelper, error) {
	// get the query, and parse it
	opts := p.GetMethodOption()
	if opts == nil {
		return nil, fmt.Errorf("no options found on proto method")
	}
	args := opts.GetArguments()
	query := opts.GetQuery()
	logrus.Debugf("query: %#v", query)
	pquery, err := sqlparser.Parse(query)
	if err != nil {
		return nil, fmt.Errorf("parsing error in spanner_helper: %s", err)
	}
	// get the fields descriptions to construct query args
	input := p.GetInputTypeStruct()
	fieldsMap := p.GetTypeDescForFieldsInStructSnakeCase(input)

	sh := &SpannerHelper{
		RawQuery: query,
		ParsedQuery: pquery,
		OptionArguments: args,
		Parent: p,
		ProtoFieldDescs: fieldsMap,
	}
	err = sh.Parse()
	if err != nil {
		return nil, err
	}
	return sh, nil
}

func (sh *SpannerHelper) Parse() error {
	// parse our query
	switch pq := sh.ParsedQuery.(type) {
	case *sqlparser.Select:
		sh.IsSelect = true
		spl := strings.Split(sh.RawQuery, "?")
		var updatedQuery string

		if len(sh.OptionArguments) != len(spl) - 1 {
			errStr := "err parsing spanner query: not correct number of option arguments"
			errStr += " for method: %s of service: %s  want: %d have: %d"
			return fmt.Errorf(errStr, sh.Parent.GetName(), sh.Parent.Service.GetName(), len(spl) - 1, len(sh.OptionArguments))
		}
		for i := 0; i < len(spl)-1; i++ {
			name := fmt.Sprintf("@%d", i)
			field := sh.ProtoFieldDescs[sh.OptionArguments[i]]
			qa := QueryArg{
				Name: name,
				IsFieldValue: true,
				Field: field,
			}
			sh.QueryArgs = append(sh.QueryArgs, qa)
			updatedQuery += (spl[i] + name)
		}
		updatedQuery += spl[len(spl)-1]
		sh.Query = updatedQuery
	case *sqlparser.Insert:
		sh.IsInsert = true
		cols, err := extractInsertColumns(pq)
		if err != nil {
			return err
		}
		table, err := extractIUDTableName(pq)
		if err != nil {
			return err
		}
		pas, err := prepareInsertValues(pq)
		if err != nil {
			return err
		}
		for _, arg := range pas.args {
			var qa QueryArg
			if ap, ok := arg.(PassedInArgPos); ok {
				index := int(ap)
				argName := sh.OptionArguments[index]
				qa = QueryArg{
					IsFieldValue: true,
					Field: sh.ProtoFieldDescs[argName],
				}
			} else {
				qa = QueryArg{
					Value: fmt.Sprintf("%#v", arg),
					IsFieldValue: false,
				}
			}
			sh.QueryArgs = append(sh.QueryArgs, qa)
		}
		sh.InsertCols = cols
		sh.TableName = table
	case *sqlparser.Delete:
		sh.IsDelete = true
		table, err := extractIUDTableName(pq)
		if err != nil {
			return err
		}
		sh.TableName = table
	case *sqlparser.Update:
		sh.IsUpdate = true
		table, err := extractIUDTableName(pq)
		if err != nil {
			return err
		}
		pam, err := extractUpdateClause(pq)
		if err != nil {
			return err
		}
		for key, arg := range pam.args {
			var qa QueryArg
			if ap, ok := arg.(PassedInArgPos); ok {
				index := int(ap)
				argName := sh.OptionArguments[index]
				qa = QueryArg{
					Name: key,
					IsFieldValue: true,
					Field: sh.ProtoFieldDescs[argName],
				}
			} else {
				qa = QueryArg{
					Name: key,
					Value: fmt.Sprintf("%#v", arg),
					IsFieldValue: false,
				}
			}
			sh.QueryArgs = append(sh.QueryArgs, qa)
		}
		sh.TableName = table
	}
	return nil
}

func (sh *SpannerHelper) InsertColsAsString() string {
	return fmt.Sprintf("%#v", sh.InsertCols)
}

func (sh *SpannerHelper) GetDeleteKeyRange() string {
	return ""
}
<commit_msg>refactor<commit_after>// Copyright 2017, TCN Inc.
// All rights reserved.

// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:

//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of TCN Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package generator

import (
	"fmt"
	"strings"
	"github.com/xwb1989/sqlparser"
	"github.com/Sirupsen/logrus"
)

type QueryArg struct {
	Name         string      // name in the map,
	Value        interface{} // generic value of the argument. If is a field, this will be empty
	IsFieldValue bool        // Whether this refers to a field passed in
	Field        TypeDesc    // if IsFieldValue is true, this will describe the Field
}

type KeyRangeDesc struct {
	Start []QueryArg
	End []QueryArg
	Kind string // a string of of a spanner.KeyRangeKind (ClosedOpen, ClosedClosed ex.)
}

type SpannerHelper struct {
	RawQuery string
	Query string
	ParsedQuery sqlparser.Statement
	TableName string
	OptionArguments []string
	IsSelect bool
	IsUpdate bool
	IsInsert bool
	IsDelete bool
	QueryArgs []QueryArg
	KeyRangeDesc *KeyRangeDesc // used for delete queries, will be set if IsDelete is true
	InsertCols []string // the column names for insert queries
	Parent *Method
	ProtoFieldDescs map[string]TypeDesc
}

func (sh *SpannerHelper) String() string {
	if sh != nil {
		return fmt.Sprintf("SpannerHelper\n\tQuery: %s\n\tIsSelect: %t\n\tIsUpdate: %t\n\tIsInsert: %t\n\tIsDelete: %t\n\n",
				sh.Query, sh.IsSelect, sh.IsUpdate, sh.IsInsert, sh.IsDelete)
	}
	return "<nil>"
}

func NewSpannerHelper(p *Method) (*SpannerHelper, error) {
	// get the query, and parse it
	opts := p.GetMethodOption()
	if opts == nil {
		return nil, fmt.Errorf("no options found on proto method")
	}
	args := opts.GetArguments()
	query := opts.GetQuery()
	logrus.Debugf("query: %#v", query)
	pquery, err := sqlparser.Parse(query)
	if err != nil {
		return nil, fmt.Errorf("parsing error in spanner_helper: %s", err)
	}
	// get the fields descriptions to construct query args
	input := p.GetInputTypeStruct()
	fieldsMap := p.GetTypeDescForFieldsInStructSnakeCase(input)

	sh := &SpannerHelper{
		RawQuery: query,
		ParsedQuery: pquery,
		OptionArguments: args,
		Parent: p,
		ProtoFieldDescs: fieldsMap,
	}
	err = sh.Parse()
	if err != nil {
		return nil, err
	}
	return sh, nil
}

func (sh *SpannerHelper) Parse() error {
	// parse our query
	switch pq := sh.ParsedQuery.(type) {
	case *sqlparser.Select:
		return sh.ParseSelect(pq)
	case *sqlparser.Insert:
		return sh.ParseInsert(pq)
	case *sqlparser.Delete:
		return sh.ParseDelete(pq)
	case *sqlparser.Update:
		return sh.ParseUpdate(pq)
	default:
		return fmt.Errorf("not a query we can parse")
	}
}

func (sh *SpannerHelper) InsertColsAsString() string {
	return fmt.Sprintf("%#v", sh.InsertCols)
}

func (sh *SpannerHelper) PopulateArgSlice(slice []interface{}) ([]QueryArg, error) {
	if len(slice) < len(sh.OptionArguments) {
		return nil, fmt.Errorf("cannot have less ? than arguments in query: %s", sh.Query)
	}
	qas := make([]QueryArg, len(slice))
	for i, arg := range slice {
		var qa QueryArg
		if ap, ok := arg.(PassedInArgPos); ok {
			index := int(ap)
			argName := sh.OptionArguments[index]
			qa = QueryArg{
				IsFieldValue: true,
				Field: sh.ProtoFieldDescs[argName],
			}
		} else {
			qa = QueryArg{
				Value: fmt.Sprintf("%#v", arg),
				IsFieldValue: false,
			}
		}
		qas[i] = qa
	}
	return qas, nil
}

func (sh *SpannerHelper) ParseInsert(pq *sqlparser.Insert) error {
	sh.IsInsert = true
	cols, err := extractInsertColumns(pq)
	if err != nil {
		return err
	}
	table, err := extractIUDTableName(pq)
	if err != nil {
		return err
	}
	pas, err := prepareInsertValues(pq)
	if err != nil {
		return err
	}
	qas, err := sh.PopulateArgSlice(pas.args)
	if err != nil {
		return err
	}
	sh.QueryArgs = qas
	sh.InsertCols = cols
	sh.TableName = table
	return nil
}
func (sh *SpannerHelper) ParseSelect(pq *sqlparser.Select) error {
	sh.IsSelect = true
	spl := strings.Split(sh.RawQuery, "?")
	var updatedQuery string

	if len(sh.OptionArguments) != len(spl) - 1 {
		errStr := "err parsing spanner query: not correct number of option arguments"
		errStr += " for method: %s of service: %s  want: %d have: %d"
		return fmt.Errorf(errStr, sh.Parent.GetName(), sh.Parent.Service.GetName(), len(spl) - 1, len(sh.OptionArguments))
	}
	for i := 0; i < len(spl)-1; i++ {
		name := fmt.Sprintf("@%d", i)
		field := sh.ProtoFieldDescs[sh.OptionArguments[i]]
		qa := QueryArg{
			Name: name,
			IsFieldValue: true,
			Field: field,
		}
		sh.QueryArgs = append(sh.QueryArgs, qa)
		updatedQuery += (spl[i] + name)
	}
	updatedQuery += spl[len(spl)-1]
	sh.Query = updatedQuery
	return nil
}
func (sh *SpannerHelper) ParseDelete(pq *sqlparser.Delete) error {
	sh.IsDelete = true
	table, err := extractIUDTableName(pq)
	if err != nil {
		return err
	}
	mkr, err := extractSpannerKeyFromDelete(pq)
	if err != nil {
		return err
	}
	start, err := sh.PopulateArgSlice(mkr.Start.args)
	if err != nil {
		return err
	}
	end, err := sh.PopulateArgSlice(mkr.End.args)
	if err != nil {
		return err
	}
	low := mkr.LowerOpen
	up := mkr.UpperOpen

	var kind string

	if low && up {
		kind = "spanner.OpenOpen"
	} else if low && !up {
		kind = "spanner.OpenClosed"
	} else if !low && up {
		kind = "spanner.ClosedOpen"
	} else {
		kind = "spanner.ClosedClosed"
	}
	sh.KeyRangeDesc = &KeyRangeDesc{
		Start: start,
		End: end,
		Kind: kind,
	}
	sh.TableName = table
	return nil
}

func (sh *SpannerHelper) ParseUpdate(pq *sqlparser.Update) error {
	sh.IsUpdate = true
	table, err := extractIUDTableName(pq)
	if err != nil {
		return err
	}
	pam, err := extractUpdateClause(pq)
	if err != nil {
		return err
	}
	for key, arg := range pam.args {
		var qa QueryArg
		if ap, ok := arg.(PassedInArgPos); ok {
			index := int(ap)
			argName := sh.OptionArguments[index]
			qa = QueryArg{
				Name: key,
				IsFieldValue: true,
				Field: sh.ProtoFieldDescs[argName],
			}
		} else {
			qa = QueryArg{
				Name: key,
				Value: fmt.Sprintf("%#v", arg),
				IsFieldValue: false,
			}
		}
		sh.QueryArgs = append(sh.QueryArgs, qa)
	}
	sh.TableName = table
	return nil
}

<|endoftext|>


<commit_before>package data

import (
	"database/sql"
	"github.com/UHERO/rest-api/models"
	"strings"
	"time"
)

type SeriesRepository struct {
	DB *sql.DB
}

type transformation struct {
	Statement        string
	PlaceholderCount int
}

const (
	Levels = "lvl"
	YoyPCh = "pc1"
	YTD    = "ytd"
)

var transformations map[string]transformation = map[string]transformation{
	"lvl": {
		Statement:        `SELECT date, value FROM data_points WHERE series_id = ? and current = 1;`,
		PlaceholderCount: 1,
	},
	"pc1": {
		Statement: `SELECT t1.date, (t1.value/t2.last_value - 1)*100 AS yoy
				FROM (SELECT value, date, DATE_SUB(date, INTERVAL 1 YEAR) AS last_year
				FROM data_points WHERE series_id = ? AND current = 1) AS t1
				LEFT JOIN (SELECT value AS last_value, date
				FROM data_points WHERE series_id = ? and current = 1) AS t2
				ON (t1.last_year = t2.date);`,
		PlaceholderCount: 2,
	},
	"ytd": {
		Statement: `SELECT t1.date, (t1.ytd/t2.last_ytd - 1)*100 AS ytd
      FROM (SELECT date, value, @sum := IF(@year = YEAR(date), @sum, 0) + value AS ytd,
            @year := year(date), DATE_SUB(date, INTERVAL 1 YEAR) AS last_year
          FROM data_points CROSS JOIN (SELECT @sum := 0, @year := 0) AS init
          WHERE series_id = ? AND current = 1 ORDER BY date) AS t1
      LEFT JOIN (SELECT date, @sum := IF(@year = YEAR(date), @sum, 0) + value AS last_ytd,
            @year := year(date)
          FROM data_points CROSS JOIN (SELECT @sum := 0, @year := 0) AS init
          WHERE series_id = ? AND current = 1 ORDER BY date) AS t2
      ON (t1.last_year = t2.date);`,
		PlaceholderCount: 2,
	},
}

var seriesPrefix = `SELECT series.id, series.name, description, frequency,
	!(series.name REGEXP '.*NS@.*') AS seasonally_adjusted,
	unitsLabel, unitsLabelShort, dataPortalName, percent, series.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	JOIN data_lists_series ON data_lists_series.series_id = series.id
	JOIN categories ON categories.data_list_id = data_lists_series.data_list_id
	WHERE categories.id = ?`
var geoFilter = ` AND series.name LIKE CONCAT('%@', ? ,'.%') `
var freqFilter = ` AND series.name LIKE CONCAT('%@%.', ?) `
var sortStmt = ` ORDER BY LOCATE(CONCAT(TRIM(TRAILING 'NS' FROM left(series.name, locate('@', series.name) - 1)), '@'),
	(SELECT list FROM data_lists JOIN categories WHERE categories.data_list_id = data_lists.id AND categories.id = ?)) +
	LOCATE(CONCAT(TRIM(TRAILING 'NS' FROM left(series.name, locate('@', series.name) - 1)), 'NS@'),
	(SELECT list FROM data_lists JOIN categories WHERE categories.data_list_id = data_lists.id AND categories.id = ?));`
var siblingsPrefix = `SELECT series.id, series.name, description, frequency,
	!(series.name REGEXP '.*NS@.*') AS seasonally_adjusted,
	unitsLabel, unitsLabelShort, dataPortalName, percent, series.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	JOIN (SELECT name FROM series where id = ?) as original_series
	WHERE TRIM(TRAILING 'NS' FROM left(series.name, locate('@', series.name) - 1))
	LIKE TRIM(TRAILING 'NS' FROM left(original_series.name, locate("@", original_series.name) - 1))`

func (r *SeriesRepository) GetSeriesByCategoryAndFreq(
	categoryId int64,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, freqFilter, sortStmt}, ""),
		categoryId,
		freq,
		categoryId,
		categoryId,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesByCategoryGeoAndFreq(
	categoryId int64,
	geoHandle string,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, geoFilter, freqFilter, sortStmt}, ""),
		categoryId,
		geoHandle,
		freq,
		categoryId,
		categoryId,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesByCategoryAndGeo(
	categoryId int64,
	geoHandle string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, geoFilter, sortStmt}, ""),
		categoryId,
		geoHandle,
		categoryId,
		categoryId,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesBySearchText(searchText string) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(`SELECT series.id, name, description, frequency,
	!(name REGEXP '.*NS@.*') AS seasonally_adjusted,
	unitsLabel, unitsLabelShort, dataPortalName, percent, series.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	WHERE
	((MATCH(name, description, dataPortalName)
	AGAINST(? IN NATURAL LANGUAGE MODE))
	OR LOWER(CONCAT(name, description, dataPortalName)) LIKE CONCAT('%', LOWER(?), '%'));`, searchText, searchText)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesByCategory(categoryId int64) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, sortStmt}, ""),
		categoryId,
		categoryId,
		categoryId,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetFreqByCategory(categoryId int64) (frequencies []models.FrequencyResult, err error) {
	rows, err := r.DB.Query(`SELECT DISTINCT(RIGHT(series.name, 1)) as freq
	FROM series
	JOIN data_lists_series ON data_lists_series.series_id = series.id
	JOIN categories ON categories.data_list_id = data_lists_series.data_list_id
	WHERE categories.id = ?;`, categoryId)
	if err != nil {
		return
	}
	for rows.Next() {
		frequency := models.Frequency{}
		err = rows.Scan(
			&frequency.Freq,
		)
		if err != nil {
			return
		}
		frequencies = append(
			frequencies,
			models.FrequencyResult{Freq: frequency.Freq, Label: freqLabel[frequency.Freq]},
		)
	}
	return

}

func (r *SeriesRepository) GetSeriesSiblingsById(seriesId int64) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(siblingsPrefix, seriesId)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsByIdAndFreq(
	seriesId int64,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(strings.Join([]string{siblingsPrefix, freqFilter}, ""), seriesId, freq)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsByIdAndGeo(
	seriesId int64,
	geo string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(strings.Join([]string{siblingsPrefix, geoFilter}, ""), seriesId, geo)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsByIdGeoAndFreq(
	seriesId int64,
	geo string,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{siblingsPrefix, geoFilter, freqFilter}, ""),
		seriesId, geo, freq)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsFreqById(
	seriesId int64,
) (frequencyList []models.FrequencyResult, err error) {
	rows, err := r.DB.Query(`SELECT DISTINCT(RIGHT(series.name, 1)) as freq
	FROM series JOIN (SELECT name FROM series where id = ?) as original_series
	WHERE series.name LIKE CONCAT(left(original_series.name, locate("@", original_series.name)), '%');`, seriesId)
	if err != nil {
		return
	}
	for rows.Next() {
		frequency := models.Frequency{}
		err = rows.Scan(
			&frequency.Freq,
		)
		if err != nil {
			return
		}
		frequencyList = append(
			frequencyList,
			models.FrequencyResult{Freq: frequency.Freq, Label: freqLabel[frequency.Freq]},
		)
	}
	return
}

func (r *SeriesRepository) GetSeriesById(seriesId int64) (dataPortalSeries models.DataPortalSeries, err error) {
	row := r.DB.QueryRow(`SELECT series.id, name, description, frequency,
	!(name REGEXP '.*NS@.*') AS seasonally_adjusted,
	unitsLabel, unitsLabelShort, dataPortalName, percent, series.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	WHERE series.id = ?;`, seriesId)
	dataPortalSeries, err = getNextSeriesFromRow(row)
	return
}

func (r *SeriesRepository) GetSeriesObservations(
	seriesId int64,
) (seriesObservations models.SeriesObservations, err error) {
	lvlTransform, start, end, err := r.GetTransformation(Levels, seriesId)
	if err != nil {
		return
	}
	yoyTransform, yoyStart, yoyEnd, err := r.GetTransformation(YoyPCh, seriesId)
	if err != nil {
		return
	}
	if yoyStart.Before(start) {
		start = yoyStart
	}
	if end.Before(yoyEnd) {
		end = yoyEnd
	}
	ytdTransform, ytdStart, ytdEnd, err := r.GetTransformation(YTD, seriesId)
	if err != nil {
		return
	}
	if ytdStart.Before(start) {
		start = ytdStart
	}
	if end.Before(ytdEnd) {
		end = ytdEnd
	}
	seriesObservations.TransformationResults = []models.TransformationResult{lvlTransform, yoyTransform, ytdTransform}
	seriesObservations.ObservationStart = start
	seriesObservations.ObservationEnd = end
	return
}

func variadicSeriesId(seriesId int64, count int) []interface{} {
	variadic := make([]interface{}, count, count)
	for i := range variadic {
		variadic[i] = seriesId
	}
	return variadic
}

func (r *SeriesRepository) GetTransformation(transformation string, seriesId int64) (
	transformationResult models.TransformationResult,
	observationStart time.Time,
	observationEnd time.Time,
	err error,
) {
	rows, err := r.DB.Query(
		transformations[transformation].Statement,
		variadicSeriesId(seriesId, transformations[transformation].PlaceholderCount)...,
	)
	if err != nil {
		return
	}
	var (
		observations []models.DataPortalObservation
	)

	for rows.Next() {
		observation := models.Observation{}
		err = rows.Scan(
			&observation.Date,
			&observation.Value,
		)
		if err != nil {
			return
		}
		if !observation.Value.Valid {
			continue
		}
		if observationStart.IsZero() || observation.Date.Before(observationStart) {
			observationStart = observation.Date
		}
		if observationEnd.IsZero() || observationEnd.Before(observation.Date) {
			observationEnd = observation.Date
		}
		observations = append(
			observations,
			models.DataPortalObservation{
				Date:  observation.Date,
				Value: observation.Value.Float64,
			},
		)
	}
	transformationResult.Transformation = transformation
	transformationResult.Observations = observations
	return
}
<commit_msg>updated seriesRepository to use the new data_list_measurements table<commit_after>package data

import (
	"database/sql"
	"github.com/UHERO/rest-api/models"
	"strings"
	"time"
)

type SeriesRepository struct {
	DB *sql.DB
}

type transformation struct {
	Statement        string
	PlaceholderCount int
}

const (
	Levels = "lvl"
	YoyPCh = "pc1"
	YTD    = "ytd"
)

var transformations map[string]transformation = map[string]transformation{
	"lvl": {
		Statement:        `SELECT date, value FROM data_points WHERE series_id = ? and current = 1;`,
		PlaceholderCount: 1,
	},
	"pc1": {
		Statement: `SELECT t1.date, (t1.value/t2.last_value - 1)*100 AS yoy
				FROM (SELECT value, date, DATE_SUB(date, INTERVAL 1 YEAR) AS last_year
				FROM data_points WHERE series_id = ? AND current = 1) AS t1
				LEFT JOIN (SELECT value AS last_value, date
				FROM data_points WHERE series_id = ? and current = 1) AS t2
				ON (t1.last_year = t2.date);`,
		PlaceholderCount: 2,
	},
	"ytd": {
		Statement: `SELECT t1.date, (t1.ytd/t2.last_ytd - 1)*100 AS ytd
      FROM (SELECT date, value, @sum := IF(@year = YEAR(date), @sum, 0) + value AS ytd,
            @year := year(date), DATE_SUB(date, INTERVAL 1 YEAR) AS last_year
          FROM data_points CROSS JOIN (SELECT @sum := 0, @year := 0) AS init
          WHERE series_id = ? AND current = 1 ORDER BY date) AS t1
      LEFT JOIN (SELECT date, @sum := IF(@year = YEAR(date), @sum, 0) + value AS last_ytd,
            @year := year(date)
          FROM data_points CROSS JOIN (SELECT @sum := 0, @year := 0) AS init
          WHERE series_id = ? AND current = 1 ORDER BY date) AS t2
      ON (t1.last_year = t2.date);`,
		PlaceholderCount: 2,
	},
}

var seriesPrefix = `SELECT series.id, series.name, description, frequency, seasonally_adjusted,
	measurements.units_label, measurements.units_label_short, measurements.data_portal_name, measurements.percent, measurements.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	JOIN measurements ON measurements.id = series.measurement_id
	JOIN data_list_measurements ON data_list_measurements.measurement_id = measurements.id
	JOIN categories ON categories.data_list_id = data_list_measurements.data_list_id
	WHERE categories.id = ?`
var geoFilter = ` AND series.name LIKE CONCAT('%@', ? ,'.%') `
var freqFilter = ` AND series.name LIKE CONCAT('%@%.', ?) `
var sortStmt = ` ORDER BY data_list_measurements.list_order;`
var siblingsPrefix = `SELECT series.id, series.name, description, frequency, seasonally_adjusted,
	measurements.units_label, measurements.units_label_short, measurements.data_portal_name, measurements.percent, series.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	JOIN (SELECT name FROM series where id = ?) as original_series
	WHERE
  		TRIM(TRAILING '&NS' FROM TRIM(TRAILING 'NS' FROM
  		  UPPER(LEFT(series.name, LOCATE('@', series.name) - 1))
  		))
	LIKE
  		TRIM(TRAILING '&NS' FROM TRIM(TRAILING 'NS' FROM
  		  UPPER(LEFT(original_series.name, LOCATE('@', original_series.name) - 1))
  		))`

func (r *SeriesRepository) GetSeriesByCategoryAndFreq(
	categoryId int64,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, freqFilter, sortStmt}, ""),
		categoryId,
		freq,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesByCategoryGeoAndFreq(
	categoryId int64,
	geoHandle string,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, geoFilter, freqFilter, sortStmt}, ""),
		categoryId,
		geoHandle,
		freq,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesByCategoryAndGeo(
	categoryId int64,
	geoHandle string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, geoFilter, sortStmt}, ""),
		categoryId,
		geoHandle,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesBySearchText(searchText string) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(`SELECT series.id, name, description, frequency, seasonally_adjusted,
	unitsLabel, unitsLabelShort, dataPortalName, percent, series.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	WHERE
	((MATCH(name, description, dataPortalName)
	AGAINST(? IN NATURAL LANGUAGE MODE))
	OR LOWER(CONCAT(name, description, dataPortalName)) LIKE CONCAT('%', LOWER(?), '%'));`, searchText, searchText)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesByCategory(categoryId int64) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{seriesPrefix, sortStmt}, ""),
		categoryId,
	)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetFreqByCategory(categoryId int64) (frequencies []models.FrequencyResult, err error) {
	rows, err := r.DB.Query(`SELECT DISTINCT(RIGHT(series.name, 1)) as freq
	FROM series
	JOIN data_lists_series ON data_lists_series.series_id = series.id
	JOIN categories ON categories.data_list_id = data_lists_series.data_list_id
	WHERE categories.id = ?;`, categoryId)
	if err != nil {
		return
	}
	for rows.Next() {
		frequency := models.Frequency{}
		err = rows.Scan(
			&frequency.Freq,
		)
		if err != nil {
			return
		}
		frequencies = append(
			frequencies,
			models.FrequencyResult{Freq: frequency.Freq, Label: freqLabel[frequency.Freq]},
		)
	}
	return

}

func (r *SeriesRepository) GetSeriesSiblingsById(seriesId int64) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(siblingsPrefix, seriesId)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsByIdAndFreq(
	seriesId int64,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(strings.Join([]string{siblingsPrefix, freqFilter}, ""), seriesId, freq)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsByIdAndGeo(
	seriesId int64,
	geo string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(strings.Join([]string{siblingsPrefix, geoFilter}, ""), seriesId, geo)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsByIdGeoAndFreq(
	seriesId int64,
	geo string,
	freq string,
) (seriesList []models.DataPortalSeries, err error) {
	rows, err := r.DB.Query(
		strings.Join([]string{siblingsPrefix, geoFilter, freqFilter}, ""),
		seriesId, geo, freq)
	if err != nil {
		return
	}
	for rows.Next() {
		dataPortalSeries, scanErr := getNextSeriesFromRows(rows)
		if scanErr != nil {
			return seriesList, scanErr
		}
		seriesList = append(seriesList, dataPortalSeries)
	}
	return
}

func (r *SeriesRepository) GetSeriesSiblingsFreqById(
	seriesId int64,
) (frequencyList []models.FrequencyResult, err error) {
	rows, err := r.DB.Query(`SELECT DISTINCT(RIGHT(series.name, 1)) as freq
	FROM series JOIN (SELECT name FROM series where id = ?) as original_series
	WHERE series.name LIKE CONCAT(left(original_series.name, locate("@", original_series.name)), '%');`, seriesId)
	if err != nil {
		return
	}
	for rows.Next() {
		frequency := models.Frequency{}
		err = rows.Scan(
			&frequency.Freq,
		)
		if err != nil {
			return
		}
		frequencyList = append(
			frequencyList,
			models.FrequencyResult{Freq: frequency.Freq, Label: freqLabel[frequency.Freq]},
		)
	}
	return
}

func (r *SeriesRepository) GetSeriesById(seriesId int64) (dataPortalSeries models.DataPortalSeries, err error) {
	row := r.DB.QueryRow(`SELECT series.id, name, description, frequency, seasonally_adjusted,
	unitsLabel, unitsLabelShort, dataPortalName, percent, series.real,
	fips, SUBSTRING_INDEX(SUBSTR(series.name, LOCATE('@', series.name) + 1), '.', 1) as shandle, display_name_short
	FROM series LEFT JOIN geographies ON name LIKE CONCAT('%@', handle, '.%')
	WHERE series.id = ?;`, seriesId)
	dataPortalSeries, err = getNextSeriesFromRow(row)
	return
}

func (r *SeriesRepository) GetSeriesObservations(
	seriesId int64,
) (seriesObservations models.SeriesObservations, err error) {
	lvlTransform, start, end, err := r.GetTransformation(Levels, seriesId)
	if err != nil {
		return
	}
	yoyTransform, yoyStart, yoyEnd, err := r.GetTransformation(YoyPCh, seriesId)
	if err != nil {
		return
	}
	if yoyStart.Before(start) {
		start = yoyStart
	}
	if end.Before(yoyEnd) {
		end = yoyEnd
	}
	ytdTransform, ytdStart, ytdEnd, err := r.GetTransformation(YTD, seriesId)
	if err != nil {
		return
	}
	if ytdStart.Before(start) {
		start = ytdStart
	}
	if end.Before(ytdEnd) {
		end = ytdEnd
	}
	seriesObservations.TransformationResults = []models.TransformationResult{lvlTransform, yoyTransform, ytdTransform}
	seriesObservations.ObservationStart = start
	seriesObservations.ObservationEnd = end
	return
}

func variadicSeriesId(seriesId int64, count int) []interface{} {
	variadic := make([]interface{}, count, count)
	for i := range variadic {
		variadic[i] = seriesId
	}
	return variadic
}

func (r *SeriesRepository) GetTransformation(transformation string, seriesId int64) (
	transformationResult models.TransformationResult,
	observationStart time.Time,
	observationEnd time.Time,
	err error,
) {
	rows, err := r.DB.Query(
		transformations[transformation].Statement,
		variadicSeriesId(seriesId, transformations[transformation].PlaceholderCount)...,
	)
	if err != nil {
		return
	}
	var (
		observations []models.DataPortalObservation
	)

	for rows.Next() {
		observation := models.Observation{}
		err = rows.Scan(
			&observation.Date,
			&observation.Value,
		)
		if err != nil {
			return
		}
		if !observation.Value.Valid {
			continue
		}
		if observationStart.IsZero() || observation.Date.Before(observationStart) {
			observationStart = observation.Date
		}
		if observationEnd.IsZero() || observationEnd.Before(observation.Date) {
			observationEnd = observation.Date
		}
		observations = append(
			observations,
			models.DataPortalObservation{
				Date:  observation.Date,
				Value: observation.Value.Float64,
			},
		)
	}
	transformationResult.Transformation = transformation
	transformationResult.Observations = observations
	return
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/common/util"
	cutil "github.com/hyperledger/fabric/core/container/util"
)

var logger = flogging.MustGetLogger("util")

//ComputeHash computes contents hash based on previous hash
func ComputeHash(contents []byte, hash []byte) []byte {
	newSlice := make([]byte, len(hash)+len(contents))

	//copy the contents
	copy(newSlice[0:len(contents)], contents[:])

	//add the previous hash
	copy(newSlice[len(contents):], hash[:])

	//compute new hash
	hash = util.ComputeSHA256(newSlice)

	return hash
}

//HashFilesInDir computes h=hash(h,file bytes) for each file in a directory
//Directory entries are traversed recursively. In the end a single
//hash value is returned for the entire directory structure
func HashFilesInDir(rootDir string, dir string, hash []byte, tw *tar.Writer) ([]byte, error) {
	currentDir := filepath.Join(rootDir, dir)
	logger.Debugf("hashFiles %s", currentDir)
	//ReadDir returns sorted list of files in dir
	fis, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return hash, fmt.Errorf("ReadDir failed %s\n", err)
	}
	for _, fi := range fis {
		name := filepath.Join(dir, fi.Name())
		if fi.IsDir() {
			var err error
			hash, err = HashFilesInDir(rootDir, name, hash, tw)
			if err != nil {
				return hash, err
			}
			continue
		}
		fqp := filepath.Join(rootDir, name)
		buf, err := ioutil.ReadFile(fqp)
		if err != nil {
			logger.Errorf("Error reading %s\n", err)
			return hash, err
		}

		//get the new hash from file contents
		hash = ComputeHash(buf, hash)

		if tw != nil {
			is := bytes.NewReader(buf)
			if err = cutil.WriteStreamToPackage(is, fqp, filepath.Join("src", name), tw); err != nil {
				return hash, fmt.Errorf("Error adding file to tar %s", err)
			}
		}
	}
	return hash, nil
}

//IsCodeExist checks the chaincode if exists
func IsCodeExist(tmppath string) error {
	file, err := os.Open(tmppath)
	if err != nil {
		return fmt.Errorf("Could not open file %s", err)
	}

	fi, err := file.Stat()
	if err != nil {
		return fmt.Errorf("Could not stat file %s", err)
	}

	if !fi.IsDir() {
		return fmt.Errorf("File %s is not dir\n", file.Name())
	}

	return nil
}

type DockerBuildOptions struct {
	Image        string
	Env          []string
	Cmd          string
	InputStream  io.Reader
	OutputStream io.Writer
}

//-------------------------------------------------------------------------------------------
// DockerBuild
//-------------------------------------------------------------------------------------------
// This function allows a "pass-through" build of chaincode within a docker container as
// an alternative to using standard "docker build" + Dockerfile mechanisms.  The plain docker
// build is somewhat limiting due to the resulting image that is a superset composition of
// the build-time and run-time environments.  This superset can be problematic on several
// fronts, such as a bloated image size, and additional security exposure associated with
// applications that are not needed, etc.
//
// Therefore, this mechanism creates a pipeline consisting of an ephemeral docker
// container that accepts source code as input, runs some function (e.g. "go build"), and
// outputs the result.  The intention is that this output will be consumed as the basis of
// a streamlined container by installing the output into a downstream docker-build based on
// an appropriate minimal image.
//
// The input parameters are fairly simple:
//      - Image:        (optional) The builder image to use or "chaincode.builder"
//      - Env:          (optional) environment variables for the build environment.
//      - Cmd:          The command to execute inside the container.
//      - InputStream:  A tarball of files that will be expanded into /chaincode/input.
//      - OutputStream: A tarball of files that will be gathered from /chaincode/output
//                      after successful execution of Cmd.
//-------------------------------------------------------------------------------------------
func DockerBuild(opts DockerBuildOptions) error {
	client, err := cutil.NewDockerClient()
	if err != nil {
		return fmt.Errorf("Error creating docker client: %s", err)
	}
	if opts.Image == "" {
		opts.Image = cutil.GetDockerfileFromConfig("chaincode.builder")
		if opts.Image == "" {
			return fmt.Errorf("No image provided and \"chaincode.builder\" default does not exist")
		}
	}

	logger.Debugf("Attempting build with image %s", opts.Image)

	//-----------------------------------------------------------------------------------
	// Ensure the image exists locally, or pull it from a registry if it doesn't
	//-----------------------------------------------------------------------------------
	_, err = client.InspectImage(opts.Image)
	if err != nil {
		logger.Debugf("Image %s does not exist locally, attempt pull", opts.Image)

		err = client.PullImage(docker.PullImageOptions{Repository: opts.Image}, docker.AuthConfiguration{})
		if err != nil {
			return fmt.Errorf("Failed to pull %s: %s", opts.Image, err)
		}
	}

	//-----------------------------------------------------------------------------------
	// Create an ephemeral container, armed with our Env/Cmd
	//-----------------------------------------------------------------------------------
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image:        opts.Image,
			Env:          opts.Env,
			Cmd:          []string{"/bin/sh", "-c", opts.Cmd},
			AttachStdout: true,
			AttachStderr: true,
		},
	})
	if err != nil {
		return fmt.Errorf("Error creating container: %s", err)
	}
	defer client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	//-----------------------------------------------------------------------------------
	// Upload our input stream
	//-----------------------------------------------------------------------------------
	err = client.UploadToContainer(container.ID, docker.UploadToContainerOptions{
		Path:        "/chaincode/input",
		InputStream: opts.InputStream,
	})
	if err != nil {
		return fmt.Errorf("Error uploading input to container: %s", err)
	}

	//-----------------------------------------------------------------------------------
	// Attach stdout buffer to capture possible compilation errors
	//-----------------------------------------------------------------------------------
	stdout := bytes.NewBuffer(nil)
	cw, err := client.AttachToContainerNonBlocking(docker.AttachToContainerOptions{
		Container:    container.ID,
		OutputStream: stdout,
		ErrorStream:  stdout,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
	if err != nil {
		return fmt.Errorf("Error attaching to container: %s", err)
	}

	//-----------------------------------------------------------------------------------
	// Launch the actual build, realizing the Env/Cmd specified at container creation
	//-----------------------------------------------------------------------------------
	err = client.StartContainer(container.ID, nil)
	if err != nil {
		cw.Close()
		return fmt.Errorf("Error executing build: %s \"%s\"", err, stdout.String())
	}

	//-----------------------------------------------------------------------------------
	// Wait for the build to complete and gather the return value
	//-----------------------------------------------------------------------------------
	retval, err := client.WaitContainer(container.ID)
	if err != nil {
		cw.Close()
		return fmt.Errorf("Error waiting for container to complete: %s", err)
	}
	cw.Close()

	if retval > 0 {
		// Wait for stream copying to complete before getting output
		if err := cw.Wait(); err != nil {
			logger.Errorf("attach wait failed: %s", err)
		}
		return fmt.Errorf("Error returned from build: %d \"%s\"", retval, stdout.String())
	}

	logger.Debugf("Build output is %s", stdout.String())

	//-----------------------------------------------------------------------------------
	// Finally, download the result
	//-----------------------------------------------------------------------------------
	err = client.DownloadFromContainer(container.ID, docker.DownloadFromContainerOptions{
		Path:         "/chaincode/output/.",
		OutputStream: opts.OutputStream,
	})
	if err != nil {
		return fmt.Errorf("Error downloading output: %s", err)
	}

	return nil
}
<commit_msg>[FAB-11683] fix data race in GenerateDockerBuild<commit_after>/*
Copyright IBM Corp. 2016 All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		 http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/common/util"
	cutil "github.com/hyperledger/fabric/core/container/util"
)

var logger = flogging.MustGetLogger("util")

//ComputeHash computes contents hash based on previous hash
func ComputeHash(contents []byte, hash []byte) []byte {
	newSlice := make([]byte, len(hash)+len(contents))

	//copy the contents
	copy(newSlice[0:len(contents)], contents[:])

	//add the previous hash
	copy(newSlice[len(contents):], hash[:])

	//compute new hash
	hash = util.ComputeSHA256(newSlice)

	return hash
}

//HashFilesInDir computes h=hash(h,file bytes) for each file in a directory
//Directory entries are traversed recursively. In the end a single
//hash value is returned for the entire directory structure
func HashFilesInDir(rootDir string, dir string, hash []byte, tw *tar.Writer) ([]byte, error) {
	currentDir := filepath.Join(rootDir, dir)
	logger.Debugf("hashFiles %s", currentDir)
	//ReadDir returns sorted list of files in dir
	fis, err := ioutil.ReadDir(currentDir)
	if err != nil {
		return hash, fmt.Errorf("ReadDir failed %s\n", err)
	}
	for _, fi := range fis {
		name := filepath.Join(dir, fi.Name())
		if fi.IsDir() {
			var err error
			hash, err = HashFilesInDir(rootDir, name, hash, tw)
			if err != nil {
				return hash, err
			}
			continue
		}
		fqp := filepath.Join(rootDir, name)
		buf, err := ioutil.ReadFile(fqp)
		if err != nil {
			logger.Errorf("Error reading %s\n", err)
			return hash, err
		}

		//get the new hash from file contents
		hash = ComputeHash(buf, hash)

		if tw != nil {
			is := bytes.NewReader(buf)
			if err = cutil.WriteStreamToPackage(is, fqp, filepath.Join("src", name), tw); err != nil {
				return hash, fmt.Errorf("Error adding file to tar %s", err)
			}
		}
	}
	return hash, nil
}

//IsCodeExist checks the chaincode if exists
func IsCodeExist(tmppath string) error {
	file, err := os.Open(tmppath)
	if err != nil {
		return fmt.Errorf("Could not open file %s", err)
	}

	fi, err := file.Stat()
	if err != nil {
		return fmt.Errorf("Could not stat file %s", err)
	}

	if !fi.IsDir() {
		return fmt.Errorf("File %s is not dir\n", file.Name())
	}

	return nil
}

type DockerBuildOptions struct {
	Image        string
	Env          []string
	Cmd          string
	InputStream  io.Reader
	OutputStream io.Writer
}

//-------------------------------------------------------------------------------------------
// DockerBuild
//-------------------------------------------------------------------------------------------
// This function allows a "pass-through" build of chaincode within a docker container as
// an alternative to using standard "docker build" + Dockerfile mechanisms.  The plain docker
// build is somewhat limiting due to the resulting image that is a superset composition of
// the build-time and run-time environments.  This superset can be problematic on several
// fronts, such as a bloated image size, and additional security exposure associated with
// applications that are not needed, etc.
//
// Therefore, this mechanism creates a pipeline consisting of an ephemeral docker
// container that accepts source code as input, runs some function (e.g. "go build"), and
// outputs the result.  The intention is that this output will be consumed as the basis of
// a streamlined container by installing the output into a downstream docker-build based on
// an appropriate minimal image.
//
// The input parameters are fairly simple:
//      - Image:        (optional) The builder image to use or "chaincode.builder"
//      - Env:          (optional) environment variables for the build environment.
//      - Cmd:          The command to execute inside the container.
//      - InputStream:  A tarball of files that will be expanded into /chaincode/input.
//      - OutputStream: A tarball of files that will be gathered from /chaincode/output
//                      after successful execution of Cmd.
//-------------------------------------------------------------------------------------------
func DockerBuild(opts DockerBuildOptions) error {
	client, err := cutil.NewDockerClient()
	if err != nil {
		return fmt.Errorf("Error creating docker client: %s", err)
	}
	if opts.Image == "" {
		opts.Image = cutil.GetDockerfileFromConfig("chaincode.builder")
		if opts.Image == "" {
			return fmt.Errorf("No image provided and \"chaincode.builder\" default does not exist")
		}
	}

	logger.Debugf("Attempting build with image %s", opts.Image)

	//-----------------------------------------------------------------------------------
	// Ensure the image exists locally, or pull it from a registry if it doesn't
	//-----------------------------------------------------------------------------------
	_, err = client.InspectImage(opts.Image)
	if err != nil {
		logger.Debugf("Image %s does not exist locally, attempt pull", opts.Image)

		err = client.PullImage(docker.PullImageOptions{Repository: opts.Image}, docker.AuthConfiguration{})
		if err != nil {
			return fmt.Errorf("Failed to pull %s: %s", opts.Image, err)
		}
	}

	//-----------------------------------------------------------------------------------
	// Create an ephemeral container, armed with our Env/Cmd
	//-----------------------------------------------------------------------------------
	container, err := client.CreateContainer(docker.CreateContainerOptions{
		Config: &docker.Config{
			Image:        opts.Image,
			Env:          opts.Env,
			Cmd:          []string{"/bin/sh", "-c", opts.Cmd},
			AttachStdout: true,
			AttachStderr: true,
		},
	})
	if err != nil {
		return fmt.Errorf("Error creating container: %s", err)
	}
	defer client.RemoveContainer(docker.RemoveContainerOptions{ID: container.ID})

	//-----------------------------------------------------------------------------------
	// Upload our input stream
	//-----------------------------------------------------------------------------------
	err = client.UploadToContainer(container.ID, docker.UploadToContainerOptions{
		Path:        "/chaincode/input",
		InputStream: opts.InputStream,
	})
	if err != nil {
		return fmt.Errorf("Error uploading input to container: %s", err)
	}

	//-----------------------------------------------------------------------------------
	// Attach stdout buffer to capture possible compilation errors
	//-----------------------------------------------------------------------------------
	stdout := bytes.NewBuffer(nil)
	cw, err := client.AttachToContainerNonBlocking(docker.AttachToContainerOptions{
		Container:    container.ID,
		OutputStream: stdout,
		ErrorStream:  stdout,
		Logs:         true,
		Stdout:       true,
		Stderr:       true,
		Stream:       true,
	})
	if err != nil {
		return fmt.Errorf("Error attaching to container: %s", err)
	}

	//-----------------------------------------------------------------------------------
	// Launch the actual build, realizing the Env/Cmd specified at container creation
	//-----------------------------------------------------------------------------------
	err = client.StartContainer(container.ID, nil)
	if err != nil {
		cw.Close()
		return fmt.Errorf("Error executing build: %s \"%s\"", err, stdout.String())
	}

	//-----------------------------------------------------------------------------------
	// Wait for the build to complete and gather the return value
	//-----------------------------------------------------------------------------------
	retval, err := client.WaitContainer(container.ID)
	if err != nil {
		cw.Close()
		return fmt.Errorf("Error waiting for container to complete: %s", err)
	}

	// Wait for stream copying to complete before accessing stdout.
	cw.Close()
	if err := cw.Wait(); err != nil {
		logger.Errorf("attach wait failed: %s", err)
	}

	if retval > 0 {
		return fmt.Errorf("Error returned from build: %d \"%s\"", retval, stdout.String())
	}

	logger.Debugf("Build output is %s", stdout.String())

	//-----------------------------------------------------------------------------------
	// Finally, download the result
	//-----------------------------------------------------------------------------------
	err = client.DownloadFromContainer(container.ID, docker.DownloadFromContainerOptions{
		Path:         "/chaincode/output/.",
		OutputStream: opts.OutputStream,
	})
	if err != nil {
		return fmt.Errorf("Error downloading output: %s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package lint

import (
	"fmt"
)

// Rules is a registry for registering and looking up rules.
type Rules map[RuleName]Rule

// Copy returns a new copy of the rules.
func (r Rules) Copy() Rules {
	n := make(Rules, len(r))
	for k, v := range r {
		n[k] = v
	}
	return n
}

// Register registers the list of rules.
// It returns an error if any of the rules is found duplicate
// in the registry.
func (r Rules) Register(rules ...Rule) error {
	for _, rl := range rules {
		if !rl.Info().Name.IsValid() {
			return fmt.Errorf("%v is not a valid RuleName", rl.Info().Name)
		}

		if _, found := r[rl.Info().Name]; found {
			return fmt.Errorf("duplicate rule name `%s`", rl.Info().Name)
		}

		r[rl.Info().Name] = rl
	}
	return nil
}

// All returns all rules.
func (r Rules) All() []Rule {
	rules := make([]Rule, 0, len(r))
	for _, r1 := range r {
		rules = append(rules, r1)
	}
	return rules
}

// NewRules returns a rule registry initialized with the given set of rules.
func NewRules(rules ...Rule) (Rules, error) {
	r := make(Rules, len(rules))
	err := r.Register(rules...)
	return r, err
}
<commit_msg>Fixed fmt string for consistency<commit_after>package lint

import (
	"fmt"
)

// Rules is a registry for registering and looking up rules.
type Rules map[RuleName]Rule

// Copy returns a new copy of the rules.
func (r Rules) Copy() Rules {
	n := make(Rules, len(r))
	for k, v := range r {
		n[k] = v
	}
	return n
}

// Register registers the list of rules.
// It returns an error if any of the rules is found duplicate
// in the registry.
func (r Rules) Register(rules ...Rule) error {
	for _, rl := range rules {
		if !rl.Info().Name.IsValid() {
			return fmt.Errorf("%q is not a valid RuleName", rl.Info().Name)
		}

		if _, found := r[rl.Info().Name]; found {
			return fmt.Errorf("duplicate rule name %q", rl.Info().Name)
		}

		r[rl.Info().Name] = rl
	}
	return nil
}

// All returns all rules.
func (r Rules) All() []Rule {
	rules := make([]Rule, 0, len(r))
	for _, r1 := range r {
		rules = append(rules, r1)
	}
	return rules
}

// NewRules returns a rule registry initialized with the given set of rules.
func NewRules(rules ...Rule) (Rules, error) {
	r := make(Rules, len(rules))
	err := r.Register(rules...)
	return r, err
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package doc extracts source code documentation from a Go AST.
package doc

import (
	"go/ast"
	"go/token"
)

// Package is the documentation for an entire package.
type Package struct {
	Doc        string
	Name       string
	ImportPath string
	Imports    []string
	Filenames  []string
	Notes      map[string][]*Note
	// DEPRECATED. For backward compatibility Bugs is still populated,
	// but all new code should use Notes instead.
	Bugs []string

	// declarations
	Consts []*Value
	Types  []*Type
	Vars   []*Value
	Funcs  []*Func
}

// Value is the documentation for a (possibly grouped) var or const declaration.
type Value struct {
	Doc   string
	Names []string // var or const names in declaration order
	Decl  *ast.GenDecl

	order int
}

// Type is the documentation for a type declaration.
type Type struct {
	Doc  string
	Name string
	Decl *ast.GenDecl

	// associated declarations
	Consts  []*Value // sorted list of constants of (mostly) this type
	Vars    []*Value // sorted list of variables of (mostly) this type
	Funcs   []*Func  // sorted list of functions returning this type
	Methods []*Func  // sorted list of methods (including embedded ones) of this type
}

// Func is the documentation for a func declaration.
type Func struct {
	Doc  string
	Name string
	Decl *ast.FuncDecl

	// methods
	// (for functions, these fields have the respective zero value)
	Recv  string // actual   receiver "T" or "*T"
	Orig  string // original receiver "T" or "*T"
	Level int    // embedding level; 0 means not embedded
}

// A Note represents marked comments starting with "MARKER(uid): note body".
// Any note with a marker of 2 or more upper case [A-Z] letters and a uid of
// at least one character is recognized. The ":" following the uid is optional.
// Notes are collected in the Package.Notes map indexed by the notes marker.
type Note struct {
	Pos  token.Pos // position of the comment containing the marker
	UID  string    // uid found with the marker
	Body string    // note body text
}

// Mode values control the operation of New.
type Mode int

const (
	// extract documentation for all package-level declarations,
	// not just exported ones
	AllDecls Mode = 1 << iota

	// show all embedded methods, not just the ones of
	// invisible (unexported) anonymous fields
	AllMethods
)

// New computes the package documentation for the given package AST.
// New takes ownership of the AST pkg and may edit or overwrite it.
//
func New(pkg *ast.Package, importPath string, mode Mode) *Package {
	var r reader
	r.readPackage(pkg, mode)
	r.computeMethodSets()
	r.cleanupTypes()
	return &Package{
		Doc:        r.doc,
		Name:       pkg.Name,
		ImportPath: importPath,
		Imports:    sortedKeys(r.imports),
		Filenames:  r.filenames,
		Notes:      r.notes,
		Bugs:       noteBodies(r.notes["BUG"]),
		Consts:     sortedValues(r.values, token.CONST),
		Types:      sortedTypes(r.types, mode&AllMethods != 0),
		Vars:       sortedValues(r.values, token.VAR),
		Funcs:      sortedFuncs(r.funcs, true),
	}
}
<commit_msg>go/doc: fix typo in comment<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package doc extracts source code documentation from a Go AST.
package doc

import (
	"go/ast"
	"go/token"
)

// Package is the documentation for an entire package.
type Package struct {
	Doc        string
	Name       string
	ImportPath string
	Imports    []string
	Filenames  []string
	Notes      map[string][]*Note
	// DEPRECATED. For backward compatibility Bugs is still populated,
	// but all new code should use Notes instead.
	Bugs []string

	// declarations
	Consts []*Value
	Types  []*Type
	Vars   []*Value
	Funcs  []*Func
}

// Value is the documentation for a (possibly grouped) var or const declaration.
type Value struct {
	Doc   string
	Names []string // var or const names in declaration order
	Decl  *ast.GenDecl

	order int
}

// Type is the documentation for a type declaration.
type Type struct {
	Doc  string
	Name string
	Decl *ast.GenDecl

	// associated declarations
	Consts  []*Value // sorted list of constants of (mostly) this type
	Vars    []*Value // sorted list of variables of (mostly) this type
	Funcs   []*Func  // sorted list of functions returning this type
	Methods []*Func  // sorted list of methods (including embedded ones) of this type
}

// Func is the documentation for a func declaration.
type Func struct {
	Doc  string
	Name string
	Decl *ast.FuncDecl

	// methods
	// (for functions, these fields have the respective zero value)
	Recv  string // actual   receiver "T" or "*T"
	Orig  string // original receiver "T" or "*T"
	Level int    // embedding level; 0 means not embedded
}

// A Note represents a marked comment starting with "MARKER(uid): note body".
// Any note with a marker of 2 or more upper case [A-Z] letters and a uid of
// at least one character is recognized. The ":" following the uid is optional.
// Notes are collected in the Package.Notes map indexed by the notes marker.
type Note struct {
	Pos  token.Pos // position of the comment containing the marker
	UID  string    // uid found with the marker
	Body string    // note body text
}

// Mode values control the operation of New.
type Mode int

const (
	// extract documentation for all package-level declarations,
	// not just exported ones
	AllDecls Mode = 1 << iota

	// show all embedded methods, not just the ones of
	// invisible (unexported) anonymous fields
	AllMethods
)

// New computes the package documentation for the given package AST.
// New takes ownership of the AST pkg and may edit or overwrite it.
//
func New(pkg *ast.Package, importPath string, mode Mode) *Package {
	var r reader
	r.readPackage(pkg, mode)
	r.computeMethodSets()
	r.cleanupTypes()
	return &Package{
		Doc:        r.doc,
		Name:       pkg.Name,
		ImportPath: importPath,
		Imports:    sortedKeys(r.imports),
		Filenames:  r.filenames,
		Notes:      r.notes,
		Bugs:       noteBodies(r.notes["BUG"]),
		Consts:     sortedValues(r.values, token.CONST),
		Types:      sortedTypes(r.types, mode&AllMethods != 0),
		Vars:       sortedValues(r.values, token.VAR),
		Funcs:      sortedFuncs(r.funcs, true),
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"container/heap"
	"sync"
)

// The Timer type represents a single event.
// When the Timer expires, the current time will be sent on C
// unless the Timer represents an AfterFunc event.
type Timer struct {
	C <-chan int64
	t int64       // The absolute time that the event should fire.
	f func(int64) // The function to call when the event fires.
	i int         // The event's index inside eventHeap.
}

type timerHeap []*Timer

// forever is the absolute time (in ns) of an event that is forever away.
const forever = 1 << 62

// maxSleepTime is the maximum length of time that a sleeper
// sleeps for before checking if it is defunct.
const maxSleepTime = 1e9

var (
	// timerMutex guards the variables inside this var group.
	timerMutex sync.Mutex

	// timers holds a binary heap of pending events, terminated with a sentinel.
	timers timerHeap

	// currentSleeper is an ever-incrementing counter which represents
	// the current sleeper. It allows older sleepers to detect that they are
	// defunct and exit.
	currentSleeper int64
)

func init() {
	timers.Push(&Timer{t: forever}) // sentinel
}

// NewTimer creates a new Timer that will send
// the current time on its channel after at least ns nanoseconds.
func NewTimer(ns int64) *Timer {
	c := make(chan int64, 1)
	e := after(ns, func(t int64) { c <- t })
	e.C = c
	return e
}

// After waits at least ns nanoseconds before sending the current time
// on the returned channel.
// It is equivalent to NewTimer(ns).C.
func After(ns int64) <-chan int64 {
	return NewTimer(ns).C
}

// AfterFunc waits at least ns nanoseconds before calling f
// in its own goroutine. It returns a Timer that can
// be used to cancel the call using its Stop method.
func AfterFunc(ns int64, f func()) *Timer {
	return after(ns, func(_ int64) {
		go f()
	})
}

// Stop prevents the Timer from firing.
// It returns true if the call stops the timer, false if the timer has already
// expired or stopped.
func (e *Timer) Stop() (ok bool) {
	timerMutex.Lock()
	// Avoid removing the first event in the queue so that
	// we don't start a new sleeper unnecessarily.
	if e.i > 0 {
		heap.Remove(timers, e.i)
	}
	ok = e.f != nil
	e.f = nil
	timerMutex.Unlock()
	return
}

// after is the implementation of After and AfterFunc.
// When the current time is after ns, it calls f with the current time.
// It assumes that f will not block.
func after(ns int64, f func(int64)) (e *Timer) {
	now := Nanoseconds()
	t := Nanoseconds() + ns
	if ns > 0 && t < now {
		panic("time: time overflow")
	}
	timerMutex.Lock()
	t0 := timers[0].t
	e = &Timer{nil, t, f, -1}
	heap.Push(timers, e)
	// Start a new sleeper if the new event is before
	// the first event in the queue. If the length of time
	// until the new event is at least maxSleepTime,
	// then we're guaranteed that the sleeper will wake up
	// in time to service it, so no new sleeper is needed.
	if t0 > t && (t0 == forever || ns < maxSleepTime) {
		currentSleeper++
		go sleeper(currentSleeper)
	}
	timerMutex.Unlock()
	return
}

// sleeper continually looks at the earliest event in the queue, waits until it happens,
// then removes any events in the queue that are due. It stops when the queue
// is empty or when another sleeper has been started.
func sleeper(sleeperId int64) {
	timerMutex.Lock()
	e := timers[0]
	t := Nanoseconds()
	for e.t != forever {
		if dt := e.t - t; dt > 0 {
			if dt > maxSleepTime {
				dt = maxSleepTime
			}
			timerMutex.Unlock()
			sysSleep(dt)
			timerMutex.Lock()
			if currentSleeper != sleeperId {
				// Another sleeper has been started, making this one redundant.
				break
			}
		}
		e = timers[0]
		t = Nanoseconds()
		for t >= e.t {
			if e.f != nil {
				e.f(t)
				e.f = nil
			}
			heap.Pop(timers)
			e = timers[0]
		}
	}
	timerMutex.Unlock()
}

func (timerHeap) Len() int {
	return len(timers)
}

func (timerHeap) Less(i, j int) bool {
	return timers[i].t < timers[j].t
}

func (timerHeap) Swap(i, j int) {
	timers[i], timers[j] = timers[j], timers[i]
	timers[i].i = i
	timers[j].i = j
}

func (timerHeap) Push(x interface{}) {
	e := x.(*Timer)
	e.i = len(timers)
	timers = append(timers, e)
}

func (timerHeap) Pop() interface{} {
	// TODO: possibly shrink array.
	n := len(timers) - 1
	e := timers[n]
	timers[n] = nil
	timers = timers[0:n]
	e.i = -1
	return e
}
<commit_msg>time: Remove unnecessary call to Nanoseconds() in after().<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

import (
	"container/heap"
	"sync"
)

// The Timer type represents a single event.
// When the Timer expires, the current time will be sent on C
// unless the Timer represents an AfterFunc event.
type Timer struct {
	C <-chan int64
	t int64       // The absolute time that the event should fire.
	f func(int64) // The function to call when the event fires.
	i int         // The event's index inside eventHeap.
}

type timerHeap []*Timer

// forever is the absolute time (in ns) of an event that is forever away.
const forever = 1 << 62

// maxSleepTime is the maximum length of time that a sleeper
// sleeps for before checking if it is defunct.
const maxSleepTime = 1e9

var (
	// timerMutex guards the variables inside this var group.
	timerMutex sync.Mutex

	// timers holds a binary heap of pending events, terminated with a sentinel.
	timers timerHeap

	// currentSleeper is an ever-incrementing counter which represents
	// the current sleeper. It allows older sleepers to detect that they are
	// defunct and exit.
	currentSleeper int64
)

func init() {
	timers.Push(&Timer{t: forever}) // sentinel
}

// NewTimer creates a new Timer that will send
// the current time on its channel after at least ns nanoseconds.
func NewTimer(ns int64) *Timer {
	c := make(chan int64, 1)
	e := after(ns, func(t int64) { c <- t })
	e.C = c
	return e
}

// After waits at least ns nanoseconds before sending the current time
// on the returned channel.
// It is equivalent to NewTimer(ns).C.
func After(ns int64) <-chan int64 {
	return NewTimer(ns).C
}

// AfterFunc waits at least ns nanoseconds before calling f
// in its own goroutine. It returns a Timer that can
// be used to cancel the call using its Stop method.
func AfterFunc(ns int64, f func()) *Timer {
	return after(ns, func(_ int64) {
		go f()
	})
}

// Stop prevents the Timer from firing.
// It returns true if the call stops the timer, false if the timer has already
// expired or stopped.
func (e *Timer) Stop() (ok bool) {
	timerMutex.Lock()
	// Avoid removing the first event in the queue so that
	// we don't start a new sleeper unnecessarily.
	if e.i > 0 {
		heap.Remove(timers, e.i)
	}
	ok = e.f != nil
	e.f = nil
	timerMutex.Unlock()
	return
}

// after is the implementation of After and AfterFunc.
// When the current time is after ns, it calls f with the current time.
// It assumes that f will not block.
func after(ns int64, f func(int64)) (e *Timer) {
	now := Nanoseconds()
	t := now + ns
	if ns > 0 && t < now {
		panic("time: time overflow")
	}
	timerMutex.Lock()
	t0 := timers[0].t
	e = &Timer{nil, t, f, -1}
	heap.Push(timers, e)
	// Start a new sleeper if the new event is before
	// the first event in the queue. If the length of time
	// until the new event is at least maxSleepTime,
	// then we're guaranteed that the sleeper will wake up
	// in time to service it, so no new sleeper is needed.
	if t0 > t && (t0 == forever || ns < maxSleepTime) {
		currentSleeper++
		go sleeper(currentSleeper)
	}
	timerMutex.Unlock()
	return
}

// sleeper continually looks at the earliest event in the queue, waits until it happens,
// then removes any events in the queue that are due. It stops when the queue
// is empty or when another sleeper has been started.
func sleeper(sleeperId int64) {
	timerMutex.Lock()
	e := timers[0]
	t := Nanoseconds()
	for e.t != forever {
		if dt := e.t - t; dt > 0 {
			if dt > maxSleepTime {
				dt = maxSleepTime
			}
			timerMutex.Unlock()
			sysSleep(dt)
			timerMutex.Lock()
			if currentSleeper != sleeperId {
				// Another sleeper has been started, making this one redundant.
				break
			}
		}
		e = timers[0]
		t = Nanoseconds()
		for t >= e.t {
			if e.f != nil {
				e.f(t)
				e.f = nil
			}
			heap.Pop(timers)
			e = timers[0]
		}
	}
	timerMutex.Unlock()
}

func (timerHeap) Len() int {
	return len(timers)
}

func (timerHeap) Less(i, j int) bool {
	return timers[i].t < timers[j].t
}

func (timerHeap) Swap(i, j int) {
	timers[i], timers[j] = timers[j], timers[i]
	timers[i].i = i
	timers[j].i = j
}

func (timerHeap) Push(x interface{}) {
	e := x.(*Timer)
	e.i = len(timers)
	timers = append(timers, e)
}

func (timerHeap) Pop() interface{} {
	// TODO: possibly shrink array.
	n := len(timers) - 1
	e := timers[n]
	timers[n] = nil
	timers = timers[0:n]
	e.i = -1
	return e
}
<|endoftext|>


<commit_before>package yamlConfig

import (
	"encoding/json"
	"io/ioutil"

	"github.com/vvvntdotorg/felicium/config"
	"github.com/vvvntdotorg/felicium/config/hashConfig"

	"github.com/vvvntdotorg/felicium/Godeps/_workspace/src/gopkg.in/yaml.v2"
)

type configuration struct {
	config.Configurator
	path string
}

func NewConfig(path, env string) (config.Configurator, error) {

	yc := &configuration{}
	yc.path = path

	//load config
	if err := yc.loadFromFile(env); err != nil {
		return nil, err
	}

	return yc, nil

}

func (yc *configuration) Reload() error {
	return yc.loadFromFile(yc.Env())
}

func (yc *configuration) loadFromFile(env string) error {

	var err error
	var rawconfig []byte
	if rawconfig, err = ioutil.ReadFile(yc.path); err != nil {
		return err
	}

	tmp := map[string]map[string]interface{}{}

	if err = yaml.Unmarshal([]byte(rawconfig), tmp); err != nil {
		return err
	}

	_, foundDefault := tmp["default"]
	_, foundEnv := tmp[env]
	configData := map[string]interface{}{}

	// your options here are you have defined the default and the env you requested
	// or you have only defined the requested env. This will error if you don't have
	// the requested env defined. This is to reduce confusion.
	switch {
	case foundDefault && foundEnv:
		var err error
		var defaultData, envData []byte

		// are you looking at this and thinking thats getto?
		// your right the merge lib on github didn't work and
		// yaml doesn't have a Raw type to decode into
		// TODO: do this a better way
		if defaultData, err = json.Marshal(tmp["default"]); err != nil {
			return err
		}
		if envData, err = json.Marshal(tmp[env]); err != nil {
			return err
		}

		if err = json.Unmarshal(defaultData, &configData); err != nil {
			return err
		}
		// if you asked for the default we are done
		if env != "default" {
			// override if needed
			if err = json.Unmarshal(envData, &configData); err != nil {
				return err
			}
		}
	case foundEnv:
		configData = tmp[env]
	default:
		return config.InvalidEnvError
	}
	yc.Configurator = hashConfig.NewConfiguration(configData, env)
	return nil
}
<commit_msg>Update yaml.go<commit_after>package yamlConfig

import (
	"encoding/json"
	"io/ioutil"

	"github.com/avant/felicium/config"
	"github.com/avant/felicium/config/hashConfig"

	"github.com/avant/felicium/Godeps/_workspace/src/gopkg.in/yaml.v2"
)

type configuration struct {
	config.Configurator
	path string
}

func NewConfig(path, env string) (config.Configurator, error) {

	yc := &configuration{}
	yc.path = path

	//load config
	if err := yc.loadFromFile(env); err != nil {
		return nil, err
	}

	return yc, nil

}

func (yc *configuration) Reload() error {
	return yc.loadFromFile(yc.Env())
}

func (yc *configuration) loadFromFile(env string) error {

	var err error
	var rawconfig []byte
	if rawconfig, err = ioutil.ReadFile(yc.path); err != nil {
		return err
	}

	tmp := map[string]map[string]interface{}{}

	if err = yaml.Unmarshal([]byte(rawconfig), tmp); err != nil {
		return err
	}

	_, foundDefault := tmp["default"]
	_, foundEnv := tmp[env]
	configData := map[string]interface{}{}

	// your options here are you have defined the default and the env you requested
	// or you have only defined the requested env. This will error if you don't have
	// the requested env defined. This is to reduce confusion.
	switch {
	case foundDefault && foundEnv:
		var err error
		var defaultData, envData []byte

		// are you looking at this and thinking thats getto?
		// your right the merge lib on github didn't work and
		// yaml doesn't have a Raw type to decode into
		// TODO: do this a better way
		if defaultData, err = json.Marshal(tmp["default"]); err != nil {
			return err
		}
		if envData, err = json.Marshal(tmp[env]); err != nil {
			return err
		}

		if err = json.Unmarshal(defaultData, &configData); err != nil {
			return err
		}
		// if you asked for the default we are done
		if env != "default" {
			// override if needed
			if err = json.Unmarshal(envData, &configData); err != nil {
				return err
			}
		}
	case foundEnv:
		configData = tmp[env]
	default:
		return config.InvalidEnvError
	}
	yc.Configurator = hashConfig.NewConfiguration(configData, env)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/pelletier/go-toml"
	//"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"sync"
	"text/template"
)

type config struct {
	emails      []string
	mailBinPath string
	ikachanUrl  string
	channel     string
}

type consulAlert struct {
	Timestamp string
	Node      string
	ServiceId string
	Service   string
	CheckId   string
	Check     string
	Status    string
	Output    string
	Notes     string
}

func (c *consulAlert) TrimmedOutput() string {
	return strings.TrimSpace(c.Output)
}

func (c *consulAlert) StatusString() string {
	status := strings.ToUpper(c.Status)
	switch c.Status {
	case "passing":
		return colorMsg(status, cGreen, cNone)
	case "critical":
		return colorMsg(status, cBlack, cRed)
	default:
		return colorMsg(status, cYellow, cNone)
	}
}

func (c *consulAlert) NodeString() string {
	return setIrcMode(ircUnderline) + c.Node + setIrcMode(ircCReset)
}

const (
	version = "0.0.1"
)

var (
	ircBodyTemplate = setIrcMode(ircBold) +
		"*** {{.Service}}({{.CheckId}}) is now {{.StatusString}}" +
		setIrcMode(ircBold) +
		" on {{.NodeString}}" +
		" - {{.TrimmedOutput}}"

	mailTitleTemplate = "Check {{.CheckId}} is now {{.Status}} on {{.Node}}"
	mailBodyTemplate  = `
{{.Service}}({{.CheckId}}) is now {{.Status}}
On node {{.Node}}

Output is:
  {{.TrimmedOutput}}
`

	logger = log.New(os.Stdout, "[consul-simple-notifier] ", log.LstdFlags)
)

func main() {
	var (
		justShowVersion bool
		configPath      string
		conf            config
		input           []consulAlert
	)

	flag.BoolVar(&justShowVersion, "v", false, "Show version")
	flag.BoolVar(&justShowVersion, "version", false, "Show version")

	flag.StringVar(&configPath, "c", "/etc/consul-simple-notifier.ini", "Config path")
	flag.Parse()

	if justShowVersion {
		showVersion()
		return
	}

	parsed, err := toml.LoadFile(configPath)
	if err != nil {
		panic(err.Error())
	}

	binPath := parsed.Get("email.binpath")
	if binPath == nil {
		conf.mailBinPath = "/bin/mail"
	} else {
		conf.mailBinPath = binPath.(string)
	}

	recipients := parsed.Get("email.recipients")
	for _, address := range recipients.([]interface{}) {
		conf.emails = append(conf.emails, address.(string))
	}

	conf.ikachanUrl = parsed.Get("ikachan.url").(string)
	conf.channel = parsed.Get("ikachan.channel").(string)
	logger.Printf("conf is: %+v\n", conf)

	err = json.NewDecoder(os.Stdin).Decode(&input)
	if err != nil {
		panic(err.Error())
	}
	logger.Printf("input json is: %+v\n", input)

	for _, content := range input {
		var wg sync.WaitGroup
		wg.Add((len(conf.emails) + 1))

		go func(_wg *sync.WaitGroup, _content *consulAlert) {
			for _, address := range conf.emails {
				err := notifyEmail(conf.mailBinPath, address, _content, _wg)
				if err != nil {
					panic(err)
				}
			}
		}(&wg, &content)
		go func(_wg *sync.WaitGroup, _content *consulAlert) {
			err = notifyIkachan(conf.ikachanUrl, conf.channel, _content, _wg)
			if err != nil {
				panic(err)
			}
		}(&wg, &content)
		wg.Wait()
	}
}

func notifyEmail(mainBinPath, address string, content *consulAlert, wg *sync.WaitGroup) error {
	var titleBuf, bodyBuf bytes.Buffer
	titleTmpl := template.Must(template.New("emailTitle").Parse(mailTitleTemplate))
	bodyTmpl := template.Must(template.New("emailBody").Parse(mailBodyTemplate))
	err := titleTmpl.Execute(&titleBuf, &content)
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	title := titleBuf.String()

	logger.Printf("Sending... %s to %s\n", title, address)
	cmd := exec.Command(mainBinPath, "-s", title, address)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	fmt.Fprint(stdin, bodyBuf.String())
	stdin.Close()
	logger.Printf("Send!\n")
	cmd.Wait()
	wg.Done()
	return nil
}

func notifyIkachan(ikachanUrl string, channel string, content *consulAlert, wg *sync.WaitGroup) error {
	joinUrl := fmt.Sprintf("%s/join", ikachanUrl)
	noticeUrl := fmt.Sprintf("%s/notice", ikachanUrl)

	values := make(url.Values)
	values.Set("channel", channel)

	resp1, err := http.PostForm(joinUrl, values)
	if err != nil {
		return err
	}
	defer resp1.Body.Close()

	var bodyBuf bytes.Buffer
	bodyTmpl := template.Must(template.New("ircBody").Parse(ircBodyTemplate))
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	body := bodyBuf.String()

	values.Set("message", body)

	logger.Printf("Posted! %+v", values)
	resp2, err := http.PostForm(noticeUrl, values)
	if err != nil {
		return err
	}
	defer resp2.Body.Close()

	wg.Done()
	return nil
}

func showVersion() {
	fmt.Printf("consul-simple-notifier version: %s\n", version)
}
<commit_msg>Concurrent along with the input array size<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/pelletier/go-toml"
	//"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"strings"
	"sync"
	"text/template"
)

type config struct {
	emails      []string
	mailBinPath string
	ikachanUrl  string
	channel     string
}

type consulAlert struct {
	Timestamp string
	Node      string
	ServiceId string
	Service   string
	CheckId   string
	Check     string
	Status    string
	Output    string
	Notes     string
}

func (c *consulAlert) TrimmedOutput() string {
	return strings.TrimSpace(c.Output)
}

func (c *consulAlert) StatusString() string {
	status := strings.ToUpper(c.Status)
	switch c.Status {
	case "passing":
		return colorMsg(status, cGreen, cNone)
	case "critical":
		return colorMsg(status, cBlack, cRed)
	default:
		return colorMsg(status, cYellow, cNone)
	}
}

func (c *consulAlert) NodeString() string {
	return setIrcMode(ircUnderline) + c.Node + setIrcMode(ircCReset)
}

const (
	version = "0.0.1"
)

var (
	ircBodyTemplate = setIrcMode(ircBold) +
		"*** {{.Service}}({{.CheckId}}) is now {{.StatusString}}" +
		setIrcMode(ircBold) +
		" on {{.NodeString}}" +
		" - {{.TrimmedOutput}}"

	mailTitleTemplate = "Check {{.CheckId}} is now {{.Status}} on {{.Node}}"
	mailBodyTemplate  = `
{{.Service}}({{.CheckId}}) is now {{.Status}}
On node {{.Node}}

Output is:
  {{.TrimmedOutput}}
`

	logger = log.New(os.Stdout, "[consul-simple-notifier] ", log.LstdFlags)
)

func main() {
	var (
		justShowVersion bool
		configPath      string
		conf            config
		input           []consulAlert
	)

	flag.BoolVar(&justShowVersion, "v", false, "Show version")
	flag.BoolVar(&justShowVersion, "version", false, "Show version")

	flag.StringVar(&configPath, "c", "/etc/consul-simple-notifier.ini", "Config path")
	flag.Parse()

	if justShowVersion {
		showVersion()
		return
	}

	parsed, err := toml.LoadFile(configPath)
	if err != nil {
		panic(err.Error())
	}

	binPath := parsed.Get("email.binpath")
	if binPath == nil {
		conf.mailBinPath = "/bin/mail"
	} else {
		conf.mailBinPath = binPath.(string)
	}

	recipients := parsed.Get("email.recipients")
	for _, address := range recipients.([]interface{}) {
		conf.emails = append(conf.emails, address.(string))
	}

	conf.ikachanUrl = parsed.Get("ikachan.url").(string)
	conf.channel = parsed.Get("ikachan.channel").(string)
	logger.Printf("conf is: %+v\n", conf)

	err = json.NewDecoder(os.Stdin).Decode(&input)
	if err != nil {
		panic(err.Error())
	}
	logger.Printf("input json is: %+v\n", input)

	var wg sync.WaitGroup
	wg.Add((len(conf.emails) + 1) * len(input))

	for _, content := range input {
		go func(_wg *sync.WaitGroup, _content consulAlert) {
			for _, address := range conf.emails {
				err := notifyEmail(conf.mailBinPath, address, &_content, _wg)
				if err != nil {
					panic(err)
				}
			}
		}(&wg, content)
		go func(_wg *sync.WaitGroup, _content consulAlert) {
			err = notifyIkachan(conf.ikachanUrl, conf.channel, &_content, _wg)
			if err != nil {
				panic(err)
			}
		}(&wg, content)
	}
	wg.Wait()
}

func notifyEmail(mainBinPath, address string, content *consulAlert, wg *sync.WaitGroup) error {
	var titleBuf, bodyBuf bytes.Buffer
	titleTmpl := template.Must(template.New("emailTitle").Parse(mailTitleTemplate))
	bodyTmpl := template.Must(template.New("emailBody").Parse(mailBodyTemplate))
	err := titleTmpl.Execute(&titleBuf, &content)
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	title := titleBuf.String()

	logger.Printf("Sending... %s to %s\n", title, address)
	cmd := exec.Command(mainBinPath, "-s", title, address)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}

	fmt.Fprint(stdin, bodyBuf.String())
	stdin.Close()
	logger.Printf("Send!\n")
	cmd.Wait()
	wg.Done()
	return nil
}

func notifyIkachan(ikachanUrl string, channel string, content *consulAlert, wg *sync.WaitGroup) error {
	joinUrl := fmt.Sprintf("%s/join", ikachanUrl)
	noticeUrl := fmt.Sprintf("%s/notice", ikachanUrl)

	values := make(url.Values)
	values.Set("channel", channel)

	resp1, err := http.PostForm(joinUrl, values)
	if err != nil {
		return err
	}
	defer resp1.Body.Close()

	var bodyBuf bytes.Buffer
	bodyTmpl := template.Must(template.New("ircBody").Parse(ircBodyTemplate))
	err = bodyTmpl.Execute(&bodyBuf, &content)
	if err != nil {
		return err
	}
	body := bodyBuf.String()

	values.Set("message", body)

	logger.Printf("Posted! %+v", values)
	resp2, err := http.PostForm(noticeUrl, values)
	if err != nil {
		return err
	}
	defer resp2.Body.Close()

	wg.Done()
	return nil
}

func showVersion() {
	fmt.Printf("consul-simple-notifier version: %s\n", version)
}
<|endoftext|>