<commit_before>package data

import (
	"database/sql"
	"errors"
	"github.com/UHERO/rest-api/models"
	"sort"
)

var freqLabel map[string]string = map[string]string{
	"A": "Annual",
	"S": "Semiannual",
	"Q": "Quarterly",
	"M": "Monthly",
	"W": "Weekly",
	"D": "Daily",
}

var indentationLevel map[string]int = map[string]int{
	"indent0": 0,
	"indent1": 1,
	"indent2": 2,
}

func getNextSeriesFromRows(rows *sql.Rows) (dataPortalSeries models.DataPortalSeries, err error) {
	series := models.Series{}
	geography := models.Geography{}
	err = rows.Scan(
		&series.Id,
		&series.Name,
		&series.Description,
		&series.Frequency,
		&series.SeasonallyAdjusted,
		&series.UnitsLabel,
		&series.UnitsLabelShort,
		&series.DataPortalName,
		&series.Percent,
		&series.Real,
		&series.SourceDescription,
		&series.SourceLink,
		&series.Indent,
		&geography.FIPS,
		&geography.Handle,
		&geography.Name,
	)
	if err != nil {
		return
	}
	dataPortalSeries = models.DataPortalSeries{
		Id:             series.Id,
		Name:           series.Name,
		FrequencyShort: series.Name[len(series.Name)-1:],
	}
	dataPortalSeries.Frequency = freqLabel[dataPortalSeries.FrequencyShort]
	if series.DataPortalName.Valid {
		dataPortalSeries.Title = series.DataPortalName.String
	}
	if series.Description.Valid {
		dataPortalSeries.Description = series.Description.String
	}
	if series.SeasonallyAdjusted.Valid && series.Name[len(series.Name)-1:] != "A" {
		dataPortalSeries.SeasonallyAdjusted = &series.SeasonallyAdjusted.Bool
	}
	if series.UnitsLabel.Valid {
		dataPortalSeries.UnitsLabel = series.UnitsLabel.String
	}
	if series.UnitsLabelShort.Valid {
		dataPortalSeries.UnitsLabelShort = series.UnitsLabelShort.String
	}
	if series.Percent.Valid {
		dataPortalSeries.Percent = &series.Percent.Bool
	}
	if series.Real.Valid {
		dataPortalSeries.Real = &series.Real.Bool
	}
	if series.SourceDescription.Valid {
		dataPortalSeries.SourceDescription = series.SourceDescription.String
	}
	if series.SourceLink.Valid {
		dataPortalSeries.SourceLink = series.SourceLink.String
	}
	if series.Indent.Valid {
		dataPortalSeries.Indent = indentationLevel[series.Indent.String]
	}
	dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
	if geography.FIPS.Valid {
		dataPortalGeography.FIPS = geography.FIPS.String
	}
	if geography.Name.Valid {
		dataPortalGeography.Name = geography.Name.String
	}
	dataPortalSeries.Geography = dataPortalGeography
	return
}

func getNextSeriesFromRow(row *sql.Row) (dataPortalSeries models.DataPortalSeries, err error) {
	series := models.Series{}
	geography := models.Geography{}
	err = row.Scan(
		&series.Id,
		&series.Name,
		&series.Description,
		&series.Frequency,
		&series.SeasonallyAdjusted,
		&series.UnitsLabel,
		&series.UnitsLabelShort,
		&series.DataPortalName,
		&series.Percent,
		&series.Real,
		&series.SourceDescription,
		&series.SourceLink,
		&geography.FIPS,
		&geography.Handle,
		&geography.Name,
	)
	if err != nil {
		return dataPortalSeries, errors.New("Series restricted or does not exist.")
	}
	dataPortalSeries = models.DataPortalSeries{
		Id:             series.Id,
		Name:           series.Name,
		FrequencyShort: series.Name[len(series.Name)-1:],
	}
	dataPortalSeries.Frequency = freqLabel[dataPortalSeries.FrequencyShort]
	if series.DataPortalName.Valid {
		dataPortalSeries.Title = series.DataPortalName.String
	}
	if series.Description.Valid {
		dataPortalSeries.Description = series.Description.String
	}
	if series.SeasonallyAdjusted.Valid && series.Name[len(series.Name)-1:] != "A" {
		dataPortalSeries.SeasonallyAdjusted = &series.SeasonallyAdjusted.Bool
	}
	if series.UnitsLabel.Valid {
		dataPortalSeries.UnitsLabel = series.UnitsLabel.String
	}
	if series.UnitsLabelShort.Valid {
		dataPortalSeries.UnitsLabelShort = series.UnitsLabelShort.String
	}
	if series.Percent.Valid {
		dataPortalSeries.Percent = &series.Percent.Bool
	}
	if series.Real.Valid {
		dataPortalSeries.Real = &series.Real.Bool
	}
	if series.SourceDescription.Valid {
		dataPortalSeries.SourceDescription = series.SourceDescription.String
	}
	if series.SourceLink.Valid {
		dataPortalSeries.SourceLink = series.SourceLink.String
	}
	dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
	if geography.FIPS.Valid {
		dataPortalGeography.FIPS = geography.FIPS.String
	}
	if geography.Name.Valid {
		dataPortalGeography.Name = geography.Name.String
	}
	dataPortalSeries.Geography = dataPortalGeography
	return
}

func getFreqGeoCombinations(r *SeriesRepository, seriesId int64) (
	[]models.GeographyFrequencies,
	[]models.FrequencyGeographies,
	error,
) {
	rows, err := r.DB.Query(`SELECT geographies.fips, geographies.display_name_short, geofreq.geo, geofreq.freq
	FROM (SELECT MAX(SUBSTRING_INDEX(SUBSTR(name, LOCATE('@', name) + 1), '.', 1)) as geo,
		   MAX(RIGHT(name, 1)) as freq
	FROM (SELECT series.name AS name FROM
		series
		WHERE series.measurement_id = (SELECT measurement_id FROM series WHERE id = ?)) AS s
	GROUP BY SUBSTR(name, LOCATE('@', name) + 1) ORDER BY COUNT(*) DESC) as geofreq
	LEFT JOIN geographies ON geographies.handle = geofreq.geo;`, seriesId)
	if err != nil {
		return nil, nil, err
	}
	geoFreqs := map[string][]models.FrequencyResult{}
	geoByHandle := map[string]models.DataPortalGeography{}
	freqGeos := map[string][]models.DataPortalGeography{}
	freqByHandle := map[string]models.FrequencyResult{}
	for rows.Next() {
		scangeo := models.Geography{}
		frequency := models.FrequencyResult{}
		err = rows.Scan(
			&scangeo.FIPS,
			&scangeo.Name,
			&scangeo.Handle,
			&frequency.Freq,
		)
		geography := models.DataPortalGeography{Handle: scangeo.Handle}
		if scangeo.FIPS.Valid {
			geography.FIPS = scangeo.FIPS.String
		}
		if scangeo.Name.Valid {
			geography.Name = scangeo.Name.String
		}
		frequency.Label = freqLabel[frequency.Freq]
		// update the freq and geo maps
		geoByHandle[geography.Handle] = geography
		freqByHandle[frequency.Freq] = frequency
		// add to the geoFreqs and freqGeos maps
		geoFreqs[geography.Handle] = append(geoFreqs[geography.Handle], frequency)
		freqGeos[frequency.Freq] = append(freqGeos[frequency.Freq], geography)
	}
	geoFreqsResult := []models.GeographyFrequencies{}
	for geo, freqs := range geoFreqs {
		sort.Sort(models.ByFrequency(freqs))
		geoFreqsResult = append(geoFreqsResult, models.GeographyFrequencies{
			DataPortalGeography: geoByHandle[geo],
			Frequencies:         freqs,
		})
	}

	freqGeosResult := []models.FrequencyGeographies{}
	for _, freq := range models.FreqOrder {
		if val, ok := freqByHandle[freq]; ok {
			freqGeosResult = append(freqGeosResult, models.FrequencyGeographies{
				FrequencyResult: val,
				Geographies:     freqGeos[freq],
			})
		}
	}

	return geoFreqsResult, freqGeosResult, err
}
<commit_msg>added indent3 to map<commit_after>package data

import (
	"database/sql"
	"errors"
	"github.com/UHERO/rest-api/models"
	"sort"
)

var freqLabel map[string]string = map[string]string{
	"A": "Annual",
	"S": "Semiannual",
	"Q": "Quarterly",
	"M": "Monthly",
	"W": "Weekly",
	"D": "Daily",
}

var indentationLevel map[string]int = map[string]int{
	"indent0": 0,
	"indent1": 1,
	"indent2": 2,
	"indent3": 3,
}

func getNextSeriesFromRows(rows *sql.Rows) (dataPortalSeries models.DataPortalSeries, err error) {
	series := models.Series{}
	geography := models.Geography{}
	err = rows.Scan(
		&series.Id,
		&series.Name,
		&series.Description,
		&series.Frequency,
		&series.SeasonallyAdjusted,
		&series.UnitsLabel,
		&series.UnitsLabelShort,
		&series.DataPortalName,
		&series.Percent,
		&series.Real,
		&series.SourceDescription,
		&series.SourceLink,
		&series.Indent,
		&geography.FIPS,
		&geography.Handle,
		&geography.Name,
	)
	if err != nil {
		return
	}
	dataPortalSeries = models.DataPortalSeries{
		Id:             series.Id,
		Name:           series.Name,
		FrequencyShort: series.Name[len(series.Name)-1:],
	}
	dataPortalSeries.Frequency = freqLabel[dataPortalSeries.FrequencyShort]
	if series.DataPortalName.Valid {
		dataPortalSeries.Title = series.DataPortalName.String
	}
	if series.Description.Valid {
		dataPortalSeries.Description = series.Description.String
	}
	if series.SeasonallyAdjusted.Valid && series.Name[len(series.Name)-1:] != "A" {
		dataPortalSeries.SeasonallyAdjusted = &series.SeasonallyAdjusted.Bool
	}
	if series.UnitsLabel.Valid {
		dataPortalSeries.UnitsLabel = series.UnitsLabel.String
	}
	if series.UnitsLabelShort.Valid {
		dataPortalSeries.UnitsLabelShort = series.UnitsLabelShort.String
	}
	if series.Percent.Valid {
		dataPortalSeries.Percent = &series.Percent.Bool
	}
	if series.Real.Valid {
		dataPortalSeries.Real = &series.Real.Bool
	}
	if series.SourceDescription.Valid {
		dataPortalSeries.SourceDescription = series.SourceDescription.String
	}
	if series.SourceLink.Valid {
		dataPortalSeries.SourceLink = series.SourceLink.String
	}
	if series.Indent.Valid {
		dataPortalSeries.Indent = indentationLevel[series.Indent.String]
	}
	dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
	if geography.FIPS.Valid {
		dataPortalGeography.FIPS = geography.FIPS.String
	}
	if geography.Name.Valid {
		dataPortalGeography.Name = geography.Name.String
	}
	dataPortalSeries.Geography = dataPortalGeography
	return
}

func getNextSeriesFromRow(row *sql.Row) (dataPortalSeries models.DataPortalSeries, err error) {
	series := models.Series{}
	geography := models.Geography{}
	err = row.Scan(
		&series.Id,
		&series.Name,
		&series.Description,
		&series.Frequency,
		&series.SeasonallyAdjusted,
		&series.UnitsLabel,
		&series.UnitsLabelShort,
		&series.DataPortalName,
		&series.Percent,
		&series.Real,
		&series.SourceDescription,
		&series.SourceLink,
		&geography.FIPS,
		&geography.Handle,
		&geography.Name,
	)
	if err != nil {
		return dataPortalSeries, errors.New("Series restricted or does not exist.")
	}
	dataPortalSeries = models.DataPortalSeries{
		Id:             series.Id,
		Name:           series.Name,
		FrequencyShort: series.Name[len(series.Name)-1:],
	}
	dataPortalSeries.Frequency = freqLabel[dataPortalSeries.FrequencyShort]
	if series.DataPortalName.Valid {
		dataPortalSeries.Title = series.DataPortalName.String
	}
	if series.Description.Valid {
		dataPortalSeries.Description = series.Description.String
	}
	if series.SeasonallyAdjusted.Valid && series.Name[len(series.Name)-1:] != "A" {
		dataPortalSeries.SeasonallyAdjusted = &series.SeasonallyAdjusted.Bool
	}
	if series.UnitsLabel.Valid {
		dataPortalSeries.UnitsLabel = series.UnitsLabel.String
	}
	if series.UnitsLabelShort.Valid {
		dataPortalSeries.UnitsLabelShort = series.UnitsLabelShort.String
	}
	if series.Percent.Valid {
		dataPortalSeries.Percent = &series.Percent.Bool
	}
	if series.Real.Valid {
		dataPortalSeries.Real = &series.Real.Bool
	}
	if series.SourceDescription.Valid {
		dataPortalSeries.SourceDescription = series.SourceDescription.String
	}
	if series.SourceLink.Valid {
		dataPortalSeries.SourceLink = series.SourceLink.String
	}
	dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
	if geography.FIPS.Valid {
		dataPortalGeography.FIPS = geography.FIPS.String
	}
	if geography.Name.Valid {
		dataPortalGeography.Name = geography.Name.String
	}
	dataPortalSeries.Geography = dataPortalGeography
	return
}

func getFreqGeoCombinations(r *SeriesRepository, seriesId int64) (
	[]models.GeographyFrequencies,
	[]models.FrequencyGeographies,
	error,
) {
	rows, err := r.DB.Query(`SELECT geographies.fips, geographies.display_name_short, geofreq.geo, geofreq.freq
	FROM (SELECT MAX(SUBSTRING_INDEX(SUBSTR(name, LOCATE('@', name) + 1), '.', 1)) as geo,
		   MAX(RIGHT(name, 1)) as freq
	FROM (SELECT series.name AS name FROM
		series
		WHERE series.measurement_id = (SELECT measurement_id FROM series WHERE id = ?)) AS s
	GROUP BY SUBSTR(name, LOCATE('@', name) + 1) ORDER BY COUNT(*) DESC) as geofreq
	LEFT JOIN geographies ON geographies.handle = geofreq.geo;`, seriesId)
	if err != nil {
		return nil, nil, err
	}
	geoFreqs := map[string][]models.FrequencyResult{}
	geoByHandle := map[string]models.DataPortalGeography{}
	freqGeos := map[string][]models.DataPortalGeography{}
	freqByHandle := map[string]models.FrequencyResult{}
	for rows.Next() {
		scangeo := models.Geography{}
		frequency := models.FrequencyResult{}
		err = rows.Scan(
			&scangeo.FIPS,
			&scangeo.Name,
			&scangeo.Handle,
			&frequency.Freq,
		)
		geography := models.DataPortalGeography{Handle: scangeo.Handle}
		if scangeo.FIPS.Valid {
			geography.FIPS = scangeo.FIPS.String
		}
		if scangeo.Name.Valid {
			geography.Name = scangeo.Name.String
		}
		frequency.Label = freqLabel[frequency.Freq]
		// update the freq and geo maps
		geoByHandle[geography.Handle] = geography
		freqByHandle[frequency.Freq] = frequency
		// add to the geoFreqs and freqGeos maps
		geoFreqs[geography.Handle] = append(geoFreqs[geography.Handle], frequency)
		freqGeos[frequency.Freq] = append(freqGeos[frequency.Freq], geography)
	}
	geoFreqsResult := []models.GeographyFrequencies{}
	for geo, freqs := range geoFreqs {
		sort.Sort(models.ByFrequency(freqs))
		geoFreqsResult = append(geoFreqsResult, models.GeographyFrequencies{
			DataPortalGeography: geoByHandle[geo],
			Frequencies:         freqs,
		})
	}

	freqGeosResult := []models.FrequencyGeographies{}
	for _, freq := range models.FreqOrder {
		if val, ok := freqByHandle[freq]; ok {
			freqGeosResult = append(freqGeosResult, models.FrequencyGeographies{
				FrequencyResult: val,
				Geographies:     freqGeos[freq],
			})
		}
	}

	return geoFreqsResult, freqGeosResult, err
}
<|endoftext|>


<commit_before>package data

import (
	"database/sql"
	"github.com/UHERO/rest-api/models"
	"strconv"
	"strings"
	"time"
)

var freqLabel map[string]string = map[string]string{
	"A": "Annual",
	"S": "Semiannual",
	"Q": "Quarterly",
	"M": "Monthly",
	"W": "Weekly",
	"D": "Daily",
}

var freqDbNames map[string]string = map[string]string{
	"A": "year",
	"S": "semi",
	"Q": "quarter",
	"M": "month",
	"W": "week",
	"D": "day",
}

var indentationLevel map[string]int = map[string]int{
	"indent0": 0,
	"indent1": 1,
	"indent2": 2,
	"indent3": 3,
}

func getNextSeriesFromRows(rows *sql.Rows) (dataPortalSeries models.DataPortalSeries, err error) {
	series := models.Series{}
	geography := models.Geography{}
	err = rows.Scan(
		&series.Id,
		&series.Name,
		&series.Universe,
		&series.Description,
		&series.Frequency,
		&series.SeasonallyAdjusted,
		&series.SeasonalAdjustment,
		&series.UnitsLabel,
		&series.UnitsLabelShort,
		&series.DataPortalName,
		&series.Percent,
		&series.Real,
		&series.SourceDescription,
		&series.SourceLink,
		&series.SourceDetails,
		&series.TablePrefix,
		&series.TablePostfix,
		&series.MeasurementId,
		&series.MeasurementName,
		&series.Indent,
		&series.BaseYear,
		&series.Decimals,
		&geography.FIPS,
		&geography.Handle,
		&geography.Name,
		&geography.ShortName,
	)
	if err != nil {
		return
	}
	dataPortalSeries = models.DataPortalSeries{
		Id:       series.Id,
		Name:     series.Name,
		Universe: series.Universe,
	}
	dataPortalSeries.FrequencyShort = dataPortalSeries.Name[len(dataPortalSeries.Name)-1:]
	dataPortalSeries.Frequency = freqLabel[dataPortalSeries.FrequencyShort]
	if series.DataPortalName.Valid {
		dataPortalSeries.Title = series.DataPortalName.String
	}
	if series.Description.Valid {
		dataPortalSeries.Description = series.Description.String
	}
	if series.SeasonallyAdjusted.Valid && dataPortalSeries.FrequencyShort != "A" {
		dataPortalSeries.SeasonallyAdjusted = &series.SeasonallyAdjusted.Bool
	}
	if series.SeasonalAdjustment.Valid {
		dataPortalSeries.SeasonalAdjustment = series.SeasonalAdjustment.String
	}
	if series.UnitsLabel.Valid {
		dataPortalSeries.UnitsLabel = series.UnitsLabel.String
	}
	if series.UnitsLabelShort.Valid {
		dataPortalSeries.UnitsLabelShort = series.UnitsLabelShort.String
	}
	if series.Percent.Valid {
		dataPortalSeries.Percent = &series.Percent.Bool
	}
	if series.Real.Valid {
		dataPortalSeries.Real = &series.Real.Bool
	}
	if series.SourceDescription.Valid {
		dataPortalSeries.SourceDescription = series.SourceDescription.String
		dataPortalSeries.SourceDescriptionDeprecated = series.SourceDescription.String
	}
	if series.SourceLink.Valid {
		dataPortalSeries.SourceLink = series.SourceLink.String
		dataPortalSeries.SourceLinkDeprecated = series.SourceLink.String
	}
	if series.SourceDetails.Valid {
		dataPortalSeries.SourceDetails = series.SourceDetails.String
	}
	if series.TablePrefix.Valid {
		dataPortalSeries.TablePrefix = series.TablePrefix.String
	}
	if series.TablePostfix.Valid {
		dataPortalSeries.TablePostfix = series.TablePostfix.String
	}
	if series.MeasurementId.Valid {
		dataPortalSeries.MeasurementId = series.MeasurementId.Int64
	}
	if series.MeasurementName.Valid {
		dataPortalSeries.MeasurementName = series.MeasurementName.String
	}
	if series.Decimals.Valid {
		dataPortalSeries.Decimals = &series.Decimals.Int64
	}
	if series.BaseYear.Valid {
		dataPortalSeries.Title = formatWithYear(dataPortalSeries.Title, series.BaseYear.Int64)
		dataPortalSeries.Description = formatWithYear(dataPortalSeries.Description, series.BaseYear.Int64)
		dataPortalSeries.UnitsLabel = formatWithYear(dataPortalSeries.UnitsLabel, series.BaseYear.Int64)
		dataPortalSeries.UnitsLabelShort = formatWithYear(dataPortalSeries.UnitsLabelShort, series.BaseYear.Int64)
		dataPortalSeries.BaseYear = &series.BaseYear.Int64
		dataPortalSeries.BaseYearDeprecated = &series.BaseYear.Int64
	}
	if series.Indent.Valid {
		dataPortalSeries.Indent = indentationLevel[series.Indent.String]
	}
	dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
	if geography.FIPS.Valid {
		dataPortalGeography.FIPS = geography.FIPS.String
	}
	if geography.Name.Valid {
		dataPortalGeography.Name = geography.Name.String
	}
	if geography.ShortName.Valid {
		dataPortalGeography.ShortName = geography.ShortName.String
	}
	dataPortalSeries.Geography = dataPortalGeography
	return
}

func getAllFreqsGeos(r *SeriesRepository, seriesId int64) (
	[]models.DataPortalGeography,
	[]models.DataPortalFrequency,
	error,
) {
	rows, err := r.DB.Query(
		`SELECT DISTINCT 'geo' AS gftype,
			ANY_VALUE(geo.handle) AS handle,
			ANY_VALUE(geo.fips) AS fips,
			ANY_VALUE(geo.display_name) AS display_name,
			ANY_VALUE(geo.display_name_short) AS display_name_short,
			MIN(pdp.date), MAX(pdp.date)
		FROM measurement_series
		LEFT JOIN measurement_series AS ms ON ms.measurement_id = measurement_series.measurement_id
		LEFT JOIN series ON series.id = ms.series_id
		LEFT JOIN geographies geo on geo.id = series.geography_id
		LEFT JOIN public_data_points pdp on pdp.series_id = series.id
		WHERE pdp.value IS NOT NULL
		AND measurement_series.series_id = ?
		GROUP BY geo.id
		   UNION
		SELECT DISTINCT 'freq' AS gftype,
			RIGHT(series.name, 1) AS handle, null, null, null, MIN(pdp.date), MAX(pdp.date)
		FROM measurement_series
		LEFT JOIN measurement_series AS ms ON ms.measurement_id = measurement_series.measurement_id
		LEFT JOIN series ON series.id = ms.series_id
		LEFT JOIN public_data_points pdp on pdp.series_id = series.id
		WHERE pdp.value IS NOT NULL
		AND measurement_series.series_id = ?
		GROUP BY RIGHT(series.name, 1)
		ORDER BY 1,2 ;`, seriesId, seriesId)
	if err != nil {
		return nil, nil, err
	}
	geosResult := []models.DataPortalGeography{}
	freqsResult := []models.DataPortalFrequency{}
	for rows.Next() {
		var gftype sql.NullString
		temp := models.Geography{} // Using Geography object as a scan buffer, because it works.
		err = rows.Scan(
			&gftype,
			&temp.Handle,
			&temp.FIPS,
			&temp.Name,
			&temp.ShortName,
			&temp.ObservationStart,
			&temp.ObservationEnd,
		)
		if gftype.String == "geo" {
			g := models.DataPortalGeography{Handle: temp.Handle}
			if temp.FIPS.Valid {
				g.FIPS = temp.FIPS.String
			}
			if temp.Name.Valid {
				g.Name = temp.Name.String
			}
			if temp.ShortName.Valid {
				g.ShortName = temp.ShortName.String
			}
			if temp.ObservationStart.Valid {
				g.ObservationStart = &temp.ObservationStart.Time
			}
			if temp.ObservationEnd.Valid {
				g.ObservationEnd = &temp.ObservationEnd.Time
			}
			geosResult = append(geosResult, g)
		} else {
			f := models.DataPortalFrequency{
				Freq:  temp.Handle,
				Label: freqLabel[temp.Handle],
			}
			if temp.ObservationStart.Valid {
				f.ObservationStart = &temp.ObservationStart.Time
			}
			if temp.ObservationEnd.Valid {
				f.ObservationEnd = &temp.ObservationEnd.Time
			}
			freqsResult = append(freqsResult, f)
		}
	}
	return geosResult, freqsResult, err
}

func formatWithYear(formatString string, year int64) string {
	return strings.Replace(formatString, "%Y", strconv.FormatInt(year, 10), -1)
}

func rangeIntersection(start1 time.Time, end1 time.Time, start2 time.Time, end2 time.Time) (iStart *time.Time, iEnd *time.Time) {
	iStart = &start1
	iEnd = &end1
	if !rangesOverlap(start1, end1, start2, end2) {
		return nil, nil
	}
	if start2.After(start1) {
		iStart = &start2
	}
	if end2.Before(end1) {
		iEnd = &end2
	}
	return
}

func rangesOverlap(start1 time.Time, end1 time.Time, start2 time.Time, end2 time.Time) bool {
	return !(end1.Before(start2) || end2.Before(start1))
}
<commit_msg>Return Frequency object arrays sorted by frequency, not alpha (UA-850)<commit_after>package data

import (
	"database/sql"
	"github.com/UHERO/rest-api/models"
	"strconv"
	"strings"
	"time"
	"sort"
)

var freqLabel map[string]string = map[string]string{
	"A": "Annual",
	"S": "Semiannual",
	"Q": "Quarterly",
	"M": "Monthly",
	"W": "Weekly",
	"D": "Daily",
}

var freqDbNames map[string]string = map[string]string{
	"A": "year",
	"S": "semi",
	"Q": "quarter",
	"M": "month",
	"W": "week",
	"D": "day",
}

var indentationLevel map[string]int = map[string]int{
	"indent0": 0,
	"indent1": 1,
	"indent2": 2,
	"indent3": 3,
}

func getNextSeriesFromRows(rows *sql.Rows) (dataPortalSeries models.DataPortalSeries, err error) {
	series := models.Series{}
	geography := models.Geography{}
	err = rows.Scan(
		&series.Id,
		&series.Name,
		&series.Universe,
		&series.Description,
		&series.Frequency,
		&series.SeasonallyAdjusted,
		&series.SeasonalAdjustment,
		&series.UnitsLabel,
		&series.UnitsLabelShort,
		&series.DataPortalName,
		&series.Percent,
		&series.Real,
		&series.SourceDescription,
		&series.SourceLink,
		&series.SourceDetails,
		&series.TablePrefix,
		&series.TablePostfix,
		&series.MeasurementId,
		&series.MeasurementName,
		&series.Indent,
		&series.BaseYear,
		&series.Decimals,
		&geography.FIPS,
		&geography.Handle,
		&geography.Name,
		&geography.ShortName,
	)
	if err != nil {
		return
	}
	dataPortalSeries = models.DataPortalSeries{
		Id:       series.Id,
		Name:     series.Name,
		Universe: series.Universe,
	}
	dataPortalSeries.FrequencyShort = dataPortalSeries.Name[len(dataPortalSeries.Name)-1:]
	dataPortalSeries.Frequency = freqLabel[dataPortalSeries.FrequencyShort]
	if series.DataPortalName.Valid {
		dataPortalSeries.Title = series.DataPortalName.String
	}
	if series.Description.Valid {
		dataPortalSeries.Description = series.Description.String
	}
	if series.SeasonallyAdjusted.Valid && dataPortalSeries.FrequencyShort != "A" {
		dataPortalSeries.SeasonallyAdjusted = &series.SeasonallyAdjusted.Bool
	}
	if series.SeasonalAdjustment.Valid {
		dataPortalSeries.SeasonalAdjustment = series.SeasonalAdjustment.String
	}
	if series.UnitsLabel.Valid {
		dataPortalSeries.UnitsLabel = series.UnitsLabel.String
	}
	if series.UnitsLabelShort.Valid {
		dataPortalSeries.UnitsLabelShort = series.UnitsLabelShort.String
	}
	if series.Percent.Valid {
		dataPortalSeries.Percent = &series.Percent.Bool
	}
	if series.Real.Valid {
		dataPortalSeries.Real = &series.Real.Bool
	}
	if series.SourceDescription.Valid {
		dataPortalSeries.SourceDescription = series.SourceDescription.String
		dataPortalSeries.SourceDescriptionDeprecated = series.SourceDescription.String
	}
	if series.SourceLink.Valid {
		dataPortalSeries.SourceLink = series.SourceLink.String
		dataPortalSeries.SourceLinkDeprecated = series.SourceLink.String
	}
	if series.SourceDetails.Valid {
		dataPortalSeries.SourceDetails = series.SourceDetails.String
	}
	if series.TablePrefix.Valid {
		dataPortalSeries.TablePrefix = series.TablePrefix.String
	}
	if series.TablePostfix.Valid {
		dataPortalSeries.TablePostfix = series.TablePostfix.String
	}
	if series.MeasurementId.Valid {
		dataPortalSeries.MeasurementId = series.MeasurementId.Int64
	}
	if series.MeasurementName.Valid {
		dataPortalSeries.MeasurementName = series.MeasurementName.String
	}
	if series.Decimals.Valid {
		dataPortalSeries.Decimals = &series.Decimals.Int64
	}
	if series.BaseYear.Valid {
		dataPortalSeries.Title = formatWithYear(dataPortalSeries.Title, series.BaseYear.Int64)
		dataPortalSeries.Description = formatWithYear(dataPortalSeries.Description, series.BaseYear.Int64)
		dataPortalSeries.UnitsLabel = formatWithYear(dataPortalSeries.UnitsLabel, series.BaseYear.Int64)
		dataPortalSeries.UnitsLabelShort = formatWithYear(dataPortalSeries.UnitsLabelShort, series.BaseYear.Int64)
		dataPortalSeries.BaseYear = &series.BaseYear.Int64
		dataPortalSeries.BaseYearDeprecated = &series.BaseYear.Int64
	}
	if series.Indent.Valid {
		dataPortalSeries.Indent = indentationLevel[series.Indent.String]
	}
	dataPortalGeography := models.DataPortalGeography{Handle: geography.Handle}
	if geography.FIPS.Valid {
		dataPortalGeography.FIPS = geography.FIPS.String
	}
	if geography.Name.Valid {
		dataPortalGeography.Name = geography.Name.String
	}
	if geography.ShortName.Valid {
		dataPortalGeography.ShortName = geography.ShortName.String
	}
	dataPortalSeries.Geography = dataPortalGeography
	return
}

func getAllFreqsGeos(r *SeriesRepository, seriesId int64) (
	[]models.DataPortalGeography,
	[]models.DataPortalFrequency,
	error,
) {
	rows, err := r.DB.Query(
		`SELECT DISTINCT 'geo' AS gftype,
			ANY_VALUE(geo.handle) AS handle,
			ANY_VALUE(geo.fips) AS fips,
			ANY_VALUE(geo.display_name) AS display_name,
			ANY_VALUE(geo.display_name_short) AS display_name_short,
			MIN(pdp.date), MAX(pdp.date)
		FROM measurement_series
		LEFT JOIN measurement_series AS ms ON ms.measurement_id = measurement_series.measurement_id
		LEFT JOIN series ON series.id = ms.series_id
		LEFT JOIN geographies geo on geo.id = series.geography_id
		LEFT JOIN public_data_points pdp on pdp.series_id = series.id
		WHERE pdp.value IS NOT NULL
		AND measurement_series.series_id = ?
		GROUP BY geo.id
		   UNION
		SELECT DISTINCT 'freq' AS gftype,
			RIGHT(series.name, 1) AS handle, null, null, null, MIN(pdp.date), MAX(pdp.date)
		FROM measurement_series
		LEFT JOIN measurement_series AS ms ON ms.measurement_id = measurement_series.measurement_id
		LEFT JOIN series ON series.id = ms.series_id
		LEFT JOIN public_data_points pdp on pdp.series_id = series.id
		WHERE pdp.value IS NOT NULL
		AND measurement_series.series_id = ?
		GROUP BY RIGHT(series.name, 1)
		ORDER BY 1,2 ;`, seriesId, seriesId)
	if err != nil {
		return nil, nil, err
	}
	geosResult := []models.DataPortalGeography{}
	freqsResult := []models.DataPortalFrequency{}
	for rows.Next() {
		var gftype sql.NullString
		temp := models.Geography{} // Using Geography object as a scan buffer, because it works.
		err = rows.Scan(
			&gftype,
			&temp.Handle,
			&temp.FIPS,
			&temp.Name,
			&temp.ShortName,
			&temp.ObservationStart,
			&temp.ObservationEnd,
		)
		if gftype.String == "geo" {
			g := models.DataPortalGeography{Handle: temp.Handle}
			if temp.FIPS.Valid {
				g.FIPS = temp.FIPS.String
			}
			if temp.Name.Valid {
				g.Name = temp.Name.String
			}
			if temp.ShortName.Valid {
				g.ShortName = temp.ShortName.String
			}
			if temp.ObservationStart.Valid {
				g.ObservationStart = &temp.ObservationStart.Time
			}
			if temp.ObservationEnd.Valid {
				g.ObservationEnd = &temp.ObservationEnd.Time
			}
			geosResult = append(geosResult, g)
		} else {
			f := models.DataPortalFrequency{
				Freq:  temp.Handle,
				Label: freqLabel[temp.Handle],
			}
			if temp.ObservationStart.Valid {
				f.ObservationStart = &temp.ObservationStart.Time
			}
			if temp.ObservationEnd.Valid {
				f.ObservationEnd = &temp.ObservationEnd.Time
			}
			freqsResult = append(freqsResult, f)
		}
	}
	sort.Sort(models.ByGeography(geosResult))
	sort.Sort(models.ByFrequency(freqsResult))
	return geosResult, freqsResult, err
}

func formatWithYear(formatString string, year int64) string {
	return strings.Replace(formatString, "%Y", strconv.FormatInt(year, 10), -1)
}

func rangeIntersection(start1 time.Time, end1 time.Time, start2 time.Time, end2 time.Time) (iStart *time.Time, iEnd *time.Time) {
	iStart = &start1
	iEnd = &end1
	if !rangesOverlap(start1, end1, start2, end2) {
		return nil, nil
	}
	if start2.After(start1) {
		iStart = &start2
	}
	if end2.Before(end1) {
		iEnd = &end2
	}
	return
}

func rangesOverlap(start1 time.Time, end1 time.Time, start2 time.Time, end2 time.Time) bool {
	return !(end1.Before(start2) || end2.Before(start1))
}
<|endoftext|>


<commit_before>package help

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
	"time"
	"unicode/utf8"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/command_registry"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/terminal"
)

type appPresenter struct {
	Name     string
	Usage    string
	Version  string
	Compiled time.Time
	Commands []groupedCommands
}

type groupedCommands struct {
	Name             string
	CommandSubGroups [][]cmdPresenter
}

type cmdPresenter struct {
	Name        string
	Description string
}

func ShowHelp(helpTemplate string) {
	translatedTemplatedHelp := T(strings.Replace(helpTemplate, "{{", "[[", -1))
	translatedTemplatedHelp = strings.Replace(translatedTemplatedHelp, "[[", "{{", -1)

	showAppHelp(translatedTemplatedHelp)
}

func showAppHelp(helpTemplate string) {
	presenter := newAppPresenter()

	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Parse(helpTemplate))
	err := t.Execute(w, presenter)
	if err != nil {
		fmt.Println("error", err)
	}
	w.Flush()
}

func newAppPresenter() (presenter appPresenter) {
	maxNameLen := command_registry.Commands.MaxCommandNameLength()

	presentNonCodegangstaCommand := func(commandName string) (presenter cmdPresenter) {
		cmd := command_registry.Commands.FindCommand(commandName)
		presenter.Name = cmd.MetaData().Name
		padding := strings.Repeat(" ", maxNameLen-utf8.RuneCountInString(presenter.Name))
		presenter.Name = presenter.Name + padding
		presenter.Description = cmd.MetaData().Description
		return
	}

	presentPluginCommands := func() []cmdPresenter {
		pluginConfig := plugin_config.NewPluginConfig(func(err error) {
			//fail silently when running help?
		})

		plugins := pluginConfig.Plugins()
		var presenters []cmdPresenter
		var pluginPresenter cmdPresenter

		for _, pluginMetadata := range plugins {
			for _, cmd := range pluginMetadata.Commands {

				if cmd.Alias == "" {
					pluginPresenter.Name = cmd.Name
				} else {
					pluginPresenter.Name = cmd.Name + ", " + cmd.Alias
				}

				padding := strings.Repeat(" ", maxNameLen-utf8.RuneCountInString(pluginPresenter.Name))
				pluginPresenter.Name = pluginPresenter.Name + padding
				pluginPresenter.Description = cmd.HelpText
				presenters = append(presenters, pluginPresenter)
			}
		}

		return presenters
	}

	presenter.Name = os.Args[0]
	presenter.Usage = T("A command line tool to interact with Cloud Foundry")
	presenter.Version = cf.Version + "-" + cf.BuiltOnDate
	compiledAtTime, err := time.Parse("2006-01-02T03:04:05+00:00", cf.BuiltOnDate)
	if err == nil {
		presenter.Compiled = compiledAtTime
	} else {
		presenter.Compiled = time.Now()
	}
	presenter.Commands = []groupedCommands{
		{
			Name: T("GETTING STARTED"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("help"),
					presentNonCodegangstaCommand("login"),
					presentNonCodegangstaCommand("logout"),
					presentNonCodegangstaCommand("passwd"),
					presentNonCodegangstaCommand("target"),
				}, {
					presentNonCodegangstaCommand("api"),
					presentNonCodegangstaCommand("auth"),
				},
			},
		}, {
			Name: T("APPS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("apps"),
					presentNonCodegangstaCommand("app"),
				}, {
					presentNonCodegangstaCommand("push"),
					presentNonCodegangstaCommand("scale"),
					presentNonCodegangstaCommand("delete"),
					presentNonCodegangstaCommand("rename"),
				}, {
					presentNonCodegangstaCommand("start"),
					presentNonCodegangstaCommand("stop"),
					presentNonCodegangstaCommand("restart"),
					presentNonCodegangstaCommand("restage"),
					presentNonCodegangstaCommand("restart-app-instance"),
				}, {
					presentNonCodegangstaCommand("events"),
					presentNonCodegangstaCommand("files"),
					presentNonCodegangstaCommand("logs"),
				}, {
					presentNonCodegangstaCommand("env"),
					presentNonCodegangstaCommand("set-env"),
					presentNonCodegangstaCommand("unset-env"),
				}, {
					presentNonCodegangstaCommand("stacks"),
					presentNonCodegangstaCommand("stack"),
				}, {
					presentNonCodegangstaCommand("copy-source"),
				}, {
					presentNonCodegangstaCommand("create-app-manifest"),
				},
			},
		}, {
			Name: T("SERVICES"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("marketplace"),
					presentNonCodegangstaCommand("services"),
					presentNonCodegangstaCommand("service"),
				}, {
					presentNonCodegangstaCommand("create-service"),
					presentNonCodegangstaCommand("update-service"),
					presentNonCodegangstaCommand("delete-service"),
					presentNonCodegangstaCommand("rename-service"),
				}, {
					presentNonCodegangstaCommand("create-service-key"),
					presentNonCodegangstaCommand("service-keys"),
					presentNonCodegangstaCommand("service-key"),
					presentNonCodegangstaCommand("delete-service-key"),
				}, {
					presentNonCodegangstaCommand("bind-service"),
					presentNonCodegangstaCommand("unbind-service"),
				}, {
					presentNonCodegangstaCommand("create-user-provided-service"),
					presentNonCodegangstaCommand("update-user-provided-service"),
				},
			},
		}, {
			Name: T("ORGS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("orgs"),
					presentNonCodegangstaCommand("org"),
				}, {
					presentNonCodegangstaCommand("create-org"),
					presentNonCodegangstaCommand("delete-org"),
					presentNonCodegangstaCommand("rename-org"),
				},
			},
		}, {
			Name: T("SPACES"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("spaces"),
					presentNonCodegangstaCommand("space"),
				}, {
					presentNonCodegangstaCommand("create-space"),
					presentNonCodegangstaCommand("delete-space"),
					presentNonCodegangstaCommand("rename-space"),
				},
			},
		}, {
			Name: T("DOMAINS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("domains"),
					presentNonCodegangstaCommand("create-domain"),
					presentNonCodegangstaCommand("delete-domain"),
					presentNonCodegangstaCommand("create-shared-domain"),
					presentNonCodegangstaCommand("delete-shared-domain"),
				},
			},
		}, {
			Name: T("ROUTES"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("routes"),
					presentNonCodegangstaCommand("create-route"),
					presentNonCodegangstaCommand("check-route"),
					presentNonCodegangstaCommand("map-route"),
					presentNonCodegangstaCommand("unmap-route"),
					presentNonCodegangstaCommand("delete-route"),
					presentNonCodegangstaCommand("delete-orphaned-routes"),
				},
			},
		}, {
			Name: T("BUILDPACKS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("buildpacks"),
					presentNonCodegangstaCommand("create-buildpack"),
					presentNonCodegangstaCommand("update-buildpack"),
					presentNonCodegangstaCommand("rename-buildpack"),
					presentNonCodegangstaCommand("delete-buildpack"),
				},
			},
		}, {
			Name: T("USER ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("create-user"),
					presentNonCodegangstaCommand("delete-user"),
				}, {
					presentNonCodegangstaCommand("org-users"),
					presentNonCodegangstaCommand("set-org-role"),
					presentNonCodegangstaCommand("unset-org-role"),
				}, {
					presentNonCodegangstaCommand("space-users"),
					presentNonCodegangstaCommand("set-space-role"),
					presentNonCodegangstaCommand("unset-space-role"),
				},
			},
		}, {
			Name: T("ORG ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("quotas"),
					presentNonCodegangstaCommand("quota"),
					presentNonCodegangstaCommand("set-quota"),
				}, {
					presentNonCodegangstaCommand("create-quota"),
					presentNonCodegangstaCommand("delete-quota"),
					presentNonCodegangstaCommand("update-quota"),
				},
				{
					presentNonCodegangstaCommand("share-private-domain"),
					presentNonCodegangstaCommand("unshare-private-domain"),
				},
			},
		}, {
			Name: T("SPACE ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("space-quotas"),
					presentNonCodegangstaCommand("space-quota"),
					presentNonCodegangstaCommand("create-space-quota"),
					presentNonCodegangstaCommand("update-space-quota"),
					presentNonCodegangstaCommand("delete-space-quota"),
					presentNonCodegangstaCommand("set-space-quota"),
					presentNonCodegangstaCommand("unset-space-quota"),
				},
			},
		}, {
			Name: T("SERVICE ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("service-auth-tokens"),
					presentNonCodegangstaCommand("create-service-auth-token"),
					presentNonCodegangstaCommand("update-service-auth-token"),
					presentNonCodegangstaCommand("delete-service-auth-token"),
				}, {
					presentNonCodegangstaCommand("service-brokers"),
					presentNonCodegangstaCommand("create-service-broker"),
					presentNonCodegangstaCommand("update-service-broker"),
					presentNonCodegangstaCommand("delete-service-broker"),
					presentNonCodegangstaCommand("rename-service-broker"),
				}, {
					presentNonCodegangstaCommand("migrate-service-instances"),
					presentNonCodegangstaCommand("purge-service-offering"),
				}, {
					presentNonCodegangstaCommand("service-access"),
					presentNonCodegangstaCommand("enable-service-access"),
					presentNonCodegangstaCommand("disable-service-access"),
				},
			},
		}, {
			Name: T("SECURITY GROUP"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("security-group"),
					presentNonCodegangstaCommand("security-groups"),
					presentNonCodegangstaCommand("create-security-group"),
					presentNonCodegangstaCommand("update-security-group"),
					presentNonCodegangstaCommand("delete-security-group"),
					presentNonCodegangstaCommand("bind-security-group"),
					presentNonCodegangstaCommand("unbind-security-group"),
				}, {
					presentNonCodegangstaCommand("bind-staging-security-group"),
					presentNonCodegangstaCommand("staging-security-groups"),
					presentNonCodegangstaCommand("unbind-staging-security-group"),
				}, {
					presentNonCodegangstaCommand("bind-running-security-group"),
					presentNonCodegangstaCommand("running-security-groups"),
					presentNonCodegangstaCommand("unbind-running-security-group"),
				},
			},
		}, {
			Name: T("ENVIRONMENT VARIABLE GROUPS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("running-environment-variable-group"),
					presentNonCodegangstaCommand("staging-environment-variable-group"),
					presentNonCodegangstaCommand("set-staging-environment-variable-group"),
					presentNonCodegangstaCommand("set-running-environment-variable-group"),
				},
			},
		},
		{
			Name: T("FEATURE FLAGS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("feature-flags"),
					presentNonCodegangstaCommand("feature-flag"),
					presentNonCodegangstaCommand("enable-feature-flag"),
					presentNonCodegangstaCommand("disable-feature-flag"),
				},
			},
		}, {
			Name: T("ADVANCED"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("curl"),
					presentNonCodegangstaCommand("config"),
					presentNonCodegangstaCommand("oauth-token"),
				},
			},
		}, {
			Name: T("ADD/REMOVE PLUGIN REPOSITORY"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("add-plugin-repo"),
					presentNonCodegangstaCommand("remove-plugin-repo"),
					presentNonCodegangstaCommand("list-plugin-repos"),
					presentNonCodegangstaCommand("repo-plugins"),
				},
			},
		}, {
			Name: T("ADD/REMOVE PLUGIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("plugins"),
					presentNonCodegangstaCommand("install-plugin"),
					presentNonCodegangstaCommand("uninstall-plugin"),
				},
			},
		}, {
			Name: T("INSTALLED PLUGIN COMMANDS"),
			CommandSubGroups: [][]cmdPresenter{
				presentPluginCommands(),
			},
		},
	}

	return
}

func (p appPresenter) Title(name string) string {
	return terminal.HeaderColor(name)
}

func (c groupedCommands) SubTitle(name string) string {
	return terminal.HeaderColor(name + ":")
}
<commit_msg>help test for get-health-check and set-health-check<commit_after>package help

import (
	"fmt"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
	"time"
	"unicode/utf8"

	"github.com/cloudfoundry/cli/cf"
	"github.com/cloudfoundry/cli/cf/command_registry"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/terminal"
)

type appPresenter struct {
	Name     string
	Usage    string
	Version  string
	Compiled time.Time
	Commands []groupedCommands
}

type groupedCommands struct {
	Name             string
	CommandSubGroups [][]cmdPresenter
}

type cmdPresenter struct {
	Name        string
	Description string
}

func ShowHelp(helpTemplate string) {
	translatedTemplatedHelp := T(strings.Replace(helpTemplate, "{{", "[[", -1))
	translatedTemplatedHelp = strings.Replace(translatedTemplatedHelp, "[[", "{{", -1)

	showAppHelp(translatedTemplatedHelp)
}

func showAppHelp(helpTemplate string) {
	presenter := newAppPresenter()

	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Parse(helpTemplate))
	err := t.Execute(w, presenter)
	if err != nil {
		fmt.Println("error", err)
	}
	w.Flush()
}

func newAppPresenter() (presenter appPresenter) {
	maxNameLen := command_registry.Commands.MaxCommandNameLength()

	presentNonCodegangstaCommand := func(commandName string) (presenter cmdPresenter) {
		cmd := command_registry.Commands.FindCommand(commandName)
		presenter.Name = cmd.MetaData().Name
		padding := strings.Repeat(" ", maxNameLen-utf8.RuneCountInString(presenter.Name))
		presenter.Name = presenter.Name + padding
		presenter.Description = cmd.MetaData().Description
		return
	}

	presentPluginCommands := func() []cmdPresenter {
		pluginConfig := plugin_config.NewPluginConfig(func(err error) {
			//fail silently when running help?
		})

		plugins := pluginConfig.Plugins()
		var presenters []cmdPresenter
		var pluginPresenter cmdPresenter

		for _, pluginMetadata := range plugins {
			for _, cmd := range pluginMetadata.Commands {

				if cmd.Alias == "" {
					pluginPresenter.Name = cmd.Name
				} else {
					pluginPresenter.Name = cmd.Name + ", " + cmd.Alias
				}

				padding := strings.Repeat(" ", maxNameLen-utf8.RuneCountInString(pluginPresenter.Name))
				pluginPresenter.Name = pluginPresenter.Name + padding
				pluginPresenter.Description = cmd.HelpText
				presenters = append(presenters, pluginPresenter)
			}
		}

		return presenters
	}

	presenter.Name = os.Args[0]
	presenter.Usage = T("A command line tool to interact with Cloud Foundry")
	presenter.Version = cf.Version + "-" + cf.BuiltOnDate
	compiledAtTime, err := time.Parse("2006-01-02T03:04:05+00:00", cf.BuiltOnDate)
	if err == nil {
		presenter.Compiled = compiledAtTime
	} else {
		presenter.Compiled = time.Now()
	}
	presenter.Commands = []groupedCommands{
		{
			Name: T("GETTING STARTED"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("help"),
					presentNonCodegangstaCommand("login"),
					presentNonCodegangstaCommand("logout"),
					presentNonCodegangstaCommand("passwd"),
					presentNonCodegangstaCommand("target"),
				}, {
					presentNonCodegangstaCommand("api"),
					presentNonCodegangstaCommand("auth"),
				},
			},
		}, {
			Name: T("APPS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("apps"),
					presentNonCodegangstaCommand("app"),
				}, {
					presentNonCodegangstaCommand("push"),
					presentNonCodegangstaCommand("scale"),
					presentNonCodegangstaCommand("delete"),
					presentNonCodegangstaCommand("rename"),
				}, {
					presentNonCodegangstaCommand("start"),
					presentNonCodegangstaCommand("stop"),
					presentNonCodegangstaCommand("restart"),
					presentNonCodegangstaCommand("restage"),
					presentNonCodegangstaCommand("restart-app-instance"),
				}, {
					presentNonCodegangstaCommand("events"),
					presentNonCodegangstaCommand("files"),
					presentNonCodegangstaCommand("logs"),
				}, {
					presentNonCodegangstaCommand("env"),
					presentNonCodegangstaCommand("set-env"),
					presentNonCodegangstaCommand("unset-env"),
				}, {
					presentNonCodegangstaCommand("stacks"),
					presentNonCodegangstaCommand("stack"),
				}, {
					presentNonCodegangstaCommand("copy-source"),
				}, {
					presentNonCodegangstaCommand("create-app-manifest"),
				}, {
					presentNonCodegangstaCommand("get-health-check"),
					presentNonCodegangstaCommand("set-health-check"),
				},
			},
		}, {
			Name: T("SERVICES"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("marketplace"),
					presentNonCodegangstaCommand("services"),
					presentNonCodegangstaCommand("service"),
				}, {
					presentNonCodegangstaCommand("create-service"),
					presentNonCodegangstaCommand("update-service"),
					presentNonCodegangstaCommand("delete-service"),
					presentNonCodegangstaCommand("rename-service"),
				}, {
					presentNonCodegangstaCommand("create-service-key"),
					presentNonCodegangstaCommand("service-keys"),
					presentNonCodegangstaCommand("service-key"),
					presentNonCodegangstaCommand("delete-service-key"),
				}, {
					presentNonCodegangstaCommand("bind-service"),
					presentNonCodegangstaCommand("unbind-service"),
				}, {
					presentNonCodegangstaCommand("create-user-provided-service"),
					presentNonCodegangstaCommand("update-user-provided-service"),
				},
			},
		}, {
			Name: T("ORGS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("orgs"),
					presentNonCodegangstaCommand("org"),
				}, {
					presentNonCodegangstaCommand("create-org"),
					presentNonCodegangstaCommand("delete-org"),
					presentNonCodegangstaCommand("rename-org"),
				},
			},
		}, {
			Name: T("SPACES"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("spaces"),
					presentNonCodegangstaCommand("space"),
				}, {
					presentNonCodegangstaCommand("create-space"),
					presentNonCodegangstaCommand("delete-space"),
					presentNonCodegangstaCommand("rename-space"),
				},
			},
		}, {
			Name: T("DOMAINS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("domains"),
					presentNonCodegangstaCommand("create-domain"),
					presentNonCodegangstaCommand("delete-domain"),
					presentNonCodegangstaCommand("create-shared-domain"),
					presentNonCodegangstaCommand("delete-shared-domain"),
				},
			},
		}, {
			Name: T("ROUTES"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("routes"),
					presentNonCodegangstaCommand("create-route"),
					presentNonCodegangstaCommand("check-route"),
					presentNonCodegangstaCommand("map-route"),
					presentNonCodegangstaCommand("unmap-route"),
					presentNonCodegangstaCommand("delete-route"),
					presentNonCodegangstaCommand("delete-orphaned-routes"),
				},
			},
		}, {
			Name: T("BUILDPACKS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("buildpacks"),
					presentNonCodegangstaCommand("create-buildpack"),
					presentNonCodegangstaCommand("update-buildpack"),
					presentNonCodegangstaCommand("rename-buildpack"),
					presentNonCodegangstaCommand("delete-buildpack"),
				},
			},
		}, {
			Name: T("USER ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("create-user"),
					presentNonCodegangstaCommand("delete-user"),
				}, {
					presentNonCodegangstaCommand("org-users"),
					presentNonCodegangstaCommand("set-org-role"),
					presentNonCodegangstaCommand("unset-org-role"),
				}, {
					presentNonCodegangstaCommand("space-users"),
					presentNonCodegangstaCommand("set-space-role"),
					presentNonCodegangstaCommand("unset-space-role"),
				},
			},
		}, {
			Name: T("ORG ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("quotas"),
					presentNonCodegangstaCommand("quota"),
					presentNonCodegangstaCommand("set-quota"),
				}, {
					presentNonCodegangstaCommand("create-quota"),
					presentNonCodegangstaCommand("delete-quota"),
					presentNonCodegangstaCommand("update-quota"),
				},
				{
					presentNonCodegangstaCommand("share-private-domain"),
					presentNonCodegangstaCommand("unshare-private-domain"),
				},
			},
		}, {
			Name: T("SPACE ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("space-quotas"),
					presentNonCodegangstaCommand("space-quota"),
					presentNonCodegangstaCommand("create-space-quota"),
					presentNonCodegangstaCommand("update-space-quota"),
					presentNonCodegangstaCommand("delete-space-quota"),
					presentNonCodegangstaCommand("set-space-quota"),
					presentNonCodegangstaCommand("unset-space-quota"),
				},
			},
		}, {
			Name: T("SERVICE ADMIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("service-auth-tokens"),
					presentNonCodegangstaCommand("create-service-auth-token"),
					presentNonCodegangstaCommand("update-service-auth-token"),
					presentNonCodegangstaCommand("delete-service-auth-token"),
				}, {
					presentNonCodegangstaCommand("service-brokers"),
					presentNonCodegangstaCommand("create-service-broker"),
					presentNonCodegangstaCommand("update-service-broker"),
					presentNonCodegangstaCommand("delete-service-broker"),
					presentNonCodegangstaCommand("rename-service-broker"),
				}, {
					presentNonCodegangstaCommand("migrate-service-instances"),
					presentNonCodegangstaCommand("purge-service-offering"),
				}, {
					presentNonCodegangstaCommand("service-access"),
					presentNonCodegangstaCommand("enable-service-access"),
					presentNonCodegangstaCommand("disable-service-access"),
				},
			},
		}, {
			Name: T("SECURITY GROUP"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("security-group"),
					presentNonCodegangstaCommand("security-groups"),
					presentNonCodegangstaCommand("create-security-group"),
					presentNonCodegangstaCommand("update-security-group"),
					presentNonCodegangstaCommand("delete-security-group"),
					presentNonCodegangstaCommand("bind-security-group"),
					presentNonCodegangstaCommand("unbind-security-group"),
				}, {
					presentNonCodegangstaCommand("bind-staging-security-group"),
					presentNonCodegangstaCommand("staging-security-groups"),
					presentNonCodegangstaCommand("unbind-staging-security-group"),
				}, {
					presentNonCodegangstaCommand("bind-running-security-group"),
					presentNonCodegangstaCommand("running-security-groups"),
					presentNonCodegangstaCommand("unbind-running-security-group"),
				},
			},
		}, {
			Name: T("ENVIRONMENT VARIABLE GROUPS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("running-environment-variable-group"),
					presentNonCodegangstaCommand("staging-environment-variable-group"),
					presentNonCodegangstaCommand("set-staging-environment-variable-group"),
					presentNonCodegangstaCommand("set-running-environment-variable-group"),
				},
			},
		},
		{
			Name: T("FEATURE FLAGS"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("feature-flags"),
					presentNonCodegangstaCommand("feature-flag"),
					presentNonCodegangstaCommand("enable-feature-flag"),
					presentNonCodegangstaCommand("disable-feature-flag"),
				},
			},
		}, {
			Name: T("ADVANCED"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("curl"),
					presentNonCodegangstaCommand("config"),
					presentNonCodegangstaCommand("oauth-token"),
				},
			},
		}, {
			Name: T("ADD/REMOVE PLUGIN REPOSITORY"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("add-plugin-repo"),
					presentNonCodegangstaCommand("remove-plugin-repo"),
					presentNonCodegangstaCommand("list-plugin-repos"),
					presentNonCodegangstaCommand("repo-plugins"),
				},
			},
		}, {
			Name: T("ADD/REMOVE PLUGIN"),
			CommandSubGroups: [][]cmdPresenter{
				{
					presentNonCodegangstaCommand("plugins"),
					presentNonCodegangstaCommand("install-plugin"),
					presentNonCodegangstaCommand("uninstall-plugin"),
				},
			},
		}, {
			Name: T("INSTALLED PLUGIN COMMANDS"),
			CommandSubGroups: [][]cmdPresenter{
				presentPluginCommands(),
			},
		},
	}

	return
}

func (p appPresenter) Title(name string) string {
	return terminal.HeaderColor(name)
}

func (c groupedCommands) SubTitle(name string) string {
	return terminal.HeaderColor(name + ":")
}
<|endoftext|>


<commit_before>package resourcepool

func (pool *Pool) getSlot(wait bool) bool {
	// Grab a slot (the right to have a resource in use).
	if wait {
		pool.semaphore <- struct{}{}
	} else {
		select {
		case pool.semaphore <- struct{}{}:
		default:
			return false
		}
	}
	return true
}

func (resource *Resource) get(wait bool) bool {
	pool := resource.pool
	if resource.inUse {
		panic("Resource is already in use")
	}
	if !pool.getSlot(wait) {
		return false
	}
	pool.lock.Lock()
	defer pool.lock.Unlock()
	if resource.allocated {
		delete(pool.unused, resource)
		resource.inUse = true
		pool.numUsed++
		return true
	}
	if pool.numUsed+uint(len(pool.unused)) >= pool.max {
		// Need to grab a free resource and release. Be lazy: do a random pick.
		var resourceToRelease *Resource
		for res := range pool.unused {
			resourceToRelease = res
			break
		}
		if resourceToRelease == nil {
			panic("No free resource to release")
		}
		if !resourceToRelease.allocated {
			panic("Resource is not allocated")
		}
		delete(pool.unused, resourceToRelease)
		resourceToRelease.releaseFunc()
		resourceToRelease.releaseFunc = nil
		resourceToRelease.allocated = false
	}
	resource.inUse = true
	resource.releaseFunc = nil
	resource.allocated = true
	pool.numUsed++
	return true
}

func (resource *Resource) put() {
	pool := resource.pool
	pool.lock.Lock()
	if !resource.allocated {
		pool.lock.Unlock()
		return
	}
	if !resource.inUse {
		pool.lock.Unlock()
		panic("Resource was not gotten")
	}
	resource.inUse = false
	if resource.releaseOnPut {
		resource.releaseFunc()
		resource.releaseFunc = nil
		resource.allocated = false
	} else {
		pool.unused[resource] = struct{}{}
	}
	pool.numUsed--
	pool.lock.Unlock()
	<-pool.semaphore // Free up a slot for someone else.
}

func (resource *Resource) release(haveLock bool) {
	pool := resource.pool
	if !haveLock {
		pool.lock.Lock()
	}
	if !resource.allocated {
		pool.lock.Unlock()
		return
	}
	resource.releaseFunc()
	resource.releaseFunc = nil
	resource.allocated = false
	delete(resource.pool.unused, resource)
	wasUsed := resource.inUse
	if resource.inUse {
		resource.inUse = false
		pool.numUsed--
	}
	pool.lock.Unlock()
	if wasUsed {
		<-pool.semaphore // Free up a slot for someone else.
	}
}

func (resource *Resource) setReleaseFunc(releaseFunc func()) {
	if releaseFunc == nil {
		panic("Cannot set nil releaseFunc")
	}
	resource.pool.lock.Lock()
	defer resource.pool.lock.Unlock()
	if !resource.inUse {
		panic("Resource was not gotten")
	}
	if resource.releaseFunc != nil {
		panic("Cannot change releaseFunc once set")
	}
	resource.releaseFunc = releaseFunc
}

func (resource *Resource) scheduleRelease() {
	resource.pool.lock.Lock()
	if resource.inUse {
		resource.releaseOnPut = true
		resource.pool.lock.Unlock()
		return
	}
	resource.release(true)
}
<commit_msg>Fix bug in last commit: did not check for nil releaseFunc prior to calling.<commit_after>package resourcepool

func (pool *Pool) getSlot(wait bool) bool {
	// Grab a slot (the right to have a resource in use).
	if wait {
		pool.semaphore <- struct{}{}
	} else {
		select {
		case pool.semaphore <- struct{}{}:
		default:
			return false
		}
	}
	return true
}

func (resource *Resource) get(wait bool) bool {
	pool := resource.pool
	if resource.inUse {
		panic("Resource is already in use")
	}
	if !pool.getSlot(wait) {
		return false
	}
	pool.lock.Lock()
	defer pool.lock.Unlock()
	if resource.allocated {
		delete(pool.unused, resource)
		resource.inUse = true
		pool.numUsed++
		return true
	}
	if pool.numUsed+uint(len(pool.unused)) >= pool.max {
		// Need to grab a free resource and release. Be lazy: do a random pick.
		var resourceToRelease *Resource
		for res := range pool.unused {
			resourceToRelease = res
			break
		}
		if resourceToRelease == nil {
			panic("No free resource to release")
		}
		if !resourceToRelease.allocated {
			panic("Resource is not allocated")
		}
		delete(pool.unused, resourceToRelease)
		if resourceToRelease.releaseFunc != nil {
			resourceToRelease.releaseFunc()
			resourceToRelease.releaseFunc = nil
		}
		resourceToRelease.allocated = false
	}
	resource.inUse = true
	resource.releaseFunc = nil
	resource.allocated = true
	pool.numUsed++
	return true
}

func (resource *Resource) put() {
	pool := resource.pool
	pool.lock.Lock()
	if !resource.allocated {
		pool.lock.Unlock()
		return
	}
	if !resource.inUse {
		pool.lock.Unlock()
		panic("Resource was not gotten")
	}
	resource.inUse = false
	if resource.releaseOnPut {
		if resource.releaseFunc != nil {
			resource.releaseFunc()
			resource.releaseFunc = nil
		}
		resource.allocated = false
	} else {
		pool.unused[resource] = struct{}{}
	}
	pool.numUsed--
	pool.lock.Unlock()
	<-pool.semaphore // Free up a slot for someone else.
}

func (resource *Resource) release(haveLock bool) {
	pool := resource.pool
	if !haveLock {
		pool.lock.Lock()
	}
	if !resource.allocated {
		pool.lock.Unlock()
		return
	}
	if resource.releaseFunc != nil {
		resource.releaseFunc()
		resource.releaseFunc = nil
	}
	resource.allocated = false
	delete(resource.pool.unused, resource)
	wasUsed := resource.inUse
	if resource.inUse {
		resource.inUse = false
		pool.numUsed--
	}
	pool.lock.Unlock()
	if wasUsed {
		<-pool.semaphore // Free up a slot for someone else.
	}
}

func (resource *Resource) setReleaseFunc(releaseFunc func()) {
	if releaseFunc == nil {
		panic("Cannot set nil releaseFunc")
	}
	resource.pool.lock.Lock()
	defer resource.pool.lock.Unlock()
	if !resource.inUse {
		panic("Resource was not gotten")
	}
	if resource.releaseFunc != nil {
		panic("Cannot change releaseFunc once set")
	}
	resource.releaseFunc = releaseFunc
}

func (resource *Resource) scheduleRelease() {
	resource.pool.lock.Lock()
	if resource.inUse {
		resource.releaseOnPut = true
		resource.pool.lock.Unlock()
		return
	}
	resource.release(true)
}
<|endoftext|>


<commit_before>package libcomfo

import (
	"reflect"
	"testing"
)

func TestTemperature_MarshalBinary(t *testing.T) {
	tests := []struct {
		name string
		temp temperature
		b    byte
		err  error
	}{
		{
			name: "average value",
			temp: temperature(25),
			b:    90,
		},
		{
			name: "high value",
			temp: temperature(107),
			b:    254,
		},
		{
			name: "low value",
			temp: temperature(-20),
			b:    0,
		},
		{
			name: "temperature too low",
			temp: temperature(-21),
			err:  errTooLow,
		},
		{
			name: "temperature too high",
			temp: temperature(108),
			err:  errTooHigh,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			// Marshal value into binary representation
			b, err := tt.temp.MarshalBinary()

			if want, got := tt.err, err; want != got {
				t.Fatalf("unexpected error marshaling temperature:\n- want: %v\n-  got: %v",
					want, got)
			}

			if want, got := tt.b, b; !reflect.DeepEqual(want, got) {
				t.Fatalf("unexpected temperature marshal:\n- want: %v\n-  got: %v",
					want, got)
			}
		})
	}
}

func TestTemperature_UnmarshalBinary(t *testing.T) {
	tests := []struct {
		name string
		temp temperature
		b    byte
		err  error
	}{
		{
			name: "average value",
			b:    90,
			temp: temperature(25),
		},
		{
			name: "high value",
			b:    255,
			temp: temperature(107),
		},
		{
			name: "low value",
			b:    0,
			temp: temperature(-20),
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			// Unmarshal binary representation into temperature value
			var temp temperature
			temp.UnmarshalBinary(tt.b)

			if want, got := tt.temp, temp; !reflect.DeepEqual(want, got) {
				t.Fatalf("unexpected temperature unmarshal:\n- want: %v\n-  got: %v",
					want, got)
			}
		})
	}
}
<commit_msg>libcomfo - mockSetReq: mock implementation of setRequest for testing<commit_after>package libcomfo

import (
	"bytes"
	"errors"
	"fmt"
	"reflect"
	"testing"
)

// mockSetReq implements the SetRequest interface
// and is used to raise errors during testing.
type mockSetReq struct {
	mockType setRequest
	mockData []byte
	mockErr  error
}

func (sr mockSetReq) Type() setRequest { return sr.mockType }
func (sr mockSetReq) MarshalBinary() (out []byte, err error) {
	return sr.mockData, sr.mockErr
}

func TestTemperature_MarshalBinary(t *testing.T) {
	tests := []struct {
		name string
		temp temperature
		b    byte
		err  error
	}{
		{
			name: "average value",
			temp: temperature(25),
			b:    90,
		},
		{
			name: "high value",
			temp: temperature(107),
			b:    254,
		},
		{
			name: "low value",
			temp: temperature(-20),
			b:    0,
		},
		{
			name: "temperature too low",
			temp: temperature(-21),
			err:  errTooLow,
		},
		{
			name: "temperature too high",
			temp: temperature(108),
			err:  errTooHigh,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			// Marshal value into binary representation
			b, err := tt.temp.MarshalBinary()

			if want, got := tt.err, err; want != got {
				t.Fatalf("unexpected error marshaling temperature:\n- want: %v\n-  got: %v",
					want, got)
			}

			if want, got := tt.b, b; !reflect.DeepEqual(want, got) {
				t.Fatalf("unexpected temperature marshal:\n- want: %v\n-  got: %v",
					want, got)
			}
		})
	}
}

func TestTemperature_UnmarshalBinary(t *testing.T) {
	tests := []struct {
		name string
		temp temperature
		b    byte
		err  error
	}{
		{
			name: "average value",
			b:    90,
			temp: temperature(25),
		},
		{
			name: "high value",
			b:    255,
			temp: temperature(107),
		},
		{
			name: "low value",
			b:    0,
			temp: temperature(-20),
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			// Unmarshal binary representation into temperature value
			var temp temperature
			temp.UnmarshalBinary(tt.b)

			if want, got := tt.temp, temp; !reflect.DeepEqual(want, got) {
				t.Fatalf("unexpected temperature unmarshal:\n- want: %v\n-  got: %v",
					want, got)
			}
		})
	}
}
<|endoftext|>


<commit_before>package collector

import (
	"fullerite/config"
	"fullerite/metric"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"time"

	l "github.com/Sirupsen/logrus"

	"github.com/fsouza/go-dockerclient"
)

const (
	endpoint = "unix:///var/run/docker.sock"
)

// DockerStats collector type.
// previousCPUValues contains the last cpu-usage values per container.
// dockerClient is the client for the Docker remote API.
type DockerStats struct {
	baseCollector
	previousCPUValues map[string]*CPUValues
	dockerClient      *docker.Client
	statsTimeout      int
	compiledRegex     map[string]*Regex
	endpoint          string
	mu                *sync.Mutex
}

// CPUValues struct contains the last cpu-usage values in order to compute properly the current values.
// (see calculateCPUPercent() for more details)
type CPUValues struct {
	totCPU, systemCPU uint64
}

// Regex struct contains the info used to get the user specific dimensions from the docker env variables
// tag: is the environmental variable you want to get the value from
// regex: is the reg exp used to extract the value from the env var
type Regex struct {
	tag   string
	regex *regexp.Regexp
}

func init() {
	RegisterCollector("DockerStats", newDockerStats)
}

// newDockerStats creates a new DockerStats collector.
func newDockerStats(channel chan metric.Metric, initialInterval int, log *l.Entry) Collector {
	d := new(DockerStats)

	d.log = log
	d.channel = channel
	d.interval = initialInterval
	d.mu = new(sync.Mutex)

	d.name = "DockerStats"
	d.previousCPUValues = make(map[string]*CPUValues)
	d.compiledRegex = make(map[string]*Regex)

	return d
}

// GetEndpoint Returns endpoint of DockerStats instance
func (d *DockerStats) GetEndpoint() string {
	return d.endpoint
}

// Configure takes a dictionary of values with which the handler can configure itself.
func (d *DockerStats) Configure(configMap map[string]interface{}) {
	if timeout, exists := configMap["dockerStatsTimeout"]; exists {
		d.statsTimeout = min(config.GetAsInt(timeout, d.interval), d.interval)
	} else {
		d.statsTimeout = d.interval
	}
	if dockerEndpoint, exists := configMap["dockerEndPoint"]; exists {
		if str, ok := dockerEndpoint.(string); ok {
			d.endpoint = str
		} else {
			d.log.Warn("Failed to cast dokerEndPoint: ", reflect.TypeOf(dockerEndpoint))
		}
	} else {
		d.endpoint = endpoint
	}
	d.dockerClient, _ = docker.NewClient(d.endpoint)
	if generatedDimensions, exists := configMap["generatedDimensions"]; exists {
		for dimension, generator := range generatedDimensions.(map[string]interface{}) {
			for key, regx := range config.GetAsMap(generator) {
				re, err := regexp.Compile(regx)
				if err != nil {
					d.log.Warn("Failed to compile regex: ", regx, err)
				} else {
					d.compiledRegex[dimension] = &Regex{regex: re, tag: key}
				}
			}
		}
	}
	d.configureCommonParams(configMap)
}

// Collect iterates on all the docker containers alive and, if possible, collects the correspondent
// memory and cpu statistics.
// For each container a gorutine is started to spin up the collection process.
func (d *DockerStats) Collect() {
	if d.dockerClient == nil {
		d.log.Error("Invalid endpoint: ", docker.ErrInvalidEndpoint)
		return
	}
	containers, err := d.dockerClient.ListContainers(docker.ListContainersOptions{All: false})
	if err != nil {
		d.log.Error("ListContainers() failed: ", err)
		return
	}
	for _, apiContainer := range containers {
		container, err := d.dockerClient.InspectContainer(apiContainer.ID)
		if err != nil {
			d.log.Error("InspectContainer() failed: ", err)
			continue
		}
		if _, ok := d.previousCPUValues[container.ID]; !ok {
			d.previousCPUValues[container.ID] = new(CPUValues)
		}
		go d.getDockerContainerInfo(container)
	}
}

// getDockerContainerInfo gets container statistics for the given container.
// results is a channel to make possible the synchronization between the main process and the gorutines (wait-notify pattern).
func (d *DockerStats) getDockerContainerInfo(container *docker.Container) {
	errC := make(chan error, 1)
	statsC := make(chan *docker.Stats, 1)
	done := make(chan bool, 1)

	go func() {
		errC <- d.dockerClient.Stats(docker.StatsOptions{container.ID, statsC, false, done, time.Second * time.Duration(d.interval)})
	}()
	select {
	case stats, ok := <-statsC:
		if !ok {
			err := <-errC
			d.log.Error("Failed to collect docker container stats: ", err)
			break
		}
		done <- true

		metrics := d.extractMetrics(container, stats)
		d.sendMetrics(metrics)

		break
	case <-time.After(time.Duration(d.statsTimeout) * time.Second):
		d.log.Error("Timed out collecting stats for container ", container.ID)
		done <- true
		break
	}
}

func (d *DockerStats) extractMetrics(container *docker.Container, stats *docker.Stats) []metric.Metric {
	d.mu.Lock()
	defer d.mu.Unlock()
	metrics := d.buildMetrics(container, stats, calculateCPUPercent(d.previousCPUValues[container.ID].totCPU, d.previousCPUValues[container.ID].systemCPU, stats))

	d.previousCPUValues[container.ID].totCPU = stats.CPUStats.CPUUsage.TotalUsage
	d.previousCPUValues[container.ID].systemCPU = stats.CPUStats.SystemCPUUsage
	return metrics
}

// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, containerStats *docker.Stats, cpuPercentage float64) []metric.Metric {
	//var netiface string
	ret := []metric.Metric{
		buildDockerMetric("DockerMemoryUsed", metric.Gauge, float64(containerStats.MemoryStats.Usage)),
		buildDockerMetric("DockerMemoryLimit", metric.Gauge, float64(containerStats.MemoryStats.Limit)),
		buildDockerMetric("DockerCpuUser", metric.Gauge, cpuPercentage),
		buildDockerMetric("DockerCpuPercentage", metric.Gauge, cpuPercentage),
	}
	for netiface, _ := range containerStats.Networks {
		// legacy format
		txb := buildDockerMetric("DockerTxBytes", metric.CumulativeCounter, float64(containerStats.Networks[netiface].TxBytes))
		txb.AddDimension("iface", netiface)
		ret = append(ret, txb)
		rxb := buildDockerMetric("DockerRxBytes", metric.CumulativeCounter, float64(containerStats.Networks[netiface].RxBytes))
		rxb.AddDimension("iface", netiface)
		ret = append(ret, rxb)
	}
	additionalDimensions := map[string]string{
		"container_id":   container.ID,
		"container_name": strings.TrimPrefix(container.Name, "/"),
	}
	metric.AddToAll(&ret, additionalDimensions)
	ret = append(ret, buildDockerMetric("DockerContainerCount", metric.Counter, 1))
	metric.AddToAll(&ret, d.extractDimensions(container))

	return ret
}

// sendMetrics writes all the metrics received to the collector channel.
func (d DockerStats) sendMetrics(metrics []metric.Metric) {
	for _, m := range metrics {
		d.Channel() <- m
	}
}

// Function that extracts additional dimensions from the docker environmental variables set up by the user
// in the configuration file.
func (d DockerStats) extractDimensions(container *docker.Container) map[string]string {
	envVars := container.Config.Env
	ret := map[string]string{}

	for dimension, r := range d.compiledRegex {
		for _, envVariable := range envVars {
			envArray := strings.Split(envVariable, "=")
			if r.tag == envArray[0] {
				subMatch := r.regex.FindStringSubmatch(envArray[1])
				if len(subMatch) > 0 {
					ret[dimension] = strings.Replace(subMatch[len(subMatch)-1], "--", "_", -1)
				}
			}
		}
	}
	d.log.Debug(ret)
	return ret
}

func buildDockerMetric(name string, metricType string, value float64) (m metric.Metric) {
	m = metric.New(name)
	m.MetricType = metricType
	m.Value = value
	return m
}

// Function that compute the current cpu usage percentage combining current and last values.
func calculateCPUPercent(previousCPU, previousSystem uint64, stats *docker.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(stats.CPUStats.CPUUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(stats.CPUStats.SystemCPUUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}
<commit_msg>remove commented variable<commit_after>package collector

import (
	"fullerite/config"
	"fullerite/metric"
	"reflect"
	"regexp"
	"strings"
	"sync"
	"time"

	l "github.com/Sirupsen/logrus"

	"github.com/fsouza/go-dockerclient"
)

const (
	endpoint = "unix:///var/run/docker.sock"
)

// DockerStats collector type.
// previousCPUValues contains the last cpu-usage values per container.
// dockerClient is the client for the Docker remote API.
type DockerStats struct {
	baseCollector
	previousCPUValues map[string]*CPUValues
	dockerClient      *docker.Client
	statsTimeout      int
	compiledRegex     map[string]*Regex
	endpoint          string
	mu                *sync.Mutex
}

// CPUValues struct contains the last cpu-usage values in order to compute properly the current values.
// (see calculateCPUPercent() for more details)
type CPUValues struct {
	totCPU, systemCPU uint64
}

// Regex struct contains the info used to get the user specific dimensions from the docker env variables
// tag: is the environmental variable you want to get the value from
// regex: is the reg exp used to extract the value from the env var
type Regex struct {
	tag   string
	regex *regexp.Regexp
}

func init() {
	RegisterCollector("DockerStats", newDockerStats)
}

// newDockerStats creates a new DockerStats collector.
func newDockerStats(channel chan metric.Metric, initialInterval int, log *l.Entry) Collector {
	d := new(DockerStats)

	d.log = log
	d.channel = channel
	d.interval = initialInterval
	d.mu = new(sync.Mutex)

	d.name = "DockerStats"
	d.previousCPUValues = make(map[string]*CPUValues)
	d.compiledRegex = make(map[string]*Regex)

	return d
}

// GetEndpoint Returns endpoint of DockerStats instance
func (d *DockerStats) GetEndpoint() string {
	return d.endpoint
}

// Configure takes a dictionary of values with which the handler can configure itself.
func (d *DockerStats) Configure(configMap map[string]interface{}) {
	if timeout, exists := configMap["dockerStatsTimeout"]; exists {
		d.statsTimeout = min(config.GetAsInt(timeout, d.interval), d.interval)
	} else {
		d.statsTimeout = d.interval
	}
	if dockerEndpoint, exists := configMap["dockerEndPoint"]; exists {
		if str, ok := dockerEndpoint.(string); ok {
			d.endpoint = str
		} else {
			d.log.Warn("Failed to cast dokerEndPoint: ", reflect.TypeOf(dockerEndpoint))
		}
	} else {
		d.endpoint = endpoint
	}
	d.dockerClient, _ = docker.NewClient(d.endpoint)
	if generatedDimensions, exists := configMap["generatedDimensions"]; exists {
		for dimension, generator := range generatedDimensions.(map[string]interface{}) {
			for key, regx := range config.GetAsMap(generator) {
				re, err := regexp.Compile(regx)
				if err != nil {
					d.log.Warn("Failed to compile regex: ", regx, err)
				} else {
					d.compiledRegex[dimension] = &Regex{regex: re, tag: key}
				}
			}
		}
	}
	d.configureCommonParams(configMap)
}

// Collect iterates on all the docker containers alive and, if possible, collects the correspondent
// memory and cpu statistics.
// For each container a gorutine is started to spin up the collection process.
func (d *DockerStats) Collect() {
	if d.dockerClient == nil {
		d.log.Error("Invalid endpoint: ", docker.ErrInvalidEndpoint)
		return
	}
	containers, err := d.dockerClient.ListContainers(docker.ListContainersOptions{All: false})
	if err != nil {
		d.log.Error("ListContainers() failed: ", err)
		return
	}
	for _, apiContainer := range containers {
		container, err := d.dockerClient.InspectContainer(apiContainer.ID)
		if err != nil {
			d.log.Error("InspectContainer() failed: ", err)
			continue
		}
		if _, ok := d.previousCPUValues[container.ID]; !ok {
			d.previousCPUValues[container.ID] = new(CPUValues)
		}
		go d.getDockerContainerInfo(container)
	}
}

// getDockerContainerInfo gets container statistics for the given container.
// results is a channel to make possible the synchronization between the main process and the gorutines (wait-notify pattern).
func (d *DockerStats) getDockerContainerInfo(container *docker.Container) {
	errC := make(chan error, 1)
	statsC := make(chan *docker.Stats, 1)
	done := make(chan bool, 1)

	go func() {
		errC <- d.dockerClient.Stats(docker.StatsOptions{container.ID, statsC, false, done, time.Second * time.Duration(d.interval)})
	}()
	select {
	case stats, ok := <-statsC:
		if !ok {
			err := <-errC
			d.log.Error("Failed to collect docker container stats: ", err)
			break
		}
		done <- true

		metrics := d.extractMetrics(container, stats)
		d.sendMetrics(metrics)

		break
	case <-time.After(time.Duration(d.statsTimeout) * time.Second):
		d.log.Error("Timed out collecting stats for container ", container.ID)
		done <- true
		break
	}
}

func (d *DockerStats) extractMetrics(container *docker.Container, stats *docker.Stats) []metric.Metric {
	d.mu.Lock()
	defer d.mu.Unlock()
	metrics := d.buildMetrics(container, stats, calculateCPUPercent(d.previousCPUValues[container.ID].totCPU, d.previousCPUValues[container.ID].systemCPU, stats))

	d.previousCPUValues[container.ID].totCPU = stats.CPUStats.CPUUsage.TotalUsage
	d.previousCPUValues[container.ID].systemCPU = stats.CPUStats.SystemCPUUsage
	return metrics
}

// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, containerStats *docker.Stats, cpuPercentage float64) []metric.Metric {
	ret := []metric.Metric{
		buildDockerMetric("DockerMemoryUsed", metric.Gauge, float64(containerStats.MemoryStats.Usage)),
		buildDockerMetric("DockerMemoryLimit", metric.Gauge, float64(containerStats.MemoryStats.Limit)),
		buildDockerMetric("DockerCpuUser", metric.Gauge, cpuPercentage),
		buildDockerMetric("DockerCpuPercentage", metric.Gauge, cpuPercentage),
	}
	for netiface, _ := range containerStats.Networks {
		// legacy format
		txb := buildDockerMetric("DockerTxBytes", metric.CumulativeCounter, float64(containerStats.Networks[netiface].TxBytes))
		txb.AddDimension("iface", netiface)
		ret = append(ret, txb)
		rxb := buildDockerMetric("DockerRxBytes", metric.CumulativeCounter, float64(containerStats.Networks[netiface].RxBytes))
		rxb.AddDimension("iface", netiface)
		ret = append(ret, rxb)
	}
	additionalDimensions := map[string]string{
		"container_id":   container.ID,
		"container_name": strings.TrimPrefix(container.Name, "/"),
	}
	metric.AddToAll(&ret, additionalDimensions)
	ret = append(ret, buildDockerMetric("DockerContainerCount", metric.Counter, 1))
	metric.AddToAll(&ret, d.extractDimensions(container))

	return ret
}

// sendMetrics writes all the metrics received to the collector channel.
func (d DockerStats) sendMetrics(metrics []metric.Metric) {
	for _, m := range metrics {
		d.Channel() <- m
	}
}

// Function that extracts additional dimensions from the docker environmental variables set up by the user
// in the configuration file.
func (d DockerStats) extractDimensions(container *docker.Container) map[string]string {
	envVars := container.Config.Env
	ret := map[string]string{}

	for dimension, r := range d.compiledRegex {
		for _, envVariable := range envVars {
			envArray := strings.Split(envVariable, "=")
			if r.tag == envArray[0] {
				subMatch := r.regex.FindStringSubmatch(envArray[1])
				if len(subMatch) > 0 {
					ret[dimension] = strings.Replace(subMatch[len(subMatch)-1], "--", "_", -1)
				}
			}
		}
	}
	d.log.Debug(ret)
	return ret
}

func buildDockerMetric(name string, metricType string, value float64) (m metric.Metric) {
	m = metric.New(name)
	m.MetricType = metricType
	m.Value = value
	return m
}

// Function that compute the current cpu usage percentage combining current and last values.
func calculateCPUPercent(previousCPU, previousSystem uint64, stats *docker.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(stats.CPUStats.CPUUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(stats.CPUStats.SystemCPUUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}
<|endoftext|>


<commit_before>package collector

import (
	"fullerite/config"
	"fullerite/metric"

	"regexp"
	"strings"
	"time"

	l "github.com/Sirupsen/logrus"

	"github.com/fsouza/go-dockerclient"
)

const (
	endpoint         = "unix:///var/run/docker.sock"
	serviceNameLabel = "SERVICE_NAME"
)

// DockerStats collector type.
// previousCPUValues contains the last cpu-usage values per container.
// dockerClient is the client for the Docker remote API.
type DockerStats struct {
	baseCollector
	previousCPUValues map[string]*CPUValues
	dockerClient      *docker.Client
	statsTimeout      int
	compiledRegex     map[string]*Regex
}

// CPUValues struct contains the last cpu-usage values in order to compute properly the current values.
// (see calculateCPUPercent() for more details)
type CPUValues struct {
	totCPU, systemCPU uint64
}

// Regex struct contains the info used to get the user specific dimensions from the docker env variables
// tag: is the environmental variable you want to get the value from
// regex: is the reg exp used to extract the value from the env var
type Regex struct {
	tag   string
	regex *regexp.Regexp
}

// NewDockerStats creates a new DockerStats collector.
func NewDockerStats(channel chan metric.Metric, initialInterval int, log *l.Entry) *DockerStats {
	d := new(DockerStats)

	d.log = log
	d.channel = channel
	d.interval = initialInterval

	d.name = "DockerStats"
	d.previousCPUValues = make(map[string]*CPUValues)
	d.dockerClient, _ = docker.NewClient(endpoint)
	d.compiledRegex = make(map[string]*Regex)

	return d
}

// Configure takes a dictionary of values with which the handler can configure itself.
func (d *DockerStats) Configure(configMap map[string]interface{}) {
	if timeout, exists := configMap["dockerStatsTimeout"]; exists {
		d.statsTimeout = min(config.GetAsInt(timeout, d.interval), d.interval)
	} else {
		d.statsTimeout = d.interval
	}
	if generatedDimensions, exists := configMap["generatedDimensions"]; exists {
		for dimension, generator := range generatedDimensions.(map[string]interface{}) {
			for key, regx := range config.GetAsMap(generator) {
				re, err := regexp.Compile(regx)
				if err != nil {
					d.log.Warn("Failed to compile regex: ", regx, err)
				} else {
					d.compiledRegex[dimension] = &Regex{regex: re, tag: key}
				}
			}
		}
	}
	d.configureCommonParams(configMap)
}

// Collect iterates on all the docker containers alive and, if possible, collects the correspondent
// memory and cpu statistics.
// For each container a gorutine is started to spin up the collection process.
func (d *DockerStats) Collect() {
	if d.dockerClient == nil {
		d.log.Error("Invalid endpoint: ", docker.ErrInvalidEndpoint)
		return
	}
	containers, err := d.dockerClient.ListContainers(docker.ListContainersOptions{All: false})
	if err != nil {
		d.log.Error("ListContainers() failed: ", err)
		return
	}
	for _, apiContainer := range containers {
		container, err := d.dockerClient.InspectContainer(apiContainer.ID)
		if err != nil {
			d.log.Error("InspectContainer() failed: ", err)
			continue
		}
		if _, ok := d.previousCPUValues[container.ID]; !ok {
			d.previousCPUValues[container.ID] = new(CPUValues)
		}
		go d.getDockerContainerInfo(container)
	}
}

// getDockerContainerInfo gets container statistics for the given container.
// results is a channel to make possible the synchronization between the main process and the gorutines (wait-notify pattern).
func (d DockerStats) getDockerContainerInfo(container *docker.Container) {
	errC := make(chan error, 1)
	statsC := make(chan *docker.Stats, 1)
	done := make(chan bool)

	go func() {
		errC <- d.dockerClient.Stats(docker.StatsOptions{container.ID, statsC, false, done, time.Second * time.Duration(d.interval)})
	}()
	select {
	case stats, ok := <-statsC:
		if !ok {
			err := <-errC
			d.log.Error("Failed to collect docker container stats: ", err)
			break
		}
		done <- true

		ret := d.buildMetrics(container, stats, calculateCPUPercent(d.previousCPUValues[container.ID].totCPU, d.previousCPUValues[container.ID].systemCPU, stats))

		d.sendMetrics(ret)

		d.previousCPUValues[container.ID].totCPU = stats.CPUStats.CPUUsage.TotalUsage
		d.previousCPUValues[container.ID].systemCPU = stats.CPUStats.SystemCPUUsage

		break
	case <-time.After(time.Duration(d.statsTimeout) * time.Second):
		d.log.Error("Timed out collecting stats for container ", container.ID)
		done <- true
		break
	}
}

// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, containerStats *docker.Stats, cpuPercentage float64) []metric.Metric {
	ret := []metric.Metric{
		buildDockerMetric("DockerRxBytes", metric.CumulativeCounter, float64(containerStats.Network.RxBytes)),
		buildDockerMetric("DockerTxBytes", metric.CumulativeCounter, float64(containerStats.Network.TxBytes)),
		buildDockerMetric("DockerMemoryUsed", metric.Gauge, float64(containerStats.MemoryStats.Usage)),
		buildDockerMetric("DockerMemoryLimit", metric.Gauge, float64(containerStats.MemoryStats.Limit)),
		buildDockerMetric("DockerCpuPercentage", metric.Gauge, cpuPercentage),
	}
	additionalDimensions := map[string]string{
		"container_id":   container.ID,
		"container_name": strings.TrimPrefix(container.Name, "/"),
	}
	metric.AddToAll(&ret, additionalDimensions)
	metric.AddToAll(&ret, d.extractDimensions(container))

	return ret
}

// sendMetrics writes all the metrics received to the collector channel.
func (d DockerStats) sendMetrics(metrics []metric.Metric) {
	for _, m := range metrics {
		d.Channel() <- m
	}
}

// Function that extracts additional dimensions from the docker environmental variables set up by the user
// in the configuration file.
func (d DockerStats) extractDimensions(container *docker.Container) map[string]string {
	envVars := container.Config.Env
	ret := map[string]string{}

	for dimension, r := range d.compiledRegex {
		for _, envVariable := range envVars {
			envArray := strings.Split(envVariable, "=")
			if r.tag == envArray[0] {
				subMatch := r.regex.FindStringSubmatch(envArray[1])
				if len(subMatch) > 0 {
					ret[dimension] = strings.Replace(subMatch[len(subMatch)-1], "--", "_", -1)
				}
			}
		}
	}
	d.log.Debug(ret)
	return ret
}

func buildDockerMetric(name string, metricType string, value float64) (m metric.Metric) {
	m = metric.New(name)
	m.MetricType = metricType
	m.Value = value
	m.AddDimension("collector", "DockerStats")
	return m
}

// Function that compute the current cpu usage percentage combining current and last values.
func calculateCPUPercent(previousCPU, previousSystem uint64, stats *docker.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(stats.CPUStats.CPUUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(stats.CPUStats.SystemCPUUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}
<commit_msg>Remove servicenamelabel<commit_after>package collector

import (
	"fullerite/config"
	"fullerite/metric"

	"regexp"
	"strings"
	"time"

	l "github.com/Sirupsen/logrus"

	"github.com/fsouza/go-dockerclient"
)

const (
	endpoint = "unix:///var/run/docker.sock"
)

// DockerStats collector type.
// previousCPUValues contains the last cpu-usage values per container.
// dockerClient is the client for the Docker remote API.
type DockerStats struct {
	baseCollector
	previousCPUValues map[string]*CPUValues
	dockerClient      *docker.Client
	statsTimeout      int
	compiledRegex     map[string]*Regex
}

// CPUValues struct contains the last cpu-usage values in order to compute properly the current values.
// (see calculateCPUPercent() for more details)
type CPUValues struct {
	totCPU, systemCPU uint64
}

// Regex struct contains the info used to get the user specific dimensions from the docker env variables
// tag: is the environmental variable you want to get the value from
// regex: is the reg exp used to extract the value from the env var
type Regex struct {
	tag   string
	regex *regexp.Regexp
}

// NewDockerStats creates a new DockerStats collector.
func NewDockerStats(channel chan metric.Metric, initialInterval int, log *l.Entry) *DockerStats {
	d := new(DockerStats)

	d.log = log
	d.channel = channel
	d.interval = initialInterval

	d.name = "DockerStats"
	d.previousCPUValues = make(map[string]*CPUValues)
	d.dockerClient, _ = docker.NewClient(endpoint)
	d.compiledRegex = make(map[string]*Regex)

	return d
}

// Configure takes a dictionary of values with which the handler can configure itself.
func (d *DockerStats) Configure(configMap map[string]interface{}) {
	if timeout, exists := configMap["dockerStatsTimeout"]; exists {
		d.statsTimeout = min(config.GetAsInt(timeout, d.interval), d.interval)
	} else {
		d.statsTimeout = d.interval
	}
	if generatedDimensions, exists := configMap["generatedDimensions"]; exists {
		for dimension, generator := range generatedDimensions.(map[string]interface{}) {
			for key, regx := range config.GetAsMap(generator) {
				re, err := regexp.Compile(regx)
				if err != nil {
					d.log.Warn("Failed to compile regex: ", regx, err)
				} else {
					d.compiledRegex[dimension] = &Regex{regex: re, tag: key}
				}
			}
		}
	}
	d.configureCommonParams(configMap)
}

// Collect iterates on all the docker containers alive and, if possible, collects the correspondent
// memory and cpu statistics.
// For each container a gorutine is started to spin up the collection process.
func (d *DockerStats) Collect() {
	if d.dockerClient == nil {
		d.log.Error("Invalid endpoint: ", docker.ErrInvalidEndpoint)
		return
	}
	containers, err := d.dockerClient.ListContainers(docker.ListContainersOptions{All: false})
	if err != nil {
		d.log.Error("ListContainers() failed: ", err)
		return
	}
	for _, apiContainer := range containers {
		container, err := d.dockerClient.InspectContainer(apiContainer.ID)
		if err != nil {
			d.log.Error("InspectContainer() failed: ", err)
			continue
		}
		if _, ok := d.previousCPUValues[container.ID]; !ok {
			d.previousCPUValues[container.ID] = new(CPUValues)
		}
		go d.getDockerContainerInfo(container)
	}
}

// getDockerContainerInfo gets container statistics for the given container.
// results is a channel to make possible the synchronization between the main process and the gorutines (wait-notify pattern).
func (d DockerStats) getDockerContainerInfo(container *docker.Container) {
	errC := make(chan error, 1)
	statsC := make(chan *docker.Stats, 1)
	done := make(chan bool)

	go func() {
		errC <- d.dockerClient.Stats(docker.StatsOptions{container.ID, statsC, false, done, time.Second * time.Duration(d.interval)})
	}()
	select {
	case stats, ok := <-statsC:
		if !ok {
			err := <-errC
			d.log.Error("Failed to collect docker container stats: ", err)
			break
		}
		done <- true

		ret := d.buildMetrics(container, stats, calculateCPUPercent(d.previousCPUValues[container.ID].totCPU, d.previousCPUValues[container.ID].systemCPU, stats))

		d.sendMetrics(ret)

		d.previousCPUValues[container.ID].totCPU = stats.CPUStats.CPUUsage.TotalUsage
		d.previousCPUValues[container.ID].systemCPU = stats.CPUStats.SystemCPUUsage

		break
	case <-time.After(time.Duration(d.statsTimeout) * time.Second):
		d.log.Error("Timed out collecting stats for container ", container.ID)
		done <- true
		break
	}
}

// buildMetrics creates the actual metrics for the given container.
func (d DockerStats) buildMetrics(container *docker.Container, containerStats *docker.Stats, cpuPercentage float64) []metric.Metric {
	ret := []metric.Metric{
		buildDockerMetric("DockerRxBytes", metric.CumulativeCounter, float64(containerStats.Network.RxBytes)),
		buildDockerMetric("DockerTxBytes", metric.CumulativeCounter, float64(containerStats.Network.TxBytes)),
		buildDockerMetric("DockerMemoryUsed", metric.Gauge, float64(containerStats.MemoryStats.Usage)),
		buildDockerMetric("DockerMemoryLimit", metric.Gauge, float64(containerStats.MemoryStats.Limit)),
		buildDockerMetric("DockerCpuPercentage", metric.Gauge, cpuPercentage),
	}
	additionalDimensions := map[string]string{
		"container_id":   container.ID,
		"container_name": strings.TrimPrefix(container.Name, "/"),
	}
	metric.AddToAll(&ret, additionalDimensions)
	metric.AddToAll(&ret, d.extractDimensions(container))

	return ret
}

// sendMetrics writes all the metrics received to the collector channel.
func (d DockerStats) sendMetrics(metrics []metric.Metric) {
	for _, m := range metrics {
		d.Channel() <- m
	}
}

// Function that extracts additional dimensions from the docker environmental variables set up by the user
// in the configuration file.
func (d DockerStats) extractDimensions(container *docker.Container) map[string]string {
	envVars := container.Config.Env
	ret := map[string]string{}

	for dimension, r := range d.compiledRegex {
		for _, envVariable := range envVars {
			envArray := strings.Split(envVariable, "=")
			if r.tag == envArray[0] {
				subMatch := r.regex.FindStringSubmatch(envArray[1])
				if len(subMatch) > 0 {
					ret[dimension] = strings.Replace(subMatch[len(subMatch)-1], "--", "_", -1)
				}
			}
		}
	}
	d.log.Debug(ret)
	return ret
}

func buildDockerMetric(name string, metricType string, value float64) (m metric.Metric) {
	m = metric.New(name)
	m.MetricType = metricType
	m.Value = value
	m.AddDimension("collector", "DockerStats")
	return m
}

// Function that compute the current cpu usage percentage combining current and last values.
func calculateCPUPercent(previousCPU, previousSystem uint64, stats *docker.Stats) float64 {
	var (
		cpuPercent = 0.0
		// calculate the change for the cpu usage of the container in between readings
		cpuDelta = float64(stats.CPUStats.CPUUsage.TotalUsage - previousCPU)
		// calculate the change for the entire system between readings
		systemDelta = float64(stats.CPUStats.SystemCPUUsage - previousSystem)
	)

	if systemDelta > 0.0 && cpuDelta > 0.0 {
		cpuPercent = (cpuDelta / systemDelta) * float64(len(stats.CPUStats.CPUUsage.PercpuUsage)) * 100.0
	}
	return cpuPercent
}

func min(x, y int) int {
	if x < y {
		return x
	}
	return y
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bufio"
	"fmt"
	"os"

	"k8s.io/contrib/test-utils/utils"
)

// constants to use for downloading data.
const (
	logFile = "build-log.txt"
)

// GoogleGCSDownloader that gets data about Google results from the GCS repository
type GoogleGCSDownloader struct {
	Builds               int
	GoogleGCSBucketUtils *utils.Utils
}

// NewGoogleGCSDownloader creates a new GoogleGCSDownloader
func NewGoogleGCSDownloader(builds int) *GoogleGCSDownloader {
	return &GoogleGCSDownloader{
		Builds:               builds,
		GoogleGCSBucketUtils: utils.NewUtils(utils.GoogleBucketURL),
	}
}

// TODO(random-liu): Only download and update new data each time.
func (g *GoogleGCSDownloader) getData() (TestToBuildData, error) {
	fmt.Print("Getting Data from GCS...\n")
	result := make(TestToBuildData)
	for job, tests := range TestConfig[utils.GoogleBucketURL] {
		lastBuildNo, err := g.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job)
		if err != nil {
			return result, err
		}
		fmt.Printf("Last build no: %v\n", lastBuildNo)
		var buildNumber int
		if lastBuildNo >= g.Builds {
			buildNumber = lastBuildNo - g.Builds + 1
		} else {
			buildNumber = 1
		}
		for ; buildNumber <= lastBuildNo; buildNumber++ {
			fmt.Printf("Fetching build %v...\n", buildNumber)
			testDataResponse, err := g.GoogleGCSBucketUtils.GetFileFromJenkinsGoogleBucket(job, buildNumber, logFile)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error while fetching data: %v\n", err)
				continue
			}

			testDataBody := testDataResponse.Body
			defer testDataBody.Close()
			testDataScanner := bufio.NewScanner(testDataBody)
			parseTestOutput(testDataScanner, job, tests, buildNumber, result)
		}
	}
	return result, nil
}
<commit_msg>Fetch build logs from latest to oldest so as to make sure metrics with newer version are parsed first<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bufio"
	"fmt"
	"os"

	"k8s.io/contrib/test-utils/utils"
)

// constants to use for downloading data.
const (
	logFile = "build-log.txt"
)

// GoogleGCSDownloader that gets data about Google results from the GCS repository
type GoogleGCSDownloader struct {
	Builds               int
	GoogleGCSBucketUtils *utils.Utils
}

// NewGoogleGCSDownloader creates a new GoogleGCSDownloader
func NewGoogleGCSDownloader(builds int) *GoogleGCSDownloader {
	return &GoogleGCSDownloader{
		Builds:               builds,
		GoogleGCSBucketUtils: utils.NewUtils(utils.GoogleBucketURL),
	}
}

// TODO(random-liu): Only download and update new data each time.
func (g *GoogleGCSDownloader) getData() (TestToBuildData, error) {
	fmt.Print("Getting Data from GCS...\n")
	result := make(TestToBuildData)
	for job, tests := range TestConfig[utils.GoogleBucketURL] {
		lastBuildNo, err := g.GoogleGCSBucketUtils.GetLastestBuildNumberFromJenkinsGoogleBucket(job)
		if err != nil {
			return result, err
		}
		fmt.Printf("Last build no: %v\n", lastBuildNo)
		for buildNumber := lastBuildNo; buildNumber > lastBuildNo-g.Builds && buildNumber > 0; buildNumber-- {
			fmt.Printf("Fetching build %v...\n", buildNumber)
			testDataResponse, err := g.GoogleGCSBucketUtils.GetFileFromJenkinsGoogleBucket(job, buildNumber, logFile)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error while fetching data: %v\n", err)
				continue
			}

			testDataBody := testDataResponse.Body
			defer testDataBody.Close()
			testDataScanner := bufio.NewScanner(testDataBody)
			parseTestOutput(testDataScanner, job, tests, buildNumber, result)
		}
	}
	return result, nil
}
<|endoftext|>