<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Source ranking is based on the following criteria in order:
// 1. More trusted source ranks higher
// 2. Latest data ranks higher
// 3. More data ranks higher

package ranking

import (
	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/server/model"
)

// RankKey represents keys used for ranking.
// Import Name should be non-empty.
// Can use "*" for wildcard match for MeasurementMethod and ObservationPeriod
type RankKey struct {
	ImportName        string
	MeasurementMethod string
	ObservationPeriod string
}

// StatsRanking is used to rank multiple source series for the same
// StatisticalVariable, where lower value means higher ranking.
var StatsRanking = map[RankKey]int{
	{"CensusPEP", "CensusPEPSurvey", "*"}:                                0, // Population
	{"CensusACS5YearSurvey", "CensusACS5yrSurvey", "*"}:                  1, // Population
	{"CensusACS5YearSurvey_AggCountry", "CensusACS5yrSurvey", "*"}:       1, // Population
	{"CensusUSAMedianAgeIncome", "CensusACS5yrSurvey", "*"}:              1, // Population
	{"USDecennialCensus_RedistrictingRelease", "USDecennialCensus", "*"}: 2, // Population
	{"EurostatData", "EurostatRegionalPopulationData", "*"}:              3, // Population
	{"WorldDevelopmentIndicators", "*", "*"}:                             4, // Population
	// Prefer Indian Census population for Indian states, over something like OECD.
	{"IndiaCensus_Primary", "*", "*"}:                 5,    // Population
	{"WikipediaStatsData", "Wikipedia", "*"}:          1001, // Population
	{"HumanCuratedStats", "HumanCuratedStats", "*"}:   1002, // Population
	{"WikidataPopulation", "WikidataPopulation", "*"}: 1003, // Population

	{"BLS_LAUS", "BLSSeasonallyUnadjusted", "*"}: 0, // Unemployment Rate
	{"BLS_CPS", "BLSSeasonallyAdjusted", "*"}:    1, // Labor Force data ranked higher than WDI (above) or Eurostat
	{"EurostatData", "", "*"}:                    2, // Unemployment Rate

	{"NYT_COVID19", "NYT_COVID19_GitHub", "*"}: 0, // Covid

	{"CDC500", "AgeAdjustedPrevalence", "*"}: 0, // CDC500

	{"UNEnergy", "", "*"}:         0, // Electricity
	{"EIA_Electricity", "*", "*"}: 1, // Electricity

	{"NASA_NEXDCP30", "NASA_Mean_CCSM4", "P1M"}:     0, // IPCC
	{"NASA_NEXGDDP", "NASA_Mean_CMIP5_CCSM4", "P1M"}:      0, // IPCC
	{"NASA_NEXDCP30_StatVarSeriesAggr", "*", "P1M"}: 0, // IPCC
	{"NASA_NEXGDDP_StatVarSeriesAggr", "*", "P1M"}:  0, // IPCC

	{"NASA_WetBulbComputation_Aggregation", "NASA_Mean_HadGEM2-AO", "*"}: 0, // Wet bulb year aggregation
	{"NASA_WetBulbComputation_Aggregation", "*", "*"}:                    1, // Wet bulb year aggregation
	{"NASA_WetBulbComputation", "NASA_Mean_HadGEM2-AO", "*"}:             2, // Wet bulb

	// Prefer FBI Hate Crime Publications over Data Commons Aggregate
	// Note: https://autopush.datacommons.org/tools/timeline#statsVar=Count_CriminalIncidents_IsHateCrime&place=country%2FUSA
	// Expected data 2010-2020: 6628, 6222, 5796, 5928, 5479, 5850, 6121, 7175, 7120, 7314, 8263
	// Note: https://autopush.datacommons.org/tools/timeline#place=geoId%2F06&statsVar=Count_CriminalIncidents_IsHateCrime
	// Expected data 2004-2010: 1393, 1379, 1297, 1400, 1381, 1015, 1092
	{"FBIHateCrimePublications", "*", "*"}: 0, // FBI Hate Crime Publications
	{"FBIHateCrime", "*", "*"}:             1, // FBI Hate Crime Aggregations
}

// BaseRank is the base ranking score for sources. If a source is prefered, it
// should be given a score lower than BaseRank in StatsRanking. If a source is not
// prefered, it should be given a score higher than BaseRank in StatsRanking
const BaseRank = 100

// CohortByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the place dcid.
//
// Note this has the same data type as SeriesByRank but is used to compare
// cohort instead of time series.
type CohortByRank []*pb.SourceSeries

// getScorePb derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func getScorePb(s *pb.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

func (a CohortByRank) Len() int {
	return len(a)
}

func (a CohortByRank) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a CohortByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := getScorePb(oi)
	oj := a[j]
	scorej := getScorePb(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}
	// Cohort with more place coverage is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// SeriesByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the observation date.
//
// This is the protobuf version of byRank.
type SeriesByRank []*pb.SourceSeries

func (a SeriesByRank) Len() int { return len(a) }

func (a SeriesByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a SeriesByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := getScorePb(oi)
	oj := a[j]
	scorej := getScorePb(oj)

	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.ImportName != oj.ImportName {
		return oi.ImportName < oj.ImportName
	}
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// TODO(shifucun): Remove `SourceSeries` and use pb.SourceSeries everywhere.
// getScore derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func getScore(s *model.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

// ByRank implements sort.Interface for []*SourceSeries based on
// the rank score.
type ByRank []*model.SourceSeries

func (a ByRank) Len() int { return len(a) }

func (a ByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a ByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := getScore(oi)
	oj := a[j]
	scorej := getScore(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceURL != oj.ProvenanceURL {
		return oi.ProvenanceURL < oj.ProvenanceURL
	}
	return true
}
<commit_msg>Prefer observational and gridded weather before IPCC. (#808)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Source ranking is based on the following criteria in order:
// 1. More trusted source ranks higher
// 2. Latest data ranks higher
// 3. More data ranks higher

package ranking

import (
	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/server/model"
)

// RankKey represents keys used for ranking.
// Import Name should be non-empty.
// Can use "*" for wildcard match for MeasurementMethod and ObservationPeriod
type RankKey struct {
	ImportName        string
	MeasurementMethod string
	ObservationPeriod string
}

// StatsRanking is used to rank multiple source series for the same
// StatisticalVariable, where lower value means higher ranking.
var StatsRanking = map[RankKey]int{
	{"CensusPEP", "CensusPEPSurvey", "*"}:                                0, // Population
	{"CensusACS5YearSurvey", "CensusACS5yrSurvey", "*"}:                  1, // Population
	{"CensusACS5YearSurvey_AggCountry", "CensusACS5yrSurvey", "*"}:       1, // Population
	{"CensusUSAMedianAgeIncome", "CensusACS5yrSurvey", "*"}:              1, // Population
	{"USDecennialCensus_RedistrictingRelease", "USDecennialCensus", "*"}: 2, // Population
	{"EurostatData", "EurostatRegionalPopulationData", "*"}:              3, // Population
	{"WorldDevelopmentIndicators", "*", "*"}:                             4, // Population
	// Prefer Indian Census population for Indian states, over something like OECD.
	{"IndiaCensus_Primary", "*", "*"}:                 5,    // Population
	{"WikipediaStatsData", "Wikipedia", "*"}:          1001, // Population
	{"HumanCuratedStats", "HumanCuratedStats", "*"}:   1002, // Population
	{"WikidataPopulation", "WikidataPopulation", "*"}: 1003, // Population

	{"BLS_LAUS", "BLSSeasonallyUnadjusted", "*"}: 0, // Unemployment Rate
	{"BLS_CPS", "BLSSeasonallyAdjusted", "*"}:    1, // Labor Force data ranked higher than WDI (above) or Eurostat
	{"EurostatData", "", "*"}:                    2, // Unemployment Rate

	{"NYT_COVID19", "NYT_COVID19_GitHub", "*"}: 0, // Covid

	{"CDC500", "AgeAdjustedPrevalence", "*"}: 0, // CDC500

	{"UNEnergy", "", "*"}:         0, // Electricity
	{"EIA_Electricity", "*", "*"}: 1, // Electricity

	// Prefer observational weather over gridded over projections
	{"NOAA_EPA_Observed_Historical_Weather", "*", "*"}:	0,  // Observational
	{"Copernicus_ECMWF_ERA5_Monthly", "*", "*"}: 0,			// Gridded reanalysis
	{"NASA_NEXDCP30", "NASA_Mean_CCSM4", "P1M"}:     0, // IPCC Projections
	{"NASA_NEXGDDP", "NASA_Mean_CMIP5_CCSM4", "P1M"}:      0, // IPCC Projections
	{"NASA_NEXDCP30_StatVarSeriesAggr", "*", "P1M"}: 0, // IPCC Projections
	{"NASA_NEXGDDP_StatVarSeriesAggr", "*", "P1M"}:  0, // IPCC Projections

	{"NASA_WetBulbComputation_Aggregation", "NASA_Mean_HadGEM2-AO", "*"}: 0, // Wet bulb year aggregation
	{"NASA_WetBulbComputation_Aggregation", "*", "*"}:                    1, // Wet bulb year aggregation
	{"NASA_WetBulbComputation", "NASA_Mean_HadGEM2-AO", "*"}:             2, // Wet bulb

	// Prefer FBI Hate Crime Publications over Data Commons Aggregate
	// Note: https://autopush.datacommons.org/tools/timeline#statsVar=Count_CriminalIncidents_IsHateCrime&place=country%2FUSA
	// Expected data 2010-2020: 6628, 6222, 5796, 5928, 5479, 5850, 6121, 7175, 7120, 7314, 8263
	// Note: https://autopush.datacommons.org/tools/timeline#place=geoId%2F06&statsVar=Count_CriminalIncidents_IsHateCrime
	// Expected data 2004-2010: 1393, 1379, 1297, 1400, 1381, 1015, 1092
	{"FBIHateCrimePublications", "*", "*"}: 0, // FBI Hate Crime Publications
	{"FBIHateCrime", "*", "*"}:             1, // FBI Hate Crime Aggregations
}

// BaseRank is the base ranking score for sources. If a source is prefered, it
// should be given a score lower than BaseRank in StatsRanking. If a source is not
// prefered, it should be given a score higher than BaseRank in StatsRanking
const BaseRank = 100

// CohortByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the place dcid.
//
// Note this has the same data type as SeriesByRank but is used to compare
// cohort instead of time series.
type CohortByRank []*pb.SourceSeries

// getScorePb derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func getScorePb(s *pb.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

func (a CohortByRank) Len() int {
	return len(a)
}

func (a CohortByRank) Swap(i, j int) {
	a[i], a[j] = a[j], a[i]
}

func (a CohortByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := getScorePb(oi)
	oj := a[j]
	scorej := getScorePb(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}
	// Cohort with more place coverage is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// SeriesByRank implements sort.Interface for []*SourceSeries based on
// the rank score. Each source series data is keyed by the observation date.
//
// This is the protobuf version of byRank.
type SeriesByRank []*pb.SourceSeries

func (a SeriesByRank) Len() int { return len(a) }

func (a SeriesByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a SeriesByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := getScorePb(oi)
	oj := a[j]
	scorej := getScorePb(oj)

	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.ImportName != oj.ImportName {
		return oi.ImportName < oj.ImportName
	}
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceUrl != oj.ProvenanceUrl {
		return oi.ProvenanceUrl < oj.ProvenanceUrl
	}
	return true
}

// TODO(shifucun): Remove `SourceSeries` and use pb.SourceSeries everywhere.
// getScore derives the ranking score for a source series.
//
// The score depends on ImportName and other SVObs properties, by checking the
// StatsRanking dict. To get the score, ImportName is required, with optional
// properties:
// - MeasurementMethod
// - ObservationPeriod
//
// When there are exact match of the properties in StatsRanking, then use that
// score, otherwise can also match to wildcard options (indicated by *).
//
// If no entry is found, a BaseRank is assigned to the source series.
func getScore(s *model.SourceSeries) int {
	for _, propCombination := range []struct {
		mm string
		op string
	}{
		// Check exact match first
		{s.MeasurementMethod, s.ObservationPeriod},
		{s.MeasurementMethod, "*"},
		{"*", s.ObservationPeriod},
		{"*", "*"},
	} {
		key := RankKey{
			ImportName:        s.ImportName,
			MeasurementMethod: propCombination.mm,
			ObservationPeriod: propCombination.op,
		}
		score, ok := StatsRanking[key]
		if ok {
			return score
		}
	}
	return BaseRank
}

// ByRank implements sort.Interface for []*SourceSeries based on
// the rank score.
type ByRank []*model.SourceSeries

func (a ByRank) Len() int { return len(a) }

func (a ByRank) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

func (a ByRank) Less(i, j int) bool {
	oi := a[i]
	scorei := getScore(oi)
	oj := a[j]
	scorej := getScore(oj)
	// Higher score value means lower rank.
	if scorei != scorej {
		return scorei < scorej
	}

	latesti := ""
	for date := range a[i].Val {
		if date > latesti {
			latesti = date
		}
	}

	latestj := ""
	for date := range a[j].Val {
		if date > latestj {
			latestj = date
		}
	}

	// Series with latest data is ranked higher
	if latesti != latestj {
		return latesti > latestj
	}

	// Series with more data is ranked higher
	if len(a[i].Val) != len(a[j].Val) {
		return len(a[i].Val) > len(a[j].Val)
	}

	// Compare other fields to get consistent ranking.
	if oi.MeasurementMethod != oj.MeasurementMethod {
		return oi.MeasurementMethod < oj.MeasurementMethod
	}
	if oi.ObservationPeriod != oj.ObservationPeriod {
		return oi.ObservationPeriod < oj.ObservationPeriod
	}
	if oi.ScalingFactor != oj.ScalingFactor {
		return oi.ScalingFactor < oj.ScalingFactor
	}
	if oi.Unit != oj.Unit {
		return oi.Unit < oj.Unit
	}
	if oi.ProvenanceURL != oj.ProvenanceURL {
		return oi.ProvenanceURL < oj.ProvenanceURL
	}
	return true
}
<|endoftext|>


<commit_before>package storage

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"time"

	"cloud.google.com/go/storage"
	"github.com/googleapis/gax-go/v2"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"google.golang.org/api/option"
)

type StorageHandle interface {
	BucketHandle(bucketName string) (bh *bucketHandle, err error)
}

type storageClient struct {
	client *storage.Client
}

type storageClientConfig struct {
	disableHTTP2        bool
	maxConnsPerHost     int
	maxIdleConnsPerHost int
	tokenSrc            oauth2.TokenSource
	timeOut             time.Duration
	maxRetryDuration    time.Duration
	retryMultiplier     float64
}

// NewStorageHandle returns the handle of Go storage client containing
// customized http client. We can configure the http client using the
// storageClientConfig parameter.
func NewStorageHandle(ctx context.Context,
	clientConfig storageClientConfig) (sh StorageHandle, err error) {
	var transport *http.Transport
	// Disabling the http2 makes the client more performant.
	if clientConfig.disableHTTP2 {
		transport = &http.Transport{
			MaxConnsPerHost:     clientConfig.maxConnsPerHost,
			MaxIdleConnsPerHost: clientConfig.maxIdleConnsPerHost,
			// This disables HTTP/2 in transport.
			TLSNextProto: make(
				map[string]func(string, *tls.Conn) http.RoundTripper,
			),
		}
	} else {
		// For http2, change in MaxConnsPerHost doesn't affect the performance.
		transport = &http.Transport{
			DisableKeepAlives: true,
			MaxConnsPerHost:   clientConfig.maxConnsPerHost,
			ForceAttemptHTTP2: true,
		}
	}

	// Custom http client for Go Client.
	httpClient := &http.Client{
		Transport: &oauth2.Transport{
			Base:   transport,
			Source: clientConfig.tokenSrc,
		},
		Timeout: clientConfig.timeOut,
	}

	var sc *storage.Client
	sc, err = storage.NewClient(ctx, option.WithHTTPClient(httpClient))
	if err != nil {
		err = fmt.Errorf("go storage client creation failed: %w", err)
		return
	}

	// RetryAlways causes all operations to be retried when the service returns a transient error, regardless of
	// idempotency considerations.
	sc.SetRetry(
		storage.WithBackoff(gax.Backoff{
			Max:        clientConfig.maxRetryDuration,
			Multiplier: clientConfig.retryMultiplier,
		}),
		storage.WithPolicy(storage.RetryAlways))

	client := &storageClient{client: sc}
	return client, err
}

func (sh *storageClient) BucketHandle(bucketName string) (bh *bucketHandle,
	err error) {
	storageBucketHandle := sh.client.Bucket(bucketName)
	_, err = storageBucketHandle.Attrs(context.Background())

	if err != nil {
		return
	}

	bh = &bucketHandle{bucket: storageBucketHandle}
	return
}
<commit_msg>Removing unused empty line<commit_after>package storage

import (
	"crypto/tls"
	"fmt"
	"net/http"
	"time"

	"cloud.google.com/go/storage"
	"github.com/googleapis/gax-go/v2"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"google.golang.org/api/option"
)

type StorageHandle interface {
	BucketHandle(bucketName string) (bh *bucketHandle, err error)
}

type storageClient struct {
	client *storage.Client
}

type storageClientConfig struct {
	disableHTTP2        bool
	maxConnsPerHost     int
	maxIdleConnsPerHost int
	tokenSrc            oauth2.TokenSource
	timeOut             time.Duration
	maxRetryDuration    time.Duration
	retryMultiplier     float64
}

// NewStorageHandle returns the handle of Go storage client containing
// customized http client. We can configure the http client using the
// storageClientConfig parameter.
func NewStorageHandle(ctx context.Context,
	clientConfig storageClientConfig) (sh StorageHandle, err error) {
	var transport *http.Transport
	// Disabling the http2 makes the client more performant.
	if clientConfig.disableHTTP2 {
		transport = &http.Transport{
			MaxConnsPerHost:     clientConfig.maxConnsPerHost,
			MaxIdleConnsPerHost: clientConfig.maxIdleConnsPerHost,
			// This disables HTTP/2 in transport.
			TLSNextProto: make(
				map[string]func(string, *tls.Conn) http.RoundTripper,
			),
		}
	} else {
		// For http2, change in MaxConnsPerHost doesn't affect the performance.
		transport = &http.Transport{
			DisableKeepAlives: true,
			MaxConnsPerHost:   clientConfig.maxConnsPerHost,
			ForceAttemptHTTP2: true,
		}
	}

	// Custom http client for Go Client.
	httpClient := &http.Client{
		Transport: &oauth2.Transport{
			Base:   transport,
			Source: clientConfig.tokenSrc,
		},
		Timeout: clientConfig.timeOut,
	}

	var sc *storage.Client
	sc, err = storage.NewClient(ctx, option.WithHTTPClient(httpClient))
	if err != nil {
		err = fmt.Errorf("go storage client creation failed: %w", err)
		return
	}

	// RetryAlways causes all operations to be retried when the service returns a transient error, regardless of
	// idempotency considerations.
	sc.SetRetry(
		storage.WithBackoff(gax.Backoff{
			Max:        clientConfig.maxRetryDuration,
			Multiplier: clientConfig.retryMultiplier,
		}),
		storage.WithPolicy(storage.RetryAlways))

	client := &storageClient{client: sc}
	return client, err
}

func (sh *storageClient) BucketHandle(bucketName string) (bh *bucketHandle,
	err error) {
	storageBucketHandle := sh.client.Bucket(bucketName)
	_, err = storageBucketHandle.Attrs(context.Background())
	if err != nil {
		return
	}

	bh = &bucketHandle{bucket: storageBucketHandle}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bitmm/bitfinex"
	"fmt"
	"github.com/grd/stat"
	"log"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.0025   // Minumum change required to update prices
	TRADENUM  = 25       // Number of trades to use in calculations
	MINO      = 0.5      // Min order size
	STDMULT   = 5        // Multiplier for standard deviation
)

var (
	api        = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))
	apiErrors  = false // Set to true on any error
	liveOrders = false // Set to true on any order
	orderTheo  = 0.0   // Theo value on which the live orders are based
	orderPos   = 0.0   // Position on which the live orders are based
	// Fed in as OS args:
	maxPos      float64 // Maximum Position size
	minEdge     float64 // Minimum edge for position entry
	exitPercent float64 // Percent of edge for position exit
)

func main() {
	if len(os.Args) < 4 {
		fmt.Printf("usage: %s <size> <minimum edge> <exit percent edge>\n", filepath.Base(os.Args[0]))
		os.Exit(1)
	}

	fmt.Println("\nInitializing...")

	// Get maxPos, minEdge, exitPercent from user input
	getArgs()

	// Check for input to break loop
	inputChan := make(chan rune)
	go checkStdin(inputChan)

	// Run loop until user input is received
	runMainLoop(inputChan)
}

func getArgs() {
	var err error
	if maxPos, err = strconv.ParseFloat(os.Args[1], 64); err != nil {
		log.Fatal(err)
	}
	if minEdge, err = strconv.ParseFloat(os.Args[2], 64); err != nil {
		log.Fatal(err)
	}
	if exitPercent, err = strconv.ParseFloat(os.Args[3], 64); err != nil {
		log.Fatal(err)
	}
}

// Check for any user input
func checkStdin(inputChan chan<- rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Infinite loop
func runMainLoop(inputChan <-chan rune) {
	positionChan := make(chan float64)

	var (
		trades    bitfinex.Trades
		orders    bitfinex.Orders
		start     time.Time
		position  float64
		theo      float64
		stdev     float64
		lastTrade int
	)

	for {
		// Record time for each iteration
		start = time.Now()

		// Cancel orders and exit if anything entered by user
		select {
		case <-inputChan:
			exit()
			return
		default: // Continue if nothing on chan
		}

		trades = getTrades()
		if !apiErrors && trades[0].TID != lastTrade { // If new trades
			go checkPosition(positionChan)
			// Do calcs on trade data while waiting for position data
			theo = calculateTheo(trades)
			stdev = calculateStdev(trades)
			position = <-positionChan
			if (math.Abs(theo-orderTheo) >= MINCHANGE || math.Abs(position-
				orderPos) >= MINO || !liveOrders) && !apiErrors {
				orders = sendOrders(theo, position, stdev)
			}
		}

		if !apiErrors {
			printResults(orders, position, stdev, theo, start)
			// Reset for next iteration
			lastTrade = trades[0].TID
		}

		// Reset for next iteration
		apiErrors = false
	}
}

// Send orders to the exchange
func sendOrders(theo, position, stdev float64) bitfinex.Orders {
	if liveOrders {
		cancelAll()
	}

	// Send new order request to the exchange
	params := calculateOrderParams(position, theo, stdev)
	orders, err := api.MultipleNewOrders(params)
	checkErr(err)
	if err == nil {
		liveOrders = true
		orderTheo = theo
		orderPos = position
	}
	return orders
}

func calculateOrderParams(position, theo, stdev float64) []bitfinex.OrderParams {
	var params []bitfinex.OrderParams

	if math.Abs(position) < MINO { // No position
		params = []bitfinex.OrderParams{
			{SYMBOL, maxPos, theo - math.Max(stdev, minEdge), "bitfinex", "buy", "limit"},
			{SYMBOL, maxPos, theo + math.Max(stdev, minEdge), "bitfinex", "sell", "limit"},
		}
	} else if position < (-1*maxPos)+MINO { // Max short postion
		params = []bitfinex.OrderParams{
			{SYMBOL, -1 * position, theo - stdev*exitPercent, "bitfinex", "buy", "limit"},
		}
	} else if position > maxPos-MINO { // Max long postion
		params = []bitfinex.OrderParams{
			{SYMBOL, position, theo + stdev*exitPercent, "bitfinex", "sell", "limit"},
		}
	} else if (-1*maxPos)+MINO <= position && position <= -1*MINO { // Partial short
		params = []bitfinex.OrderParams{
			{SYMBOL, maxPos, theo - math.Max(stdev, minEdge), "bitfinex", "buy", "limit"},
			{SYMBOL, -1 * position, theo - stdev*exitPercent, "bitfinex", "buy", "limit"},
			{SYMBOL, maxPos + position, theo + math.Max(stdev, minEdge), "bitfinex", "sell", "limit"},
		}
	} else if MINO <= position && position <= maxPos-MINO { // Partial long
		params = []bitfinex.OrderParams{
			{SYMBOL, maxPos - position, theo - math.Max(stdev, minEdge), "bitfinex", "buy", "limit"},
			{SYMBOL, position, theo + stdev*exitPercent, "bitfinex", "sell", "limit"},
			{SYMBOL, maxPos, theo + math.Max(stdev, minEdge), "bitfinex", "sell", "limit"},
		}
	}

	return params
}

func checkPosition(positionChan chan<- float64) {
	var position float64
	posSlice, err := api.ActivePositions()
	checkErr(err)
	for _, pos := range posSlice {
		if pos.Symbol == SYMBOL {
			position = pos.Amount
		}
	}

	positionChan <- position
}

// Get trade data
func getTrades() bitfinex.Trades {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	return trades
}

// Calculate a volume and time weighted average of traded prices
func calculateTheo(trades bitfinex.Trades) float64 {
	weightDuration := 60 // number of seconds back for a 50% weight relative to most recent
	mostRecent := trades[0].Timestamp
	var weight, timeDivisor, sum, weightTotal float64

	for _, trade := range trades {
		timeDivisor = float64(mostRecent - trade.Timestamp + weightDuration)
		weight = trade.Amount / timeDivisor
		sum += trade.Price * weight
		weightTotal += weight
	}

	return sum / weightTotal
}

func calculateStdev(trades bitfinex.Trades) float64 {
	x := make(stat.Float64Slice, len(trades)-1)
	for i := 1; i < len(trades); i++ {
		x[i-1] = trades[i-1].Price - trades[i].Price
	}
	return STDMULT * stat.Sd(x)
}

// Called on any error
func checkErr(err error) {
	if err != nil {
		cancelAll()
		apiErrors = true
	}
}

// Call on exit
func exit() {
	cancelAll()
	fmt.Println("\nCancelled all orders.")
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
	liveOrders = false
}

// Print results
func printResults(orders bitfinex.Orders, position, stdev, theo float64, start time.Time) {

	clearScreen()

	fmt.Printf("\nPosition: %.2f\n", position)
	fmt.Printf("Stdev:    %.4f\n", stdev)
	fmt.Printf("Theo:     %.4f\n", theo)

	fmt.Println("\nActive orders:")
	for _, order := range orders.Orders {
		fmt.Printf("%7.2f %s @ %6.4f\n", order.Amount, SYMBOL, order.Price)
	}

	fmt.Printf("\n%v processing time...", time.Since(start))
}

// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}
<commit_msg>make stdMult an os arg<commit_after>package main

import (
	"bitmm/bitfinex"
	"fmt"
	"github.com/grd/stat"
	"log"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"time"
)

// Trade inputs
const (
	SYMBOL    = "ltcusd" // Instrument to trade
	MINCHANGE = 0.0025   // Minumum change required to update prices
	TRADENUM  = 25       // Number of trades to use in calculations
	MINO      = 0.5      // Min order size
)

var (
	api        = bitfinex.New(os.Getenv("BITFINEX_KEY"), os.Getenv("BITFINEX_SECRET"))
	apiErrors  = false // Set to true on any error
	liveOrders = false // Set to true on any order
	orderTheo  = 0.0   // Theo value on which the live orders are based
	orderPos   = 0.0   // Position on which the live orders are based
	// Fed in as OS args:
	maxPos      float64 // Maximum Position size
	minEdge     float64 // Minimum edge for position entry
	stdMult     float64 // Multiplier for standard deviation
	exitPercent float64 // Percent of edge for position exit
)

func main() {
	if len(os.Args) < 5 {
		fmt.Printf("usage: %s <size> <minimum edge> <stdev multiplier> <exit percent edge>\n",
			filepath.Base(os.Args[0]))
		os.Exit(1)
	}

	fmt.Println("\nInitializing...")

	// Get maxPos, minEdge, exitPercent from user input
	getArgs()

	// Check for input to break loop
	inputChan := make(chan rune)
	go checkStdin(inputChan)

	// Run loop until user input is received
	runMainLoop(inputChan)
}

func getArgs() {
	var err error
	if maxPos, err = strconv.ParseFloat(os.Args[1], 64); err != nil {
		log.Fatal(err)
	}
	if minEdge, err = strconv.ParseFloat(os.Args[2], 64); err != nil {
		log.Fatal(err)
	}
	if stdMult, err = strconv.ParseFloat(os.Args[3], 64); err != nil {
		log.Fatal(err)
	}
	if exitPercent, err = strconv.ParseFloat(os.Args[4], 64); err != nil {
		log.Fatal(err)
	}
}

// Check for any user input
func checkStdin(inputChan chan<- rune) {
	var ch rune
	fmt.Scanf("%c", &ch)
	inputChan <- ch
}

// Infinite loop
func runMainLoop(inputChan <-chan rune) {
	positionChan := make(chan float64)

	var (
		trades    bitfinex.Trades
		orders    bitfinex.Orders
		start     time.Time
		position  float64
		theo      float64
		stdev     float64
		lastTrade int
	)

	for {
		// Record time for each iteration
		start = time.Now()

		// Cancel orders and exit if anything entered by user
		select {
		case <-inputChan:
			exit()
			return
		default: // Continue if nothing on chan
		}

		trades = getTrades()
		if !apiErrors && trades[0].TID != lastTrade { // If new trades
			go checkPosition(positionChan)
			// Do calcs on trade data while waiting for position data
			theo = calculateTheo(trades)
			stdev = calculateStdev(trades)
			position = <-positionChan
			if (math.Abs(theo-orderTheo) >= MINCHANGE || math.Abs(position-
				orderPos) >= MINO || !liveOrders) && !apiErrors {
				orders = sendOrders(theo, position, stdev)
			}
		}

		if !apiErrors {
			printResults(orders, position, stdev, theo, start)
			// Reset for next iteration
			lastTrade = trades[0].TID
		}

		// Reset for next iteration
		apiErrors = false
	}
}

// Send orders to the exchange
func sendOrders(theo, position, stdev float64) bitfinex.Orders {
	if liveOrders {
		cancelAll()
	}

	// Send new order request to the exchange
	params := calculateOrderParams(position, theo, stdev)
	orders, err := api.MultipleNewOrders(params)
	checkErr(err)
	if err == nil {
		liveOrders = true
		orderTheo = theo
		orderPos = position
	}
	return orders
}

func calculateOrderParams(position, theo, stdev float64) []bitfinex.OrderParams {
	var params []bitfinex.OrderParams

	if math.Abs(position) < MINO { // No position
		params = []bitfinex.OrderParams{
			{SYMBOL, maxPos, theo - math.Max(stdev, minEdge), "bitfinex", "buy", "limit"},
			{SYMBOL, maxPos, theo + math.Max(stdev, minEdge), "bitfinex", "sell", "limit"},
		}
	} else if position < (-1*maxPos)+MINO { // Max short postion
		params = []bitfinex.OrderParams{
			{SYMBOL, -1 * position, theo - stdev*exitPercent, "bitfinex", "buy", "limit"},
		}
	} else if position > maxPos-MINO { // Max long postion
		params = []bitfinex.OrderParams{
			{SYMBOL, position, theo + stdev*exitPercent, "bitfinex", "sell", "limit"},
		}
	} else if (-1*maxPos)+MINO <= position && position <= -1*MINO { // Partial short
		params = []bitfinex.OrderParams{
			{SYMBOL, maxPos, theo - math.Max(stdev, minEdge), "bitfinex", "buy", "limit"},
			{SYMBOL, -1 * position, theo - stdev*exitPercent, "bitfinex", "buy", "limit"},
			{SYMBOL, maxPos + position, theo + math.Max(stdev, minEdge), "bitfinex", "sell", "limit"},
		}
	} else if MINO <= position && position <= maxPos-MINO { // Partial long
		params = []bitfinex.OrderParams{
			{SYMBOL, maxPos - position, theo - math.Max(stdev, minEdge), "bitfinex", "buy", "limit"},
			{SYMBOL, position, theo + stdev*exitPercent, "bitfinex", "sell", "limit"},
			{SYMBOL, maxPos, theo + math.Max(stdev, minEdge), "bitfinex", "sell", "limit"},
		}
	}

	return params
}

func checkPosition(positionChan chan<- float64) {
	var position float64
	posSlice, err := api.ActivePositions()
	checkErr(err)
	for _, pos := range posSlice {
		if pos.Symbol == SYMBOL {
			position = pos.Amount
		}
	}

	positionChan <- position
}

// Get trade data
func getTrades() bitfinex.Trades {
	trades, err := api.Trades(SYMBOL, TRADENUM)
	checkErr(err)

	return trades
}

// Calculate a volume and time weighted average of traded prices
func calculateTheo(trades bitfinex.Trades) float64 {
	weightDuration := 60 // number of seconds back for a 50% weight relative to most recent
	mostRecent := trades[0].Timestamp
	var weight, timeDivisor, sum, weightTotal float64

	for _, trade := range trades {
		timeDivisor = float64(mostRecent - trade.Timestamp + weightDuration)
		weight = trade.Amount / timeDivisor
		sum += trade.Price * weight
		weightTotal += weight
	}

	return sum / weightTotal
}

func calculateStdev(trades bitfinex.Trades) float64 {
	x := make(stat.Float64Slice, len(trades)-1)
	for i := 1; i < len(trades); i++ {
		x[i-1] = trades[i-1].Price - trades[i].Price
	}
	return stdMult * stat.Sd(x)
}

// Called on any error
func checkErr(err error) {
	if err != nil {
		cancelAll()
		apiErrors = true
	}
}

// Call on exit
func exit() {
	cancelAll()
	fmt.Println("\nCancelled all orders.")
}

// Cancel all orders
func cancelAll() {
	cancelled := false
	for !cancelled {
		cancelled, _ = api.CancelAll()
	}
	liveOrders = false
}

// Print results
func printResults(orders bitfinex.Orders, position, stdev, theo float64, start time.Time) {

	clearScreen()

	fmt.Printf("\nPosition: %.2f\n", position)
	fmt.Printf("Stdev:    %.4f\n", stdev)
	fmt.Printf("Theo:     %.4f\n", theo)

	fmt.Println("\nActive orders:")
	for _, order := range orders.Orders {
		fmt.Printf("%7.2f %s @ %6.4f\n", order.Amount, SYMBOL, order.Price)
	}

	fmt.Printf("\n%v processing time...", time.Since(start))
}

// Clear the terminal between prints
func clearScreen() {
	c := exec.Command("clear")
	c.Stdout = os.Stdout
	c.Run()
}
<|endoftext|>


<commit_before>// Package npmblame captures useful informations and common errors from npm node_modules
package npmblame

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/gosuri/uitable"
)

// PackageError represents common npm packages errors
type PackageError int

const (
	// ExecError marks a package containing executables
	ExecError PackageError = iota
	// TestError marks a package test files
	TestError
	// BenchError marks a package benchmark files
	BenchError
	// ImageError marks a package images files
	ImageError
	// CIError marks a package containing continous integration files
	CIError
	// DotfileError marks a package lint files
	DotfileError
)

// NpmPackage represents a npm package
type NpmPackage struct {
	BugsURL  string
	Homepage string
	Errors   map[int]int
}

// NpmPackages is a map of all the packages and there given errors
type NpmPackages map[string]map[PackageError]int

// NewNpmPackages returns a new npm package instance
func NewNpmPackages() NpmPackages {
	return make(NpmPackages)
}

// ExtractPackageName returns the npm package name from a given path
// TODO improve to handle nested dependencies
func (np NpmPackages) ExtractPackageName(path string) string {
	dir := filepath.Dir(path)
	//fmt.Println("HAS_NODEMODULES", strings.Contains(dir, "node_modules"), strings.LastIndex(dir, "node_modules"))
	subPkgIndex := strings.LastIndex(dir, "node_modules")
	var module []string
	if subPkgIndex != -1 {
		module = strings.Split(dir[subPkgIndex:], "/")
		if len(module) > 1 {
			return module[1]
		}
	}
	module = strings.Split(dir, "/")
	if len(module[0]) > 0 {
		return module[0]
	}
	return module[1]
}

//ExtractPackageInformations extracts the package information from its package.json
func (np NpmPackages) ExtractPackageInformations() error {
	return nil
}

// AppendError appends an error to a given package
func (np NpmPackages) AppendError(pkgName string, err PackageError) {
	if len(np[pkgName]) == 0 {
		np[pkgName] = make(map[PackageError]int)
	}
	np[pkgName][err] = np[pkgName][err] + 1
}

func (np NpmPackages) checkTests(path string, pkg string) {
	if strings.Contains(path, "test") ||
		strings.Contains(path, "tests") ||
		strings.Contains(path, ".zuul.yml") ||
		strings.Contains(path, "coverage") ||
		strings.Contains(path, ".coveralls.yml") {
		np.AppendError(pkg, TestError)
	}
}

func (np NpmPackages) checkDotFiles(path string, pkg string) {
	if strings.Contains(path, ".editorconfig") ||
		strings.Contains(path, ".eslintrc") ||
		strings.Contains(path, ".sass-lint.yml") ||
		strings.Contains(path, ".jshintrc") {
		np.AppendError(pkg, DotfileError)
	}
}

func (np NpmPackages) checkExecutables(info os.FileInfo, pkg string) {
	if !info.Mode().IsDir() && (info.Mode()&0111) != 0 {
		np.AppendError(pkg, ExecError)
	}
}

func (np NpmPackages) checkImages(path string, pkg string) {
	if filepath.Ext(path) == ".png" ||
		filepath.Ext(path) == ".jpg" ||
		filepath.Ext(path) == ".ico" {
		np.AppendError(pkg, ImageError)
	}
}

// Blame reports on error for a given npm package
func (np NpmPackages) Blame(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	pkg := np.ExtractPackageName(path)
	if pkg == "" || pkg == ".bin" {
		return nil
	}
	if np[pkg] == nil {

		np[pkg] = make(map[PackageError]int)
	}

	np.checkExecutables(info, pkg)
	np.checkTests(path, pkg)
	np.checkDotFiles(path, pkg)
	np.checkImages(path, pkg)

	if strings.Contains(path, "bench") {
		np.AppendError(pkg, BenchError)
	}

	if strings.Contains(path, ".travis.yml") {
		np.AppendError(pkg, CIError)
	}

	return nil
}

// TotalErrors return the total amount of errors
func (np NpmPackages) TotalErrors(pkgName string) int {
	totalErrors := 0
	for _, err := range np[pkgName] {
		totalErrors += err
	}
	return totalErrors
}

// String returns the printalbe representation of the NpmPackages
func (np NpmPackages) String() string {
	buf := &bytes.Buffer{}
	var totalErr int

	var keys []string
	for k := range np {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	table := uitable.New()
	table.MaxColWidth = 50

	table.AddRow("PACKAGE", "ERRORS", "EXECUTABLE FILE", "TESTS", "BENCH",
		"IMAGES", "TRAVIS_FILES", "EDITOR_LINT_FILES")
	for _, name := range keys {
		errors := np[name]

		if len(errors) > 0 {
			pkgErr := np.TotalErrors(name)
			totalErr++
			table.AddRow(name, pkgErr, errors[ExecError], errors[TestError],
				errors[BenchError], errors[ImageError], errors[CIError],
				errors[DotfileError])
		}
	}

	fmt.Fprintf(buf, "Your node_modules contains %d packages with errors out of %d packages\n\n", totalErr, len(np))
	fmt.Fprintln(buf, table)
	return buf.String()
}
<commit_msg>Fix misspell<commit_after>// Package npmblame captures useful informations and common errors from npm node_modules
package npmblame

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/gosuri/uitable"
)

// PackageError represents common npm packages errors
type PackageError int

const (
	// ExecError marks a package containing executables
	ExecError PackageError = iota
	// TestError marks a package test files
	TestError
	// BenchError marks a package benchmark files
	BenchError
	// ImageError marks a package images files
	ImageError
	// CIError marks a package containing continuous integration files
	CIError
	// DotfileError marks a package lint files
	DotfileError
)

// NpmPackage represents a npm package
type NpmPackage struct {
	BugsURL  string
	Homepage string
	Errors   map[int]int
}

// NpmPackages is a map of all the packages and there given errors
type NpmPackages map[string]map[PackageError]int

// NewNpmPackages returns a new npm package instance
func NewNpmPackages() NpmPackages {
	return make(NpmPackages)
}

// ExtractPackageName returns the npm package name from a given path
// TODO improve to handle nested dependencies
func (np NpmPackages) ExtractPackageName(path string) string {
	dir := filepath.Dir(path)
	//fmt.Println("HAS_NODEMODULES", strings.Contains(dir, "node_modules"), strings.LastIndex(dir, "node_modules"))
	subPkgIndex := strings.LastIndex(dir, "node_modules")
	var module []string
	if subPkgIndex != -1 {
		module = strings.Split(dir[subPkgIndex:], "/")
		if len(module) > 1 {
			return module[1]
		}
	}
	module = strings.Split(dir, "/")
	if len(module[0]) > 0 {
		return module[0]
	}
	return module[1]
}

//ExtractPackageInformations extracts the package information from its package.json
func (np NpmPackages) ExtractPackageInformations() error {
	return nil
}

// AppendError appends an error to a given package
func (np NpmPackages) AppendError(pkgName string, err PackageError) {
	if len(np[pkgName]) == 0 {
		np[pkgName] = make(map[PackageError]int)
	}
	np[pkgName][err] = np[pkgName][err] + 1
}

func (np NpmPackages) checkTests(path string, pkg string) {
	if strings.Contains(path, "test") ||
		strings.Contains(path, "tests") ||
		strings.Contains(path, ".zuul.yml") ||
		strings.Contains(path, "coverage") ||
		strings.Contains(path, ".coveralls.yml") {
		np.AppendError(pkg, TestError)
	}
}

func (np NpmPackages) checkDotFiles(path string, pkg string) {
	if strings.Contains(path, ".editorconfig") ||
		strings.Contains(path, ".eslintrc") ||
		strings.Contains(path, ".sass-lint.yml") ||
		strings.Contains(path, ".jshintrc") {
		np.AppendError(pkg, DotfileError)
	}
}

func (np NpmPackages) checkExecutables(info os.FileInfo, pkg string) {
	if !info.Mode().IsDir() && (info.Mode()&0111) != 0 {
		np.AppendError(pkg, ExecError)
	}
}

func (np NpmPackages) checkImages(path string, pkg string) {
	if filepath.Ext(path) == ".png" ||
		filepath.Ext(path) == ".jpg" ||
		filepath.Ext(path) == ".ico" {
		np.AppendError(pkg, ImageError)
	}
}

// Blame reports on error for a given npm package
func (np NpmPackages) Blame(path string, info os.FileInfo, err error) error {
	if err != nil {
		return err
	}

	pkg := np.ExtractPackageName(path)
	if pkg == "" || pkg == ".bin" {
		return nil
	}
	if np[pkg] == nil {

		np[pkg] = make(map[PackageError]int)
	}

	np.checkExecutables(info, pkg)
	np.checkTests(path, pkg)
	np.checkDotFiles(path, pkg)
	np.checkImages(path, pkg)

	if strings.Contains(path, "bench") {
		np.AppendError(pkg, BenchError)
	}

	if strings.Contains(path, ".travis.yml") {
		np.AppendError(pkg, CIError)
	}

	return nil
}

// TotalErrors return the total amount of errors
func (np NpmPackages) TotalErrors(pkgName string) int {
	totalErrors := 0
	for _, err := range np[pkgName] {
		totalErrors += err
	}
	return totalErrors
}

// String returns the printalbe representation of the NpmPackages
func (np NpmPackages) String() string {
	buf := &bytes.Buffer{}
	var totalErr int

	var keys []string
	for k := range np {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	table := uitable.New()
	table.MaxColWidth = 50

	table.AddRow("PACKAGE", "ERRORS", "EXECUTABLE FILE", "TESTS", "BENCH",
		"IMAGES", "TRAVIS_FILES", "EDITOR_LINT_FILES")
	for _, name := range keys {
		errors := np[name]

		if len(errors) > 0 {
			pkgErr := np.TotalErrors(name)
			totalErr++
			table.AddRow(name, pkgErr, errors[ExecError], errors[TestError],
				errors[BenchError], errors[ImageError], errors[CIError],
				errors[DotfileError])
		}
	}

	fmt.Fprintf(buf, "Your node_modules contains %d packages with errors out of %d packages\n\n", totalErr, len(np))
	fmt.Fprintln(buf, table)
	return buf.String()
}
<|endoftext|>


<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zap

import (
	"bytes"
	"fmt"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/index/scorch/segment"
	"github.com/couchbase/vellum"
	"github.com/couchbase/vellum/levenshtein"
	"github.com/couchbase/vellum/regexp"
)

// Dictionary is the zap representation of the term dictionary
type Dictionary struct {
	sb      *SegmentBase
	field   string
	fieldID uint16
	fst     *vellum.FST
}

// PostingsList returns the postings list for the specified term
func (d *Dictionary) PostingsList(term string, except *roaring.Bitmap) (segment.PostingsList, error) {
	return d.postingsList([]byte(term), except, nil)
}

func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	if d.fst == nil {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	postingsOffset, exists, err := d.fst.Get(term)
	if err != nil {
		return nil, fmt.Errorf("vellum err: %v", err)
	}
	if !exists {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	return d.postingsListFromOffset(postingsOffset, except, rv)
}

func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	rv = d.postingsListInit(rv, except)

	err := rv.read(postingsOffset, d)
	if err != nil {
		return nil, err
	}

	return rv, nil
}

func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList {
	if rv == nil || rv == emptyPostingsList {
		rv = &PostingsList{}
	} else {
		postings := rv.postings
		if postings != nil {
			postings.Clear()
		}

		*rv = PostingsList{} // clear the struct

		rv.postings = postings
	}
	rv.sb = d.sb
	rv.except = except
	return rv
}

// Iterator returns an iterator for this dictionary
func (d *Dictionary) Iterator() segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator(nil, nil)
		if err == nil {
			rv.itr = itr
		}
	}

	return rv
}

// PrefixIterator returns an iterator which only visits terms having the
// the specified prefix
func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		r, err := regexp.New(prefix + ".*")
		if err == nil {
			itr, err := d.fst.Search(r, nil, nil)
			if err == nil {
				rv.itr = itr
			}
		}
	}

	return rv
}

// RangeIterator returns an iterator which only visits terms between the
// start and end terms.  NOTE: bleve.index API specifies the end is inclusive.
func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	// need to increment the end position to be inclusive
	endBytes := []byte(end)
	if endBytes[len(endBytes)-1] < 0xff {
		endBytes[len(endBytes)-1]++
	} else {
		endBytes = append(endBytes, 0xff)
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator([]byte(start), endBytes)
		if err == nil {
			rv.itr = itr
		}
	}

	return rv
}

// RegexIterator returns an iterator which only visits terms having the
// the specified regex
func (d *Dictionary) RegexpIterator(regex string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		r, err := regexp.New(regex)
		if err == nil {
			itr, err := d.fst.Search(r, nil, nil)
			if err == nil {
				rv.itr = itr
			}
		}
	}

	return rv
}

// FuzzyIterator returns an iterator which only visits terms having the
// the specified edit/levenshtein distance
func (d *Dictionary) FuzzyIterator(term string,
	fuzziness int) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		la, err := levenshtein.New(term, fuzziness)
		if err == nil {
			itr, err := d.fst.Search(la, nil, nil)
			if err == nil {
				rv.itr = itr
			}
		}
	}

	return rv
}

func (d *Dictionary) OnlyIterator(onlyTerms [][]byte,
	includeCount bool) segment.DictionaryIterator {

	rv := &DictionaryIterator{
		d:         d,
		omitCount: !includeCount,
	}

	var buf bytes.Buffer
	builder, err := vellum.New(&buf, nil)
	if err != nil {
		rv.err = err
		return rv
	}
	for _, term := range onlyTerms {
		err = builder.Insert(term, 0)
		if err != nil {
			rv.err = err
			return rv
		}
	}
	err = builder.Close()
	if err != nil {
		rv.err = err
		return rv
	}

	onlyFST, err := vellum.Load(buf.Bytes())
	if err != nil {
		rv.err = err
		return rv
	}

	itr, err := d.fst.Search(onlyFST, nil, nil)
	if err == nil {
		rv.itr = itr
	}

	return rv
}

// DictionaryIterator is an iterator for term dictionary
type DictionaryIterator struct {
	d         *Dictionary
	itr       vellum.Iterator
	err       error
	tmp       PostingsList
	entry     index.DictEntry
	omitCount bool
}

// Next returns the next entry in the dictionary
func (i *DictionaryIterator) Next() (*index.DictEntry, error) {
	if i.itr == nil || i.err == vellum.ErrIteratorDone {
		return nil, nil
	} else if i.err != nil {
		return nil, i.err
	}
	term, postingsOffset := i.itr.Current()
	i.entry.Term = string(term)
	if !i.omitCount {
		i.err = i.tmp.read(postingsOffset, i.d)
		if i.err != nil {
			return nil, i.err
		}
		i.entry.Count = i.tmp.Count()
	}
	i.err = i.itr.Next()
	return &i.entry, nil
}
<commit_msg>make error handing inside iterator creation mroe consistent<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package zap

import (
	"bytes"
	"fmt"

	"github.com/RoaringBitmap/roaring"
	"github.com/blevesearch/bleve/index"
	"github.com/blevesearch/bleve/index/scorch/segment"
	"github.com/couchbase/vellum"
	"github.com/couchbase/vellum/levenshtein"
	"github.com/couchbase/vellum/regexp"
)

// Dictionary is the zap representation of the term dictionary
type Dictionary struct {
	sb      *SegmentBase
	field   string
	fieldID uint16
	fst     *vellum.FST
}

// PostingsList returns the postings list for the specified term
func (d *Dictionary) PostingsList(term string, except *roaring.Bitmap) (segment.PostingsList, error) {
	return d.postingsList([]byte(term), except, nil)
}

func (d *Dictionary) postingsList(term []byte, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	if d.fst == nil {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	postingsOffset, exists, err := d.fst.Get(term)
	if err != nil {
		return nil, fmt.Errorf("vellum err: %v", err)
	}
	if !exists {
		if rv == nil || rv == emptyPostingsList {
			return emptyPostingsList, nil
		}
		return d.postingsListInit(rv, except), nil
	}

	return d.postingsListFromOffset(postingsOffset, except, rv)
}

func (d *Dictionary) postingsListFromOffset(postingsOffset uint64, except *roaring.Bitmap, rv *PostingsList) (*PostingsList, error) {
	rv = d.postingsListInit(rv, except)

	err := rv.read(postingsOffset, d)
	if err != nil {
		return nil, err
	}

	return rv, nil
}

func (d *Dictionary) postingsListInit(rv *PostingsList, except *roaring.Bitmap) *PostingsList {
	if rv == nil || rv == emptyPostingsList {
		rv = &PostingsList{}
	} else {
		postings := rv.postings
		if postings != nil {
			postings.Clear()
		}

		*rv = PostingsList{} // clear the struct

		rv.postings = postings
	}
	rv.sb = d.sb
	rv.except = except
	return rv
}

// Iterator returns an iterator for this dictionary
func (d *Dictionary) Iterator() segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator(nil, nil)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// PrefixIterator returns an iterator which only visits terms having the
// the specified prefix
func (d *Dictionary) PrefixIterator(prefix string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		r, err := regexp.New(prefix + ".*")
		if err == nil {
			itr, err := d.fst.Search(r, nil, nil)
			if err == nil {
				rv.itr = itr
			} else if err != nil && err != vellum.ErrIteratorDone {
				rv.err = err
			}
		} else {
			rv.err = err
		}
	}

	return rv
}

// RangeIterator returns an iterator which only visits terms between the
// start and end terms.  NOTE: bleve.index API specifies the end is inclusive.
func (d *Dictionary) RangeIterator(start, end string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	// need to increment the end position to be inclusive
	endBytes := []byte(end)
	if endBytes[len(endBytes)-1] < 0xff {
		endBytes[len(endBytes)-1]++
	} else {
		endBytes = append(endBytes, 0xff)
	}

	if d.fst != nil {
		itr, err := d.fst.Iterator([]byte(start), endBytes)
		if err == nil {
			rv.itr = itr
		} else if err != nil && err != vellum.ErrIteratorDone {
			rv.err = err
		}
	}

	return rv
}

// RegexpIterator returns an iterator which only visits terms having the
// the specified regex
func (d *Dictionary) RegexpIterator(regex string) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		r, err := regexp.New(regex)
		if err == nil {
			itr, err2 := d.fst.Search(r, nil, nil)
			if err2 == nil {
				rv.itr = itr
			} else if err2 != nil && err2 != vellum.ErrIteratorDone {
				rv.err = err2
			}
		} else {
			rv.err = err
		}
	}

	return rv
}

// FuzzyIterator returns an iterator which only visits terms having the
// the specified edit/levenshtein distance
func (d *Dictionary) FuzzyIterator(term string,
	fuzziness int) segment.DictionaryIterator {
	rv := &DictionaryIterator{
		d: d,
	}

	if d.fst != nil {
		la, err := levenshtein.New(term, fuzziness)
		if err == nil {
			itr, err2 := d.fst.Search(la, nil, nil)
			if err2 == nil {
				rv.itr = itr
			} else if err2 != nil && err2 != vellum.ErrIteratorDone {
				rv.err = err2
			}
		} else {
			rv.err = err
		}
	}

	return rv
}

func (d *Dictionary) OnlyIterator(onlyTerms [][]byte,
	includeCount bool) segment.DictionaryIterator {

	rv := &DictionaryIterator{
		d:         d,
		omitCount: !includeCount,
	}

	var buf bytes.Buffer
	builder, err := vellum.New(&buf, nil)
	if err != nil {
		rv.err = err
		return rv
	}
	for _, term := range onlyTerms {
		err = builder.Insert(term, 0)
		if err != nil {
			rv.err = err
			return rv
		}
	}
	err = builder.Close()
	if err != nil {
		rv.err = err
		return rv
	}

	onlyFST, err := vellum.Load(buf.Bytes())
	if err != nil {
		rv.err = err
		return rv
	}

	itr, err := d.fst.Search(onlyFST, nil, nil)
	if err == nil {
		rv.itr = itr
	} else if err != nil && err != vellum.ErrIteratorDone {
		rv.err = err
	}

	return rv
}

// DictionaryIterator is an iterator for term dictionary
type DictionaryIterator struct {
	d         *Dictionary
	itr       vellum.Iterator
	err       error
	tmp       PostingsList
	entry     index.DictEntry
	omitCount bool
}

// Next returns the next entry in the dictionary
func (i *DictionaryIterator) Next() (*index.DictEntry, error) {
	if i.itr == nil || i.err == vellum.ErrIteratorDone {
		return nil, nil
	} else if i.err != nil {
		return nil, i.err
	}
	term, postingsOffset := i.itr.Current()
	i.entry.Term = string(term)
	if !i.omitCount {
		i.err = i.tmp.read(postingsOffset, i.d)
		if i.err != nil {
			return nil, i.err
		}
		i.entry.Count = i.tmp.Count()
	}
	i.err = i.itr.Next()
	return &i.entry, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"strconv"
	"strings"
	"time"

	"github.com/docker/docker/api"
	"github.com/docker/docker/pkg/integration/checker"
	icmd "github.com/docker/docker/pkg/integration/cmd"
	"github.com/go-check/check"
)

func (s *DockerSuite) TestAPIOptionsRoute(c *check.C) {
	status, _, err := sockRequest("OPTIONS", "/", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusOK)
}

func (s *DockerSuite) TestAPIGetEnabledCORS(c *check.C) {
	res, body, err := sockRequestRaw("GET", "/version", nil, "")
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
	body.Close()
	// TODO: @runcom incomplete tests, why old integration tests had this headers
	// and here none of the headers below are in the response?
	//c.Log(res.Header)
	//c.Assert(res.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
	//c.Assert(res.Header.Get("Access-Control-Allow-Headers"), check.Equals, "Origin, X-Requested-With, Content-Type, Accept, X-Registry-Auth")
}

func (s *DockerSuite) TestAPIVersionStatusCode(c *check.C) {
	conn, err := sockConn(time.Duration(10*time.Second), "")
	c.Assert(err, checker.IsNil)

	client := httputil.NewClientConn(conn, nil)
	defer client.Close()

	req, err := http.NewRequest("GET", "/v999.0/version", nil)
	c.Assert(err, checker.IsNil)
	req.Header.Set("User-Agent", "Docker-Client/999.0 (os)")

	res, err := client.Do(req)
	c.Assert(res.StatusCode, checker.Equals, http.StatusBadRequest)
}

func (s *DockerSuite) TestAPIClientVersionNewerThanServer(c *check.C) {
	v := strings.Split(api.DefaultVersion, ".")
	vMinInt, err := strconv.Atoi(v[1])
	c.Assert(err, checker.IsNil)
	vMinInt++
	v[1] = strconv.Itoa(vMinInt)
	version := strings.Join(v, ".")

	status, body, err := sockRequest("GET", "/v"+version+"/version", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusBadRequest)
	expected := fmt.Sprintf("client is newer than server (client API version: %s, server API version: %s)", version, api.DefaultVersion)
	c.Assert(getErrorMessage(c, body), checker.Equals, expected)
}

func (s *DockerSuite) TestAPIClientVersionOldNotSupported(c *check.C) {
	v := strings.Split(api.MinVersion, ".")
	vMinInt, err := strconv.Atoi(v[1])
	c.Assert(err, checker.IsNil)
	vMinInt--
	v[1] = strconv.Itoa(vMinInt)
	version := strings.Join(v, ".")

	status, body, err := sockRequest("GET", "/v"+version+"/version", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusBadRequest)
	expected := fmt.Sprintf("client version %s is too old. Minimum supported API version is %s, please upgrade your client to a newer version", version, api.MinVersion)
	c.Assert(strings.TrimSpace(string(body)), checker.Contains, expected)
}

func (s *DockerSuite) TestAPIDockerAPIVersion(c *check.C) {
	var svrVersion string

	server := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			url := r.URL.Path
			svrVersion = url
		}))
	defer server.Close()

	// Test using the env var first
	result := icmd.RunCmd(icmd.Cmd{
		Command: binaryWithArgs("-H="+server.URL[7:], "version"),
		Env:     appendBaseEnv(false, "DOCKER_API_VERSION=xxx"),
	})
	c.Assert(result, icmd.Matches, icmd.Expected{Out: "API version:  xxx", ExitCode: 1})
	c.Assert(svrVersion, check.Equals, "/vxxx/version", check.Commentf("%s", result.Compare(icmd.Success)))
}

func (s *DockerSuite) TestAPIErrorJSON(c *check.C) {
	httpResp, body, err := sockRequestRaw("POST", "/containers/create", strings.NewReader(`{}`), "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusInternalServerError)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Equals, "application/json")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(getErrorMessage(c, b), checker.Equals, "Config cannot be empty in order to create a container")
}

func (s *DockerSuite) TestAPIErrorPlainText(c *check.C) {
	// Windows requires API 1.25 or later. This test is validating a behaviour which was present
	// in v1.23, but changed in 1.24, hence not applicable on Windows. See apiVersionSupportsJSONErrors
	testRequires(c, DaemonIsLinux)
	httpResp, body, err := sockRequestRaw("POST", "/v1.23/containers/create", strings.NewReader(`{}`), "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusInternalServerError)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Contains, "text/plain")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(strings.TrimSpace(string(b)), checker.Equals, "Config cannot be empty in order to create a container")
}

func (s *DockerSuite) TestAPIErrorNotFoundJSON(c *check.C) {
	// 404 is a different code path to normal errors, so test separately
	httpResp, body, err := sockRequestRaw("GET", "/notfound", nil, "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusNotFound)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Equals, "application/json")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(getErrorMessage(c, b), checker.Equals, "page not found")
}

func (s *DockerSuite) TestAPIErrorNotFoundPlainText(c *check.C) {
	httpResp, body, err := sockRequestRaw("GET", "/v1.23/notfound", nil, "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusNotFound)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Contains, "text/plain")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(strings.TrimSpace(string(b)), checker.Equals, "page not found")
}
<commit_msg>Fix TestAPIClientVersionOldNotSupported<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/docker/docker/api"
	"github.com/docker/docker/pkg/integration/checker"
	icmd "github.com/docker/docker/pkg/integration/cmd"
	"github.com/go-check/check"
)

func (s *DockerSuite) TestAPIOptionsRoute(c *check.C) {
	status, _, err := sockRequest("OPTIONS", "/", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusOK)
}

func (s *DockerSuite) TestAPIGetEnabledCORS(c *check.C) {
	res, body, err := sockRequestRaw("GET", "/version", nil, "")
	c.Assert(err, checker.IsNil)
	c.Assert(res.StatusCode, checker.Equals, http.StatusOK)
	body.Close()
	// TODO: @runcom incomplete tests, why old integration tests had this headers
	// and here none of the headers below are in the response?
	//c.Log(res.Header)
	//c.Assert(res.Header.Get("Access-Control-Allow-Origin"), check.Equals, "*")
	//c.Assert(res.Header.Get("Access-Control-Allow-Headers"), check.Equals, "Origin, X-Requested-With, Content-Type, Accept, X-Registry-Auth")
}

func (s *DockerSuite) TestAPIVersionStatusCode(c *check.C) {
	conn, err := sockConn(time.Duration(10*time.Second), "")
	c.Assert(err, checker.IsNil)

	client := httputil.NewClientConn(conn, nil)
	defer client.Close()

	req, err := http.NewRequest("GET", "/v999.0/version", nil)
	c.Assert(err, checker.IsNil)
	req.Header.Set("User-Agent", "Docker-Client/999.0 (os)")

	res, err := client.Do(req)
	c.Assert(res.StatusCode, checker.Equals, http.StatusBadRequest)
}

func (s *DockerSuite) TestAPIClientVersionNewerThanServer(c *check.C) {
	v := strings.Split(api.DefaultVersion, ".")
	vMinInt, err := strconv.Atoi(v[1])
	c.Assert(err, checker.IsNil)
	vMinInt++
	v[1] = strconv.Itoa(vMinInt)
	version := strings.Join(v, ".")

	status, body, err := sockRequest("GET", "/v"+version+"/version", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusBadRequest)
	expected := fmt.Sprintf("client is newer than server (client API version: %s, server API version: %s)", version, api.DefaultVersion)
	c.Assert(getErrorMessage(c, body), checker.Equals, expected)
}

func (s *DockerSuite) TestAPIClientVersionOldNotSupported(c *check.C) {
	if daemonPlatform != runtime.GOOS {
		c.Skip("Daemon platform doesn't match test platform")
	}
	if api.MinVersion == api.DefaultVersion {
		c.Skip("API MinVersion==DefaultVersion")
	}
	v := strings.Split(api.MinVersion, ".")
	vMinInt, err := strconv.Atoi(v[1])
	c.Assert(err, checker.IsNil)
	vMinInt--
	v[1] = strconv.Itoa(vMinInt)
	version := strings.Join(v, ".")

	status, body, err := sockRequest("GET", "/v"+version+"/version", nil)
	c.Assert(err, checker.IsNil)
	c.Assert(status, checker.Equals, http.StatusBadRequest)
	expected := fmt.Sprintf("client version %s is too old. Minimum supported API version is %s, please upgrade your client to a newer version", version, api.MinVersion)
	c.Assert(strings.TrimSpace(string(body)), checker.Contains, expected)
}

func (s *DockerSuite) TestAPIDockerAPIVersion(c *check.C) {
	var svrVersion string

	server := httptest.NewServer(http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			url := r.URL.Path
			svrVersion = url
		}))
	defer server.Close()

	// Test using the env var first
	result := icmd.RunCmd(icmd.Cmd{
		Command: binaryWithArgs("-H="+server.URL[7:], "version"),
		Env:     appendBaseEnv(false, "DOCKER_API_VERSION=xxx"),
	})
	c.Assert(result, icmd.Matches, icmd.Expected{Out: "API version:  xxx", ExitCode: 1})
	c.Assert(svrVersion, check.Equals, "/vxxx/version", check.Commentf("%s", result.Compare(icmd.Success)))
}

func (s *DockerSuite) TestAPIErrorJSON(c *check.C) {
	httpResp, body, err := sockRequestRaw("POST", "/containers/create", strings.NewReader(`{}`), "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusInternalServerError)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Equals, "application/json")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(getErrorMessage(c, b), checker.Equals, "Config cannot be empty in order to create a container")
}

func (s *DockerSuite) TestAPIErrorPlainText(c *check.C) {
	// Windows requires API 1.25 or later. This test is validating a behaviour which was present
	// in v1.23, but changed in 1.24, hence not applicable on Windows. See apiVersionSupportsJSONErrors
	testRequires(c, DaemonIsLinux)
	httpResp, body, err := sockRequestRaw("POST", "/v1.23/containers/create", strings.NewReader(`{}`), "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusInternalServerError)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Contains, "text/plain")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(strings.TrimSpace(string(b)), checker.Equals, "Config cannot be empty in order to create a container")
}

func (s *DockerSuite) TestAPIErrorNotFoundJSON(c *check.C) {
	// 404 is a different code path to normal errors, so test separately
	httpResp, body, err := sockRequestRaw("GET", "/notfound", nil, "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusNotFound)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Equals, "application/json")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(getErrorMessage(c, b), checker.Equals, "page not found")
}

func (s *DockerSuite) TestAPIErrorNotFoundPlainText(c *check.C) {
	httpResp, body, err := sockRequestRaw("GET", "/v1.23/notfound", nil, "application/json")
	c.Assert(err, checker.IsNil)
	c.Assert(httpResp.StatusCode, checker.Equals, http.StatusNotFound)
	c.Assert(httpResp.Header.Get("Content-Type"), checker.Contains, "text/plain")
	b, err := readBody(body)
	c.Assert(err, checker.IsNil)
	c.Assert(strings.TrimSpace(string(b)), checker.Equals, "page not found")
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	. "github.com/clearcontainers/tests"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("restart", func() {
	var (
		args []string
		id   string
	)

	BeforeEach(func() {
		id = randomDockerName()
		args = []string{"run", "-td", "--name", id, Image, "sh"}
		runDockerCommand(0, args...)
	})

	AfterEach(func() {
		Expect(RemoveDockerContainer(id)).To(BeTrue())
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
	})

	Describe("restart with docker", func() {
		Context("restart a container", func() {
			It("should be running", func() {
				Expect(StopDockerContainer(id)).To(BeTrue())
				Expect(IsRunningDockerContainer(id)).To(BeFalse())
				args = []string{"restart", id}
				runDockerCommand(0, args...)
				Expect(IsRunningDockerContainer(id)).To(BeTrue())
			})
		})
	})
})
<commit_msg>tests: add DockerRun function to restart test<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package docker

import (
	. "github.com/clearcontainers/tests"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("restart", func() {
	var (
		args []string
		id   string
	)

	BeforeEach(func() {
		id = randomDockerName()
		_, _, exitCode := DockerRun("-td", "--name", id, Image, "sh")
		Expect(exitCode).To(Equal(0))
	})

	AfterEach(func() {
		Expect(RemoveDockerContainer(id)).To(BeTrue())
		Expect(ExistDockerContainer(id)).NotTo(BeTrue())
	})

	Describe("restart with docker", func() {
		Context("restart a container", func() {
			It("should be running", func() {
				Expect(StopDockerContainer(id)).To(BeTrue())
				Expect(IsRunningDockerContainer(id)).To(BeFalse())
				args = []string{"restart", id}
				runDockerCommand(0, args...)
				Expect(IsRunningDockerContainer(id)).To(BeTrue())
			})
		})
	})
})
<|endoftext|>


<commit_before>// +build linux

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package integration

import (
	"fmt"
	"os/exec"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
)

func TestVolumeCopyUp(t *testing.T) {
	const (
		testImage   = "gcr.io/k8s-cri-containerd/volume-copy-up:1.0"
		execTimeout = time.Minute
	)

	t.Logf("Create a sandbox")
	sbConfig := PodSandboxConfig("sandbox", "volume-copy-up")
	sb, err := runtimeService.RunPodSandbox(sbConfig, *runtimeHandler)
	require.NoError(t, err)
	defer func() {
		assert.NoError(t, runtimeService.StopPodSandbox(sb))
		assert.NoError(t, runtimeService.RemovePodSandbox(sb))
	}()

	t.Logf("Pull test image")
	_, err = imageService.PullImage(&runtime.ImageSpec{Image: testImage}, nil, sbConfig)
	require.NoError(t, err)

	t.Logf("Create a container with volume-copy-up test image")
	cnConfig := ContainerConfig(
		"container",
		testImage,
		WithCommand("tail", "-f", "/dev/null"),
	)
	cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
	require.NoError(t, err)

	t.Logf("Start the container")
	require.NoError(t, runtimeService.StartContainer(cn))

	// gcr.io/k8s-cri-containerd/volume-copy-up:1.0 contains a test_dir
	// volume, which contains a test_file with content "test_content".
	t.Logf("Check whether volume contains the test file")
	stdout, stderr, err := runtimeService.ExecSync(cn, []string{
		"cat",
		"/test_dir/test_file",
	}, execTimeout)
	require.NoError(t, err)
	assert.Empty(t, stderr)
	assert.Equal(t, "test_content\n", string(stdout))

	t.Logf("Check host path of the volume")
	hostCmd := fmt.Sprintf("find %s/containers/%s/volumes/*/test_file | xargs cat", *criRoot, cn)
	output, err := exec.Command("sh", "-c", hostCmd).CombinedOutput()
	require.NoError(t, err)
	assert.Equal(t, "test_content\n", string(output))

	t.Logf("Update volume from inside the container")
	_, _, err = runtimeService.ExecSync(cn, []string{
		"sh",
		"-c",
		"echo new_content > /test_dir/test_file",
	}, execTimeout)
	require.NoError(t, err)

	t.Logf("Check whether host path of the volume is updated")
	output, err = exec.Command("sh", "-c", hostCmd).CombinedOutput()
	require.NoError(t, err)
	assert.Equal(t, "new_content\n", string(output))
}

func TestVolumeOwnership(t *testing.T) {
	const (
		testImage   = "gcr.io/k8s-cri-containerd/volume-ownership:1.0"
		execTimeout = time.Minute
	)

	t.Logf("Create a sandbox")
	sbConfig := PodSandboxConfig("sandbox", "volume-ownership")
	sb, err := runtimeService.RunPodSandbox(sbConfig, *runtimeHandler)
	require.NoError(t, err)
	defer func() {
		assert.NoError(t, runtimeService.StopPodSandbox(sb))
		assert.NoError(t, runtimeService.RemovePodSandbox(sb))
	}()

	t.Logf("Pull test image")
	_, err = imageService.PullImage(&runtime.ImageSpec{Image: testImage}, nil, sbConfig)
	require.NoError(t, err)

	t.Logf("Create a container with volume-ownership test image")
	cnConfig := ContainerConfig(
		"container",
		testImage,
		WithCommand("tail", "-f", "/dev/null"),
	)
	cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
	require.NoError(t, err)

	t.Logf("Start the container")
	require.NoError(t, runtimeService.StartContainer(cn))

	// gcr.io/k8s-cri-containerd/volume-ownership:1.0 contains a test_dir
	// volume, which is owned by nobody:nogroup.
	t.Logf("Check ownership of test directory inside container")
	stdout, stderr, err := runtimeService.ExecSync(cn, []string{
		"stat", "-c", "%U:%G", "/test_dir",
	}, execTimeout)
	require.NoError(t, err)
	assert.Empty(t, stderr)
	assert.Equal(t, "nobody:nogroup\n", string(stdout))

	t.Logf("Check ownership of test directory on the host")
	hostCmd := fmt.Sprintf("find %s/containers/%s/volumes/* | xargs stat -c %%U:%%G", *criRoot, cn)
	output, err := exec.Command("sh", "-c", hostCmd).CombinedOutput()
	require.NoError(t, err)
	assert.Equal(t, "nobody:nogroup\n", string(output))
}
<commit_msg>Use the multi-arch version of the test images<commit_after>// +build linux

/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package integration

import (
	"fmt"
	"os/exec"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
)

func TestVolumeCopyUp(t *testing.T) {
	const (
		testImage   = "gcr.io/k8s-cri-containerd/volume-copy-up:2.0"
		execTimeout = time.Minute
	)

	t.Logf("Create a sandbox")
	sbConfig := PodSandboxConfig("sandbox", "volume-copy-up")
	sb, err := runtimeService.RunPodSandbox(sbConfig, *runtimeHandler)
	require.NoError(t, err)
	defer func() {
		assert.NoError(t, runtimeService.StopPodSandbox(sb))
		assert.NoError(t, runtimeService.RemovePodSandbox(sb))
	}()

	t.Logf("Pull test image")
	_, err = imageService.PullImage(&runtime.ImageSpec{Image: testImage}, nil, sbConfig)
	require.NoError(t, err)

	t.Logf("Create a container with volume-copy-up test image")
	cnConfig := ContainerConfig(
		"container",
		testImage,
		WithCommand("tail", "-f", "/dev/null"),
	)
	cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
	require.NoError(t, err)

	t.Logf("Start the container")
	require.NoError(t, runtimeService.StartContainer(cn))

	// gcr.io/k8s-cri-containerd/volume-copy-up:2.0 contains a test_dir
	// volume, which contains a test_file with content "test_content".
	t.Logf("Check whether volume contains the test file")
	stdout, stderr, err := runtimeService.ExecSync(cn, []string{
		"cat",
		"/test_dir/test_file",
	}, execTimeout)
	require.NoError(t, err)
	assert.Empty(t, stderr)
	assert.Equal(t, "test_content\n", string(stdout))

	t.Logf("Check host path of the volume")
	hostCmd := fmt.Sprintf("find %s/containers/%s/volumes/*/test_file | xargs cat", *criRoot, cn)
	output, err := exec.Command("sh", "-c", hostCmd).CombinedOutput()
	require.NoError(t, err)
	assert.Equal(t, "test_content\n", string(output))

	t.Logf("Update volume from inside the container")
	_, _, err = runtimeService.ExecSync(cn, []string{
		"sh",
		"-c",
		"echo new_content > /test_dir/test_file",
	}, execTimeout)
	require.NoError(t, err)

	t.Logf("Check whether host path of the volume is updated")
	output, err = exec.Command("sh", "-c", hostCmd).CombinedOutput()
	require.NoError(t, err)
	assert.Equal(t, "new_content\n", string(output))
}

func TestVolumeOwnership(t *testing.T) {
	const (
		testImage   = "gcr.io/k8s-cri-containerd/volume-ownership:2.0"
		execTimeout = time.Minute
	)

	t.Logf("Create a sandbox")
	sbConfig := PodSandboxConfig("sandbox", "volume-ownership")
	sb, err := runtimeService.RunPodSandbox(sbConfig, *runtimeHandler)
	require.NoError(t, err)
	defer func() {
		assert.NoError(t, runtimeService.StopPodSandbox(sb))
		assert.NoError(t, runtimeService.RemovePodSandbox(sb))
	}()

	t.Logf("Pull test image")
	_, err = imageService.PullImage(&runtime.ImageSpec{Image: testImage}, nil, sbConfig)
	require.NoError(t, err)

	t.Logf("Create a container with volume-ownership test image")
	cnConfig := ContainerConfig(
		"container",
		testImage,
		WithCommand("tail", "-f", "/dev/null"),
	)
	cn, err := runtimeService.CreateContainer(sb, cnConfig, sbConfig)
	require.NoError(t, err)

	t.Logf("Start the container")
	require.NoError(t, runtimeService.StartContainer(cn))

	// gcr.io/k8s-cri-containerd/volume-ownership:2.0 contains a test_dir
	// volume, which is owned by nobody:nogroup.
	t.Logf("Check ownership of test directory inside container")
	stdout, stderr, err := runtimeService.ExecSync(cn, []string{
		"stat", "-c", "%U:%G", "/test_dir",
	}, execTimeout)
	require.NoError(t, err)
	assert.Empty(t, stderr)
	assert.Equal(t, "nobody:nogroup\n", string(stdout))

	t.Logf("Check ownership of test directory on the host")
	hostCmd := fmt.Sprintf("find %s/containers/%s/volumes/* | xargs stat -c %%U:%%G", *criRoot, cn)
	output, err := exec.Command("sh", "-c", hostCmd).CombinedOutput()
	require.NoError(t, err)
	assert.Equal(t, "nobody:nogroup\n", string(output))
}
<|endoftext|>