<commit_before>package fetch

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/slotix/dataflowkit/splash"
)

// NewHTTPClient returns an Fetch Service backed by an HTTP server living at the
// remote instance. We expect instance to come from a service discovery system,
// so likely of the form "host:port". We bake-in certain middlewares,
// implementing the client library pattern.
func NewHTTPClient(instance string) (Service, error) {
	// Quickly sanitize the instance string.
	if !strings.HasPrefix(instance, "http") {
		instance = "http://" + instance
	}
	u, err := url.Parse(instance)
	if err != nil {
		return nil, err
	}

	// Each individual endpoint is an http/transport.Client (which implements
	// endpoint.Endpoint) that gets wrapped with various middlewares. If you
	// made your own client library, you'd do this work there, so your server
	// could rely on a consistent set of client behavior.
	var splashFetchEndpoint endpoint.Endpoint
	{
		splashFetchEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/fetch/splash"),
			encodeHTTPGenericRequest,
			decodeSplashFetcherContent,
		).Endpoint()
	}

	var splashResponseEndpoint endpoint.Endpoint
	{
		splashResponseEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/response/splash"),
			encodeHTTPGenericRequest,
			decodeSplashFetcherResponse,
		).Endpoint()
	}

	var baseFetchEndpoint endpoint.Endpoint
	{
		baseFetchEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/fetch/base"),
			encodeHTTPGenericRequest,
			decodeBaseFetcherContent,
		).Endpoint()
	}

	var baseResponseEndpoint endpoint.Endpoint
	{
		baseResponseEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/response/base"),
			encodeHTTPGenericRequest,
			decodeBaseFetcherResponse,
		).Endpoint()
	}
	// Returning the endpoint.Set as a service.Service relies on the
	// endpoint.Set implementing the Service methods. That's just a simple bit
	// of glue code.
	return Endpoints{
		SplashFetchEndpoint:    splashFetchEndpoint,
		SplashResponseEndpoint: splashResponseEndpoint,
		BaseFetchEndpoint:      baseFetchEndpoint,
		BaseResponseEndpoint:   baseResponseEndpoint,
	}, nil
}

// encodeHTTPGenericRequest is a transport/http.EncodeRequestFunc that
// JSON-encodes any request to the request body. Primarily useful in a client.
func encodeHTTPGenericRequest(ctx context.Context, r *http.Request, request interface{}) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(request); err != nil {
		return err
	}
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// decodeSplashFetcherContent is a transport/http.DecodeResponseFunc that decodes a
// JSON-encoded splash fetcher response from the HTTP response body. If the response has a
// non-200 status code, we will interpret that as an error and attempt to decode
// the specific error message from the response body. Primarily useful in a
// client.
func decodeSplashFetcherContent(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func decodeSplashFetcherResponse(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	var resp splash.Response
	err := json.NewDecoder(r.Body).Decode(&resp)
	return resp, err
}

func decodeBaseFetcherContent(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	var resp BaseFetcherResponse
	err := json.NewDecoder(r.Body).Decode(&resp)
	return resp, err
}

func decodeBaseFetcherResponse(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	var resp BaseFetcherResponse
	err := json.NewDecoder(r.Body).Decode(&resp)
	return resp, err
}

func copyURL(base *url.URL, path string) *url.URL {
	next := *base
	next.Path = path
	return &next
}

func (e Endpoints) Fetch(req FetchRequester) (io.ReadCloser, error) {
	r, err := e.Response(req)
	if err != nil {
		return nil, err
	}
	return r.GetHTML()
}

func (e Endpoints) Response(req FetchRequester) (FetchResponser, error) {
	ctx := context.Background()
	//switch req.(type) {
	switch req.Type() {
	//case  BaseFetcherRequest, *BaseFetcherRequest:
	case "base":
		resp, err := e.BaseResponseEndpoint(ctx, req)
		if err != nil {
			return nil, err
		}
		response := resp.(BaseFetcherResponse)
		return &response, nil
	//case splash.Request, *splash.Request:
	case "splash":
		resp, err := e.SplashResponseEndpoint(ctx, req)
		if err != nil {
			return nil, err
		}
		response := resp.(splash.Response)
		return &response, nil
	default:
		return nil, errors.New("invalid fetcher request")
	}
}
<commit_msg>fix endpoints logic for fetching content<commit_after>package fetch

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/go-kit/kit/endpoint"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/slotix/dataflowkit/splash"
)

// NewHTTPClient returns an Fetch Service backed by an HTTP server living at the
// remote instance. We expect instance to come from a service discovery system,
// so likely of the form "host:port". We bake-in certain middlewares,
// implementing the client library pattern.
func NewHTTPClient(instance string) (Service, error) {
	// Quickly sanitize the instance string.
	if !strings.HasPrefix(instance, "http") {
		instance = "http://" + instance
	}
	u, err := url.Parse(instance)
	if err != nil {
		return nil, err
	}

	// Each individual endpoint is an http/transport.Client (which implements
	// endpoint.Endpoint) that gets wrapped with various middlewares. If you
	// made your own client library, you'd do this work there, so your server
	// could rely on a consistent set of client behavior.
	var splashFetchEndpoint endpoint.Endpoint
	{
		splashFetchEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/fetch/splash"),
			encodeHTTPGenericRequest,
			decodeSplashFetcherContent,
		).Endpoint()
	}

	var splashResponseEndpoint endpoint.Endpoint
	{
		splashResponseEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/response/splash"),
			encodeHTTPGenericRequest,
			decodeSplashFetcherResponse,
		).Endpoint()
	}

	var baseFetchEndpoint endpoint.Endpoint
	{
		baseFetchEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/fetch/base"),
			encodeHTTPGenericRequest,
			decodeBaseFetcherContent,
		).Endpoint()
	}

	var baseResponseEndpoint endpoint.Endpoint
	{
		baseResponseEndpoint = httptransport.NewClient(
			"POST",
			copyURL(u, "/response/base"),
			encodeHTTPGenericRequest,
			decodeBaseFetcherResponse,
		).Endpoint()
	}
	// Returning the endpoint.Set as a service.Service relies on the
	// endpoint.Set implementing the Service methods. That's just a simple bit
	// of glue code.
	return Endpoints{
		SplashFetchEndpoint:    splashFetchEndpoint,
		SplashResponseEndpoint: splashResponseEndpoint,
		BaseFetchEndpoint:      baseFetchEndpoint,
		BaseResponseEndpoint:   baseResponseEndpoint,
	}, nil
}

// encodeHTTPGenericRequest is a transport/http.EncodeRequestFunc that
// JSON-encodes any request to the request body. Primarily useful in a client.
func encodeHTTPGenericRequest(ctx context.Context, r *http.Request, request interface{}) error {
	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(request); err != nil {
		return err
	}
	r.Body = ioutil.NopCloser(&buf)
	return nil
}

// decodeSplashFetcherContent is a transport/http.DecodeResponseFunc that decodes a
// JSON-encoded splash fetcher response from the HTTP response body. If the response has a
// non-200 status code, we will interpret that as an error and attempt to decode
// the specific error message from the response body. Primarily useful in a
// client.
func decodeSplashFetcherContent(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func decodeSplashFetcherResponse(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	var resp splash.Response
	err := json.NewDecoder(r.Body).Decode(&resp)
	return resp, err
}

func decodeBaseFetcherContent(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	data, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	return data, nil
}

func decodeBaseFetcherResponse(ctx context.Context, r *http.Response) (interface{}, error) {
	if r.StatusCode != http.StatusOK {
		return nil, errors.New(r.Status)
	}
	var resp BaseFetcherResponse
	err := json.NewDecoder(r.Body).Decode(&resp)
	return resp, err
}

func copyURL(base *url.URL, path string) *url.URL {
	next := *base
	next.Path = path
	return &next
}

func (e Endpoints) Fetch(req FetchRequester) (io.ReadCloser, error) {
	ctx := context.Background()
	var resp interface{}
	var err error
	switch req.Type() {
	case "base":
		resp, err = e.BaseFetchEndpoint(ctx, req)
		if err != nil {
			return nil, err
		}
	case "splash":
		resp, err = e.SplashFetchEndpoint(ctx, req)
		if err != nil {
			return nil, err
		}
	}
	readCloser := ioutil.NopCloser(bytes.NewReader(resp.([]byte)))
	return readCloser, nil
}

func (e Endpoints) Response(req FetchRequester) (FetchResponser, error) {
	ctx := context.Background()
	var r FetchResponser
	var err error
	switch req.Type() {
	case "base":
		resp, err := e.BaseResponseEndpoint(ctx, req)
		if err != nil {
			return nil, err
		}
		response := resp.(BaseFetcherResponse)
		r = &response
	//case splash.Request, *splash.Request:
	case "splash":
		resp, err := e.SplashResponseEndpoint(ctx, req)
		if err != nil {
			return nil, err
		}
		response := resp.(splash.Response)
		r = &response
	}
	return r, err
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"github.com/concourse/concourse/fly/rc"
	"net/url"
	"os"
	"os/exec"
	"strings"
)

type CurlCommand struct {
	Args struct {
		Path string   `positional-arg-name:"PATH" required:"true"`
		Rest []string `positional-arg-name:"curl flags"`
	} `positional-args:"yes"`
	PrintAndExit bool `long:"print-and-exit" description:"Print curl command and exit"`
}

func (command *CurlCommand) Execute([]string) error {
	target, err := rc.LoadTarget(Fly.Target, Fly.Verbose)
	if err != nil {
		return err
	}

	err = target.Validate()
	if err != nil {
		return err
	}

	if target.CACert() != "" {
		return fmt.Errorf("not implemented for custom CA Certs")
	}

	fullUrl, err := command.makeFullUrl(target.URL(), command.Args.Path)
	if err != nil {
		return err
	}

	argsList := command.makeArgsList(target.Token(), fullUrl, command.Args.Rest)

	cmd := exec.Command("curl", argsList...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	if command.PrintAndExit {
		fmt.Println(printableCommand(cmd.Args))
		return nil
	}

	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func (command *CurlCommand) makeFullUrl(host, path string) (string, error) {
	u, err := url.Parse(host)
	if err != nil {
		return "", err
	}
	u.Path = path
	return u.String(), nil
}

func (command *CurlCommand) makeArgsList(token *rc.TargetToken, url string, options []string) (args []string) {
	authTokenHeader := []string{"-H", fmt.Sprintf("Authorization: %s %s", token.Type, token.Value)}
	args = append(args, authTokenHeader...)
	args = append(args, options...)
	args = append(args, url)
	return
}

func printableCommand(args []string) string {
	for i,_ := range args {
		if strings.Contains(args[i], " ") {
			args[i] = fmt.Sprintf(`"%s"`, args[i])
		}
	}

	return strings.Join(args, " ")
}
<commit_msg>Remove check for ca-certs<commit_after>package commands

import (
	"fmt"
	"github.com/concourse/concourse/fly/rc"
	"net/url"
	"os"
	"os/exec"
	"strings"
)

type CurlCommand struct {
	Args struct {
		Path string   `positional-arg-name:"PATH" required:"true"`
		Rest []string `positional-arg-name:"curl flags"`
	} `positional-args:"yes"`
	PrintAndExit bool `long:"print-and-exit" description:"Print curl command and exit"`
}

func (command *CurlCommand) Execute([]string) error {
	target, err := rc.LoadTarget(Fly.Target, Fly.Verbose)
	if err != nil {
		return err
	}

	err = target.Validate()
	if err != nil {
		return err
	}

	fullUrl, err := command.makeFullUrl(target.URL(), command.Args.Path)
	if err != nil {
		return err
	}

	argsList := command.makeArgsList(target.Token(), fullUrl, command.Args.Rest)

	cmd := exec.Command("curl", argsList...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Stdin = os.Stdin

	if command.PrintAndExit {
		fmt.Println(printableCommand(cmd.Args))
		return nil
	}

	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func (command *CurlCommand) makeFullUrl(host, path string) (string, error) {
	u, err := url.Parse(host)
	if err != nil {
		return "", err
	}
	u.Path = path
	return u.String(), nil
}

func (command *CurlCommand) makeArgsList(token *rc.TargetToken, url string, options []string) (args []string) {
	authTokenHeader := []string{"-H", fmt.Sprintf("Authorization: %s %s", token.Type, token.Value)}
	args = append(args, authTokenHeader...)
	args = append(args, options...)
	args = append(args, url)
	return
}

func printableCommand(args []string) string {
	for i, arg := range args {
		if strings.Contains(arg, " ") {
			args[i] = fmt.Sprintf(`"%s"`, arg)
		}
	}

	return strings.Join(args, " ")
}
<|endoftext|>


<commit_before>package forecast

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// response from https://api.darksky.net/forecast/
// comes back like:
// {
//     "alerts": [
//         {
//             "description": "...BLOWING DUST ADVISORY REMAINS IN EFFECT UNTIL 11 PM MST THIS\nEVENING...\nA BLOWING DUST ADVISORY REMAINS IN EFFECT UNTIL 11 PM MST THIS\nEVENING.\n* AFFECTED AREA...NORTHWEST AND NORTH-CENTRAL PINAL COUNTY...\nEXTENDING NORTH INTO EAST CENTRAL MARICOPA COUNTY...INCLUDING\nCHANDLER...GILBERT...MESA...TEMPE...AND THE GREATER PHOENIX\nAREA\n* TIMING...UNTIL 11 PM.\n* WINDS...SOUTH-SOUTHEAST 30 MPH WITH GUSTS UP TO 45 MPH\nPOSSIBLE.\n* VISIBILITY...3 MILES IN BLOWING DUST...LOCALLY DOWN TO AS LOW\nAS ONE MILE.\n* IMPACTS...SUDDEN DROPS IN VISIBILITIES DUE TO BLOWING DUST\nWILL CREATE HAZARDOUS TRAVEL CONDITIONS...ESPECIALLY AT NIGHT.\nTRAVELERS NEED TO BE READY FOR RAPIDLY CHANGING ROAD\nCONDITIONS.\n",
//             "expires": 1405317600,
//             "time": 1405313940,
//             "title": "Blowing Dust Advisory for Maricopa, AZ",
//             "uri": "http://alerts.weather.gov/cap/wwacapget.php?x=AZ1251601B31CC.BlowingDustAdvisory.1251601B68E0AZ.PSRNPWPSR.5957bb654b0ef12b8351fd6d518e951e"
//         }
//     ],
//     "currently": {
//         "apparentTemperature": 91.72,
//         "cloudCover": 0.92,
//         "dewPoint": 64.97,
//         "humidity": 0.44,
//         "icon": "partly-cloudy-night",
//         "nearestStormDistance": 0,
//         "nearestStormBearing": 0,
//         "ozone": 295.8,
//         "precipIntensity": 0,
//         "precipProbability": 0,
//         "pressure": 1010.85,
//         "summary": "Mostly Cloudy",
//         "temperature": 89.72,
//         "time": 1405315610,
//         "visibility": 9.99,
//         "windBearing": 208,
//         "windSpeed": 6.48
//     },
//     "daily": {
//         "data": [
//             {
//                 "apparentTemperatureMax": 107.25,
//                 "apparentTemperatureMaxTime": 1405288800,
//                 "apparentTemperatureMin": 86.87,
//                 "apparentTemperatureMinTime": 1405252800,
//                 "cloudCover": 0.68,
//                 "dewPoint": 62.06,
//                 "humidity": 0.35,
//                 "icon": "rain",
//                 "moonPhase": 0.55,
//                 "ozone": 299.93,
//                 "precipIntensity": 0.0047,
//                 "precipIntensityMax": 0.0308,
//                 "precipIntensityMaxTime": 1405310400,
//                 "precipProbability": 0.95,
//                 "precipType": "rain",
//                 "pressure": 1009.52,
//                 "summary": "Light rain starting in the afternoon.",
//                 "sunriseTime": 1405254533,
//                 "sunsetTime": 1405305631,
//                 "temperatureMax": 105.3,
//                 "temperatureMaxTime": 1405288800,
//                 "temperatureMin": 86.14,
//                 "temperatureMinTime": 1405252800,
//                 "time": 1405234800,
//                 "visibility": 9.88,
//                 "windBearing": 149,
//                 "windSpeed": 2.95
//             }...
//         ],
//         "icon": "rain",
//         "summary": "Light rain today and tomorrow, with temperatures rising to 109°F on Thursday."
//     },
//     "flags": {
//         "units": "us"
//     },
//     "hourly": {
//         "data": [
//             {
//                 "apparentTemperature": 92.39,
//                 "cloudCover": 0.91,
//                 "dewPoint": 64.12,
//                 "humidity": 0.41,
//                 "icon": "rain",
//                 "ozone": 295.87,
//                 "precipIntensity": 0.0178,
//                 "precipProbability": 0.52,
//                 "precipType": "rain",
//                 "pressure": 1010.44,
//                 "summary": "Light Rain",
//                 "temperature": 90.77,
//                 "time": 1405314000,
//                 "visibility": 10,
//                 "windBearing": 195,
//                 "windSpeed": 6.06
//             }...
//         ],
//         "icon": "rain",
//         "summary": "Drizzle starting tomorrow afternoon, continuing until tomorrow evening."
//     },
//     "latitude": 33.4962205,
//     "longitude": -111.9641728,
//     "minutely": {
//         "data": [
//             {
//                 "precipIntensity": 0,
//                 "precipProbability": 0,
//                 "time": 1405315920
//             },
//             {
//                 "precipIntensity": 0.0027,
//                 "precipIntensityError": 0.0002,
//                 "precipProbability": 0.01,
//                 "precipType": "rain",
//                 "time": 1405315980
//             }...
//         ],
//         "icon": "partly-cloudy-night",
//         "summary": "Mostly cloudy for the hour."
//     },
//     "offset": -7,
//     "timezone": "America/Phoenix"
// }

// Forecast contains the information returned from the server
// when requesting the forecast.
type Forecast struct {
	Alerts    []Alert       `json:"alerts"`
	Currently Weather       `json:"currently"`
	Code      int           `json:"code"`
	Daily     TimeDelimited `json:"daily"`
	Error     string        `json:"error"`
	Flags     Flags         `json:"flags"`
	Hourly    TimeDelimited `json:"hourly"`
	Latitude  float64       `json:"latitude"`
	Longitude float64       `json:"longitude"`
	Offset    float64       `json:"offset"`
	Timezone  string        `json:"timezone"`
}

// Alert contains any weather alerts happening at the location.
type Alert struct {
	Description string `json:"description"`
	Expires     int64  `json:"expires"`
	Time        int64  `json:"time"`
	Title       string `json:"title"`
	URI         string `json:"uri"`
}

// Flags describes the flags on a forecast.
type Flags struct {
	Units string `json:"units"`
}

// Weather describes details about the weather for the location.
type Weather struct {
	ApparentTemperature        float64 `json:"apparentTemperature"`
	ApparentTemperatureMax     float64 `json:"apparentTemperatureMax"`
	ApparentTemperatureMaxTime int64   `json:"apparentTemperatureMaxTime"`
	ApparentTemperatureMin     float64 `json:"apparentTemperatureMin"`
	ApparentTemperatureMinTime int64   `json:"apparentTemperatureMinTime"`
	CloudCover                 float64 `json:"cloudCover"`
	DewPoint                   float64 `json:"dewPoint"`
	Humidity                   float64 `json:"humidity"`
	Icon                       string  `json:"icon"`
	NearestStormDistance       float64 `json:"nearestStormDistance"`
	NearestStormBearing        float64 `json:"nearestStormBearing"`
	Ozone                      float64 `json:"ozone"`
	PrecipIntensity            float64 `json:"precipIntensity"`
	PrecipIntensityMax         float64 `json:"precipIntensityMax"`
	PrecipIntensityMaxTime     int64   `json:"precipIntensityMaxTime"`
	PrecipProbability          float64 `json:"precipProbability"`
	PrecipType                 string  `json:"precipType"`
	Pressure                   float64 `json:"pressure"`
	Summary                    string  `json:"summary"`
	SunriseTime                int64   `json:"sunriseTime"`
	SunsetTime                 int64   `json:"sunsetTime"`
	Temperature                float64 `json:"temperature"`
	TemperatureMax             float64 `json:"temperatureMax"`
	TemperatureMaxTime         int64   `json:"temperatureMaxTime"`
	TemperatureMin             float64 `json:"temperatureMin"`
	TemperatureMinTime         int64   `json:"temperatureMinTime"`
	Time                       int64   `json:"time"`
	Visibility                 float64 `json:"visibility"`
	WindBearing                float64 `json:"windBearing"`
	WindSpeed                  float64 `json:"windSpeed"`
}

// TimeDelimited describes the data for the time series.
type TimeDelimited struct {
	Data    []Weather `json:"data"`
	Icon    string    `json:"icon"`
	Summary string    `json:"summary"`
}

// Request describes the request posted to the forecast api.
type Request struct {
	Latitude  float64  `json:"lat"`
	Longitude float64  `json:"lng"`
	Units     string   `json:"units"`
	Exclude   []string `json:"exclude"`
}

// Get performs a request to get the forecast data for a location.
func Get(uri string, data Request) (forecast Forecast, err error) {
	// create json data
	jsonByte, err := json.Marshal(data)
	if err != nil {
		return forecast, fmt.Errorf("Marshaling forecast json failed: %v", err)
	}

	// send the request
	req, err := http.NewRequest("POST", uri, bytes.NewReader(jsonByte))
	if err != nil {
		return forecast, nil
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return forecast, fmt.Errorf("Http request to %s failed: %s", req.URL, err.Error())
	}
	defer resp.Body.Close()

	// decode the body
	dec := json.NewDecoder(resp.Body)
	if err = dec.Decode(&forecast); err != nil {
		return forecast, fmt.Errorf("Decoding forecast response failed: %v", err)
	}

	if forecast.Error != "" {
		return forecast, fmt.Errorf("Forecast API response error: %s", forecast.Error)
	}

	return forecast, nil
}
<commit_msg>Check HTTP status of forcast API and fail gracefully if it's down (#56)<commit_after>package forecast

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
)

// response from https://api.darksky.net/forecast/
// comes back like:
// {
//     "alerts": [
//         {
//             "description": "...BLOWING DUST ADVISORY REMAINS IN EFFECT UNTIL 11 PM MST THIS\nEVENING...\nA BLOWING DUST ADVISORY REMAINS IN EFFECT UNTIL 11 PM MST THIS\nEVENING.\n* AFFECTED AREA...NORTHWEST AND NORTH-CENTRAL PINAL COUNTY...\nEXTENDING NORTH INTO EAST CENTRAL MARICOPA COUNTY...INCLUDING\nCHANDLER...GILBERT...MESA...TEMPE...AND THE GREATER PHOENIX\nAREA\n* TIMING...UNTIL 11 PM.\n* WINDS...SOUTH-SOUTHEAST 30 MPH WITH GUSTS UP TO 45 MPH\nPOSSIBLE.\n* VISIBILITY...3 MILES IN BLOWING DUST...LOCALLY DOWN TO AS LOW\nAS ONE MILE.\n* IMPACTS...SUDDEN DROPS IN VISIBILITIES DUE TO BLOWING DUST\nWILL CREATE HAZARDOUS TRAVEL CONDITIONS...ESPECIALLY AT NIGHT.\nTRAVELERS NEED TO BE READY FOR RAPIDLY CHANGING ROAD\nCONDITIONS.\n",
//             "expires": 1405317600,
//             "time": 1405313940,
//             "title": "Blowing Dust Advisory for Maricopa, AZ",
//             "uri": "http://alerts.weather.gov/cap/wwacapget.php?x=AZ1251601B31CC.BlowingDustAdvisory.1251601B68E0AZ.PSRNPWPSR.5957bb654b0ef12b8351fd6d518e951e"
//         }
//     ],
//     "currently": {
//         "apparentTemperature": 91.72,
//         "cloudCover": 0.92,
//         "dewPoint": 64.97,
//         "humidity": 0.44,
//         "icon": "partly-cloudy-night",
//         "nearestStormDistance": 0,
//         "nearestStormBearing": 0,
//         "ozone": 295.8,
//         "precipIntensity": 0,
//         "precipProbability": 0,
//         "pressure": 1010.85,
//         "summary": "Mostly Cloudy",
//         "temperature": 89.72,
//         "time": 1405315610,
//         "visibility": 9.99,
//         "windBearing": 208,
//         "windSpeed": 6.48
//     },
//     "daily": {
//         "data": [
//             {
//                 "apparentTemperatureMax": 107.25,
//                 "apparentTemperatureMaxTime": 1405288800,
//                 "apparentTemperatureMin": 86.87,
//                 "apparentTemperatureMinTime": 1405252800,
//                 "cloudCover": 0.68,
//                 "dewPoint": 62.06,
//                 "humidity": 0.35,
//                 "icon": "rain",
//                 "moonPhase": 0.55,
//                 "ozone": 299.93,
//                 "precipIntensity": 0.0047,
//                 "precipIntensityMax": 0.0308,
//                 "precipIntensityMaxTime": 1405310400,
//                 "precipProbability": 0.95,
//                 "precipType": "rain",
//                 "pressure": 1009.52,
//                 "summary": "Light rain starting in the afternoon.",
//                 "sunriseTime": 1405254533,
//                 "sunsetTime": 1405305631,
//                 "temperatureMax": 105.3,
//                 "temperatureMaxTime": 1405288800,
//                 "temperatureMin": 86.14,
//                 "temperatureMinTime": 1405252800,
//                 "time": 1405234800,
//                 "visibility": 9.88,
//                 "windBearing": 149,
//                 "windSpeed": 2.95
//             }...
//         ],
//         "icon": "rain",
//         "summary": "Light rain today and tomorrow, with temperatures rising to 109°F on Thursday."
//     },
//     "flags": {
//         "units": "us"
//     },
//     "hourly": {
//         "data": [
//             {
//                 "apparentTemperature": 92.39,
//                 "cloudCover": 0.91,
//                 "dewPoint": 64.12,
//                 "humidity": 0.41,
//                 "icon": "rain",
//                 "ozone": 295.87,
//                 "precipIntensity": 0.0178,
//                 "precipProbability": 0.52,
//                 "precipType": "rain",
//                 "pressure": 1010.44,
//                 "summary": "Light Rain",
//                 "temperature": 90.77,
//                 "time": 1405314000,
//                 "visibility": 10,
//                 "windBearing": 195,
//                 "windSpeed": 6.06
//             }...
//         ],
//         "icon": "rain",
//         "summary": "Drizzle starting tomorrow afternoon, continuing until tomorrow evening."
//     },
//     "latitude": 33.4962205,
//     "longitude": -111.9641728,
//     "minutely": {
//         "data": [
//             {
//                 "precipIntensity": 0,
//                 "precipProbability": 0,
//                 "time": 1405315920
//             },
//             {
//                 "precipIntensity": 0.0027,
//                 "precipIntensityError": 0.0002,
//                 "precipProbability": 0.01,
//                 "precipType": "rain",
//                 "time": 1405315980
//             }...
//         ],
//         "icon": "partly-cloudy-night",
//         "summary": "Mostly cloudy for the hour."
//     },
//     "offset": -7,
//     "timezone": "America/Phoenix"
// }

// Forecast contains the information returned from the server
// when requesting the forecast.
type Forecast struct {
	Alerts    []Alert       `json:"alerts"`
	Currently Weather       `json:"currently"`
	Code      int           `json:"code"`
	Daily     TimeDelimited `json:"daily"`
	Error     string        `json:"error"`
	Flags     Flags         `json:"flags"`
	Hourly    TimeDelimited `json:"hourly"`
	Latitude  float64       `json:"latitude"`
	Longitude float64       `json:"longitude"`
	Offset    float64       `json:"offset"`
	Timezone  string        `json:"timezone"`
}

// Alert contains any weather alerts happening at the location.
type Alert struct {
	Description string `json:"description"`
	Expires     int64  `json:"expires"`
	Time        int64  `json:"time"`
	Title       string `json:"title"`
	URI         string `json:"uri"`
}

// Flags describes the flags on a forecast.
type Flags struct {
	Units string `json:"units"`
}

// Weather describes details about the weather for the location.
type Weather struct {
	ApparentTemperature        float64 `json:"apparentTemperature"`
	ApparentTemperatureMax     float64 `json:"apparentTemperatureMax"`
	ApparentTemperatureMaxTime int64   `json:"apparentTemperatureMaxTime"`
	ApparentTemperatureMin     float64 `json:"apparentTemperatureMin"`
	ApparentTemperatureMinTime int64   `json:"apparentTemperatureMinTime"`
	CloudCover                 float64 `json:"cloudCover"`
	DewPoint                   float64 `json:"dewPoint"`
	Humidity                   float64 `json:"humidity"`
	Icon                       string  `json:"icon"`
	NearestStormDistance       float64 `json:"nearestStormDistance"`
	NearestStormBearing        float64 `json:"nearestStormBearing"`
	Ozone                      float64 `json:"ozone"`
	PrecipIntensity            float64 `json:"precipIntensity"`
	PrecipIntensityMax         float64 `json:"precipIntensityMax"`
	PrecipIntensityMaxTime     int64   `json:"precipIntensityMaxTime"`
	PrecipProbability          float64 `json:"precipProbability"`
	PrecipType                 string  `json:"precipType"`
	Pressure                   float64 `json:"pressure"`
	Summary                    string  `json:"summary"`
	SunriseTime                int64   `json:"sunriseTime"`
	SunsetTime                 int64   `json:"sunsetTime"`
	Temperature                float64 `json:"temperature"`
	TemperatureMax             float64 `json:"temperatureMax"`
	TemperatureMaxTime         int64   `json:"temperatureMaxTime"`
	TemperatureMin             float64 `json:"temperatureMin"`
	TemperatureMinTime         int64   `json:"temperatureMinTime"`
	Time                       int64   `json:"time"`
	Visibility                 float64 `json:"visibility"`
	WindBearing                float64 `json:"windBearing"`
	WindSpeed                  float64 `json:"windSpeed"`
}

// TimeDelimited describes the data for the time series.
type TimeDelimited struct {
	Data    []Weather `json:"data"`
	Icon    string    `json:"icon"`
	Summary string    `json:"summary"`
}

// Request describes the request posted to the forecast api.
type Request struct {
	Latitude  float64  `json:"lat"`
	Longitude float64  `json:"lng"`
	Units     string   `json:"units"`
	Exclude   []string `json:"exclude"`
}

// Get performs a request to get the forecast data for a location.
func Get(uri string, data Request) (forecast Forecast, err error) {
	// create json data
	jsonByte, err := json.Marshal(data)
	if err != nil {
		return forecast, fmt.Errorf("Marshaling forecast json failed: %v", err)
	}

	// send the request
	req, err := http.NewRequest("POST", uri, bytes.NewReader(jsonByte))
	if err != nil {
		return forecast, nil
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return forecast, fmt.Errorf("Http request to %s failed: %s", req.URL, err.Error())
	}

	if resp.StatusCode != http.StatusOK {
		return forecast, fmt.Errorf("Http request to %s failed with status code: %v", req.URL, resp.StatusCode)
	}
	defer resp.Body.Close()

	// decode the body
	dec := json.NewDecoder(resp.Body)
	if err = dec.Decode(&forecast); err != nil {
		return forecast, fmt.Errorf("Decoding forecast response failed: %v", err)
	}

	if forecast.Error != "" {
		return forecast, fmt.Errorf("Forecast API response error: %s", forecast.Error)
	}

	return forecast, nil
}
<|endoftext|>


<commit_before>package gost

import "testing"

var bypassTests = []struct {
	patterns []string
	reversed bool
	addr     string
	bypassed bool
}{
	// IP address
	{[]string{"192.168.1.1"}, false, "192.168.1.1", true},
	{[]string{"192.168.1.1"}, false, "192.168.1.2", false},
	{[]string{"0.0.0.0"}, false, "0.0.0.0", true},

	// CIDR address
	{[]string{"192.168.1.0/0"}, false, "1.2.3.4", true},
	{[]string{"192.168.1.0/8"}, false, "192.1.0.255", true},
	{[]string{"192.168.1.0/16"}, false, "192.168.0.255", true},
	{[]string{"192.168.1.0/24"}, false, "192.168.1.255", true},
	{[]string{"192.168.1.1/32"}, false, "192.168.1.1", true},
	{[]string{"192.168.1.1/32"}, false, "192.168.1.2", false},

	// plain domain
	{[]string{"www.example.com"}, false, "www.example.com", true},
	{[]string{"http://www.example.com"}, false, "http://www.example.com", true},
	{[]string{"http://www.example.com"}, false, "http://example.com", false},
	{[]string{"www.example.com"}, false, "example.com", false},

	// domain wildcard

	// sub-domain
	{[]string{"*.example.com"}, false, "example.com", false},
	{[]string{"*.example.com"}, false, "http://example.com", false},
	{[]string{"*.example.com"}, false, "www.example.com", true},
	{[]string{"*.example.com"}, false, "http://www.example.com", true},
	{[]string{"*.example.com"}, false, "abc.def.example.com", true},

	{[]string{"*.*.example.com"}, false, "example.com", false},
	{[]string{"*.*.example.com"}, false, "www.example.com", false},
	{[]string{"*.*.example.com"}, false, "abc.def.example.com", true},
	{[]string{"*.*.example.com"}, false, "abc.def.ghi.example.com", true},

	{[]string{"**.example.com"}, false, "example.com", false},
	{[]string{"**.example.com"}, false, "www.example.com", true},
	{[]string{"**.example.com"}, false, "abc.def.ghi.example.com", true},

	// prefix wildcard
	{[]string{"*example.com"}, false, "example.com", true},
	{[]string{"*example.com"}, false, "www.example.com", true},
	{[]string{"*example.com"}, false, "abc.defexample.com", true},
	{[]string{"*example.com"}, false, "abc.def-example.com", true},
	{[]string{"*example.com"}, false, "abc.def.example.com", true},
	{[]string{"*example.com"}, false, "http://www.example.com", true},
	{[]string{"*example.com"}, false, "e-xample.com", false},

	{[]string{"http://*.example.com"}, false, "example.com", false},
	{[]string{"http://*.example.com"}, false, "http://example.com", false},
	{[]string{"http://*.example.com"}, false, "http://www.example.com", true},
	{[]string{"http://*.example.com"}, false, "https://www.example.com", false},
	{[]string{"http://*.example.com"}, false, "http://abc.def.example.com", true},

	{[]string{"www.*.com"}, false, "www.example.com", true},
	{[]string{"www.*.com"}, false, "www.abc.def.com", true},

	{[]string{"www.*.*.com"}, false, "www.example.com", false},
	{[]string{"www.*.*.com"}, false, "www.abc.def.com", true},
	{[]string{"www.*.*.com"}, false, "www.abc.def.ghi.com", true},

	{[]string{"www.*example*.com"}, false, "www.example.com", true},
	{[]string{"www.*example*.com"}, false, "www.abc.example.def.com", true},
	{[]string{"www.*example*.com"}, false, "www.e-xample.com", false},

	{[]string{"www.example.*"}, false, "www.example.com", true},
	{[]string{"www.example.*"}, false, "www.example.io", true},
	{[]string{"www.example.*"}, false, "www.example.com.cn", true},

	{[]string{".example.com"}, false, "www.example.com", true},
	{[]string{".example.com"}, false, "example.com", true},
	{[]string{".example.com"}, false, "www.example.com.cn", false},
}

func TestBypass(t *testing.T) {
	for _, test := range bypassTests {
		bp := NewBypassPatterns(test.patterns, test.reversed)
		if bp.Contains(test.addr) != test.bypassed {
			t.Errorf("test failed: %v, %s", test.patterns, test.addr)
		}

		rbp := NewBypassPatterns(test.patterns, !test.reversed)
		if rbp.Contains(test.addr) == test.bypassed {
			t.Errorf("reverse test failed: %v, %s", test.patterns, test.addr)
		}
	}
}
<commit_msg>fix bypass test<commit_after>package gost

import "testing"

var bypassTests = []struct {
	patterns []string
	reversed bool
	addr     string
	bypassed bool
}{
	// IP address
	{[]string{"192.168.1.1"}, false, "192.168.1.1", true},
	{[]string{"192.168.1.1"}, false, "192.168.1.2", false},
	{[]string{"0.0.0.0"}, false, "0.0.0.0", true},

	// CIDR address
	{[]string{"192.168.1.0/0"}, false, "1.2.3.4", true},
	{[]string{"192.168.1.0/8"}, false, "192.1.0.255", true},
	{[]string{"192.168.1.0/16"}, false, "192.168.0.255", true},
	{[]string{"192.168.1.0/24"}, false, "192.168.1.255", true},
	{[]string{"192.168.1.1/32"}, false, "192.168.1.1", true},
	{[]string{"192.168.1.1/32"}, false, "192.168.1.2", false},

	// plain domain
	{[]string{"www.example.com"}, false, "www.example.com", true},
	{[]string{"http://www.example.com"}, false, "http://www.example.com", true},
	{[]string{"http://www.example.com"}, false, "http://example.com", false},
	{[]string{"www.example.com"}, false, "example.com", false},

	// domain wildcard

	// sub-domain
	{[]string{"*.example.com"}, false, "example.com", false},
	{[]string{"*.example.com"}, false, "http://example.com", false},
	{[]string{"*.example.com"}, false, "www.example.com", true},
	{[]string{"*.example.com"}, false, "http://www.example.com", true},
	{[]string{"*.example.com"}, false, "abc.def.example.com", true},

	{[]string{"*.*.example.com"}, false, "example.com", false},
	{[]string{"*.*.example.com"}, false, "www.example.com", false},
	{[]string{"*.*.example.com"}, false, "abc.def.example.com", true},
	{[]string{"*.*.example.com"}, false, "abc.def.ghi.example.com", true},

	{[]string{"**.example.com"}, false, "example.com", false},
	{[]string{"**.example.com"}, false, "www.example.com", true},
	{[]string{"**.example.com"}, false, "abc.def.ghi.example.com", true},

	// prefix wildcard
	{[]string{"*example.com"}, false, "example.com", true},
	{[]string{"*example.com"}, false, "www.example.com", true},
	{[]string{"*example.com"}, false, "abc.defexample.com", true},
	{[]string{"*example.com"}, false, "abc.def-example.com", true},
	{[]string{"*example.com"}, false, "abc.def.example.com", true},
	{[]string{"*example.com"}, false, "http://www.example.com", true},
	{[]string{"*example.com"}, false, "e-xample.com", false},

	{[]string{"http://*.example.com"}, false, "example.com", false},
	{[]string{"http://*.example.com"}, false, "http://example.com", false},
	{[]string{"http://*.example.com"}, false, "http://www.example.com", true},
	{[]string{"http://*.example.com"}, false, "https://www.example.com", false},
	{[]string{"http://*.example.com"}, false, "http://abc.def.example.com", true},

	{[]string{"www.*.com"}, false, "www.example.com", true},
	{[]string{"www.*.com"}, false, "www.abc.def.com", true},

	{[]string{"www.*.*.com"}, false, "www.example.com", false},
	{[]string{"www.*.*.com"}, false, "www.abc.def.com", true},
	{[]string{"www.*.*.com"}, false, "www.abc.def.ghi.com", true},

	{[]string{"www.*example*.com"}, false, "www.example.com", true},
	{[]string{"www.*example*.com"}, false, "www.abc.example.def.com", true},
	{[]string{"www.*example*.com"}, false, "www.e-xample.com", false},

	{[]string{"www.example.*"}, false, "www.example.com", true},
	{[]string{"www.example.*"}, false, "www.example.io", true},
	{[]string{"www.example.*"}, false, "www.example.com.cn", true},

	{[]string{".example.com"}, false, "www.example.com", true},
	{[]string{".example.com"}, false, "example.com", true},
	{[]string{".example.com"}, false, "www.example.com.cn", false},
}

func TestBypass(t *testing.T) {
	for _, test := range bypassTests {
		bp := NewBypassPatterns(test.reversed, test.patterns...)
		if bp.Contains(test.addr) != test.bypassed {
			t.Errorf("test failed: %v, %s", test.patterns, test.addr)
		}

		rbp := NewBypassPatterns(!test.reversed, test.patterns...)
		if rbp.Contains(test.addr) == test.bypassed {
			t.Errorf("reverse test failed: %v, %s", test.patterns, test.addr)
		}
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"bytes"
	"compress/gzip"
	"strings"
	"time"
)

type Object struct {
	Data    []byte
	ModTime time.Time
}

type Filler interface {
	// Fill adds an object to the cache and also returns the object.
	// When the file may be compressed or not, Fill may add both versions to the cache, but should return
	// requested version of the item.
	Fill(cache Cache, path string) (Object, error)
}

type Cache interface {
	Get(path string, filler Filler) (Object, error)
	Set(path string, object Object, writeThrough bool) error
	// Delete an item from the cache. Include a "*" wildcard at the end to purge multiple items.
	Del(path string)
}

// Helper function for adding things to a Cache.
// It would be nice to have this as as member of Cache, but I need to figure out how to do that in Go.
func CompressAndSet(cache Cache, path string, data []byte, modTime time.Time) (uncompressed Object, compressed Object, err error) {

	compressedPath := path
	uncompressedPath := path

	if strings.HasSuffix(path, ".gz") {
		uncompressedPath = path[0 : len(path)-3]
	} else {
		compressedPath = path + ".gz"
	}

	gzBuf := new(bytes.Buffer)
	compressor, err := gzip.NewWriterLevel(gzBuf, gzip.BestCompression)
	if err != nil {
		return Object{}, Object{}, err
	}

	_, err = compressor.Write(data)
	compressor.Close()
	if err != nil {
		return Object{}, Object{}, err
	}

	compressedItem := Object{gzBuf.Bytes(), modTime}

	// Add the compressed version to the cache.
	err = cache.Set(compressedPath, compressedItem, true)
	if err != nil {
		return Object{}, Object{}, err
	}

	// Also add the uncompressed version.
	uncompressedItem := Object{data, modTime}
	err = cache.Set(uncompressedPath, uncompressedItem, true)

	return uncompressedItem, compressedItem, err
}
<commit_msg>Update comment<commit_after>package cache

import (
	"bytes"
	"compress/gzip"
	"strings"
	"time"
)

type Object struct {
	Data    []byte
	ModTime time.Time
}

type Filler interface {
	// Fill adds an object to the cache and also returns the object.
	// When the file may be compressed or not, Fill may add both versions to the cache, but should return
	// requested version of the item.
	Fill(cache Cache, path string) (Object, error)
}

type Cache interface {
	Get(path string, filler Filler) (Object, error)
	Set(path string, object Object, writeThrough bool) error
	// Delete an item from the cache. Include a "*" wildcard at the end to purge multiple items.
	Del(path string)
}

// Helper function for adding things to a Cache.
// It would be nice to have this as a member of Cache, but I need to figure out how to do that in Go.
func CompressAndSet(cache Cache, path string, data []byte, modTime time.Time) (uncompressed Object, compressed Object, err error) {

	compressedPath := path
	uncompressedPath := path

	if strings.HasSuffix(path, ".gz") {
		uncompressedPath = path[0 : len(path)-3]
	} else {
		compressedPath = path + ".gz"
	}

	gzBuf := new(bytes.Buffer)
	compressor, err := gzip.NewWriterLevel(gzBuf, gzip.BestCompression)
	if err != nil {
		return Object{}, Object{}, err
	}

	_, err = compressor.Write(data)
	compressor.Close()
	if err != nil {
		return Object{}, Object{}, err
	}

	compressedItem := Object{gzBuf.Bytes(), modTime}

	// Add the compressed version to the cache.
	err = cache.Set(compressedPath, compressedItem, true)
	if err != nil {
		return Object{}, Object{}, err
	}

	// Also add the uncompressed version.
	uncompressedItem := Object{data, modTime}
	err = cache.Set(uncompressedPath, uncompressedItem, true)

	return uncompressedItem, compressedItem, err
}
<|endoftext|>


<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The implementation borrows heavily from SmallLRUCache (originally by Nathan
// Schrenk). The object maintains a doubly-linked list of elements in the
// When an element is accessed it is promoted to the head of the list, and when
// space is needed the element at the tail of the list (the least recently used
// element) is evicted.

// ------
// Contents of the original LICENSE file from https://code.google.com/p/vitess/source/browse/LICENSE

// Copyright 2012, Google Inc.
// All rights reserved.

// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:

//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.

// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ------

package cache

import (
	"container/list"
	"fmt"
	"sync"
	"time"
)

type LRUCache struct {
	mu sync.Mutex

	// list & table of *entry objects
	list  *list.List
	table map[string]*list.Element

	// Our current size, in bytes. Obviously a gross simplification and low-grade
	// approximation.
	size uint64

	// How many bytes we are limiting the cache to.
	capacity uint64
}

// Values that go into LRUCache need to satisfy this interface.
type Value interface {
	Size() int
}

type Item struct {
	Key   string
	Value Value
}

type entry struct {
	key           string
	value         Value
	size          int
	time_accessed time.Time
}

func NewLRUCache(capacity uint64) *LRUCache {
	return &LRUCache{
		list:     list.New(),
		table:    make(map[string]*list.Element),
		capacity: capacity,
	}
}

func (lru *LRUCache) Get(key string) (v Value, ok bool) {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	element := lru.table[key]
	if element == nil {
		return nil, false
	}
	lru.moveToFront(element)
	return element.Value.(*entry).value, true
}

func (lru *LRUCache) Set(key string, value Value) {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	if element := lru.table[key]; element != nil {
		lru.updateInplace(element, value)
	} else {
		lru.addNew(key, value)
	}
}

func (lru *LRUCache) SetIfAbsent(key string, value Value) {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	if element := lru.table[key]; element != nil {
		lru.moveToFront(element)
	} else {
		lru.addNew(key, value)
	}
}

func (lru *LRUCache) Delete(key string) bool {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	element := lru.table[key]
	if element == nil {
		return false
	}

	lru.list.Remove(element)
	delete(lru.table, key)
	lru.size -= uint64(element.Value.(*entry).size)
	return true
}

func (lru *LRUCache) Clear() {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	lru.list.Init()
	lru.table = make(map[string]*list.Element)
	lru.size = 0
}

func (lru *LRUCache) SetCapacity(capacity uint64) {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	lru.capacity = capacity
	lru.checkCapacity()
}

func (lru *LRUCache) Stats() (length, size, capacity uint64, oldest time.Time) {
	lru.mu.Lock()
	defer lru.mu.Unlock()
	if lastElem := lru.list.Back(); lastElem != nil {
		oldest = lastElem.Value.(*entry).time_accessed
	}
	return uint64(lru.list.Len()), lru.size, lru.capacity, oldest
}

func (lru *LRUCache) StatsJSON() string {
	if lru == nil {
		return "{}"
	}
	l, s, c, o := lru.Stats()
	return fmt.Sprintf("{\"Length\": %v, \"Size\": %v, \"Capacity\": %v, \"OldestAccess\": \"%v\"}", l, s, c, o)
}

func (lru *LRUCache) Keys() []string {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	keys := make([]string, 0, lru.list.Len())
	for e := lru.list.Front(); e != nil; e = e.Next() {
		keys = append(keys, e.Value.(*entry).key)
	}
	return keys
}

func (lru *LRUCache) Items() []Item {
	lru.mu.Lock()
	defer lru.mu.Unlock()

	items := make([]Item, 0, lru.list.Len())
	for e := lru.list.Front(); e != nil; e = e.Next() {
		v := e.Value.(*entry)
		items = append(items, Item{Key: v.key, Value: v.value})
	}
	return items
}

func (lru *LRUCache) updateInplace(element *list.Element, value Value) {
	valueSize := value.Size()
	sizeDiff := valueSize - element.Value.(*entry).size
	element.Value.(*entry).value = value
	element.Value.(*entry).size = valueSize
	lru.size += uint64(sizeDiff)
	lru.moveToFront(element)
	lru.checkCapacity()
}

func (lru *LRUCache) moveToFront(element *list.Element) {
	lru.list.MoveToFront(element)
	element.Value.(*entry).time_accessed = time.Now()
}

func (lru *LRUCache) addNew(key string, value Value) {
	newEntry := &entry{key, value, value.Size(), time.Now()}
	element := lru.list.PushFront(newEntry)
	lru.table[key] = element
	lru.size += uint64(newEntry.size)
	lru.checkCapacity()
}

func (lru *LRUCache) checkCapacity() {
	// Partially duplicated from Delete
	for lru.size > lru.capacity {
		delElem := lru.list.Back()
		delValue := delElem.Value.(*entry)
		lru.list.Remove(delElem)
		delete(lru.table, delValue.key)
		lru.size -= uint64(delValue.size)
	}
}
<commit_msg>Remove another folder related to cache<commit_after><|endoftext|>


<commit_before><commit_msg>fixed compile error<commit_after><|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package generate implements code generation for mock classes. This is an
// implementation detail of the createmock command, which you probably want to
// use directly instead.
package generate

import (
	"bytes"
	"errors"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
	"path"
	"reflect"
	"text/template"
)

const tmplStr =
`package {{.Pkg}}

import (
	{{range $identifier, $import := .Imports}}
		{{$identifier}} "{{$import}}"
	{{end}}
)

{{range .Interfaces}}
	{{$structName := printf "mock%s" .Name}}

	type {{$structName}} struct {
		controller oglemock.Controller
		description string
	}

	func New{{printf "Mock%s" .Name}}(
		c oglemock.Controller,
		desc string) *{{$structName}} {
	  return &{{$structName}}{
			controller: c,
			description: desc,
		}
	}

	func (m *{{$structName}}) Oglemock_Id() uintptr {
		return uintptr(unsafe.Pointer(m))
	}

	func (m *{{$structName}}) Oglemock_Description() string {
		return m.description
	}

	{{range getMethods .}}
	  {{$inputTypes := getInputs .Type}}
	  {{$outputTypes := getOutputs .Type}}

		func (m *{{$structName}}) {{.Name}}({{range $i, $type := $inputTypes}}p{{$i}} {{getTypeString $type}}, {{end}}
		) ({{range $i, $type := $outputTypes}}o{{$i}} {{getTypeString $type}}, {{end}}
		){
			// Hand the call off to the controller, which does most of the work.
			retVals := m.controller.HandleMethodCall(m, "{{.Name}}")
			if len(retVals != {{len $outputTypes}}) {
				panic(fmt.Sprintf("{{$structName}}.{{.Name}}: invalid return values: %v", retVals))
			}

			var v reflect.Value
			{{range $i, $type := $outputTypes}}
				// o{{$i}} {{getTypeString $type}}
				v = reflect.ValueOf(retVals[{{$i}}])
				if v.Type() != reflect.TypeOf(o{{$i}}) {
					panic(fmt.Sprintf("{{$structName}}.{{.Name}}: invalid return value {{$i}}: %v", v))
				}
				o{{$i}} = v.Interface().({{getTypeString $type}})
			{{end}}

			return
		}
	{{end}}
{{end}}
`

var tmpl *template.Template

func init() {
	extraFuncs := make(template.FuncMap)
	extraFuncs["getMethods"] = getMethods
	extraFuncs["getInputs"] = getInputs
	extraFuncs["getOutputs"] = getOutputs
	extraFuncs["getTypeString"] = getTypeString

	tmpl = template.New("code")
	tmpl.Funcs(extraFuncs)
	tmpl.Parse(tmplStr)
}

func getTypeString(t reflect.Type) string {
	return t.String()
}

func getMethods(it reflect.Type) []reflect.Method {
	numMethods := it.NumMethod()
	methods := make([]reflect.Method, numMethods)

	for i := 0; i < numMethods; i++ {
		methods[i] = it.Method(i)
	}

	return methods
}

func getInputs(ft reflect.Type) []reflect.Type {
	numIn := ft.NumIn()
	inputs := make([]reflect.Type, numIn)

	for i := 0; i < numIn; i++ {
		inputs[i] = ft.In(i)
	}

	return inputs
}

func getOutputs(ft reflect.Type) []reflect.Type {
	numOut := ft.NumOut()
	outputs := make([]reflect.Type, numOut)

	for i := 0; i < numOut; i++ {
		outputs[i] = ft.Out(i)
	}

	return outputs
}

// A map from import identifier to package to use that identifier for,
// containing elements for each import needed by a set of mocked interfaces.
type importMap map[string]string

type tmplArg struct {
	// The package of the generated code.
	Pkg string

	// Imports needed by the interfaces.
	Imports importMap

	// The set of interfaces to mock.
	Interfaces []reflect.Type
}

// Add an import for the supplied type, without recursing.
func addImportForType(imports importMap, t reflect.Type) {
	// If there is no package path, this is a built-in type and we don't need an
	// import.
	pkgPath := t.PkgPath()
	if pkgPath == "" {
		return
	}

	// Work around a bug in Go:
	//
	//     http://code.google.com/p/go/issues/detail?id=2660
	//
	var errorPtr *error
	if t == reflect.TypeOf(errorPtr).Elem() {
		return
	}

	// Use the package's base name as an identifier.
	imports[path.Base(pkgPath)] = pkgPath
}

// Add all necessary imports for the type, recursing as appropriate.
func addImportsForType(imports importMap, t reflect.Type) {
	// Add any import needed for the type itself.
	addImportForType(imports, t)

	// Handle special cases where recursion is needed.
	switch t.Kind() {
	case reflect.Array, reflect.Chan, reflect.Ptr, reflect.Slice:
		addImportsForType(imports, t.Elem())

	case reflect.Func:
		// Input parameters.
		for i := 0; i < t.NumIn(); i++ {
			addImportsForType(imports, t.In(i))
		}

		// Return values.
		for i := 0; i < t.NumOut(); i++ {
			addImportsForType(imports, t.Out(i))
		}

	case reflect.Map:
		addImportsForType(imports, t.Key())
		addImportsForType(imports, t.Elem())
	}
}

// Add imports for each of the methods of the interface, but not the interface
// itself.
func addImportsForInterfaceMethods(imports importMap, it reflect.Type) {
	// Handle each method.
	for i := 0; i < it.NumMethod(); i++ {
		m := it.Method(i)
		addImportsForType(imports, m.Type)
	}
}

// Given a set of interfaces, return a map from import identifier to package to
// use that identifier for, containing elements for each import needed by the
// mock versions of those interfaces.
func getImports(interfaces []reflect.Type) importMap {
	imports := make(importMap)
	for _, it := range interfaces {
		addImportsForInterfaceMethods(imports, it)
	}

	return imports
}

// Given a set of interfaces to mock, write out source code for a package named
// `pkg` that contains mock implementations of those interfaces.
func GenerateMockSource(w io.Writer, pkg string, interfaces []reflect.Type) error {
	// Sanity-check arguments.
	if pkg == "" {
		return errors.New("Package name must be non-empty.")
	}

	if len(interfaces) == 0 {
		return errors.New("List of interfaces must be non-empty.")
	}

	// Make sure each type is indeed an interface.
	for _, it := range interfaces {
		if it.Kind() != reflect.Interface {
			return errors.New("Invalid type: " + it.String())
		}
	}

	// Create an appropriate template arg, then execute the template. Write the
	// raw output into a buffer.
	var arg tmplArg
	arg.Pkg = pkg
	arg.Imports = getImports(interfaces)
	arg.Interfaces = interfaces

	buf := new(bytes.Buffer)
	if err := tmpl.Execute(buf, arg); err != nil {
		return err
	}

	// Pretty-print the output.
	fset := token.NewFileSet()
	astFile, err := parser.ParseFile(fset, pkg + ".go", buf, parser.ParseComments)
	if err != nil {
		return errors.New("Error parsing generated code: " + err.Error())
	}

	if err = printer.Fprint(w, fset, astFile); err != nil {
		return errors.New("Error pretty printing: " + err.Error())
	}

	return nil
}
<commit_msg>Fixed struct ordering.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package generate implements code generation for mock classes. This is an
// implementation detail of the createmock command, which you probably want to
// use directly instead.
package generate

import (
	"bytes"
	"errors"
	"go/parser"
	"go/printer"
	"go/token"
	"io"
	"path"
	"reflect"
	"text/template"
)

const tmplStr =
`package {{.Pkg}}

import (
	{{range $identifier, $import := .Imports}}
		{{$identifier}} "{{$import}}"
	{{end}}
)

{{range .Interfaces}}
	{{$structName := printf "mock%s" .Name}}

	type {{$structName}} struct {
		controller oglemock.Controller
		description string
	}

	func New{{printf "Mock%s" .Name}}(
		c oglemock.Controller,
		desc string) *{{$structName}} {
	  return &{{$structName}}{
			controller: c,
			description: desc,
		}
	}

	func (m *{{$structName}}) Oglemock_Id() uintptr {
		return uintptr(unsafe.Pointer(m))
	}

	func (m *{{$structName}}) Oglemock_Description() string {
		return m.description
	}

	{{range getMethods .}}
	  {{$inputTypes := getInputs .Type}}
	  {{$outputTypes := getOutputs .Type}}

		func (m *{{$structName}}) {{.Name}}({{range $i, $type := $inputTypes}}p{{$i}} {{getTypeString $type}}, {{end}}
		) ({{range $i, $type := $outputTypes}}o{{$i}} {{getTypeString $type}}, {{end}}
		){
			// Hand the call off to the controller, which does most of the work.
			retVals := m.controller.HandleMethodCall(m, "{{.Name}}")
			if len(retVals != {{len $outputTypes}}) {
				panic(fmt.Sprintf("{{$structName}}.{{.Name}}: invalid return values: %v", retVals))
			}

			var v reflect.Value
			{{range $i, $type := $outputTypes}}
				// o{{$i}} {{getTypeString $type}}
				v = reflect.ValueOf(retVals[{{$i}}])
				if v.Type() != reflect.TypeOf(o{{$i}}) {
					panic(fmt.Sprintf("{{$structName}}.{{.Name}}: invalid return value {{$i}}: %v", v))
				}
				o{{$i}} = v.Interface().({{getTypeString $type}})
			{{end}}

			return
		}
	{{end}}
{{end}}
`

type tmplArg struct {
	// The package of the generated code.
	Pkg string

	// Imports needed by the interfaces.
	Imports importMap

	// The set of interfaces to mock.
	Interfaces []reflect.Type
}

var tmpl *template.Template

func init() {
	extraFuncs := make(template.FuncMap)
	extraFuncs["getMethods"] = getMethods
	extraFuncs["getInputs"] = getInputs
	extraFuncs["getOutputs"] = getOutputs
	extraFuncs["getTypeString"] = getTypeString

	tmpl = template.New("code")
	tmpl.Funcs(extraFuncs)
	tmpl.Parse(tmplStr)
}

func getTypeString(t reflect.Type) string {
	return t.String()
}

func getMethods(it reflect.Type) []reflect.Method {
	numMethods := it.NumMethod()
	methods := make([]reflect.Method, numMethods)

	for i := 0; i < numMethods; i++ {
		methods[i] = it.Method(i)
	}

	return methods
}

func getInputs(ft reflect.Type) []reflect.Type {
	numIn := ft.NumIn()
	inputs := make([]reflect.Type, numIn)

	for i := 0; i < numIn; i++ {
		inputs[i] = ft.In(i)
	}

	return inputs
}

func getOutputs(ft reflect.Type) []reflect.Type {
	numOut := ft.NumOut()
	outputs := make([]reflect.Type, numOut)

	for i := 0; i < numOut; i++ {
		outputs[i] = ft.Out(i)
	}

	return outputs
}

// A map from import identifier to package to use that identifier for,
// containing elements for each import needed by a set of mocked interfaces.
type importMap map[string]string

// Add an import for the supplied type, without recursing.
func addImportForType(imports importMap, t reflect.Type) {
	// If there is no package path, this is a built-in type and we don't need an
	// import.
	pkgPath := t.PkgPath()
	if pkgPath == "" {
		return
	}

	// Work around a bug in Go:
	//
	//     http://code.google.com/p/go/issues/detail?id=2660
	//
	var errorPtr *error
	if t == reflect.TypeOf(errorPtr).Elem() {
		return
	}

	// Use the package's base name as an identifier.
	imports[path.Base(pkgPath)] = pkgPath
}

// Add all necessary imports for the type, recursing as appropriate.
func addImportsForType(imports importMap, t reflect.Type) {
	// Add any import needed for the type itself.
	addImportForType(imports, t)

	// Handle special cases where recursion is needed.
	switch t.Kind() {
	case reflect.Array, reflect.Chan, reflect.Ptr, reflect.Slice:
		addImportsForType(imports, t.Elem())

	case reflect.Func:
		// Input parameters.
		for i := 0; i < t.NumIn(); i++ {
			addImportsForType(imports, t.In(i))
		}

		// Return values.
		for i := 0; i < t.NumOut(); i++ {
			addImportsForType(imports, t.Out(i))
		}

	case reflect.Map:
		addImportsForType(imports, t.Key())
		addImportsForType(imports, t.Elem())
	}
}

// Add imports for each of the methods of the interface, but not the interface
// itself.
func addImportsForInterfaceMethods(imports importMap, it reflect.Type) {
	// Handle each method.
	for i := 0; i < it.NumMethod(); i++ {
		m := it.Method(i)
		addImportsForType(imports, m.Type)
	}
}

// Given a set of interfaces, return a map from import identifier to package to
// use that identifier for, containing elements for each import needed by the
// mock versions of those interfaces.
func getImports(interfaces []reflect.Type) importMap {
	imports := make(importMap)
	for _, it := range interfaces {
		addImportsForInterfaceMethods(imports, it)
	}

	return imports
}

// Given a set of interfaces to mock, write out source code for a package named
// `pkg` that contains mock implementations of those interfaces.
func GenerateMockSource(w io.Writer, pkg string, interfaces []reflect.Type) error {
	// Sanity-check arguments.
	if pkg == "" {
		return errors.New("Package name must be non-empty.")
	}

	if len(interfaces) == 0 {
		return errors.New("List of interfaces must be non-empty.")
	}

	// Make sure each type is indeed an interface.
	for _, it := range interfaces {
		if it.Kind() != reflect.Interface {
			return errors.New("Invalid type: " + it.String())
		}
	}

	// Create an appropriate template arg, then execute the template. Write the
	// raw output into a buffer.
	var arg tmplArg
	arg.Pkg = pkg
	arg.Imports = getImports(interfaces)
	arg.Interfaces = interfaces

	buf := new(bytes.Buffer)
	if err := tmpl.Execute(buf, arg); err != nil {
		return err
	}

	// Pretty-print the output.
	fset := token.NewFileSet()
	astFile, err := parser.ParseFile(fset, pkg + ".go", buf, parser.ParseComments)
	if err != nil {
		return errors.New("Error parsing generated code: " + err.Error())
	}

	if err = printer.Fprint(w, fset, astFile); err != nil {
		return errors.New("Error pretty printing: " + err.Error())
	}

	return nil
}
<|endoftext|>


<commit_before>//
//   Copyright 2017 Deadlock X42 <deadlock.x42@gmail.com>
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

package generate

import (
	"fmt"
	"strings"
	"unicode"
)

// ValidationResults is a list of warnings and errors that are produced by validating
// a generator.
type ValidationResults struct {
	Warnings []string
	Errors   []string
}

// Validate checks the correctness of the generator structure.
//
// For a generator it will
// 	- warn if the copyright is empty
//  - error if the "Begin" type isn't defined
//
// For an object it will
//  - warn if the documentation is empty
//	- error if the name of the object is not a valid identifier
//	- error if the name of the object is a reserved or predeclared word
//
// For each field of an object it will
//	- TODO: error if the name of the field is not a valid identifier
//	- TODO: error if the type of the field is not defined
//
// For fields with precludes defined it will
//	- TODO: error if the a field name in precludes does not exist
//	- TODO: error if the field in the precludes is set
//
// For an array it will
//  - warn if the documentation is empty
//	- error if the name of the array is not a valid identifier
//	- TODO: error if the type of the array is not defined
//
func Validate(g Generator) (*ValidationResults, error) {
	r := &ValidationResults{
		Warnings: []string{},
		Errors:   []string{},
	}
	err := g.Accept(r)
	return r, err
}

// VisitGenerator validates a generator.
func (r *ValidationResults) VisitGenerator(g Generator) error {
	// Warn if the copyright isn't defined.
	if len(g.Copyright()) == 0 {
		r.Warnings = append(r.Warnings, "Empty copyright")
	}
	// Error if the begin type isn't defined.
	if maps.objects[g.Begin()] == nil && maps.arrays[g.Begin()] == nil {
		r.Errors = append(r.Errors, fmt.Sprintf("Begin %q not defined.", g.Begin()))
	}
	return nil
}

// VisitObject validates a generator object.
func (r *ValidationResults) VisitObject(o Object) error {
	name := strings.ToLower(o.Name())
	// Error if the name of the object is not a valid identifier.
	if !isIdentifier(name) {
		r.Errors = append(r.Errors, fmt.Sprintf("Object name %q is not a valid identifier.", o.Name()))
	}
	// Error if the name of the object is a reserved or predeclared word.
	if isReserved(name) || isPredeclared(name) {
		r.Errors = append(r.Errors, fmt.Sprintf("Object name %q cannot be a reserved or predeclared word.", o.Name()))
	}
	// Warn if the documentation doesn't exist.
	if len(o.Documentation()) == 0 {
		r.Warnings = append(r.Warnings, fmt.Sprintf("Documentation for object %q does not exist.", o.Name()))
	}
	return nil
}

// VisitArray validates a generator array.
func (r *ValidationResults) VisitArray(a Array) error {
	name := strings.ToLower(a.Name())
	// Error if the name of the array is not a valid identifier.
	if !isIdentifier(name) {
		r.Errors = append(r.Errors, fmt.Sprintf("Array name %q is not a valid identifier.", a.Name()))
	}
	// Warn if the documentation doesn't exist.
	if len(a.Documentation()) == 0 {
		r.Warnings = append(r.Warnings, fmt.Sprintf("Documentation for array %q does not exist.", a.Name()))
	}
	return nil
}

var reserved = map[string]bool{
	"break": true, "default": true, "func": true, "interface": true, "select": true,
	"case": true, "defer": true, "go": true, "map": true, "struct": true,
	"chan": true, "else": true, "goto": true, "package": true, "switch": true,
	"const": true, "fallthrough": true, "if": true, "range": true, "type": true,
	"continue": true, "for": true, "import": true, "return": true, "var": true,
}

func isReserved(s string) bool {
	return reserved[s]
}

var predeclared = map[string]bool{
	"bool": true, "byte": true, "complex64": true, "complex128": true, "error": true,
	"float32": true, "float64": true, "int": true, "int8": true, "int16": true,
	"int32": true, "int64": true, "rune": true, "string": true, "uint": true,
	"uint8": true, "uint16": true, "uint32": true, "uint64": true, "uintptr": true,
	"true": true, "false": true, "iota": true, "nil": true, "append": true,
	"cap": true, "close": true, "complex": true, "copy": true, "delete": true,
	"imag": true, "len": true, "make": true, "new": true, "panic": true,
	"print": true, "println": true, "real": true, "recover": true,
}

func isPredeclared(s string) bool {
	return predeclared[s]
}

func isIdentifier(s string) bool {
	first := true
	for _, r := range s {
		switch first {
		case true:
			first = false
			if unicode.IsLetter(r) || r == '_' {
				continue
			}
			return false
		case false:
			if unicode.IsLetter(r) || r == '_' || unicode.IsDigit(r) {
				continue
			}
			return false
		}
	}
	return true
}
<commit_msg>error if the type of the array is not defined<commit_after>//
//   Copyright 2017 Deadlock X42 <deadlock.x42@gmail.com>
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.
//

package generate

import (
	"fmt"
	"strings"
	"unicode"
)

// ValidationResults is a list of warnings and errors that are produced by validating
// a generator.
type ValidationResults struct {
	Warnings []string
	Errors   []string
}

// Validate checks the correctness of the generator structure.
//
// For a generator it will
// 	- warn if the copyright is empty
//  - error if the "Begin" type isn't defined
//
// For an object it will
//  - warn if the documentation is empty
//	- error if the name of the object is not a valid identifier
//	- error if the name of the object is a reserved or predeclared word
//
// For each field of an object it will
//	- TODO: error if the name of the field is not a valid identifier
//	- TODO: error if the type of the field is not defined
//
// For fields with precludes defined it will
//	- TODO: error if the a field name in precludes does not exist
//	- TODO: error if the field in the precludes is set
//
// For an array it will
//  - warn if the documentation is empty
//	- error if the name of the array is not a valid identifier
//	- error if the type of the array is not defined
//
func Validate(g Generator) (*ValidationResults, error) {
	r := &ValidationResults{
		Warnings: []string{},
		Errors:   []string{},
	}
	err := g.Accept(r)
	return r, err
}

// VisitGenerator validates a generator.
func (r *ValidationResults) VisitGenerator(g Generator) error {
	// Warn if the copyright isn't defined.
	if len(g.Copyright()) == 0 {
		r.Warnings = append(r.Warnings, "Empty copyright")
	}
	// Error if the begin type isn't defined.
	if !isType(g.Begin()) {
		r.Errors = append(r.Errors, fmt.Sprintf("Begin %q not defined.", g.Begin()))
	}
	return nil
}

// VisitObject validates a generator object.
func (r *ValidationResults) VisitObject(o Object) error {
	name := strings.ToLower(o.Name())
	// Error if the name of the object is not a valid identifier.
	if !isIdentifier(name) {
		r.Errors = append(r.Errors, fmt.Sprintf("Object name %q is not a valid identifier.", o.Name()))
	}
	// Error if the name of the object is a reserved or predeclared word.
	if isReserved(name) || isPredeclared(name) {
		r.Errors = append(r.Errors, fmt.Sprintf("Object name %q cannot be a reserved or predeclared word.", o.Name()))
	}
	// Warn if the documentation doesn't exist.
	if len(o.Documentation()) == 0 {
		r.Warnings = append(r.Warnings, fmt.Sprintf("Documentation for object %q does not exist.", o.Name()))
	}
	return nil
}

// VisitArray validates a generator array.
func (r *ValidationResults) VisitArray(a Array) error {
	name := strings.ToLower(a.Name())
	// Error if the name of the array is not a valid identifier.
	if !isIdentifier(name) {
		r.Errors = append(r.Errors, fmt.Sprintf("Array name %q is not a valid identifier.", a.Name()))
	}
	// Warn if the documentation doesn't exist.
	if len(a.Documentation()) == 0 {
		r.Warnings = append(r.Warnings, fmt.Sprintf("Documentation for array %q does not exist.", a.Name()))
	}
	// Error if the array type isn't defined.
	if !isType(a.Type()) {
		r.Errors = append(r.Errors, fmt.Sprintf("Array type %q not defined.", a.Type()))
	}
	return nil
}

var reserved = map[string]bool{
	"break": true, "default": true, "func": true, "interface": true, "select": true,
	"case": true, "defer": true, "go": true, "map": true, "struct": true,
	"chan": true, "else": true, "goto": true, "package": true, "switch": true,
	"const": true, "fallthrough": true, "if": true, "range": true, "type": true,
	"continue": true, "for": true, "import": true, "return": true, "var": true,
}

func isReserved(s string) bool {
	return reserved[s]
}

var predeclared = map[string]bool{
	"bool": true, "byte": true, "complex64": true, "complex128": true, "error": true,
	"float32": true, "float64": true, "int": true, "int8": true, "int16": true,
	"int32": true, "int64": true, "rune": true, "string": true, "uint": true,
	"uint8": true, "uint16": true, "uint32": true, "uint64": true, "uintptr": true,
	"true": true, "false": true, "iota": true, "nil": true, "append": true,
	"cap": true, "close": true, "complex": true, "copy": true, "delete": true,
	"imag": true, "len": true, "make": true, "new": true, "panic": true,
	"print": true, "println": true, "real": true, "recover": true,
}

func isPredeclared(s string) bool {
	return predeclared[s]
}

func isIdentifier(s string) bool {
	first := true
	for _, r := range s {
		switch first {
		case true:
			first = false
			if unicode.IsLetter(r) || r == '_' {
				continue
			}
			return false
		case false:
			if unicode.IsLetter(r) || r == '_' || unicode.IsDigit(r) {
				continue
			}
			return false
		}
	}
	return true
}

var types = map[string]bool{
	"bool": true, "uint8": true, "uint16": true, "uint32": true, "uint64": true,
	"int8": true, "int16": true, "int32": true, "int64": true, "float32": true,
	"float64": true, "complex64": true, "complex128": true, "byte": true, "rune": true,
	"uint": true, "int": true, "uintptr": true, "string": true,
}

func isType(s string) bool {
	if maps.objects[s] != nil || maps.arrays[s] != nil {
		return true
	}
	return types[s]
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/crabmusket/gosunspec/smdx"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"sort"
	"strings"
	"text/template"
)

const file = `
{{define "point"}}{{.Id}} {{.Type|goType}} ` + "`" + `sunspec:"offset={{.Offset}}{{if gt .Length 0}},len={{.Length}}{{end}}{{if ne .ScaleFactor ""}},sf={{.ScaleFactor}}{{end}}{{if ne .Access ""}},access={{.Access}}{{end}}"` + "`" + `{{end}}
{{define "model"}}
{{$model := index .Models 0 }}
{{$strings := index .Strings 0 }}

// Block{{$model.Id}} - {{$strings.ModelStrings.Label}} - {{$strings.ModelStrings.Description}}

const (
	ModelID = {{$model.Id}}
)

const (
{{range (uniqueNames $model)}}   {{.|title}} = "{{.}}"
{{end}}
)

{{range (repeatingBlocks $model)}}
type Block{{$model.Id}}Repeat struct {
	{{range .Points}}{{template "point" .}}
{{end}}
}
{{end}}

type Block{{$model.Id}} struct {
{{range (fixedBlocks $model)}}
{{range .Points}}    {{template "point" .}}
{{end}}
	{{end}}
{{range (repeatingBlocks $model)}}
  Repeats []Block{{$model.Id}}Repeat
{{end}}
}

func (self *Block{{$model.Id}}) GetId() sunspec.ModelId {
  return ModelID
}

func init() {
	smdx.RegisterModel(&smdx.ModelElement{
	Id: ModelID,
	Name: "{{$model.Name}}",
	Length: {{$model.Length}},
	Blocks: []smdx.BlockElement{ {{range $model.Blocks}}
			smdx.BlockElement{ {{if gt (len .Name) 0 }}Name: "{{.Name}}",{{end}}
				Length: {{.Length}},
				{{if gt (len .Type) 0 }}Type: "{{.Type}}",{{end}}
				Points: []smdx.PointElement{ {{range .Points}}
					smdx.PointElement{Id: {{.Id|title}},Offset: {{.Offset}},Type: typelabel.{{.Type|title}}{{optF "ScaleFactor" .ScaleFactor}}{{optF "Units" .Units}}{{optF "Access" .Access}}{{if gt (.Length) 0}},Length: {{.Length}}{{end}}{{if .Mandatory}},Mandatory: {{.Mandatory}}{{end}},}, {{end}}
				},
			}, {{end}}
	}})
}
{{end}}
// NOTICE
// This file was automatically generated by ../../generators/models.go. Do not edit it!
// You can regenerate it by running 'go generate ./models' from the directory above.

package model{{(index .Model.Models 0).Id}}

import (
	"github.com/crabmusket/gosunspec/smdx"
	"github.com/crabmusket/gosunspec"
	"github.com/crabmusket/gosunspec/typelabel"
)
{{template "model" .Model}}
`

const loader = `
// NOTICE
// This file was automatically generated by ../generators/models.go. Do not edit it!
// You can regenerate it by running 'go generate ./models' from the directory above.

package models

import (
{{range .Names}}   _ "github.com/crabmusket/gosunspec/models/{{.}}"
{{end}}
)
`

func main() {

	var smdxDir string

	flag.StringVar(&smdxDir, "smdx-dir", "../spec/smdx/", "The location of the SMDX directory")
	flag.Parse()

	files, err := ioutil.ReadDir(smdxDir)
	if err != nil {
		log.Fatal(err)
	}

	models := []smdx.ModelDefinitionElement{}
	for _, file := range files {
		if strings.HasPrefix(file.Name(), "smdx_") {
			smdxFile, err := os.OpenFile(smdxDir+file.Name(), os.O_RDONLY, 0644)
			if err != nil {
				log.Fatal(err)
			}

			def, err := smdx.FromXML(smdxFile)
			if err != nil {
				smdxFile.Close()
				log.Fatal(err)
			}
			smdxFile.Close()

			if len(def.Models) < 1 {
				log.Printf("failed to find any models in %s", file.Name())
				continue
			} else {
				if len(def.Models) > 1 {
					log.Printf("found additional models in %s", file.Name())
				}
				models = append(models, def)
			}
		}
	}

	t := template.New("file")
	t.Funcs(map[string]interface{}{
		"goType": func(sstype string) string {
			switch sstype {
			case "uint16", "uint32", "uint64", "int16", "float32", "int32", "int64", "string":
				return sstype
			case "sunssf":
				return "sunspec.ScaleFactor"
			default:
				return "sunspec." + strings.Title(sstype)
			}
		},
		"repeatingBlocks": func(m smdx.ModelElement) []smdx.BlockElement {
			if len(m.Blocks) == 1 && m.Blocks[0].Type == "repeating" {
				return m.Blocks
			} else if len(m.Blocks) > 1 {
				return m.Blocks[1:]
			} else {
				return []smdx.BlockElement{}
			}
		},
		"fixedBlocks": func(m smdx.ModelElement) []smdx.BlockElement {
			if len(m.Blocks) > 0 && m.Blocks[0].Type != "repeating" {
				return m.Blocks[0:1]
			} else {
				return []smdx.BlockElement{}
			}
		},
		"optF": func(n, v string) string {
			if len(v) > 0 {
				return "," + n + ": \"" + v + "\""
			} else {
				return ""
			}
		},
		"title": func(in string) string {
			switch in {
			case "sunssf":
				return "ScaleFactor"
			default:
				return strings.Title(in)
			}
		},
		"uniqueNames": func(m smdx.ModelElement) []string {
			seen := map[string]bool{}
			for _, b := range m.Blocks {
				for _, p := range b.Points {
					seen[p.Id] = true
				}
			}
			result := []string{}
			for n, _ := range seen {
				result = append(result, n)
			}
			sort.Strings(result)
			return result
		},
	})
	modelTemplate := template.Must(t.Parse(file))

	// write out all the model files

	for _, m := range models {
		dir := fmt.Sprintf("../models/model%d", m.Models[0].Id)
		if err := os.MkdirAll(dir, 0777); err != nil {
			log.Fatalf("%v", err)
		}
		outputFilename := dir + "/model.go.tmp"
		outputFile, err := os.OpenFile(outputFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if err != nil {
			log.Fatal(err)
		}
		if err := modelTemplate.Execute(outputFile, map[string]interface{}{
			"Model": m,
		}); err != nil {
			log.Fatalf("template execution failed: %v", err)
		}
		outputFile.Close()
		cmd := exec.Command("/bin/sh", "-c", "gofmt -w "+outputFilename)
		if err := cmd.Run(); err != nil {
			log.Fatalf("gofmt failed: %v", err)
		}
		if err := os.Rename(outputFilename, dir+"/model.go"); err != nil {
			log.Fatalf("replacing models.go failed: %v", err)
		}
	}

	// write out a loader module

	files, err = ioutil.ReadDir("../models/")
	if err != nil {
		log.Fatal(err)
	}
	names := []string{}
	for _, f := range files {
		if f.IsDir() && strings.HasPrefix(f.Name(), "model") {
			names = append(names, f.Name())
		}
	}
	outputFile, err := os.OpenFile("../models/loader.go", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		log.Fatal(err)
	}
	loaderTemplate := template.Must(t.Parse(loader))
	loaderTemplate.Execute(outputFile, map[string]interface{}{
		"Names": names,
	})
	outputFile.Close()
	cmd := exec.Command("/bin/sh", "-c", "gofmt -w ../models/loader.go")
	if err := cmd.Run(); err != nil {
		log.Fatalf("gofmt failed: %v", err)
	}

}
<commit_msg>generators: Add top-package as a parameter so that the loader works.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/crabmusket/gosunspec/smdx"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"sort"
	"strings"
	"text/template"
)

const file = `
{{define "point"}}{{.Id}} {{.Type|goType}} ` + "`" + `sunspec:"offset={{.Offset}}{{if gt .Length 0}},len={{.Length}}{{end}}{{if ne .ScaleFactor ""}},sf={{.ScaleFactor}}{{end}}{{if ne .Access ""}},access={{.Access}}{{end}}"` + "`" + `{{end}}
{{define "model"}}
{{$model := index .Models 0 }}
{{$strings := index .Strings 0 }}

// Block{{$model.Id}} - {{$strings.ModelStrings.Label}} - {{$strings.ModelStrings.Description}}

const (
	ModelID = {{$model.Id}}
)

const (
{{range (uniqueNames $model)}}   {{.|title}} = "{{.}}"
{{end}}
)

{{range (repeatingBlocks $model)}}
type Block{{$model.Id}}Repeat struct {
	{{range .Points}}{{template "point" .}}
{{end}}
}
{{end}}

type Block{{$model.Id}} struct {
{{range (fixedBlocks $model)}}
{{range .Points}}    {{template "point" .}}
{{end}}
	{{end}}
{{range (repeatingBlocks $model)}}
  Repeats []Block{{$model.Id}}Repeat
{{end}}
}

func (self *Block{{$model.Id}}) GetId() sunspec.ModelId {
  return ModelID
}

func init() {
	smdx.RegisterModel(&smdx.ModelElement{
	Id: ModelID,
	Name: "{{$model.Name}}",
	Length: {{$model.Length}},
	Blocks: []smdx.BlockElement{ {{range $model.Blocks}}
			smdx.BlockElement{ {{if gt (len .Name) 0 }}Name: "{{.Name}}",{{end}}
				Length: {{.Length}},
				{{if gt (len .Type) 0 }}Type: "{{.Type}}",{{end}}
				Points: []smdx.PointElement{ {{range .Points}}
					smdx.PointElement{Id: {{.Id|title}},Offset: {{.Offset}},Type: typelabel.{{.Type|title}}{{optF "ScaleFactor" .ScaleFactor}}{{optF "Units" .Units}}{{optF "Access" .Access}}{{if gt (.Length) 0}},Length: {{.Length}}{{end}}{{if .Mandatory}},Mandatory: {{.Mandatory}}{{end}},}, {{end}}
				},
			}, {{end}}
	}})
}
{{end}}
// NOTICE
// This file was automatically generated by ../../generators/models.go. Do not edit it!
// You can regenerate it by running 'go generate ./models' from the directory above.

package model{{(index .Model.Models 0).Id}}

import (
	"github.com/crabmusket/gosunspec/smdx"
	"github.com/crabmusket/gosunspec"
	"github.com/crabmusket/gosunspec/typelabel"
)
{{template "model" .Model}}
`

const loader = `
// NOTICE
// This file was automatically generated by ../generators/models.go. Do not edit it!
// You can regenerate it by running 'go generate ./models' from the directory above.

package models

{{$top := .TopPackage}}
import (
{{range .Names}}   _ "{{$top}}/models/{{.}}"
{{end}}
)
`

func main() {

	var smdxDir string
	var topPackage string

	flag.StringVar(&smdxDir, "smdx-dir", "../spec/smdx/", "The location of the SMDX directory")
	flag.StringVar(&topPackage, "top-package", "github.com/crabmusket/gosunspec", "The go path to the package containing the models package.")
	flag.Parse()

	files, err := ioutil.ReadDir(smdxDir)
	if err != nil {
		log.Fatal(err)
	}

	models := []smdx.ModelDefinitionElement{}
	for _, file := range files {
		if strings.HasPrefix(file.Name(), "smdx_") {
			smdxFile, err := os.OpenFile(smdxDir+file.Name(), os.O_RDONLY, 0644)
			if err != nil {
				log.Fatal(err)
			}

			def, err := smdx.FromXML(smdxFile)
			if err != nil {
				smdxFile.Close()
				log.Fatal(err)
			}
			smdxFile.Close()

			if len(def.Models) < 1 {
				log.Printf("failed to find any models in %s", file.Name())
				continue
			} else {
				if len(def.Models) > 1 {
					log.Printf("found additional models in %s", file.Name())
				}
				models = append(models, def)
			}
		}
	}

	t := template.New("file")
	t.Funcs(map[string]interface{}{
		"goType": func(sstype string) string {
			switch sstype {
			case "uint16", "uint32", "uint64", "int16", "float32", "int32", "int64", "string":
				return sstype
			case "sunssf":
				return "sunspec.ScaleFactor"
			default:
				return "sunspec." + strings.Title(sstype)
			}
		},
		"repeatingBlocks": func(m smdx.ModelElement) []smdx.BlockElement {
			if len(m.Blocks) == 1 && m.Blocks[0].Type == "repeating" {
				return m.Blocks
			} else if len(m.Blocks) > 1 {
				return m.Blocks[1:]
			} else {
				return []smdx.BlockElement{}
			}
		},
		"fixedBlocks": func(m smdx.ModelElement) []smdx.BlockElement {
			if len(m.Blocks) > 0 && m.Blocks[0].Type != "repeating" {
				return m.Blocks[0:1]
			} else {
				return []smdx.BlockElement{}
			}
		},
		"optF": func(n, v string) string {
			if len(v) > 0 {
				return "," + n + ": \"" + v + "\""
			} else {
				return ""
			}
		},
		"title": func(in string) string {
			switch in {
			case "sunssf":
				return "ScaleFactor"
			default:
				return strings.Title(in)
			}
		},
		"uniqueNames": func(m smdx.ModelElement) []string {
			seen := map[string]bool{}
			for _, b := range m.Blocks {
				for _, p := range b.Points {
					seen[p.Id] = true
				}
			}
			result := []string{}
			for n, _ := range seen {
				result = append(result, n)
			}
			sort.Strings(result)
			return result
		},
	})
	modelTemplate := template.Must(t.Parse(file))

	// write out all the model files

	for _, m := range models {
		dir := fmt.Sprintf("../models/model%d", m.Models[0].Id)
		if err := os.MkdirAll(dir, 0777); err != nil {
			log.Fatalf("%v", err)
		}
		outputFilename := dir + "/model.go.tmp"
		outputFile, err := os.OpenFile(outputFilename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
		if err != nil {
			log.Fatal(err)
		}
		if err := modelTemplate.Execute(outputFile, map[string]interface{}{
			"Model": m,
		}); err != nil {
			log.Fatalf("template execution failed: %v", err)
		}
		outputFile.Close()
		cmd := exec.Command("/bin/sh", "-c", "gofmt -w "+outputFilename)
		if err := cmd.Run(); err != nil {
			log.Fatalf("gofmt failed: %v", err)
		}
		if err := os.Rename(outputFilename, dir+"/model.go"); err != nil {
			log.Fatalf("replacing models.go failed: %v", err)
		}
	}

	// write out a loader module

	files, err = ioutil.ReadDir("../models/")
	if err != nil {
		log.Fatal(err)
	}
	names := []string{}
	for _, f := range files {
		if f.IsDir() && strings.HasPrefix(f.Name(), "model") {
			names = append(names, f.Name())
		}
	}
	outputFile, err := os.OpenFile("../models/loader.go", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		log.Fatal(err)
	}
	loaderTemplate := template.Must(t.Parse(loader))
	loaderTemplate.Execute(outputFile, map[string]interface{}{
		"Names":      names,
		"TopPackage": topPackage,
	})
	outputFile.Close()
	cmd := exec.Command("/bin/sh", "-c", "gofmt -w ../models/loader.go")
	if err := cmd.Run(); err != nil {
		log.Fatalf("gofmt failed: %v", err)
	}

}
<|endoftext|>