<commit_before>package app

import (
	"io/ioutil"
	"time"

	"github.com/mohae/contour"
	log "github.com/mohae/logwrap"
)

var (
	// Name is the name of the application
	Name string = "car"

	// ConfigFile is the name of the configuration file for the application.
	CfgFilename string = "app.json"
)

// Variables for configuration entries, or just hard code them.
var (
	CfgFile     string = "cfgfile"     // configuration filename; format type is inferred from ext.
	Log         string = "log"         // to log or not to log
	LogFile     string = "logfile"     // output filename for log output, stderr if empty
	Verbose            = "Verbose"     // Verbose output bool
	VerboseFile        = "VerboseFile" // output filename for Verbose output; stdout if empty.

	Format string = "format" // default archive format
	Type   string = "type"   // default compression type; does not apply to zip archives
)

var unsetTime time.Time

// set-up the application defaults and let contour know about the app.
// Setting also saves them to their relative environment variable.
func init() {
	// Calling Register* saves the configuration setting information
	// without writing it to its respective environment variable. This
	// allows any already set environment variables to override non-core
	// vars.
	//
	// Only settings that have been initialized are recognized by contour.

	// The config filename is handled differently, calling this function
	// also sets the ConfigFile format automatically,based on the
	// extension, if it can be determined. If it cannot, the extension is
	// left blank and must be set.
	contour.RegisterCfgFilename(CfgFile, CfgFilename)

	//// Alternative way of setting configuration file info; manually setting the values
	//contour.RegisterString("configfilename", ConfigFilename)
	//contour.RegisterString("configfileext", "json")

	// Core settings are only settable by the application, and once set are
	// immutable
	contour.RegisterStringCore("name", Name)

	// Immutable settings are only settable once. If its value isn't set
	// during registration, it can be set at a later time. Once it is set,
	// immutable values cannot be changed. Because of this, and the fact
	// that initialization causes bools to be set, bools cannot be made
	// immutable.

	// Set*Flag allows you to add settings that are also exposed as
	// command-line flags. Default implicit values to settings:
	//	IsFlag = true
	//	IsIdempotent = false
	//	IsCore = false
	// The shortcode, 2nd parameter, can be left as an empty string, ""
	// if this flag doesn't support a shortcode.

	// Logging and output related
	contour.RegisterBoolFlag(Log, "l", false, "false", "enable/disable logging")
	contour.RegisterBoolFlag(Verbose, "v", false, "false", "Bool for verbose output")

	// AddSettingAlias sets an alias for the setting.
	// contour doesn't support alias yet
	//	contour.AddSettingAlias(CfgLog, "logenabled")
	initApp()

	// Now that the configuration is set, set app logging. May be overridden later.
	SetAppLogging()
}

// InitApp is the best place to add custom defaults for your application,
func initApp() {
	contour.RegisterStringFlag(Format, "f", "tar", "tar", "create an archive using the tar format")
	contour.RegisterStringFlag(Type, "t", "gzip", "gzip", "create an archive using the zip format")

	// Create operation modifiers
	contour.RegisterIntFlag("owner", "", 0, "0", "force UID as owner for added files")
	contour.RegisterIntFlag("group", "", 0, "0", "force GID as group for added files")
	contour.RegisterInt64Flag("mode", "", 0, "0", "force MASK as mode for added files")

	//	contour.RegisterBoolFlag("usefullpath", "u", false, ")

	// Extract Operation Modifiers
	//	contour.RegisterBoolFlag("keep-old-files", "k", false, "don't replace existing files when extracting")
	//	contour.RegisterBoolFlag("keep-newer-files", "", false, "don't replace existing files that are newer than their archive copies")
	//	contour.RegisterBoolFlag("overwrite", "", true, "overwrite existing files when extracting")
	////	contour.RegisterBoolFlag("atime-preserve", "", true)
	//	contour.RegisterBoolFlag("modification-time", "m", false, "don't extract file modified time")
	//	contour.RegisterBoolFlag("same-owner", "", true, "try extracting files with the same ownership")
	//	contour.RegisterBoolFlag("no-same-owner", "", false, "extract files as yourself")
	//	contour.RegisterBoolFlag("numeric-owner", "", true, "always use numbers for user/group names")
	//	contour.RegisterBoolFlag("same-permissions", "p", true, "extract permissions information")
	//	contour.RegisterBoolFlag("no-same-permissions", "", false, "do not extract permissions information")

	// Create Operation Local file selection
	contour.RegisterBoolFlag("delete-files", "D", false, "false", "remove files after adding them to the archive")
	contour.RegisterStringFlag("exclude", "", "", "", "exclude files, given as a PATTERN")
	contour.RegisterStringFlag("exclude-ext", "e", "", "", "exclude files with EXTENSIONS")
	contour.RegisterStringFlag("exclude-anchored", "", "", "", "exclude patterns match file name start")
	contour.RegisterStringFlag("include", "", "", "", "include files, given as a PATTERN")
	contour.RegisterStringFlag("include-ext", "i", "", "", "include files with EXTENSIONS")
	contour.RegisterStringFlag("include-anchored", "", "", "", "include patterns match file name start")
	//	contour.RegisterBoolFlag("wildcards", "", false, "false", "patterns use wildcards")
	//	contour.RegisterBoolFlag("no-wildcards", "", true, "true", "patters do not use wildcards")
	//	contour.RegisterTimeFlag("newer", "N", unsetTime, "only store files newer than DATE or File")
	//contour.RegisterTimeFlag("newer-mtime", "M", unsetTime, "not set", "only store files modified since DATE")
	//	contour.RegisterStringFlag("newer-file", "", "only store files newere than the DATE for FILENAMEE")

	// Register option aliases
	//	contour.RegisterFlagAlias("newer-date", "after-date")
	//	contour.RegisterFlagAlias("same-permissions", "preserve-permissions")
}

// InitConfig initialized the application's configuration. When the config is
// has been initialized, the preset-enivronment variables, application
// defaults, and your application's configuration file, should it have one,
// will all be merged according to the setting properties.
//
// After this, only overrides can occur via command flags.
func SetCfg() error {
	// Set config:
	//    Checks environment variables for settings, follows update rules.
	//    Retrieves config file and applies those settings, if and where
	//      applicable.
	//    Writes the resulting configuration settings to their env vars.
	//  After set config, only command flags can override the settings.
	//  If this is an interactive application, preference changes would
	//    also override certain settings. It may necessitate an additional
	//    flag or two.
	err := contour.SetCfg()
	if err != nil {
		log.Print(err)
		return err
	}

	// setting the cfg may have altered logging information. Set it.
	SetAppLogging()
	return nil
}

// SetAppLog sets the logger for package loggers and allow for custom-
// ization of the applications log. This is where app specific code for
// setting up the application's log should be.
//
// SetAppLog assumes that log is enabled if it has been called as its
// caller should be SetLog(). If you are going to call this from elsewhere,
// first make sure that log is enabled.
//
func SetAppLogging() {
	if !contour.GetBool(Log) {
		log.SetOutput(ioutil.Discard)
		goto verbose
	}
	// get the logfilename, if it's not set, use stderr
	log.SetOutputFile(contour.GetString(LogFile))

verbose:
	if !contour.GetBool(Verbose) {
		log.SetVerboseOutput(ioutil.Discard)
		return
	}
	log.SetOutputFile(contour.GetString(VerboseFile))
	return
}

// CloseLogs closes the logfiles; if there are any open
func CloseLogs() {
	log.CloseOutput()
}
<commit_msg>comment updated<commit_after>package app

import (
	"io/ioutil"
	"time"

	"github.com/mohae/contour"
	log "github.com/mohae/logwrap"
)

var (
	// Name is the name of the application
	Name string = "car"

	// ConfigFile is the name of the configuration file for the application.
	CfgFilename string = "app.json"
)

// Variables for configuration entries, or just hard code them.
var (
	CfgFile     string = "cfgfile"     // configuration filename; format type is inferred from ext.
	Log         string = "log"         // to log or not to log
	LogFile     string = "logfile"     // output filename for log output, stderr if empty
	Verbose            = "Verbose"     // Verbose output bool
	VerboseFile        = "VerboseFile" // output filename for Verbose output; stdout if empty.

	Format string = "format" // default archive format
	Type   string = "type"   // default compression type; does not apply to zip archives
)

var unsetTime time.Time

// set-up the application defaults and let contour know about the app.
// Setting also saves them to their relative environment variable.
func init() {
	// Calling Register* saves the configuration setting information
	// without writing it to its respective environment variable. This
	// allows any already set environment variables to override non-core
	// vars.
	//
	// Only settings that have been initialized are recognized by contour.

	// The config filename is handled differently, calling this function
	// also sets the ConfigFile format automatically,based on the
	// extension, if it can be determined. If it cannot, the extension is
	// left blank and must be set.
	contour.RegisterCfgFilename(CfgFile, CfgFilename)

	//// Alternative way of setting configuration file info; manually setting the values
	//contour.RegisterString("configfilename", ConfigFilename)
	//contour.RegisterString("configfileext", "json")

	// Core settings are only settable by the application, and once set are
	// immutable
	contour.RegisterStringCore("name", Name)

	// Immutable settings are only settable once. If its value isn't set
	// during registration, it can be set at a later time. Once it is set,
	// immutable values cannot be changed. Because of this, and the fact
	// that initialization causes bools to be set, bools cannot be made
	// immutable.

	// Set*Flag allows you to add settings that are also exposed as
	// command-line flags. Default implicit values to settings:
	//	IsFlag = true
	//	IsIdempotent = false
	//	IsCore = false
	// The shortcode, 2nd parameter, can be left as an empty string, ""
	// if this flag doesn't support a shortcode.

	// Logging and output related
	contour.RegisterBoolFlag(Log, "l", false, "false", "enable/disable logging")
	contour.RegisterBoolFlag(Verbose, "v", false, "false", "Bool for verbose output")

	// AddSettingAlias sets an alias for the setting.
	// contour doesn't support alias yet
	//	contour.AddSettingAlias(CfgLog, "logenabled")
	initApp()

	// Now that the configuration is set, set app logging. May be overridden later.
	SetAppLogging()
}

// InitApp is the best place to add custom defaults for your application,
func initApp() {
	contour.RegisterStringFlag(Format, "f", "tar", "tar", "create an archive using the tar format")
	contour.RegisterStringFlag(Type, "t", "gzip", "gzip", "create an archive using the zip format")

	// Create operation modifiers
	contour.RegisterIntFlag("owner", "", 0, "0", "force UID as owner for added files")
	contour.RegisterIntFlag("group", "", 0, "0", "force GID as group for added files")
	contour.RegisterInt64Flag("mode", "", 0, "0", "force MASK as mode for added files")

	//	contour.RegisterBoolFlag("usefullpath", "u", false, ")

	// Extract Operation Modifiers
	//	contour.RegisterBoolFlag("keep-old-files", "k", false, "don't replace existing files when extracting")
	//	contour.RegisterBoolFlag("keep-newer-files", "", false, "don't replace existing files that are newer than their archive copies")
	//	contour.RegisterBoolFlag("overwrite", "", true, "overwrite existing files when extracting")
	////	contour.RegisterBoolFlag("atime-preserve", "", true)
	//	contour.RegisterBoolFlag("modification-time", "m", false, "don't extract file modified time")
	//	contour.RegisterBoolFlag("same-owner", "", true, "try extracting files with the same ownership")
	//	contour.RegisterBoolFlag("no-same-owner", "", false, "extract files as yourself")
	//	contour.RegisterBoolFlag("numeric-owner", "", true, "always use numbers for user/group names")
	//	contour.RegisterBoolFlag("same-permissions", "p", true, "extract permissions information")
	//	contour.RegisterBoolFlag("no-same-permissions", "", false, "do not extract permissions information")

	// Create Operation Local file selection
	contour.RegisterBoolFlag("delete-files", "D", false, "false", "remove files after adding them to the archive")
	contour.RegisterStringFlag("exclude", "", "", "", "exclude files, given as a PATTERN")
	contour.RegisterStringFlag("exclude-ext", "e", "", "", "exclude files with EXTENSIONS")
	contour.RegisterStringFlag("exclude-anchored", "", "", "", "exclude patterns match file name start")
	contour.RegisterStringFlag("include", "", "", "", "include files, given as a PATTERN")
	contour.RegisterStringFlag("include-ext", "i", "", "", "include files with EXTENSIONS")
	contour.RegisterStringFlag("include-anchored", "", "", "", "include patterns match file name start")
	//	contour.RegisterBoolFlag("wildcards", "", false, "false", "patterns use wildcards")
	//	contour.RegisterBoolFlag("no-wildcards", "", true, "true", "patters do not use wildcards")
	//	contour.RegisterTimeFlag("newer", "N", unsetTime, "only store files newer than DATE or File")
	//contour.RegisterTimeFlag("newer-mtime", "M", unsetTime, "not set", "only store files modified since DATE")
	//	contour.RegisterStringFlag("newer-file", "", "only store files newere than the DATE for FILENAMEE")

	// Register option aliases
	//	contour.RegisterFlagAlias("newer-date", "after-date")
	//	contour.RegisterFlagAlias("same-permissions", "preserve-permissions")
}

// SetCfg initialized the application's configuration. When the config is
// has been initialized, the preset-enivronment variables, application
// defaults, and your application's configuration file, should it have one,
// will all be merged according to the setting properties.
//
// After this, only overrides can occur via command flags.
func SetCfg() error {
	// Set config:
	//    Checks environment variables for settings, follows update rules.
	//    Retrieves config file and applies those settings, if and where
	//      applicable.
	//    Writes the resulting configuration settings to their env vars.
	//  After set config, only command flags can override the settings.
	//  If this is an interactive application, preference changes would
	//    also override certain settings. It may necessitate an additional
	//    flag or two.
	err := contour.SetCfg()
	if err != nil {
		log.Print(err)
		return err
	}

	// setting the cfg may have altered logging information. Set it.
	SetAppLogging()
	return nil
}

// SetAppLog sets the logger for package loggers and allow for custom-
// ization of the applications log. This is where app specific code for
// setting up the application's log should be.
//
// SetAppLog assumes that log is enabled if it has been called as its
// caller should be SetLog(). If you are going to call this from elsewhere,
// first make sure that log is enabled.
//
func SetAppLogging() {
	if !contour.GetBool(Log) {
		log.SetOutput(ioutil.Discard)
		goto verbose
	}
	// get the logfilename, if it's not set, use stderr
	log.SetOutputFile(contour.GetString(LogFile))

verbose:
	if !contour.GetBool(Verbose) {
		log.SetVerboseOutput(ioutil.Discard)
		return
	}
	log.SetOutputFile(contour.GetString(VerboseFile))
	return
}

// CloseLogs closes the logfiles; if there are any open
func CloseLogs() {
	log.CloseOutput()
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcp

import (
	"context"
	"fmt"
	"log"
	"net/url"

	apiv1 "cloud-android-orchestration/api/v1"
	"cloud-android-orchestration/app"

	"google.golang.org/api/compute/v1"
)

const (
	labelPrefix          = "cf-"
	labelAcloudCreatedBy = "created_by" // required for acloud backwards compatibility
	labelCreatedBy       = labelPrefix + "created_by"
)

// GCP implementation of the instance manager.
type InstanceManager struct {
	Config                app.IMConfig
	Service               *compute.Service
	InstanceNameGenerator NameGenerator
}

func (m *InstanceManager) GetHostAddr(zone string, host string) (string, error) {
	instance, err := m.getHostInstance(zone, host)
	if err != nil {
		return "", err
	}
	ilen := len(instance.NetworkInterfaces)
	if ilen == 0 {
		log.Printf("host instance %s in zone %s is missing a network interface", host, zone)
		return "", app.NewInternalError("host instance missing a network interface", nil)
	}
	if ilen > 1 {
		log.Printf("host instance %s in zone %s has %d network interfaces", host, zone, ilen)
	}
	return instance.NetworkInterfaces[0].NetworkIP, nil
}

const (
	hostURLScheme = "http"
	hostURLPort   = 1080
)

func (m *InstanceManager) GetHostURL(zone string, host string) (*url.URL, error) {
	addr, err := m.GetHostAddr(zone, host)
	if err != nil {
		return nil, err
	}
	return url.Parse(fmt.Sprintf("%s://%s:%d", hostURLScheme, addr, hostURLPort))
}

const operationStatusDone = "DONE"

func (m *InstanceManager) CreateHost(zone string, req *apiv1.CreateHostRequest, user app.UserInfo) (*apiv1.Operation, error) {
	if err := validateRequest(req); err != nil {
		return nil, err
	}
	labels := map[string]string{
		labelAcloudCreatedBy: user.Username(),
		labelCreatedBy:       user.Username(),
	}
	payload := &compute.Instance{
		Name:           m.InstanceNameGenerator.NewName(),
		MachineType:    req.CreateHostInstanceRequest.GCP.MachineType,
		MinCpuPlatform: req.CreateHostInstanceRequest.GCP.MinCPUPlatform,
		Disks: []*compute.AttachedDisk{
			{
				InitializeParams: &compute.AttachedDiskInitializeParams{
					DiskSizeGb:  int64(req.CreateHostInstanceRequest.GCP.DiskSizeGB),
					SourceImage: m.Config.GCP.HostImage,
				},
				Boot: true,
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			{
				Name: buildDefaultNetworkName(m.Config.GCP.ProjectID),
				AccessConfigs: []*compute.AccessConfig{
					{
						Name: "External NAT",
						Type: "ONE_TO_ONE_NAT",
					},
				},
			},
		},
		Labels: labels,
	}
	op, err := m.Service.Instances.
		Insert(m.Config.GCP.ProjectID, zone, payload).
		Context(context.TODO()).
		Do()
	if err != nil {
		return nil, err
	}
	result := &apiv1.Operation{
		Name: op.Name,
		Done: op.Status == operationStatusDone,
	}
	return result, nil
}

const listHostsRequestMaxResultsLimit uint32 = 500

func (m *InstanceManager) ListHosts(zone string, user app.UserInfo, req *app.ListHostsRequest) (*apiv1.ListHostsResponse, error) {
	var maxResults uint32
	if req.MaxResults <= listHostsRequestMaxResultsLimit {
		maxResults = req.MaxResults
	} else {
		maxResults = listHostsRequestMaxResultsLimit
	}
	res, err := m.Service.Instances.
		List(m.Config.GCP.ProjectID, zone).
		Context(context.TODO()).
		MaxResults(int64(maxResults)).
		PageToken(req.PageToken).
		Filter(fmt.Sprintf("labels.%s:%s", labelCreatedBy, user.Username())).
		Do()
	if err != nil {
		return nil, err
	}
	var items []*apiv1.HostInstance
	for _, i := range res.Items {
		hi, err := BuildHostInstance(i)
		if err != nil {
			return nil, err
		}
		items = append(items, hi)
	}
	return &apiv1.ListHostsResponse{
		Items:         items,
		NextPageToken: res.NextPageToken,
	}, nil
}

func (m *InstanceManager) getHostInstance(zone string, host string) (*compute.Instance, error) {
	return m.Service.Instances.
		Get(m.Config.GCP.ProjectID, zone, host).
		Context(context.TODO()).
		Do()
}

func validateRequest(r *apiv1.CreateHostRequest) error {
	if r.CreateHostInstanceRequest == nil ||
		r.CreateHostInstanceRequest.GCP == nil ||
		r.CreateHostInstanceRequest.GCP.DiskSizeGB == 0 ||
		r.CreateHostInstanceRequest.GCP.MachineType == "" {
		return app.NewBadRequestError("invalid CreateHostRequest", nil)
	}
	return nil
}

func buildDefaultNetworkName(projectID string) string {
	return fmt.Sprintf("projects/%s/global/networks/default", projectID)
}

func BuildHostInstance(in *compute.Instance) (*apiv1.HostInstance, error) {
	disksLen := len(in.Disks)
	if disksLen == 0 {
		log.Printf("invalid host instance %q: has 0 disks", in.SelfLink)
		return nil, app.NewInternalError("invalid host instance: has 0 disks", nil)
	}
	if disksLen > 1 {
		log.Printf("invalid host instance %q: has %d (more than one) disks", in.SelfLink, disksLen)
	}
	return &apiv1.HostInstance{
		Name: in.Name,
		GCP: &apiv1.GCPInstance{
			DiskSizeGB:     in.Disks[0].DiskSizeGb,
			MachineType:    in.MachineType,
			MinCPUPlatform: in.MinCpuPlatform,
		},
	}, nil
}

const hostInstanceNamePrefix = "cf-"

type NameGenerator interface {
	NewName() string
}

type InstanceNameGenerator struct {
	UUIDFactory func() string
}

func (g *InstanceNameGenerator) NewName() string {
	return hostInstanceNamePrefix + g.UUIDFactory()
}
<commit_msg>Revert "Use acloud ownership label. (#24)" (#25)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcp

import (
	"context"
	"fmt"
	"log"
	"net/url"

	apiv1 "cloud-android-orchestration/api/v1"
	"cloud-android-orchestration/app"

	"google.golang.org/api/compute/v1"
)

const (
	labelPrefix          = "cf-"
	labelAcloudCreatedBy = "created_by" // required for acloud backwards compatibility
	labelCreatedBy       = labelPrefix + "created_by"
)

// GCP implementation of the instance manager.
type InstanceManager struct {
	Config                app.IMConfig
	Service               *compute.Service
	InstanceNameGenerator NameGenerator
}

func (m *InstanceManager) GetHostAddr(zone string, host string) (string, error) {
	instance, err := m.getHostInstance(zone, host)
	if err != nil {
		return "", err
	}
	ilen := len(instance.NetworkInterfaces)
	if ilen == 0 {
		log.Printf("host instance %s in zone %s is missing a network interface", host, zone)
		return "", app.NewInternalError("host instance missing a network interface", nil)
	}
	if ilen > 1 {
		log.Printf("host instance %s in zone %s has %d network interfaces", host, zone, ilen)
	}
	return instance.NetworkInterfaces[0].NetworkIP, nil
}

const (
	hostURLScheme = "http"
	hostURLPort   = 1080
)

func (m *InstanceManager) GetHostURL(zone string, host string) (*url.URL, error) {
	addr, err := m.GetHostAddr(zone, host)
	if err != nil {
		return nil, err
	}
	return url.Parse(fmt.Sprintf("%s://%s:%d", hostURLScheme, addr, hostURLPort))
}

const operationStatusDone = "DONE"

func (m *InstanceManager) CreateHost(zone string, req *apiv1.CreateHostRequest, user app.UserInfo) (*apiv1.Operation, error) {
	if err := validateRequest(req); err != nil {
		return nil, err
	}
	labels := map[string]string{
		labelAcloudCreatedBy: user.Username(),
		labelCreatedBy:       user.Username(),
	}
	payload := &compute.Instance{
		Name:           m.InstanceNameGenerator.NewName(),
		MachineType:    req.CreateHostInstanceRequest.GCP.MachineType,
		MinCpuPlatform: req.CreateHostInstanceRequest.GCP.MinCPUPlatform,
		Disks: []*compute.AttachedDisk{
			{
				InitializeParams: &compute.AttachedDiskInitializeParams{
					DiskSizeGb:  int64(req.CreateHostInstanceRequest.GCP.DiskSizeGB),
					SourceImage: m.Config.GCP.HostImage,
				},
				Boot: true,
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			{
				Name: buildDefaultNetworkName(m.Config.GCP.ProjectID),
				AccessConfigs: []*compute.AccessConfig{
					{
						Name: "External NAT",
						Type: "ONE_TO_ONE_NAT",
					},
				},
			},
		},
		Labels: labels,
	}
	op, err := m.Service.Instances.
		Insert(m.Config.GCP.ProjectID, zone, payload).
		Context(context.TODO()).
		Do()
	if err != nil {
		return nil, err
	}
	result := &apiv1.Operation{
		Name: op.Name,
		Done: op.Status == operationStatusDone,
	}
	return result, nil
}

const listHostsRequestMaxResultsLimit uint32 = 500

func (m *InstanceManager) ListHosts(zone string, user app.UserInfo, req *app.ListHostsRequest) (*apiv1.ListHostsResponse, error) {
	var maxResults uint32
	if req.MaxResults <= listHostsRequestMaxResultsLimit {
		maxResults = req.MaxResults
	} else {
		maxResults = listHostsRequestMaxResultsLimit
	}
	res, err := m.Service.Instances.
		List(m.Config.GCP.ProjectID, zone).
		Context(context.TODO()).
		MaxResults(int64(maxResults)).
		PageToken(req.PageToken).
		Filter("labels.cf-created_by:" + user.Username()).
		Do()
	if err != nil {
		return nil, err
	}
	var items []*apiv1.HostInstance
	for _, i := range res.Items {
		hi, err := BuildHostInstance(i)
		if err != nil {
			return nil, err
		}
		items = append(items, hi)
	}
	return &apiv1.ListHostsResponse{
		Items:         items,
		NextPageToken: res.NextPageToken,
	}, nil
}

func (m *InstanceManager) getHostInstance(zone string, host string) (*compute.Instance, error) {
	return m.Service.Instances.
		Get(m.Config.GCP.ProjectID, zone, host).
		Context(context.TODO()).
		Do()
}

func validateRequest(r *apiv1.CreateHostRequest) error {
	if r.CreateHostInstanceRequest == nil ||
		r.CreateHostInstanceRequest.GCP == nil ||
		r.CreateHostInstanceRequest.GCP.DiskSizeGB == 0 ||
		r.CreateHostInstanceRequest.GCP.MachineType == "" {
		return app.NewBadRequestError("invalid CreateHostRequest", nil)
	}
	return nil
}

func buildDefaultNetworkName(projectID string) string {
	return fmt.Sprintf("projects/%s/global/networks/default", projectID)
}

func BuildHostInstance(in *compute.Instance) (*apiv1.HostInstance, error) {
	disksLen := len(in.Disks)
	if disksLen == 0 {
		log.Printf("invalid host instance %q: has 0 disks", in.SelfLink)
		return nil, app.NewInternalError("invalid host instance: has 0 disks", nil)
	}
	if disksLen > 1 {
		log.Printf("invalid host instance %q: has %d (more than one) disks", in.SelfLink, disksLen)
	}
	return &apiv1.HostInstance{
		Name: in.Name,
		GCP: &apiv1.GCPInstance{
			DiskSizeGB:     in.Disks[0].DiskSizeGb,
			MachineType:    in.MachineType,
			MinCPUPlatform: in.MinCpuPlatform,
		},
	}, nil
}

const hostInstanceNamePrefix = "cf-"

type NameGenerator interface {
	NewName() string
}

type InstanceNameGenerator struct {
	UUIDFactory func() string
}

func (g *InstanceNameGenerator) NewName() string {
	return hostInstanceNamePrefix + g.UUIDFactory()
}
<|endoftext|>


<commit_before>// Package backend provides local and remote storage for restic repositories.
// All backends need to implement the Backend interface. There is a
// MockBackend, which can be used for mocking in tests, and a MemBackend, which
// stores all data in a hash internally.
package backend
<commit_msg>Correct comment<commit_after>// Package backend provides local and remote storage for restic repositories.
// All backends need to implement the Backend interface. There is a MemBackend,
// which stores all data in a map internally and can be used for testing.
package backend
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"

	"bufio"
	"flag"
	"net"
	"net/http"
	"os"
	"regexp"
)

// modes
const VirtualizeMode = "virtualize"
const SynthesizeMode = "synthesize"
const ModifyMode = "modify"
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

func main() {
	// Output to stderr instead of stdout, could also be a file.
	log.SetOutput(os.Stderr)
	log.SetFormatter(&log.TextFormatter{})

	// getting proxy configuration
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	// modes
	capture := flag.Bool("capture", false, "should proxy capture requests")
	synthesize := flag.Bool("synthesize", false, "should proxy capture requests")
	modify := flag.Bool("modify", false, "should proxy only modify requests")

	destination := flag.String("destination", ".", "destination URI to catch")
	middleware := flag.String("middleware", "", "should proxy use middleware")
	flag.Parse()

	// getting settings
	cfg := InitSettings()

	if *verbose {
		// Only log the warning severity or above.
		log.SetLevel(log.DebugLevel)
	}
	cfg.verbose = *verbose

	// overriding default middleware setting
	cfg.middleware = *middleware

	// setting default mode
	mode := VirtualizeMode

	if *capture {
		mode = CaptureMode
		// checking whether user supplied other modes
		if *synthesize == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *synthesize {
		mode = SynthesizeMode

		if AppConfig.middleware == "" {
			log.Fatal("Synthesize mode chosen although middleware not supplied")
		}

		if *capture == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *modify {
		mode = ModifyMode

		if AppConfig.middleware == "" {
			log.Fatal("Modify mode chosen although middleware not supplied")
		}

		if *capture == true || *synthesize == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	}

	// overriding default settings
	AppConfig.mode = mode

	// overriding destination
	AppConfig.destination = *destination

	// getting default database
	port := os.Getenv("ProxyPort")
	if port == "" {
		port = DefaultPort
	} else {
		port = fmt.Sprintf(":%s", port)
	}

	// getting boltDB
	db := getDB(AppConfig.databaseName)
	defer db.Close()

	cache := Cache{
		db:             db,
		requestsBucket: []byte(requestsBucketName),
	}

	// getting connections
	d := DBClient{
		cache: cache,
		http:  &http.Client{},
	}

	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(*destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
		defer func() {
			if e := recover(); e != nil {
				ctx.Logf("error connecting to remote: %v", e)
				client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
			}
			client.Close()
		}()
		clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
		remote, err := net.Dial("tcp", req.URL.Host)
		orPanic(err)
		remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
		for {
			req, err := http.ReadRequest(clientBuf.Reader)
			orPanic(err)
			orPanic(req.Write(remoteBuf))
			orPanic(remoteBuf.Flush())
			resp, err := http.ReadResponse(remoteBuf.Reader, req)

			orPanic(err)
			orPanic(resp.Write(clientBuf.Writer))
			orPanic(clientBuf.Flush())
		}
	})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(*destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	go d.startAdminInterface()

	proxy.Verbose = *verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": *destination,
		"ProxyPort":   port,
		"Mode":        AppConfig.mode,
	}).Info("Proxy is starting...")

	log.Warn(http.ListenAndServe(port, proxy))
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	if AppConfig.mode == CaptureMode {
		log.Info("*** Capture ***")
		newResponse, err := d.captureRequest(req)
		if err != nil {
			// something bad happened, passing through
			return req, nil
		} else {
			// discarding original requests and returns supplied response
			return req, newResponse
		}

	} else if AppConfig.mode == SynthesizeMode {
		log.Info("*** Sinthesize ***")
		response := synthesizeResponse(req, AppConfig.middleware)
		return req, response

	} else if AppConfig.mode == ModifyMode {
		log.Info("*** Modify ***")
		response, err := d.modifyRequestResponse(req, AppConfig.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": AppConfig.middleware,
			}).Error("Got error when performing request modification")
			return req, nil
		}

		// returning modified response
		return req, response

	} else {
		log.Info("*** Virtualize ***")
		newResponse := d.getResponse(req)
		return req, newResponse

	}
}
<commit_msg>starting hoverfly function separated from main<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"

	"bufio"
	"flag"
	"net"
	"net/http"
	"os"
	"regexp"
)

// modes
const VirtualizeMode = "virtualize"
const SynthesizeMode = "synthesize"
const ModifyMode = "modify"
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

func main() {
	// Output to stderr instead of stdout, could also be a file.
	log.SetOutput(os.Stderr)
	log.SetFormatter(&log.TextFormatter{})

	// getting proxy configuration
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	// modes
	capture := flag.Bool("capture", false, "should proxy capture requests")
	synthesize := flag.Bool("synthesize", false, "should proxy capture requests")
	modify := flag.Bool("modify", false, "should proxy only modify requests")

	destination := flag.String("destination", ".", "destination URI to catch")
	middleware := flag.String("middleware", "", "should proxy use middleware")
	flag.Parse()

	// getting settings
	cfg := InitSettings()

	if *verbose {
		// Only log the warning severity or above.
		log.SetLevel(log.DebugLevel)
	}
	cfg.verbose = *verbose

	// overriding default middleware setting
	cfg.middleware = *middleware

	// setting default mode
	mode := VirtualizeMode

	if *capture {
		mode = CaptureMode
		// checking whether user supplied other modes
		if *synthesize == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *synthesize {
		mode = SynthesizeMode

		if cfg.middleware == "" {
			log.Fatal("Synthesize mode chosen although middleware not supplied")
		}

		if *capture == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *modify {
		mode = ModifyMode

		if cfg.middleware == "" {
			log.Fatal("Modify mode chosen although middleware not supplied")
		}

		if *capture == true || *synthesize == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	}

	// overriding default settings
	cfg.mode = mode

	// overriding destination
	cfg.destination = *destination

	proxy, dbClient := getNewHoverfly(&cfg)
	defer dbClient.cache.db.Close()

	log.Warn(http.ListenAndServe(cfg.proxyPort, proxy))
}

// getNewHoverfly returns a configured ProxyHttpServer and DBClient, also starts admin interface on configured port
func getNewHoverfly(cfg *Configuration) (*goproxy.ProxyHttpServer, DBClient) {

	// getting boltDB
	db := getDB(cfg.databaseName)

	cache := Cache{
		db:             db,
		requestsBucket: []byte(requestsBucketName),
	}

	// getting connections
	d := DBClient{
		cache: cache,
		http:  &http.Client{},
		cfg:   cfg,
	}

	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.cfg.destination))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.cfg.destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
		defer func() {
			if e := recover(); e != nil {
				ctx.Logf("error connecting to remote: %v", e)
				client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
			}
			client.Close()
		}()
		clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
		remote, err := net.Dial("tcp", req.URL.Host)
		orPanic(err)
		remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
		for {
			req, err := http.ReadRequest(clientBuf.Reader)
			orPanic(err)
			orPanic(req.Write(remoteBuf))
			orPanic(remoteBuf.Flush())
			resp, err := http.ReadResponse(remoteBuf.Reader, req)

			orPanic(err)
			orPanic(resp.Write(clientBuf.Writer))
			orPanic(clientBuf.Flush())
		}
	})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(cfg.destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	go d.startAdminInterface()

	proxy.Verbose = d.cfg.verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": d.cfg.destination,
		"ProxyPort":   d.cfg.proxyPort,
		"Mode":        d.cfg.mode,
	}).Info("Proxy prepared...")

	return proxy, d
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	log.WithFields(log.Fields{
		"mode": d.cfg.mode,
	}).Info("current mode")

	if d.cfg.mode == CaptureMode {
		log.Info("*** Capture ***")
		newResponse, err := d.captureRequest(req)
		if err != nil {
			// something bad happened, passing through
			return req, nil
		} else {
			// discarding original requests and returns supplied response
			return req, newResponse
		}

	} else if d.cfg.mode == SynthesizeMode {
		log.Info("*** Sinthesize ***")
		response := synthesizeResponse(req, d.cfg.middleware)
		return req, response

	} else if d.cfg.mode == ModifyMode {
		log.Info("*** Modify ***")
		response, err := d.modifyRequestResponse(req, d.cfg.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": d.cfg.middleware,
			}).Error("Got error when performing request modification")
			return req, nil
		}

		// returning modified response
		return req, response

	} else {
		log.Info("*** Virtualize ***")
		newResponse := d.getResponse(req)
		return req, newResponse

	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"time"

	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/timeutil"
)

const baseUpdatesURL = `https://register.cockroachdb.com/api/clusters/updates`
const baseReportingURL = `https://register.cockroachdb.com/api/report`

const updateCheckFrequency = time.Hour * 24
const updateCheckJitterSeconds = 120
const updateCheckRetryFrequency = time.Hour

type versionInfo struct {
	Version string `json:"version"`
	Details string `json:"details"`
}

type reportingInfo struct {
	Node   nodeInfo    `json:"node"`
	Stores []storeInfo `json:"stores"`
}

type nodeInfo struct {
	NodeID     roachpb.NodeID `json:"node_id"`
	Bytes      int            `json:"bytes"`
	KeyCount   int            `json:"key_count"`
	RangeCount int            `json:"range_count"`
}

type storeInfo struct {
	NodeID     roachpb.NodeID  `json:"node_id"`
	StoreID    roachpb.StoreID `json:"store_id"`
	Bytes      int             `json:"bytes"`
	KeyCount   int             `json:"key_count"`
	RangeCount int             `json:"range_count"`
}

// SetupReportingURLs parses the phone-home for version updates URL and should be
// called before server starts except in tests.
func (s *Server) SetupReportingURLs() error {
	var err error
	s.parsedUpdatesURL, err = url.Parse(baseUpdatesURL)
	if err != nil {
		return err
	}
	s.parsedReportingURL, err = url.Parse(baseReportingURL)
	if err != nil {
		return err
	}
	return nil
}

func (s *Server) periodicallyCheckForUpdates() {
	s.stopper.RunWorker(func() {
		startup := timeutil.Now()

		for {
			// `maybeCheckForUpdates` and `maybeReportUsage` both return the
			// duration until they should next be checked.
			// Wait for the shorter of the durations returned by the two checks.
			wait := s.maybeCheckForUpdates()
			if reportWait := s.maybeReportUsage(timeutil.Now().Sub(startup)); reportWait < wait {
				wait = reportWait
			}
			jitter := rand.Intn(updateCheckJitterSeconds) - updateCheckJitterSeconds/2
			wait = wait + (time.Duration(jitter) * time.Second)
			select {
			case <-s.stopper.ShouldDrain():
				return
			case <-time.After(wait):
			}
		}
	})
}

// Determines if it is time to check for updates and does so if it is.
// Returns a duration indicating when to make the next call to this method.
func (s *Server) maybeCheckForUpdates() time.Duration {
	return s.maybeRunPeriodicCheck("updates check", keys.UpdateCheckCluster, s.checkForUpdates)
}

// If the time is greater than the timestamp stored at `key`, run `f`.
// Before running `f`, the timestamp is updated forward by a small amount via
// a compare-and-swap to ensure at-most-one concurrent execution. After `f`
// executes the timestamp is set to the next execution time.
// Returns how long until `f` should be run next (i.e. when this method should
// be called again).
func (s *Server) maybeRunPeriodicCheck(op string, key roachpb.Key, f func()) time.Duration {
	resp, err := s.db.Get(key)
	if err != nil {
		log.Infof("Error reading %s time: %v", op, err)
		return updateCheckRetryFrequency
	}

	// We should early returned below if either the next check time is in the
	// future or if the atomic compare-and-set of that time failed (which
	// would happen if two nodes tried at the same time).
	if resp.Exists() {
		whenToCheck, pErr := resp.Value.GetTime()
		if pErr != nil {
			log.Warningf("Error decoding %s time: %v", op, err)
			return updateCheckRetryFrequency
		} else if delay := whenToCheck.Sub(timeutil.Now()); delay > 0 {
			return delay
		}

		nextRetry := whenToCheck.Add(updateCheckRetryFrequency)
		if err := s.db.CPut(key, nextRetry, whenToCheck); err != nil {
			if log.V(2) {
				log.Infof("Could not set next version check time (maybe another node checked?)", err)
			}
			return updateCheckRetryFrequency
		}
	} else {
		log.Infof("No previous %s time.", op)
		nextRetry := timeutil.Now().Add(updateCheckRetryFrequency)
		// CPut with `nil` prev value to assert that no other node has checked.
		if err := s.db.CPut(key, nextRetry, nil); err != nil {
			if log.V(2) {
				log.Infof("Could not set %s time (maybe another node checked?): %v", op, err)
			}
			return updateCheckRetryFrequency
		}
	}

	f()

	if err := s.db.Put(key, timeutil.Now().Add(updateCheckFrequency)); err != nil {
		log.Infof("Error updating %s time: %v", op, err)
	}
	return updateCheckFrequency
}

func (s *Server) checkForUpdates() {
	// TestServer.Start nils these out to prevent tests possibly phoning home.
	if s.parsedUpdatesURL == nil {
		return
	}

	q := s.parsedUpdatesURL.Query()
	q.Set("version", util.GetBuildInfo().Tag)
	q.Set("uuid", s.node.ClusterID.String())
	s.parsedUpdatesURL.RawQuery = q.Encode()

	res, err := http.Get(s.parsedUpdatesURL.String())
	if err != nil {
		// This is probably going to be relatively common in production
		// environments where network access is usually curtailed.
		if log.V(2) {
			log.Warning("Error checking for updates: ", err)
		}
	}
	defer res.Body.Close()

	decoder := json.NewDecoder(res.Body)
	r := struct {
		Details []versionInfo `json:"details"`
	}{}

	err = decoder.Decode(&r)
	if err != nil && err != io.EOF {
		log.Warning("Error decoding updates info: ", err)
		return
	}

	for _, v := range r.Details {
		log.Info("A new version is available: %s\n\t%s", v.Version, v.Details)
	}
}

func (s *Server) usageReportingEnabled() bool {
	resp, err := s.db.Get(keys.UpdateCheckReportUsage)
	if err != nil {
		log.Warning(err)
		return false
	}
	return resp.Exists() && resp.ValueInt() != 0
}

func (s *Server) maybeReportUsage(running time.Duration) time.Duration {
	if running < updateCheckRetryFrequency {
		// On first check, we decline to report usage as metrics may not yet
		// be stable, so instead we request re-evaluation after a retry delay.
		return updateCheckRetryFrequency - running
	}
	if !s.usageReportingEnabled() {
		return updateCheckFrequency
	}
	return s.maybeRunPeriodicCheck("metrics reporting", keys.NodeLastUsageReportKey(int32(s.node.Descriptor.NodeID)), s.reportUsage)
}

func (s *Server) getReportingInfo() reportingInfo {
	n := s.node.recorder.GetStatusSummary()

	summary := nodeInfo{NodeID: s.node.Descriptor.NodeID}

	stores := make([]storeInfo, len(n.StoreStatuses))
	for i, r := range n.StoreStatuses {
		stores[i].NodeID = r.Desc.Node.NodeID
		stores[i].StoreID = r.Desc.StoreID
		stores[i].KeyCount = int(r.Metrics["keycount"])
		summary.KeyCount += stores[i].KeyCount
		stores[i].RangeCount = int(r.Metrics["replicas"])
		summary.RangeCount += stores[i].RangeCount
		bytes := int(r.Metrics["sysbytes"] + r.Metrics["intentbytes"] + r.Metrics["valbytes"] + r.Metrics["keybytes"])
		stores[i].Bytes = bytes
		summary.Bytes += bytes
	}
	return reportingInfo{summary, stores}
}

func (s *Server) reportUsage() {
	// TestServer.Start nils these out to prevent tests possibly phoning home.
	if s.parsedReportingURL == nil {
		return
	}

	b := new(bytes.Buffer)
	if err := json.NewEncoder(b).Encode(s.getReportingInfo()); err != nil {
		log.Warning(err)
		return
	}

	q := s.parsedReportingURL.Query()
	q.Set("version", util.GetBuildInfo().Tag)
	q.Set("uuid", s.node.ClusterID.String())
	s.parsedReportingURL.RawQuery = q.Encode()

	_, err := http.Post(s.parsedReportingURL.String(), "application/json", b)
	if err != nil && log.V(2) {
		// This is probably going to be relatively common in production
		// environments where network access is usually curtailed.
		log.Warning("Error checking reporting node usage metrics: ", err)
	}
}
<commit_msg>server: bail out of update check on http error<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package server

import (
	"bytes"
	"encoding/json"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"time"

	"github.com/cockroachdb/cockroach/keys"
	"github.com/cockroachdb/cockroach/roachpb"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/timeutil"
)

const baseUpdatesURL = `https://register.cockroachdb.com/api/clusters/updates`
const baseReportingURL = `https://register.cockroachdb.com/api/report`

const updateCheckFrequency = time.Hour * 24
const updateCheckJitterSeconds = 120
const updateCheckRetryFrequency = time.Hour

type versionInfo struct {
	Version string `json:"version"`
	Details string `json:"details"`
}

type reportingInfo struct {
	Node   nodeInfo    `json:"node"`
	Stores []storeInfo `json:"stores"`
}

type nodeInfo struct {
	NodeID     roachpb.NodeID `json:"node_id"`
	Bytes      int            `json:"bytes"`
	KeyCount   int            `json:"key_count"`
	RangeCount int            `json:"range_count"`
}

type storeInfo struct {
	NodeID     roachpb.NodeID  `json:"node_id"`
	StoreID    roachpb.StoreID `json:"store_id"`
	Bytes      int             `json:"bytes"`
	KeyCount   int             `json:"key_count"`
	RangeCount int             `json:"range_count"`
}

// SetupReportingURLs parses the phone-home for version updates URL and should be
// called before server starts except in tests.
func (s *Server) SetupReportingURLs() error {
	var err error
	s.parsedUpdatesURL, err = url.Parse(baseUpdatesURL)
	if err != nil {
		return err
	}
	s.parsedReportingURL, err = url.Parse(baseReportingURL)
	if err != nil {
		return err
	}
	return nil
}

func (s *Server) periodicallyCheckForUpdates() {
	s.stopper.RunWorker(func() {
		startup := timeutil.Now()

		for {
			// `maybeCheckForUpdates` and `maybeReportUsage` both return the
			// duration until they should next be checked.
			// Wait for the shorter of the durations returned by the two checks.
			wait := s.maybeCheckForUpdates()
			if reportWait := s.maybeReportUsage(timeutil.Now().Sub(startup)); reportWait < wait {
				wait = reportWait
			}
			jitter := rand.Intn(updateCheckJitterSeconds) - updateCheckJitterSeconds/2
			wait = wait + (time.Duration(jitter) * time.Second)
			select {
			case <-s.stopper.ShouldDrain():
				return
			case <-time.After(wait):
			}
		}
	})
}

// Determines if it is time to check for updates and does so if it is.
// Returns a duration indicating when to make the next call to this method.
func (s *Server) maybeCheckForUpdates() time.Duration {
	return s.maybeRunPeriodicCheck("updates check", keys.UpdateCheckCluster, s.checkForUpdates)
}

// If the time is greater than the timestamp stored at `key`, run `f`.
// Before running `f`, the timestamp is updated forward by a small amount via
// a compare-and-swap to ensure at-most-one concurrent execution. After `f`
// executes the timestamp is set to the next execution time.
// Returns how long until `f` should be run next (i.e. when this method should
// be called again).
func (s *Server) maybeRunPeriodicCheck(op string, key roachpb.Key, f func()) time.Duration {
	resp, err := s.db.Get(key)
	if err != nil {
		log.Infof("Error reading %s time: %v", op, err)
		return updateCheckRetryFrequency
	}

	// We should early returned below if either the next check time is in the
	// future or if the atomic compare-and-set of that time failed (which
	// would happen if two nodes tried at the same time).
	if resp.Exists() {
		whenToCheck, pErr := resp.Value.GetTime()
		if pErr != nil {
			log.Warningf("Error decoding %s time: %v", op, err)
			return updateCheckRetryFrequency
		} else if delay := whenToCheck.Sub(timeutil.Now()); delay > 0 {
			return delay
		}

		nextRetry := whenToCheck.Add(updateCheckRetryFrequency)
		if err := s.db.CPut(key, nextRetry, whenToCheck); err != nil {
			if log.V(2) {
				log.Infof("Could not set next version check time (maybe another node checked?)", err)
			}
			return updateCheckRetryFrequency
		}
	} else {
		log.Infof("No previous %s time.", op)
		nextRetry := timeutil.Now().Add(updateCheckRetryFrequency)
		// CPut with `nil` prev value to assert that no other node has checked.
		if err := s.db.CPut(key, nextRetry, nil); err != nil {
			if log.V(2) {
				log.Infof("Could not set %s time (maybe another node checked?): %v", op, err)
			}
			return updateCheckRetryFrequency
		}
	}

	f()

	if err := s.db.Put(key, timeutil.Now().Add(updateCheckFrequency)); err != nil {
		log.Infof("Error updating %s time: %v", op, err)
	}
	return updateCheckFrequency
}

func (s *Server) checkForUpdates() {
	// TestServer.Start nils these out to prevent tests possibly phoning home.
	if s.parsedUpdatesURL == nil {
		return
	}

	q := s.parsedUpdatesURL.Query()
	q.Set("version", util.GetBuildInfo().Tag)
	q.Set("uuid", s.node.ClusterID.String())
	s.parsedUpdatesURL.RawQuery = q.Encode()

	res, err := http.Get(s.parsedUpdatesURL.String())
	if err != nil {
		// This is probably going to be relatively common in production
		// environments where network access is usually curtailed.
		if log.V(2) {
			log.Warning("Error checking for updates: ", err)
		}
		return
	}
	defer res.Body.Close()

	decoder := json.NewDecoder(res.Body)
	r := struct {
		Details []versionInfo `json:"details"`
	}{}

	err = decoder.Decode(&r)
	if err != nil && err != io.EOF {
		log.Warning("Error decoding updates info: ", err)
		return
	}

	for _, v := range r.Details {
		log.Info("A new version is available: %s\n\t%s", v.Version, v.Details)
	}
}

func (s *Server) usageReportingEnabled() bool {
	resp, err := s.db.Get(keys.UpdateCheckReportUsage)
	if err != nil {
		log.Warning(err)
		return false
	}
	return resp.Exists() && resp.ValueInt() != 0
}

func (s *Server) maybeReportUsage(running time.Duration) time.Duration {
	if running < updateCheckRetryFrequency {
		// On first check, we decline to report usage as metrics may not yet
		// be stable, so instead we request re-evaluation after a retry delay.
		return updateCheckRetryFrequency - running
	}
	if !s.usageReportingEnabled() {
		return updateCheckFrequency
	}
	return s.maybeRunPeriodicCheck("metrics reporting", keys.NodeLastUsageReportKey(int32(s.node.Descriptor.NodeID)), s.reportUsage)
}

func (s *Server) getReportingInfo() reportingInfo {
	n := s.node.recorder.GetStatusSummary()

	summary := nodeInfo{NodeID: s.node.Descriptor.NodeID}

	stores := make([]storeInfo, len(n.StoreStatuses))
	for i, r := range n.StoreStatuses {
		stores[i].NodeID = r.Desc.Node.NodeID
		stores[i].StoreID = r.Desc.StoreID
		stores[i].KeyCount = int(r.Metrics["keycount"])
		summary.KeyCount += stores[i].KeyCount
		stores[i].RangeCount = int(r.Metrics["replicas"])
		summary.RangeCount += stores[i].RangeCount
		bytes := int(r.Metrics["sysbytes"] + r.Metrics["intentbytes"] + r.Metrics["valbytes"] + r.Metrics["keybytes"])
		stores[i].Bytes = bytes
		summary.Bytes += bytes
	}
	return reportingInfo{summary, stores}
}

func (s *Server) reportUsage() {
	// TestServer.Start nils these out to prevent tests possibly phoning home.
	if s.parsedReportingURL == nil {
		return
	}

	b := new(bytes.Buffer)
	if err := json.NewEncoder(b).Encode(s.getReportingInfo()); err != nil {
		log.Warning(err)
		return
	}

	q := s.parsedReportingURL.Query()
	q.Set("version", util.GetBuildInfo().Tag)
	q.Set("uuid", s.node.ClusterID.String())
	s.parsedReportingURL.RawQuery = q.Encode()

	_, err := http.Post(s.parsedReportingURL.String(), "application/json", b)
	if err != nil && log.V(2) {
		// This is probably going to be relatively common in production
		// environments where network access is usually curtailed.
		log.Warning("Error checking reporting node usage metrics: ", err)
	}
}
<|endoftext|>


<commit_before>package lexer

// TokenType defines the type of a Token (see below)
type TokenType string

// Token represents one token of a pecific type and its literal representation
type Token struct {
	Type    TokenType
	Literal string
	Line    int
}

const (
	// ILLEGAL represents a not character the lexer does not know
	ILLEGAL = "ILLEGAL"
	// EOF indicates the end of the file
	EOF = "EOF"
	// DOUBLEQUOTEDSTRING represents double quoted strings
	DOUBLEQUOTEDSTRING = "DOUBLEQUOTEDSTRING"
	// SINGLEQUOTEDSTRING represents double quoted strings
	SINGLEQUOTEDSTRING = "SINGLEQUOTEDSTRING"
	// COMMENT is a comment
	COMMENT = "COMMENT"

	// Identifiers and literals

	// IDENT represents all kinds of identifiers, like variables, function names, internal functions etc.
	IDENT = "IDENT" // $foo, $bar, foo, print
	// INT is an integer
	INT = "INT" // 123456

	// Operators

	// ASSIGN is "="
	ASSIGN = "ASSIGN"
	// REFERENCE is "=&"
	REFERENCE = "REFERENCE"
	// PLUS is "+"
	PLUS = "PLUS"
	// MINUS is "-"
	MINUS = "MINUS"
	// MULTIPLY is "*"
	MULTIPLY = "MULTIPLY"
	// DIVIDE is "/"
	DIVIDE = "DIVIDE"
	// EQUALS is "=="
	EQUALS = "EQUALS"
	// IDENTITY is "==="
	IDENTITY = "IDENTITY"
	// LESSTHAN is "<"
	LESSTHAN = "LESSTHAN"
	// GREATERTHAN is ">"
	GREATERTHAN = "GREATERTHAN"
	// LESSTHANOREQUAL is "<="
	LESSTHANOREQUAL = "LESSTHANOREQUAL"
	// GREATERTHANOREQUAL is ">="
	GREATERTHANOREQUAL = "GREATERTHANOREQUAL"
	// OR is "||"
	OR = "OR"
	// AND is "&&"
	AND = "AND"
	// NOT is "!"
	NOT = "NOT"
	// INC is "++" - used for both preinc and postinc. It is the job of the parser to differentiate
	INC = "INC"
	// DEC is "--" - used for both preinc and postinc. It is the job of the parser to differentiate
	DEC = "DEC"
	// QUESTIONMARK is "?" - as used in ternary operations
	QUESTIONMARK = "QUESTIONMARK"
	// COLON is ":" - as used in ternary operations
	COLON = "COLON"

	// Delimiters

	// COMMA is ","
	COMMA = "COMMA"
	// DOT is "."
	DOT = "DOT"
	// SEMICOLON is ";"
	SEMICOLON = "SEMICOLON"
	// LPAREN is "("
	LPAREN = "LPAREN"
	// RPAREN is ")"
	RPAREN = "RPAREN"
	// LBRACE is "{"
	LBRACE = "LBRACE"
	// RBRACE is "}"
	RBRACE = "RBRACE"
	// LSQUAREBRACKET is "["
	LSQUAREBRACKET = "LSQUAREBRACKET"
	// RSQUAREBRACKET is "]"
	RSQUAREBRACKET = "RSQUAREBRACKET"

	// Keywords

	// FUNCTION is "function"
	FUNCTION = "FUNCTION"
	// RETURN is "return"
	RETURN = "RETURN"
	// PUBLIC is "public"
	PUBLIC = "PUBLIC"
	// PRIVATE is "private"
	PRIVATE = "PRIVATE"
	// STATIC is "static"
	STATIC = "STATIC"
	// PROTECTED is "protected"
	PROTECTED = "PROTECTED"
	// PHPTAG is "<?php"
	PHPTAG = "PHPTAG"
	// CLASS is "class"
	CLASS = "CLASS"
	// IMPLEMENTS is "implements"
	IMPLEMENTS = "IMPLEMENTS"
	// EXTENDS is "extends"
	EXTENDS = "EXTENDS"
	// IF is "if"
	IF = "IF"
	// TRUE is "true"
	TRUE = "TRUE"
	// FALSE is "false"
	FALSE = "FALSE"
	// USE is "use"
	USE = "USE"
	// FOR is "for"
	FOR = "FOR"
	// FOREACH is "foreach"
	FOREACH = "FOREACH"
	// AS is as as used in a foreach loop
	AS = "AS"
	// DOUBLEARROW is => as used in a foreach loop
	DOUBLEARROW = "DOUBLEARROW"
	// ARROW is -> as used in attribute access
	ARROW = "ARROW"

	// PHP7
	// SPACESHIP is the spaceship operator added in php7: <=>
	SPACESHIP = "SPACESHIP"
)

var keywords = map[string]TokenType{
	"function":   FUNCTION,
	"return":     RETURN,
	"public":     PUBLIC,
	"private":    PRIVATE,
	"protected":  PROTECTED,
	"static":     STATIC,
	"<?php":      PHPTAG,
	"class":      CLASS,
	"if":         IF,
	"true":       TRUE,
	"false":      FALSE,
	"use":        USE,
	"for":        FOR,
	"foreach":    FOREACH,
	"as":         AS,
	"implements": IMPLEMENTS,
	"extends":    EXTENDS,
}

// LookupIdent will search for possible keywords and return the
// appropriate TokenType
func LookupIdent(ident string) TokenType {
	if tok, ok := keywords[ident]; ok {
		return tok
	}

	return IDENT
}
<commit_msg>add blank line to satisfy linter<commit_after>package lexer

// TokenType defines the type of a Token (see below)
type TokenType string

// Token represents one token of a pecific type and its literal representation
type Token struct {
	Type    TokenType
	Literal string
	Line    int
}

const (
	// ILLEGAL represents a not character the lexer does not know
	ILLEGAL = "ILLEGAL"
	// EOF indicates the end of the file
	EOF = "EOF"
	// DOUBLEQUOTEDSTRING represents double quoted strings
	DOUBLEQUOTEDSTRING = "DOUBLEQUOTEDSTRING"
	// SINGLEQUOTEDSTRING represents double quoted strings
	SINGLEQUOTEDSTRING = "SINGLEQUOTEDSTRING"
	// COMMENT is a comment
	COMMENT = "COMMENT"

	// Identifiers and literals

	// IDENT represents all kinds of identifiers, like variables, function names, internal functions etc.
	IDENT = "IDENT" // $foo, $bar, foo, print
	// INT is an integer
	INT = "INT" // 123456

	// Operators

	// ASSIGN is "="
	ASSIGN = "ASSIGN"
	// REFERENCE is "=&"
	REFERENCE = "REFERENCE"
	// PLUS is "+"
	PLUS = "PLUS"
	// MINUS is "-"
	MINUS = "MINUS"
	// MULTIPLY is "*"
	MULTIPLY = "MULTIPLY"
	// DIVIDE is "/"
	DIVIDE = "DIVIDE"
	// EQUALS is "=="
	EQUALS = "EQUALS"
	// IDENTITY is "==="
	IDENTITY = "IDENTITY"
	// LESSTHAN is "<"
	LESSTHAN = "LESSTHAN"
	// GREATERTHAN is ">"
	GREATERTHAN = "GREATERTHAN"
	// LESSTHANOREQUAL is "<="
	LESSTHANOREQUAL = "LESSTHANOREQUAL"
	// GREATERTHANOREQUAL is ">="
	GREATERTHANOREQUAL = "GREATERTHANOREQUAL"
	// OR is "||"
	OR = "OR"
	// AND is "&&"
	AND = "AND"
	// NOT is "!"
	NOT = "NOT"
	// INC is "++" - used for both preinc and postinc. It is the job of the parser to differentiate
	INC = "INC"
	// DEC is "--" - used for both preinc and postinc. It is the job of the parser to differentiate
	DEC = "DEC"
	// QUESTIONMARK is "?" - as used in ternary operations
	QUESTIONMARK = "QUESTIONMARK"
	// COLON is ":" - as used in ternary operations
	COLON = "COLON"

	// Delimiters

	// COMMA is ","
	COMMA = "COMMA"
	// DOT is "."
	DOT = "DOT"
	// SEMICOLON is ";"
	SEMICOLON = "SEMICOLON"
	// LPAREN is "("
	LPAREN = "LPAREN"
	// RPAREN is ")"
	RPAREN = "RPAREN"
	// LBRACE is "{"
	LBRACE = "LBRACE"
	// RBRACE is "}"
	RBRACE = "RBRACE"
	// LSQUAREBRACKET is "["
	LSQUAREBRACKET = "LSQUAREBRACKET"
	// RSQUAREBRACKET is "]"
	RSQUAREBRACKET = "RSQUAREBRACKET"

	// Keywords

	// FUNCTION is "function"
	FUNCTION = "FUNCTION"
	// RETURN is "return"
	RETURN = "RETURN"
	// PUBLIC is "public"
	PUBLIC = "PUBLIC"
	// PRIVATE is "private"
	PRIVATE = "PRIVATE"
	// STATIC is "static"
	STATIC = "STATIC"
	// PROTECTED is "protected"
	PROTECTED = "PROTECTED"
	// PHPTAG is "<?php"
	PHPTAG = "PHPTAG"
	// CLASS is "class"
	CLASS = "CLASS"
	// IMPLEMENTS is "implements"
	IMPLEMENTS = "IMPLEMENTS"
	// EXTENDS is "extends"
	EXTENDS = "EXTENDS"
	// IF is "if"
	IF = "IF"
	// TRUE is "true"
	TRUE = "TRUE"
	// FALSE is "false"
	FALSE = "FALSE"
	// USE is "use"
	USE = "USE"
	// FOR is "for"
	FOR = "FOR"
	// FOREACH is "foreach"
	FOREACH = "FOREACH"
	// AS is as as used in a foreach loop
	AS = "AS"
	// DOUBLEARROW is => as used in a foreach loop
	DOUBLEARROW = "DOUBLEARROW"
	// ARROW is -> as used in attribute access
	ARROW = "ARROW"

	// PHP7

	// SPACESHIP is the spaceship operator added in php7: <=>
	SPACESHIP = "SPACESHIP"
)

var keywords = map[string]TokenType{
	"function":   FUNCTION,
	"return":     RETURN,
	"public":     PUBLIC,
	"private":    PRIVATE,
	"protected":  PROTECTED,
	"static":     STATIC,
	"<?php":      PHPTAG,
	"class":      CLASS,
	"if":         IF,
	"true":       TRUE,
	"false":      FALSE,
	"use":        USE,
	"for":        FOR,
	"foreach":    FOREACH,
	"as":         AS,
	"implements": IMPLEMENTS,
	"extends":    EXTENDS,
}

// LookupIdent will search for possible keywords and return the
// appropriate TokenType
func LookupIdent(ident string) TokenType {
	if tok, ok := keywords[ident]; ok {
		return tok
	}

	return IDENT
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-interpreter Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package exec

import (
	"math"
)

func (vm *VM) fetchBaseAddr() int {
	return int(vm.fetchUint32() + uint32(vm.popInt32()))
}

// curMem returns a slice to the memeory segment pointed to by
// the current base address on the bytecode stream.
func (vm *VM) curMem() []byte {
	return vm.memory[vm.fetchBaseAddr():]
}

func (vm *VM) i32Load() {
	vm.pushUint32(endianess.Uint32(vm.curMem()))
}

func (vm *VM) i32Load8s() {
	vm.pushInt32(int32(int8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i32Load8u() {
	vm.pushUint32(uint32(uint8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i32Load16s() {
	vm.pushInt32(int32(int16(endianess.Uint16(vm.curMem()))))
}

func (vm *VM) i32Load16u() {
	vm.pushUint32(uint32(endianess.Uint16(vm.curMem())))
}

func (vm *VM) i64Load() {
	vm.pushUint64(endianess.Uint64(vm.curMem()))
}

func (vm *VM) i64Load8s() {
	vm.pushInt64(int64(int8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i64Load8u() {
	vm.pushUint64(uint64(uint8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i64Load16s() {
	vm.pushInt64(int64(int16(endianess.Uint16(vm.curMem()))))
}

func (vm *VM) i64Load16u() {
	vm.pushUint64(uint64(endianess.Uint16(vm.curMem())))
}

func (vm *VM) i64Load32s() {
	vm.pushInt64(int64(int32(endianess.Uint32(vm.curMem()))))
}

func (vm *VM) i64Load32u() {
	vm.pushUint64(uint64(endianess.Uint32(vm.curMem())))
}

func (vm *VM) f32Store() {
	v := math.Float32bits(vm.popFloat32())
	endianess.PutUint32(vm.curMem(), v)
}

func (vm *VM) f32Load() {
	vm.pushFloat32(math.Float32frombits(endianess.Uint32(vm.curMem())))
}

func (vm *VM) f64Store() {
	v := math.Float64bits(vm.popFloat64())
	endianess.PutUint64(vm.curMem(), v)
}

func (vm *VM) f64Load() {
	vm.pushFloat64(math.Float64frombits(endianess.Uint64(vm.curMem())))
}

func (vm *VM) i32Store() {
	v := vm.popUint32()
	endianess.PutUint32(vm.curMem(), v)
}

func (vm *VM) i32Store8() {
	v := byte(uint8(vm.popUint32()))
	vm.memory[vm.fetchBaseAddr()] = v
}

func (vm *VM) i32Store16() {
	v := uint16(vm.popUint32())
	endianess.PutUint16(vm.curMem(), v)
}

func (vm *VM) i64Store() {
	v := vm.popUint64()
	endianess.PutUint64(vm.curMem(), v)
}

func (vm *VM) i64Store8() {
	v := byte(uint8(vm.popUint64()))
	vm.memory[vm.fetchBaseAddr()] = v
}

func (vm *VM) i64Store16() {
	v := uint16(vm.popUint64())
	endianess.PutUint16(vm.curMem(), v)
}

func (vm *VM) i64Store32() {
	v := uint32(vm.popUint64())
	endianess.PutUint32(vm.curMem(), v)
}

func (vm *VM) currentMemory() {
	_ = vm.fetchInt8() // reserved (https://github.com/WebAssembly/design/blob/27ac254c854994103c24834a994be16f74f54186/BinaryEncoding.md#memory-related-operators-described-here)
	vm.pushInt32(int32(len(vm.memory) / wasmPageSize))
}

func (vm *VM) growMemory() {
	curLen := len(vm.memory) / wasmPageSize
	n := vm.popInt32()
	vm.memory = append(vm.memory, make([]byte, n*wasmPageSize)...)
	vm.pushInt32(int32(curLen))
}
<commit_msg>exec: Trap the VM on out of bounds memory access.<commit_after>// Copyright 2017 The go-interpreter Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package exec

import (
	"errors"
	"math"
)

// ErrOutOfBoundsMemoryAccess is the error value used while trapping the VM
// when it detects an out of bounds access to the linear memory.
var ErrOutOfBoundsMemoryAccess = errors.New("exec: out of bounds memory access")

func (vm *VM) fetchBaseAddr() int {
	return int(vm.fetchUint32() + uint32(vm.popInt32()))
}

// inBounds returns true when the next vm.fetchBaseAddr() + offset
// indices are in bounds accesses to the linear memory.
func (vm *VM) inBounds(offset int) bool {
	addr := endianess.Uint32(vm.ctx.code[vm.ctx.pc:]) + uint32(vm.ctx.stack[len(vm.ctx.stack)-1])
	return int(addr)+offset < len(vm.memory)
}

// curMem returns a slice to the memeory segment pointed to by
// the current base address on the bytecode stream.
func (vm *VM) curMem() []byte {
	return vm.memory[vm.fetchBaseAddr():]
}

func (vm *VM) i32Load() {
	if !vm.inBounds(3) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushUint32(endianess.Uint32(vm.curMem()))
}

func (vm *VM) i32Load8s() {
	if !vm.inBounds(0) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushInt32(int32(int8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i32Load8u() {
	if !vm.inBounds(0) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushUint32(uint32(uint8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i32Load16s() {
	if !vm.inBounds(1) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushInt32(int32(int16(endianess.Uint16(vm.curMem()))))
}

func (vm *VM) i32Load16u() {
	if !vm.inBounds(1) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushUint32(uint32(endianess.Uint16(vm.curMem())))
}

func (vm *VM) i64Load() {
	if !vm.inBounds(7) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushUint64(endianess.Uint64(vm.curMem()))
}

func (vm *VM) i64Load8s() {
	if !vm.inBounds(0) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushInt64(int64(int8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i64Load8u() {
	if !vm.inBounds(0) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushUint64(uint64(uint8(vm.memory[vm.fetchBaseAddr()])))
}

func (vm *VM) i64Load16s() {
	if !vm.inBounds(1) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushInt64(int64(int16(endianess.Uint16(vm.curMem()))))
}

func (vm *VM) i64Load16u() {
	if !vm.inBounds(1) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushUint64(uint64(endianess.Uint16(vm.curMem())))
}

func (vm *VM) i64Load32s() {
	if !vm.inBounds(3) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushInt64(int64(int32(endianess.Uint32(vm.curMem()))))
}

func (vm *VM) i64Load32u() {
	if !vm.inBounds(3) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushUint64(uint64(endianess.Uint32(vm.curMem())))
}

func (vm *VM) f32Store() {
	v := math.Float32bits(vm.popFloat32())
	if !vm.inBounds(3) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	endianess.PutUint32(vm.curMem(), v)
}

func (vm *VM) f32Load() {
	if !vm.inBounds(3) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushFloat32(math.Float32frombits(endianess.Uint32(vm.curMem())))
}

func (vm *VM) f64Store() {
	v := math.Float64bits(vm.popFloat64())
	if !vm.inBounds(7) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	endianess.PutUint64(vm.curMem(), v)
}

func (vm *VM) f64Load() {
	if !vm.inBounds(7) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.pushFloat64(math.Float64frombits(endianess.Uint64(vm.curMem())))
}

func (vm *VM) i32Store() {
	v := vm.popUint32()
	if !vm.inBounds(3) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	endianess.PutUint32(vm.curMem(), v)
}

func (vm *VM) i32Store8() {
	v := byte(uint8(vm.popUint32()))
	if !vm.inBounds(0) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.memory[vm.fetchBaseAddr()] = v
}

func (vm *VM) i32Store16() {
	v := uint16(vm.popUint32())
	if !vm.inBounds(1) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	endianess.PutUint16(vm.curMem(), v)
}

func (vm *VM) i64Store() {
	v := vm.popUint64()
	if !vm.inBounds(7) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	endianess.PutUint64(vm.curMem(), v)
}

func (vm *VM) i64Store8() {
	v := byte(uint8(vm.popUint64()))
	if !vm.inBounds(0) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	vm.memory[vm.fetchBaseAddr()] = v
}

func (vm *VM) i64Store16() {
	v := uint16(vm.popUint64())
	if !vm.inBounds(1) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	endianess.PutUint16(vm.curMem(), v)
}

func (vm *VM) i64Store32() {
	v := uint32(vm.popUint64())
	if !vm.inBounds(3) {
		panic(ErrOutOfBoundsMemoryAccess)
	}
	endianess.PutUint32(vm.curMem(), v)
}

func (vm *VM) currentMemory() {
	_ = vm.fetchInt8() // reserved (https://github.com/WebAssembly/design/blob/27ac254c854994103c24834a994be16f74f54186/BinaryEncoding.md#memory-related-operators-described-here)
	vm.pushInt32(int32(len(vm.memory) / wasmPageSize))
}

func (vm *VM) growMemory() {
	curLen := len(vm.memory) / wasmPageSize
	n := vm.popInt32()
	vm.memory = append(vm.memory, make([]byte, n*wasmPageSize)...)
	vm.pushInt32(int32(curLen))
}
<|endoftext|>


<commit_before>package exp12

import (
	"sync"

	"github.com/shurcooL/go/exp/13"
	"github.com/shurcooL/go/vcs"

	. "gist.github.com/7802150.git"
)

// TODO: Use FileUri or similar type instead of string for clean path to repo root.
// rootPath -> *VcsState
var repos = make(map[string]*exp13.VcsState)
var reposLock sync.Mutex

// TODO: Use FileUri or similar type instead of string for clean path to repo root.
// path -> *Directory
var directories = make(map[string]*Directory)
var directoriesLock sync.Mutex

type Directory struct {
	path string

	Repo *exp13.VcsState

	DepNode2
}

func (this *Directory) Update() {
	if vcs := vcs.New(this.path); vcs != nil {
		reposLock.Lock()
		if repo, ok := repos[vcs.RootPath()]; ok {
			this.Repo = repo
		} else {
			this.Repo = exp13.NewVcsState(vcs)
			repos[vcs.RootPath()] = this.Repo
		}
		reposLock.Unlock()
	}
}

func newDirectory(path string) *Directory {
	this := &Directory{path: path}
	// No DepNode2I sources, so each instance can only be updated (i.e. initialized) once
	return this
}

func LookupDirectory(path string) *Directory {
	directoriesLock.Lock()
	defer directoriesLock.Unlock()
	if dir := directories[path]; dir != nil {
		return dir
	} else {
		dir = newDirectory(path)
		directories[path] = dir
		return dir
	}
}
<commit_msg>Add doc for LookupDirectory() that it's safe for concurrent calls.<commit_after>package exp12

import (
	"sync"

	"github.com/shurcooL/go/exp/13"
	"github.com/shurcooL/go/vcs"

	. "gist.github.com/7802150.git"
)

// TODO: Use FileUri or similar type instead of string for clean path to repo root.
// rootPath -> *VcsState
var repos = make(map[string]*exp13.VcsState)
var reposLock sync.Mutex

// TODO: Use FileUri or similar type instead of string for clean path to repo root.
// path -> *Directory
var directories = make(map[string]*Directory)
var directoriesLock sync.Mutex

type Directory struct {
	path string

	Repo *exp13.VcsState

	DepNode2
}

func (this *Directory) Update() {
	if vcs := vcs.New(this.path); vcs != nil {
		reposLock.Lock()
		if repo, ok := repos[vcs.RootPath()]; ok {
			this.Repo = repo
		} else {
			this.Repo = exp13.NewVcsState(vcs)
			repos[vcs.RootPath()] = this.Repo
		}
		reposLock.Unlock()
	}
}

func newDirectory(path string) *Directory {
	this := &Directory{path: path}
	// No DepNode2I sources, so each instance can only be updated (i.e. initialized) once
	return this
}

// Safe to call concurrently.
func LookupDirectory(path string) *Directory {
	directoriesLock.Lock()
	defer directoriesLock.Unlock()
	if dir := directories[path]; dir != nil {
		return dir
	} else {
		dir = newDirectory(path)
		directories[path] = dir
		return dir
	}
}
<|endoftext|>


<commit_before>package journal

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
)

type Priority int

const (
	PRI_EMERG Priority = iota
	PRI_ALERT
	PRI_CRIT
	PRI_ERR
	PRI_WARNING
	PRI_NOTICE
	PRI_INFO
	PRI_DEBUG
)

var conn net.Conn

func init() {
	var err error
	conn, err = net.Dial("unixgram", "/run/systemd/journal/socket")
	if err != nil {
		fmt.Fprintln(os.Stderr, "socket connection error:", err)
	}
}

func Enabled() bool {
	return conn != nil
}

func Send(message string, priority Priority, vars map[string]string) {
	if conn == nil {
		fmt.Fprintln(os.Stderr, "journal error: connection uninitialized")
		return
	}

	data := new(bytes.Buffer)
	appendVariable(data, "PRIORITY", strconv.Itoa(int(priority)))
	appendVariable(data, "MESSAGE", message)
	for k, v := range vars {
		appendVariable(data, k, v)
	}

	_, err := io.Copy(conn, data)
	if err != nil && isSocketSpaceError(err) {
		file, err := tempFd()
		if err != nil {
			fmt.Fprintln(os.Stderr, "journal error:", err)
			return
		}
		_, err = io.Copy(file, data)
		if err != nil {
			fmt.Fprintln(os.Stderr, "journal error:", err)
			return
		}

		rights := syscall.UnixRights(int(file.Fd()))

		/* this connection should always be a UnixConn, but better safe than sorry */
		unixConn, ok := conn.(*net.UnixConn)
		if !ok {
			fmt.Fprintln(os.Stderr, "journal error: can't send file through non-Unix connection")
			return
		}

		unixConn.WriteMsgUnix([]byte{}, rights, nil)
	} else if err != nil {
		fmt.Fprintln(os.Stderr, "journal error:", err)
	}
}

func appendVariable(w io.Writer, name, value string) {
	if !validVarName(name) {
		fmt.Fprintln(os.Stderr, "journal error: variable name contains invalid character, ignoring")
		return
	}
	if strings.ContainsRune(value, '\n') {
		/* When the value contains a newline, we write:
		 * - the variable name, followed by a newline
		 * - the size (in 64bit little endian format)
		 * - the data, followed by a newline
		 */
		fmt.Fprintln(w, name)
		w.Write(toBytes(uint64(len(value))))
		fmt.Fprintln(w, value)
	} else {
		/* just write the variable and value all on one line */
		fmt.Fprintf(w, "%s=%s\n", name, value)
	}
}

func validVarName(name string) bool {
	/* The variable name must be in uppercase and consist only of characters,
	 * numbers and underscores, and may not begin with an underscore. (from the docs)
	 */
	valid := true
	valid = valid && name[0] != '_'
	for _, c := range name {
		valid = valid && ('A' <= c && c <= 'Z') || ('0' <= c && c <='9') || c == '_'
	}
	return valid
}

func isSocketSpaceError(err error) bool {
	opErr, ok := err.(*net.OpError)
	if !ok {
		return false
	}

	sysErr, ok := opErr.Err.(syscall.Errno)
	if !ok {
		return false
	}

	return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS
}

func toBytes(n uint64) []byte {
	var b [8]byte
	for i := 0; i < 8; i++ {
		b[i] = byte(n >> uint(i*8))
	}
	return b[:]
}

func tempFd() (*os.File, error) {
	file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX")
	if err != nil {
		return nil, err
	}
	syscall.Unlink(file.Name())
	if err != nil {
		return nil, err
	}
	return file, nil
}
<commit_msg>Better fit Go conventions<commit_after>package journal

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"strconv"
	"strings"
	"syscall"
)

type Priority int

const (
	PriEmerg Priority = iota
	PriAlert
	PriCrit
	PriErr
	PriWarning
	PriNotice
	PriInfo
	PriDebug
)

var conn net.Conn

func init() {
	var err error
	conn, err = net.Dial("unixgram", "/run/systemd/journal/socket")
	if err != nil {
		fmt.Fprintln(os.Stderr, "socket connection error:", err)
	}
}

func Enabled() bool {
	return conn != nil
}

func Send(message string, priority Priority, vars map[string]string) {
	if conn == nil {
		fmt.Fprintln(os.Stderr, "journal error: connection uninitialized")
		return
	}

	data := new(bytes.Buffer)
	appendVariable(data, "PRIORITY", strconv.Itoa(int(priority)))
	appendVariable(data, "MESSAGE", message)
	for k, v := range vars {
		appendVariable(data, k, v)
	}

	_, err := io.Copy(conn, data)
	if err != nil && isSocketSpaceError(err) {
		file, err := tempFd()
		if err != nil {
			fmt.Fprintln(os.Stderr, "journal error:", err)
			return
		}
		_, err = io.Copy(file, data)
		if err != nil {
			fmt.Fprintln(os.Stderr, "journal error:", err)
			return
		}

		rights := syscall.UnixRights(int(file.Fd()))

		/* this connection should always be a UnixConn, but better safe than sorry */
		unixConn, ok := conn.(*net.UnixConn)
		if !ok {
			fmt.Fprintln(os.Stderr, "journal error: can't send file through non-Unix connection")
			return
		}

		unixConn.WriteMsgUnix([]byte{}, rights, nil)
	} else if err != nil {
		fmt.Fprintln(os.Stderr, "journal error:", err)
	}
}

func appendVariable(w io.Writer, name, value string) {
	if !validVarName(name) {
		fmt.Fprintln(os.Stderr, "journal error: variable name contains invalid character, ignoring")
		return
	}
	if strings.ContainsRune(value, '\n') {
		/* When the value contains a newline, we write:
		 * - the variable name, followed by a newline
		 * - the size (in 64bit little endian format)
		 * - the data, followed by a newline
		 */
		fmt.Fprintln(w, name)
		w.Write(toBytes(uint64(len(value))))
		fmt.Fprintln(w, value)
	} else {
		/* just write the variable and value all on one line */
		fmt.Fprintf(w, "%s=%s\n", name, value)
	}
}

func validVarName(name string) bool {
	/* The variable name must be in uppercase and consist only of characters,
	 * numbers and underscores, and may not begin with an underscore. (from the docs)
	 */
	valid := true
	valid = valid && name[0] != '_'
	for _, c := range name {
		valid = valid && ('A' <= c && c <= 'Z') || ('0' <= c && c <='9') || c == '_'
	}
	return valid
}

func isSocketSpaceError(err error) bool {
	opErr, ok := err.(*net.OpError)
	if !ok {
		return false
	}

	sysErr, ok := opErr.Err.(syscall.Errno)
	if !ok {
		return false
	}

	return sysErr == syscall.EMSGSIZE || sysErr == syscall.ENOBUFS
}

func toBytes(n uint64) []byte {
	b := make([]byte, 8)
	for i := 0; i < 8; i++ {
		b[i] = byte(n >> uint(i*8))
	}
	return b
}

func tempFd() (*os.File, error) {
	file, err := ioutil.TempFile("/dev/shm/", "journal.XXXXX")
	if err != nil {
		return nil, err
	}
	syscall.Unlink(file.Name())
	if err != nil {
		return nil, err
	}
	return file, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/memcache"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/user"
)

type RootData struct {
	Posts   *[]Entry
	IsAdmin bool
	Page    int64
	Prev    int64
	Next    int64
}

const perPage = 30

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	if r.Request.URL.Path == "/page/0" {
		http.Redirect(w, r.Request, "/", 301)
	}

	c := appengine.NewContext(r.Request)
	pg, err := strconv.ParseInt(r.Param("page"), 10, 64)
	if err != nil {
		log.Infof(c, "Error parsing: %+v", err)
		pg = 0
	}

	entries, err := Pagination(c, perPage, int(pg*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{
		Posts:   entries,
		IsAdmin: user.IsAdmin(c),
		Page:    pg,
		Next:    pg + 1,
		Prev:    pg - 1,
	}

	// If there are no posts left, don't show next button.
	if len(*entries) == 0 {
		data.Next = -1
	}

	// Get next page's posts so we don't show the next page if there is none.
	nxt_entr, err := Pagination(c, perPage, int((pg+1)*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if len(*nxt_entr) == 0 {
		data.Next = -1
	}

	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("about", nil)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func RedirectHomeHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "/", 302)
}

func CleanWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	q := datastore.NewQuery("Entry")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for _, p := range *entries {
		// TODO: Figure out how to unset all public.
		if &p.Draft == nil {
			p.Draft = false
		}
		if len(p.Title) == 0 {
			p.Title = fmt.Sprintf("Untitled #%d", p.Id)
		}
		p.Save(c)
	}
}

type SiteMapData struct {
	Posts  *[]Entry
	Newest time.Time
}

// http://www.sitemaps.org/protocol.html
func SitemapHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &SiteMapData{
		Posts:  entries,
		Newest: (*entries)[0].Modified,
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	w.Render("sitemap", data)
}

// This queues lots of work every fifteen minutes.
func WorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	t := taskqueue.NewPOSTTask("/archive/work", url.Values{})
	_, err := taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Download all the links.
	t = taskqueue.NewPOSTTask("/link/work", url.Values{})
	_, err = taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	//// Update the stats
	//t = taskqueue.NewPOSTTask("/stats/work", url.Values{})
	//_, err = taskqueue.Add(c, t, "tasks")

	//if err != nil {
	//	log.Errorf(c, "Error queueing work: %v", err.Error())
	//	http.Error(w, err.Error(), 500)
	//	return
	//}

	// Update the longform data.
	t = taskqueue.NewPOSTTask("/longform/work", url.Values{})
	_, err = taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Clean the database
	t = taskqueue.NewPOSTTask("/clean/work", url.Values{})
	_, err = taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the Search Index
	t = taskqueue.NewPOSTTask("/search/work", url.Values{})
	_, err = taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Delete some caches
	hour, min, _ := time.Now().Clock()
	if hour == 0 && (min > 0 && min < 20) {
		err = memcache.Delete(c, "stats_data")
		if err != nil {
			log.Errorf(c, "Error cleaning stats cache: %v", err.Error())
		}
	}

	fmt.Fprint(w, "success.\n")
}
<commit_msg>refactor task queueing<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/user"
)

type RootData struct {
	Posts   *[]Entry
	IsAdmin bool
	Page    int64
	Prev    int64
	Next    int64
}

const perPage = 30

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	if r.Request.URL.Path == "/page/0" {
		http.Redirect(w, r.Request, "/", 301)
	}

	c := appengine.NewContext(r.Request)
	pg, err := strconv.ParseInt(r.Param("page"), 10, 64)
	if err != nil {
		log.Infof(c, "Error parsing: %+v", err)
		pg = 0
	}

	entries, err := Pagination(c, perPage, int(pg*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{
		Posts:   entries,
		IsAdmin: user.IsAdmin(c),
		Page:    pg,
		Next:    pg + 1,
		Prev:    pg - 1,
	}

	// If there are no posts left, don't show next button.
	if len(*entries) == 0 {
		data.Next = -1
	}

	// Get next page's posts so we don't show the next page if there is none.
	nxt_entr, err := Pagination(c, perPage, int((pg+1)*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	if len(*nxt_entr) == 0 {
		data.Next = -1
	}

	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	w.Render("about", nil)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func RedirectHomeHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "/", 302)
}

func CleanWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	q := datastore.NewQuery("Entry")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for _, p := range *entries {
		// TODO: Figure out how to unset all public.
		if &p.Draft == nil {
			p.Draft = false
		}
		if len(p.Title) == 0 {
			p.Title = fmt.Sprintf("Untitled #%d", p.Id)
		}
		p.Save(c)
	}
}

type SiteMapData struct {
	Posts  *[]Entry
	Newest time.Time
}

// http://www.sitemaps.org/protocol.html
func SitemapHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &SiteMapData{
		Posts:  entries,
		Newest: (*entries)[0].Modified,
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	w.Render("sitemap", data)
}

func queueWork(c context.Context, uri string) error {
	r := &taskqueue.RetryOptions{
		RetryLimit: 1,
	}

	t := taskqueue.NewPOSTTask(uri, url.Values{})
	t.RetryOptions = r
	_, err := taskqueue.Add(c, t, "tasks")

	if err != nil {
		log.Errorf(c, "Error queueing %s: %v", err.Error())
		return err
	}

	return nil
}

// This queues lots of work every fifteen minutes.
func WorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	err := queueWork(c, "/archive/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Download all the links.
	err = queueWork(c, "/link/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	//// Update the stats
	//err = queueWork(c, "/stats/work")
	//if err != nil {
	//	http.Error(w, err.Error(), 500)
	//	return
	//}

	// Update the longform data.
	err = queueWork(c, "/longform/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Clean the database
	err = queueWork(c, "/clean/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the Search Index
	err = queueWork(c, "/search/work")
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	fmt.Fprint(w, "success.\n")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/user"
)

type RootData struct {
	Posts   *[]Entry
	IsAdmin bool
	Page    int64
	Prev    int64
	Next    int64
}

const perPage = 30

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	if r.Request.URL.Path == "/page/0" {
		http.Redirect(w, r.Request, "/", 301)
	}

	c := appengine.NewContext(r.Request)
	pg, err := strconv.ParseInt(r.Param("page"), 10, 64)
	if err != nil {
		log.Infof(c, "Error parsing: %+v", err)
		pg = 0
	}

	entries, err := Pagination(c, perPage, int(pg*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{
		Posts:   entries,
		IsAdmin: user.IsAdmin(c),
		Page:    pg,
		Next:    pg + 1,
		Prev:    pg - 1,
	}

	if len(*entries) == 0 {
		data.Next = -1
	}

	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "http://natwelch.com", 301)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func RedirectHomeHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "/", 302)
}

func CleanWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	q := datastore.NewQuery("Entry")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for _, p := range *entries {
		if &p.Draft == nil {
			p.Draft = false
		}
		p.Save(c)
	}
}

type SiteMapData struct {
	Posts  *[]Entry
	Newest time.Time
}

// http://www.sitemaps.org/protocol.html
func SitemapHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &SiteMapData{
		Posts:  entries,
		Newest: (*entries)[0].Modified,
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	w.Render("sitemap", data)
}

// This queues lots of work every fifteen minutes.
func WorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	t := taskqueue.NewPOSTTask("/archive/work", url.Values{})
	_, err := taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Download all the links.
	t = taskqueue.NewPOSTTask("/link/work", url.Values{})
	_, err = taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the longform data.
	t = taskqueue.NewPOSTTask("/longform/work", url.Values{})
	_, err = taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Clean the database
	t = taskqueue.NewPOSTTask("/clean/work", url.Values{})
	_, err = taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	fmt.Fprint(w, "success.\n")
}
<commit_msg>add cache cron<commit_after>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/pilu/traffic"
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/user"
)

type RootData struct {
	Posts   *[]Entry
	IsAdmin bool
	Page    int64
	Prev    int64
	Next    int64
}

const perPage = 30

func RootHandler(w traffic.ResponseWriter, r *traffic.Request) {
	if r.Request.URL.Path == "/page/0" {
		http.Redirect(w, r.Request, "/", 301)
	}

	c := appengine.NewContext(r.Request)
	pg, err := strconv.ParseInt(r.Param("page"), 10, 64)
	if err != nil {
		log.Infof(c, "Error parsing: %+v", err)
		pg = 0
	}

	entries, err := Pagination(c, perPage, int(pg*perPage))
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &RootData{
		Posts:   entries,
		IsAdmin: user.IsAdmin(c),
		Page:    pg,
		Next:    pg + 1,
		Prev:    pg - 1,
	}

	if len(*entries) == 0 {
		data.Next = -1
	}

	w.Render("index", data)
}

func AboutHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "http://natwelch.com", 301)
}

func UnimplementedHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Error(w, "Sorry, I haven't implemented this yet", 500)
}

func RedirectHomeHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, "/", 302)
}

func CleanWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	q := datastore.NewQuery("Entry")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	for _, p := range *entries {
		if &p.Draft == nil {
			p.Draft = false
		}
		p.Save(c)
	}
}

type SiteMapData struct {
	Posts  *[]Entry
	Newest time.Time
}

// http://www.sitemaps.org/protocol.html
func SitemapHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := AllPosts(c)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	data := &SiteMapData{
		Posts:  entries,
		Newest: (*entries)[0].Modified,
	}

	w.Header().Set("Content-Type", "application/xml; charset=utf-8")
	w.Render("sitemap", data)
}

// This queues lots of work every fifteen minutes.
func WorkQueueHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Build data for the Archive Page
	t := taskqueue.NewPOSTTask("/archive/work", url.Values{})
	_, err := taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Download all the links.
	t = taskqueue.NewPOSTTask("/link/work", url.Values{})
	_, err = taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Update the longform data.
	t = taskqueue.NewPOSTTask("/longform/work", url.Values{})
	_, err = taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Clean the database
	t = taskqueue.NewPOSTTask("/clean/work", url.Values{})
	_, err = taskqueue.Add(c, t, "")

	if err != nil {
		log.Errorf(c, "Error queueing work: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Delete some caches
	hour, min, _ = time.Now().Clock()
	if hour == 0 && (min > 0 && min < 20) {
		err = memcache.Delete(c, "stats_data")
		if err != nil {
			log.Errorf(c, "Error cleaning stats cache: %v", err.Error())
		}
	}

	fmt.Fprint(w, "success.\n")
}
<|endoftext|>