<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"k8s.io/release/pkg/announce"
)

// releaseNotesCmd represents the subcommand for `krel release-notes`
var githubPageCmd = &cobra.Command{
	Use:   "github",
	Short: "Updates the github page of a release",
	Long: `publish-release github

This command updates the GitHub release page for a given tag. It will
update the page using a built in template or you can update it using
a custom template.

Before updating the page, the tag has to exist already on github.

To publish the page, --nomock has to be defined. Otherwise, the rendered
page will be printed to stdout and the program will exit.

CUSTOM TEMPLATES
================
You can define a custom golang template to use in your release page. Your
template can contain string substitutions and you can define those using
the --substitution flag:

  --substitution="releaseTheme:Accentuate the Paw-sitive"
  --substitution="releaseLogo:accentuate-the-pawsitive.png"

ASSET FILES
===========
This command supports uploading release assets to the github page. You
can add asset files with the --asset flag:

  --asset=_output/kubernetes-1.18.2-2.fc33.x86_64.rpm

You can also specify a label for the assets by appending it with a colon
to the asset file:

  --asset="_output/kubernetes-1.18.2-2.fc33.x86_64.rpm:RPM Package for amd64"

`,
	RunE: func(cmd *cobra.Command, args []string) error {
		// Run the PR creation function
		return runGithubPage(ghPageOpts)
	},
}

type githubPageCmdLineOptions struct {
	noupdate         bool
	draft            bool
	sbom             bool
	name             string
	repo             string
	template         string
	repoPath         string
	ReleaseNotesFile string
	substitutions    []string
	assets           []string
}

var ghPageOpts = &githubPageCmdLineOptions{}

func init() {
	githubPageCmd.PersistentFlags().StringVarP(
		&ghPageOpts.repo,
		"repo",
		"r",
		"",
		"repository slug containing the release page",
	)
	githubPageCmd.PersistentFlags().StringVar(
		&ghPageOpts.template,
		"template",
		"",
		"path to a custom page template",
	)
	githubPageCmd.PersistentFlags().StringVarP(
		&ghPageOpts.name,
		"name",
		"n",
		"",
		"name for the release",
	)
	githubPageCmd.PersistentFlags().StringSliceVarP(
		&ghPageOpts.assets,
		"asset",
		"a",
		[]string{},
		"Path to asset file for the release. Can be specified multiple times.",
	)
	githubPageCmd.PersistentFlags().StringSliceVarP(
		&ghPageOpts.substitutions,
		"substitution",
		"s",
		[]string{},
		"String substitution for the page template",
	)
	githubPageCmd.PersistentFlags().BoolVar(
		&ghPageOpts.noupdate,
		"noupdate",
		false,
		"Fail if the release already exists",
	)
	githubPageCmd.PersistentFlags().BoolVar(
		&ghPageOpts.draft,
		"draft",
		false,
		"Mark the release as a draft in GitHub so you can finish editing and publish it manually.",
	)
	githubPageCmd.PersistentFlags().BoolVar(
		&ghPageOpts.sbom,
		"sbom",
		true,
		"Generate an SPDX bill of materials and attach it to the release",
	)

	githubPageCmd.PersistentFlags().StringVar(
		&ghPageOpts.repoPath,
		"repo-path",
		".",
		"Path to the source code repository",
	)

	githubPageCmd.PersistentFlags().StringVar(
		&ghPageOpts.ReleaseNotesFile,
		"release-notes-file",
		"",
		"Path to a release notes markdown file to include in the release",
	)

	for _, f := range []string{"template", "asset"} {
		if err := githubPageCmd.MarkPersistentFlagFilename(f); err != nil {
			logrus.Error(err)
		}
	}

	if err := githubPageCmd.MarkPersistentFlagRequired("repo"); err != nil {
		logrus.Error(err)
	}

	rootCmd.AddCommand(githubPageCmd)
}

func getAssetsFromStrings(assetStrings []string) []announce.Asset {
	r := []announce.Asset{}
	for _, s := range assetStrings {
		parts := strings.Split(s, ":")
		l := ""
		if len(parts) > 0 {
			l = parts[1]
		}
		r = append(r, announce.Asset{
			Path:     filepath.Base(parts[0]),
			ReadFrom: parts[0],
			Label:    l,
		})
	}
	return r
}

func runGithubPage(opts *githubPageCmdLineOptions) (err error) {
	// Generate the release SBOM
	assets := getAssetsFromStrings(opts.assets)
	sbom := ""
	if opts.sbom {
		// Generate the assets file
		sbom, err = announce.GenerateReleaseSBOM(&announce.SBOMOptions{
			ReleaseName:   opts.name,
			Repo:          opts.repo,
			RepoDirectory: opts.repoPath,
			Assets:        assets,
			Tag:           commandLineOpts.tag,
		})
		if err != nil {
			return errors.Wrap(err, "generating sbom")
		}
		opts.assets = append(opts.assets, sbom+":SPDX Software Bill of Materials (SBOM)")
		// Delete the temporary sbom  when we're done
		if commandLineOpts.nomock {
			defer os.Remove(sbom)
		}
	}

	// Build the release page options
	announceOpts := announce.GitHubPageOptions{
		AssetFiles:            opts.assets,
		Tag:                   commandLineOpts.tag,
		NoMock:                commandLineOpts.nomock,
		UpdateIfReleaseExists: !opts.noupdate,
		Name:                  opts.name,
		Draft:                 opts.draft,
		ReleaseNotesFile:      opts.ReleaseNotesFile,
	}

	// Assign the repository data
	if err := announceOpts.SetRepository(opts.repo); err != nil {
		return errors.Wrap(err, "assigning the repository slug")
	}

	// Assign the substitutions
	if err := announceOpts.ParseSubstitutions(opts.substitutions); err != nil {
		return errors.Wrap(err, "parsing template substitutions")
	}

	// Read the csutom template data
	if err := announceOpts.ReadTemplate(opts.template); err != nil {
		return errors.Wrap(err, "reading the template file")
	}

	// Validate the options
	if err := announceOpts.Validate(); err != nil {
		return errors.Wrap(err, "validating options")
	}

	// Run the update process
	return announce.UpdateGitHubPage(&announceOpts)
}
<commit_msg>Fix bug where incorrect count was used when assets had no tag<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cmd

import (
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"k8s.io/release/pkg/announce"
)

// releaseNotesCmd represents the subcommand for `krel release-notes`
var githubPageCmd = &cobra.Command{
	Use:   "github",
	Short: "Updates the github page of a release",
	Long: `publish-release github

This command updates the GitHub release page for a given tag. It will
update the page using a built in template or you can update it using
a custom template.

Before updating the page, the tag has to exist already on github.

To publish the page, --nomock has to be defined. Otherwise, the rendered
page will be printed to stdout and the program will exit.

CUSTOM TEMPLATES
================
You can define a custom golang template to use in your release page. Your
template can contain string substitutions and you can define those using
the --substitution flag:

  --substitution="releaseTheme:Accentuate the Paw-sitive"
  --substitution="releaseLogo:accentuate-the-pawsitive.png"

ASSET FILES
===========
This command supports uploading release assets to the github page. You
can add asset files with the --asset flag:

  --asset=_output/kubernetes-1.18.2-2.fc33.x86_64.rpm

You can also specify a label for the assets by appending it with a colon
to the asset file:

  --asset="_output/kubernetes-1.18.2-2.fc33.x86_64.rpm:RPM Package for amd64"

`,
	RunE: func(cmd *cobra.Command, args []string) error {
		// Run the PR creation function
		return runGithubPage(ghPageOpts)
	},
}

type githubPageCmdLineOptions struct {
	noupdate         bool
	draft            bool
	sbom             bool
	name             string
	repo             string
	template         string
	repoPath         string
	ReleaseNotesFile string
	substitutions    []string
	assets           []string
}

var ghPageOpts = &githubPageCmdLineOptions{}

func init() {
	githubPageCmd.PersistentFlags().StringVarP(
		&ghPageOpts.repo,
		"repo",
		"r",
		"",
		"repository slug containing the release page",
	)
	githubPageCmd.PersistentFlags().StringVar(
		&ghPageOpts.template,
		"template",
		"",
		"path to a custom page template",
	)
	githubPageCmd.PersistentFlags().StringVarP(
		&ghPageOpts.name,
		"name",
		"n",
		"",
		"name for the release",
	)
	githubPageCmd.PersistentFlags().StringSliceVarP(
		&ghPageOpts.assets,
		"asset",
		"a",
		[]string{},
		"Path to asset file for the release. Can be specified multiple times.",
	)
	githubPageCmd.PersistentFlags().StringSliceVarP(
		&ghPageOpts.substitutions,
		"substitution",
		"s",
		[]string{},
		"String substitution for the page template",
	)
	githubPageCmd.PersistentFlags().BoolVar(
		&ghPageOpts.noupdate,
		"noupdate",
		false,
		"Fail if the release already exists",
	)
	githubPageCmd.PersistentFlags().BoolVar(
		&ghPageOpts.draft,
		"draft",
		false,
		"Mark the release as a draft in GitHub so you can finish editing and publish it manually.",
	)
	githubPageCmd.PersistentFlags().BoolVar(
		&ghPageOpts.sbom,
		"sbom",
		true,
		"Generate an SPDX bill of materials and attach it to the release",
	)

	githubPageCmd.PersistentFlags().StringVar(
		&ghPageOpts.repoPath,
		"repo-path",
		".",
		"Path to the source code repository",
	)

	githubPageCmd.PersistentFlags().StringVar(
		&ghPageOpts.ReleaseNotesFile,
		"release-notes-file",
		"",
		"Path to a release notes markdown file to include in the release",
	)

	for _, f := range []string{"template", "asset"} {
		if err := githubPageCmd.MarkPersistentFlagFilename(f); err != nil {
			logrus.Error(err)
		}
	}

	if err := githubPageCmd.MarkPersistentFlagRequired("repo"); err != nil {
		logrus.Error(err)
	}

	rootCmd.AddCommand(githubPageCmd)
}

func getAssetsFromStrings(assetStrings []string) []announce.Asset {
	r := []announce.Asset{}
	for _, s := range assetStrings {
		parts := strings.Split(s, ":")
		l := ""
		if len(parts) > 1 {
			l = parts[1]
		}
		r = append(r, announce.Asset{
			Path:     filepath.Base(parts[0]),
			ReadFrom: parts[0],
			Label:    l,
		})
	}
	return r
}

func runGithubPage(opts *githubPageCmdLineOptions) (err error) {
	// Generate the release SBOM
	assets := getAssetsFromStrings(opts.assets)
	sbom := ""
	if opts.sbom {
		// Generate the assets file
		sbom, err = announce.GenerateReleaseSBOM(&announce.SBOMOptions{
			ReleaseName:   opts.name,
			Repo:          opts.repo,
			RepoDirectory: opts.repoPath,
			Assets:        assets,
			Tag:           commandLineOpts.tag,
		})
		if err != nil {
			return errors.Wrap(err, "generating sbom")
		}
		opts.assets = append(opts.assets, sbom+":SPDX Software Bill of Materials (SBOM)")
		// Delete the temporary sbom  when we're done
		if commandLineOpts.nomock {
			defer os.Remove(sbom)
		}
	}

	// Build the release page options
	announceOpts := announce.GitHubPageOptions{
		AssetFiles:            opts.assets,
		Tag:                   commandLineOpts.tag,
		NoMock:                commandLineOpts.nomock,
		UpdateIfReleaseExists: !opts.noupdate,
		Name:                  opts.name,
		Draft:                 opts.draft,
		ReleaseNotesFile:      opts.ReleaseNotesFile,
	}

	// Assign the repository data
	if err := announceOpts.SetRepository(opts.repo); err != nil {
		return errors.Wrap(err, "assigning the repository slug")
	}

	// Assign the substitutions
	if err := announceOpts.ParseSubstitutions(opts.substitutions); err != nil {
		return errors.Wrap(err, "parsing template substitutions")
	}

	// Read the csutom template data
	if err := announceOpts.ReadTemplate(opts.template); err != nil {
		return errors.Wrap(err, "reading the template file")
	}

	// Validate the options
	if err := announceOpts.Validate(); err != nil {
		return errors.Wrap(err, "validating options")
	}

	// Run the update process
	return announce.UpdateGitHubPage(&announceOpts)
}
<|endoftext|>


<commit_before>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <metric_root>.collect) to collect
// program and queue metrics.
package collect

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 50

	// Debug enables debug logging.
	Debug = false

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               opentsdb.MultiDataPoint
	qlock, mlock, slock sync.Mutex   // Locks for queues, maps, stats.
	counters                         = make(map[string]*addMetric)
	sets                             = make(map[string]*setMetric)
	client              *http.Client = &http.Client{Timeout: time.Minute}
)

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost, metric_root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(metric_root, "metric root"); err != nil {
		return err
	}
	if tsdbhost == "" {
		return fmt.Errorf("must specify non-empty tsdb host")
	}
	u := url.URL{
		Scheme: "http",
		Path:   "/api/put",
	}
	if !strings.Contains(tsdbhost, ":") {
		tsdbhost += ":4242"
	}
	u.Host = tsdbhost
	tsdbURL = u.String()
	metricRoot = metric_root + "."
	tchan = ch
	go func() {
		for dp := range tchan {
			qlock.Lock()
			for {
				if len(queue) > MaxQueueLen {
					slock.Lock()
					dropped++
					slock.Unlock()
					break
				}
				queue = append(queue, dp)
				select {
				case dp = <-tchan:
					continue
				default:
				}
				break
			}
			qlock.Unlock()
		}
	}()
	go send()

	go collect()
	Set("collect.dropped", nil, func() (i float64) {
		slock.Lock()
		i = float64(dropped)
		slock.Unlock()
		return
	})
	Set("collect.sent", nil, func() (i float64) {
		slock.Lock()
		i = float64(sent)
		slock.Unlock()
		return
	})
	Set("collect.alloc", nil, func() float64 {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return float64(ms.Alloc)
	})
	Set("collect.goroutines", nil, func() float64 {
		return float64(runtime.NumGoroutine())
	})
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost, metric_root string) error {
	return InitChan(tsdbhost, metric_root, make(chan *opentsdb.DataPoint))
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() float64
}

func Set(metric string, ts opentsdb.TagSet, f func() float64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  float64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc float64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if (*ts)["host"] == "" {
		(*ts)["host"] = osHostname
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<commit_msg>cmd/scollector: Clear idle connections every 5 minutes<commit_after>// Package collect provides functions for sending data to OpenTSDB.
//
// The "collect" namespace is used (i.e., <metric_root>.collect) to collect
// program and queue metrics.
package collect

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

var (
	// Freq is how often metrics are sent to OpenTSDB.
	Freq = time.Second * 15

	// MaxQueueLen is the maximum size of the queue, above which incoming data will
	// be discarded. Defaults to about 150MB.
	MaxQueueLen = 200000

	// BatchSize is the maximum length of data points sent at once to OpenTSDB.
	BatchSize = 50

	// Debug enables debug logging.
	Debug = false

	// Dropped is the number of dropped data points due to a full queue.
	dropped int64

	// Sent is the number of sent data points.
	sent int64

	tchan               chan *opentsdb.DataPoint
	tsdbURL             string
	osHostname          string
	metricRoot          string
	queue               opentsdb.MultiDataPoint
	qlock, mlock, slock sync.Mutex   // Locks for queues, maps, stats.
	counters                         = make(map[string]*addMetric)
	sets                             = make(map[string]*setMetric)
	client              *http.Client = &http.Client{
		Transport: new(timeoutTransport),
		Timeout:   time.Minute,
	}
)

type timeoutTransport struct {
	Transport *http.Transport
	Timeout   time.Time
}

func (t *timeoutTransport) RoundTrip(r *http.Request) (*http.Response, error) {
	if time.Now().After(t.Timeout) {
		t.Transport.CloseIdleConnections()
		t.Timeout = time.Now().Add(time.Minute * 5)
	}
	return t.Transport.RoundTrip(r)
}

// InitChan is similar to Init, but uses the given channel instead of creating a
// new one.
func InitChan(tsdbhost, metric_root string, ch chan *opentsdb.DataPoint) error {
	if tchan != nil {
		return fmt.Errorf("cannot init twice")
	}
	if err := checkClean(metric_root, "metric root"); err != nil {
		return err
	}
	if tsdbhost == "" {
		return fmt.Errorf("must specify non-empty tsdb host")
	}
	u := url.URL{
		Scheme: "http",
		Path:   "/api/put",
	}
	if !strings.Contains(tsdbhost, ":") {
		tsdbhost += ":4242"
	}
	u.Host = tsdbhost
	tsdbURL = u.String()
	metricRoot = metric_root + "."
	tchan = ch
	go func() {
		for dp := range tchan {
			qlock.Lock()
			for {
				if len(queue) > MaxQueueLen {
					slock.Lock()
					dropped++
					slock.Unlock()
					break
				}
				queue = append(queue, dp)
				select {
				case dp = <-tchan:
					continue
				default:
				}
				break
			}
			qlock.Unlock()
		}
	}()
	go send()

	go collect()
	Set("collect.dropped", nil, func() (i float64) {
		slock.Lock()
		i = float64(dropped)
		slock.Unlock()
		return
	})
	Set("collect.sent", nil, func() (i float64) {
		slock.Lock()
		i = float64(sent)
		slock.Unlock()
		return
	})
	Set("collect.alloc", nil, func() float64 {
		var ms runtime.MemStats
		runtime.ReadMemStats(&ms)
		return float64(ms.Alloc)
	})
	Set("collect.goroutines", nil, func() float64 {
		return float64(runtime.NumGoroutine())
	})
	return nil
}

// Init sets up the channels and the queue for sending data to OpenTSDB. It also
// sets up the basename for all metrics.
func Init(tsdbhost, metric_root string) error {
	return InitChan(tsdbhost, metric_root, make(chan *opentsdb.DataPoint))
}

func setHostName() error {
	h, err := os.Hostname()
	if err != nil {
		return err
	}
	osHostname = strings.ToLower(strings.SplitN(h, ".", 2)[0])
	if err := checkClean(osHostname, "host tag"); err != nil {
		return err
	}
	return nil
}

type setMetric struct {
	metric string
	ts     opentsdb.TagSet
	f      func() float64
}

func Set(metric string, ts opentsdb.TagSet, f func() float64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	sets[tss] = &setMetric{metric, ts.Copy(), f}
	mlock.Unlock()
	return nil
}

type addMetric struct {
	metric string
	ts     opentsdb.TagSet
	value  float64
}

// Add takes a metric and increments a counter for that metric. The metric name
// is appended to the basename specified in the Init function.
func Add(metric string, ts opentsdb.TagSet, inc float64) error {
	if err := check(metric, &ts); err != nil {
		return err
	}
	tss := metric + ts.String()
	mlock.Lock()
	if counters[tss] == nil {
		counters[tss] = &addMetric{
			metric: metric,
			ts:     ts.Copy(),
		}
	}
	counters[tss].value += inc
	mlock.Unlock()
	return nil
}

func check(metric string, ts *opentsdb.TagSet) error {
	if err := checkClean(metric, "metric"); err != nil {
		return err
	}
	for k, v := range *ts {
		if err := checkClean(k, "tagk"); err != nil {
			return err
		}
		if err := checkClean(v, "tagv"); err != nil {
			return err
		}
	}
	if osHostname == "" {
		if err := setHostName(); err != nil {
			return err
		}
	}
	if *ts == nil {
		*ts = make(opentsdb.TagSet)
	}
	if (*ts)["host"] == "" {
		(*ts)["host"] = osHostname
	}
	return nil
}

func checkClean(s, t string) error {
	if sc, err := opentsdb.Clean(s); s != sc || err != nil {
		if err != nil {
			return err
		}
		return fmt.Errorf("%s %s may only contain a to z, A to Z, 0 to 9, -, _, ., / or Unicode letters and may not be empty", t, s)
	}
	return nil
}

func collect() {
	for {
		mlock.Lock()
		now := time.Now().Unix()
		for _, c := range counters {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + c.metric,
				Timestamp: now,
				Value:     c.value,
				Tags:      c.ts,
			}
			tchan <- dp
		}
		for _, s := range sets {
			dp := &opentsdb.DataPoint{
				Metric:    metricRoot + s.metric,
				Timestamp: now,
				Value:     s.f(),
				Tags:      s.ts,
			}
			tchan <- dp
		}
		mlock.Unlock()
		time.Sleep(Freq)
	}
}
<|endoftext|>


<commit_before>package main

import (
	_ "code.google.com/p/go.image/bmp"
	_ "code.google.com/p/go.image/tiff"
	_ "code.google.com/p/go.image/webp"
	"github.com/hullerob/go.farbfeld"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"os"
)

func main() {
	if len(os.Args) != 1 {
		usage()
		os.Exit(1)
	}
	m, _, err := image.Decode(os.Stdin)
	if err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
	err = imagefile.Encode(os.Stdout, m)
	os.Stdout.Sync()
	if err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
}

func usage() {
	os.Stderr.WriteString("usage: img2ff\n")
}
<commit_msg>update import paths<commit_after>package main

import (
	"github.com/hullerob/go.farbfeld"
	_ "golang.org/x/image/bmp"
	_ "golang.org/x/image/tiff"
	_ "golang.org/x/image/webp"
	"image"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"os"
)

func main() {
	if len(os.Args) != 1 {
		usage()
		os.Exit(1)
	}
	m, _, err := image.Decode(os.Stdin)
	if err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
	err = imagefile.Encode(os.Stdout, m)
	os.Stdout.Sync()
	if err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Exit(1)
	}
}

func usage() {
	os.Stderr.WriteString("usage: img2ff\n")
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poll

import (
	"internal/syscall/unix"
	"sync"
	"syscall"
)

var (
	kernelVersion53Once sync.Once
	kernelVersion53     bool
)

const maxCopyFileRangeRound = 1 << 30

// CopyFileRange copies at most remain bytes of data from src to dst, using
// the copy_file_range system call. dst and src must refer to regular files.
func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) {
	kernelVersion53Once.Do(func() {
		major, minor := unix.KernelVersion()
		// copy_file_range(2) is broken in various ways on kernels older than 5.3,
		// see issue #42400 and
		// https://man7.org/linux/man-pages/man2/copy_file_range.2.html#VERSIONS
		if major > 5 || (major == 5 && minor >= 3) {
			kernelVersion53 = true
		}
	})

	if !kernelVersion53 {
		return 0, false, nil
	}

	for remain > 0 {
		max := remain
		if max > maxCopyFileRangeRound {
			max = maxCopyFileRangeRound
		}
		n, err := copyFileRange(dst, src, int(max))
		switch err {
		case syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM:
			// EINVAL is what we see if, for example,
			// dst or src refers to a pipe rather than a regular
			// file. This is another case where no data has been
			// transferred, so we consider it unhandled.
			//
			// If src and dst are on CIFS, we can see EIO.
			// See issue #42334.
			//
			// If the file is on NFS, we can see EOPNOTSUPP.
			// See issue #40731.
			//
			// If the process is running inside a Docker container,
			// we might see EPERM instead of ENOSYS. See issue #40893.
			// Since EPERM might also be a legitimate error: operation not permitted,
			// we should still keep this error even if we have the previous kernel version 5.3 check
			// and don't mark copy_file_range(2) as unsupported.
			return 0, false, nil
		case nil:
			if n == 0 {
				// If we did not read any bytes at all,
				// then this file may be in a file system
				// where copy_file_range silently fails.
				// https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0
				if written == 0 {
					return 0, false, nil
				}
				// Otherwise, src is at EOF, which means
				// we are done.
				return written, true, nil
			}
			remain -= n
			written += n
		default:
			return written, true, err
		}
	}
	return written, true, nil
}

// copyFileRange performs one round of copy_file_range(2).
func copyFileRange(dst, src *FD, max int) (written int64, err error) {
	// The signature of copy_file_range(2) is:
	//
	// ssize_t copy_file_range(int fd_in, loff_t *off_in,
	//                         int fd_out, loff_t *off_out,
	//                         size_t len, unsigned int flags);
	//
	// Note that in the call to unix.CopyFileRange below, we use nil
	// values for off_in and off_out. For the system call, this means
	// "use and update the file offsets". That is why we must acquire
	// locks for both file descriptors (and why this whole machinery is
	// in the internal/poll package to begin with).
	if err := dst.writeLock(); err != nil {
		return 0, err
	}
	defer dst.writeUnlock()
	if err := src.readLock(); err != nil {
		return 0, err
	}
	defer src.readUnlock()
	var n int
	for {
		n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0)
		if err != syscall.EINTR {
			break
		}
	}
	return int64(n), err
}
<commit_msg>Revert "internal/poll: drop redundant ENOSYS and EXDEV error checks in CopyFileRange()"<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package poll

import (
	"internal/syscall/unix"
	"sync"
	"syscall"
)

var (
	kernelVersion53Once sync.Once
	kernelVersion53     bool
)

const maxCopyFileRangeRound = 1 << 30

// CopyFileRange copies at most remain bytes of data from src to dst, using
// the copy_file_range system call. dst and src must refer to regular files.
func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) {
	kernelVersion53Once.Do(func() {
		major, minor := unix.KernelVersion()
		// copy_file_range(2) is broken in various ways on kernels older than 5.3,
		// see issue #42400 and
		// https://man7.org/linux/man-pages/man2/copy_file_range.2.html#VERSIONS
		if major > 5 || (major == 5 && minor >= 3) {
			kernelVersion53 = true
		}
	})

	if !kernelVersion53 {
		return 0, false, nil
	}

	for remain > 0 {
		max := remain
		if max > maxCopyFileRangeRound {
			max = maxCopyFileRangeRound
		}
		n, err := copyFileRange(dst, src, int(max))
		switch err {
		case syscall.ENOSYS:
			// copy_file_range(2) was introduced in Linux 4.5.
			// Go supports Linux >= 2.6.33, so the system call
			// may not be present.
			//
			// If we see ENOSYS, we have certainly not transferred
			// any data, so we can tell the caller that we
			// couldn't handle the transfer and let them fall
			// back to more generic code.
			return 0, false, nil
		case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM:
			// Prior to Linux 5.3, it was not possible to
			// copy_file_range across file systems. Similarly to
			// the ENOSYS case above, if we see EXDEV, we have
			// not transferred any data, and we can let the caller
			// fall back to generic code.
			//
			// As for EINVAL, that is what we see if, for example,
			// dst or src refer to a pipe rather than a regular
			// file. This is another case where no data has been
			// transferred, so we consider it unhandled.
			//
			// If src and dst are on CIFS, we can see EIO.
			// See issue #42334.
			//
			// If the file is on NFS, we can see EOPNOTSUPP.
			// See issue #40731.
			//
			// If the process is running inside a Docker container,
			// we might see EPERM instead of ENOSYS. See issue
			// #40893. Since EPERM might also be a legitimate error,
			// don't mark copy_file_range(2) as unsupported.
			return 0, false, nil
		case nil:
			if n == 0 {
				// If we did not read any bytes at all,
				// then this file may be in a file system
				// where copy_file_range silently fails.
				// https://lore.kernel.org/linux-fsdevel/20210126233840.GG4626@dread.disaster.area/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0
				if written == 0 {
					return 0, false, nil
				}
				// Otherwise src is at EOF, which means
				// we are done.
				return written, true, nil
			}
			remain -= n
			written += n
		default:
			return written, true, err
		}
	}
	return written, true, nil
}

// copyFileRange performs one round of copy_file_range(2).
func copyFileRange(dst, src *FD, max int) (written int64, err error) {
	// The signature of copy_file_range(2) is:
	//
	// ssize_t copy_file_range(int fd_in, loff_t *off_in,
	//                         int fd_out, loff_t *off_out,
	//                         size_t len, unsigned int flags);
	//
	// Note that in the call to unix.CopyFileRange below, we use nil
	// values for off_in and off_out. For the system call, this means
	// "use and update the file offsets". That is why we must acquire
	// locks for both file descriptors (and why this whole machinery is
	// in the internal/poll package to begin with).
	if err := dst.writeLock(); err != nil {
		return 0, err
	}
	defer dst.writeUnlock()
	if err := src.readLock(); err != nil {
		return 0, err
	}
	defer src.readUnlock()
	var n int
	for {
		n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0)
		if err != syscall.EINTR {
			break
		}
	}
	return int64(n), err
}
<|endoftext|>


<commit_before>package pac

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/txthinking/x"
)

func PAC(proxy, mode, domainURL, cidrURL string) (io.Reader, error) {
	t := template.New("pac")
	t, err := t.Parse(js)
	if err != nil {
		return nil, err
	}
	b := &bytes.Buffer{}

	if mode == "global" {
		if err := t.Execute(b, map[string]interface{}{
			"mode":  "global",
			"proxy": proxy,
		}); err != nil {
			return nil, err
		}
		return b, nil
	}

	var ds []string
	var cs []map[string]int64
	if domainURL != "" {
		data, err := readData(domainURL)
		if err != nil {
			return nil, err
		}
		ds = makeDomains(data)
	}
	if cidrURL != "" {
		data, err := readData(cidrURL)
		if err != nil {
			return nil, err
		}
		cs = makeCIDRs(data)
	}

	if err := t.Execute(b, map[string]interface{}{
		"proxy":   proxy,
		"mode":    mode,
		"domains": ds,
		"cidrs":   cs,
	}); err != nil {
		return nil, err
	}
	return b, nil
}

func readData(url string) ([]byte, error) {
	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
		c := &http.Client{
			Timeout: 9 * time.Second,
		}
		r, err := c.Get(url)
		if err != nil {
			return nil, err
		}
		defer r.Body.Close()
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return nil, err
		}
		return data, nil
	}
	if strings.HasPrefix(url, "file://") {
		data, err := ioutil.ReadFile(url[7:])
		if err != nil {
			return nil, err
		}
		return data, nil
	}
	return nil, errors.New("Unsupport URL")
}

func makeDomains(data []byte) []string {
	data = bytes.TrimSpace(data)
	data = bytes.Replace(data, []byte{0x20}, []byte{}, -1)
	data = bytes.Replace(data, []byte{0x0d, 0x0a}, []byte{0x0a}, -1)
	ds := strings.Split(string(data), "\n")
	return ds
}

func makeCIDRs(data []byte) []map[string]int64 {
	cs := make([]map[string]int64, 0)
	data = bytes.TrimSpace(data)
	data = bytes.Replace(data, []byte{0x20}, []byte{}, -1)
	data = bytes.Replace(data, []byte{0x0d, 0x0a}, []byte{0x0a}, -1)
	ss := strings.Split(string(data), "\n")
	for _, s := range ss {
		c, err := x.CIDR(s)
		if err != nil {
			continue
		}
		first, err := x.IP2Decimal(c.First)
		if err != nil {
			continue
		}
		last, err := x.IP2Decimal(c.Last)
		if err != nil {
			continue
		}
		m := make(map[string]int64)
		m["first"] = first
		m["last"] = last
		cs = append(cs, m)
	}
	return cs
}

const js = `
//
// https://github.com/txthinking/pac
//

var proxy="{{.proxy}}";

var mode = "{{.mode}}";

{{if .domains}}
var domains = {
	{{range .domains}}
	"{{.}}": 1,
	{{end}}
};
{{end}}

{{if .cidrs}}
var cidrs = [
    {{range .cidrs}}
    [{{.first}},{{.last}}],
	{{end}}
];
{{end}}

function ip2decimal(ip) {
    var d = ip.split('.');
    return ((((((+d[0])*256)+(+d[1]))*256)+(+d[2]))*256)+(+d[3]);
}

function FindProxyForURL(url, host){
    if(/\d+\.\d+\.\d+\.\d+/.test(host)){
        if (isInNet(dnsResolve(host), "10.0.0.0", "255.0.0.0") ||
                isInNet(dnsResolve(host), "172.16.0.0",  "255.240.0.0") ||
                isInNet(dnsResolve(host), "192.168.0.0", "255.255.0.0") ||
                isInNet(dnsResolve(host), "127.0.0.0", "255.255.255.0")){
            return "DIRECT";
        }
        {{if .cidrs}}
        var d = ip2decimal(host);
        var l = cidrs.length;
        var min = 0;
        var max = l;
        for(;;){
            if (min+1 > max) {
                break;
            }
            var mid = Math.floor(min+(max-min)/2);
            if(d >= cidrs[mid][0] && d <= cidrs[mid][1]){
				if(mode == "white"){
					return "DIRECT";
				}
				if(mode == "black"){
					return proxy;
				}
            }else if(d < cidrs[mid][0]){
                max = mid;
            }else{
                min = mid+1;
            }
        }
		{{end}}
    }

    if (isPlainHostName(host)){
        return "DIRECT";
    }

    {{if .domains}}
    var a = host.split(".");
    for(var i=a.length-1; i>=0; i--){
        if (domains.hasOwnProperty(a.slice(i).join("."))){
			if(mode == "white"){
				return "DIRECT";
			}
			if(mode == "black"){
				return proxy;
			}
        }
    }
	if(mode == "white"){
		return proxy;
	}
	if(mode == "black"){
		return "DIRECT";
	}
	{{end}}

	if(mode == "global"){
		return proxy;
	}
}
`
<commit_msg>pac from string<commit_after>package pac

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/txthinking/x"
)

func PAC(proxy, mode, domainURL, cidrURL string) (io.Reader, error) {
	t := template.New("pac")
	t, err := t.Parse(js)
	if err != nil {
		return nil, err
	}
	b := &bytes.Buffer{}

	if mode == "global" {
		if err := t.Execute(b, map[string]interface{}{
			"mode":  "global",
			"proxy": proxy,
		}); err != nil {
			return nil, err
		}
		return b, nil
	}

	var ds []string
	var cs []map[string]int64
	if domainURL != "" {
		data, err := readData(domainURL)
		if err != nil {
			return nil, err
		}
		ds = makeDomains(data)
	}
	if cidrURL != "" {
		data, err := readData(cidrURL)
		if err != nil {
			return nil, err
		}
		cs = makeCIDRs(data)
	}

	if err := t.Execute(b, map[string]interface{}{
		"proxy":   proxy,
		"mode":    mode,
		"domains": ds,
		"cidrs":   cs,
	}); err != nil {
		return nil, err
	}
	return b, nil
}

func PACFromString(proxy, mode, domains, cidrs string) (io.Reader, error) {
	t := template.New("pac")
	t, err := t.Parse(js)
	if err != nil {
		return nil, err
	}
	b := &bytes.Buffer{}

	if mode == "global" {
		if err := t.Execute(b, map[string]interface{}{
			"mode":  "global",
			"proxy": proxy,
		}); err != nil {
			return nil, err
		}
		return b, nil
	}

	var ds []string
	var cs []map[string]int64
	if domains != "" {
		ds = makeDomains([]byte(domains))
	}
	if cidrs != "" {
		cs = makeCIDRs([]byte(cidrs))
	}

	if err := t.Execute(b, map[string]interface{}{
		"proxy":   proxy,
		"mode":    mode,
		"domains": ds,
		"cidrs":   cs,
	}); err != nil {
		return nil, err
	}
	return b, nil
}

func readData(url string) ([]byte, error) {
	if strings.HasPrefix(url, "http://") || strings.HasPrefix(url, "https://") {
		c := &http.Client{
			Timeout: 9 * time.Second,
		}
		r, err := c.Get(url)
		if err != nil {
			return nil, err
		}
		defer r.Body.Close()
		data, err := ioutil.ReadAll(r.Body)
		if err != nil {
			return nil, err
		}
		return data, nil
	}
	if strings.HasPrefix(url, "file://") {
		data, err := ioutil.ReadFile(url[7:])
		if err != nil {
			return nil, err
		}
		return data, nil
	}
	return nil, errors.New("Unsupport URL")
}

func makeDomains(data []byte) []string {
	data = bytes.TrimSpace(data)
	data = bytes.Replace(data, []byte{0x20}, []byte{}, -1)
	data = bytes.Replace(data, []byte{0x0d, 0x0a}, []byte{0x0a}, -1)
	ds := strings.Split(string(data), "\n")
	return ds
}

func makeCIDRs(data []byte) []map[string]int64 {
	cs := make([]map[string]int64, 0)
	data = bytes.TrimSpace(data)
	data = bytes.Replace(data, []byte{0x20}, []byte{}, -1)
	data = bytes.Replace(data, []byte{0x0d, 0x0a}, []byte{0x0a}, -1)
	ss := strings.Split(string(data), "\n")
	for _, s := range ss {
		c, err := x.CIDR(s)
		if err != nil {
			continue
		}
		first, err := x.IP2Decimal(c.First)
		if err != nil {
			continue
		}
		last, err := x.IP2Decimal(c.Last)
		if err != nil {
			continue
		}
		m := make(map[string]int64)
		m["first"] = first
		m["last"] = last
		cs = append(cs, m)
	}
	return cs
}

const js = `
//
// https://github.com/txthinking/pac
//

var proxy="{{.proxy}}";

var mode = "{{.mode}}";

{{if .domains}}
var domains = {
	{{range .domains}}
	"{{.}}": 1,
	{{end}}
};
{{end}}

{{if .cidrs}}
var cidrs = [
    {{range .cidrs}}
    [{{.first}},{{.last}}],
	{{end}}
];
{{end}}

function ip2decimal(ip) {
    var d = ip.split('.');
    return ((((((+d[0])*256)+(+d[1]))*256)+(+d[2]))*256)+(+d[3]);
}

function FindProxyForURL(url, host){
    if(/\d+\.\d+\.\d+\.\d+/.test(host)){
        if (isInNet(dnsResolve(host), "10.0.0.0", "255.0.0.0") ||
                isInNet(dnsResolve(host), "172.16.0.0",  "255.240.0.0") ||
                isInNet(dnsResolve(host), "192.168.0.0", "255.255.0.0") ||
                isInNet(dnsResolve(host), "127.0.0.0", "255.255.255.0")){
            return "DIRECT";
        }
        {{if .cidrs}}
        var d = ip2decimal(host);
        var l = cidrs.length;
        var min = 0;
        var max = l;
        for(;;){
            if (min+1 > max) {
                break;
            }
            var mid = Math.floor(min+(max-min)/2);
            if(d >= cidrs[mid][0] && d <= cidrs[mid][1]){
				if(mode == "white"){
					return "DIRECT";
				}
				if(mode == "black"){
					return proxy;
				}
            }else if(d < cidrs[mid][0]){
                max = mid;
            }else{
                min = mid+1;
            }
        }
		{{end}}
    }

    if (isPlainHostName(host)){
        return "DIRECT";
    }

    {{if .domains}}
    var a = host.split(".");
    for(var i=a.length-1; i>=0; i--){
        if (domains.hasOwnProperty(a.slice(i).join("."))){
			if(mode == "white"){
				return "DIRECT";
			}
			if(mode == "black"){
				return proxy;
			}
        }
    }
	if(mode == "white"){
		return proxy;
	}
	if(mode == "black"){
		return "DIRECT";
	}
	{{end}}

	if(mode == "global"){
		return proxy;
	}
}
`
<|endoftext|>


<commit_before>package d3d9

import "unsafe"

const MAX_DEVICE_IDENTIFIER_STRING = 512

// ADAPTER_IDENTIFIER contains information identifying the adapter.
type ADAPTER_IDENTIFIER struct {
	Driver           [MAX_DEVICE_IDENTIFIER_STRING]byte
	Description      [MAX_DEVICE_IDENTIFIER_STRING]byte
	DeviceName       [32]byte
	DriverVersion    int64
	VendorId         uint32
	DeviceId         uint32
	SubSysId         uint32
	Revision         uint32
	DeviceIdentifier GUID
	WHQLLevel        uint32
}

// GUID is a globally unique identifier.
type GUID struct {
	Data1 uint32
	Data2 uint16
	Data3 uint16
	Data4 [8]uint8
}

type DISPLAYMODE struct {
	Width       uint32
	Height      uint32
	RefreshRate uint32
	Format      FORMAT
}

// CAPS represents the capabilities of the hardware exposed through the Direct3D
// object.
type CAPS struct {
	DeviceType                        DEVTYPE
	AdapterOrdinal                    uint32
	Caps                              uint32
	Caps2                             uint32
	Caps3                             uint32
	PresentationIntervals             uint32
	CursorCaps                        uint32
	DevCaps                           uint32
	PrimitiveMiscCaps                 uint32
	RasterCaps                        uint32
	ZCmpCaps                          uint32
	SrcBlendCaps                      uint32
	DestBlendCaps                     uint32
	AlphaCmpCaps                      uint32
	ShadeCaps                         uint32
	TextureCaps                       uint32
	TextureFilterCaps                 uint32
	CubeTextureFilterCaps             uint32
	VolumeTextureFilterCaps           uint32
	TextureAddressCaps                uint32
	VolumeTextureAddressCaps          uint32
	LineCaps                          uint32
	MaxTextureWidth                   uint32
	MaxTextureHeight                  uint32
	MaxVolumeExtent                   uint32
	MaxTextureRepeat                  uint32
	MaxTextureAspectRatio             uint32
	MaxAnisotropy                     uint32
	MaxVertexW                        float32
	GuardBandLeft                     float32
	GuardBandTop                      float32
	GuardBandRight                    float32
	GuardBandBottom                   float32
	ExtentsAdjust                     float32
	StencilCaps                       uint32
	FVFCaps                           uint32
	TextureOpCaps                     uint32
	MaxTextureBlendStages             uint32
	MaxSimultaneousTextures           uint32
	VertexProcessingCaps              uint32
	MaxActiveLights                   uint32
	MaxUserClipPlanes                 uint32
	MaxVertexBlendMatrices            uint32
	MaxVertexBlendMatrixIndex         uint32
	MaxPointSize                      float32
	MaxPrimitiveCount                 uint32
	MaxVertexIndex                    uint32
	MaxStreams                        uint32
	MaxStreamStride                   uint32
	VertexShaderVersion               uint32
	MaxVertexShaderConst              uint32
	PixelShaderVersion                uint32
	PixelShader1xMaxValue             float32
	DevCaps2                          uint32
	MasterAdapterOrdinal              uint32
	AdapterOrdinalInGroup             uint32
	NumberOfAdaptersInGroup           uint32
	DeclTypes                         uint32
	NumSimultaneousRTs                uint32
	StretchRectFilterCaps             uint32
	VS20Caps                          VSHADERCAPS2_0
	PS20Caps                          PSHADERCAPS2_0
	VertexTextureFilterCaps           uint32
	MaxVShaderInstructionsExecuted    uint32
	MaxPShaderInstructionsExecuted    uint32
	MaxVertexShader30InstructionSlots uint32
	MaxPixelShader30InstructionSlots  uint32
}

// VSHADERCAPS2_0 contains vertex shader capabilities.
type VSHADERCAPS2_0 struct {
	Caps                    uint32
	DynamicFlowControlDepth int32
	NumTemps                int32
	StaticFlowControlDepth  int32
}

// PSHADERCAPS2_0 describes pixel shader driver caps.
type PSHADERCAPS2_0 struct {
	Caps                    uint32
	DynamicFlowControlDepth int32
	NumTemps                int32
	StaticFlowControlDepth  int32
	NumInstructionSlots     int32
}

// PRESENT_PARAMETERS describes the presentation parameters.
type PRESENT_PARAMETERS struct {
	BackBufferWidth            uint32
	BackBufferHeight           uint32
	BackBufferFormat           FORMAT
	BackBufferCount            uint32
	MultiSampleType            MULTISAMPLE_TYPE
	MultiSampleQuality         uint32
	SwapEffect                 SWAPEFFECT
	HDeviceWindow              HWND
	Windowed                   int32
	EnableAutoDepthStencil     int32
	AutoDepthStencilFormat     FORMAT
	Flags                      uint32
	FullScreen_RefreshRateInHz uint32
	PresentationInterval       uint32
}

type (
	HANDLE   uintptr
	HWND     HANDLE
	HMONITOR HANDLE
	HDC      HANDLE
)

// RECT describes a rectangle.
type RECT struct {
	Left   int32
	Top    int32
	Right  int32
	Bottom int32
}

// RGNDATA contains region data.
type RGNDATA struct {
	Rdh    RGNDATAHEADER
	Buffer [1]byte
}

// RGNDATAHEADER describes region data.
type RGNDATAHEADER struct {
	DwSize   uint32
	IType    uint32
	NCount   uint32
	NRgnSize uint32
	RcBound  RECT
}

// DEVICE_CREATION_PARAMETERS describes the creation parameters for a device.
type DEVICE_CREATION_PARAMETERS struct {
	AdapterOrdinal uint32
	DeviceType     DEVTYPE
	HFocusWindow   HWND
	BehaviorFlags  uint32
}

// RASTER_STATUS describes the raster status.
type RASTER_STATUS struct {
	InVBlank int32
	ScanLine uint32
}

// GAMMARAMP contains red, green, and blue ramp data.
type GAMMARAMP struct {
	Red   [256]uint16
	Green [256]uint16
	Blue  [256]uint16
}

// POINT describes a 2D point.
type POINT struct {
	X int32
	Y int32
}

// MATRIX describes a matrix.
type MATRIX [16]float32

// VIEWPORT defines the window dimensions of a render-target surface onto which
// a 3D volume projects.
type VIEWPORT struct {
	X      uint32
	Y      uint32
	Width  uint32
	Height uint32
	MinZ   float32
	MaxZ   float32
}

// MATERIAL specifies material properties.
type MATERIAL struct {
	Diffuse  COLORVALUE
	Ambient  COLORVALUE
	Specular COLORVALUE
	Emissive COLORVALUE
	Power    float32
}

// COLORVALUE describes color values.
type COLORVALUE struct {
	R float32
	G float32
	B float32
	A float32
}

// LIGHT defines a set of lighting properties.
type LIGHT struct {
	Type         LIGHTTYPE
	Diffuse      COLORVALUE
	Specular     COLORVALUE
	Ambient      COLORVALUE
	Position     VECTOR
	Direction    VECTOR
	Range        float32
	Falloff      float32
	Attenuation0 float32
	Attenuation1 float32
	Attenuation2 float32
	Theta        float32
	Phi          float32
}

// VECTOR defines a vector.
type VECTOR struct {
	X float32
	Y float32
	Z float32
}

// CLIPSTATUS describes the current clip status.
type CLIPSTATUS struct {
	ClipUnion        uint32
	ClipIntersection uint32
}

// PALETTEENTRY specifies the color and usage of an entry in a logical palette.
type PALETTEENTRY struct {
	PeRed   byte
	PeGreen byte
	PeBlue  byte
	PeFlags byte
}

// VERTEXELEMENT defines the vertex data layout. Each vertex can contain one or
// more data types, and each data type is described by a vertex element.
type VERTEXELEMENT struct {
	Stream     uint16
	Offset     uint16
	Type       DECLTYPE
	Method     DECLMETHOD
	Usage      DECLUSAGE
	UsageIndex byte
}

// RECTPATCH_INFO describes a rectangular high-order patch.
type RECTPATCH_INFO struct {
	StartVertexOffsetWidth  uint32
	StartVertexOffsetHeight uint32
	Width                   uint32
	Height                  uint32
	Stride                  uint32
	Basis                   BASISTYPE
	Degree                  DEGREETYPE
}

// TRIPATCH_INFO describes a triangular high-order patch.
type TRIPATCH_INFO struct {
	StartVertexOffset uint32
	NumVertices       uint32
	Basis             BASISTYPE
	Degree            DEGREETYPE
}

// BOX defines a volume.
type BOX struct {
	Left   uint32
	Top    uint32
	Right  uint32
	Bottom uint32
	Front  uint32
	Back   uint32
}

// VOLUME_DESC describes a volume.
type VOLUME_DESC struct {
	Format FORMAT
	Type   RESOURCETYPE
	Usage  uint32
	Pool   POOL
	Width  uint32
	Height uint32
	Depth  uint32
}

// LOCKED_BOX describes a locked box (volume).
type LOCKED_BOX struct {
	RowPitch   int32
	SlicePitch int32
	PBits      uintptr
}

// VERTEXBUFFER_DESC describes a vertex buffer.
type VERTEXBUFFER_DESC struct {
	Format FORMAT
	Type   RESOURCETYPE
	Usage  uint32
	Pool   POOL
	Size   uint32
	FVF    uint32
}

// INDEXBUFFER_DESC describes an index buffer.
type INDEXBUFFER_DESC struct {
	Format FORMAT
	Type   RESOURCETYPE
	Usage  uint32
	Pool   POOL
	Size   uint32
}

// SURFACE_DESC describes a surface.
type SURFACE_DESC struct {
	Format             FORMAT
	Type               RESOURCETYPE
	Usage              uint32
	Pool               POOL
	MultiSampleType    MULTISAMPLE_TYPE
	MultiSampleQuality uint32
	Width              uint32
	Height             uint32
}

// LOCKED_RECT describes a locked rectangular region.
type LOCKED_RECT struct {
	Pitch int32
	PBits uintptr
}

// SetAllBytes will fill the whole rect with the given data, taking into account
// the rect's pitch. The given byte slice is expected to have the given stride
// in bytes, i.e. one line in the given data is <srcStride> bytes in length.
func (r LOCKED_RECT) SetAllBytes(data []byte, srcStride int) {
	dest := r.PBits
	destStride := int(r.Pitch)
	src := uintptr(unsafe.Pointer(&data[0]))
	height := len(data) / srcStride

	stride := srcStride
	if destStride < srcStride {
		stride = destStride
	}
	destSkip := uintptr(destStride - stride)
	srcSkip := uintptr(srcStride - stride)
	d := dest
	s := src
	for y := 0; y < height; y++ {
		for x := 0; x < stride; x++ {
			*((*byte)(unsafe.Pointer(d))) = *((*byte)(unsafe.Pointer(s)))
			d++
			s++
		}
		d += destSkip
		s += srcSkip
	}
}

// DEVINFO_D3D9BANDWIDTHTIMINGS contains throughput metrics for help in
// understanding the performance of an application.
type DEVINFO_D3D9BANDWIDTHTIMINGS struct {
	MaxBandwidthUtilized                float32
	FrontEndUploadMemoryUtilizedPercent float32
	VertexRateUtilizedPercent           float32
	TriangleSetupRateUtilizedPercent    float32
	FillRateUtilizedPercent             float32
}

// DEVINFO_D3D9CACHEUTILIZATION measures the cache hit rate performance for
// textures and indexed vertices.
type DEVINFO_D3D9CACHEUTILIZATION struct {
	TextureCacheHitRate             float32
	PostTransformVertexCacheHitRate float32
}

// DEVINFO_D3D9INTERFACETIMINGS contains the percent of time processing data in
// the driver. These statistics may help identify cases when the driver is
// waiting for other resources.
type DEVINFO_D3D9INTERFACETIMINGS struct {
	WaitingForGPUToUseApplicationResourceTimePercent float32
	WaitingForGPUToAcceptMoreCommandsTimePercent     float32
	WaitingForGPUToStayWithinLatencyTimePercent      float32
	WaitingForGPUExclusiveResourceTimePercent        float32
	WaitingForGPUOtherTimePercent                    float32
}

// DEVINFO_D3D9PIPELINETIMINGS contains the percent of time processing data in
// the pipeline.
type DEVINFO_D3D9PIPELINETIMINGS struct {
	VertexProcessingTimePercent   float32
	PixelProcessingTimePercent    float32
	OtherGPUProcessingTimePercent float32
	GPUIdleTimePercent            float32
}

// DEVINFO_D3D9STAGETIMINGS contains the percent of time processing shader data.
type DEVINFO_D3D9STAGETIMINGS struct {
	MemoryProcessingPercent      float32
	ComputationProcessingPercent float32
}

// DEVINFO_D3DVERTEXSTATS reports the number of triangles that have been
// processed and clipped by the runtime's software vertex processing.
type DEVINFO_D3DVERTEXSTATS struct {
	NumRenderedTriangles      uint32
	NumExtraClippingTriangles uint32
}

// DEVINFO_VCACHE contains vertex cache optimization hints.
type DEVINFO_VCACHE struct {
	Pattern     uint32
	OptMethod   uint32
	CacheSize   uint32
	MagicNumber uint32
}

// RESOURCESTATS contains resource statistics gathered by the
// DEVINFO_ResourceManager when using the asynchronous query mechanism.
type RESOURCESTATS struct {
	BThrashing            uint32
	ApproxBytesDownloaded uint32
	NumEvicts             uint32
	NumVidCreates         uint32
	LastPri               uint32
	NumUsed               uint32
	NumUsedInVidMem       uint32
	WorkingSet            uint32
	WorkingSetBytes       uint32
	TotalManaged          uint32
	TotalBytes            uint32
}

// LUID is a 64-bit value guaranteed to be unique only on the system on which it
// was generated. The uniqueness of a locally unique identifier (LUID) is
// guaranteed only until the system is restarted.
type LUID struct {
	LowPart  uint32
	HighPart int32
}

// RANGE defines a range.
type RANGE struct {
	Offset uint32
	Size   uint32
}
<commit_msg>Optimize texture data uploading<commit_after>package d3d9

import "unsafe"

const MAX_DEVICE_IDENTIFIER_STRING = 512

// ADAPTER_IDENTIFIER contains information identifying the adapter.
type ADAPTER_IDENTIFIER struct {
	Driver           [MAX_DEVICE_IDENTIFIER_STRING]byte
	Description      [MAX_DEVICE_IDENTIFIER_STRING]byte
	DeviceName       [32]byte
	DriverVersion    int64
	VendorId         uint32
	DeviceId         uint32
	SubSysId         uint32
	Revision         uint32
	DeviceIdentifier GUID
	WHQLLevel        uint32
}

// GUID is a globally unique identifier.
type GUID struct {
	Data1 uint32
	Data2 uint16
	Data3 uint16
	Data4 [8]uint8
}

type DISPLAYMODE struct {
	Width       uint32
	Height      uint32
	RefreshRate uint32
	Format      FORMAT
}

// CAPS represents the capabilities of the hardware exposed through the Direct3D
// object.
type CAPS struct {
	DeviceType                        DEVTYPE
	AdapterOrdinal                    uint32
	Caps                              uint32
	Caps2                             uint32
	Caps3                             uint32
	PresentationIntervals             uint32
	CursorCaps                        uint32
	DevCaps                           uint32
	PrimitiveMiscCaps                 uint32
	RasterCaps                        uint32
	ZCmpCaps                          uint32
	SrcBlendCaps                      uint32
	DestBlendCaps                     uint32
	AlphaCmpCaps                      uint32
	ShadeCaps                         uint32
	TextureCaps                       uint32
	TextureFilterCaps                 uint32
	CubeTextureFilterCaps             uint32
	VolumeTextureFilterCaps           uint32
	TextureAddressCaps                uint32
	VolumeTextureAddressCaps          uint32
	LineCaps                          uint32
	MaxTextureWidth                   uint32
	MaxTextureHeight                  uint32
	MaxVolumeExtent                   uint32
	MaxTextureRepeat                  uint32
	MaxTextureAspectRatio             uint32
	MaxAnisotropy                     uint32
	MaxVertexW                        float32
	GuardBandLeft                     float32
	GuardBandTop                      float32
	GuardBandRight                    float32
	GuardBandBottom                   float32
	ExtentsAdjust                     float32
	StencilCaps                       uint32
	FVFCaps                           uint32
	TextureOpCaps                     uint32
	MaxTextureBlendStages             uint32
	MaxSimultaneousTextures           uint32
	VertexProcessingCaps              uint32
	MaxActiveLights                   uint32
	MaxUserClipPlanes                 uint32
	MaxVertexBlendMatrices            uint32
	MaxVertexBlendMatrixIndex         uint32
	MaxPointSize                      float32
	MaxPrimitiveCount                 uint32
	MaxVertexIndex                    uint32
	MaxStreams                        uint32
	MaxStreamStride                   uint32
	VertexShaderVersion               uint32
	MaxVertexShaderConst              uint32
	PixelShaderVersion                uint32
	PixelShader1xMaxValue             float32
	DevCaps2                          uint32
	MasterAdapterOrdinal              uint32
	AdapterOrdinalInGroup             uint32
	NumberOfAdaptersInGroup           uint32
	DeclTypes                         uint32
	NumSimultaneousRTs                uint32
	StretchRectFilterCaps             uint32
	VS20Caps                          VSHADERCAPS2_0
	PS20Caps                          PSHADERCAPS2_0
	VertexTextureFilterCaps           uint32
	MaxVShaderInstructionsExecuted    uint32
	MaxPShaderInstructionsExecuted    uint32
	MaxVertexShader30InstructionSlots uint32
	MaxPixelShader30InstructionSlots  uint32
}

// VSHADERCAPS2_0 contains vertex shader capabilities.
type VSHADERCAPS2_0 struct {
	Caps                    uint32
	DynamicFlowControlDepth int32
	NumTemps                int32
	StaticFlowControlDepth  int32
}

// PSHADERCAPS2_0 describes pixel shader driver caps.
type PSHADERCAPS2_0 struct {
	Caps                    uint32
	DynamicFlowControlDepth int32
	NumTemps                int32
	StaticFlowControlDepth  int32
	NumInstructionSlots     int32
}

// PRESENT_PARAMETERS describes the presentation parameters.
type PRESENT_PARAMETERS struct {
	BackBufferWidth            uint32
	BackBufferHeight           uint32
	BackBufferFormat           FORMAT
	BackBufferCount            uint32
	MultiSampleType            MULTISAMPLE_TYPE
	MultiSampleQuality         uint32
	SwapEffect                 SWAPEFFECT
	HDeviceWindow              HWND
	Windowed                   int32
	EnableAutoDepthStencil     int32
	AutoDepthStencilFormat     FORMAT
	Flags                      uint32
	FullScreen_RefreshRateInHz uint32
	PresentationInterval       uint32
}

type (
	HANDLE   uintptr
	HWND     HANDLE
	HMONITOR HANDLE
	HDC      HANDLE
)

// RECT describes a rectangle.
type RECT struct {
	Left   int32
	Top    int32
	Right  int32
	Bottom int32
}

// RGNDATA contains region data.
type RGNDATA struct {
	Rdh    RGNDATAHEADER
	Buffer [1]byte
}

// RGNDATAHEADER describes region data.
type RGNDATAHEADER struct {
	DwSize   uint32
	IType    uint32
	NCount   uint32
	NRgnSize uint32
	RcBound  RECT
}

// DEVICE_CREATION_PARAMETERS describes the creation parameters for a device.
type DEVICE_CREATION_PARAMETERS struct {
	AdapterOrdinal uint32
	DeviceType     DEVTYPE
	HFocusWindow   HWND
	BehaviorFlags  uint32
}

// RASTER_STATUS describes the raster status.
type RASTER_STATUS struct {
	InVBlank int32
	ScanLine uint32
}

// GAMMARAMP contains red, green, and blue ramp data.
type GAMMARAMP struct {
	Red   [256]uint16
	Green [256]uint16
	Blue  [256]uint16
}

// POINT describes a 2D point.
type POINT struct {
	X int32
	Y int32
}

// MATRIX describes a matrix.
type MATRIX [16]float32

// VIEWPORT defines the window dimensions of a render-target surface onto which
// a 3D volume projects.
type VIEWPORT struct {
	X      uint32
	Y      uint32
	Width  uint32
	Height uint32
	MinZ   float32
	MaxZ   float32
}

// MATERIAL specifies material properties.
type MATERIAL struct {
	Diffuse  COLORVALUE
	Ambient  COLORVALUE
	Specular COLORVALUE
	Emissive COLORVALUE
	Power    float32
}

// COLORVALUE describes color values.
type COLORVALUE struct {
	R float32
	G float32
	B float32
	A float32
}

// LIGHT defines a set of lighting properties.
type LIGHT struct {
	Type         LIGHTTYPE
	Diffuse      COLORVALUE
	Specular     COLORVALUE
	Ambient      COLORVALUE
	Position     VECTOR
	Direction    VECTOR
	Range        float32
	Falloff      float32
	Attenuation0 float32
	Attenuation1 float32
	Attenuation2 float32
	Theta        float32
	Phi          float32
}

// VECTOR defines a vector.
type VECTOR struct {
	X float32
	Y float32
	Z float32
}

// CLIPSTATUS describes the current clip status.
type CLIPSTATUS struct {
	ClipUnion        uint32
	ClipIntersection uint32
}

// PALETTEENTRY specifies the color and usage of an entry in a logical palette.
type PALETTEENTRY struct {
	PeRed   byte
	PeGreen byte
	PeBlue  byte
	PeFlags byte
}

// VERTEXELEMENT defines the vertex data layout. Each vertex can contain one or
// more data types, and each data type is described by a vertex element.
type VERTEXELEMENT struct {
	Stream     uint16
	Offset     uint16
	Type       DECLTYPE
	Method     DECLMETHOD
	Usage      DECLUSAGE
	UsageIndex byte
}

// RECTPATCH_INFO describes a rectangular high-order patch.
type RECTPATCH_INFO struct {
	StartVertexOffsetWidth  uint32
	StartVertexOffsetHeight uint32
	Width                   uint32
	Height                  uint32
	Stride                  uint32
	Basis                   BASISTYPE
	Degree                  DEGREETYPE
}

// TRIPATCH_INFO describes a triangular high-order patch.
type TRIPATCH_INFO struct {
	StartVertexOffset uint32
	NumVertices       uint32
	Basis             BASISTYPE
	Degree            DEGREETYPE
}

// BOX defines a volume.
type BOX struct {
	Left   uint32
	Top    uint32
	Right  uint32
	Bottom uint32
	Front  uint32
	Back   uint32
}

// VOLUME_DESC describes a volume.
type VOLUME_DESC struct {
	Format FORMAT
	Type   RESOURCETYPE
	Usage  uint32
	Pool   POOL
	Width  uint32
	Height uint32
	Depth  uint32
}

// LOCKED_BOX describes a locked box (volume).
type LOCKED_BOX struct {
	RowPitch   int32
	SlicePitch int32
	PBits      uintptr
}

// VERTEXBUFFER_DESC describes a vertex buffer.
type VERTEXBUFFER_DESC struct {
	Format FORMAT
	Type   RESOURCETYPE
	Usage  uint32
	Pool   POOL
	Size   uint32
	FVF    uint32
}

// INDEXBUFFER_DESC describes an index buffer.
type INDEXBUFFER_DESC struct {
	Format FORMAT
	Type   RESOURCETYPE
	Usage  uint32
	Pool   POOL
	Size   uint32
}

// SURFACE_DESC describes a surface.
type SURFACE_DESC struct {
	Format             FORMAT
	Type               RESOURCETYPE
	Usage              uint32
	Pool               POOL
	MultiSampleType    MULTISAMPLE_TYPE
	MultiSampleQuality uint32
	Width              uint32
	Height             uint32
}

// LOCKED_RECT describes a locked rectangular region.
type LOCKED_RECT struct {
	Pitch int32
	PBits uintptr
}

// SetAllBytes will fill the whole rect with the given data, taking into account
// the rect's pitch. The given byte slice is expected to have the given stride
// in bytes, i.e. one line in the given data is <srcStride> bytes in length.
func (r LOCKED_RECT) SetAllBytes(data []byte, srcStride int) {
	dest := r.PBits
	destStride := int(r.Pitch)
	src := uintptr(unsafe.Pointer(&data[0]))
	height := len(data) / srcStride

	stride := srcStride
	if destStride < srcStride {
		stride = destStride
	}
	destSkip := uintptr(destStride - stride)
	srcSkip := uintptr(srcStride - stride)
	d := dest
	s := src
	if stride%8 == 0 {
		// in this case we can speed up copying by using 8 byte wide uint64s
		// instead of copying byte for byte
		for y := 0; y < height; y++ {
			for x := 0; x < stride; x += 8 {
				*((*uint64)(unsafe.Pointer(d))) = *((*uint64)(unsafe.Pointer(s)))
				d += 8
				s += 8
			}
			d += destSkip
			s += srcSkip
		}
	} else if stride%4 == 0 {
		// in this case we can speed up copying by using 4 byte wide uint32s
		// instead of copying byte for byte
		for y := 0; y < height; y++ {
			for x := 0; x < stride; x += 4 {
				*((*uint32)(unsafe.Pointer(d))) = *((*uint32)(unsafe.Pointer(s)))
				d += 4
				s += 4
			}
			d += destSkip
			s += srcSkip
		}
	} else {
		// in the unlikely case that stride is neither a multiple of 8 nor 4
		// bytes, just copy byte for byte
		for y := 0; y < height; y++ {
			for x := 0; x < stride; x++ {
				*((*byte)(unsafe.Pointer(d))) = *((*byte)(unsafe.Pointer(s)))
				d++
				s++
			}
			d += destSkip
			s += srcSkip
		}
	}
}

// DEVINFO_D3D9BANDWIDTHTIMINGS contains throughput metrics for help in
// understanding the performance of an application.
type DEVINFO_D3D9BANDWIDTHTIMINGS struct {
	MaxBandwidthUtilized                float32
	FrontEndUploadMemoryUtilizedPercent float32
	VertexRateUtilizedPercent           float32
	TriangleSetupRateUtilizedPercent    float32
	FillRateUtilizedPercent             float32
}

// DEVINFO_D3D9CACHEUTILIZATION measures the cache hit rate performance for
// textures and indexed vertices.
type DEVINFO_D3D9CACHEUTILIZATION struct {
	TextureCacheHitRate             float32
	PostTransformVertexCacheHitRate float32
}

// DEVINFO_D3D9INTERFACETIMINGS contains the percent of time processing data in
// the driver. These statistics may help identify cases when the driver is
// waiting for other resources.
type DEVINFO_D3D9INTERFACETIMINGS struct {
	WaitingForGPUToUseApplicationResourceTimePercent float32
	WaitingForGPUToAcceptMoreCommandsTimePercent     float32
	WaitingForGPUToStayWithinLatencyTimePercent      float32
	WaitingForGPUExclusiveResourceTimePercent        float32
	WaitingForGPUOtherTimePercent                    float32
}

// DEVINFO_D3D9PIPELINETIMINGS contains the percent of time processing data in
// the pipeline.
type DEVINFO_D3D9PIPELINETIMINGS struct {
	VertexProcessingTimePercent   float32
	PixelProcessingTimePercent    float32
	OtherGPUProcessingTimePercent float32
	GPUIdleTimePercent            float32
}

// DEVINFO_D3D9STAGETIMINGS contains the percent of time processing shader data.
type DEVINFO_D3D9STAGETIMINGS struct {
	MemoryProcessingPercent      float32
	ComputationProcessingPercent float32
}

// DEVINFO_D3DVERTEXSTATS reports the number of triangles that have been
// processed and clipped by the runtime's software vertex processing.
type DEVINFO_D3DVERTEXSTATS struct {
	NumRenderedTriangles      uint32
	NumExtraClippingTriangles uint32
}

// DEVINFO_VCACHE contains vertex cache optimization hints.
type DEVINFO_VCACHE struct {
	Pattern     uint32
	OptMethod   uint32
	CacheSize   uint32
	MagicNumber uint32
}

// RESOURCESTATS contains resource statistics gathered by the
// DEVINFO_ResourceManager when using the asynchronous query mechanism.
type RESOURCESTATS struct {
	BThrashing            uint32
	ApproxBytesDownloaded uint32
	NumEvicts             uint32
	NumVidCreates         uint32
	LastPri               uint32
	NumUsed               uint32
	NumUsedInVidMem       uint32
	WorkingSet            uint32
	WorkingSetBytes       uint32
	TotalManaged          uint32
	TotalBytes            uint32
}

// LUID is a 64-bit value guaranteed to be unique only on the system on which it
// was generated. The uniqueness of a locally unique identifier (LUID) is
// guaranteed only until the system is restarted.
type LUID struct {
	LowPart  uint32
	HighPart int32
}

// RANGE defines a range.
type RANGE struct {
	Offset uint32
	Size   uint32
}
<|endoftext|>


<commit_before>// +build appengine

/*

A Google App Engine Memcache session store implementation.

The implementation stores sessions in the Memcache and also saves sessions to the Datastore as a backup
in case data would be removed from the Memcache. This behaviour is optional, Datastore can be disabled completely.
You can also choose whether saving to Datastore happens synchronously (in the same goroutine)
or asynchronously (in another goroutine).

Limitations based on GAE Memcache:

- Since session ids are used in the Memcache keys, session ids can't be longer than 250 chars (bytes, but with Base64 charset it's the same).
If you also specify a key prefix (in MemcacheStoreOptions), that also counts into it.

- The size of a Session cannot be larger than 1 MB (marshalled into a byte slice).

Note that the Store will automatically "flush" sessions accessed from it when the Store is closed,
so it is very important to close the Store at the end of your request; this is usually done by closing
the session manager to which you passed the store (preferably with the defer statement).

Check out the GAE session demo application which shows how to use it properly:

https://github.com/icza/session/blob/master/gae_session_demo/session_demo.go

*/

package session

import (
	"appengine"
	"appengine/datastore"
	"appengine/memcache"
	"net/http"
	"sync"
	"time"
)

// A Google App Engine Memcache session store implementation.
type memcacheStore struct {
	ctx appengine.Context // Appengine context used when accessing the Memcache

	keyPrefix string // Prefix to use in front of session ids to construct Memcache key
	retries   int    // Number of retries to perform in case of general Memcache failures

	codec memcache.Codec // Codec used to marshal and unmarshal a Session to a byte slice

	onlyMemcache      bool   // Tells if sessions are not to be saved in Datastore
	syncDatastoreSave bool   // Tells if saving to Datastore should happen synchronously, in the same goroutine
	dsEntityName      string // Name of the datastore entity to use to save sessions

	// Map of sessions (mapped from ID) that were accessed using this store; usually it will only be 1.
	// It is also used as a cache, should the user call Get() with the same id multiple times.
	sessions map[string]Session

	mux *sync.RWMutex // mutex to synchronize access to sessions
}

// MemcacheStoreOptions defines options that may be passed when creating a new Memcache session store.
// All fields are optional; default value will be used for any field that has the zero value.
type MemcacheStoreOptions struct {
	// Prefix to use when storing sessions in the Memcache, cannot contain a null byte
	// and cannot be longer than 250 chars (bytes) when concatenated with the session id; default value is the empty string
	// The Memcache key will be this prefix and the session id concatenated.
	KeyPrefix string

	// Number of retries to perform if Memcache operations fail due to general service error;
	// default value is 3
	Retries int

	// Codec used to marshal and unmarshal a Session to a byte slice;
	// Default value is &memcache.Gob (which uses the gob package).
	Codec *memcache.Codec

	// Tells if sessions are only to be stored in Memcache, and do not store them in Datastore as backup;
	// as Memcache has no guarantees, it may lose content from time to time, but if Datastore is
	// also used, the session will automatically be retrieved from the Datastore if not found in Memcache;
	// default value is false (which means to also save sessions in the Datastore)
	OnlyMemcache bool

	// Tells if saving to Datastore should happen synchronously (in the same goroutine, before returning),
	// if false, session saving to Datastore will happen in the background (in another goroutine)
	// which gives smaller latency (and is enough most of the times as Memcache is always checked first);
	// default value is false which means to save sessions to Datastore in the background and return immedately
	// Not used if OnlyMemcache=true.
	SyncDatastoreSave bool

	// Name of the entity to use for saving sessions;
	// default value is "sess_"
	// Not used if OnlyMemcache=true.
	DSEntityName string
}

// SessEntity models the session entity saved to Datastore.
// The Key is the session id.
type SessEntity struct {
	Expires time.Time `datastore:"exp"`
	Value   []byte    `datastore:"val"`
}

// Pointer to zero value of MemcacheStoreOptions to be reused for efficiency.
var zeroMemcacheStoreOptions = new(MemcacheStoreOptions)

// NewMemcacheStore returns a new, GAE Memcache session Store with default options.
// Default values of options are listed in the MemcacheStoreOptions type.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStore(ctx appengine.Context) Store {
	return NewMemcacheStoreOptions(ctx, zeroMemcacheStoreOptions)
}

const defaultDSEntityName = "sess_" // Default value of DSEntityName.

// NewMemcacheStoreOptions returns a new, GAE Memcache session Store with the specified options.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStoreOptions(ctx appengine.Context, o *MemcacheStoreOptions) Store {
	s := &memcacheStore{
		ctx:               ctx,
		keyPrefix:         o.KeyPrefix,
		retries:           o.Retries,
		onlyMemcache:      o.OnlyMemcache,
		syncDatastoreSave: o.SyncDatastoreSave,
		dsEntityName:      o.DSEntityName,
		sessions:          make(map[string]Session, 2),
		mux:               &sync.RWMutex{},
	}
	if s.retries <= 0 {
		s.retries = 3
	}
	if o.Codec != nil {
		s.codec = *o.Codec
	} else {
		s.codec = memcache.Gob
	}
	if s.dsEntityName == "" {
		s.dsEntityName = defaultDSEntityName
	}
	return s
}

// Get is to implement Store.Get().
// Important! Since sessions are marshalled and stored in the Memcache,
// the mutex of the Session (Session.RWMutex()) will be different for each
// Session value (even though they might have the same session id)!
func (s *memcacheStore) Get(id string) Session {
	s.mux.RLock()
	defer s.mux.RUnlock()

	// First check our "cache"
	if sess := s.sessions[id]; sess != nil {
		return sess
	}

	// Next check in Memcache
	var err error
	var sess *sessionImpl

	for i := 0; i < s.retries; i++ {
		var sess_ sessionImpl
		_, err = s.codec.Get(s.ctx, s.keyPrefix+id, &sess_)
		if err == memcache.ErrCacheMiss {
			break // It's not in the Memcache (e.g. invalid sess id or was removed from Memcache by AppEngine)
		}
		if err == nil {
			sess = &sess_
			break
		}
		// Service error? Retry..
	}

	if sess == nil {
		if err != nil && err != memcache.ErrCacheMiss {
			s.ctx.Errorf("Failed to get session from memcache, id: %s, error: %v", id, err)
		}

		// Ok, we didn't get it from Memcace (either was not there or Memcache service is unavailable).
		// Now it's time to check in the Datastore.
		key := datastore.NewKey(s.ctx, s.dsEntityName, id, 0, nil)
		for i := 0; i < s.retries; i++ {
			e := SessEntity{}
			err = datastore.Get(s.ctx, key, &e)
			if err == datastore.ErrNoSuchEntity {
				return nil // It's not in the Datastore either
			}
			if err != nil {
				// Service error? Retry..
				continue
			}
			if e.Expires.Before(time.Now()) {
				// Session expired.
				datastore.Delete(s.ctx, key) // Omitting error check...
				return nil
			}
			var sess_ sessionImpl
			if err = s.codec.Unmarshal(e.Value, &sess_); err != nil {
				break // Invalid data in stored session entity...
			}
			sess = &sess_
			break
		}
	}

	if sess == nil {
		s.ctx.Errorf("Failed to get session from datastore, id: %s, error: %v", id, err)
		return nil
	}

	// Yes! We have it! "Actualize" it.
	sess.Access()
	// Mutex is not marshalled, so create a new one:
	sess.mux = &sync.RWMutex{}
	s.sessions[id] = sess
	return sess
}

// Add is to implement Store.Add().
func (s *memcacheStore) Add(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	if s.setMemcacheSession(sess) {
		s.ctx.Infof("Session added: %s", sess.Id())
		s.sessions[sess.Id()] = sess
		return
	}
}

// setMemcacheSession sets the specified session in the Memcache.
func (s *memcacheStore) setMemcacheSession(sess Session) (success bool) {
	item := &memcache.Item{
		Key:        s.keyPrefix + sess.Id(),
		Object:     sess,
		Expiration: sess.Timeout(),
	}

	var err error
	for i := 0; i < s.retries; i++ {
		if err = s.codec.Set(s.ctx, item); err == nil {
			return true
		}
	}

	s.ctx.Errorf("Failed to add session to memcache, id: %s, error: %v", sess.Id(), err)
	return false
}

// Remove is to implement Store.Remove().
func (s *memcacheStore) Remove(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	var err error
	for i := 0; i < s.retries; i++ {
		if err = memcache.Delete(s.ctx, s.keyPrefix+sess.Id()); err == nil || err == memcache.ErrCacheMiss {
			s.ctx.Infof("Session removed: %s", sess.Id())
			delete(s.sessions, sess.Id())
			if !s.onlyMemcache {
				// Also from the Datastore:
				key := datastore.NewKey(s.ctx, s.dsEntityName, sess.Id(), 0, nil)
				datastore.Delete(s.ctx, key) // Omitting error check...
			}
			return
		}
	}
	s.ctx.Errorf("Failed to remove session from memcache, id: %s, error: %v", sess.Id(), err)
}

// Close is to implement Store.Close().
func (s *memcacheStore) Close() {
	// Flush out sessions that were accessed from this store. No need locking, we're closing...
	// We could use Cocec.SetMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		s.setMemcacheSession(sess)
	}

	if s.onlyMemcache {
		return // Don't save to Datastore
	}

	if s.syncDatastoreSave {
		s.saveToDatastore()
	} else {
		go s.saveToDatastore()
	}
}

// saveToDatastore saves the sessions of the Store to the Datastore
// in the caller's goroutine.
func (s *memcacheStore) saveToDatastore() {
	// Save sessions that were accessed from this store. No need locking, we're closing...
	// We could use datastore.PutMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		value, err := s.codec.Marshal(sess)
		if err != nil {
			s.ctx.Errorf("Failed to marshal session: %s, error: %v", sess.Id(), err)
			continue
		}
		e := SessEntity{
			Expires: sess.Accessed().Add(sess.Timeout()),
			Value:   value,
		}
		key := datastore.NewKey(s.ctx, s.dsEntityName, sess.Id(), 0, nil)
		for i := 0; i < s.retries; i++ {
			if _, err = datastore.Put(s.ctx, key, &e); err == nil {
				break
			}
		}
		if err != nil {
			s.ctx.Errorf("Failed to save session to datastore: %s, error: %v", sess.Id(), err)
		}
	}
}

// PurgeExpiredSessFromDSFunc returns a request handler function which deletes expired sessions
// from the Datastore.
// dsEntityName is the name of the entity used for saving sessions; pass an empty string
// to use the default value (which is "sess_").
//
// It is recommended to register the returned handler function to a path which then can be defined
// as a cron job to be called periodically, e.g. in every 30 minutes or so (your choice).
// As cron handlers may run up to 10 minutes, the returned handler will stop at 8 minutes
// to complete safely even if there are more expired, undeleted sessions.
//
// The response of the handler func is a JSON text telling if the handler was able to delete all expired sessions,
// or that it was finished early due to the time. Examle of a respone where all expired sessions were deleted:
//
//     {"completed":true}
func PurgeExpiredSessFromDSFunc(dsEntityName string) http.HandlerFunc {
	if dsEntityName == "" {
		dsEntityName = defaultDSEntityName
	}

	return func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)
		// Delete in batches of 100
		q := datastore.NewQuery(dsEntityName).Filter("exp<", time.Now()).KeysOnly().Limit(100)

		deadline := time.Now().Add(time.Minute * 8)

		for {
			var err error
			var keys []*datastore.Key

			if keys, err = q.GetAll(c, nil); err != nil {
				// Datastore error.
				c.Errorf("Failed to query expired sessions: %v", err)
				http.Error(w, "Failed to query expired sessions!", http.StatusInternalServerError)
			}
			if len(keys) == 0 {
				// We're done, no more expired sessions
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":true}`))
				return
			}

			if err = datastore.DeleteMulti(c, keys); err != nil {
				c.Errorf("Error while deleting expired sessions: %v", err)
			}

			if time.Now().After(deadline) {
				// Our time is up, return
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":false}`))
				return
			}
			// We have time to continue
		}
	}
}
<commit_msg>Changed default op to save sessions in Datastore synchronously. See issue #3<commit_after>// +build appengine

/*

A Google App Engine Memcache session store implementation.

The implementation stores sessions in the Memcache and also saves sessions to the Datastore as a backup
in case data would be removed from the Memcache. This behaviour is optional, Datastore can be disabled completely.
You can also choose whether saving to Datastore happens synchronously (in the same goroutine)
or asynchronously (in another goroutine).

Limitations based on GAE Memcache:

- Since session ids are used in the Memcache keys, session ids can't be longer than 250 chars (bytes, but with Base64 charset it's the same).
If you also specify a key prefix (in MemcacheStoreOptions), that also counts into it.

- The size of a Session cannot be larger than 1 MB (marshalled into a byte slice).

Note that the Store will automatically "flush" sessions accessed from it when the Store is closed,
so it is very important to close the Store at the end of your request; this is usually done by closing
the session manager to which you passed the store (preferably with the defer statement).

Check out the GAE session demo application which shows how to use it properly:

https://github.com/icza/session/blob/master/gae_session_demo/session_demo.go

*/

package session

import (
	"net/http"
	"sync"
	"time"

	"appengine"
	"appengine/datastore"
	"appengine/memcache"
)

// A Google App Engine Memcache session store implementation.
type memcacheStore struct {
	ctx appengine.Context // Appengine context used when accessing the Memcache

	keyPrefix string // Prefix to use in front of session ids to construct Memcache key
	retries   int    // Number of retries to perform in case of general Memcache failures

	codec memcache.Codec // Codec used to marshal and unmarshal a Session to a byte slice

	onlyMemcache       bool   // Tells if sessions are not to be saved in Datastore
	asyncDatastoreSave bool   // Tells if saving in Datastore should happen asynchronously, in a new goroutine
	dsEntityName       string // Name of the datastore entity to use to save sessions

	// Map of sessions (mapped from ID) that were accessed using this store; usually it will only be 1.
	// It is also used as a cache, should the user call Get() with the same id multiple times.
	sessions map[string]Session

	mux *sync.RWMutex // mutex to synchronize access to sessions
}

// MemcacheStoreOptions defines options that may be passed when creating a new Memcache session store.
// All fields are optional; default value will be used for any field that has the zero value.
type MemcacheStoreOptions struct {
	// Prefix to use when storing sessions in the Memcache, cannot contain a null byte
	// and cannot be longer than 250 chars (bytes) when concatenated with the session id; default value is the empty string
	// The Memcache key will be this prefix and the session id concatenated.
	KeyPrefix string

	// Number of retries to perform if Memcache operations fail due to general service error;
	// default value is 3
	Retries int

	// Codec used to marshal and unmarshal a Session to a byte slice;
	// Default value is &memcache.Gob (which uses the gob package).
	Codec *memcache.Codec

	// Tells if sessions are only to be stored in Memcache, and do not store them in Datastore as backup;
	// as Memcache has no guarantees, it may lose content from time to time, but if Datastore is
	// also used, the session will automatically be retrieved from the Datastore if not found in Memcache;
	// default value is false (which means to also save sessions in the Datastore)
	OnlyMemcache bool

	// Tells if saving in Datastore should happen asynchronously (in a new goroutine, possibly after returning),
	// if false, session saving in Datastore will happen in the same goroutine, before returning from the request.
	// Asynchronous saving gives smaller latency (and is enough most of the time as Memcache is always checked first);
	// default value is false which means to save sessions in the Datastore in the same goroutine, synchronously
	// Not used if OnlyMemcache=true.
	// FIXME: See https://github.com/icza/session/issues/3
	AsyncDatastoreSave bool

	// Name of the entity to use for saving sessions;
	// default value is "sess_"
	// Not used if OnlyMemcache=true.
	DSEntityName string
}

// SessEntity models the session entity saved to Datastore.
// The Key is the session id.
type SessEntity struct {
	Expires time.Time `datastore:"exp"`
	Value   []byte    `datastore:"val"`
}

// Pointer to zero value of MemcacheStoreOptions to be reused for efficiency.
var zeroMemcacheStoreOptions = new(MemcacheStoreOptions)

// NewMemcacheStore returns a new, GAE Memcache session Store with default options.
// Default values of options are listed in the MemcacheStoreOptions type.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStore(ctx appengine.Context) Store {
	return NewMemcacheStoreOptions(ctx, zeroMemcacheStoreOptions)
}

const defaultDSEntityName = "sess_" // Default value of DSEntityName.

// NewMemcacheStoreOptions returns a new, GAE Memcache session Store with the specified options.
//
// Important! Since accessing the Memcache relies on Appengine Context
// which is bound to an http.Request, the returned Store can only be used for the lifetime of a request!
func NewMemcacheStoreOptions(ctx appengine.Context, o *MemcacheStoreOptions) Store {
	s := &memcacheStore{
		ctx:                ctx,
		keyPrefix:          o.KeyPrefix,
		retries:            o.Retries,
		onlyMemcache:       o.OnlyMemcache,
		asyncDatastoreSave: o.AsyncDatastoreSave,
		dsEntityName:       o.DSEntityName,
		sessions:           make(map[string]Session, 2),
		mux:                &sync.RWMutex{},
	}
	if s.retries <= 0 {
		s.retries = 3
	}
	if o.Codec != nil {
		s.codec = *o.Codec
	} else {
		s.codec = memcache.Gob
	}
	if s.dsEntityName == "" {
		s.dsEntityName = defaultDSEntityName
	}
	return s
}

// Get is to implement Store.Get().
// Important! Since sessions are marshalled and stored in the Memcache,
// the mutex of the Session (Session.RWMutex()) will be different for each
// Session value (even though they might have the same session id)!
func (s *memcacheStore) Get(id string) Session {
	s.mux.RLock()
	defer s.mux.RUnlock()

	// First check our "cache"
	if sess := s.sessions[id]; sess != nil {
		return sess
	}

	// Next check in Memcache
	var err error
	var sess *sessionImpl

	for i := 0; i < s.retries; i++ {
		var sess_ sessionImpl
		_, err = s.codec.Get(s.ctx, s.keyPrefix+id, &sess_)
		if err == memcache.ErrCacheMiss {
			break // It's not in the Memcache (e.g. invalid sess id or was removed from Memcache by AppEngine)
		}
		if err == nil {
			sess = &sess_
			break
		}
		// Service error? Retry..
	}

	if sess == nil {
		if err != nil && err != memcache.ErrCacheMiss {
			s.ctx.Errorf("Failed to get session from memcache, id: %s, error: %v", id, err)
		}

		// Ok, we didn't get it from Memcace (either was not there or Memcache service is unavailable).
		// Now it's time to check in the Datastore.
		key := datastore.NewKey(s.ctx, s.dsEntityName, id, 0, nil)
		for i := 0; i < s.retries; i++ {
			e := SessEntity{}
			err = datastore.Get(s.ctx, key, &e)
			if err == datastore.ErrNoSuchEntity {
				return nil // It's not in the Datastore either
			}
			if err != nil {
				// Service error? Retry..
				continue
			}
			if e.Expires.Before(time.Now()) {
				// Session expired.
				datastore.Delete(s.ctx, key) // Omitting error check...
				return nil
			}
			var sess_ sessionImpl
			if err = s.codec.Unmarshal(e.Value, &sess_); err != nil {
				break // Invalid data in stored session entity...
			}
			sess = &sess_
			break
		}
	}

	if sess == nil {
		s.ctx.Errorf("Failed to get session from datastore, id: %s, error: %v", id, err)
		return nil
	}

	// Yes! We have it! "Actualize" it.
	sess.Access()
	// Mutex is not marshalled, so create a new one:
	sess.mux = &sync.RWMutex{}
	s.sessions[id] = sess
	return sess
}

// Add is to implement Store.Add().
func (s *memcacheStore) Add(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	if s.setMemcacheSession(sess) {
		s.ctx.Infof("Session added: %s", sess.Id())
		s.sessions[sess.Id()] = sess
		return
	}
}

// setMemcacheSession sets the specified session in the Memcache.
func (s *memcacheStore) setMemcacheSession(sess Session) (success bool) {
	item := &memcache.Item{
		Key:        s.keyPrefix + sess.Id(),
		Object:     sess,
		Expiration: sess.Timeout(),
	}

	var err error
	for i := 0; i < s.retries; i++ {
		if err = s.codec.Set(s.ctx, item); err == nil {
			return true
		}
	}

	s.ctx.Errorf("Failed to add session to memcache, id: %s, error: %v", sess.Id(), err)
	return false
}

// Remove is to implement Store.Remove().
func (s *memcacheStore) Remove(sess Session) {
	s.mux.Lock()
	defer s.mux.Unlock()

	var err error
	for i := 0; i < s.retries; i++ {
		if err = memcache.Delete(s.ctx, s.keyPrefix+sess.Id()); err == nil || err == memcache.ErrCacheMiss {
			s.ctx.Infof("Session removed: %s", sess.Id())
			delete(s.sessions, sess.Id())
			if !s.onlyMemcache {
				// Also from the Datastore:
				key := datastore.NewKey(s.ctx, s.dsEntityName, sess.Id(), 0, nil)
				datastore.Delete(s.ctx, key) // Omitting error check...
			}
			return
		}
	}
	s.ctx.Errorf("Failed to remove session from memcache, id: %s, error: %v", sess.Id(), err)
}

// Close is to implement Store.Close().
func (s *memcacheStore) Close() {
	// Flush out sessions that were accessed from this store. No need locking, we're closing...
	// We could use Cocec.SetMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		s.setMemcacheSession(sess)
	}

	if s.onlyMemcache {
		return // Don't save to Datastore
	}

	if s.asyncDatastoreSave {
		go s.saveToDatastore()
	} else {
		s.saveToDatastore()
	}
}

// saveToDatastore saves the sessions of the Store to the Datastore
// in the caller's goroutine.
func (s *memcacheStore) saveToDatastore() {
	// Save sessions that were accessed from this store. No need locking, we're closing...
	// We could use datastore.PutMulti(), but sessions will contain at most 1 session like all the times.
	for _, sess := range s.sessions {
		value, err := s.codec.Marshal(sess)
		if err != nil {
			s.ctx.Errorf("Failed to marshal session: %s, error: %v", sess.Id(), err)
			continue
		}
		e := SessEntity{
			Expires: sess.Accessed().Add(sess.Timeout()),
			Value:   value,
		}
		key := datastore.NewKey(s.ctx, s.dsEntityName, sess.Id(), 0, nil)
		for i := 0; i < s.retries; i++ {
			if _, err = datastore.Put(s.ctx, key, &e); err == nil {
				break
			}
		}
		if err != nil {
			s.ctx.Errorf("Failed to save session to datastore: %s, error: %v", sess.Id(), err)
		}
	}
}

// PurgeExpiredSessFromDSFunc returns a request handler function which deletes expired sessions
// from the Datastore.
// dsEntityName is the name of the entity used for saving sessions; pass an empty string
// to use the default value (which is "sess_").
//
// It is recommended to register the returned handler function to a path which then can be defined
// as a cron job to be called periodically, e.g. in every 30 minutes or so (your choice).
// As cron handlers may run up to 10 minutes, the returned handler will stop at 8 minutes
// to complete safely even if there are more expired, undeleted sessions.
//
// The response of the handler func is a JSON text telling if the handler was able to delete all expired sessions,
// or that it was finished early due to the time. Examle of a respone where all expired sessions were deleted:
//
//     {"completed":true}
func PurgeExpiredSessFromDSFunc(dsEntityName string) http.HandlerFunc {
	if dsEntityName == "" {
		dsEntityName = defaultDSEntityName
	}

	return func(w http.ResponseWriter, r *http.Request) {
		c := appengine.NewContext(r)
		// Delete in batches of 100
		q := datastore.NewQuery(dsEntityName).Filter("exp<", time.Now()).KeysOnly().Limit(100)

		deadline := time.Now().Add(time.Minute * 8)

		for {
			var err error
			var keys []*datastore.Key

			if keys, err = q.GetAll(c, nil); err != nil {
				// Datastore error.
				c.Errorf("Failed to query expired sessions: %v", err)
				http.Error(w, "Failed to query expired sessions!", http.StatusInternalServerError)
			}
			if len(keys) == 0 {
				// We're done, no more expired sessions
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":true}`))
				return
			}

			if err = datastore.DeleteMulti(c, keys); err != nil {
				c.Errorf("Error while deleting expired sessions: %v", err)
			}

			if time.Now().After(deadline) {
				// Our time is up, return
				w.Header().Set("Content-Type", "application/json")
				w.Write([]byte(`{"completed":false}`))
				return
			}
			// We have time to continue
		}
	}
}
<|endoftext|>