<commit_before>// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"os"

	"github.com/golang/glog"
	"github.com/google/go-licenses/licenses"
	"github.com/spf13/cobra"
)

var (
	csvCmd = &cobra.Command{
		Use:   "csv <package>",
		Short: "Prints all licenses that apply to a Go package and its dependencies",
		Args:  cobra.MinimumNArgs(1),
		RunE:  csvMain,
	}

	gitRemotes []string
)

func init() {
	csvCmd.Flags().StringArrayVar(&gitRemotes, "git_remote", []string{"origin", "upstream"}, "Remote Git repositories to try")

	rootCmd.AddCommand(csvCmd)
}

func csvMain(_ *cobra.Command, args []string) error {
	classifier, err := licenses.NewClassifier(confidenceThreshold)
	if err != nil {
		return err
	}

	libs, err := licenses.Libraries(context.Background(), classifier, args...)
	if err != nil {
		return err
	}
	for _, lib := range libs {
		licenseName := "Unknown"
		licenseURL := "Unknown"
		if lib.LicensePath != "" {
			licenseName, _, err = classifier.Identify(lib.LicensePath)
			if err != nil {
				glog.Errorf("Error identifying license in %q: %v", lib.LicensePath, err)
				licenseName = "Unknown"
			}
			licenseURL, err = lib.FileURL(context.Background(), lib.LicensePath)
			if err != nil {
				glog.Warningf("Error discovering license URL: %s", err)
			}
			if licenseURL == "" {
				licenseURL = "Unknown"
			}
		}
		// Adding spaces after each "," makes vscode/terminal recognize the correct license URL.
		// Otherwise, if there's no space, vscode interprets the URL as concatenated with the license name.
		if _, err := os.Stdout.WriteString(fmt.Sprintf("%s, %s, %s\n", lib.Name(), licenseURL, licenseName)); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>chore: apply feedback from @wlynch<commit_after>// Copyright 2019 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"os"
	"strings"

	"github.com/golang/glog"
	"github.com/google/go-licenses/licenses"
	"github.com/spf13/cobra"
)

var (
	csvCmd = &cobra.Command{
		Use:   "csv <package>",
		Short: "Prints all licenses that apply to a Go package and its dependencies",
		Args:  cobra.MinimumNArgs(1),
		RunE:  csvMain,
	}

	gitRemotes []string
)

func init() {
	csvCmd.Flags().StringArrayVar(&gitRemotes, "git_remote", []string{"origin", "upstream"}, "Remote Git repositories to try")

	rootCmd.AddCommand(csvCmd)
}

func csvMain(_ *cobra.Command, args []string) error {
	classifier, err := licenses.NewClassifier(confidenceThreshold)
	if err != nil {
		return err
	}

	libs, err := licenses.Libraries(context.Background(), classifier, args...)
	if err != nil {
		return err
	}
	for _, lib := range libs {
		licenseName := "Unknown"
		licenseURL := "Unknown"
		if lib.LicensePath != "" {
			name, _, err := classifier.Identify(lib.LicensePath)
			if err == nil {
				licenseName = name
			} else {
				glog.Errorf("Error identifying license in %q: %v", lib.LicensePath, err)
			}
			url, err := lib.FileURL(context.Background(), lib.LicensePath)
			if err == nil {
				licenseURL = url
			} else {
				glog.Warningf("Error discovering license URL: %s", err)
			}
		}
		// Using ", " to join words makes vscode/terminal recognize the
		// correct license URL. Otherwise, if there's no space after
		// comma, vscode interprets the URL as concatenated with the
		// license name after it.
		// Also, the extra spaces does not affect csv syntax much, we
		// can still copy the csv text and paste into Excel / Google
		// Sheets.
		if _, err := fmt.Fprintln(os.Stdout, strings.Join([]string{lib.Name(), licenseURL, licenseName}, ", ")); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package release

import (
	"crypto"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"github.com/hashicorp/go-version"
	binaryUpdate "github.com/inconshreveable/go-update"

	"github.com/Shopify/themekit/src/colors"
)

var (
	builds = map[string]string{
		"darwin-amd64":  "theme",
		"darwin-386":    "theme",
		"linux-386":     "theme",
		"linux-amd64":   "theme",
		"freebsd-386":   "theme",
		"freebsd-amd64": "theme",
		"windows-386":   "theme.exe",
		"windows-amd64": "theme.exe",
	}
	// ThemeKitVersion is the version build of the library
	ThemeKitVersion, _ = version.NewVersion("0.8.1")
)

const (
	releasesS3URL = "https://shopify-themekit.s3.amazonaws.com/releases/all.json"
	latestS3URL   = "https://shopify-themekit.s3.amazonaws.com/releases/latest.json"
)

type release struct {
	Version   string     `json:"version"`
	Platforms []platform `json:"platforms"`
}

func (r release) isValid() bool {
	return len(r.Platforms) > 0
}

func (r release) isApplicable() bool {
	version, err := version.NewVersion(r.Version)
	if err != nil {
		return false
	}
	return ThemeKitVersion.LessThan(version) && version.Metadata() == "" && version.Prerelease() == ""
}

func (r release) getVersion() *version.Version {
	version, _ := version.NewVersion(r.Version)
	return version
}

func (r release) forCurrentPlatform() platform {
	platformKey := runtime.GOOS + "-" + runtime.GOARCH
	for _, p := range r.Platforms {
		if p.Name == platformKey {
			return p
		}
	}
	return platform{}
}

// IsUpdateAvailable will check if there is an update to the theme kit command
// and if there is one it will return true. Otherwise it will return false.
func IsUpdateAvailable() bool {
	return checkUpdateAvailable(latestS3URL)
}

// Install will take a semver string and parse it then check if that
// update is available and install it. If the string is 'latest' it will install
// the most current. If the string is latest and there is no update it will return an
// error. An error will also be returned if the requested version does not exist.
func Install(ver string) error {
	installer := func(p platform) error {
		return applyUpdate(p, "")
	}
	if ver == "latest" {
		return installLatest(latestS3URL, installer)
	}
	return installVersion(ver, releasesS3URL, installer)
}

// Update will update the details of a release or deploy a new release with the
// deploy feed
func Update(key, secret, ver string, force bool) error {
	return update(ver, releasesS3URL, filepath.Join("build", "dist"), force, newS3Uploader(key, secret))
}

// Remove will remove a themekit release from the deployed releases list. This will
// prevent any users from installing the version again. This can only be done will
// appropriate S3 priviledges
func Remove(key, secret, ver string) error {
	return remove(ver, releasesS3URL, newS3Uploader(key, secret))
}

func checkUpdateAvailable(latestURL string) bool {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return false
	}
	return release.isApplicable()
}

func installLatest(latestURL string, install func(platform) error) error {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return err
	}
	if !release.isApplicable() {
		return fmt.Errorf("no applicable update available")
	}
	return install(release.forCurrentPlatform())
}

func installVersion(ver, releasesURL string, install func(platform) error) error {
	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}
	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return fmt.Errorf("version %s not found", ver)
	}
	return install(requestedRelease.forCurrentPlatform())
}

func applyUpdate(platformRelease platform, targetPath string) error {
	checksum, err := hex.DecodeString(platformRelease.Digest)
	if err != nil {
		return err
	}

	updateFile, err := http.Get(platformRelease.URL)
	if err != nil {
		return err
	}
	defer updateFile.Body.Close()

	err = binaryUpdate.Apply(updateFile.Body, binaryUpdate.Options{
		TargetPath: targetPath,
		Hash:       crypto.MD5,
		Checksum:   checksum,
	})

	if err != nil {
		if rerr := binaryUpdate.RollbackError(err); rerr != nil {
			return fmt.Errorf("Failed to rollback from bad update: %v", rerr)
		}
		return fmt.Errorf("Could not update and had to roll back. %v", err)
	}

	return nil
}

func update(ver, releasesURL, distDir string, force bool, u uploader) error {
	if !force {
		requestedVersion, _ := version.NewVersion(ver)
		if !requestedVersion.Equal(ThemeKitVersion) {
			return errors.New("deploy version does not match themekit version")
		}
	}

	_, err := os.Stat(distDir)
	if os.IsNotExist(err) {
		return errors.New("Dist folder does not exist. Run 'make dist' before attempting to create a new release")
	}

	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}

	if requestedRelease := releases.get(ver); !force && requestedRelease.isValid() {
		return errors.New("version has already been deployed")
	}

	newRelease, err := buildRelease(ver, distDir, u)
	if err != nil {
		return err
	}

	return updateDeploy(releases.add(newRelease), u)
}

func remove(ver, releaseURL string, u uploader) error {
	releases, err := fetchReleases(releaseURL)
	if err != nil {
		return err
	}

	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return errors.New("version has not be deployed")
	}

	return updateDeploy(releases.del(ver), u)
}

func updateDeploy(releases releasesList, u uploader) error {
	colors.ColorStdOut.Printf("Updating releases")
	if err := u.JSON("releases/all.json", releases); err != nil {
		return err
	}
	return u.JSON("releases/latest.json", releases.get("latest"))
}

func fetchLatest(url string) (release, error) {
	var latest release
	resp, err := http.Get(url)
	if err != nil {
		return latest, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		err = json.Unmarshal(data, &latest)
	}
	return latest, err
}

func buildRelease(ver, distDir string, u uploader) (release, error) {
	colors.ColorStdOut.Printf("Building %s", colors.Green(ver))
	newRelease := release{Version: ver, Platforms: []platform{}}

	for platformName, binName := range builds {
		plat, err := buildPlatform(ver, platformName, distDir, binName, u)
		if err != nil {
			return newRelease, err
		}
		newRelease.Platforms = append(newRelease.Platforms, plat)
	}

	return newRelease, nil
}
<commit_msg>Update version<commit_after>package release

import (
	"crypto"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"github.com/hashicorp/go-version"
	binaryUpdate "github.com/inconshreveable/go-update"

	"github.com/Shopify/themekit/src/colors"
)

var (
	builds = map[string]string{
		"darwin-amd64":  "theme",
		"darwin-386":    "theme",
		"linux-386":     "theme",
		"linux-amd64":   "theme",
		"freebsd-386":   "theme",
		"freebsd-amd64": "theme",
		"windows-386":   "theme.exe",
		"windows-amd64": "theme.exe",
	}
	// ThemeKitVersion is the version build of the library
	ThemeKitVersion, _ = version.NewVersion("1.0.0")
)

const (
	releasesS3URL = "https://shopify-themekit.s3.amazonaws.com/releases/all.json"
	latestS3URL   = "https://shopify-themekit.s3.amazonaws.com/releases/latest.json"
)

type release struct {
	Version   string     `json:"version"`
	Platforms []platform `json:"platforms"`
}

func (r release) isValid() bool {
	return len(r.Platforms) > 0
}

func (r release) isApplicable() bool {
	version, err := version.NewVersion(r.Version)
	if err != nil {
		return false
	}
	return ThemeKitVersion.LessThan(version) && version.Metadata() == "" && version.Prerelease() == ""
}

func (r release) getVersion() *version.Version {
	version, _ := version.NewVersion(r.Version)
	return version
}

func (r release) forCurrentPlatform() platform {
	platformKey := runtime.GOOS + "-" + runtime.GOARCH
	for _, p := range r.Platforms {
		if p.Name == platformKey {
			return p
		}
	}
	return platform{}
}

// IsUpdateAvailable will check if there is an update to the theme kit command
// and if there is one it will return true. Otherwise it will return false.
func IsUpdateAvailable() bool {
	return checkUpdateAvailable(latestS3URL)
}

// Install will take a semver string and parse it then check if that
// update is available and install it. If the string is 'latest' it will install
// the most current. If the string is latest and there is no update it will return an
// error. An error will also be returned if the requested version does not exist.
func Install(ver string) error {
	installer := func(p platform) error {
		return applyUpdate(p, "")
	}
	if ver == "latest" {
		return installLatest(latestS3URL, installer)
	}
	return installVersion(ver, releasesS3URL, installer)
}

// Update will update the details of a release or deploy a new release with the
// deploy feed
func Update(key, secret, ver string, force bool) error {
	return update(ver, releasesS3URL, filepath.Join("build", "dist"), force, newS3Uploader(key, secret))
}

// Remove will remove a themekit release from the deployed releases list. This will
// prevent any users from installing the version again. This can only be done will
// appropriate S3 priviledges
func Remove(key, secret, ver string) error {
	return remove(ver, releasesS3URL, newS3Uploader(key, secret))
}

func checkUpdateAvailable(latestURL string) bool {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return false
	}
	return release.isApplicable()
}

func installLatest(latestURL string, install func(platform) error) error {
	release, err := fetchLatest(latestURL)
	if err != nil {
		return err
	}
	if !release.isApplicable() {
		return fmt.Errorf("no applicable update available")
	}
	return install(release.forCurrentPlatform())
}

func installVersion(ver, releasesURL string, install func(platform) error) error {
	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}
	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return fmt.Errorf("version %s not found", ver)
	}
	return install(requestedRelease.forCurrentPlatform())
}

func applyUpdate(platformRelease platform, targetPath string) error {
	checksum, err := hex.DecodeString(platformRelease.Digest)
	if err != nil {
		return err
	}

	updateFile, err := http.Get(platformRelease.URL)
	if err != nil {
		return err
	}
	defer updateFile.Body.Close()

	err = binaryUpdate.Apply(updateFile.Body, binaryUpdate.Options{
		TargetPath: targetPath,
		Hash:       crypto.MD5,
		Checksum:   checksum,
	})

	if err != nil {
		if rerr := binaryUpdate.RollbackError(err); rerr != nil {
			return fmt.Errorf("Failed to rollback from bad update: %v", rerr)
		}
		return fmt.Errorf("Could not update and had to roll back. %v", err)
	}

	return nil
}

func update(ver, releasesURL, distDir string, force bool, u uploader) error {
	if !force {
		requestedVersion, _ := version.NewVersion(ver)
		if !requestedVersion.Equal(ThemeKitVersion) {
			return errors.New("deploy version does not match themekit version")
		}
	}

	_, err := os.Stat(distDir)
	if os.IsNotExist(err) {
		return errors.New("Dist folder does not exist. Run 'make dist' before attempting to create a new release")
	}

	releases, err := fetchReleases(releasesURL)
	if err != nil {
		return err
	}

	if requestedRelease := releases.get(ver); !force && requestedRelease.isValid() {
		return errors.New("version has already been deployed")
	}

	newRelease, err := buildRelease(ver, distDir, u)
	if err != nil {
		return err
	}

	return updateDeploy(releases.add(newRelease), u)
}

func remove(ver, releaseURL string, u uploader) error {
	releases, err := fetchReleases(releaseURL)
	if err != nil {
		return err
	}

	requestedRelease := releases.get(ver)
	if !requestedRelease.isValid() {
		return errors.New("version has not be deployed")
	}

	return updateDeploy(releases.del(ver), u)
}

func updateDeploy(releases releasesList, u uploader) error {
	colors.ColorStdOut.Printf("Updating releases")
	if err := u.JSON("releases/all.json", releases); err != nil {
		return err
	}
	return u.JSON("releases/latest.json", releases.get("latest"))
}

func fetchLatest(url string) (release, error) {
	var latest release
	resp, err := http.Get(url)
	if err != nil {
		return latest, err
	}
	defer resp.Body.Close()
	data, err := ioutil.ReadAll(resp.Body)
	if err == nil {
		err = json.Unmarshal(data, &latest)
	}
	return latest, err
}

func buildRelease(ver, distDir string, u uploader) (release, error) {
	colors.ColorStdOut.Printf("Building %s", colors.Green(ver))
	newRelease := release{Version: ver, Platforms: []platform{}}

	for platformName, binName := range builds {
		plat, err := buildPlatform(ver, platformName, distDir, binName, u)
		if err != nil {
			return newRelease, err
		}
		newRelease.Platforms = append(newRelease.Platforms, plat)
	}

	return newRelease, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"image"

	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	"github.com/fogleman/gg"
	"github.com/pborman/getopt"
	"golang.org/x/image/font/inconsolata"

	"discord-auto-upload/web"
)

const currentVersion = "0.6"

var lastCheck = time.Now()
var newLastCheck = time.Now()

// Config for the application
type Config struct {
	webhookURL  string
	path        string
	watch       int
	username    string
	noWatermark bool
	exclude     string
}

func main() {

	config := parseOptions()
	checkPath(config.path)
	web.Init()

	checkUpdates()

	log.Print("Waiting for images to appear in ", config.path)
	// wander the path, forever
	for {
		err := filepath.Walk(config.path,
			func(path string, f os.FileInfo, err error) error { return checkFile(path, f, err, config) })
		if err != nil {
			log.Fatal("could not watch path", err)
		}
		lastCheck = newLastCheck
		time.Sleep(time.Duration(config.watch) * time.Second)
	}
}

func checkPath(path string) {
	src, err := os.Stat(path)
	if err != nil {
		log.Fatal(err)
	}
	if !src.IsDir() {
		log.Fatal(path, " is not a directory")
		os.Exit(1)
	}
}

func checkUpdates() {

	type GithubRelease struct {
		HTMLURL string
		TagName string
		Name    string
		Body    string
	}

	client := &http.Client{Timeout: time.Second * 5}
	resp, err := client.Get("https://api.github.com/repos/tardisx/discord-auto-upload/releases/latest")
	if err != nil {
		log.Fatal("could not check for updates:", err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("could not check read update response")
	}

	var latest GithubRelease
	err = json.Unmarshal(body, &latest)

	if err != nil {
		log.Fatal("could not parse JSON: ", err)
	}

	if currentVersion < latest.TagName {
		fmt.Printf("You are currently on version %s, but version %s is available\n", currentVersion, latest.TagName)
		fmt.Println("----------- Release Info -----------")
		fmt.Println(latest.Body)
		fmt.Println("------------------------------------")
		fmt.Println("Upgrade at https://github.com/tardisx/discord-auto-upload/releases/latest")
	}

}

func parseOptions() Config {

	var newConfig Config
	// Declare the flags to be used
	webhookFlag := getopt.StringLong("webhook", 'w', "", "discord webhook URL")
	pathFlag := getopt.StringLong("directory", 'd', "", "directory to scan, optional, defaults to current directory")
	watchFlag := getopt.Int16Long("watch", 's', 10, "time between scans")
	usernameFlag := getopt.StringLong("username", 'u', "", "username for the bot upload")
	excludeFlag := getopt.StringLong("exclude", 'x', "", "exclude files containing this string")
	noWatermarkFlag := getopt.BoolLong("no-watermark", 'n', "do not put a watermark on images before uploading")
	helpFlag := getopt.BoolLong("help", 'h', "help")
	versionFlag := getopt.BoolLong("version", 'v', "show version")
	getopt.SetParameters("")

	getopt.Parse()

	if *helpFlag {
		getopt.PrintUsage(os.Stderr)
		os.Exit(0)
	}

	if *versionFlag {
		fmt.Println("dau - https://github.com/tardisx/discord-auto-upload")
		fmt.Printf("Version: %s\n", currentVersion)
		os.Exit(0)
	}

	if !getopt.IsSet("directory") {
		*pathFlag = "./"
		log.Println("Defaulting to current directory")
	}

	if !getopt.IsSet("webhook") {
		log.Fatal("ERROR: You must specify a --webhook URL")
	}

	newConfig.path = *pathFlag
	newConfig.webhookURL = *webhookFlag
	newConfig.watch = int(*watchFlag)
	newConfig.username = *usernameFlag
	newConfig.noWatermark = *noWatermarkFlag
	newConfig.exclude = *excludeFlag

	return newConfig
}

func checkFile(path string, f os.FileInfo, err error, config Config) error {

	if f.ModTime().After(lastCheck) && f.Mode().IsRegular() {

		if fileEligible(config, path) {
			// process file
			processFile(config, path)
		}

		if newLastCheck.Before(f.ModTime()) {
			newLastCheck = f.ModTime()
		}
	}

	return nil
}

func fileEligible(config Config, file string) bool {

	if config.exclude != "" && strings.Contains(file, config.exclude) {
		return false
	}

	extension := strings.ToLower(filepath.Ext(file))
	if extension == ".png" || extension == ".jpg" || extension == ".gif" {
		return true
	}

	return false
}

func processFile(config Config, file string) {

	if !config.noWatermark {
		log.Print("Copying to temp location and watermarking ", file)
		file = mungeFile(file)
	}

	log.Print("Uploading ", file)

	extraParams := map[string]string{}

	if config.username != "" {
		extraParams["username"] = config.username
	}

	type DiscordAPIResponseAttachment struct {
		URL      string
		ProxyURL string
		Size     int
		Width    int
		Height   int
		Filename string
	}

	type DiscordAPIResponse struct {
		Attachments []DiscordAPIResponseAttachment
		ID          int64 `json:",string"`
	}

	var retriesRemaining = 5
	for retriesRemaining > 0 {
		request, err := newfileUploadRequest(config.webhookURL, extraParams, "file", file)
		if err != nil {
			log.Fatal(err)
		}
		start := time.Now()
		client := &http.Client{Timeout: time.Second * 30}
		resp, err := client.Do(request)
		if err != nil {
			log.Print("Error performing request:", err)
			retriesRemaining--
			sleepForRetries(retriesRemaining)
			continue
		} else {

			if resp.StatusCode != 200 {
				log.Print("Bad response from server:", resp.StatusCode)
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}

			resBody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				log.Print("could not deal with body: ", err)
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			resp.Body.Close()

			var res DiscordAPIResponse
			err = json.Unmarshal(resBody, &res)

			if err != nil {
				log.Print("could not parse JSON: ", err)
				fmt.Println("Response was:", string(resBody[:]))
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			if len(res.Attachments) < 1 {
				log.Print("bad response - no attachments?")
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			var a = res.Attachments[0]
			elapsed := time.Since(start)
			rate := float64(a.Size) / elapsed.Seconds() / 1024.0

			log.Printf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height)
			log.Printf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate)
			break
		}
	}

	if !config.noWatermark {
		log.Print("Removing temporary file ", file)
		os.Remove(file)
	}

	if retriesRemaining == 0 {
		log.Fatal("Failed to upload, even after retries")
	}
}

func sleepForRetries(retry int) {
	if retry == 0 {
		return
	}
	retryTime := (6-retry)*(6-retry) + 6
	log.Printf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry)
	// time.Sleep(time.Duration(retryTime) * time.Second)
}

func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		log.Fatal("Could not copy: ", err)
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, err
}

func mungeFile(path string) string {

	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	im, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}
	bounds := im.Bounds()
	// var S float64 = float64(bounds.Max.X)

	dc := gg.NewContext(bounds.Max.X, bounds.Max.Y)
	dc.Clear()
	dc.SetRGB(0, 0, 0)

	dc.SetFontFace(inconsolata.Regular8x16)

	dc.DrawImage(im, 0, 0)

	dc.DrawRoundedRectangle(0, float64(bounds.Max.Y-18.0), 320, float64(bounds.Max.Y), 0)
	dc.SetRGB(0, 0, 0)
	dc.Fill()

	dc.SetRGB(1, 1, 1)

	dc.DrawString("github.com/tardisx/discord-auto-upload", 5.0, float64(bounds.Max.Y)-5.0)

	tempfile, err := ioutil.TempFile("", "dau")
	if err != nil {
		log.Fatal(err)
	}
	tempfile.Close()
	os.Remove(tempfile.Name())
	actualName := tempfile.Name() + ".png"

	dc.SavePNG(actualName)
	return actualName
}
<commit_msg>Improve update check error and make it non-fatal<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"image"

	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"

	"github.com/fogleman/gg"
	"github.com/pborman/getopt"
	"golang.org/x/image/font/inconsolata"

	"discord-auto-upload/web"
)

const currentVersion = "0.7"

var lastCheck = time.Now()
var newLastCheck = time.Now()

// Config for the application
type Config struct {
	webhookURL  string
	path        string
	watch       int
	username    string
	noWatermark bool
	exclude     string
}

func main() {

	config := parseOptions()
	checkPath(config.path)
	web.Init()

	checkUpdates()

	log.Print("Waiting for images to appear in ", config.path)
	// wander the path, forever
	for {
		err := filepath.Walk(config.path,
			func(path string, f os.FileInfo, err error) error { return checkFile(path, f, err, config) })
		if err != nil {
			log.Fatal("could not watch path", err)
		}
		lastCheck = newLastCheck
		time.Sleep(time.Duration(config.watch) * time.Second)
	}
}

func checkPath(path string) {
	src, err := os.Stat(path)
	if err != nil {
		log.Fatal(err)
	}
	if !src.IsDir() {
		log.Fatal(path, " is not a directory")
		os.Exit(1)
	}
}

func checkUpdates() {

	type GithubRelease struct {
		HTMLURL string
		TagName string
		Name    string
		Body    string
	}

	client := &http.Client{Timeout: time.Second * 5}
	resp, err := client.Get("https://api.github.com/repos/tardisx/discord-auto-upload/releases/latest")
	if err != nil {
		log.Print("WARNING: Update check failed: ", err)
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal("could not check read update response")
	}

	var latest GithubRelease
	err = json.Unmarshal(body, &latest)

	if err != nil {
		log.Fatal("could not parse JSON: ", err)
	}

	if currentVersion < latest.TagName {
		fmt.Printf("You are currently on version %s, but version %s is available\n", currentVersion, latest.TagName)
		fmt.Println("----------- Release Info -----------")
		fmt.Println(latest.Body)
		fmt.Println("------------------------------------")
		fmt.Println("Upgrade at https://github.com/tardisx/discord-auto-upload/releases/latest")
	}

}

func parseOptions() Config {

	var newConfig Config
	// Declare the flags to be used
	webhookFlag := getopt.StringLong("webhook", 'w', "", "discord webhook URL")
	pathFlag := getopt.StringLong("directory", 'd', "", "directory to scan, optional, defaults to current directory")
	watchFlag := getopt.Int16Long("watch", 's', 10, "time between scans")
	usernameFlag := getopt.StringLong("username", 'u', "", "username for the bot upload")
	excludeFlag := getopt.StringLong("exclude", 'x', "", "exclude files containing this string")
	noWatermarkFlag := getopt.BoolLong("no-watermark", 'n', "do not put a watermark on images before uploading")
	helpFlag := getopt.BoolLong("help", 'h', "help")
	versionFlag := getopt.BoolLong("version", 'v', "show version")
	getopt.SetParameters("")

	getopt.Parse()

	if *helpFlag {
		getopt.PrintUsage(os.Stderr)
		os.Exit(0)
	}

	if *versionFlag {
		fmt.Println("dau - https://github.com/tardisx/discord-auto-upload")
		fmt.Printf("Version: %s\n", currentVersion)
		os.Exit(0)
	}

	if !getopt.IsSet("directory") {
		*pathFlag = "./"
		log.Println("Defaulting to current directory")
	}

	if !getopt.IsSet("webhook") {
		log.Fatal("ERROR: You must specify a --webhook URL")
	}

	newConfig.path = *pathFlag
	newConfig.webhookURL = *webhookFlag
	newConfig.watch = int(*watchFlag)
	newConfig.username = *usernameFlag
	newConfig.noWatermark = *noWatermarkFlag
	newConfig.exclude = *excludeFlag

	return newConfig
}

func checkFile(path string, f os.FileInfo, err error, config Config) error {

	if f.ModTime().After(lastCheck) && f.Mode().IsRegular() {

		if fileEligible(config, path) {
			// process file
			processFile(config, path)
		}

		if newLastCheck.Before(f.ModTime()) {
			newLastCheck = f.ModTime()
		}
	}

	return nil
}

func fileEligible(config Config, file string) bool {

	if config.exclude != "" && strings.Contains(file, config.exclude) {
		return false
	}

	extension := strings.ToLower(filepath.Ext(file))
	if extension == ".png" || extension == ".jpg" || extension == ".gif" {
		return true
	}

	return false
}

func processFile(config Config, file string) {

	if !config.noWatermark {
		log.Print("Copying to temp location and watermarking ", file)
		file = mungeFile(file)
	}

	log.Print("Uploading ", file)

	extraParams := map[string]string{}

	if config.username != "" {
		extraParams["username"] = config.username
	}

	type DiscordAPIResponseAttachment struct {
		URL      string
		ProxyURL string
		Size     int
		Width    int
		Height   int
		Filename string
	}

	type DiscordAPIResponse struct {
		Attachments []DiscordAPIResponseAttachment
		ID          int64 `json:",string"`
	}

	var retriesRemaining = 5
	for retriesRemaining > 0 {
		request, err := newfileUploadRequest(config.webhookURL, extraParams, "file", file)
		if err != nil {
			log.Fatal(err)
		}
		start := time.Now()
		client := &http.Client{Timeout: time.Second * 30}
		resp, err := client.Do(request)
		if err != nil {
			log.Print("Error performing request:", err)
			retriesRemaining--
			sleepForRetries(retriesRemaining)
			continue
		} else {

			if resp.StatusCode != 200 {
				log.Print("Bad response from server:", resp.StatusCode)
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}

			resBody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				log.Print("could not deal with body: ", err)
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			resp.Body.Close()

			var res DiscordAPIResponse
			err = json.Unmarshal(resBody, &res)

			if err != nil {
				log.Print("could not parse JSON: ", err)
				fmt.Println("Response was:", string(resBody[:]))
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			if len(res.Attachments) < 1 {
				log.Print("bad response - no attachments?")
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			var a = res.Attachments[0]
			elapsed := time.Since(start)
			rate := float64(a.Size) / elapsed.Seconds() / 1024.0

			log.Printf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height)
			log.Printf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate)
			break
		}
	}

	if !config.noWatermark {
		log.Print("Removing temporary file ", file)
		os.Remove(file)
	}

	if retriesRemaining == 0 {
		log.Fatal("Failed to upload, even after retries")
	}
}

func sleepForRetries(retry int) {
	if retry == 0 {
		return
	}
	retryTime := (6-retry)*(6-retry) + 6
	log.Printf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry)
	// time.Sleep(time.Duration(retryTime) * time.Second)
}

func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		log.Fatal("Could not copy: ", err)
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, err
}

func mungeFile(path string) string {

	reader, err := os.Open(path)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	im, _, err := image.Decode(reader)
	if err != nil {
		log.Fatal(err)
	}
	bounds := im.Bounds()
	// var S float64 = float64(bounds.Max.X)

	dc := gg.NewContext(bounds.Max.X, bounds.Max.Y)
	dc.Clear()
	dc.SetRGB(0, 0, 0)

	dc.SetFontFace(inconsolata.Regular8x16)

	dc.DrawImage(im, 0, 0)

	dc.DrawRoundedRectangle(0, float64(bounds.Max.Y-18.0), 320, float64(bounds.Max.Y), 0)
	dc.SetRGB(0, 0, 0)
	dc.Fill()

	dc.SetRGB(1, 1, 1)

	dc.DrawString("github.com/tardisx/discord-auto-upload", 5.0, float64(bounds.Max.Y)-5.0)

	tempfile, err := ioutil.TempFile("", "dau")
	if err != nil {
		log.Fatal(err)
	}
	tempfile.Close()
	os.Remove(tempfile.Name())
	actualName := tempfile.Name() + ".png"

	dc.SavePNG(actualName)
	return actualName
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

/*
	Some HTML presented at http://machine:port/debug/rpc
	Lists services, their methods, and some statistics, still rudimentary.
*/

import (
	"fmt"
	"http"
	"sort"
	"template"
)

const debugText = `

	<title>Services</title>
	{.repeated section @}

	Service {Name}

		Method	Calls
		{.repeated section Method}

			{Name}({Type.ArgType}, {Type.ReplyType}) os.Error
			{Type.NumCalls}

		{.end}

	{.end}

	`

var debug = template.MustParse(debugText, nil)

type debugMethod struct {
	Type *methodType
	Name string
}

type methodArray []debugMethod

type debugService struct {
	Service *service
	Name    string
	Method  methodArray
}

type serviceArray []debugService

func (s serviceArray) Len() int           { return len(s) }
func (s serviceArray) Less(i, j int) bool { return s[i].Name < s[j].Name }
func (s serviceArray) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func (m methodArray) Len() int           { return len(m) }
func (m methodArray) Less(i, j int) bool { return m[i].Name < m[j].Name }
func (m methodArray) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }

type debugHTTP struct {
	*Server
}

// Runs at /debug/rpc
func (server debugHTTP) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Build a sorted version of the data.
	var services = make(serviceArray, len(server.serviceMap))
	i := 0
	server.Lock()
	for sname, service := range server.serviceMap {
		services[i] = debugService{service, sname, make(methodArray, len(service.method))}
		j := 0
		for mname, method := range service.method {
			services[i].Method[j] = debugMethod{method, mname}
			j++
		}
		sort.Sort(services[i].Method)
		i++
	}
	server.Unlock()
	sort.Sort(services)
	err := debug.Execute(w, services)
	if err != nil {
		fmt.Fprintln(w, "rpc: error executing template:", err.String())
	}
}
<commit_msg>rpc: convert /debug/rpc handler to exp/template<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

/*
	Some HTML presented at http://machine:port/debug/rpc
	Lists services, their methods, and some statistics, still rudimentary.
*/

import (
	"exp/template"
	"fmt"
	"http"
	"sort"
)

const debugText = `

	<title>Services</title>
	{{range .}}

	Service {{.Name}}

		Method	Calls
		{{range .Method}}

			{{.Name}}({{.Type.ArgType}}, {{.Type.ReplyType}}) os.Error
			{{.Type.NumCalls}}

		{{end}}

	{{end}}

	`

var debug = template.New("RPC debug").MustParse(debugText)

type debugMethod struct {
	Type *methodType
	Name string
}

type methodArray []debugMethod

type debugService struct {
	Service *service
	Name    string
	Method  methodArray
}

type serviceArray []debugService

func (s serviceArray) Len() int           { return len(s) }
func (s serviceArray) Less(i, j int) bool { return s[i].Name < s[j].Name }
func (s serviceArray) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func (m methodArray) Len() int           { return len(m) }
func (m methodArray) Less(i, j int) bool { return m[i].Name < m[j].Name }
func (m methodArray) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }

type debugHTTP struct {
	*Server
}

// Runs at /debug/rpc
func (server debugHTTP) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// Build a sorted version of the data.
	var services = make(serviceArray, len(server.serviceMap))
	i := 0
	server.Lock()
	for sname, service := range server.serviceMap {
		services[i] = debugService{service, sname, make(methodArray, len(service.method))}
		j := 0
		for mname, method := range service.method {
			services[i].Method[j] = debugMethod{method, mname}
			j++
		}
		sort.Sort(services[i].Method)
		i++
	}
	server.Unlock()
	sort.Sort(services)
	err := debug.Execute(w, services)
	if err != nil {
		fmt.Fprintln(w, "rpc: error executing template:", err.String())
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The sort package provides primitives for sorting arrays
// and user-defined collections.
package sort

// A type, typically a collection, that satisfies sort.Interface can be
// sorted by the routines in this package.  The methods require that the
// elements of the collection be enumerated by an integer index.
type Interface interface {
	// Len is the number of elements in the collection.
	Len() int;
	// Less returns whether the element with index i is should sort
	// before the element with index j.
	// TODO(r): should this method be renamed Before?
	Less(i, j int) bool;
	// Swap swaps the elements with indexes i and j.
	Swap(i, j int);
}

func min(a, b int) int {
	if a < b {
		return a;
	}
	return b;
}

// Insertion sort
func insertionSort(data Interface, a, b int) {
	for i := a+1; i < b; i++ {
		for j := i; j > a && data.Less(j, j-1); j-- {
			data.Swap(j, j-1);
		}
	}
}

// Quicksort, following Bentley and McIlroy,
// ``Engineering a Sort Function,'' SP&E November 1993.

// Move the median of the three values data[a], data[b], data[c] into data[a].
func medianOfThree(data Interface, a, b, c int) {
	m0 := b;
	m1 := a;
	m2 := c;
	// bubble sort on 3 elements
	if data.Less(m1, m0) {
		data.Swap(m1, m0);
	}
	if data.Less(m2, m1) {
		data.Swap(m2, m1);
	}
	if data.Less(m1, m0) {
		data.Swap(m1, m0);
	}
// now data[m0] <= data[m1] <= data[m2]
}

func swapRange(data Interface, a, b, n int) {
	for i := 0; i < n; i++ {
		data.Swap(a+i, b+i);
	}
}

func doPivot(data Interface, lo, hi int) (midlo, midhi int) {
	m := (lo+hi)/2;
	if hi-lo > 40 {
		// Tukey's ``Ninther,'' median of three medians of three.
		s := (hi-lo)/8;
		medianOfThree(data, lo, lo+s, lo + 2*s);
		medianOfThree(data, m, m-s, m+s);
		medianOfThree(data, hi-1, hi-1-s, hi - 1 - 2*s);
	}
	medianOfThree(data, lo, m, hi-1);

	// Invariants are:
	//	data[lo] = pivot (set up by ChoosePivot)
	//	data[lo <= i < a] = pivot
	//	data[a <= i < b] < pivot
	//	data[b <= i < c] is unexamined
	//	data[c <= i < d] > pivot
	//	data[d <= i < hi] = pivot
	//
	// Once b meets c, can swap the "= pivot" sections
	// into the middle of the array.
	pivot := lo;
	a, b, c, d := lo+1, lo+1, hi, hi;
	for b < c {
		if data.Less(b, pivot) {	// data[b] < pivot
			b++;
			continue;
		}
		if !data.Less(pivot, b) {	// data[b] = pivot
			data.Swap(a, b);
			a++;
			b++;
			continue;
		}
		if data.Less(pivot, c-1) {	// data[c-1] > pivot
			c--;
			continue;
		}
		if !data.Less(c-1, pivot) {	// data[c-1] = pivot
			data.Swap(c-1, d-1);
			c--;
			d--;
			continue;
		}
		// data[b] > pivot; data[c-1] < pivot
		data.Swap(b, c-1);
		b++;
		c--;
	}

	n := min(b-a, a-lo);
	swapRange(data, lo, b-n, n);

	n = min(hi-d, d-c);
	swapRange(data, c, hi-n, n);

	return lo+b-a, hi-(d-c);
}

func quickSort(data Interface, a, b int) {
	if b-a > 7 {
		mlo, mhi := doPivot(data, a, b);
		quickSort(data, a, mlo);
		quickSort(data, mhi, b);
	} else if b-a > 1 {
		insertionSort(data, a, b);
	}
}

func Sort(data Interface) {
	quickSort(data, 0, data.Len());
}

func IsSorted(data Interface) bool {
	n := data.Len();
	for i := n-1; i > 0; i-- {
		if data.Less(i, i-1) {
			return false;
		}
	}
	return true;
}

// Convenience types for common cases

// IntArray attaches the methods of Interface to []int, sorting in increasing order.
type IntArray []int

func (p IntArray) Len() int {
	return len(p);
}
func (p IntArray) Less(i, j int) bool {
	return p[i] < p[j];
}
func (p IntArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i];
}

// Sort is a convenience method.
func (p IntArray) Sort() {
	Sort(p);
}

// FloatArray attaches the methods of Interface to []float, sorting in increasing order.
type FloatArray []float

func (p FloatArray) Len() int {
	return len(p);
}
func (p FloatArray) Less(i, j int) bool {
	return p[i] < p[j];
}
func (p FloatArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i];
}

// Sort is a convenience method.
func (p FloatArray) Sort() {
	Sort(p);
}

// StringArray attaches the methods of Interface to []string, sorting in increasing order.
type StringArray []string

func (p StringArray) Len() int {
	return len(p);
}
func (p StringArray) Less(i, j int) bool {
	return p[i] < p[j];
}
func (p StringArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i];
}

// Sort is a convenience method.
func (p StringArray) Sort() {
	Sort(p);
}

// Convenience wrappers for common cases

// SortInts sorts an array of ints in increasing order.
func SortInts(a []int) {
	Sort(IntArray(a));
}
// SortFloats sorts an array of floats in increasing order.
func SortFloats(a []float) {
	Sort(FloatArray(a));
}
// SortStrings sorts an array of strings in increasing order.
func SortStrings(a []string) {
	Sort(StringArray(a));
}

// IntsAreSorted tests whether an array of ints is sorted in increasing order.
func IntsAreSorted(a []int) bool {
	return IsSorted(IntArray(a));
}
// FloatsAreSorted tests whether an array of floats is sorted in increasing order.
func FloatsAreSorted(a []float) bool {
	return IsSorted(FloatArray(a));
}
// StringsAreSorted tests whether an array of strings is sorted in increasing order.
func StringsAreSorted(a []string) bool {
	return IsSorted(StringArray(a));
}
<commit_msg>delete silly TODO<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The sort package provides primitives for sorting arrays
// and user-defined collections.
package sort

// A type, typically a collection, that satisfies sort.Interface can be
// sorted by the routines in this package.  The methods require that the
// elements of the collection be enumerated by an integer index.
type Interface interface {
	// Len is the number of elements in the collection.
	Len() int;
	// Less returns whether the element with index i is should sort
	// before the element with index j.
	Less(i, j int) bool;
	// Swap swaps the elements with indexes i and j.
	Swap(i, j int);
}

func min(a, b int) int {
	if a < b {
		return a;
	}
	return b;
}

// Insertion sort
func insertionSort(data Interface, a, b int) {
	for i := a+1; i < b; i++ {
		for j := i; j > a && data.Less(j, j-1); j-- {
			data.Swap(j, j-1);
		}
	}
}

// Quicksort, following Bentley and McIlroy,
// ``Engineering a Sort Function,'' SP&E November 1993.

// Move the median of the three values data[a], data[b], data[c] into data[a].
func medianOfThree(data Interface, a, b, c int) {
	m0 := b;
	m1 := a;
	m2 := c;
	// bubble sort on 3 elements
	if data.Less(m1, m0) {
		data.Swap(m1, m0);
	}
	if data.Less(m2, m1) {
		data.Swap(m2, m1);
	}
	if data.Less(m1, m0) {
		data.Swap(m1, m0);
	}
// now data[m0] <= data[m1] <= data[m2]
}

func swapRange(data Interface, a, b, n int) {
	for i := 0; i < n; i++ {
		data.Swap(a+i, b+i);
	}
}

func doPivot(data Interface, lo, hi int) (midlo, midhi int) {
	m := (lo+hi)/2;
	if hi-lo > 40 {
		// Tukey's ``Ninther,'' median of three medians of three.
		s := (hi-lo)/8;
		medianOfThree(data, lo, lo+s, lo + 2*s);
		medianOfThree(data, m, m-s, m+s);
		medianOfThree(data, hi-1, hi-1-s, hi - 1 - 2*s);
	}
	medianOfThree(data, lo, m, hi-1);

	// Invariants are:
	//	data[lo] = pivot (set up by ChoosePivot)
	//	data[lo <= i < a] = pivot
	//	data[a <= i < b] < pivot
	//	data[b <= i < c] is unexamined
	//	data[c <= i < d] > pivot
	//	data[d <= i < hi] = pivot
	//
	// Once b meets c, can swap the "= pivot" sections
	// into the middle of the array.
	pivot := lo;
	a, b, c, d := lo+1, lo+1, hi, hi;
	for b < c {
		if data.Less(b, pivot) {	// data[b] < pivot
			b++;
			continue;
		}
		if !data.Less(pivot, b) {	// data[b] = pivot
			data.Swap(a, b);
			a++;
			b++;
			continue;
		}
		if data.Less(pivot, c-1) {	// data[c-1] > pivot
			c--;
			continue;
		}
		if !data.Less(c-1, pivot) {	// data[c-1] = pivot
			data.Swap(c-1, d-1);
			c--;
			d--;
			continue;
		}
		// data[b] > pivot; data[c-1] < pivot
		data.Swap(b, c-1);
		b++;
		c--;
	}

	n := min(b-a, a-lo);
	swapRange(data, lo, b-n, n);

	n = min(hi-d, d-c);
	swapRange(data, c, hi-n, n);

	return lo+b-a, hi-(d-c);
}

func quickSort(data Interface, a, b int) {
	if b-a > 7 {
		mlo, mhi := doPivot(data, a, b);
		quickSort(data, a, mlo);
		quickSort(data, mhi, b);
	} else if b-a > 1 {
		insertionSort(data, a, b);
	}
}

func Sort(data Interface) {
	quickSort(data, 0, data.Len());
}

func IsSorted(data Interface) bool {
	n := data.Len();
	for i := n-1; i > 0; i-- {
		if data.Less(i, i-1) {
			return false;
		}
	}
	return true;
}

// Convenience types for common cases

// IntArray attaches the methods of Interface to []int, sorting in increasing order.
type IntArray []int

func (p IntArray) Len() int {
	return len(p);
}
func (p IntArray) Less(i, j int) bool {
	return p[i] < p[j];
}
func (p IntArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i];
}

// Sort is a convenience method.
func (p IntArray) Sort() {
	Sort(p);
}

// FloatArray attaches the methods of Interface to []float, sorting in increasing order.
type FloatArray []float

func (p FloatArray) Len() int {
	return len(p);
}
func (p FloatArray) Less(i, j int) bool {
	return p[i] < p[j];
}
func (p FloatArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i];
}

// Sort is a convenience method.
func (p FloatArray) Sort() {
	Sort(p);
}

// StringArray attaches the methods of Interface to []string, sorting in increasing order.
type StringArray []string

func (p StringArray) Len() int {
	return len(p);
}
func (p StringArray) Less(i, j int) bool {
	return p[i] < p[j];
}
func (p StringArray) Swap(i, j int) {
	p[i], p[j] = p[j], p[i];
}

// Sort is a convenience method.
func (p StringArray) Sort() {
	Sort(p);
}

// Convenience wrappers for common cases

// SortInts sorts an array of ints in increasing order.
func SortInts(a []int) {
	Sort(IntArray(a));
}
// SortFloats sorts an array of floats in increasing order.
func SortFloats(a []float) {
	Sort(FloatArray(a));
}
// SortStrings sorts an array of strings in increasing order.
func SortStrings(a []string) {
	Sort(StringArray(a));
}

// IntsAreSorted tests whether an array of ints is sorted in increasing order.
func IntsAreSorted(a []int) bool {
	return IsSorted(IntArray(a));
}
// FloatsAreSorted tests whether an array of floats is sorted in increasing order.
func FloatsAreSorted(a []float) bool {
	return IsSorted(FloatArray(a));
}
// StringsAreSorted tests whether an array of strings is sorted in increasing order.
func StringsAreSorted(a []string) bool {
	return IsSorted(StringArray(a));
}
<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package domain

import (
	"math/rand"
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/util/testleak"
)

type leaseGrantItem struct {
	leaseGrantTS uint64
	oldVer       int64
	schemaVer    int64
}

func (*testSuite) TestSchemaValidator(c *C) {
	defer testleak.AfterTest(c)()

	lease := 5 * time.Millisecond
	leaseGrantCh := make(chan leaseGrantItem)
	oracleCh := make(chan uint64)
	exit := make(chan struct{})
	go serverFunc(lease, leaseGrantCh, oracleCh, exit)

	validator := NewSchemaValidator(lease).(*schemaValidator)

	for i := 0; i < 10; i++ {
		delay := time.Duration(100+rand.Intn(900)) * time.Microsecond
		time.Sleep(delay)
		// Reload can run arbitrarily, at any time.
		reload(validator, leaseGrantCh, 0)
	}

	// Take a lease, check it's valid.
	item := <-leaseGrantCh
	validator.Update(item.leaseGrantTS, item.oldVer, item.schemaVer, []int64{10})
	valid := validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	c.Assert(valid, Equals, ResultSucc)

	// Stop the validator, validator's items value is nil.
	validator.Stop()
	isTablesChanged := validator.isRelatedTablesChanged(item.schemaVer, []int64{10})
	c.Assert(isTablesChanged, IsTrue)
	valid = validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	c.Assert(valid, Equals, ResultFail)
	validator.Restart()

	// Sleep for a long time, check schema is invalid.
	time.Sleep(lease)
	ts := <-oracleCh
	valid = validator.Check(ts, item.schemaVer, []int64{10})
	c.Assert(valid, Equals, ResultUnknown)

	currVer := reload(validator, leaseGrantCh, 0)
	valid = validator.Check(ts, item.schemaVer, []int64{0})
	c.Assert(valid, Equals, ResultFail)
	// Check the latest schema version must changed.
	c.Assert(item.schemaVer, Less, validator.latestSchemaVer)

	// Make sure newItem's version is bigger than currVer.
	time.Sleep(lease * 2)
	newItem := <-leaseGrantCh

	// Update current schema version to newItem's version and the delta table IDs is 1, 2, 3.
	validator.Update(ts, currVer, newItem.schemaVer, []int64{1, 2, 3})
	// Make sure the updated table IDs don't be covered with the same schema version.
	validator.Update(ts, newItem.schemaVer, newItem.schemaVer, nil)
	isTablesChanged = validator.isRelatedTablesChanged(currVer, nil)
	c.Assert(isTablesChanged, IsFalse)
	isTablesChanged = validator.isRelatedTablesChanged(currVer, []int64{2})
	c.Assert(isTablesChanged, IsTrue)
	// The current schema version is older than the oldest schema version.
	isTablesChanged = validator.isRelatedTablesChanged(-1, nil)
	c.Assert(isTablesChanged, IsTrue)

	// All schema versions is expired.
	ts = uint64(time.Now().Add(lease).UnixNano())
	valid = validator.Check(ts, newItem.schemaVer, nil)
	c.Assert(valid, Equals, ResultUnknown)

	exit <- struct{}{}
}

func reload(validator SchemaValidator, leaseGrantCh chan leaseGrantItem, ids ...int64) int64 {
	item := <-leaseGrantCh
	validator.Update(item.leaseGrantTS, item.oldVer, item.schemaVer, ids)
	return item.schemaVer
}

// serverFunc plays the role as a remote server, runs in a separate goroutine.
// It can grant lease and provide timestamp oracle.
// Caller should communicate with it through channel to mock network.
func serverFunc(lease time.Duration, requireLease chan leaseGrantItem, oracleCh chan uint64, exit chan struct{}) {
	var version int64
	leaseTS := uint64(time.Now().UnixNano())
	ticker := time.NewTicker(lease)
	for {
		select {
		case <-ticker.C:
			version++
			leaseTS = uint64(time.Now().UnixNano())
		case requireLease <- leaseGrantItem{
			leaseGrantTS: leaseTS,
			oldVer:       version - 1,
			schemaVer:    version,
		}:
		case oracleCh <- uint64(time.Now().UnixNano()):
		case <-exit:
			return
		}
	}
}
<commit_msg>domain: make schema validator test more stable (#6128)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package domain

import (
	"math/rand"
	"time"

	. "github.com/pingcap/check"
	"github.com/pingcap/tidb/util/testleak"
)

type leaseGrantItem struct {
	leaseGrantTS uint64
	oldVer       int64
	schemaVer    int64
}

func (*testSuite) TestSchemaValidator(c *C) {
	defer testleak.AfterTest(c)()

	lease := 10 * time.Millisecond
	leaseGrantCh := make(chan leaseGrantItem)
	oracleCh := make(chan uint64)
	exit := make(chan struct{})
	go serverFunc(lease, leaseGrantCh, oracleCh, exit)

	validator := NewSchemaValidator(lease).(*schemaValidator)

	for i := 0; i < 10; i++ {
		delay := time.Duration(100+rand.Intn(900)) * time.Microsecond
		time.Sleep(delay)
		// Reload can run arbitrarily, at any time.
		reload(validator, leaseGrantCh, 0)
	}

	// Take a lease, check it's valid.
	item := <-leaseGrantCh
	validator.Update(item.leaseGrantTS, item.oldVer, item.schemaVer, []int64{10})
	valid := validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	c.Assert(valid, Equals, ResultSucc)

	// Stop the validator, validator's items value is nil.
	validator.Stop()
	isTablesChanged := validator.isRelatedTablesChanged(item.schemaVer, []int64{10})
	c.Assert(isTablesChanged, IsTrue)
	valid = validator.Check(item.leaseGrantTS, item.schemaVer, []int64{10})
	c.Assert(valid, Equals, ResultFail)
	validator.Restart()

	// Sleep for a long time, check schema is invalid.
	time.Sleep(lease)
	ts := <-oracleCh
	valid = validator.Check(ts, item.schemaVer, []int64{10})
	c.Assert(valid, Equals, ResultUnknown)

	currVer := reload(validator, leaseGrantCh, 0)
	valid = validator.Check(ts, item.schemaVer, []int64{0})
	c.Assert(valid, Equals, ResultFail)
	// Check the latest schema version must changed.
	c.Assert(item.schemaVer, Less, validator.latestSchemaVer)

	// Make sure newItem's version is bigger than currVer.
	time.Sleep(lease * 2)
	newItem := <-leaseGrantCh

	// Update current schema version to newItem's version and the delta table IDs is 1, 2, 3.
	validator.Update(ts, currVer, newItem.schemaVer, []int64{1, 2, 3})
	// Make sure the updated table IDs don't be covered with the same schema version.
	validator.Update(ts, newItem.schemaVer, newItem.schemaVer, nil)
	isTablesChanged = validator.isRelatedTablesChanged(currVer, nil)
	c.Assert(isTablesChanged, IsFalse)
	isTablesChanged = validator.isRelatedTablesChanged(currVer, []int64{2})
	c.Assert(isTablesChanged, IsTrue)
	// The current schema version is older than the oldest schema version.
	isTablesChanged = validator.isRelatedTablesChanged(-1, nil)
	c.Assert(isTablesChanged, IsTrue)

	// All schema versions is expired.
	ts = uint64(time.Now().Add(lease).UnixNano())
	valid = validator.Check(ts, newItem.schemaVer, nil)
	c.Assert(valid, Equals, ResultUnknown)

	close(exit)
}

func reload(validator SchemaValidator, leaseGrantCh chan leaseGrantItem, ids ...int64) int64 {
	item := <-leaseGrantCh
	validator.Update(item.leaseGrantTS, item.oldVer, item.schemaVer, ids)
	return item.schemaVer
}

// serverFunc plays the role as a remote server, runs in a separate goroutine.
// It can grant lease and provide timestamp oracle.
// Caller should communicate with it through channel to mock network.
func serverFunc(lease time.Duration, requireLease chan leaseGrantItem, oracleCh chan uint64, exit chan struct{}) {
	var version int64
	leaseTS := uint64(time.Now().UnixNano())
	ticker := time.NewTicker(lease)
	for {
		select {
		case now := <-ticker.C:
			version++
			leaseTS = uint64(now.UnixNano())
		case requireLease <- leaseGrantItem{
			leaseGrantTS: leaseTS,
			oldVer:       version - 1,
			schemaVer:    version,
		}:
		case oracleCh <- uint64(time.Now().UnixNano()):
		case <-exit:
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(rsc): All the prints in this file should go to standard error.

package net

import (
	"net";
	"once";
	"os";
	"sync";
	"syscall";
)

// Network file descriptor.
type netFD struct {
	// immutable until Close
	fd int64;
	file *os.File;
	cr chan *netFD;
	cw chan *netFD;
	net string;
	laddr string;
	raddr string;

	// owned by client
	rdeadline_delta int64;
	rdeadline int64;
	rio sync.Mutex;
	wdeadline_delta int64;
	wdeadline int64;
	wio sync.Mutex;

	// owned by fd wait server
	ncr, ncw int;
}

// Make reads and writes on fd return EAGAIN instead of blocking.
func setNonblock(fd int64) *os.Error {
	flags, e := syscall.Fcntl(fd, syscall.F_GETFL, 0);
	if e != 0 {
		return os.ErrnoToError(e)
	}
	flags, e = syscall.Fcntl(fd, syscall.F_SETFL, flags | syscall.O_NONBLOCK);
	if e != 0 {
		return os.ErrnoToError(e)
	}
	return nil
}

// Make reads/writes blocking; last gasp, so no error checking.
func setBlock(fd int64) {
	flags, e := syscall.Fcntl(fd, syscall.F_GETFL, 0);
	if e != 0 {
		return;
	}
	syscall.Fcntl(fd, syscall.F_SETFL, flags & ^syscall.O_NONBLOCK);
}

// A pollServer helps FDs determine when to retry a non-blocking
// read or write after they get EAGAIN.  When an FD needs to wait,
// send the fd on s.cr (for a read) or s.cw (for a write) to pass the
// request to the poll server.  Then receive on fd.cr/fd.cw.
// When the pollServer finds that i/o on FD should be possible
// again, it will send fd on fd.cr/fd.cw to wake any waiting processes.
// This protocol is implemented as s.WaitRead() and s.WaitWrite().
//
// There is one subtlety: when sending on s.cr/s.cw, the
// poll server is probably in a system call, waiting for an fd
// to become ready.  It's not looking at the request channels.
// To resolve this, the poll server waits not just on the FDs it has
// been given but also its own pipe.  After sending on the
// buffered channel s.cr/s.cw, WaitRead/WaitWrite writes a
// byte to the pipe, causing the pollServer's poll system call to
// return.  In response to the pipe being readable, the pollServer
// re-polls its request channels.
//
// Note that the ordering is "send request" and then "wake up server".
// If the operations were reversed, there would be a race: the poll
// server might wake up and look at the request channel, see that it
// was empty, and go back to sleep, all before the requester managed
// to send the request.  Because the send must complete before the wakeup,
// the request channel must be buffered.  A buffer of size 1 is sufficient
// for any request load.  If many processes are trying to submit requests,
// one will succeed, the pollServer will read the request, and then the
// channel will be empty for the next process's request.  A larger buffer
// might help batch requests.

type pollServer struct {
	cr, cw chan *netFD;	// buffered >= 1
	pr, pw *os.File;
	pending map[int64] *netFD;
	poll *pollster;	// low-level OS hooks
	deadline int64;	// next deadline (nsec since 1970)
}
func (s *pollServer) Run();

func newPollServer() (s *pollServer, err *os.Error) {
	s = new(pollServer);
	s.cr = make(chan *netFD, 1);
	s.cw = make(chan *netFD, 1);
	if s.pr, s.pw, err = os.Pipe(); err != nil {
		return nil, err
	}
	if err = setNonblock(s.pr.Fd()); err != nil {
	Error:
		s.pr.Close();
		s.pw.Close();
		return nil, err
	}
	if err = setNonblock(s.pw.Fd()); err != nil {
		goto Error
	}
	if s.poll, err = newpollster(); err != nil {
		goto Error
	}
	if err = s.poll.AddFD(s.pr.Fd(), 'r', true); err != nil {
		s.poll.Close();
		goto Error
	}
	s.pending = make(map[int64] *netFD);
	go s.Run();
	return s, nil
}

func (s *pollServer) AddFD(fd *netFD, mode int) {
	if err := s.poll.AddFD(fd.fd, mode, false); err != nil {
		panicln("pollServer AddFD ", fd.fd, ": ", err.String(), "\n");
		return
	}

	var t int64;
	key := fd.fd << 1;
	if mode == 'r' {
		fd.ncr++;
		t = fd.rdeadline;
	} else {
		fd.ncw++;
		key++;
		t = fd.wdeadline;
	}
	s.pending[key] = fd;
	if t > 0 && (s.deadline == 0 || t < s.deadline) {
		s.deadline = t;
	}
}

func (s *pollServer) LookupFD(fd int64, mode int) *netFD {
	key := fd << 1;
	if mode == 'w' {
		key++;
	}
	netfd, ok := s.pending[key];
	if !ok {
		return nil
	}
	s.pending[key] = nil, false;
	return netfd
}

func (s *pollServer) WakeFD(fd *netFD, mode int) {
	if mode == 'r' {
		for fd.ncr > 0 {
			fd.ncr--;
			fd.cr <- fd
		}
	} else {
		for fd.ncw > 0 {
			fd.ncw--;
			fd.cw <- fd
		}
	}
}

func (s *pollServer) Now() int64 {
	sec, nsec, err := os.Time();
	if err != nil {
		panic("net: os.Time: ", err.String());
	}
	nsec += sec * 1e9;
	return nsec;
}

func (s *pollServer) CheckDeadlines() {
	now := s.Now();
	// TODO(rsc): This will need to be handled more efficiently,
	// probably with a heap indexed by wakeup time.

	var next_deadline int64;
	for key, fd := range s.pending {
		var t int64;
		var mode int;
		if key&1 == 0 {
			mode = 'r';
		} else {
			mode = 'w';
		}
		if mode == 'r' {
			t = fd.rdeadline;
		} else {
			t = fd.wdeadline;
		}
		if t > 0 {
			if t <= now {
				s.pending[key] = nil, false;
				if mode == 'r' {
					s.poll.DelFD(fd.fd, mode);
					fd.rdeadline = -1;
				} else {
					s.poll.DelFD(fd.fd, mode);
					fd.wdeadline = -1;
				}
				s.WakeFD(fd, mode);
			} else if next_deadline == 0 || t < next_deadline {
				next_deadline = t;
			}
		}
	}
	s.deadline = next_deadline;
}

func (s *pollServer) Run() {
	var scratch [100]byte;
	for {
		var t = s.deadline;
		if t > 0 {
			t = t - s.Now();
			if t < 0 {
				s.CheckDeadlines();
				continue;
			}
		}
		fd, mode, err := s.poll.WaitFD(t);
		if err != nil {
			print("pollServer WaitFD: ", err.String(), "\n");
			return
		}
		if fd < 0 {
			// Timeout happened.
			s.CheckDeadlines();
			continue;
		}
		if fd == s.pr.Fd() {
			// Drain our wakeup pipe.
			for nn, e := s.pr.Read(scratch); nn > 0; {
				nn, e = s.pr.Read(scratch)
			}

			// Read from channels
			for fd, ok := <-s.cr; ok; fd, ok = <-s.cr {
				s.AddFD(fd, 'r')
			}
			for fd, ok := <-s.cw; ok; fd, ok = <-s.cw {
				s.AddFD(fd, 'w')
			}
		} else {
			netfd := s.LookupFD(fd, mode);
			if netfd == nil {
				print("pollServer: unexpected wakeup for fd=", netfd, " mode=", string(mode), "\n");
				continue
			}
			s.WakeFD(netfd, mode);
		}
	}
}

func (s *pollServer) Wakeup() {
	var b [1]byte;
	s.pw.Write(b)
}

func (s *pollServer) WaitRead(fd *netFD) {
	s.cr <- fd;
	s.Wakeup();
	<-fd.cr
}

func (s *pollServer) WaitWrite(fd *netFD) {
	s.cr <- fd;
	s.Wakeup();
	<-fd.cr
}

// Network FD methods.
// All the network FDs use a single pollServer.

var pollserver *pollServer

func _StartServer() {
	p, err := newPollServer();
	if err != nil {
		print("Start pollServer: ", err.String(), "\n")
	}
	pollserver = p
}

func newFD(fd int64, net, laddr, raddr string) (f *netFD, err *os.Error) {
	if pollserver == nil {
		once.Do(_StartServer);
	}
	if err = setNonblock(fd); err != nil {
		return nil, err
	}
	f = new(netFD);
	f.fd = fd;
	f.net = net;
	f.laddr = laddr;
	f.raddr = raddr;
	f.file = os.NewFile(fd, "net: " + net + " " + laddr + " " + raddr);
	f.cr = make(chan *netFD, 1);
	f.cw = make(chan *netFD, 1);
	return f, nil
}

func (fd *netFD) Close() *os.Error {
	if fd == nil || fd.file == nil {
		return os.EINVAL
	}

	// In case the user has set linger,
	// switch to blocking mode so the close blocks.
	// As long as this doesn't happen often,
	// we can handle the extra OS processes.
	// Otherwise we'll need to use the pollserver
	// for Close too.  Sigh.
	setBlock(fd.file.Fd());

	e := fd.file.Close();
	fd.file = nil;
	fd.fd = -1;
	return e
}

func (fd *netFD) Read(p []byte) (n int, err *os.Error) {
	if fd == nil || fd.file == nil {
		return -1, os.EINVAL
	}
	fd.rio.Lock();
	defer fd.rio.Unlock();
	if fd.rdeadline_delta > 0 {
		fd.rdeadline = pollserver.Now() + fd.rdeadline_delta;
	} else {
		fd.rdeadline = 0;
	}
	n, err = fd.file.Read(p);
	for err == os.EAGAIN && fd.rdeadline >= 0 {
		pollserver.WaitRead(fd);
		n, err = fd.file.Read(p)
	}
	return n, err
}

func (fd *netFD) Write(p []byte) (n int, err *os.Error) {
	if fd == nil || fd.file == nil {
		return -1, os.EINVAL
	}
	fd.wio.Lock();
	defer fd.wio.Unlock();
	if fd.wdeadline_delta > 0 {
		fd.wdeadline = pollserver.Now() + fd.wdeadline_delta;
	} else {
		fd.wdeadline = 0;
	}
	err = nil;
	nn := 0;
	for nn < len(p) {
		n, err = fd.file.Write(p[nn:len(p)]);
		if n > 0 {
			nn += n
		}
		if nn == len(p) {
			break;
		}
		if err == os.EAGAIN && fd.wdeadline >= 0 {
			pollserver.WaitWrite(fd);
			continue;
		}
		if n == 0 || err != nil {
			break;
		}
	}
	return nn, err
}

func sockaddrToHostPort(sa *syscall.Sockaddr) (hostport string, err *os.Error)

func (fd *netFD) Accept(sa *syscall.Sockaddr) (nfd *netFD, err *os.Error) {
	if fd == nil || fd.file == nil {
		return nil, os.EINVAL
	}

	// See ../syscall/exec.go for description of ForkLock.
	// It is okay to hold the lock across syscall.Accept
	// because we have put fd.fd into non-blocking mode.
	syscall.ForkLock.RLock();
	var s, e int64;
	for {
		s, e = syscall.Accept(fd.fd, sa);
		if e != syscall.EAGAIN {
			break;
		}
		syscall.ForkLock.RUnlock();
		pollserver.WaitRead(fd);
		syscall.ForkLock.RLock();
	}
	if e != 0 {
		syscall.ForkLock.RUnlock();
		return nil, os.ErrnoToError(e)
	}
	syscall.CloseOnExec(s);
	syscall.ForkLock.RUnlock();

	raddr, err1 := sockaddrToHostPort(sa);
	if err1 != nil {
		raddr = "invalid-address";
	}
	if nfd, err = newFD(s, fd.net, fd.laddr, raddr); err != nil {
		syscall.Close(s);
		return nil, err
	}
	return nfd, nil
}

<commit_msg>document and partially fix a race<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// TODO(rsc): All the prints in this file should go to standard error.

package net

import (
	"net";
	"once";
	"os";
	"sync";
	"syscall";
)

// Network file descriptor.
type netFD struct {
	// immutable until Close
	fd int64;
	file *os.File;
	cr chan *netFD;
	cw chan *netFD;
	net string;
	laddr string;
	raddr string;

	// owned by client
	rdeadline_delta int64;
	rdeadline int64;
	rio sync.Mutex;
	wdeadline_delta int64;
	wdeadline int64;
	wio sync.Mutex;

	// owned by fd wait server
	ncr, ncw int;
}

// Make reads and writes on fd return EAGAIN instead of blocking.
func setNonblock(fd int64) *os.Error {
	flags, e := syscall.Fcntl(fd, syscall.F_GETFL, 0);
	if e != 0 {
		return os.ErrnoToError(e)
	}
	flags, e = syscall.Fcntl(fd, syscall.F_SETFL, flags | syscall.O_NONBLOCK);
	if e != 0 {
		return os.ErrnoToError(e)
	}
	return nil
}

// Make reads/writes blocking; last gasp, so no error checking.
func setBlock(fd int64) {
	flags, e := syscall.Fcntl(fd, syscall.F_GETFL, 0);
	if e != 0 {
		return;
	}
	syscall.Fcntl(fd, syscall.F_SETFL, flags & ^syscall.O_NONBLOCK);
}

// A pollServer helps FDs determine when to retry a non-blocking
// read or write after they get EAGAIN.  When an FD needs to wait,
// send the fd on s.cr (for a read) or s.cw (for a write) to pass the
// request to the poll server.  Then receive on fd.cr/fd.cw.
// When the pollServer finds that i/o on FD should be possible
// again, it will send fd on fd.cr/fd.cw to wake any waiting processes.
// This protocol is implemented as s.WaitRead() and s.WaitWrite().
//
// There is one subtlety: when sending on s.cr/s.cw, the
// poll server is probably in a system call, waiting for an fd
// to become ready.  It's not looking at the request channels.
// To resolve this, the poll server waits not just on the FDs it has
// been given but also its own pipe.  After sending on the
// buffered channel s.cr/s.cw, WaitRead/WaitWrite writes a
// byte to the pipe, causing the pollServer's poll system call to
// return.  In response to the pipe being readable, the pollServer
// re-polls its request channels.
//
// Note that the ordering is "send request" and then "wake up server".
// If the operations were reversed, there would be a race: the poll
// server might wake up and look at the request channel, see that it
// was empty, and go back to sleep, all before the requester managed
// to send the request.  Because the send must complete before the wakeup,
// the request channel must be buffered.  A buffer of size 1 is sufficient
// for any request load.  If many processes are trying to submit requests,
// one will succeed, the pollServer will read the request, and then the
// channel will be empty for the next process's request.  A larger buffer
// might help batch requests.

type pollServer struct {
	cr, cw chan *netFD;	// buffered >= 1
	pr, pw *os.File;
	pending map[int64] *netFD;
	poll *pollster;	// low-level OS hooks
	deadline int64;	// next deadline (nsec since 1970)
}
func (s *pollServer) Run();

func newPollServer() (s *pollServer, err *os.Error) {
	s = new(pollServer);
	s.cr = make(chan *netFD, 1);
	s.cw = make(chan *netFD, 1);
	if s.pr, s.pw, err = os.Pipe(); err != nil {
		return nil, err
	}
	if err = setNonblock(s.pr.Fd()); err != nil {
	Error:
		s.pr.Close();
		s.pw.Close();
		return nil, err
	}
	if err = setNonblock(s.pw.Fd()); err != nil {
		goto Error
	}
	if s.poll, err = newpollster(); err != nil {
		goto Error
	}
	if err = s.poll.AddFD(s.pr.Fd(), 'r', true); err != nil {
		s.poll.Close();
		goto Error
	}
	s.pending = make(map[int64] *netFD);
	go s.Run();
	return s, nil
}

func (s *pollServer) AddFD(fd *netFD, mode int) {
	// TODO(rsc): This check handles a race between
	// one goroutine reading and another one closing,
	// but it doesn't solve the race completely:
	// it still could happen that one goroutine closes
	// but we read fd.fd before it does, and then
	// another goroutine creates a new open file with
	// that fd, which we'd now be referring to.
	// The fix is probably to send the Close call
	// through the poll server too, except that
	// not all Reads and Writes go through the poll
	// server even now.
	intfd := fd.fd;
	if intfd < 0 {
		// fd closed underfoot
		if mode == 'r' {
			fd.cr <- fd
		} else {
			fd.cw <- fd
		}
		return
	}
	if err := s.poll.AddFD(intfd, mode, false); err != nil {
		panicln("pollServer AddFD ", intfd, ": ", err.String(), "\n");
		return
	}

	var t int64;
	key := intfd << 1;
	if mode == 'r' {
		fd.ncr++;
		t = fd.rdeadline;
	} else {
		fd.ncw++;
		key++;
		t = fd.wdeadline;
	}
	s.pending[key] = fd;
	if t > 0 && (s.deadline == 0 || t < s.deadline) {
		s.deadline = t;
	}
}

func (s *pollServer) LookupFD(fd int64, mode int) *netFD {
	key := fd << 1;
	if mode == 'w' {
		key++;
	}
	netfd, ok := s.pending[key];
	if !ok {
		return nil
	}
	s.pending[key] = nil, false;
	return netfd
}

func (s *pollServer) WakeFD(fd *netFD, mode int) {
	if mode == 'r' {
		for fd.ncr > 0 {
			fd.ncr--;
			fd.cr <- fd
		}
	} else {
		for fd.ncw > 0 {
			fd.ncw--;
			fd.cw <- fd
		}
	}
}

func (s *pollServer) Now() int64 {
	sec, nsec, err := os.Time();
	if err != nil {
		panic("net: os.Time: ", err.String());
	}
	nsec += sec * 1e9;
	return nsec;
}

func (s *pollServer) CheckDeadlines() {
	now := s.Now();
	// TODO(rsc): This will need to be handled more efficiently,
	// probably with a heap indexed by wakeup time.

	var next_deadline int64;
	for key, fd := range s.pending {
		var t int64;
		var mode int;
		if key&1 == 0 {
			mode = 'r';
		} else {
			mode = 'w';
		}
		if mode == 'r' {
			t = fd.rdeadline;
		} else {
			t = fd.wdeadline;
		}
		if t > 0 {
			if t <= now {
				s.pending[key] = nil, false;
				if mode == 'r' {
					s.poll.DelFD(fd.fd, mode);
					fd.rdeadline = -1;
				} else {
					s.poll.DelFD(fd.fd, mode);
					fd.wdeadline = -1;
				}
				s.WakeFD(fd, mode);
			} else if next_deadline == 0 || t < next_deadline {
				next_deadline = t;
			}
		}
	}
	s.deadline = next_deadline;
}

func (s *pollServer) Run() {
	var scratch [100]byte;
	for {
		var t = s.deadline;
		if t > 0 {
			t = t - s.Now();
			if t < 0 {
				s.CheckDeadlines();
				continue;
			}
		}
		fd, mode, err := s.poll.WaitFD(t);
		if err != nil {
			print("pollServer WaitFD: ", err.String(), "\n");
			return
		}
		if fd < 0 {
			// Timeout happened.
			s.CheckDeadlines();
			continue;
		}
		if fd == s.pr.Fd() {
			// Drain our wakeup pipe.
			for nn, e := s.pr.Read(scratch); nn > 0; {
				nn, e = s.pr.Read(scratch)
			}

			// Read from channels
			for fd, ok := <-s.cr; ok; fd, ok = <-s.cr {
				s.AddFD(fd, 'r')
			}
			for fd, ok := <-s.cw; ok; fd, ok = <-s.cw {
				s.AddFD(fd, 'w')
			}
		} else {
			netfd := s.LookupFD(fd, mode);
			if netfd == nil {
				print("pollServer: unexpected wakeup for fd=", netfd, " mode=", string(mode), "\n");
				continue
			}
			s.WakeFD(netfd, mode);
		}
	}
}

func (s *pollServer) Wakeup() {
	var b [1]byte;
	s.pw.Write(b)
}

func (s *pollServer) WaitRead(fd *netFD) {
	s.cr <- fd;
	s.Wakeup();
	<-fd.cr
}

func (s *pollServer) WaitWrite(fd *netFD) {
	s.cr <- fd;
	s.Wakeup();
	<-fd.cr
}

// Network FD methods.
// All the network FDs use a single pollServer.

var pollserver *pollServer

func _StartServer() {
	p, err := newPollServer();
	if err != nil {
		print("Start pollServer: ", err.String(), "\n")
	}
	pollserver = p
}

func newFD(fd int64, net, laddr, raddr string) (f *netFD, err *os.Error) {
	if pollserver == nil {
		once.Do(_StartServer);
	}
	if err = setNonblock(fd); err != nil {
		return nil, err
	}
	f = new(netFD);
	f.fd = fd;
	f.net = net;
	f.laddr = laddr;
	f.raddr = raddr;
	f.file = os.NewFile(fd, "net: " + net + " " + laddr + " " + raddr);
	f.cr = make(chan *netFD, 1);
	f.cw = make(chan *netFD, 1);
	return f, nil
}

func (fd *netFD) Close() *os.Error {
	if fd == nil || fd.file == nil {
		return os.EINVAL
	}

	// In case the user has set linger,
	// switch to blocking mode so the close blocks.
	// As long as this doesn't happen often,
	// we can handle the extra OS processes.
	// Otherwise we'll need to use the pollserver
	// for Close too.  Sigh.
	setBlock(fd.file.Fd());

	e := fd.file.Close();
	fd.file = nil;
	fd.fd = -1;
	return e
}

func (fd *netFD) Read(p []byte) (n int, err *os.Error) {
	if fd == nil || fd.file == nil {
		return -1, os.EINVAL
	}
	fd.rio.Lock();
	defer fd.rio.Unlock();
	if fd.rdeadline_delta > 0 {
		fd.rdeadline = pollserver.Now() + fd.rdeadline_delta;
	} else {
		fd.rdeadline = 0;
	}
	n, err = fd.file.Read(p);
	for err == os.EAGAIN && fd.rdeadline >= 0 {
		pollserver.WaitRead(fd);
		n, err = fd.file.Read(p)
	}
	return n, err
}

func (fd *netFD) Write(p []byte) (n int, err *os.Error) {
	if fd == nil || fd.file == nil {
		return -1, os.EINVAL
	}
	fd.wio.Lock();
	defer fd.wio.Unlock();
	if fd.wdeadline_delta > 0 {
		fd.wdeadline = pollserver.Now() + fd.wdeadline_delta;
	} else {
		fd.wdeadline = 0;
	}
	err = nil;
	nn := 0;
	for nn < len(p) {
		n, err = fd.file.Write(p[nn:len(p)]);
		if n > 0 {
			nn += n
		}
		if nn == len(p) {
			break;
		}
		if err == os.EAGAIN && fd.wdeadline >= 0 {
			pollserver.WaitWrite(fd);
			continue;
		}
		if n == 0 || err != nil {
			break;
		}
	}
	return nn, err
}

func sockaddrToHostPort(sa *syscall.Sockaddr) (hostport string, err *os.Error)

func (fd *netFD) Accept(sa *syscall.Sockaddr) (nfd *netFD, err *os.Error) {
	if fd == nil || fd.file == nil {
		return nil, os.EINVAL
	}

	// See ../syscall/exec.go for description of ForkLock.
	// It is okay to hold the lock across syscall.Accept
	// because we have put fd.fd into non-blocking mode.
	syscall.ForkLock.RLock();
	var s, e int64;
	for {
		s, e = syscall.Accept(fd.fd, sa);
		if e != syscall.EAGAIN {
			break;
		}
		syscall.ForkLock.RUnlock();
		pollserver.WaitRead(fd);
		syscall.ForkLock.RLock();
	}
	if e != 0 {
		syscall.ForkLock.RUnlock();
		return nil, os.ErrnoToError(e)
	}
	syscall.CloseOnExec(s);
	syscall.ForkLock.RUnlock();

	raddr, err1 := sockaddrToHostPort(sa);
	if err1 != nil {
		raddr = "invalid-address";
	}
	if nfd, err = newFD(s, fd.net, fd.laddr, raddr); err != nil {
		syscall.Close(s);
		return nil, err
	}
	return nfd, nil
}

<|endoftext|>


<commit_before>package interpolate

import (
	"fmt"
)

type BiCubic struct {
	xs, ys []float64
	vals []float64
	nx int

	lastY float64
	ySplines []*Spline
	xSplineVals []float64
	xSpline *Spline
}

func NewBiCubic(xs, ys, vals []float64) *BiCubic {
	if len(xs) * len(ys) != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), len(xs), len(ys),
		))
	}

	bi := &BiCubic{}
	bi.nx = len(xs)
	bi.vals = vals

	bi.xs, bi.ys = xs, ys

	bi.initSplines()

	return bi
}

func NewUniformBiCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	vals []float64,
) *BiCubic {
	if nx*ny != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), nx, ny,
		))
	}

	bi := &BiCubic{}
	bi.nx = nx
	bi.vals = vals

	bi.xs = make([]float64, nx)
	bi.ys = make([]float64, ny)
	for i := range bi.xs { bi.xs[i] = x0 + float64(i)*dx }
	for i := range bi.ys { bi.ys[i] = y0 + float64(i)*dy }

	bi.initSplines()

	return bi
}

func (bi *BiCubic) initSplines() {
	bi.ySplines = make([]*Spline, len(bi.xs))

	for xi := range bi.xs {
		yVals := make([]float64, len(bi.ys))
		for yi := range bi.ys {
			yVals[yi] = bi.vals[bi.nx * yi + xi]
		}

		bi.ySplines[xi] = NewSpline(bi.ys, yVals)
	}

	bi.lastY = bi.ys[0]
	bi.xSplineVals = make([]float64, len(bi.xs))
	for i := range bi.xSplineVals {
		bi.xSplineVals[i] = bi.ySplines[i].Eval(bi.lastY)
	}

	bi.xSpline = NewSpline(bi.xs, bi.xSplineVals)
}

func (bi *BiCubic) Eval(x, y float64) float64 {
	if y != bi.lastY {
		bi.lastY = y
		for i := range bi.xSplineVals {
			bi.xSplineVals[i] = bi.ySplines[i].Eval(y)
		}

		bi.xSpline.Init(bi.xs, bi.xSplineVals)
	}

	return bi.xSpline.Eval(x)
}

func (bi *BiCubic) EvalAll(xs, ys []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = bi.Eval(xs[i], ys[i]) }
	return out[0]
}

type TriCubic struct {
}

func NewTriCubic(xs, ys, zs, vals []float64) *TriCubic {
	panic("NYI")
}

func NewUniformTriCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	z0, dz float64, nz int,
	vals []float64,
) *TriCubic {
	panic("NYI")
}

func (tri *TriCubic) Eval(x, y, z float64) float64 {
	panic("NYI")
}

func (tri *TriCubic) EvalAll(xs, ys, zs []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = tri.Eval(xs[i], ys[i], zs[i]) }
	return out[0]
}
<commit_msg>Wrote TriCubic interpolator.<commit_after>package interpolate

import (
	"fmt"
)

////////////////////////////
// BiCubic Implementation //
////////////////////////////

type BiCubic struct {
	xs, ys []float64
	vals []float64
	nx int

	lastY float64
	ySplines []*Spline
	xSplineVals []float64
	xSpline *Spline
}

func NewBiCubic(xs, ys, vals []float64) *BiCubic {
	if len(xs) * len(ys) != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), len(xs), len(ys),
		))
	}

	bi := &BiCubic{}
	bi.nx = len(xs)
	bi.vals = vals

	bi.xs, bi.ys = xs, ys

	bi.initSplines()

	return bi
}

func NewUniformBiCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	vals []float64,
) *BiCubic {
	if nx*ny != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d and len(ys) = %d",
			len(vals), nx, ny,
		))
	}

	bi := &BiCubic{}
	bi.nx = nx
	bi.vals = vals

	bi.xs = make([]float64, nx)
	bi.ys = make([]float64, ny)
	for i := range bi.xs { bi.xs[i] = x0 + float64(i)*dx }
	for i := range bi.ys { bi.ys[i] = y0 + float64(i)*dy }

	bi.initSplines()

	return bi
}

func (bi *BiCubic) initSplines() {
	bi.ySplines = make([]*Spline, len(bi.xs))

	for xi := range bi.xs {
		yVals := make([]float64, len(bi.ys))
		for yi := range bi.ys {
			yVals[yi] = bi.vals[bi.nx * yi + xi]
		}

		bi.ySplines[xi] = NewSpline(bi.ys, yVals)
	}

	bi.lastY = bi.ys[0]
	bi.xSplineVals = make([]float64, len(bi.xs))
	for i := range bi.xSplineVals {
		bi.xSplineVals[i] = bi.ySplines[i].Eval(bi.lastY)
	}

	bi.xSpline = NewSpline(bi.xs, bi.xSplineVals)
}

func (bi *BiCubic) Eval(x, y float64) float64 {
	if y != bi.lastY {
		bi.lastY = y
		for i := range bi.xSplineVals {
			bi.xSplineVals[i] = bi.ySplines[i].Eval(y)
		}

		bi.xSpline.Init(bi.xs, bi.xSplineVals)
	}

	return bi.xSpline.Eval(x)
}

func (bi *BiCubic) EvalAll(xs, ys []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = bi.Eval(xs[i], ys[i]) }
	return out[0]
}

/////////////////////////////
// TriCubic Implementation //
/////////////////////////////

type TriCubic struct {
	xs, ys, zs []float64
	vals []float64
	nx, ny int

	lastY, lastZ float64
	zSplines []*Spline
	ySplineVals [][]float64
	ySplines []*Spline
	xSplineVals []float64
	xSpline *Spline
}

func NewTriCubic(xs, ys, zs, vals []float64) *TriCubic {
	if len(xs)*len(ys)*len(zs) != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d, len(ys) = %d, and len(zs) = %d",
			len(vals), len(xs), len(ys), len(zs),
		))
	}

	tri := &TriCubic{}
	tri.nx = len(xs)
	tri.ny = len(ys)
	tri.vals = vals

	tri.xs, tri.ys, tri.zs = xs, ys, zs

	tri.initSplines()

	return tri
}

func NewUniformTriCubic(
	x0, dx float64, nx int,
	y0, dy float64, ny int,
	z0, dz float64, nz int,
	vals []float64,
) *TriCubic {
	if nx*ny*nz != len(vals) {
		panic(fmt.Sprintf(
			"len(vals) = %d, but len(xs) = %d, len(ys) = %d, len(zs) = %d",
			len(vals), nx, ny, nz,
		))
	}

	tri := &TriCubic{}
	tri.nx = nx
	tri.ny = ny
	tri.vals = vals

	tri.xs = make([]float64, nx)
	tri.ys = make([]float64, ny)
	tri.ys = make([]float64, nz)
	for i := range tri.xs { tri.xs[i] = x0 + float64(i)*dx }
	for i := range tri.ys { tri.ys[i] = y0 + float64(i)*dy }
	for i := range tri.zs { tri.zs[i] = z0 + float64(i)*dz }

	tri.initSplines()

	return tri
}

func (tri *TriCubic) initSplines() {
	tri.zSplines = make([]*Spline, len(tri.xs) * len(tri.ys))
	for xi := range tri.xs {
		for yi := range tri.ys {
			zVals := make([]float64, len(tri.zs))
			for zi := range tri.zs {
				zVals[yi] = tri.vals[tri.nx*tri.ny*zi + tri.nx*yi + xi]
			}

			tri.zSplines[yi*tri.nx + xi] = NewSpline(tri.zs, zVals)
		}
	}

	tri.lastZ = tri.zs[0]

	tri.ySplineVals = make([][]float64, len(tri.xs))
	for xi := range tri.xs {
		tri.ySplineVals[xi] = make([]float64, len(tri.ys))
		for yi := range tri.ys {
			tri.ySplineVals[xi][yi] =
				tri.zSplines[yi*tri.nx + xi].Eval(tri.lastZ)
		}
		tri.ySplines[xi] = NewSpline(tri.xs, tri.ySplineVals[xi])
	}

	tri.lastY = tri.ys[0]

	tri.xSplineVals = make([]float64, len(tri.xs))
	for xi := range tri.xSplineVals {
		tri.xSplineVals[xi] = tri.ySplines[xi].Eval(tri.lastY)
	}

	tri.xSpline = NewSpline(tri.xs, tri.xSplineVals)
}

func (tri *TriCubic) Eval(x, y, z float64) float64 {
	if y != tri.lastY || z != tri.lastZ {
		if z != tri.lastZ {
			tri.lastZ = z

			tri.ySplineVals = make([][]float64, len(tri.xs))
			for xi := range tri.xs {
				tri.ySplineVals[xi] = make([]float64, len(tri.ys))
				for yi := range tri.ys {
					tri.ySplineVals[xi][yi] =
						tri.zSplines[yi*tri.nx + xi].Eval(tri.lastZ)
				}
				tri.ySplines[xi].Init(tri.xs, tri.ySplineVals[xi])
			}
		}

		tri.lastY = y

		tri.xSplineVals = make([]float64, len(tri.xs))
		for xi := range tri.xSplineVals {
			tri.xSplineVals[xi] = tri.ySplines[xi].Eval(tri.lastY)
		}

		tri.xSpline.Init(tri.xs, tri.xSplineVals)

	}

	return tri.xSpline.Eval(x)
}

func (tri *TriCubic) EvalAll(xs, ys, zs []float64, out ...[]float64) []float64 {
	if len(out) == 0 { out = [][]float64{ make([]float64, len(xs)) } }
	for i := range xs { out[0][i] = tri.Eval(xs[i], ys[i], zs[i]) }
	return out[0]
}
<|endoftext|>