<commit_before>/*
Quick and simple monitoring system

Usage:
 jsonmon config.yml
 jsonmon -v # Prints version to stdout and exits

Environment:
 HOST
  - defaults to localhost
  - the JSON API network interface
 PORT
  - defaults to 3000
  - the JSON API port
 GOMAXPROCS
  - defaults to CPU number + 1
  - number of threads to start
*/
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"syscall"
	"time"
)

// Application version.
const Version = "1.2.2"

// This one is for internal use.
type ver struct {
	App  string `json:"jsonmon"`
	Go   string `json:"runtime"`
	Os   string `json:"os"`
	Arch string `json:"arch"`
}

var version ver

// Check details.
type Check struct {
	Name   string      `json:"name,omitempty" yaml:"name"`
	Web    string      `json:"web,omitempty" yaml:"web"`
	Shell  string      `json:"shell,omitempty" yaml:"shell"`
	Match  string      `json:"-" yaml:"match"`
	Return int         `json:"-" yaml:"return"`
	Notify interface{} `json:"-" yaml:"notify"`
	Alert  interface{} `json:"-", yaml:"alert`
	Tries  int         `json:"-" yaml:"tries"`
	Repeat int         `json:"-" yaml:"repeat"`
	Failed bool        `json:"failed" yaml:"-"`
	Since  string      `json:"since,omitempty" yaml:"-"`
}

// Global checks list. Need to share it with workers and Web UI.
var checks []Check

// Global last modified date for HTTP caching.
var modified string

// Construct the last modified string.
func etag() {
	modified = "W/\"" + strconv.FormatInt(time.Now().UnixNano(), 10) + "\""
}

// The main loop.
func main() {
	// Parse CLI args.
	usage := "Usage: " + path.Base(os.Args[0]) + " config.yml\n" +
		"Docs:  https://github.com/chillum/jsonmon/wiki"
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(1)
	}
	// -v for version.
	version.App = Version
	version.Go = runtime.Version()
	version.Os = runtime.GOOS
	version.Arch = runtime.GOARCH
	switch os.Args[1] {
	case "-h":
		fallthrough
	case "-help":
		fallthrough
	case "--help":
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(0)
	case "-v":
		fallthrough
	case "-version":
		fallthrough
	case "--version":
		json, _ := json.MarshalIndent(&version, "", "  ")
		fmt.Println(string(json))
		os.Exit(0)
	}
	// Tune concurrency.
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU() + 1)
	}
	// Read config file or exit with error.
	config, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, "ERROR:", err)
		os.Exit(3)
	}
	err = yaml.Unmarshal(config, &checks)
	if err != nil {
		fmt.Fprintln(os.Stderr, "ERROR:", config[0], err)
		os.Exit(3)
	}
	// Exit with return code 0 on kill.
	done := make(chan os.Signal, 1)
	signal.Notify(done, syscall.SIGTERM)
	go func() {
		<-done
		os.Exit(0)
	}()
	// Run checks.
	etag()
	for i := range checks {
		go worker(&checks[i])
	}
	// Launch the JSON API.
	host := os.Getenv("HOST")
	port := os.Getenv("PORT")
	if host == "" {
		host = "localhost"
	}
	if port == "" {
		port = "3000"
	}
	http.HandleFunc("/version", getVersion)
	http.HandleFunc("/", getChecks)
	err = http.ListenAndServe(host+":"+port, nil)
	if err != nil {
		fmt.Fprintln(os.Stderr, "ERROR:", err)
	}
	os.Exit(4)
}

// Background worker.
func worker(check *Check) {
	for {
		if check.Repeat == 0 { // Set default timeout.
			check.Repeat = 60
		}
		if check.Tries == 0 { // Default to 1 attempt.
			check.Tries = 1
		}
		if check.Web != "" {
			web(check)
		}
		if check.Shell != "" {
			shell(check)
		}
		time.Sleep(time.Second * time.Duration(check.Repeat))
	}
}

// Shell worker.
func shell(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Shell
	}
	// Execute with shell in N attemps.
	var out []byte
	var err error
	for i := 0; i < check.Tries; i++ {
		out, err = exec.Command("/bin/sh", "-c", check.Shell).CombinedOutput()
		if err == nil {
			if check.Match != "" { // Match regexp.
				var regex *regexp.Regexp
				regex, err = regexp.Compile(check.Match)
				if err == nil && !regex.Match(out) {
					err = errors.New("ERROR: output did not match " + check.Match)
				}
			}
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			check.Failed = false
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)
		}
	} else {
		if !check.Failed {
			check.Failed = true
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			msg := string(out) + err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// Web worker.
func web(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Web
	}
	if check.Return == 0 { // Successful HTTP return code is 200.
		check.Return = 200
	}
	// Get the URL in N attempts.
	var err error
	for i := 0; i < check.Tries; i++ {
		err = fetch(check.Web, check.Match, check.Return)
		if err == nil {
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			check.Failed = false
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)

		}
	} else {
		if !check.Failed {
			check.Failed = true
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			msg := err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// The actual HTTP GET.
func fetch(url string, match string, code int) error {
	var err error
	var resp *http.Response
	resp, err = http.Get(url)
	if err == nil {
		if resp.StatusCode != code { // Check status code.
			err = errors.New(url + " returned " + strconv.Itoa(resp.StatusCode))
		} else { // Match regexp.
			if resp != nil && match != "" {
				var regex *regexp.Regexp
				regex, err = regexp.Compile(match)
				if err == nil {
					var body []byte
					body, _ = ioutil.ReadAll(resp.Body)
					if !regex.Match(body) {
						err = errors.New(url + " output did not match " + match)
					}
				}
			}
		}
	}
	if resp != nil {
		resp.Body.Close()
	}
	return err
}

// Logs and mail alerting.
func notify(mail interface{}, subject string, message *string) {
	// Log the alerts.
	if message == nil {
		fmt.Println(subject)
	} else {
		fmt.Println(subject + "\n" + *message)
	}
	// Mail the alerts.
	if mail != nil {
		// Make the message.
		var rcpt string
		var ok bool
		if rcpt, ok = mail.(string); !ok {
			for i, v := range mail.([]interface{}) {
				if i != 0 {
					rcpt += ", "
				}
				rcpt += v.(string)
			}
		}
		msg := "To: " + rcpt + "\nSubject: " + subject + "\n\n"
		if message != nil {
			msg += *message
		}
		msg += "\n.\n"
		// And send it.
		sendmail := exec.Command("/usr/sbin/sendmail", "-t")
		stdin, _ := sendmail.StdinPipe()
		err := sendmail.Start()
		if err != nil {
			fmt.Fprintln(os.Stderr, "ERROR:", err)
		}
		io.WriteString(stdin, msg)
		sendmail.Wait()
	}
}

// Executes callback. Passes args: true/false, check's name, message.
func alert(check *Check, name *string, msg *string) {
	if check.Alert != nil {
		plugin, ok := check.Alert.(string)
		if ok { // check.Alert is a string.
			out, err := exec.Command(plugin, strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
			if err != nil {
				fmt.Fprintln(os.Stderr, "ERROR:", string(out)+err.Error())
			}
		} else { // check.Alert is a list.
			for _, i := range check.Alert.([]interface{}) {
				out, err := exec.Command(i.(string), strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
				if err != nil {
					fmt.Fprintln(os.Stderr, "ERROR:", string(out)+err.Error())
				}
			}
		}
	}
}

// Display checks' details.
func getChecks(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" { // Serve for root page only, 404 otherwise.
		w.Header().Set("Server", "jsonmon")
		http.NotFound(w, r)
		return
	}
	displayJSON(w, r, &checks)
}

// Display application version.
func getVersion(w http.ResponseWriter, r *http.Request) {
	displayJSON(w, r, &version)
}

// Output JSON.
func displayJSON(w http.ResponseWriter, r *http.Request, data interface{}) {
	h := w.Header()
	h.Set("Server", "jsonmon")
	if r.Header.Get("If-None-Match") == modified {
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(http.StatusNotModified)
	} else {
		h.Set("Cache-Control", "no-cache")
		h.Set("ETag", modified)
		h.Set("Access-Control-Allow-Origin", "*")
		h.Set("Content-Type", "application/json; charset=utf-8")
		json, _ := json.MarshalIndent(&data, "", "  ")
		w.Write(json)
		fmt.Fprintln(w, "") // Trailing newline.
	}
}
<commit_msg>Set GOMAXPROCS to CPU count (as Go 1.5 does)<commit_after>/*
Quick and simple monitoring system

Usage:
 jsonmon config.yml
 jsonmon -v # Prints version to stdout and exits

Environment:
 HOST
  - defaults to localhost
  - the JSON API network interface
 PORT
  - defaults to 3000
  - the JSON API port
*/
package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"gopkg.in/yaml.v2"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"syscall"
	"time"
)

// Application version.
const Version = "1.2.2"

// This one is for internal use.
type ver struct {
	App  string `json:"jsonmon"`
	Go   string `json:"runtime"`
	Os   string `json:"os"`
	Arch string `json:"arch"`
}

var version ver

// Check details.
type Check struct {
	Name   string      `json:"name,omitempty" yaml:"name"`
	Web    string      `json:"web,omitempty" yaml:"web"`
	Shell  string      `json:"shell,omitempty" yaml:"shell"`
	Match  string      `json:"-" yaml:"match"`
	Return int         `json:"-" yaml:"return"`
	Notify interface{} `json:"-" yaml:"notify"`
	Alert  interface{} `json:"-", yaml:"alert`
	Tries  int         `json:"-" yaml:"tries"`
	Repeat int         `json:"-" yaml:"repeat"`
	Failed bool        `json:"failed" yaml:"-"`
	Since  string      `json:"since,omitempty" yaml:"-"`
}

// Global checks list. Need to share it with workers and Web UI.
var checks []Check

// Global last modified date for HTTP caching.
var modified string

// Construct the last modified string.
func etag() {
	modified = "W/\"" + strconv.FormatInt(time.Now().UnixNano(), 10) + "\""
}

// The main loop.
func main() {
	// Parse CLI args.
	usage := "Usage: " + path.Base(os.Args[0]) + " config.yml\n" +
		"Docs:  https://github.com/chillum/jsonmon/wiki"
	if len(os.Args) != 2 {
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(1)
	}
	// -v for version.
	version.App = Version
	version.Go = runtime.Version()
	version.Os = runtime.GOOS
	version.Arch = runtime.GOARCH
	switch os.Args[1] {
	case "-h":
		fallthrough
	case "-help":
		fallthrough
	case "--help":
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(0)
	case "-v":
		fallthrough
	case "-version":
		fallthrough
	case "--version":
		json, _ := json.MarshalIndent(&version, "", "  ")
		fmt.Println(string(json))
		os.Exit(0)
	}
	// Tune concurrency.
	if os.Getenv("GOMAXPROCS") == "" {
		runtime.GOMAXPROCS(runtime.NumCPU())
	}
	// Read config file or exit with error.
	config, err := ioutil.ReadFile(os.Args[1])
	if err != nil {
		fmt.Fprintln(os.Stderr, "ERROR:", err)
		os.Exit(3)
	}
	err = yaml.Unmarshal(config, &checks)
	if err != nil {
		fmt.Fprintln(os.Stderr, "ERROR:", config[0], err)
		os.Exit(3)
	}
	// Exit with return code 0 on kill.
	done := make(chan os.Signal, 1)
	signal.Notify(done, syscall.SIGTERM)
	go func() {
		<-done
		os.Exit(0)
	}()
	// Run checks.
	etag()
	for i := range checks {
		go worker(&checks[i])
	}
	// Launch the JSON API.
	host := os.Getenv("HOST")
	port := os.Getenv("PORT")
	if host == "" {
		host = "localhost"
	}
	if port == "" {
		port = "3000"
	}
	http.HandleFunc("/version", getVersion)
	http.HandleFunc("/", getChecks)
	err = http.ListenAndServe(host+":"+port, nil)
	if err != nil {
		fmt.Fprintln(os.Stderr, "ERROR:", err)
	}
	os.Exit(4)
}

// Background worker.
func worker(check *Check) {
	for {
		if check.Repeat == 0 { // Set default timeout.
			check.Repeat = 60
		}
		if check.Tries == 0 { // Default to 1 attempt.
			check.Tries = 1
		}
		if check.Web != "" {
			web(check)
		}
		if check.Shell != "" {
			shell(check)
		}
		time.Sleep(time.Second * time.Duration(check.Repeat))
	}
}

// Shell worker.
func shell(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Shell
	}
	// Execute with shell in N attemps.
	var out []byte
	var err error
	for i := 0; i < check.Tries; i++ {
		out, err = exec.Command("/bin/sh", "-c", check.Shell).CombinedOutput()
		if err == nil {
			if check.Match != "" { // Match regexp.
				var regex *regexp.Regexp
				regex, err = regexp.Compile(check.Match)
				if err == nil && !regex.Match(out) {
					err = errors.New("ERROR: output did not match " + check.Match)
				}
			}
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			check.Failed = false
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)
		}
	} else {
		if !check.Failed {
			check.Failed = true
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			msg := string(out) + err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// Web worker.
func web(check *Check) {
	// Set check's display name.
	var name string
	if check.Name != "" {
		name = check.Name
	} else {
		name = check.Web
	}
	if check.Return == 0 { // Successful HTTP return code is 200.
		check.Return = 200
	}
	// Get the URL in N attempts.
	var err error
	for i := 0; i < check.Tries; i++ {
		err = fetch(check.Web, check.Match, check.Return)
		if err == nil {
			break
		}
	}
	// Process results.
	if err == nil {
		if check.Failed {
			check.Failed = false
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			notify(check.Notify, "Fixed: "+name, nil)
			alert(check, &name, nil)

		}
	} else {
		if !check.Failed {
			check.Failed = true
			check.Since = time.Now().Format(time.RFC3339)
			etag()
			msg := err.Error()
			notify(check.Notify, "Failed: "+name, &msg)
			alert(check, &name, &msg)
		}
	}
}

// The actual HTTP GET.
func fetch(url string, match string, code int) error {
	var err error
	var resp *http.Response
	resp, err = http.Get(url)
	if err == nil {
		if resp.StatusCode != code { // Check status code.
			err = errors.New(url + " returned " + strconv.Itoa(resp.StatusCode))
		} else { // Match regexp.
			if resp != nil && match != "" {
				var regex *regexp.Regexp
				regex, err = regexp.Compile(match)
				if err == nil {
					var body []byte
					body, _ = ioutil.ReadAll(resp.Body)
					if !regex.Match(body) {
						err = errors.New(url + " output did not match " + match)
					}
				}
			}
		}
	}
	if resp != nil {
		resp.Body.Close()
	}
	return err
}

// Logs and mail alerting.
func notify(mail interface{}, subject string, message *string) {
	// Log the alerts.
	if message == nil {
		fmt.Println(subject)
	} else {
		fmt.Println(subject + "\n" + *message)
	}
	// Mail the alerts.
	if mail != nil {
		// Make the message.
		var rcpt string
		var ok bool
		if rcpt, ok = mail.(string); !ok {
			for i, v := range mail.([]interface{}) {
				if i != 0 {
					rcpt += ", "
				}
				rcpt += v.(string)
			}
		}
		msg := "To: " + rcpt + "\nSubject: " + subject + "\n\n"
		if message != nil {
			msg += *message
		}
		msg += "\n.\n"
		// And send it.
		sendmail := exec.Command("/usr/sbin/sendmail", "-t")
		stdin, _ := sendmail.StdinPipe()
		err := sendmail.Start()
		if err != nil {
			fmt.Fprintln(os.Stderr, "ERROR:", err)
		}
		io.WriteString(stdin, msg)
		sendmail.Wait()
	}
}

// Executes callback. Passes args: true/false, check's name, message.
func alert(check *Check, name *string, msg *string) {
	if check.Alert != nil {
		plugin, ok := check.Alert.(string)
		if ok { // check.Alert is a string.
			out, err := exec.Command(plugin, strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
			if err != nil {
				fmt.Fprintln(os.Stderr, "ERROR:", string(out)+err.Error())
			}
		} else { // check.Alert is a list.
			for _, i := range check.Alert.([]interface{}) {
				out, err := exec.Command(i.(string), strconv.FormatBool(check.Failed), *name, *msg).CombinedOutput()
				if err != nil {
					fmt.Fprintln(os.Stderr, "ERROR:", string(out)+err.Error())
				}
			}
		}
	}
}

// Display checks' details.
func getChecks(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" { // Serve for root page only, 404 otherwise.
		w.Header().Set("Server", "jsonmon")
		http.NotFound(w, r)
		return
	}
	displayJSON(w, r, &checks)
}

// Display application version.
func getVersion(w http.ResponseWriter, r *http.Request) {
	displayJSON(w, r, &version)
}

// Output JSON.
func displayJSON(w http.ResponseWriter, r *http.Request, data interface{}) {
	h := w.Header()
	h.Set("Server", "jsonmon")
	if r.Header.Get("If-None-Match") == modified {
		delete(h, "Content-Type")
		delete(h, "Content-Length")
		w.WriteHeader(http.StatusNotModified)
	} else {
		h.Set("Cache-Control", "no-cache")
		h.Set("ETag", modified)
		h.Set("Access-Control-Allow-Origin", "*")
		h.Set("Content-Type", "application/json; charset=utf-8")
		json, _ := json.MarshalIndent(&data, "", "  ")
		w.Write(json)
		fmt.Fprintln(w, "") // Trailing newline.
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Takbok, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package demo

/* import packages we're using */
import (
	"bytes"
	"encoding/csv"
	"encoding/xml"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"sort"
	"strings"
	"time"

	"appengine"

	newappengine "google.golang.org/appengine"
)

const feedUrl = `https://www.google.com/m8/feeds/contacts/%s/full?v=3.0`

var inp_file multipart.File

var exampleEntry = `
  <atom:entry xmlns:atom='http://www.w3.org/2005/Atom'
    xmlns:gd='http://schemas.google.com/g/2005'>
  <atom:category scheme='http://schemas.google.com/g/2005#kind'
    term='http://schemas.google.com/contact/2008#contact' />
  <gd:name>
     <gd:givenName>My External Contact 1</gd:givenName>
     <gd:familyName>My External Contact 1 family name</gd:familyName>
     <gd:fullName>My External Contact 1</gd:fullName>
  </gd:name>
  <atom:content type='text'>Notes</atom:content>
  <gd:email rel='http://schemas.google.com/g/2005#work'
    primary='true'
    address='myexternalcontact1@testmail.com' displayName='E. Bennet' />
  <gd:email rel='http://schemas.google.com/g/2005#home'
    address='myexternalcontact1@testmail.com' />
  <gd:phoneNumber rel='http://schemas.google.com/g/2005#work'
    primary='true'>
    (206)555-1212
  </gd:phoneNumber>
  <gd:phoneNumber rel='http://schemas.google.com/g/2005#home'>
    (111)111-1111
  </gd:phoneNumber>
  <gd:im address='myexternalcontact1@testmail.com'
    protocol='http://schemas.google.com/g/2005#GOOGLE_TALK'
    primary='true'
    rel='http://schemas.google.com/g/2005#home' />
  <gd:structuredPostalAddress
      rel='http://schemas.google.com/g/2005#work'
      primary='true'>
    <gd:city>testouser</gd:city>
    <gd:street>1600 Amphitheatre Pkwy</gd:street>
    <gd:region>CA</gd:region>
    <gd:postcode>94043</gd:postcode>
    <gd:country>United States</gd:country>
    <gd:formattedAddress>
      1600 Amphitheatre Pkwy Mountain View
    </gd:formattedAddress>
  </gd:structuredPostalAddress>
</atom:entry>`

// xmlns:gd=http://schemas.google.com/g/2005

type Feed struct {
	XMLName xml.Name  `xml:"http://www.w3.org/2005/Atom feed"`
	Title   string    `xml:"title"`
	Id      string    `xml:"id"`
	Link    []Link    `xml:"link"`
	Updated time.Time `xml:"updated,attr"`
	Author  Person    `xml:"author"`
	Entry   []Entry   `xml:"entry"`
}

type Entry struct {
	Title                   string                      `xml:"title"`
	Id                      string                      `xml:"id"`
	Link                    []Link                      `xml:"link"`
	Updated                 time.Time                   `xml:"updated" datastore:",noindex"`
	Author                  Person                      `xml:"author"`
	Summary                 Text                        `xml:"summary"`
	Name                    GDName                      `xml:"http://schemas.google.com/g/2005 name"`
	Im                      []GDIm                      `xml:"http://schemas.google.com/g/2005 im"`
	Email                   []GDEmail                   `xml:"http://schemas.google.com/g/2005 email"`
	PhoneNumber             []GDPhoneNumber             `xml:"http://schemas.google.com/g/2005 phoneNumber"`
	StructuredPostalAddress []GDStructuredPostalAddress `xml:"http://schemas.google.com/g/2005 formattedAddress"`
	ExtendedProperty        []GDExtendedProperty        `xml:"http://schemas.google.com/g/2005 extendedProperty"`
}

type EntryArb struct {
	Field []*Field
}

type Field struct {
	Name  string
	Value string
}

type Link struct {
	Rel  string `xml:"rel,attr,omitempty"`
	Href string `xml:"href,attr"`
}

type Person struct {
	Name     string `xml:"name"`
	URI      string `xml:"uri"`
	Email    string `xml:"email"`
	InnerXML string `xml:",innerxml"`
}

type Text struct {
	Type string `xml:"type,attr,omitempty"`
	Body string `xml:",chardata"`
}

type GDName struct {
	FullName   string `xml:"http://schemas.google.com/g/2005 fullName"`
	GivenName  string `xml:"http://schemas.google.com/g/2005 givenName"`
	FamilyName string `xml:"http://schemas.google.com/g/2005 familyName"`
}

type GDIm struct {
	Address  string `xml:"address,attr"`
	Protocol string `xml:"protocol,attr"`
	Primary  bool   `xml:"primary,attr"`
}

type GDEmail struct {
	Address string `xml:"address,attr"`
	Primary bool   `xml:"primary,attr"`
}

type GDPhoneNumber struct {
	PhoneNumber string `xml:",chardata"`
	Primary     bool   `xml:"primary,attr"`
}

type GDStructuredPostalAddress struct {
	Primary          bool   `xml:"primary,attr"`
	City             string `xml:"city"`
	Street           string `xml:"street"`
	Region           string `xml:"region"`
	Postcode         string `xml:"postcode"`
	Country          string `xml:"country"`
	FormattedAddress string `xml:"formattedAddress"`
}

type GDExtendedProperty struct {
	Name  string `xml:"name,attr"`
	Value string `xml:"value,attr"`
}

type ImportData struct {
	Data []byte
}

func writeCSV(ctx appengine.Context, w http.ResponseWriter, data []byte) {
	var feed Feed
	if err := xml.Unmarshal(data, &feed); err != nil {
		ctx.Errorf("unmarshal feed: %v", err)
		return
	}

	contacts, names := [][]string{}, []string{}

	upboundEmail, upboundPhone, upboundIm := 0, 0, 0
	propNames := []string{}
	names = append(names, "Id", "FullName", "GivenName", "FamilyName")
	for _, entry := range feed.Entry {
		if n := len(entry.Email); upboundEmail < n {
			upboundEmail = n
		}
		if n := len(entry.PhoneNumber); upboundPhone < n {
			upboundPhone = n
		}
		if n := len(entry.Im); upboundIm < n {
			upboundIm = n
		}
		for _, p := range entry.ExtendedProperty {
			existed := false
			for _, s := range propNames {
				if s == p.Name {
					existed = true
					break
				}
			}
			if !existed {
				propNames = append(propNames, p.Name)
			}
		}
	}
	for n := 1; n <= upboundEmail; n++ {
		names = append(names, fmt.Sprintf("Email %v", n))
	}
	for n := 1; n <= upboundPhone; n++ {
		names = append(names, fmt.Sprintf("Phone %v", n))
	}
	for n := 1; n <= upboundIm; n++ {
		names = append(names, fmt.Sprintf("Im %v", n))
	}
	propStart := len(names)
	sort.Strings(propNames[0:])
	names = append(names, propNames...)
	contacts = append(contacts, names)

	for _, entry := range feed.Entry {
		values := []string{
			entry.Id,
			entry.Name.FullName,
			entry.Name.GivenName,
			entry.Name.FamilyName,
		}
		for n, m := 0, len(entry.Email); n < upboundEmail; n++ {
			s := ""
			if n < m {
				s = entry.Email[n].Address
			}
			values = append(values, strings.TrimSpace(s))
		}
		for n, m := 0, len(entry.PhoneNumber); n < upboundPhone; n++ {
			s := ""
			if n < m {
				s = entry.PhoneNumber[n].PhoneNumber
			}
			values = append(values, strings.TrimSpace(s))
		}
		for n, m := 0, len(entry.Im); n < upboundIm; n++ {
			s := ""
			if n < m {
				s = entry.Im[n].Address
			}
			values = append(values, strings.TrimSpace(s))
		}
		for n, m := 0, len(propNames); n < m; n++ {
			values = append(values, "")
		}
		for n, m, x := 0, len(entry.ExtendedProperty), len(propNames); n < m; n++ {
			ep := &entry.ExtendedProperty[n]
			for i := 0; i < x; i++ {
				if ss := propNames[i]; ss == ep.Name {
					values[propStart+i] = strings.TrimSpace(ep.Value)
					break
				}
			}
		}
		contacts = append(contacts, values)
	}

	out := csv.NewWriter(w)
	out.WriteAll(contacts)
	if err := out.Error(); err != nil {
		ctx.Errorf("error writing csv:", err)
	}
}

func loadFullFeed(domain string, ctx appengine.Context, r *http.Request) (buf *bytes.Buffer) {
	newctx := newappengine.NewContext(r)

	tok, err := config.Exchange(newctx, r.FormValue("code"))
	if err != nil {
		ctx.Errorf("exchange error: %v", err)
		return
	}

	ctx.Infof("tok: %v", tok)

	client := config.Client(newctx, tok)

	res, err := client.Get(fmt.Sprintf(feedUrl, domain))
	if err != nil {
		ctx.Errorf("get: %v", err)
		return
	}

	defer res.Body.Close()

	buf = new(bytes.Buffer)
	io.Copy(buf, res.Body)
	return
}

// https://developers.google.com/admin-sdk/domain-shared-contacts
/*
<feed xmlns='http://www.w3.org/2005/Atom'
    xmlns:openSearch='http://a9.com/-/spec/opensearchrss/1.0/'
    xmlns:gd='http://schemas.google.com/g/2005'
    xmlns:gContact='http://schemas.google.com/contact/2008'
    xmlns:batch='http://schemas.google.com/gdata/batch'>
  <id>https://www.google.com/m8/feeds/contacts/example.com/base</id>
  <updated>2008-03-05T12:36:38.836Z</updated>
  <category scheme='http://schemas.google.com/g/2005#kind'
    term='http://schemas.google.com/contact/2008#contact' />
  <title type='text'>example.com's Contacts</title>
  <link rel='http://schemas.google.com/g/2005#feed'
    type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full' />
  <link rel='http://schemas.google.com/g/2005#post'
    type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full' />
  <link rel='http://schemas.google.com/g/2005#batch'
    type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full/batch' />
  <link rel='self' type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full?max-results=25' />
  <author>
    <name>example.com</name>
    <email>example.com</email>
  </author>
  <generator version='1.0' uri='https://www.google.com/m8/feeds/contacts'>
    Contacts
  </generator>
  <openSearch:totalResults>1</openSearch:totalResults>
  <openSearch:startIndex>1</openSearch:startIndex>
  <openSearch:itemsPerPage>25</openSearch:itemsPerPage>
  <entry>
    <id>
      https://www.google.com/m8/feeds/contacts/example.com/base/c9012de
    </id>
    <updated>2008-03-05T12:36:38.835Z</updated>
    <category scheme='http://schemas.google.com/g/2005#kind'
      term='http://schemas.google.com/contact/2008#contact' />
    <title type='text'>Fitzgerald</title>
    <gd:name>
      <gd:fullName>Fitzgerald</gd:fullName>
    </gd:name>
    <link rel="http://schemas.google.com/contacts/2008/rel#photo" type="image/*"
      href="http://google.com/m8/feeds/photos/media/example.com/c9012de"/>
    <link rel='self' type='application/atom+xml'
      href='https://www.google.com/m8/feeds/contacts/example.com/full/c9012de' />
    <link rel='edit' type='application/atom+xml'
      href='https://www.google.com/m8/feeds/contacts/example.com/full/c9012de/1204720598835000' />
    <gd:phoneNumber rel='http://schemas.google.com/g/2005#home'
      primary='true'>
      456
    </gd:phoneNumber>
    <gd:extendedProperty name="pet" value="hamster" />
  </entry>
</feed>
*/
<commit_msg>fix for issue#7 add query 'max-results=50000' in export feed url to get all contact<commit_after>// Copyright 2016 Takbok, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package demo

/* import packages we're using */
import (
	"bytes"
	"encoding/csv"
	"encoding/xml"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"sort"
	"strings"
	"time"

	"appengine"

	newappengine "google.golang.org/appengine"
)

const feedUrl = `https://www.google.com/m8/feeds/contacts/%s/full?v=3.0`

var inp_file multipart.File

var exampleEntry = `
  <atom:entry xmlns:atom='http://www.w3.org/2005/Atom'
    xmlns:gd='http://schemas.google.com/g/2005'>
  <atom:category scheme='http://schemas.google.com/g/2005#kind'
    term='http://schemas.google.com/contact/2008#contact' />
  <gd:name>
     <gd:givenName>My External Contact 1</gd:givenName>
     <gd:familyName>My External Contact 1 family name</gd:familyName>
     <gd:fullName>My External Contact 1</gd:fullName>
  </gd:name>
  <atom:content type='text'>Notes</atom:content>
  <gd:email rel='http://schemas.google.com/g/2005#work'
    primary='true'
    address='myexternalcontact1@testmail.com' displayName='E. Bennet' />
  <gd:email rel='http://schemas.google.com/g/2005#home'
    address='myexternalcontact1@testmail.com' />
  <gd:phoneNumber rel='http://schemas.google.com/g/2005#work'
    primary='true'>
    (206)555-1212
  </gd:phoneNumber>
  <gd:phoneNumber rel='http://schemas.google.com/g/2005#home'>
    (111)111-1111
  </gd:phoneNumber>
  <gd:im address='myexternalcontact1@testmail.com'
    protocol='http://schemas.google.com/g/2005#GOOGLE_TALK'
    primary='true'
    rel='http://schemas.google.com/g/2005#home' />
  <gd:structuredPostalAddress
      rel='http://schemas.google.com/g/2005#work'
      primary='true'>
    <gd:city>testouser</gd:city>
    <gd:street>1600 Amphitheatre Pkwy</gd:street>
    <gd:region>CA</gd:region>
    <gd:postcode>94043</gd:postcode>
    <gd:country>United States</gd:country>
    <gd:formattedAddress>
      1600 Amphitheatre Pkwy Mountain View
    </gd:formattedAddress>
  </gd:structuredPostalAddress>
</atom:entry>`

// xmlns:gd=http://schemas.google.com/g/2005

type Feed struct {
	XMLName xml.Name  `xml:"http://www.w3.org/2005/Atom feed"`
	Title   string    `xml:"title"`
	Id      string    `xml:"id"`
	Link    []Link    `xml:"link"`
	Updated time.Time `xml:"updated,attr"`
	Author  Person    `xml:"author"`
	Entry   []Entry   `xml:"entry"`
}

type Entry struct {
	Title                   string                      `xml:"title"`
	Id                      string                      `xml:"id"`
	Link                    []Link                      `xml:"link"`
	Updated                 time.Time                   `xml:"updated" datastore:",noindex"`
	Author                  Person                      `xml:"author"`
	Summary                 Text                        `xml:"summary"`
	Name                    GDName                      `xml:"http://schemas.google.com/g/2005 name"`
	Im                      []GDIm                      `xml:"http://schemas.google.com/g/2005 im"`
	Email                   []GDEmail                   `xml:"http://schemas.google.com/g/2005 email"`
	PhoneNumber             []GDPhoneNumber             `xml:"http://schemas.google.com/g/2005 phoneNumber"`
	StructuredPostalAddress []GDStructuredPostalAddress `xml:"http://schemas.google.com/g/2005 formattedAddress"`
	ExtendedProperty        []GDExtendedProperty        `xml:"http://schemas.google.com/g/2005 extendedProperty"`
}

type EntryArb struct {
	Field []*Field
}

type Field struct {
	Name  string
	Value string
}

type Link struct {
	Rel  string `xml:"rel,attr,omitempty"`
	Href string `xml:"href,attr"`
}

type Person struct {
	Name     string `xml:"name"`
	URI      string `xml:"uri"`
	Email    string `xml:"email"`
	InnerXML string `xml:",innerxml"`
}

type Text struct {
	Type string `xml:"type,attr,omitempty"`
	Body string `xml:",chardata"`
}

type GDName struct {
	FullName   string `xml:"http://schemas.google.com/g/2005 fullName"`
	GivenName  string `xml:"http://schemas.google.com/g/2005 givenName"`
	FamilyName string `xml:"http://schemas.google.com/g/2005 familyName"`
}

type GDIm struct {
	Address  string `xml:"address,attr"`
	Protocol string `xml:"protocol,attr"`
	Primary  bool   `xml:"primary,attr"`
}

type GDEmail struct {
	Address string `xml:"address,attr"`
	Primary bool   `xml:"primary,attr"`
}

type GDPhoneNumber struct {
	PhoneNumber string `xml:",chardata"`
	Primary     bool   `xml:"primary,attr"`
}

type GDStructuredPostalAddress struct {
	Primary          bool   `xml:"primary,attr"`
	City             string `xml:"city"`
	Street           string `xml:"street"`
	Region           string `xml:"region"`
	Postcode         string `xml:"postcode"`
	Country          string `xml:"country"`
	FormattedAddress string `xml:"formattedAddress"`
}

type GDExtendedProperty struct {
	Name  string `xml:"name,attr"`
	Value string `xml:"value,attr"`
}

type ImportData struct {
	Data []byte
}

func writeCSV(ctx appengine.Context, w http.ResponseWriter, data []byte) {
	var feed Feed
	if err := xml.Unmarshal(data, &feed); err != nil {
		ctx.Errorf("unmarshal feed: %v", err)
		return
	}

	contacts, names := [][]string{}, []string{}

	upboundEmail, upboundPhone, upboundIm := 0, 0, 0
	propNames := []string{}
	names = append(names, "Id", "FullName", "GivenName", "FamilyName")
	for _, entry := range feed.Entry {
		if n := len(entry.Email); upboundEmail < n {
			upboundEmail = n
		}
		if n := len(entry.PhoneNumber); upboundPhone < n {
			upboundPhone = n
		}
		if n := len(entry.Im); upboundIm < n {
			upboundIm = n
		}
		for _, p := range entry.ExtendedProperty {
			existed := false
			for _, s := range propNames {
				if s == p.Name {
					existed = true
					break
				}
			}
			if !existed {
				propNames = append(propNames, p.Name)
			}
		}
	}
	for n := 1; n <= upboundEmail; n++ {
		names = append(names, fmt.Sprintf("Email %v", n))
	}
	for n := 1; n <= upboundPhone; n++ {
		names = append(names, fmt.Sprintf("Phone %v", n))
	}
	for n := 1; n <= upboundIm; n++ {
		names = append(names, fmt.Sprintf("Im %v", n))
	}
	propStart := len(names)
	sort.Strings(propNames[0:])
	names = append(names, propNames...)
	contacts = append(contacts, names)

	for _, entry := range feed.Entry {
		values := []string{
			entry.Id,
			entry.Name.FullName,
			entry.Name.GivenName,
			entry.Name.FamilyName,
		}
		for n, m := 0, len(entry.Email); n < upboundEmail; n++ {
			s := ""
			if n < m {
				s = entry.Email[n].Address
			}
			values = append(values, strings.TrimSpace(s))
		}
		for n, m := 0, len(entry.PhoneNumber); n < upboundPhone; n++ {
			s := ""
			if n < m {
				s = entry.PhoneNumber[n].PhoneNumber
			}
			values = append(values, strings.TrimSpace(s))
		}
		for n, m := 0, len(entry.Im); n < upboundIm; n++ {
			s := ""
			if n < m {
				s = entry.Im[n].Address
			}
			values = append(values, strings.TrimSpace(s))
		}
		for n, m := 0, len(propNames); n < m; n++ {
			values = append(values, "")
		}
		for n, m, x := 0, len(entry.ExtendedProperty), len(propNames); n < m; n++ {
			ep := &entry.ExtendedProperty[n]
			for i := 0; i < x; i++ {
				if ss := propNames[i]; ss == ep.Name {
					values[propStart+i] = strings.TrimSpace(ep.Value)
					break
				}
			}
		}
		contacts = append(contacts, values)
	}

	out := csv.NewWriter(w)
	out.WriteAll(contacts)
	if err := out.Error(); err != nil {
		ctx.Errorf("error writing csv:", err)
	}
}

func loadFullFeed(domain string, ctx appengine.Context, r *http.Request) (buf *bytes.Buffer) {
	newctx := newappengine.NewContext(r)

	tok, err := config.Exchange(newctx, r.FormValue("code"))
	if err != nil {
		ctx.Errorf("exchange error: %v", err)
		return
	}

	ctx.Infof("tok: %v", tok)

    client := config.Client(newctx, tok)
    expFeedUrl := fmt.Sprintf(feedUrl, domain)
    expFeedUrl = expFeedUrl + "&max-results=50000"
    res, err := client.Get(expFeedUrl)
	if err != nil {
		ctx.Errorf("get: %v", err)
		return
	}

	defer res.Body.Close()

	buf = new(bytes.Buffer)
	io.Copy(buf, res.Body)
	return
}

// https://developers.google.com/admin-sdk/domain-shared-contacts
/*
<feed xmlns='http://www.w3.org/2005/Atom'
    xmlns:openSearch='http://a9.com/-/spec/opensearchrss/1.0/'
    xmlns:gd='http://schemas.google.com/g/2005'
    xmlns:gContact='http://schemas.google.com/contact/2008'
    xmlns:batch='http://schemas.google.com/gdata/batch'>
  <id>https://www.google.com/m8/feeds/contacts/example.com/base</id>
  <updated>2008-03-05T12:36:38.836Z</updated>
  <category scheme='http://schemas.google.com/g/2005#kind'
    term='http://schemas.google.com/contact/2008#contact' />
  <title type='text'>example.com's Contacts</title>
  <link rel='http://schemas.google.com/g/2005#feed'
    type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full' />
  <link rel='http://schemas.google.com/g/2005#post'
    type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full' />
  <link rel='http://schemas.google.com/g/2005#batch'
    type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full/batch' />
  <link rel='self' type='application/atom+xml'
    href='https://www.google.com/m8/feeds/contacts/example.com/full?max-results=25' />
  <author>
    <name>example.com</name>
    <email>example.com</email>
  </author>
  <generator version='1.0' uri='https://www.google.com/m8/feeds/contacts'>
    Contacts
  </generator>
  <openSearch:totalResults>1</openSearch:totalResults>
  <openSearch:startIndex>1</openSearch:startIndex>
  <openSearch:itemsPerPage>25</openSearch:itemsPerPage>
  <entry>
    <id>
      https://www.google.com/m8/feeds/contacts/example.com/base/c9012de
    </id>
    <updated>2008-03-05T12:36:38.835Z</updated>
    <category scheme='http://schemas.google.com/g/2005#kind'
      term='http://schemas.google.com/contact/2008#contact' />
    <title type='text'>Fitzgerald</title>
    <gd:name>
      <gd:fullName>Fitzgerald</gd:fullName>
    </gd:name>
    <link rel="http://schemas.google.com/contacts/2008/rel#photo" type="image/*"
      href="http://google.com/m8/feeds/photos/media/example.com/c9012de"/>
    <link rel='self' type='application/atom+xml'
      href='https://www.google.com/m8/feeds/contacts/example.com/full/c9012de' />
    <link rel='edit' type='application/atom+xml'
      href='https://www.google.com/m8/feeds/contacts/example.com/full/c9012de/1204720598835000' />
    <gd:phoneNumber rel='http://schemas.google.com/g/2005#home'
      primary='true'>
      456
    </gd:phoneNumber>
    <gd:extendedProperty name="pet" value="hamster" />
  </entry>
</feed>
*/
<|endoftext|>


<commit_before>package ledger

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"
)

type Entry struct {
	Date        string // "Y-m-d"
	Description string
	Change      int // in cents
}

type Locale struct {
	Date              string
	Description       string
	Change            string
	DateSeperator     string
	DecimalPoint      string
	IntegralSeperator string
}

var locales = map[string]Locale{
	"en-US": {
		Date:              "Date",
		Description:       "Description",
		Change:            "Change",
		DateSeperator:     "/",
		DecimalPoint:      ".",
		IntegralSeperator: ",",
	},
	"nl-NL": {
		Date:              "Datum",
		Description:       "Omschrijving",
		Change:            "Verandering",
		DateSeperator:     "-",
		DecimalPoint:      ",",
		IntegralSeperator: ".",
	},
}

var currencyToSymbol = map[string]string{
	"USD": "$",
	"EUR": "€",
}

func FormatLedger(currency string, locale string, entries []Entry) (output string, e error) {
	if !isValidCurrency(currency) {
		return "", errors.New("invalid currency")
	}
	if !isValidLocale(locale) {
		return "", errors.New("invalid locale")
	}
	if !isValidDate(entries) {
		return "", errors.New("invalid date")
	}

	entriesCopy := append([]Entry{}, entries...)

	m1 := map[bool]int{true: 0, false: 1}
	m2 := map[bool]int{true: -1, false: 1}
	es := entriesCopy
	for len(es) > 1 {
		first, rest := es[0], es[1:]
		success := false
		for !success {
			success = true
			for i, e := range rest {
				if (m1[e.Date == first.Date]*m2[e.Date < first.Date]*4 +
					m1[e.Description == first.Description]*m2[e.Description < first.Description]*2 +
					m1[e.Change == first.Change]*m2[e.Change < first.Change]*1) < 0 {
					es[0], es[i+1] = es[i+1], es[0]
					success = false
				}
			}
		}
		es = es[1:]
	}

	output += header(locale)
	for _, entry := range entriesCopy {
		output += formatEntry(locale, currency, entry)
	}
	return output, nil
}

func formatEntry(locale string, currency string, entry Entry) (formatted string) {
	date := formatDate(locale, entry.Date)
	description := formatDescription(entry.Description)
	change := formatChange(locale, currency, entry.Change)

	// This conditional is necessary because expected output is aligned
	// differently for negative vs. non-negative values
	if isNegative(entry.Change) {
		return fmt.Sprintf("%-10s | %-25s | %13s\n", date, description, change)
	} else {
		return fmt.Sprintf("%-10s | %-25s | %12s\n", date, description, change)
	}
}

func header(locale string) (output string) {
	return fmt.Sprintf("%-10s | %-25s | %s\n", locales[locale].Date, locales[locale].Description, locales[locale].Change)
}

func formatDate(locale string, date string) string {
	year, month, day := date[0:4], date[5:7], date[8:10]

	if locale == "nl-NL" {
		return strings.Join([]string{day, month, year}, locales[locale].DateSeperator)
	} else if locale == "en-US" {
		return strings.Join([]string{month, day, year}, locales[locale].DateSeperator)
	}
	panic("invalid locale")
}

// formatDescription will ellipsize the description if it is longer than 25
// characters
func formatDescription(description string) string {
	if len(description) > 25 {
		return description[:22] + "..."
	}
	return description
}

func formatChange(locale string, currency string, cents int) (change string) {
	isNegative := isNegative(cents)
	paddedChange := fmt.Sprintf("%03s", strconv.Itoa(absoluteValue(cents)))
	change += formatCurrencySymbol(locale, currency)
	change += formatNumber(locale, paddedChange)
	if isNegative {
		change = formatNegative(locale, change)
	}
	return change
}

func formatNumber(locale string, paddedChange string) (formatted string) {
	integralPart := formatIntegralPart(paddedChange, locale)
	decimalPart := paddedChange[len(paddedChange)-2:]
	return strings.Join([]string{integralPart, decimalPart}, locales[locale].DecimalPoint)
}

func formatCurrencySymbol(locale string, currency string) (formatted string) {
	if locale == "en-US" {
		return currencyToSymbol[currency]
	} else if locale == "nl-NL" {
		return currencyToSymbol[currency] + " "
	}
	panic("invalid locale")
}

func formatNegative(locale string, change string) (formatted string) {
	if locale == "en-US" {
		return fmt.Sprintf("(%s)", change)
	} else if locale == "nl-NL" {
		return fmt.Sprintf("%s-", change)
	}
	panic("invalid locale")
}

func formatIntegralPart(centsStr string, locale string) (formatted string) {
	var parts []string
	remainder := centsStr[:len(centsStr)-2]
	for len(remainder) > 3 {
		parts = append([]string{remainder[len(remainder)-3:]}, parts...)
		remainder = remainder[:len(remainder)-3]
	}
	if len(remainder) > 0 {
		parts = append([]string{remainder}, parts...)
	}
	return strings.Join(parts, locales[locale].IntegralSeperator)
}

func isValidCurrency(currency string) bool {
	_, ok := currencyToSymbol[currency]
	return ok
}

func isValidLocale(locale string) bool {
	_, ok := locales[locale]
	return ok
}

func isValidDate(entries []Entry) bool {
	for _, entry := range entries {
		if len(entry.Date) != 10 {
			return false
		}
		_, d2, _, d4, _ := entry.Date[0:4], entry.Date[4], entry.Date[5:7], entry.Date[7], entry.Date[8:10]
		if d2 != '-' || d4 != '-' {
			return false
		}
	}
	return true
}

func isNegative(cents int) bool {
	return cents < 0
}

func absoluteValue(n int) int {
	return int(math.Abs(float64(n)))
}
<commit_msg>Refactor sorting<commit_after>package ledger

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"strconv"
	"strings"
)

type Entry struct {
	Date        string // "Y-m-d"
	Description string
	Change      int // in cents
}

type Locale struct {
	Date              string
	Description       string
	Change            string
	DateSeperator     string
	DecimalPoint      string
	IntegralSeperator string
}

var locales = map[string]Locale{
	"en-US": {
		Date:              "Date",
		Description:       "Description",
		Change:            "Change",
		DateSeperator:     "/",
		DecimalPoint:      ".",
		IntegralSeperator: ",",
	},
	"nl-NL": {
		Date:              "Datum",
		Description:       "Omschrijving",
		Change:            "Verandering",
		DateSeperator:     "-",
		DecimalPoint:      ",",
		IntegralSeperator: ".",
	},
}

var currencyToSymbol = map[string]string{
	"USD": "$",
	"EUR": "€",
}

func FormatLedger(currency string, locale string, entries []Entry) (output string, e error) {
	if !isValidCurrency(currency) {
		return "", errors.New("invalid currency")
	}
	if !isValidLocale(locale) {
		return "", errors.New("invalid locale")
	}
	if !isValidDate(entries) {
		return "", errors.New("invalid date")
	}

	entriesCopy := make([]Entry, len(entries))
	copy(entriesCopy, entries)
	sort.Slice(entriesCopy, func(i int, j int) bool {
		a := entries[i]
		b := entries[j]
		if a.Date != b.Date {
			return a.Date < b.Date
		} else if a.Description != b.Description {
			return a.Description < b.Description
		} else {
			return a.Change < b.Change
		}
	})

	output += header(locale)
	for _, entry := range entriesCopy {
		output += formatEntry(locale, currency, entry)
	}
	return output, nil
}

func formatEntry(locale string, currency string, entry Entry) (formatted string) {
	date := formatDate(locale, entry.Date)
	description := formatDescription(entry.Description)
	change := formatChange(locale, currency, entry.Change)

	// This conditional is necessary because expected output is aligned
	// differently for negative vs. non-negative values
	if isNegative(entry.Change) {
		return fmt.Sprintf("%-10s | %-25s | %13s\n", date, description, change)
	} else {
		return fmt.Sprintf("%-10s | %-25s | %12s\n", date, description, change)
	}
}

func header(locale string) (output string) {
	return fmt.Sprintf("%-10s | %-25s | %s\n", locales[locale].Date, locales[locale].Description, locales[locale].Change)
}

func formatDate(locale string, date string) string {
	year, month, day := date[0:4], date[5:7], date[8:10]

	if locale == "nl-NL" {
		return strings.Join([]string{day, month, year}, locales[locale].DateSeperator)
	} else if locale == "en-US" {
		return strings.Join([]string{month, day, year}, locales[locale].DateSeperator)
	}
	panic("invalid locale")
}

// formatDescription will ellipsize the description if it is longer than 25
// characters
func formatDescription(description string) string {
	if len(description) > 25 {
		return description[:22] + "..."
	}
	return description
}

func formatChange(locale string, currency string, cents int) (change string) {
	isNegative := isNegative(cents)
	paddedChange := fmt.Sprintf("%03s", strconv.Itoa(absoluteValue(cents)))
	change += formatCurrencySymbol(locale, currency)
	change += formatNumber(locale, paddedChange)
	if isNegative {
		change = formatNegative(locale, change)
	}
	return change
}

func formatNumber(locale string, paddedChange string) (formatted string) {
	integralPart := formatIntegralPart(paddedChange, locale)
	decimalPart := paddedChange[len(paddedChange)-2:]
	return strings.Join([]string{integralPart, decimalPart}, locales[locale].DecimalPoint)
}

func formatCurrencySymbol(locale string, currency string) (formatted string) {
	if locale == "en-US" {
		return currencyToSymbol[currency]
	} else if locale == "nl-NL" {
		return currencyToSymbol[currency] + " "
	}
	panic("invalid locale")
}

func formatNegative(locale string, change string) (formatted string) {
	if locale == "en-US" {
		return fmt.Sprintf("(%s)", change)
	} else if locale == "nl-NL" {
		return fmt.Sprintf("%s-", change)
	}
	panic("invalid locale")
}

func formatIntegralPart(centsStr string, locale string) (formatted string) {
	var parts []string
	remainder := centsStr[:len(centsStr)-2]
	for len(remainder) > 3 {
		parts = append([]string{remainder[len(remainder)-3:]}, parts...)
		remainder = remainder[:len(remainder)-3]
	}
	if len(remainder) > 0 {
		parts = append([]string{remainder}, parts...)
	}
	return strings.Join(parts, locales[locale].IntegralSeperator)
}

func isValidCurrency(currency string) bool {
	_, ok := currencyToSymbol[currency]
	return ok
}

func isValidLocale(locale string) bool {
	_, ok := locales[locale]
	return ok
}

func isValidDate(entries []Entry) bool {
	for _, entry := range entries {
		if len(entry.Date) != 10 {
			return false
		}
		_, d2, _, d4, _ := entry.Date[0:4], entry.Date[4], entry.Date[5:7], entry.Date[7], entry.Date[8:10]
		if d2 != '-' || d4 != '-' {
			return false
		}
	}
	return true
}

func isNegative(cents int) bool {
	return cents < 0
}

func absoluteValue(n int) int {
	return int(math.Abs(float64(n)))
}
<|endoftext|>


<commit_before>package matrix

import (
	"fmt"
	"strconv"
	"strings"
)

type Matrix struct {
	grid [][]int
}

func New(s string) (*Matrix, error) {
	if isUnevenRows(s) {
		return &Matrix{}, fmt.Errorf("uneven rows for %s", s)
	}
	rows := strings.Split(s, "\n")
	grid := [][]int{}
	for _, row := range rows {
		r := []int{}
		cols := strings.Split(strings.Trim(row, " "), " ")
		for _, col := range cols {
			val, err := strconv.Atoi(col)
			if err != nil {
				return &Matrix{}, err
			}
			r = append(r, val)
		}
		grid = append(grid, r)
	}
	return &Matrix{
		grid: grid,
	}, nil
}

func (m Matrix) Cols() [][]int {
	return [][]int{}
}

func (m Matrix) Rows() [][]int {
	return m.grid
}

func (m Matrix) Set(row int, column int, value int) bool {
	m.grid[row][column] = value
	return true
}

func isUnevenRows(s string) bool {
	rowLengths := map[int]bool{}
	rows := strings.Split(s, "\n")
	for _, row := range rows {
		cols := strings.Split(strings.Trim(row, " "), " ")
		rowLengths[len(cols)] = true
	}
	return len(rowLengths) != 1
}
<commit_msg>Fix Set impl<commit_after>package matrix

import (
	"fmt"
	"strconv"
	"strings"
)

type Matrix struct {
	grid [][]int
}

func New(s string) (*Matrix, error) {
	if isUnevenRows(s) {
		return &Matrix{}, fmt.Errorf("uneven rows for %s", s)
	}
	rows := strings.Split(s, "\n")
	grid := [][]int{}
	for _, row := range rows {
		r := []int{}
		cols := strings.Split(strings.Trim(row, " "), " ")
		for _, col := range cols {
			val, err := strconv.Atoi(col)
			if err != nil {
				return &Matrix{}, err
			}
			r = append(r, val)
		}
		grid = append(grid, r)
	}
	return &Matrix{
		grid: grid,
	}, nil
}

func (m Matrix) Cols() [][]int {
	result := initializeMatrix(len(m.grid[0]), len(m.grid))
	for x, row := range m.grid {
		for y, val := range row {
			result[y][x] = val
		}
	}
	return result
}

func (m Matrix) Rows() [][]int {
	return m.grid
}

func (m Matrix) Set(row int, column int, value int) bool {
	if row >= len(m.grid) || row < 0 {
		return false
	}
	if column >= len(m.grid[0]) || column < 0 {
		return false
	}
	m.grid[row][column] = value
	return true
}

func isUnevenRows(s string) bool {
	rowLengths := map[int]bool{}
	rows := strings.Split(s, "\n")
	for _, row := range rows {
		cols := strings.Split(strings.Trim(row, " "), " ")
		rowLengths[len(cols)] = true
	}
	return len(rowLengths) != 1
}

func initializeMatrix(dy int, dx int) [][]int {
	a := make([][]int, dy)
	for i := range a {
		a[i] = make([]int, dx)
	}
	return a
}
<|endoftext|>


<commit_before>type GoalConjunction struct {
	Goals []Goal

	choicepoints []choicepoint
}

type choicepoint struct {
	index   int
	barrier Barrier
}

func conjunction(goals ...Goal) Goal {
	return &GoalConjunction{goals}
}

func (self *GoalConjunction) Next(c Context) (bool, bool) {
	goalIndex := 0
	if len(choicepoints) > 0 {
		p := choicepoints[len(choicepoints)-1]
		self.choicepoints = self.choicepoints[0 : len(choicepoints)-1]
		goalIndex = p.index
		c.Trail().BacktrackTo(p.barrier)
	}

	for goalIndex < len(self.Goals) {
		goal := self.Goals[goalIndex]
		barrier := c.Trail().PushBarrier()
		ok, more := goal.Next(c)
		if ok {
			if more {
				// record a choicepoint
				p := &choicepoint{goalIndex, barrier}
				self.choicepoints = append(self.choicepoints, p)
			}
			goalIndex++
			continue
		} else {
			// TODO backtrack to previous choicepoint
		}
	}

	more := len(self.choicepoints) > 0
	if !more {
		for _, goal := range self.Goals {
			if x, ok := goal.(NeedsCleanup); ok {
				x.Cleanup()
			}
		}
	}
	return true, more
}
<commit_msg>drop barrier<commit_after>type GoalConjunction struct {
	Goals []Goal

	choicepoints []choicepoint
}

type choicepoint struct {
	index   int
	barrier Barrier
}

func conjunction(goals ...Goal) Goal {
	return &GoalConjunction{goals}
}

func (self *GoalConjunction) Next(c Context) (bool, bool) {
	goalIndex := 0
	if len(choicepoints) > 0 {
		p := choicepoints[len(choicepoints)-1]
		self.choicepoints = self.choicepoints[0 : len(choicepoints)-1]
		goalIndex = p.index
		c.Trail().BacktrackTo(p.barrier)
		c.Trail().DropBarrier(p.barrier)
	}

	for goalIndex < len(self.Goals) {
		goal := self.Goals[goalIndex]
		barrier := c.Trail().PushBarrier()
		ok, more := goal.Next(c)
		if ok {
			if more {
				// record a choicepoint
				p := &choicepoint{goalIndex, barrier}
				self.choicepoints = append(self.choicepoints, p)
			}
			goalIndex++
			continue
		} else {
			// TODO backtrack to previous choicepoint
		}
	}

	more := len(self.choicepoints) > 0
	if !more {
		for _, goal := range self.Goals {
			if x, ok := goal.(NeedsCleanup); ok {
				x.Cleanup()
			}
		}
	}
	return true, more
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"io/ioutil"
	"encoding/json"
	"strings"
	"github.com/line/line-bot-sdk-go/linebot"
	"github.com/go-redis/redis"
	"strconv"
)

type device struct {
	// Gps_num float32 `json:"gps_num"`
	// App string `json:"app"`
	// Gps_alt float32 `json:"gps_alt"`
	// Fmt_opt int `json:"fmt_opt"`
	// Device string `json:"device"`
	// S_d2 float32 `json:"s_d2"`
	S_d0 float32 `json:"s_d0"`
	// S_d1 float32 `json:"s_d1"`
	S_h0 float32 `json:"s_h0"`
	SiteName string `json:"SiteName"`
	// Gps_fix float32 `json:"gps_fix"`
	// Ver_app string `json:"ver_app"`
	Gps_lat float32 `json:"gps_lat"`
	S_t0 float32 `json:"s_t0"`
	Timestamp string `json:"timestamp"`
	Gps_lon float32 `json:"gps_lon"`
	// Date string `json:"date"`
	// Tick float32 `json:"tick"`
	Device_id string `json:"device_id"`
	// S_1 float32 `json:"s_1"`
	// S_0 float32 `json:"s_0"`
	// S_3 float32 `json:"s_3"`
	// S_2 float32 `json:"s_2"`
	// Ver_format string `json:"ver_format"`
	// Time string `json:"time"`
}

type airbox struct {
	Source string `json:"source"`
	Feeds []device `json:"feeds"`
	Version string `json:"version"`
	Num_of_records int `json:"num_of_records"`
}

var bot *linebot.Client
var airbox_json airbox
var	client=redis.NewClient(&redis.Options{
		Addr:"hipposerver.ddns.net:6379",
		Password:"",
		DB:0,
	})

func main() {
	url := "https://data.lass-net.org/data/last-all-airbox.json"
	req, _ := http.NewRequest("GET", url, nil)
	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)
	errs := json.Unmarshal(body, &airbox_json)
	if errs != nil {
		fmt.Println(errs)
	}

	// fmt.Println(airbox_json)
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				var txtmessage string
				inText := strings.ToLower(message.Text)
				if strings.Contains(inText,"訂閱"){
					userID:=event.Source.UserID
					// pong, _ := client.Ping().Result()
					// txtmessage=pong
					for i:=0; i<len(airbox_json.Feeds); i++ {
						if strings.Contains(inText,strings.ToLower(airbox_json.Feeds[i].Device_id)) {
							// client.Set(airbox_json.Feeds[i].Device_id,userID,0)
							// txtmessage="訂閱成功!"
							// break
							val, err:=client.Get(airbox_json.Feeds[i].Device_id).Result()
							if err!=nil{
								client.Set(airbox_json.Feeds[i].Device_id,userID,0)
								// if err!=nil{
								// 	panic(err)
								// }
								txtmessage="訂閱成功!"
								break
							}
							stringSlice:=strings.Split(val,",")
							if stringInSlice(userID,stringSlice){
								txtmessage="您已訂閱過此ID!"
								break
							} else{
								val=val+","+userID
								client.Set(airbox_json.Feeds[i].Device_id,val,0)
								txtmessage="訂閱成功!"
								break
							}
							// if err!=nil{
							// 	panic(err)
							// }
						}
					}
				} else{
					for i:=0; i<len(airbox_json.Feeds); i++ {
						if strings.Contains(inText,strings.ToLower(airbox_json.Feeds[i].Device_id)) {
							txtmessage="Device_id: "+airbox_json.Feeds[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+airbox_json.Feeds[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(airbox_json.Feeds[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(airbox_json.Feeds[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+airbox_json.Feeds[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(airbox_json.Feeds[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(airbox_json.Feeds[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(airbox_json.Feeds[i].S_t0),'f',0,64)
							break
						}
					}
				}
				if len(txtmessage)==0{
					txtmessage="Sorry! No this device ID, please check again."
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(txtmessage)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}

func stringInSlice(a string, list []string) bool {
    for _, b := range list {
        if b == a {
            return true
        }
    }
    return false
}
<commit_msg>test push message<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"io/ioutil"
	"encoding/json"
	"strings"
	"github.com/line/line-bot-sdk-go/linebot"
	"github.com/go-redis/redis"
	"strconv"
	"time"
)

type device struct {
	// Gps_num float32 `json:"gps_num"`
	// App string `json:"app"`
	// Gps_alt float32 `json:"gps_alt"`
	// Fmt_opt int `json:"fmt_opt"`
	// Device string `json:"device"`
	// S_d2 float32 `json:"s_d2"`
	S_d0 float32 `json:"s_d0"`
	// S_d1 float32 `json:"s_d1"`
	S_h0 float32 `json:"s_h0"`
	SiteName string `json:"SiteName"`
	// Gps_fix float32 `json:"gps_fix"`
	// Ver_app string `json:"ver_app"`
	Gps_lat float32 `json:"gps_lat"`
	S_t0 float32 `json:"s_t0"`
	Timestamp string `json:"timestamp"`
	Gps_lon float32 `json:"gps_lon"`
	// Date string `json:"date"`
	// Tick float32 `json:"tick"`
	Device_id string `json:"device_id"`
	// S_1 float32 `json:"s_1"`
	// S_0 float32 `json:"s_0"`
	// S_3 float32 `json:"s_3"`
	// S_2 float32 `json:"s_2"`
	// Ver_format string `json:"ver_format"`
	// Time string `json:"time"`
}

type airbox struct {
	Source string `json:"source"`
	Feeds []device `json:"feeds"`
	Version string `json:"version"`
	Num_of_records int `json:"num_of_records"`
}

var bot *linebot.Client
var airbox_json airbox
var	client=redis.NewClient(&redis.Options{
		Addr:"hipposerver.ddns.net:6379",
		Password:"",
		DB:0,
	})

func main() {
	url := "https://data.lass-net.org/data/last-all-airbox.json"
	req, _ := http.NewRequest("GET", url, nil)
	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()
	body, _ := ioutil.ReadAll(res.Body)
	errs := json.Unmarshal(body, &airbox_json)
	if errs != nil {
		fmt.Println(errs)
	}

	// fmt.Println(airbox_json)
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)

	t:=time.Now()
	_, min, _:=t.Clock()
	if min==5{
		pushmessage()
	}
}
func pushmessage(){
	_,err:=bot.PushMessage("U3617adbdd46283d7e859f36302f4f471", "hi!").Do()
	if err!=nil{
		panic(err)
	}
}
func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				var txtmessage string
				inText := strings.ToLower(message.Text)
				if strings.Contains(inText,"訂閱"){
					userID:=event.Source.UserID
					// pong, _ := client.Ping().Result()
					// txtmessage=pong
					for i:=0; i<len(airbox_json.Feeds); i++ {
						if strings.Contains(inText,strings.ToLower(airbox_json.Feeds[i].Device_id)) {
							val, err:=client.Get(airbox_json.Feeds[i].Device_id).Result()
							if err!=nil{
								client.Set(airbox_json.Feeds[i].Device_id,userID,0)
								txtmessage="訂閱成功!"
								break
							}
							stringSlice:=strings.Split(val,",")
							if stringInSlice(userID,stringSlice){
								txtmessage="您已訂閱過此ID!"
								break
							} else{
								val=val+","+userID
								client.Set(airbox_json.Feeds[i].Device_id,val,0)
								txtmessage="訂閱成功!"
								break
							}
						}
					}
				} else{
					for i:=0; i<len(airbox_json.Feeds); i++ {
						if strings.Contains(inText,strings.ToLower(airbox_json.Feeds[i].Device_id)) {
							txtmessage="Device_id: "+airbox_json.Feeds[i].Device_id+"\n"
							txtmessage=txtmessage+"Site Name: "+airbox_json.Feeds[i].SiteName+"\n"
							txtmessage=txtmessage+"Location: ("+strconv.FormatFloat(float64(airbox_json.Feeds[i].Gps_lon),'f',3,64)+","+strconv.FormatFloat(float64(airbox_json.Feeds[i].Gps_lat),'f',3,64)+")"+"\n"
							txtmessage=txtmessage+"Timestamp: "+airbox_json.Feeds[i].Timestamp+"\n"
							txtmessage=txtmessage+"PM2.5: "+strconv.FormatFloat(float64(airbox_json.Feeds[i].S_d0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Humidity: "+strconv.FormatFloat(float64(airbox_json.Feeds[i].S_h0),'f',0,64)+"\n"
							txtmessage=txtmessage+"Temperature: "+strconv.FormatFloat(float64(airbox_json.Feeds[i].S_t0),'f',0,64)
							break
						}
					}
				}
				if len(txtmessage)==0{
					txtmessage="Sorry! No this device ID, please check again."
				}
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(txtmessage)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}

func stringInSlice(a string, list []string) bool {
    for _, b := range list {
        if b == a {
            return true
        }
    }
    return false
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/howeyc/fsnotify"
)

var (
	watcher  *fsnotify.Watcher
	watched  = make(map[string]struct{})
	mtimes   = make(map[string]time.Time)
	exitCode = make(chan int)
	rootPath string

	buildQueued = true

	// flags
	useGodeps    bool
	afterAllOk   string
	afterNotOk   string
	verbose      bool
	buildArgs    string
	vetArgs      string
	testArgs     string
	ignoreVendor bool
)

func runCmd(name string, args ...string) (buf *commandBuffer, err error) {
	buf = new(commandBuffer)

	cmd := exec.Command(name, args...)
	cmd.Stdout = buf
	cmd.Stderr = buf
	cmd.Dir = rootPath

	if err = cmd.Run(); err != nil || verbose {
		print(buf.String())
	}

	return
}

func getPackages() []string {
	output, err := runCmd("go", "list", "./...")
	if err != nil {
		return []string{}
	}

	f := strings.Split(output.String(), "\n")
	var files []string

	for _, file := range f {
		if !strings.Contains(file, "/vendor") && file != "\n" {
			files = append(files, strings.TrimSpace(file))
		}
	}

	return files
}

func fullBuild() {
	var err error
	vetArrayArgs := []string{"vet"}
	testArrayArgs := []string{"test"}

	if ignoreVendor == true {
		packages := getPackages()
		vetArrayArgs = append(vetArrayArgs, packages...)
		testArrayArgs = append(testArrayArgs, packages...)
	}

	log.Println("glitch: building")
	if _, err = runCmd("go", "build", buildArgs); err == nil {
		log.Println("glitch: build OK - vetting")

		if _, err = runCmd("go", vetArrayArgs...); err == nil {
			log.Println("glitch: vet OK - testing")

			if useGodeps {
				_, err = runCmd("godep", "go", "test", testArgs)
			} else {
				_, err = runCmd("go", testArrayArgs...)
			}
			if err == nil {
				log.Println("glitch: test OK")

				if len(afterAllOk) > 0 {
					if _, err = runCmd("bash", "-c", afterAllOk); err == nil {
						log.Println("glitch: after-all-ok OK")
					}
				}
			}
		}
	}

	if err != nil && len(afterNotOk) > 0 {
		if _, err = runCmd("bash", "-c", afterNotOk); err == nil {
			log.Println("glitch: after-not-ok OK")
		}
	}

	if err != nil {
		log.Println("glitch: failed")
	} else {
		log.Println("glitch: all OK")
	}
}

func maybeQueueBuild(path string) {
	if !hasSuffix(path, ".go") {
		return
	}

	// Check whether the modified time has actually changed. This is
	// useful on systems that may emit multiple change events for a
	// single change (e.g. OSX + Spotlight)
	fi, err := os.Stat(path)
	if err == nil {
		mtime := fi.ModTime()
		lasttime := mtimes[path]
		if !mtime.Equal(lasttime) {
			mtimes[path] = mtime
			buildQueued = true
		}
	}
}

func handleCreate(path string) {
	watch(path)
	maybeQueueBuild(path)
}

func handleDelete(path string) {
	if _, watching := watched[path]; watching {
		_ = watcher.RemoveWatch(path)
		delete(watched, path)
	}
	maybeQueueBuild(path)
}

func handleModify(path string) {
	maybeQueueBuild(path)
}

func handleEvent(ev *fsnotify.FileEvent) {
	if len(ev.Name) > 0 {
		switch {
		case ev.IsCreate():
			handleCreate(ev.Name)
		case ev.IsDelete():
			handleDelete(ev.Name)
		case ev.IsModify():
			handleModify(ev.Name)
		}
	}
}

var (
	gitSuffix   = sprintf("%v.git", string(filepath.Separator))
	gitContains = sprintf("%v%v", gitSuffix, string(filepath.Separator))
)

func watch(dir string) {
	const watchFlags = fsnotify.FSN_CREATE | fsnotify.FSN_DELETE | fsnotify.FSN_MODIFY

	if _, watching := watched[dir]; watching {
		return
	}

	walker := func(path string, fileInfo os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		if hasSuffix(path, gitSuffix) || contains(path, gitContains) {
			return nil
		}

		if fileInfo.IsDir() {
			if err = watcher.WatchFlags(path, watchFlags); err == nil {
				watched[path] = emptyStruct
			}
		}

		return err
	}

	_ = filepath.Walk(dir, walker)
}

func periodicallyLogWatchedCount() {
	logWatchedCount := func() {
		log.Printf("glitch: watching: %v paths", len(watched))
	}

	logWatchedCount()
	for _ = range time.Tick(5 * time.Second) {
		logWatchedCount()
	}
}

func periodicallyLogWatchedPaths() {
	logWatchedPaths := func() {
		log.Printf("glitch: watching: %v paths", len(watched))
		for path := range watched {
			log.Println("glitch: watching:", path)
		}
	}

	logWatchedPaths()
	for _ = range time.Tick(5 * time.Second) {
		logWatchedPaths()
	}
}

func runEventLoop() {
	for {
		select {
		case ev := <-watcher.Event:
			handleEvent(ev)
		case err := <-watcher.Error:
			panicIf(err)
		}
	}
}

func runBuildLoop() {
	consumeBuildQueue := func() {
		if buildQueued {
			buildQueued = false
			clearScrollBuffer()
			fullBuild()
		}
	}

	for _ = range time.Tick(1 * time.Second) {
		go consumeBuildQueue()
	}
}

func main() {
	flag.StringVar(&afterAllOk, "after-all-ok", "", "command to run after build, vet and test succeed")
	flag.StringVar(&afterNotOk, "after-not-ok", "", "command to run after all OK")
	flag.StringVar(&buildArgs, "build", "./...", "arguments passed to `go build`")
	flag.StringVar(&testArgs, "test", "./...", "arguments passed to `go test`")
	flag.StringVar(&vetArgs, "vet", "./...", "arguments passed to `go vet`")
	flag.BoolVar(&useGodeps, "use-godeps", false, "use godep for testing")
	flag.BoolVar(&verbose, "verbose", false, "be verbose")
	flag.BoolVar(&ignoreVendor, "exclude-vendor", false, "excludes vendor folder from vet and test")

	flag.Parse()

	wd, err := os.Getwd()
	panicIf(err)
	rootPath = wd

	w, err := fsnotify.NewWatcher()
	panicIf(err)
	watcher = w
	defer watcher.Close()

	//go periodicallyLogWatchedPaths()
	//go periodicallyLogWatchedCount()
	go runEventLoop()
	go runBuildLoop()

	watch(rootPath)
	os.Exit(<-exitCode)
}
<commit_msg>Added bell on fail to alert terminal window<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/howeyc/fsnotify"
)

var (
	watcher  *fsnotify.Watcher
	watched  = make(map[string]struct{})
	mtimes   = make(map[string]time.Time)
	exitCode = make(chan int)
	rootPath string

	buildQueued = true

	// flags
	useGodeps    bool
	afterAllOk   string
	afterNotOk   string
	verbose      bool
	buildArgs    string
	vetArgs      string
	testArgs     string
	ignoreVendor bool
)

func runCmd(name string, args ...string) (buf *commandBuffer, err error) {
	buf = new(commandBuffer)

	cmd := exec.Command(name, args...)
	cmd.Stdout = buf
	cmd.Stderr = buf
	cmd.Dir = rootPath

	if err = cmd.Run(); err != nil || verbose {
		print(buf.String())
	}

	return
}

func getPackages() []string {
	output, err := runCmd("go", "list", "./...")
	if err != nil {
		return []string{}
	}

	f := strings.Split(output.String(), "\n")
	var files []string

	for _, file := range f {
		if !strings.Contains(file, "/vendor") && file != "\n" {
			files = append(files, strings.TrimSpace(file))
		}
	}

	return files
}

func fullBuild() {
	var err error
	vetArrayArgs := []string{"vet"}
	testArrayArgs := []string{"test"}

	if ignoreVendor == true {
		packages := getPackages()
		vetArrayArgs = append(vetArrayArgs, packages...)
		testArrayArgs = append(testArrayArgs, packages...)
	}

	log.Println("glitch: building")
	if _, err = runCmd("go", "build", buildArgs); err == nil {
		log.Println("glitch: build OK - vetting")

		if _, err = runCmd("go", vetArrayArgs...); err == nil {
			log.Println("glitch: vet OK - testing")

			if useGodeps {
				_, err = runCmd("godep", "go", "test", testArgs)
			} else {
				_, err = runCmd("go", testArrayArgs...)
			}
			if err == nil {
				log.Println("glitch: test OK")

				if len(afterAllOk) > 0 {
					if _, err = runCmd("bash", "-c", afterAllOk); err == nil {
						log.Println("glitch: after-all-ok OK")
					}
				}
			}
		}
	}

	if err != nil && len(afterNotOk) > 0 {
		if _, err = runCmd("bash", "-c", afterNotOk); err == nil {
			log.Println("glitch: after-not-ok OK")
		}
	}

	if err != nil {
		log.Println("glitch: failed")
		log.Print("\a")
	} else {
		log.Println("glitch: all OK")
	}
}

func maybeQueueBuild(path string) {
	if !hasSuffix(path, ".go") {
		return
	}

	// Check whether the modified time has actually changed. This is
	// useful on systems that may emit multiple change events for a
	// single change (e.g. OSX + Spotlight)
	fi, err := os.Stat(path)
	if err == nil {
		mtime := fi.ModTime()
		lasttime := mtimes[path]
		if !mtime.Equal(lasttime) {
			mtimes[path] = mtime
			buildQueued = true
		}
	}
}

func handleCreate(path string) {
	watch(path)
	maybeQueueBuild(path)
}

func handleDelete(path string) {
	if _, watching := watched[path]; watching {
		_ = watcher.RemoveWatch(path)
		delete(watched, path)
	}
	maybeQueueBuild(path)
}

func handleModify(path string) {
	maybeQueueBuild(path)
}

func handleEvent(ev *fsnotify.FileEvent) {
	if len(ev.Name) > 0 {
		switch {
		case ev.IsCreate():
			handleCreate(ev.Name)
		case ev.IsDelete():
			handleDelete(ev.Name)
		case ev.IsModify():
			handleModify(ev.Name)
		}
	}
}

var (
	gitSuffix   = sprintf("%v.git", string(filepath.Separator))
	gitContains = sprintf("%v%v", gitSuffix, string(filepath.Separator))
)

func watch(dir string) {
	const watchFlags = fsnotify.FSN_CREATE | fsnotify.FSN_DELETE | fsnotify.FSN_MODIFY

	if _, watching := watched[dir]; watching {
		return
	}

	walker := func(path string, fileInfo os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		if hasSuffix(path, gitSuffix) || contains(path, gitContains) {
			return nil
		}

		if fileInfo.IsDir() {
			if err = watcher.WatchFlags(path, watchFlags); err == nil {
				watched[path] = emptyStruct
			}
		}

		return err
	}

	_ = filepath.Walk(dir, walker)
}

func periodicallyLogWatchedCount() {
	logWatchedCount := func() {
		log.Printf("glitch: watching: %v paths", len(watched))
	}

	logWatchedCount()
	for _ = range time.Tick(5 * time.Second) {
		logWatchedCount()
	}
}

func periodicallyLogWatchedPaths() {
	logWatchedPaths := func() {
		log.Printf("glitch: watching: %v paths", len(watched))
		for path := range watched {
			log.Println("glitch: watching:", path)
		}
	}

	logWatchedPaths()
	for _ = range time.Tick(5 * time.Second) {
		logWatchedPaths()
	}
}

func runEventLoop() {
	for {
		select {
		case ev := <-watcher.Event:
			handleEvent(ev)
		case err := <-watcher.Error:
			panicIf(err)
		}
	}
}

func runBuildLoop() {
	consumeBuildQueue := func() {
		if buildQueued {
			buildQueued = false
			clearScrollBuffer()
			fullBuild()
		}
	}

	for _ = range time.Tick(1 * time.Second) {
		go consumeBuildQueue()
	}
}

func main() {
	flag.StringVar(&afterAllOk, "after-all-ok", "", "command to run after build, vet and test succeed")
	flag.StringVar(&afterNotOk, "after-not-ok", "", "command to run after all OK")
	flag.StringVar(&buildArgs, "build", "./...", "arguments passed to `go build`")
	flag.StringVar(&testArgs, "test", "./...", "arguments passed to `go test`")
	flag.StringVar(&vetArgs, "vet", "./...", "arguments passed to `go vet`")
	flag.BoolVar(&useGodeps, "use-godeps", false, "use godep for testing")
	flag.BoolVar(&verbose, "verbose", false, "be verbose")
	flag.BoolVar(&ignoreVendor, "exclude-vendor", false, "excludes vendor folder from vet and test")

	flag.Parse()

	wd, err := os.Getwd()
	panicIf(err)
	rootPath = wd

	w, err := fsnotify.NewWatcher()
	panicIf(err)
	watcher = w
	defer watcher.Close()

	//go periodicallyLogWatchedPaths()
	//go periodicallyLogWatchedCount()
	go runEventLoop()
	go runBuildLoop()

	watch(rootPath)
	os.Exit(<-exitCode)
}
<|endoftext|>


<commit_before>package golgoquery

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/abhishekkr/gol/golhttpclient"
)

var (
	CacheDir string
)

func createCache(cachePath string, cacheContent string) {
	cacheDir := path.Dir(cachePath)
	if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
		os.Mkdir(CacheDir, 0777)
	}
}

func readCache(cachePath string) string {
	dat, err := ioutil.ReadFile(cachePath)
	return string(dat)
}

func urlToFilename(url string) string {
	var replacer = strings.NewReplacer(" ", "-",
		"\t", "-",
		"%", "-",
		"?", "-",
		"=", "-",
		"\\", "-",
		"/", "-",
	)
	return replacer.Replace(url)
}

func CacheUrl(url string) (*goquery.Document, error) {
	urlFile = urlToFilename(url)
	if CacheDir == "" {
		if _, err := os.Stat("/tmp"); os.IsNotExist(err) {
			log.Fatalf("cannot find default CacheDir(%s), default cachedir is not created", CacheDir)
		}
		CacheDir = "/tmp"
	}
	cachePath := fmt.Sprintf("%s%s%s", CacheDir, filepath.Separator, urlFile)

	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		body, err := golhttpclient.HttpGet(url)
		if err != nil {
			log.Fatalf("not able to fetch %s", url)
		}
		createCache(cachePath, body)
	}

	cache := readCache(cachePath)

	utfBody, err := iconv.NewReader(cache, charset, "utf-8")
	if err != nil {
		log.Fatalf("not able to read data of %s", url)
	}

	return goquery.NewDocumentFromReader(strings.NewReader(utfBody))
}
<commit_msg>[golgoquery] fixing cache of query<commit_after>package golgoquery

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/abhishekkr/gol/golhttpclient"
)

var (
	CacheDir = "/tmp/.tune.cli"
)

func createCache(cachePath string, cacheContent string) {
	cacheDir := path.Dir(cachePath)
	if _, err := os.Stat(cacheDir); os.IsNotExist(err) {
		log.Printf("[warn] cannot find default CacheDir(%s), creating cachedir", CacheDir)
		os.Mkdir(CacheDir, 0777)
	}

	err := ioutil.WriteFile(cachePath, []byte(cacheContent), 0644)
	if err != nil {
		log.Printf("[warn] cannot create cache (%s), creating cachedir", cachePath)
	}
}

func readCache(cachePath string) string {
	dat, err := ioutil.ReadFile(cachePath)
	if err != nil {
		log.Printf("[warn] missing cache file %s", cachePath)
	}
	return string(dat)
}

func urlToFilename(url string) string {
	var replacer = strings.NewReplacer(" ", "-",
		"\t", "-",
		"%", "-",
		"?", "-",
		"=", "-",
		"\\", "-",
		"/", "-",
		":", "-",
	)
	return replacer.Replace(url)
}

func CacheUrl(url string) (*goquery.Document, error) {
	urlFile := urlToFilename(url)
	cachePath := fmt.Sprintf("%s%s%s", CacheDir, string(filepath.Separator), urlFile)

	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		body, err := golhttpclient.HttpGet(url, map[string]string{}, map[string]string{})
		if err != nil {
			log.Fatalf("not able to fetch %s", url)
		}
		createCache(cachePath, body)
	}

	cache := readCache(cachePath)

	return goquery.NewDocumentFromReader(strings.NewReader(cache))
}
<|endoftext|>


<commit_before>package awsutil

import (
	"fmt"
	"net/http"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/defaults"
)

type CredentialsConfig struct {
	// The access key if static credentials are being used
	AccessKey string

	// The secret key if static credentials are being used
	SecretKey string

	// The session token if it is being used
	SessionToken string

	// If specified, the region will be provided to the config of the
	// EC2RoleProvider's client. This may be useful if you want to e.g. reuse
	// the client elsewhere.
	Region string

	// The filename for the shared credentials provider, if being used
	Filename string

	// The profile for the shared credentials provider, if being used
	Profile string

	// The http.Client to use, or nil for the client to use its default
	HTTPClient *http.Client
}

func (c *CredentialsConfig) GenerateCredentialChain() (*credentials.Credentials, error) {
	var providers []credentials.Provider

	switch {
	case c.AccessKey != "" && c.SecretKey != "":
		// Add the static credential provider
		providers = append(providers, &credentials.StaticProvider{
			Value: credentials.Value{
				AccessKeyID:     c.AccessKey,
				SecretAccessKey: c.SecretKey,
				SessionToken:    c.SessionToken,
			}})
	case c.AccessKey == "" && c.SecretKey == "":
		// Attempt to get credentials from the IAM instance role below

	default: // Have one or the other but not both and not neither
		return nil, fmt.Errorf(
			"static AWS client credentials haven't been properly configured (the access key or secret key were provided but not both)")
	}

	// Add the environment credential provider
	providers = append(providers, &credentials.EnvProvider{})

	// Add the shared credentials provider
	providers = append(providers, &credentials.SharedCredentialsProvider{
		Filename: c.Filename,
		Profile:  c.Profile,
	})

	// Add the remote provider
	def := defaults.Get()
	if c.Region != "" {
		def.Config.Region = aws.String(c.Region)
	}
	if c.HTTPClient != nil {
		def.Config.HTTPClient = c.HTTPClient
	}

	providers = append(providers, defaults.RemoteCredProvider(*def.Config, def.Handlers))

	// Create the credentials required to access the API.
	creds := credentials.NewChainCredentials(providers)
	if creds == nil {
		return nil, fmt.Errorf("could not compile valid credential providers from static config, environment, shared, or instance metadata")
	}

	return creds, nil
}
<commit_msg>feat: replace the way AWS sessions are created and added a new WebIdentity provider to the credentials chain (#7738)<commit_after>package awsutil

import (
	"fmt"
	"net/http"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/pkg/errors"
)

type CredentialsConfig struct {
	// The access key if static credentials are being used
	AccessKey string

	// The secret key if static credentials are being used
	SecretKey string

	// The session token if it is being used
	SessionToken string

	// If specified, the region will be provided to the config of the
	// EC2RoleProvider's client. This may be useful if you want to e.g. reuse
	// the client elsewhere.
	Region string

	// The filename for the shared credentials provider, if being used
	Filename string

	// The profile for the shared credentials provider, if being used
	Profile string

	// The http.Client to use, or nil for the client to use its default
	HTTPClient *http.Client
}

func (c *CredentialsConfig) GenerateCredentialChain() (*credentials.Credentials, error) {
	var providers []credentials.Provider

	switch {
	case c.AccessKey != "" && c.SecretKey != "":
		// Add the static credential provider
		providers = append(providers, &credentials.StaticProvider{
			Value: credentials.Value{
				AccessKeyID:     c.AccessKey,
				SecretAccessKey: c.SecretKey,
				SessionToken:    c.SessionToken,
			}})
	case c.AccessKey == "" && c.SecretKey == "":
		// Attempt to get credentials from the IAM instance role below

	default: // Have one or the other but not both and not neither
		return nil, fmt.Errorf(
			"static AWS client credentials haven't been properly configured (the access key or secret key were provided but not both)")
	}

	roleARN := os.Getenv("AWS_ROLE_ARN")
	tokenPath := os.Getenv("AWS_WEB_IDENTITY_TOKEN_FILE")
	sessionName := os.Getenv("AWS_ROLE_SESSION_NAME")
	if roleARN != "" && tokenPath != "" && sessionName != "" {
		// this session is only created to create the WebIdentityRoleProvider, as the env variables are already there
		// this automatically assumes the role, but the provider needs to be added to the chain
		sess, err := session.NewSession()
		if err != nil {
			return nil, errors.Wrap(err, "error creating a new session to create a WebIdentityRoleProvider")
		}
		//Add the web identity role credential provider
		providers = append(providers, stscreds.NewWebIdentityRoleProvider(sts.New(sess), roleARN, sessionName, tokenPath))
	}

	// Add the environment credential provider
	providers = append(providers, &credentials.EnvProvider{})

	// Add the shared credentials provider
	providers = append(providers, &credentials.SharedCredentialsProvider{
		Filename: c.Filename,
		Profile:  c.Profile,
	})

	// Add the remote provider
	def := defaults.Get()
	if c.Region != "" {
		def.Config.Region = aws.String(c.Region)
	}
	if c.HTTPClient != nil {
		def.Config.HTTPClient = c.HTTPClient
	}

	providers = append(providers, defaults.RemoteCredProvider(*def.Config, def.Handlers))

	// Create the credentials required to access the API.
	creds := credentials.NewChainCredentials(providers)
	if creds == nil {
		return nil, fmt.Errorf("could not compile valid credential providers from static config, environment, shared, web identity or instance metadata")
	}

	return creds, nil
}
<|endoftext|>


<commit_before>package pool

import (
	"errors"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/go-redis/redis/internal"
)

var ErrClosed = errors.New("redis: client is closed")
var ErrPoolTimeout = errors.New("redis: connection pool timeout")

var timers = sync.Pool{
	New: func() interface{} {
		t := time.NewTimer(time.Hour)
		t.Stop()
		return t
	},
}

// Stats contains pool state information and accumulated stats.
type Stats struct {
	Requests uint32 // number of times a connection was requested by the pool
	Hits     uint32 // number of times free connection was found in the pool
	Timeouts uint32 // number of times a wait timeout occurred

	TotalConns uint32 // the number of total connections in the pool
	FreeConns  uint32 // the number of free connections in the pool
}

type Pooler interface {
	NewConn() (*Conn, error)
	CloseConn(*Conn) error

	Get() (*Conn, bool, error)
	Put(*Conn) error
	Remove(*Conn) error

	Len() int
	FreeLen() int
	Stats() *Stats

	Close() error
}

type Options struct {
	Dialer  func() (net.Conn, error)
	OnClose func(*Conn) error

	PoolSize           int
	PoolTimeout        time.Duration
	IdleTimeout        time.Duration
	IdleCheckFrequency time.Duration
}

type ConnPool struct {
	opt *Options

	dialErrorsNum  uint32 // atomic
	_lastDialError atomic.Value

	queue chan struct{}

	connsMu sync.Mutex
	conns   []*Conn

	freeConnsMu sync.Mutex
	freeConns   []*Conn

	stats Stats

	_closed uint32 // atomic
}

var _ Pooler = (*ConnPool)(nil)

func NewConnPool(opt *Options) *ConnPool {
	p := &ConnPool{
		opt: opt,

		queue:     make(chan struct{}, opt.PoolSize),
		conns:     make([]*Conn, 0, opt.PoolSize),
		freeConns: make([]*Conn, 0, opt.PoolSize),
	}
	if opt.IdleTimeout > 0 && opt.IdleCheckFrequency > 0 {
		go p.reaper(opt.IdleCheckFrequency)
	}
	return p
}

func (p *ConnPool) NewConn() (*Conn, error) {
	if p.closed() {
		return nil, ErrClosed
	}

	if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.opt.PoolSize) {
		return nil, p.lastDialError()
	}

	netConn, err := p.opt.Dialer()
	if err != nil {
		p.setLastDialError(err)
		if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.opt.PoolSize) {
			go p.tryDial()
		}
		return nil, err
	}

	cn := NewConn(netConn)
	p.connsMu.Lock()
	p.conns = append(p.conns, cn)
	p.connsMu.Unlock()

	return cn, nil
}

func (p *ConnPool) tryDial() {
	for {
		conn, err := p.opt.Dialer()
		if err != nil {
			p.setLastDialError(err)
			time.Sleep(time.Second)
			continue
		}

		atomic.StoreUint32(&p.dialErrorsNum, 0)
		_ = conn.Close()
		return
	}
}

func (p *ConnPool) setLastDialError(err error) {
	p._lastDialError.Store(err)
}

func (p *ConnPool) lastDialError() error {
	return p._lastDialError.Load().(error)
}

// Get returns existed connection from the pool or creates a new one.
func (p *ConnPool) Get() (*Conn, bool, error) {
	if p.closed() {
		return nil, false, ErrClosed
	}

	atomic.AddUint32(&p.stats.Requests, 1)

	select {
	case p.queue <- struct{}{}:
	default:
		timer := timers.Get().(*time.Timer)
		timer.Reset(p.opt.PoolTimeout)

		select {
		case p.queue <- struct{}{}:
			if !timer.Stop() {
				<-timer.C
			}
			timers.Put(timer)
		case <-timer.C:
			timers.Put(timer)
			atomic.AddUint32(&p.stats.Timeouts, 1)
			return nil, false, ErrPoolTimeout
		}
	}

	for {
		p.freeConnsMu.Lock()
		cn := p.popFree()
		p.freeConnsMu.Unlock()

		if cn == nil {
			break
		}

		if cn.IsStale(p.opt.IdleTimeout) {
			p.CloseConn(cn)
			continue
		}

		atomic.AddUint32(&p.stats.Hits, 1)
		return cn, false, nil
	}

	newcn, err := p.NewConn()
	if err != nil {
		<-p.queue
		return nil, false, err
	}

	return newcn, true, nil
}

func (p *ConnPool) popFree() *Conn {
	if len(p.freeConns) == 0 {
		return nil
	}

	idx := len(p.freeConns) - 1
	cn := p.freeConns[idx]
	p.freeConns = p.freeConns[:idx]
	return cn
}

func (p *ConnPool) Put(cn *Conn) error {
	if data := cn.Rd.PeekBuffered(); data != nil {
		internal.Logf("connection has unread data: %q", data)
		return p.Remove(cn)
	}
	p.freeConnsMu.Lock()
	p.freeConns = append(p.freeConns, cn)
	p.freeConnsMu.Unlock()
	<-p.queue
	return nil
}

func (p *ConnPool) Remove(cn *Conn) error {
	_ = p.CloseConn(cn)
	<-p.queue
	return nil
}

func (p *ConnPool) CloseConn(cn *Conn) error {
	p.connsMu.Lock()
	for i, c := range p.conns {
		if c == cn {
			p.conns = append(p.conns[:i], p.conns[i+1:]...)
			break
		}
	}
	p.connsMu.Unlock()

	return p.closeConn(cn)
}

func (p *ConnPool) closeConn(cn *Conn) error {
	if p.opt.OnClose != nil {
		_ = p.opt.OnClose(cn)
	}
	return cn.Close()
}

// Len returns total number of connections.
func (p *ConnPool) Len() int {
	p.connsMu.Lock()
	l := len(p.conns)
	p.connsMu.Unlock()
	return l
}

// FreeLen returns number of free connections.
func (p *ConnPool) FreeLen() int {
	p.freeConnsMu.Lock()
	l := len(p.freeConns)
	p.freeConnsMu.Unlock()
	return l
}

func (p *ConnPool) Stats() *Stats {
	return &Stats{
		Requests:   atomic.LoadUint32(&p.stats.Requests),
		Hits:       atomic.LoadUint32(&p.stats.Hits),
		Timeouts:   atomic.LoadUint32(&p.stats.Timeouts),
		TotalConns: uint32(p.Len()),
		FreeConns:  uint32(p.FreeLen()),
	}
}

func (p *ConnPool) closed() bool {
	return atomic.LoadUint32(&p._closed) == 1
}

func (p *ConnPool) Filter(fn func(*Conn) bool) error {
	var firstErr error
	p.connsMu.Lock()
	for _, cn := range p.conns {
		if fn(cn) {
			if err := p.closeConn(cn); err != nil && firstErr == nil {
				firstErr = err
			}
		}
	}
	p.connsMu.Unlock()
	return firstErr
}

func (p *ConnPool) Close() error {
	if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) {
		return ErrClosed
	}

	var firstErr error
	p.connsMu.Lock()
	for _, cn := range p.conns {
		if err := p.closeConn(cn); err != nil && firstErr == nil {
			firstErr = err
		}
	}
	p.conns = nil
	p.connsMu.Unlock()

	p.freeConnsMu.Lock()
	p.freeConns = nil
	p.freeConnsMu.Unlock()

	return firstErr
}

func (p *ConnPool) reapStaleConn() bool {
	if len(p.freeConns) == 0 {
		return false
	}

	cn := p.freeConns[0]
	if !cn.IsStale(p.opt.IdleTimeout) {
		return false
	}

	p.CloseConn(cn)
	p.freeConns = append(p.freeConns[:0], p.freeConns[1:]...)

	return true
}

func (p *ConnPool) ReapStaleConns() (int, error) {
	var n int
	for {
		p.queue <- struct{}{}
		p.freeConnsMu.Lock()

		reaped := p.reapStaleConn()

		p.freeConnsMu.Unlock()
		<-p.queue

		if reaped {
			n++
		} else {
			break
		}
	}
	return n, nil
}

func (p *ConnPool) reaper(frequency time.Duration) {
	ticker := time.NewTicker(frequency)
	defer ticker.Stop()

	for range ticker.C {
		if p.closed() {
			break
		}
		n, err := p.ReapStaleConns()
		if err != nil {
			internal.Logf("ReapStaleConns failed: %s", err)
			continue
		}
		s := p.Stats()
		internal.Logf(
			"reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)",
			n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts,
		)
	}
}
<commit_msg>stop ConnPool.tryDial goroutine if pool was closed<commit_after>package pool

import (
	"errors"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"github.com/go-redis/redis/internal"
)

var ErrClosed = errors.New("redis: client is closed")
var ErrPoolTimeout = errors.New("redis: connection pool timeout")

var timers = sync.Pool{
	New: func() interface{} {
		t := time.NewTimer(time.Hour)
		t.Stop()
		return t
	},
}

// Stats contains pool state information and accumulated stats.
type Stats struct {
	Requests uint32 // number of times a connection was requested by the pool
	Hits     uint32 // number of times free connection was found in the pool
	Timeouts uint32 // number of times a wait timeout occurred

	TotalConns uint32 // the number of total connections in the pool
	FreeConns  uint32 // the number of free connections in the pool
}

type Pooler interface {
	NewConn() (*Conn, error)
	CloseConn(*Conn) error

	Get() (*Conn, bool, error)
	Put(*Conn) error
	Remove(*Conn) error

	Len() int
	FreeLen() int
	Stats() *Stats

	Close() error
}

type Options struct {
	Dialer  func() (net.Conn, error)
	OnClose func(*Conn) error

	PoolSize           int
	PoolTimeout        time.Duration
	IdleTimeout        time.Duration
	IdleCheckFrequency time.Duration
}

type ConnPool struct {
	opt *Options

	dialErrorsNum  uint32 // atomic
	_lastDialError atomic.Value

	queue chan struct{}

	connsMu sync.Mutex
	conns   []*Conn

	freeConnsMu sync.Mutex
	freeConns   []*Conn

	stats Stats

	_closed uint32 // atomic
}

var _ Pooler = (*ConnPool)(nil)

func NewConnPool(opt *Options) *ConnPool {
	p := &ConnPool{
		opt: opt,

		queue:     make(chan struct{}, opt.PoolSize),
		conns:     make([]*Conn, 0, opt.PoolSize),
		freeConns: make([]*Conn, 0, opt.PoolSize),
	}
	if opt.IdleTimeout > 0 && opt.IdleCheckFrequency > 0 {
		go p.reaper(opt.IdleCheckFrequency)
	}
	return p
}

func (p *ConnPool) NewConn() (*Conn, error) {
	if p.closed() {
		return nil, ErrClosed
	}

	if atomic.LoadUint32(&p.dialErrorsNum) >= uint32(p.opt.PoolSize) {
		return nil, p.lastDialError()
	}

	netConn, err := p.opt.Dialer()
	if err != nil {
		p.setLastDialError(err)
		if atomic.AddUint32(&p.dialErrorsNum, 1) == uint32(p.opt.PoolSize) {
			go p.tryDial()
		}
		return nil, err
	}

	cn := NewConn(netConn)
	p.connsMu.Lock()
	p.conns = append(p.conns, cn)
	p.connsMu.Unlock()

	return cn, nil
}

func (p *ConnPool) tryDial() {
	for {
		if p.closed() {
			return
		}
		conn, err := p.opt.Dialer()
		if err != nil {
			p.setLastDialError(err)
			time.Sleep(time.Second)
			continue
		}

		atomic.StoreUint32(&p.dialErrorsNum, 0)
		_ = conn.Close()
		return
	}
}

func (p *ConnPool) setLastDialError(err error) {
	p._lastDialError.Store(err)
}

func (p *ConnPool) lastDialError() error {
	return p._lastDialError.Load().(error)
}

// Get returns existed connection from the pool or creates a new one.
func (p *ConnPool) Get() (*Conn, bool, error) {
	if p.closed() {
		return nil, false, ErrClosed
	}

	atomic.AddUint32(&p.stats.Requests, 1)

	select {
	case p.queue <- struct{}{}:
	default:
		timer := timers.Get().(*time.Timer)
		timer.Reset(p.opt.PoolTimeout)

		select {
		case p.queue <- struct{}{}:
			if !timer.Stop() {
				<-timer.C
			}
			timers.Put(timer)
		case <-timer.C:
			timers.Put(timer)
			atomic.AddUint32(&p.stats.Timeouts, 1)
			return nil, false, ErrPoolTimeout
		}
	}

	for {
		p.freeConnsMu.Lock()
		cn := p.popFree()
		p.freeConnsMu.Unlock()

		if cn == nil {
			break
		}

		if cn.IsStale(p.opt.IdleTimeout) {
			p.CloseConn(cn)
			continue
		}

		atomic.AddUint32(&p.stats.Hits, 1)
		return cn, false, nil
	}

	newcn, err := p.NewConn()
	if err != nil {
		<-p.queue
		return nil, false, err
	}

	return newcn, true, nil
}

func (p *ConnPool) popFree() *Conn {
	if len(p.freeConns) == 0 {
		return nil
	}

	idx := len(p.freeConns) - 1
	cn := p.freeConns[idx]
	p.freeConns = p.freeConns[:idx]
	return cn
}

func (p *ConnPool) Put(cn *Conn) error {
	if data := cn.Rd.PeekBuffered(); data != nil {
		internal.Logf("connection has unread data: %q", data)
		return p.Remove(cn)
	}
	p.freeConnsMu.Lock()
	p.freeConns = append(p.freeConns, cn)
	p.freeConnsMu.Unlock()
	<-p.queue
	return nil
}

func (p *ConnPool) Remove(cn *Conn) error {
	_ = p.CloseConn(cn)
	<-p.queue
	return nil
}

func (p *ConnPool) CloseConn(cn *Conn) error {
	p.connsMu.Lock()
	for i, c := range p.conns {
		if c == cn {
			p.conns = append(p.conns[:i], p.conns[i+1:]...)
			break
		}
	}
	p.connsMu.Unlock()

	return p.closeConn(cn)
}

func (p *ConnPool) closeConn(cn *Conn) error {
	if p.opt.OnClose != nil {
		_ = p.opt.OnClose(cn)
	}
	return cn.Close()
}

// Len returns total number of connections.
func (p *ConnPool) Len() int {
	p.connsMu.Lock()
	l := len(p.conns)
	p.connsMu.Unlock()
	return l
}

// FreeLen returns number of free connections.
func (p *ConnPool) FreeLen() int {
	p.freeConnsMu.Lock()
	l := len(p.freeConns)
	p.freeConnsMu.Unlock()
	return l
}

func (p *ConnPool) Stats() *Stats {
	return &Stats{
		Requests:   atomic.LoadUint32(&p.stats.Requests),
		Hits:       atomic.LoadUint32(&p.stats.Hits),
		Timeouts:   atomic.LoadUint32(&p.stats.Timeouts),
		TotalConns: uint32(p.Len()),
		FreeConns:  uint32(p.FreeLen()),
	}
}

func (p *ConnPool) closed() bool {
	return atomic.LoadUint32(&p._closed) == 1
}

func (p *ConnPool) Filter(fn func(*Conn) bool) error {
	var firstErr error
	p.connsMu.Lock()
	for _, cn := range p.conns {
		if fn(cn) {
			if err := p.closeConn(cn); err != nil && firstErr == nil {
				firstErr = err
			}
		}
	}
	p.connsMu.Unlock()
	return firstErr
}

func (p *ConnPool) Close() error {
	if !atomic.CompareAndSwapUint32(&p._closed, 0, 1) {
		return ErrClosed
	}

	var firstErr error
	p.connsMu.Lock()
	for _, cn := range p.conns {
		if err := p.closeConn(cn); err != nil && firstErr == nil {
			firstErr = err
		}
	}
	p.conns = nil
	p.connsMu.Unlock()

	p.freeConnsMu.Lock()
	p.freeConns = nil
	p.freeConnsMu.Unlock()

	return firstErr
}

func (p *ConnPool) reapStaleConn() bool {
	if len(p.freeConns) == 0 {
		return false
	}

	cn := p.freeConns[0]
	if !cn.IsStale(p.opt.IdleTimeout) {
		return false
	}

	p.CloseConn(cn)
	p.freeConns = append(p.freeConns[:0], p.freeConns[1:]...)

	return true
}

func (p *ConnPool) ReapStaleConns() (int, error) {
	var n int
	for {
		p.queue <- struct{}{}
		p.freeConnsMu.Lock()

		reaped := p.reapStaleConn()

		p.freeConnsMu.Unlock()
		<-p.queue

		if reaped {
			n++
		} else {
			break
		}
	}
	return n, nil
}

func (p *ConnPool) reaper(frequency time.Duration) {
	ticker := time.NewTicker(frequency)
	defer ticker.Stop()

	for range ticker.C {
		if p.closed() {
			break
		}
		n, err := p.ReapStaleConns()
		if err != nil {
			internal.Logf("ReapStaleConns failed: %s", err)
			continue
		}
		s := p.Stats()
		internal.Logf(
			"reaper: removed %d stale conns (TotalConns=%d FreeConns=%d Requests=%d Hits=%d Timeouts=%d)",
			n, s.TotalConns, s.FreeConns, s.Requests, s.Hits, s.Timeouts,
		)
	}
}
<|endoftext|>