<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/pkg/js"
	"github.com/StackExchange/dnscontrol/pkg/nameservers"
	"github.com/StackExchange/dnscontrol/pkg/normalize"
	"github.com/StackExchange/dnscontrol/providers"
	_ "github.com/StackExchange/dnscontrol/providers/_all"
	"github.com/StackExchange/dnscontrol/providers/config"
)

//go:generate go run build/generate/generate.go

var jsFile = flag.String("js", "dnsconfig.js", "Javascript file containing dns config")
var credsFile = flag.String("creds", "creds.json", "Provider credentials JSON file")
var jsonFile = flag.String("json", "", "File containing intermediate JSON")

var jsonOutputPre = flag.String("debugrawjson", "", "Write JSON intermediate to this file pre-normalization.")
var jsonOutputPost = flag.String("debugjson", "", "During preview, write JSON intermediate to this file instead of stdout.")

var devMode = flag.Bool("dev", false, "Use helpers.js from disk instead of embedded")

var flagProviders = flag.String("providers", "", "Providers to enable (comma seperated list); default is all-but-bind. Specify 'all' for all (including bind)")
var domains = flag.String("domains", "", "Comma seperated list of domain names to include")

var interactive = flag.Bool("i", false, "Confirm or Exclude each correction before they run")

var delay = flag.Int64("d", 0, "delay between domains to avoid rate limits (in ms)")

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	flag.Parse()
	command := flag.Arg(0)
	if command == "version" {
		fmt.Println(versionString())
		return
	}

	var dnsConfig *models.DNSConfig
	if *jsonFile != "" {
		text, err := ioutil.ReadFile(*jsonFile)
		if err != nil {
			log.Fatalf("Error reading %v: %v\n", *jsonFile, err)
		}
		err = json.Unmarshal(text, &dnsConfig)
		if err != nil {
			log.Fatalf("Error parsing JSON in (%v): %v", *jsonFile, err)
		}
	} else if *jsFile != "" {
		text, err := ioutil.ReadFile(*jsFile)
		if err != nil {
			log.Fatalf("Error reading %v: %v\n", *jsFile, err)
		}
		dnsConfig, err = js.ExecuteJavascript(string(text), *devMode)
		if err != nil {
			log.Fatalf("Error executing javasscript in (%v): %v", *jsFile, err)
		}
	}

	if dnsConfig == nil {
		log.Fatal("No config specified.")
	}

	if flag.NArg() != 1 {
		fmt.Println(`Usage: dnscontrol [options] cmd
		        cmd:
		           preview: Show changed that would happen.
		           push:    Make changes for real.
		           version: Print program version string.
		           print:   Print compiled data.
		           create-domains:   Pre-create domains in R53
		`)
		flag.PrintDefaults()
		return
	}
	if *jsonOutputPre != "" {
		dat, _ := json.MarshalIndent(dnsConfig, "", "  ")
		err := ioutil.WriteFile(*jsonOutputPre, dat, 0644)
		if err != nil {
			panic(err)
		}
	}

	errs := normalize.NormalizeAndValidateConfig(dnsConfig)
	if len(errs) > 0 {
		fmt.Printf("%d Validation errors:\n", len(errs))
		fatal := false
		for _, err := range errs {
			if _, ok := err.(normalize.Warning); ok {
				fmt.Printf("WARNING: %s\n", err)
			} else {
				fatal = true
				fmt.Printf("ERROR: %s\n", err)
			}
		}
		if fatal {
			log.Fatal("Exiting due to validation errors")
		}
	}

	if command == "print" {
		dat, _ := json.MarshalIndent(dnsConfig, "", "  ")
		if *jsonOutputPost == "" {
			fmt.Println("While running JS:", string(dat))
		} else {
			err := ioutil.WriteFile(*jsonOutputPost, dat, 0644)
			if err != nil {
				panic(err)
			}
		}
		return
	}

	providerConfigs, err := config.LoadProviderConfigs(*credsFile)
	if err != nil {
		log.Fatalf(err.Error())
	}
	nonDefaultProviders := []string{}
	for name, vals := range providerConfigs {
		// add "_exclude_from_defaults":"true" to a domain to exclude it from being run unless
		// -providers=all or -providers=name
		if vals["_exclude_from_defaults"] == "true" {
			nonDefaultProviders = append(nonDefaultProviders, name)
		}
	}
	registrars, err := providers.CreateRegistrars(dnsConfig, providerConfigs)
	if err != nil {
		log.Fatalf("Error creating registrars: %v\n", err)
	}
	dsps, err := providers.CreateDsps(dnsConfig, providerConfigs)
	if err != nil {
		log.Fatalf("Error creating dsps: %v\n", err)
	}

	fmt.Printf("Initialized %d registrars and %d dns service providers.\n", len(registrars), len(dsps))
	anyErrors, totalCorrections := false, 0
	switch command {
	case "create-domains":
		for _, domain := range dnsConfig.Domains {
			fmt.Println("*** ", domain.Name)
			for prov := range domain.DNSProviders {
				dsp, ok := dsps[prov]
				if !ok {
					log.Fatalf("DSP %s not declared.", prov)
				}
				if creator, ok := dsp.(providers.DomainCreator); ok {
					fmt.Println("  -", prov)
					err := creator.EnsureDomainExists(domain.Name)
					if err != nil {
						fmt.Printf("Error creating domain: %s\n", err)
					}
				}
			}
		}
	case "preview", "push":
	DomainLoop:
		for _, domain := range dnsConfig.Domains {
			if !shouldRunDomain(domain.Name) {
				continue
			}
			fmt.Printf("******************** Domain: %s\n", domain.Name)
			nsList, err := nameservers.DetermineNameservers(domain, 0, dsps)
			if err != nil {
				log.Fatal(err)
			}
			domain.Nameservers = nsList
			nameservers.AddNSRecords(domain)
			for prov := range domain.DNSProviders {
				dc, err := domain.Copy()
				if err != nil {
					log.Fatal(err)
				}
				shouldrun := shouldRunProvider(prov, dc, nonDefaultProviders)
				statusLbl := ""
				if !shouldrun {
					statusLbl = "(skipping)"
				}
				fmt.Printf("----- DNS Provider: %s... %s", prov, statusLbl)
				if !shouldrun {
					fmt.Println()
					continue
				}
				dsp, ok := dsps[prov]
				if !ok {
					log.Fatalf("DSP %s not declared.", prov)
				}
				corrections, err := dsp.GetDomainCorrections(dc)
				if err != nil {
					fmt.Println("ERROR")
					anyErrors = true
					fmt.Printf("Error getting corrections: %s\n", err)
					continue DomainLoop
				}
				totalCorrections += len(corrections)
				plural := "s"
				if len(corrections) == 1 {
					plural = ""
				}
				fmt.Printf("%d correction%s\n", len(corrections), plural)
				anyErrors = printOrRunCorrections(corrections, command) || anyErrors
			}
			if run := shouldRunProvider(domain.Registrar, domain, nonDefaultProviders); !run {
				continue
			}
			fmt.Printf("----- Registrar: %s\n", domain.Registrar)
			reg, ok := registrars[domain.Registrar]
			if !ok {
				log.Fatalf("Registrar %s not declared.", reg)
			}
			if len(domain.Nameservers) == 0 && domain.Metadata["no_ns"] != "true" {
				fmt.Printf("No nameservers declared; skipping registrar. Add {no_ns:'true'} to force.\n")
				continue
			}
			dc, err := domain.Copy()
			if err != nil {
				log.Fatal(err)
			}
			corrections, err := reg.GetRegistrarCorrections(dc)
			if err != nil {
				fmt.Printf("Error getting corrections: %s\n", err)
				anyErrors = true
				continue
			}
			totalCorrections += len(corrections)
			anyErrors = printOrRunCorrections(corrections, command) || anyErrors

			time.Sleep(time.Duration(*delay) * time.Millisecond)
		}
	default:
		log.Fatalf("Unknown command %s", command)
	}
	if os.Getenv("TEAMCITY_VERSION") != "" {
		fmt.Fprintf(os.Stderr, "##teamcity[buildStatus status='SUCCESS' text='%d corrections']", totalCorrections)
	}
	fmt.Printf("Done. %d corrections.\n", totalCorrections)
	if anyErrors {
		os.Exit(1)
	}
}

var reader = bufio.NewReader(os.Stdin)

func printOrRunCorrections(corrections []*models.Correction, command string) (anyErrors bool) {
	anyErrors = false
	if len(corrections) == 0 {
		return anyErrors
	}
	for i, correction := range corrections {
		fmt.Printf("#%d: %s\n", i+1, correction.Msg)
		if command == "push" {
			if *interactive {
				fmt.Print("Run? (Y/n): ")
				txt, err := reader.ReadString('\n')
				run := true
				if err != nil {
					run = false
				}
				txt = strings.ToLower(strings.TrimSpace(txt))
				if txt != "y" {
					run = false
				}
				if !run {
					fmt.Println("Skipping")
					continue
				}
			}
			err := correction.F()
			if err != nil {
				fmt.Println("FAILURE!", err)
				anyErrors = true
			} else {
				fmt.Println("SUCCESS!")
			}
		}
	}
	return anyErrors
}

func shouldRunProvider(p string, dc *models.DomainConfig, nonDefaultProviders []string) bool {
	if *flagProviders == "all" {
		return true
	}
	if *flagProviders == "" {
		for _, pr := range nonDefaultProviders {
			if pr == p {
				return false
			}
		}
		return true
	}
	for _, prov := range strings.Split(*flagProviders, ",") {
		if prov == p {
			return true
		}
	}
	return false
}

func shouldRunDomain(d string) bool {
	if *domains == "" {
		return true
	}
	for _, dom := range strings.Split(*domains, ",") {
		if dom == d {
			return true
		}
	}
	return false
}

// Version management. 2 Goals:
// 1. Someone who just does "go get" has at least some information.
// 2. If built with build.sh, more specific build information gets put in.
// Update the number here manually each release, so at least we have a range for go-get people.
var (
	SHA       = ""
	Version   = "0.1.0"
	BuildTime = ""
)

// printVersion prints the version banner.
func versionString() string {
	var version string
	if SHA != "" {
		version = fmt.Sprintf("%s (%s)", Version, SHA)
	} else {
		version = fmt.Sprintf("%s-dev", Version) //no SHA. '0.x.y-dev' indeicates it is run form source without build script.
	}
	if BuildTime != "" {
		i, err := strconv.ParseInt(BuildTime, 10, 64)
		if err == nil {
			tm := time.Unix(i, 0)
			version += fmt.Sprintf(" built %s", tm.Format(time.RFC822))
		}
	}
	return fmt.Sprintf("dnscontrol %s", version)
}
<commit_msg>allow args before or after flags (#179)<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/StackExchange/dnscontrol/models"
	"github.com/StackExchange/dnscontrol/pkg/js"
	"github.com/StackExchange/dnscontrol/pkg/nameservers"
	"github.com/StackExchange/dnscontrol/pkg/normalize"
	"github.com/StackExchange/dnscontrol/providers"
	_ "github.com/StackExchange/dnscontrol/providers/_all"
	"github.com/StackExchange/dnscontrol/providers/config"
)

//go:generate go run build/generate/generate.go

var jsFile = flag.String("js", "dnsconfig.js", "Javascript file containing dns config")
var credsFile = flag.String("creds", "creds.json", "Provider credentials JSON file")
var jsonFile = flag.String("json", "", "File containing intermediate JSON")

var jsonOutputPre = flag.String("debugrawjson", "", "Write JSON intermediate to this file pre-normalization.")
var jsonOutputPost = flag.String("debugjson", "", "During preview, write JSON intermediate to this file instead of stdout.")

var devMode = flag.Bool("dev", false, "Use helpers.js from disk instead of embedded")

var flagProviders = flag.String("providers", "", "Providers to enable (comma seperated list); default is all-but-bind. Specify 'all' for all (including bind)")
var domains = flag.String("domains", "", "Comma seperated list of domain names to include")

var interactive = flag.Bool("i", false, "Confirm or Exclude each correction before they run")

var delay = flag.Int64("d", 0, "delay between domains to avoid rate limits (in ms)")

// for forward compatibility:
// allow command to be before or after flags
func permuteArgs() {
	args := os.Args
	if len(args) <= 1 {
		return
	}
	first := args[1]
	if strings.HasPrefix(first, "-") {
		return
	}
	//non-flag is first arg. Move it to back
	for i := 2; i < len(args); i++ {
		args[i-1] = args[i]
	}
	args[len(args)-1] = first
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	permuteArgs()
	flag.Parse()
	if flag.NArg() != 1 {
		fmt.Println(`Usage: dnscontrol [options] cmd
		        cmd:
		           preview: Show changed that would happen.
		           push:    Make changes for real.
		           version: Print program version string.
		           print:   Print compiled data.
		           create-domains:   Pre-create domains in R53
		`)
		flag.PrintDefaults()
		return
	}
	command := flag.Arg(0)
	if command == "version" {
		fmt.Println(versionString())
		return
	}

	var dnsConfig *models.DNSConfig
	if *jsonFile != "" {
		text, err := ioutil.ReadFile(*jsonFile)
		if err != nil {
			log.Fatalf("Error reading %v: %v\n", *jsonFile, err)
		}
		err = json.Unmarshal(text, &dnsConfig)
		if err != nil {
			log.Fatalf("Error parsing JSON in (%v): %v", *jsonFile, err)
		}
	} else if *jsFile != "" {
		text, err := ioutil.ReadFile(*jsFile)
		if err != nil {
			log.Fatalf("Error reading %v: %v\n", *jsFile, err)
		}
		dnsConfig, err = js.ExecuteJavascript(string(text), *devMode)
		if err != nil {
			log.Fatalf("Error executing javasscript in (%v): %v", *jsFile, err)
		}
	}
	if dnsConfig == nil {
		log.Fatal("No config specified.")
	}
	if *jsonOutputPre != "" {
		dat, _ := json.MarshalIndent(dnsConfig, "", "  ")
		err := ioutil.WriteFile(*jsonOutputPre, dat, 0644)
		if err != nil {
			panic(err)
		}
	}

	errs := normalize.NormalizeAndValidateConfig(dnsConfig)
	if len(errs) > 0 {
		fmt.Printf("%d Validation errors:\n", len(errs))
		fatal := false
		for _, err := range errs {
			if _, ok := err.(normalize.Warning); ok {
				fmt.Printf("WARNING: %s\n", err)
			} else {
				fatal = true
				fmt.Printf("ERROR: %s\n", err)
			}
		}
		if fatal {
			log.Fatal("Exiting due to validation errors")
		}
	}

	if command == "print" {
		dat, _ := json.MarshalIndent(dnsConfig, "", "  ")
		if *jsonOutputPost == "" {
			fmt.Println("While running JS:", string(dat))
		} else {
			err := ioutil.WriteFile(*jsonOutputPost, dat, 0644)
			if err != nil {
				panic(err)
			}
		}
		return
	}

	providerConfigs, err := config.LoadProviderConfigs(*credsFile)
	if err != nil {
		log.Fatalf(err.Error())
	}
	nonDefaultProviders := []string{}
	for name, vals := range providerConfigs {
		// add "_exclude_from_defaults":"true" to a domain to exclude it from being run unless
		// -providers=all or -providers=name
		if vals["_exclude_from_defaults"] == "true" {
			nonDefaultProviders = append(nonDefaultProviders, name)
		}
	}
	registrars, err := providers.CreateRegistrars(dnsConfig, providerConfigs)
	if err != nil {
		log.Fatalf("Error creating registrars: %v\n", err)
	}
	dsps, err := providers.CreateDsps(dnsConfig, providerConfigs)
	if err != nil {
		log.Fatalf("Error creating dsps: %v\n", err)
	}

	fmt.Printf("Initialized %d registrars and %d dns service providers.\n", len(registrars), len(dsps))
	anyErrors, totalCorrections := false, 0
	switch command {
	case "create-domains":
		for _, domain := range dnsConfig.Domains {
			fmt.Println("*** ", domain.Name)
			for prov := range domain.DNSProviders {
				dsp, ok := dsps[prov]
				if !ok {
					log.Fatalf("DSP %s not declared.", prov)
				}
				if creator, ok := dsp.(providers.DomainCreator); ok {
					fmt.Println("  -", prov)
					err := creator.EnsureDomainExists(domain.Name)
					if err != nil {
						fmt.Printf("Error creating domain: %s\n", err)
					}
				}
			}
		}
	case "preview", "push":
	DomainLoop:
		for _, domain := range dnsConfig.Domains {
			if !shouldRunDomain(domain.Name) {
				continue
			}
			fmt.Printf("******************** Domain: %s\n", domain.Name)
			nsList, err := nameservers.DetermineNameservers(domain, 0, dsps)
			if err != nil {
				log.Fatal(err)
			}
			domain.Nameservers = nsList
			nameservers.AddNSRecords(domain)
			for prov := range domain.DNSProviders {
				dc, err := domain.Copy()
				if err != nil {
					log.Fatal(err)
				}
				shouldrun := shouldRunProvider(prov, dc, nonDefaultProviders)
				statusLbl := ""
				if !shouldrun {
					statusLbl = "(skipping)"
				}
				fmt.Printf("----- DNS Provider: %s... %s", prov, statusLbl)
				if !shouldrun {
					fmt.Println()
					continue
				}
				dsp, ok := dsps[prov]
				if !ok {
					log.Fatalf("DSP %s not declared.", prov)
				}
				corrections, err := dsp.GetDomainCorrections(dc)
				if err != nil {
					fmt.Println("ERROR")
					anyErrors = true
					fmt.Printf("Error getting corrections: %s\n", err)
					continue DomainLoop
				}
				totalCorrections += len(corrections)
				plural := "s"
				if len(corrections) == 1 {
					plural = ""
				}
				fmt.Printf("%d correction%s\n", len(corrections), plural)
				anyErrors = printOrRunCorrections(corrections, command) || anyErrors
			}
			if run := shouldRunProvider(domain.Registrar, domain, nonDefaultProviders); !run {
				continue
			}
			fmt.Printf("----- Registrar: %s\n", domain.Registrar)
			reg, ok := registrars[domain.Registrar]
			if !ok {
				log.Fatalf("Registrar %s not declared.", reg)
			}
			if len(domain.Nameservers) == 0 && domain.Metadata["no_ns"] != "true" {
				fmt.Printf("No nameservers declared; skipping registrar. Add {no_ns:'true'} to force.\n")
				continue
			}
			dc, err := domain.Copy()
			if err != nil {
				log.Fatal(err)
			}
			corrections, err := reg.GetRegistrarCorrections(dc)
			if err != nil {
				fmt.Printf("Error getting corrections: %s\n", err)
				anyErrors = true
				continue
			}
			totalCorrections += len(corrections)
			anyErrors = printOrRunCorrections(corrections, command) || anyErrors

			time.Sleep(time.Duration(*delay) * time.Millisecond)
		}
	default:
		log.Fatalf("Unknown command %s", command)
	}
	if os.Getenv("TEAMCITY_VERSION") != "" {
		fmt.Fprintf(os.Stderr, "##teamcity[buildStatus status='SUCCESS' text='%d corrections']", totalCorrections)
	}
	fmt.Printf("Done. %d corrections.\n", totalCorrections)
	if anyErrors {
		os.Exit(1)
	}
}

var reader = bufio.NewReader(os.Stdin)

func printOrRunCorrections(corrections []*models.Correction, command string) (anyErrors bool) {
	anyErrors = false
	if len(corrections) == 0 {
		return anyErrors
	}
	for i, correction := range corrections {
		fmt.Printf("#%d: %s\n", i+1, correction.Msg)
		if command == "push" {
			if *interactive {
				fmt.Print("Run? (Y/n): ")
				txt, err := reader.ReadString('\n')
				run := true
				if err != nil {
					run = false
				}
				txt = strings.ToLower(strings.TrimSpace(txt))
				if txt != "y" {
					run = false
				}
				if !run {
					fmt.Println("Skipping")
					continue
				}
			}
			err := correction.F()
			if err != nil {
				fmt.Println("FAILURE!", err)
				anyErrors = true
			} else {
				fmt.Println("SUCCESS!")
			}
		}
	}
	return anyErrors
}

func shouldRunProvider(p string, dc *models.DomainConfig, nonDefaultProviders []string) bool {
	if *flagProviders == "all" {
		return true
	}
	if *flagProviders == "" {
		for _, pr := range nonDefaultProviders {
			if pr == p {
				return false
			}
		}
		return true
	}
	for _, prov := range strings.Split(*flagProviders, ",") {
		if prov == p {
			return true
		}
	}
	return false
}

func shouldRunDomain(d string) bool {
	if *domains == "" {
		return true
	}
	for _, dom := range strings.Split(*domains, ",") {
		if dom == d {
			return true
		}
	}
	return false
}

// Version management. 2 Goals:
// 1. Someone who just does "go get" has at least some information.
// 2. If built with build.sh, more specific build information gets put in.
// Update the number here manually each release, so at least we have a range for go-get people.
var (
	SHA       = ""
	Version   = "0.1.0"
	BuildTime = ""
)

// printVersion prints the version banner.
func versionString() string {
	var version string
	if SHA != "" {
		version = fmt.Sprintf("%s (%s)", Version, SHA)
	} else {
		version = fmt.Sprintf("%s-dev", Version) //no SHA. '0.x.y-dev' indeicates it is run form source without build script.
	}
	if BuildTime != "" {
		i, err := strconv.ParseInt(BuildTime, 10, 64)
		if err == nil {
			tm := time.Unix(i, 0)
			version += fmt.Sprintf(" built %s", tm.Format(time.RFC822))
		}
	}
	return fmt.Sprintf("dnscontrol %s", version)
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "log"

    "flag"

    "bytes"
    "strings"

    "os/exec"

    "net/http"

    "encoding/json"
    "encoding/base64"

    "html/template"

    "strconv"

    "github.com/andreadipersio/pugo/libs/cache"
)

const (
    githubContentURL = "https://api.github.com/repos/%v/%v/contents/%v"
)

var (
    httpPort int

    repo     string
    owner    string
    token    string

    staticsPath  string
    templatesPath string

)

type GithubResponse struct {
    Content string `json:"content"`
}

func getTemplatePath(filename string) string {
    return fmt.Sprintf("%v/%v", templatesPath, filename)
}

func init() {
    flag.IntVar(&httpPort, "port", 8002, "HTTP Server port")

    flag.StringVar(&repo, "repo", "", "Github public repository name")
    flag.StringVar(&owner, "owner", "", "Github public repository owner")
    flag.StringVar(&staticsPath, "statics", "./statics", "path to static files")
    flag.StringVar(&templatesPath, "templates", "./templates", "path to html templates")
    flag.StringVar(&token, "token", "", "Github Personal Token")

    flag.Parse()
}

func getFilename(urlPath string) string {
    parts := strings.Split(urlPath, "/")
    filename := parts[len(parts) - 1]

    if filename == "" {
        return "index.md"
    }

    return fmt.Sprintf("%v.md", filename)
}

func rootHandler(w http.ResponseWriter, r *http.Request, localCache *cache.Cache) {
    var content string

    filename := getFilename(r.URL.Path)

    log.Printf("[REQ] %v", filename)

    refreshFlag := r.FormValue("refresh")

    getFromCache := func() string {
        localCache.GetChan <- filename
        return <-localCache.GetChan
    }

    getFromGithub := func() (content string) {
        githubResp := &GithubResponse{}

        fetch := func() error {
            remoteURL := fmt.Sprintf(githubContentURL, owner, repo, filename)

            log.Println(remoteURL)

            client := http.Client{}
            req, err := http.NewRequest("GET", remoteURL, nil)

            r.Header.Set("User-Agent", "andreadipersio/pugo")

            if token != "" {
                r.SetBasicAuth(owner, token)
            }

            if err != nil {
                panic(err)
            }

            resp, err := client.Do(req)

            if err != nil {
                panic(err)
            }

            defer resp.Body.Close()

            dec := json.NewDecoder(resp.Body)

            if err := dec.Decode(githubResp); err != nil {
                return err
            }

            return nil
        }

        decode := func() error {
            b64 := base64.StdEncoding

            if v, err := b64.DecodeString(githubResp.Content); err != nil {
                return err
            } else {
                content = string(v)
            }

            return nil
        }

        toHTML := func() error {
            cmd := exec.Command("markdown")

            if pipeWriter, err := cmd.StdinPipe(); err != nil {
                return err
            } else {
                pipeWriter.Write([]byte(content))
                pipeWriter.Close()
            }

            if output, err := cmd.CombinedOutput(); err != nil {
                return err
            } else {
                content = string(output)
            }

            if tmpl, err := template.ParseFiles(getTemplatePath("base.html"), getTemplatePath("article.html")); err != nil {
                return err
            } else {
                context := &struct { Content template.HTML } { template.HTML(content), }

                wr := bytes.NewBufferString("")

                if parseErr := tmpl.Execute(wr, context); parseErr != nil {
                    return parseErr
                }

                content = wr.String()
            }

            localCache.PutChan <-cache.CacheRequest{filename, content}

            return nil
        }

        if err := fetch(); err != nil {
            log.Panicf("Cannot get data from github: %v", err)
        }

        if err := decode(); err != nil {
            log.Panicf("Cannot decode github response: %v", err)
        }

        if err := toHTML(); err != nil {
            log.Panicf("Cannot convert markdown to html: %v", err)
        }

        log.Println("[Github] OK")

        return
    }

    if content = getFromCache(); content == "" || refreshFlag != "" {
        content = getFromGithub()
    }

    w.Header().Set("Content-Type", "text/html")
    w.Header().Set("Content-Length", strconv.Itoa(len(content)))

    fmt.Fprint(w, content)
}

func main() {
    httpAddr := fmt.Sprintf(":%v", httpPort)

    log.Printf("Serving static files from %v", staticsPath)

    log.Printf("Starting server on %v", httpAddr)

    localCache := cache.NewCache()
    go localCache.Run()

    http.HandleFunc("/favicon.ico", http.NotFound)
    http.Handle("/s/", http.StripPrefix("/s/", http.FileServer(http.Dir(staticsPath))))

    http.HandleFunc("/", func (w http.ResponseWriter, r *http.Request) {
        rootHandler(w, r, localCache)
    })

    log.Fatal(http.ListenAndServe(httpAddr, nil))
}
<commit_msg>Point index to README.md<commit_after>package main

import (
    "fmt"
    "log"

    "flag"

    "bytes"
    "strings"

    "os/exec"

    "net/http"

    "encoding/json"
    "encoding/base64"

    "html/template"

    "strconv"

    "github.com/andreadipersio/pugo/libs/cache"
)

const (
    githubContentURL = "https://api.github.com/repos/%v/%v/contents/%v"
)

var (
    httpPort int

    repo     string
    owner    string
    token    string

    staticsPath  string
    templatesPath string

)

type GithubResponse struct {
    Content string `json:"content"`
}

func getTemplatePath(filename string) string {
    return fmt.Sprintf("%v/%v", templatesPath, filename)
}

func init() {
    flag.IntVar(&httpPort, "port", 8002, "HTTP Server port")

    flag.StringVar(&repo, "repo", "", "Github public repository name")
    flag.StringVar(&owner, "owner", "", "Github public repository owner")
    flag.StringVar(&staticsPath, "statics", "./statics", "path to static files")
    flag.StringVar(&templatesPath, "templates", "./templates", "path to html templates")
    flag.StringVar(&token, "token", "", "Github Personal Token")

    flag.Parse()
}

func getFilename(urlPath string) string {
    parts := strings.Split(urlPath, "/")
    filename := parts[len(parts) - 1]

    if filename == "" {
        return "README.md"
    }

    return fmt.Sprintf("%v.md", filename)
}

func rootHandler(w http.ResponseWriter, r *http.Request, localCache *cache.Cache) {
    var content string

    filename := getFilename(r.URL.Path)

    log.Printf("[REQ] %v", filename)

    refreshFlag := r.FormValue("refresh")

    getFromCache := func() string {
        localCache.GetChan <- filename
        return <-localCache.GetChan
    }

    getFromGithub := func() (content string) {
        githubResp := &GithubResponse{}

        fetch := func() error {
            remoteURL := fmt.Sprintf(githubContentURL, owner, repo, filename)

            log.Println(remoteURL)

            client := http.Client{}
            req, err := http.NewRequest("GET", remoteURL, nil)

            r.Header.Set("User-Agent", "andreadipersio/pugo")

            if token != "" {
                r.SetBasicAuth(owner, token)
            }

            if err != nil {
                panic(err)
            }

            resp, err := client.Do(req)

            if err != nil {
                panic(err)
            }

            defer resp.Body.Close()

            dec := json.NewDecoder(resp.Body)

            if err := dec.Decode(githubResp); err != nil {
                return err
            }

            return nil
        }

        decode := func() error {
            b64 := base64.StdEncoding

            if v, err := b64.DecodeString(githubResp.Content); err != nil {
                return err
            } else {
                content = string(v)
            }

            return nil
        }

        toHTML := func() error {
            cmd := exec.Command("markdown")

            if pipeWriter, err := cmd.StdinPipe(); err != nil {
                return err
            } else {
                pipeWriter.Write([]byte(content))
                pipeWriter.Close()
            }

            if output, err := cmd.CombinedOutput(); err != nil {
                return err
            } else {
                content = string(output)
            }

            if tmpl, err := template.ParseFiles(getTemplatePath("base.html"), getTemplatePath("article.html")); err != nil {
                return err
            } else {
                context := &struct { Content template.HTML } { template.HTML(content), }

                wr := bytes.NewBufferString("")

                if parseErr := tmpl.Execute(wr, context); parseErr != nil {
                    return parseErr
                }

                content = wr.String()
            }

            localCache.PutChan <-cache.CacheRequest{filename, content}

            return nil
        }

        if err := fetch(); err != nil {
            log.Panicf("Cannot get data from github: %v", err)
        }

        if err := decode(); err != nil {
            log.Panicf("Cannot decode github response: %v", err)
        }

        if err := toHTML(); err != nil {
            log.Panicf("Cannot convert markdown to html: %v", err)
        }

        log.Println("[Github] OK")

        return
    }

    if content = getFromCache(); content == "" || refreshFlag != "" {
        content = getFromGithub()
    }

    w.Header().Set("Content-Type", "text/html")
    w.Header().Set("Content-Length", strconv.Itoa(len(content)))

    fmt.Fprint(w, content)
}

func main() {
    httpAddr := fmt.Sprintf(":%v", httpPort)

    log.Printf("Serving static files from %v", staticsPath)

    log.Printf("Starting server on %v", httpAddr)

    localCache := cache.NewCache()
    go localCache.Run()

    http.HandleFunc("/favicon.ico", http.NotFound)
    http.Handle("/s/", http.StripPrefix("/s/", http.FileServer(http.Dir(staticsPath))))

    http.HandleFunc("/", func (w http.ResponseWriter, r *http.Request) {
        rootHandler(w, r, localCache)
    })

    log.Fatal(http.ListenAndServe(httpAddr, nil))
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/aidan-/cloudtrailbeat/beater"
	"github.com/elastic/beats/libbeat/beat"
)

var Version = "0.0.1"
var Name = "cloudtrailbeat"

func main() {
	if err := beat.Run(Name, Version, beater.New()); err != nil {
		os.Exit(1)
	}
}
<commit_msg>bump version to 0.0.2<commit_after>package main

import (
	"os"

	"github.com/aidan-/cloudtrailbeat/beater"
	"github.com/elastic/beats/libbeat/beat"
)

var Version = "0.0.2"
var Name = "cloudtrailbeat"

func main() {
	if err := beat.Run(Name, Version, beater.New()); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//
package main

import (
	"flag"
	"log"
	"net/http"
)

var httpPort = flag.String("http", "80", "http port")
var httpsPort = flag.String("https", "443", "https port")
var certPem = flag.String("cert", "certs/fullchain.pem", "path to cert pem file")
var keyPem = flag.String("key", "certs/privkey.pem", "path key pem file")
var domain = flag.String("domain", "localhost", "web domain")
var pubdir = flag.String("pubdir", "public", "path to public directory")

func main() {
	flag.Parse()
	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(*pubdir))))
	go func() {
		err := http.ListenAndServeTLS(":"+*httpsPort, *certPem, *keyPem, nil)
		if err != nil {
			log.Fatal("ListenAndServeTLS:", err)
		}
	}()
	err := http.ListenAndServe(":"+*httpPort, http.RedirectHandler(func() string {
		if *httpsPort == "443" {
			return "https://" + *domain
		}
		return "https://" + *domain + ":" + *httpsPort
	}(), 301))
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<commit_msg>Removed redundant https url assembling code.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

//
package main

import (
	"flag"
	"log"
	"net/http"
)

var httpPort = flag.String("http", "80", "http port")
var httpsPort = flag.String("https", "443", "https port")
var certPem = flag.String("cert", "certs/fullchain.pem", "path to cert pem file")
var keyPem = flag.String("key", "certs/privkey.pem", "path key pem file")
var domain = flag.String("domain", "localhost", "web domain")
var pubdir = flag.String("pubdir", "public", "path to public directory")

func main() {
	flag.Parse()
	http.Handle("/", http.StripPrefix("/", http.FileServer(http.Dir(*pubdir))))
	go func() {
		err := http.ListenAndServeTLS(":"+*httpsPort, *certPem, *keyPem, nil)
		if err != nil {
			log.Fatal("ListenAndServeTLS:", err)
		}
	}()
	err := http.ListenAndServe(":"+*httpPort, http.RedirectHandler("https://"+*domain+":"+*httpsPort, 301))
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
)

func main() {
	// TODO: check if git binary present
	// TODO: load args from CLI
	// TODO: read config
	newAppName := "migrations"

	createAppDir(newAppName)
	addAppToGitIgnore(newAppName)
	commitToMasterRepo(newAppName)
	//fmt.Println("Add container to docker-compose.apps.yml")
	goToAppDir(newAppName)
	createReadme(newAppName)
	createDockerfile(newAppName)
	initializeGitRepo(newAppName)
}

func createAppDir(newAppName string) {
	err := os.Mkdir(newAppName, 0755)
	if err != nil {
		// Check if directory exists
		fmt.Println("Can't create dir")
	}
}

func addAppToGitIgnore(newAppName string) {
	//TODO: check if this line exists in file
	wd, _ := os.Getwd()
	f, err := os.OpenFile(wd + "/.gitignore", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println(err)
	}
	_, err = f.WriteString(newAppName + "\n")
	if err != nil {
		fmt.Println(err)
	}
	f.Close()
}

func commitToMasterRepo(newAppName string) {
	cmd := exec.Command("git", "add", ".gitignore")
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}

	cmd = exec.Command("git", "commit", "-am", "\"Add " + newAppName + " app\"")
	err = cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
}

func goToAppDir(newAppName string) {
	err := os.Chdir(newAppName)
	if err != nil {
		fmt.Println(err)
	}
}

func createReadme(newAppName string) {
	readmeFile, err := os.Create("README.md")
	if err != nil {
		fmt.Println(err)
	}
	readmeFile.Close()
}

func initializeGitRepo(newAppName string) {
	cmd := exec.Command("git", "init")
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}

	cmd = exec.Command("git", "add", ".")
	err = cmd.Run()
	if err != nil {
		fmt.Println(err)
	}

	cmd = exec.Command("git", "commit", "-am", "\"Initial commit\"")
	err = cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
}

func createDockerfile(newAppName string) {
	dockerfile, err := os.Create("Dockerfile")
	if err != nil {
		fmt.Println(err)
	}
	dockerfile.Close()
}

func createBuildScript(newAppName string) {
	buildScript, err := os.Create("build.sh")
	if err != nil {
		fmt.Println(err)
	}
	buildScript.Chmod(0755)
	buildScript.Close()
}
<commit_msg>Add todos<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
)

// TODO: implement the following features
// fres init [project name]
// fres create [app name] --lang ruby
// fres remove [app name]
// fres deploy [app name]
// fres deploy

func main() {
	// TODO: check if git binary present
	// TODO: load args from CLI
	// TODO: read config
	newAppName := "migrations"

	createAppDir(newAppName)
	addAppToGitIgnore(newAppName)
	commitToMasterRepo(newAppName)
	//fmt.Println("Add container to docker-compose.apps.yml")
	goToAppDir(newAppName)
	createReadme(newAppName)
	createDockerfile(newAppName)
	initializeGitRepo(newAppName)
	// TODO: create remote repo and push
}

func createAppDir(newAppName string) {
	err := os.Mkdir(newAppName, 0755)
	if err != nil {
		// Check if directory exists
		fmt.Println("Can't create dir")
	}
}

func addAppToGitIgnore(newAppName string) {
	//TODO: check if this line exists in file
	wd, _ := os.Getwd()
	f, err := os.OpenFile(wd + "/.gitignore", os.O_APPEND|os.O_WRONLY, os.ModeAppend)
	if err != nil {
		fmt.Println(err)
	}
	_, err = f.WriteString(newAppName + "\n")
	if err != nil {
		fmt.Println(err)
	}
	f.Close()
}

func commitToMasterRepo(newAppName string) {
	// TODO: use git lib for dealing with repos
	cmd := exec.Command("git", "add", ".gitignore")
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}

	cmd = exec.Command("git", "commit", "-am", "\"Add " + newAppName + " app\"")
	err = cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
}

func goToAppDir(newAppName string) {
	err := os.Chdir(newAppName)
	if err != nil {
		fmt.Println(err)
	}
}

func createReadme(newAppName string) {
	readmeFile, err := os.Create("README.md")
	if err != nil {
		fmt.Println(err)
	}
	readmeFile.Close()
}

func initializeGitRepo(newAppName string) {
	cmd := exec.Command("git", "init")
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
	}

	cmd = exec.Command("git", "add", ".")
	err = cmd.Run()
	if err != nil {
		fmt.Println(err)
	}

	cmd = exec.Command("git", "commit", "-am", "\"Initial commit\"")
	err = cmd.Run()
	if err != nil {
		fmt.Println(err)
	}
}

func createDockerfile(newAppName string) {
	dockerfile, err := os.Create("Dockerfile")
	if err != nil {
		fmt.Println(err)
	}
	dockerfile.Close()
}

func createBuildScript(newAppName string) {
	buildScript, err := os.Create("build.sh")
	if err != nil {
		fmt.Println(err)
	}
	buildScript.Chmod(0755)
	buildScript.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/aws/aws-sdk-go/service/route53/route53iface"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

const (
	envPrefix = "DDNS_"
)

var version string

type Config struct {
	Tag      string
	Domain   string
	Prefix   string
	Interval time.Duration
	Debug    bool
}

func initLogging(debug bool) {
	log.SetFormatter(&log.TextFormatter{})
	log.SetOutput(os.Stderr)
	log.SetLevel(log.InfoLevel)
	if debug {
		log.SetLevel(log.DebugLevel)
	}
}

func getInstances(tag Tag, client ec2iface.EC2API) ([]*ec2.Instance, error) {
	params := &ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			{
				Name: aws.String("tag:" + tag.Name),
				Values: aws.StringSlice([]string{tag.Value}),
			},
		},
	}

	resp, err := client.DescribeInstances(params)
	if err != nil {
		return []*ec2.Instance{}, errors.Wrapf(err, "Describing instances with tag [%s]", tag)
	}

	instances := []*ec2.Instance{}
	for _, r := range resp.Reservations {
		for _, i := range r.Instances {
			instances = append(instances, i)
		}
	}

	log.WithFields(log.Fields{
		"num": len(instances),
		"tag": tag,
	}).Debug("Fetched instaces")
	return instances, nil
}

func getIPs(tag Tag, client ec2iface.EC2API) ([]string, error) {
	instances, err := getInstances(tag, client)
	if err != nil {
		return []string{}, err
	}

	ips := []string{}
	for _, i := range instances {
		ips = append(ips, *i.PublicIpAddress)
	}
	return ips, nil
}

func getHostedZoneId(domain string, client route53iface.Route53API) (string, error) {
	hostedZoneRequest := &route53.ListHostedZonesByNameInput{
		DNSName:      aws.String(domain),
	}
	hostedZones, err := client.ListHostedZonesByName(hostedZoneRequest)
	if err != nil {
		return "", errors.Wrapf(err, "Listing hosted zones by name [%s]", domain)
	}
	if len(hostedZones.HostedZones) != 1 {
		return "", fmt.Errorf("Invalid number of hosted zones [%d] for domain [%s]", len(hostedZones.HostedZones), domain)
	}

	hostedZoneId := *hostedZones.HostedZones[0].Id
	log.WithField("id", hostedZoneId).Info("Fetched hosted zone id")
	return hostedZoneId, nil
}

func getRecords(hostedZoneId string, client route53iface.Route53API) ([]*route53.ResourceRecordSet, error) {
	recordSetsRequest := &route53.ListResourceRecordSetsInput{
		HostedZoneId: &hostedZoneId,
	}
	resp, err := client.ListResourceRecordSets(recordSetsRequest)
	if err != nil {
		return []*route53.ResourceRecordSet{}, errors.Wrapf(err, "Listing record sets for zone with ID [%s]", hostedZoneId)
	}
	return resp.ResourceRecordSets, nil
}

func getDNSNames(hostedZoneId, prefix string, client route53iface.Route53API) ([]string, error) {
	records, err := getRecords(hostedZoneId, client)
	if err != nil {
		return []string{}, err
	}

	names := []string{}
	for _, r := range records {
		if strings.HasPrefix(*r.Name, prefix) {
			names = append(names, *r.Name)
		}
	}
	log.WithFields(log.Fields{
		"num": len(names),
		"prefix": prefix,
		"hostedZone": hostedZoneId,
	}).Debug("Fetched DNS records")
	return names, nil
}

func StringSet(s []string) map[string]bool {
	set := map[string]bool{}
	for _, i := range s {
		set[i] = true
	}
	return set
}

func getIPFromDNS(dnsName, prefix string) string {
	pieces := strings.SplitN(dnsName, ".", 2)
	return strings.Replace(pieces[0][len(prefix):], "-", ".", -1)
}

func getDNSFromIP(ip, prefix string) string {
	return prefix + strings.Replace(ip, ".", "-", -1)
}

func findCreatedInstances(instanceIPSet, dnsIPSet map[string]bool) []string {
	createdInstances := []string{}
	for ip := range instanceIPSet {
		if _, ok := dnsIPSet[ip]; !ok {
			newIP := string(ip)
			log.WithField("ip", newIP).Debug("Found new instance")
			createdInstances = append(createdInstances, newIP)
		}
	}
	return createdInstances
}

func findDeletedInstances(instanceIPSet, dnsIPSet map[string]bool) []string {
	deletedInstances := []string{}
	for ip := range dnsIPSet {
		if _, ok := instanceIPSet[ip]; !ok {
			newIP := string(ip)
			log.WithField("ip", newIP).Debug("Found removed instance")
			deletedInstances = append(deletedInstances, newIP)
		}
	}
	return deletedInstances
}

func registerCreatedInstances(createdInstanceIPs []string, prefix, domain, hostedZoneId string, client route53iface.Route53API) error {
	if len(createdInstanceIPs) == 0 {
		return nil
	}

	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			Changes: []*route53.Change{},
		},
		HostedZoneId: aws.String(hostedZoneId),
	}

	for _, ip := range createdInstanceIPs {
		change := &route53.Change{
			Action: aws.String("CREATE"),
			ResourceRecordSet: &route53.ResourceRecordSet{
				Name: aws.String(getDNSFromIP(ip, prefix) + "." + domain),
				Type: aws.String("A"),
				ResourceRecords: []*route53.ResourceRecord{{Value: aws.String(ip)}},
				TTL: aws.Int64(60),
			},
		}
		params.ChangeBatch.Changes = append(params.ChangeBatch.Changes, change)
	}

	_, err := client.ChangeResourceRecordSets(params)
	if err != nil {
		return errors.Wrap(err, "Registering instance IPs")
	}

	log.WithFields(log.Fields{
		"number": len(createdInstanceIPs),
	}).Info("Registered DNS name for created instance")
	return nil
}

func removeDeletedInstances(deletedInstanceIPs []string, prefix, domain, hostedZoneId string, client route53iface.Route53API) error {
	if len(deletedInstanceIPs) == 0 {
		return nil
	}

	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			Changes: []*route53.Change{},
		},
		HostedZoneId: aws.String(hostedZoneId),
	}

	for _, ip := range deletedInstanceIPs {
		change := &route53.Change{
			Action: aws.String("DELETE"),
			ResourceRecordSet: &route53.ResourceRecordSet{
				Name: aws.String(getDNSFromIP(ip, prefix) + "." + domain),
			},
		}
		params.ChangeBatch.Changes = append(params.ChangeBatch.Changes, change)
	}

	_, err := client.ChangeResourceRecordSets(params)
	if err != nil {
		return errors.Wrap(err, "Removing instance IPs")
	}

	log.WithFields(log.Fields{
		"number": len(deletedInstanceIPs),
	}).Info("Removed DNS name for deleted instance")
	return nil
}

func reconcile(instanceIPs, dnsNames []string, prefix, domain, hostedZoneId string, client route53iface.Route53API) error {
	dnsIPs := []string{}
	for _, dnsName := range dnsNames {
		dnsIPs = append(dnsIPs, getIPFromDNS(dnsName, prefix))
	}

	dnsIPSet := StringSet(dnsIPs)
	instanceIPSet := StringSet(instanceIPs)

	createdInstanceIPs := findCreatedInstances(instanceIPSet, dnsIPSet)
	registerErr := registerCreatedInstances(createdInstanceIPs, prefix, domain, hostedZoneId, client)
	if registerErr != nil {
		return registerErr
	}

	deletedInstanceIPs := findDeletedInstances(instanceIPSet, dnsIPSet)
	removeErr := removeDeletedInstances(deletedInstanceIPs, prefix, domain, hostedZoneId, client)
	if removeErr != nil {
		return removeErr
	}

	return nil
}

func run(c *Config, clients *Clients) {
	initLogging(c.Debug)
	log.WithField("version", version).Info("DDNS")

	tag, err := NewTag(c.Tag)
	if err != nil {
		log.Fatal(err)
	}

	hostedZoneId, err := getHostedZoneId(c.Domain, clients.Route53Client)
	if err != nil {
		log.Fatal(err)
	}

	for {
		time.Sleep(c.Interval)
		ips, err := getIPs(tag, clients.EC2Client)
		if err != nil {
			log.Warnf("Error while fetching IPs: %s", err)
			continue
		}

		dnsEntries, err := getDNSNames(hostedZoneId, c.Prefix, clients.Route53Client)
		if err != nil {
			log.Warnf("Error while fetching DNS entries: %s", err)
			continue
		}

		err = reconcile(ips, dnsEntries, c.Prefix, c.Domain, hostedZoneId, clients.Route53Client)
		if err != nil {
			log.Warnf("Error while reconciling DNS entries: %s", err)
			continue
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "ddns"
	app.Usage = "Command line tool for dynamically generating domain entries for EC2 instances"
	app.Version = version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "tag, t",
			Usage:  "Tag used to filter EC2 instances. Format tag=value",
			EnvVar: envPrefix + "TAG",
		},
		cli.StringFlag{
			Name:   "domain, d",
			Usage:  "Domain id where records will be added",
			EnvVar: envPrefix + "DOMAIN",
		},
		cli.StringFlag{
			Name:   "prefix, p",
			Usage:  "Prefix to be added to the IP when creating records",
			Value:  "ddns-",
			EnvVar: envPrefix + "PREFIX",
		},
		cli.DurationFlag{
			Name:   "interval, i",
			Usage:  "Interval for querying the EC2 isntances",
			Value:  30 * time.Second,
			EnvVar: envPrefix + "INTERVAL",
		},
		cli.BoolFlag{
			Name:   "debug, D",
			Usage:  "Enable debug logging",
			EnvVar: envPrefix + "DEBUG",
		},
	}

	app.Action = func(c *cli.Context) error {
		run(&Config{
			Tag: c.String("tag"),
			Domain: c.String("domain"),
			Prefix: c.String("prefix"),
			Interval: c.Duration("interval"),
			Debug: c.Bool("debug"),
		}, &Clients{
			EC2Client: NewEC2Client(),
			Route53Client: NewRoute53Client(),
		})
		return nil
	}

	app.Run(os.Args)
}
<commit_msg>Filter to running or pending instances<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
	"github.com/aws/aws-sdk-go/service/route53"
	"github.com/aws/aws-sdk-go/service/route53/route53iface"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

const (
	envPrefix = "DDNS_"
)

var version string

type Config struct {
	Tag      string
	Domain   string
	Prefix   string
	Interval time.Duration
	Debug    bool
}

func initLogging(debug bool) {
	log.SetFormatter(&log.TextFormatter{})
	log.SetOutput(os.Stderr)
	log.SetLevel(log.InfoLevel)
	if debug {
		log.SetLevel(log.DebugLevel)
	}
}

func getInstances(tag Tag, client ec2iface.EC2API) ([]*ec2.Instance, error) {
	params := &ec2.DescribeInstancesInput{
		Filters: []*ec2.Filter{
			{
				Name: aws.String("tag:" + tag.Name),
				Values: aws.StringSlice([]string{tag.Value}),
			},
			{
				Name: aws.String("instance-state-name"),
				Values: aws.StringSlice([]string{"pending", "running"}),
			},
		},
	}

	resp, err := client.DescribeInstances(params)
	if err != nil {
		return []*ec2.Instance{}, errors.Wrapf(err, "Describing instances with tag [%s]", tag)
	}

	instances := []*ec2.Instance{}
	for _, r := range resp.Reservations {
		for _, i := range r.Instances {
			instances = append(instances, i)
		}
	}

	log.WithFields(log.Fields{
		"num": len(instances),
		"tag": tag,
	}).Debug("Fetched instaces")
	return instances, nil
}

func getIPs(tag Tag, client ec2iface.EC2API) ([]string, error) {
	instances, err := getInstances(tag, client)
	if err != nil {
		return []string{}, err
	}

	ips := []string{}
	for _, i := range instances {
		ips = append(ips, *i.PublicIpAddress)
	}
	return ips, nil
}

func getHostedZoneId(domain string, client route53iface.Route53API) (string, error) {
	hostedZoneRequest := &route53.ListHostedZonesByNameInput{
		DNSName:      aws.String(domain),
	}
	hostedZones, err := client.ListHostedZonesByName(hostedZoneRequest)
	if err != nil {
		return "", errors.Wrapf(err, "Listing hosted zones by name [%s]", domain)
	}
	if len(hostedZones.HostedZones) != 1 {
		return "", fmt.Errorf("Invalid number of hosted zones [%d] for domain [%s]", len(hostedZones.HostedZones), domain)
	}

	hostedZoneId := *hostedZones.HostedZones[0].Id
	log.WithField("id", hostedZoneId).Info("Fetched hosted zone id")
	return hostedZoneId, nil
}

func getRecords(hostedZoneId string, client route53iface.Route53API) ([]*route53.ResourceRecordSet, error) {
	recordSetsRequest := &route53.ListResourceRecordSetsInput{
		HostedZoneId: &hostedZoneId,
	}
	resp, err := client.ListResourceRecordSets(recordSetsRequest)
	if err != nil {
		return []*route53.ResourceRecordSet{}, errors.Wrapf(err, "Listing record sets for zone with ID [%s]", hostedZoneId)
	}
	return resp.ResourceRecordSets, nil
}

func getDNSNames(hostedZoneId, prefix string, client route53iface.Route53API) ([]string, error) {
	records, err := getRecords(hostedZoneId, client)
	if err != nil {
		return []string{}, err
	}

	names := []string{}
	for _, r := range records {
		if strings.HasPrefix(*r.Name, prefix) {
			names = append(names, *r.Name)
		}
	}
	log.WithFields(log.Fields{
		"num": len(names),
		"prefix": prefix,
		"hostedZone": hostedZoneId,
	}).Debug("Fetched DNS records")
	return names, nil
}

func StringSet(s []string) map[string]bool {
	set := map[string]bool{}
	for _, i := range s {
		set[i] = true
	}
	return set
}

func getIPFromDNS(dnsName, prefix string) string {
	pieces := strings.SplitN(dnsName, ".", 2)
	return strings.Replace(pieces[0][len(prefix):], "-", ".", -1)
}

func getDNSFromIP(ip, prefix string) string {
	return prefix + strings.Replace(ip, ".", "-", -1)
}

func findCreatedInstances(instanceIPSet, dnsIPSet map[string]bool) []string {
	createdInstances := []string{}
	for ip := range instanceIPSet {
		if _, ok := dnsIPSet[ip]; !ok {
			newIP := string(ip)
			log.WithField("ip", newIP).Debug("Found new instance")
			createdInstances = append(createdInstances, newIP)
		}
	}
	return createdInstances
}

func findDeletedInstances(instanceIPSet, dnsIPSet map[string]bool) []string {
	deletedInstances := []string{}
	for ip := range dnsIPSet {
		if _, ok := instanceIPSet[ip]; !ok {
			newIP := string(ip)
			log.WithField("ip", newIP).Debug("Found removed instance")
			deletedInstances = append(deletedInstances, newIP)
		}
	}
	return deletedInstances
}

func registerCreatedInstances(createdInstanceIPs []string, prefix, domain, hostedZoneId string, client route53iface.Route53API) error {
	if len(createdInstanceIPs) == 0 {
		return nil
	}

	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			Changes: []*route53.Change{},
		},
		HostedZoneId: aws.String(hostedZoneId),
	}

	for _, ip := range createdInstanceIPs {
		change := &route53.Change{
			Action: aws.String("CREATE"),
			ResourceRecordSet: &route53.ResourceRecordSet{
				Name: aws.String(getDNSFromIP(ip, prefix) + "." + domain),
				Type: aws.String("A"),
				ResourceRecords: []*route53.ResourceRecord{{Value: aws.String(ip)}},
				TTL: aws.Int64(60),
			},
		}
		params.ChangeBatch.Changes = append(params.ChangeBatch.Changes, change)
	}

	_, err := client.ChangeResourceRecordSets(params)
	if err != nil {
		return errors.Wrap(err, "Registering instance IPs")
	}

	log.WithFields(log.Fields{
		"number": len(createdInstanceIPs),
	}).Info("Registered DNS name for created instance")
	return nil
}

func removeDeletedInstances(deletedInstanceIPs []string, prefix, domain, hostedZoneId string, client route53iface.Route53API) error {
	if len(deletedInstanceIPs) == 0 {
		return nil
	}

	params := &route53.ChangeResourceRecordSetsInput{
		ChangeBatch: &route53.ChangeBatch{
			Changes: []*route53.Change{},
		},
		HostedZoneId: aws.String(hostedZoneId),
	}

	for _, ip := range deletedInstanceIPs {
		change := &route53.Change{
			Action: aws.String("DELETE"),
			ResourceRecordSet: &route53.ResourceRecordSet{
				Name: aws.String(getDNSFromIP(ip, prefix) + "." + domain),
				Type: aws.String("A"),
				ResourceRecords: []*route53.ResourceRecord{{Value: aws.String(ip)}},
				TTL: aws.Int64(60),
			},
		}
		params.ChangeBatch.Changes = append(params.ChangeBatch.Changes, change)
	}

	_, err := client.ChangeResourceRecordSets(params)
	if err != nil {
		return errors.Wrap(err, "Removing instance IPs")
	}

	log.WithFields(log.Fields{
		"number": len(deletedInstanceIPs),
	}).Info("Removed DNS name for deleted instance")
	return nil
}

func reconcile(instanceIPs, dnsNames []string, prefix, domain, hostedZoneId string, client route53iface.Route53API) error {
	dnsIPs := []string{}
	for _, dnsName := range dnsNames {
		dnsIPs = append(dnsIPs, getIPFromDNS(dnsName, prefix))
	}

	dnsIPSet := StringSet(dnsIPs)
	instanceIPSet := StringSet(instanceIPs)

	createdInstanceIPs := findCreatedInstances(instanceIPSet, dnsIPSet)
	registerErr := registerCreatedInstances(createdInstanceIPs, prefix, domain, hostedZoneId, client)
	if registerErr != nil {
		return registerErr
	}

	deletedInstanceIPs := findDeletedInstances(instanceIPSet, dnsIPSet)
	removeErr := removeDeletedInstances(deletedInstanceIPs, prefix, domain, hostedZoneId, client)
	if removeErr != nil {
		return removeErr
	}

	return nil
}

func run(c *Config, clients *Clients) {
	initLogging(c.Debug)
	log.WithField("version", version).Info("DDNS")

	tag, err := NewTag(c.Tag)
	if err != nil {
		log.Fatal(err)
	}

	hostedZoneId, err := getHostedZoneId(c.Domain, clients.Route53Client)
	if err != nil {
		log.Fatal(err)
	}

	for {
		time.Sleep(c.Interval)
		ips, err := getIPs(tag, clients.EC2Client)
		if err != nil {
			log.Warnf("Error while fetching IPs: %s", err)
			continue
		}

		dnsEntries, err := getDNSNames(hostedZoneId, c.Prefix, clients.Route53Client)
		if err != nil {
			log.Warnf("Error while fetching DNS entries: %s", err)
			continue
		}

		err = reconcile(ips, dnsEntries, c.Prefix, c.Domain, hostedZoneId, clients.Route53Client)
		if err != nil {
			log.Warnf("Error while reconciling DNS entries: %s", err)
			continue
		}
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "ddns"
	app.Usage = "Command line tool for dynamically generating domain entries for EC2 instances"
	app.Version = version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "tag, t",
			Usage:  "Tag used to filter EC2 instances. Format tag=value",
			EnvVar: envPrefix + "TAG",
		},
		cli.StringFlag{
			Name:   "domain, d",
			Usage:  "Domain id where records will be added",
			EnvVar: envPrefix + "DOMAIN",
		},
		cli.StringFlag{
			Name:   "prefix, p",
			Usage:  "Prefix to be added to the IP when creating records",
			Value:  "ddns-",
			EnvVar: envPrefix + "PREFIX",
		},
		cli.DurationFlag{
			Name:   "interval, i",
			Usage:  "Interval for querying the EC2 isntances",
			Value:  30 * time.Second,
			EnvVar: envPrefix + "INTERVAL",
		},
		cli.BoolFlag{
			Name:   "debug, D",
			Usage:  "Enable debug logging",
			EnvVar: envPrefix + "DEBUG",
		},
	}

	app.Action = func(c *cli.Context) error {
		run(&Config{
			Tag: c.String("tag"),
			Domain: c.String("domain"),
			Prefix: c.String("prefix"),
			Interval: c.Duration("interval"),
			Debug: c.Bool("debug"),
		}, &Clients{
			EC2Client: NewEC2Client(),
			Route53Client: NewRoute53Client(),
		})
		return nil
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"reflect"

	"code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/v2"
	"code.cloudfoundry.org/cli/utils/panichandler"
	"github.com/jessevdk/go-flags"
)

func main() {
	defer panichandler.HandlePanic()
	parse(os.Args[1:])
}

func parse(args []string) {
	parser := flags.NewParser(&v2.Commands, flags.HelpFlag)
	parser.CommandHandler = myCommandHandler
	extraArgs, err := parser.ParseArgs(args)
	if err == nil {
		return
	}

	if flagErr, ok := err.(*flags.Error); ok {
		switch flagErr.Type {
		case flags.ErrHelp, flags.ErrUnknownFlag:
			field, found := reflect.TypeOf(v2.Commands).FieldByNameFunc(
				func(fieldName string) bool {
					field, _ := reflect.TypeOf(v2.Commands).FieldByName(fieldName)
					return parser.Active != nil && parser.Active.Name == field.Tag.Get("command")
				},
			)

			if found {
				parse([]string{"help", field.Name})
				return
			}

			switch len(extraArgs) {
			case 0:
				parse([]string{"help"})
			case 1:
				parse([]string{"help", extraArgs[0]})
			default:
				parse(extraArgs[1:])
			}
		case flags.ErrRequired:
			fmt.Printf("%s\n\n", flagErr.Error())
			parse(append([]string{"help"}, args...))
		case flags.ErrUnknownCommand:
			cmd.Main(os.Getenv("CF_TRACE"), os.Args)
		default:
			fmt.Printf("unexpected flag error\ntype: %s\nmessage: %s\n", flagErr.Type, flagErr.Error())
		}
	} else {
		fmt.Println("unexpected non-flag error:", err.Error())
	}
}

func myCommandHandler(cmd flags.Commander, args []string) error {
	if extendedCmd, ok := cmd.(commands.ExtendedCommander); ok {
		err := extendedCmd.Setup()
		if err != nil {
			return err
		}
		return extendedCmd.Execute(args)
	}

	return fmt.Errorf("unable to setup command")
}
<commit_msg>support additional error cases in help<commit_after>package main

import (
	"fmt"
	"os"
	"reflect"

	"code.cloudfoundry.org/cli/cf/cmd"
	"code.cloudfoundry.org/cli/commands"
	"code.cloudfoundry.org/cli/commands/v2"
	"code.cloudfoundry.org/cli/utils/panichandler"
	"github.com/jessevdk/go-flags"
)

func main() {
	defer panichandler.HandlePanic()
	parse(os.Args[1:])
}

func parse(args []string) {
	parser := flags.NewParser(&v2.Commands, flags.HelpFlag)
	parser.CommandHandler = myCommandHandler
	extraArgs, err := parser.ParseArgs(args)
	if err == nil {
		return
	}

	if flagErr, ok := err.(*flags.Error); ok {
		switch flagErr.Type {
		case flags.ErrHelp, flags.ErrUnknownFlag, flags.ErrExpectedArgument:
			_, found := reflect.TypeOf(v2.Commands).FieldByNameFunc(
				func(fieldName string) bool {
					field, _ := reflect.TypeOf(v2.Commands).FieldByName(fieldName)
					return parser.Active != nil && parser.Active.Name == field.Tag.Get("command")
				},
			)

			if found {
				parse([]string{"help", parser.Active.Name})
			} else {
				switch len(extraArgs) {
				case 0:
					parse([]string{"help"})
				case 1:
					parse([]string{"help", extraArgs[0]})
				default:
					parse(extraArgs[1:])
				}
			}

			if flagErr.Type == flags.ErrUnknownFlag || flagErr.Type == flags.ErrExpectedArgument {
				os.Exit(1)
			}
		case flags.ErrRequired:
			fmt.Printf("%s\n\n", flagErr.Error())
			parse(append([]string{"help"}, args...))
			os.Exit(1)
		case flags.ErrUnknownCommand:
			cmd.Main(os.Getenv("CF_TRACE"), os.Args)
		default:
			fmt.Printf("unexpected flag error\ntype: %s\nmessage: %s\n", flagErr.Type, flagErr.Error())
		}
	} else {
		fmt.Println("unexpected error:", err.Error())
	}
}

func myCommandHandler(cmd flags.Commander, args []string) error {
	if extendedCmd, ok := cmd.(commands.ExtendedCommander); ok {
		err := extendedCmd.Setup()
		if err != nil {
			return err
		}
		return extendedCmd.Execute(args)
	}

	return fmt.Errorf("unable to setup command")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/canopener/PongPlusPlus-Server/server/connection"
	"log"
	"net"
)

func main() {
	fmt.Println("Server listening on localhost:3000")
	listener, err := net.Listen("tcp", "localhost:3000")
	if err != nil {
		log.Fatalln(err)
	}
	defer listener.Close()

	for {
		socket, err := listener.Accept()
		if err != nil {
			log.Fatalln(err)
		}
		conn := connection.NewConnection(socket)
		go conn.StartReader()
		go conn.StartWriter()
		go listenMessage(&conn)
	}
}

func listenMessage(conn *connection.Connection) {
	for {
		select {
		case message := <-conn.IncommingMessages:
			fmt.Println("New message: ", string(message))
			conn.OutgoingMessages <- message
		}
	}
}
<commit_msg>More refactoring<commit_after>package main

import (
	"fmt"
	"github.com/canopener/PongPlusPlus-Server/connection"
	"log"
	"net"
)

func main() {
	fmt.Println("Server listening on localhost:3000")
	listener, err := net.Listen("tcp", "localhost:3000")
	if err != nil {
		log.Fatalln(err)
	}
	defer listener.Close()

	for {
		socket, err := listener.Accept()
		if err != nil {
			log.Fatalln(err)
		}
		conn := connection.NewConnection(socket)
		go conn.StartReader()
		go conn.StartWriter()
		go listenMessage(&conn)
	}
}

func listenMessage(conn *connection.Connection) {
	for {
		select {
		case message := <-conn.IncommingMessages:
			fmt.Println("New message: ", string(message))
			conn.OutgoingMessages <- message
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/cheggaaa/pb"
	"io"
	"log"
	"math"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"strings"
)

var scratchDir string

var (
	subdivisions *int = flag.Int("subdivisions", 10, "Slices per axis")
	tolerance    *int = flag.Int("tolerance", 100, "Color delta tolerance, higher = more tolerant")
)

func init() {
	flag.Parse()

	if flag.NArg() < 1 {
		fmt.Println("usage: imgdedup [options] [<directories>/files]")
		flag.PrintDefaults()
		os.Exit(1)
	}
}

func init() {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	scratchDir = path.Join(usr.HomeDir, ".imgdedup")

	if _, err := os.Stat(scratchDir); err != nil {
		if os.IsNotExist(err) {
			os.Mkdir(scratchDir, 0700)
		} else {
			log.Fatal(err)
		}
	}
}

func main() {

	imgdata := make(map[string]*imageInfo)

	fileList := getFiles(flag.Args())

	bar := pb.StartNew(len(fileList))
	bar.Output = os.Stderr

	for _, imgpath := range fileList {
		bar.Increment()

		file, err := os.Open(imgpath)
		if err != nil {
			log.Fatal(err)
		}

		fExt := strings.ToLower(filepath.Ext(imgpath))
		if fExt == ".png" || fExt == ".jpg" || fExt == ".jpeg" || fExt == ".gif" {

			fi, err := file.Stat()
			if err != nil {
				log.Fatal(err)
			}

			h := md5.New()

			cacheUnit := imgpath + "|" + string(*subdivisions) + "|" + string(fi.Size()) + string(fi.ModTime().Unix())

			io.WriteString(h, cacheUnit)
			cachename := path.Join(scratchDir, fmt.Sprintf("%x", h.Sum(nil))+".tmp")

			imginfo, err := loadCache(cachename)

			if err != nil {

				imginfo, err = scanImg(file)
				if err != nil {
					log.Print(imgpath, " - ", err)
					continue
				}

				storeCache(cachename, imginfo)
			}

			imgdata[imgpath] = imginfo

			file.Close()

		} else {
			file.Close()
		}
	}

	bar.Finish()

	fileLength := len(fileList)

	for i := 0; i < fileLength-1; i++ {
		for j := i + 1; j < fileLength-1; j++ {

			filename1 := fileList[i]
			filename2 := fileList[j]

			imgdata1, ok1 := imgdata[filename1]
			imgdata2, ok2 := imgdata[filename2]

			if ok1 && ok2 {

				avgdata1 := imgdata1.Data
				avgdata2 := imgdata2.Data

				if filename1 == filename2 {
					continue
				}

				var xdiff uint64 = 0

				for rX := 0; rX < *subdivisions; rX++ {
					for rY := 0; rY < *subdivisions; rY++ {
						aa := avgdata1[rX][rY]
						bb := avgdata2[rX][rY]

						xdiff += absdiff(absdiff(absdiff(aa[0], bb[0]), absdiff(aa[1], bb[1])), absdiff(aa[2], bb[2]))
					}
				}

				if xdiff < uint64(*tolerance) {
					fmt.Println(filename1, filename2)
					fmt.Println(xdiff)
				}

			}

		}
	}

}

func getFiles(paths []string) []string {
	var fileList []string

	for _, imgpath := range paths {

		file, err := os.Open(imgpath)
		if err != nil {
			log.Fatal(err)
		}

		fi, err := file.Stat()
		if err != nil {
			log.Fatal(err)
		}

		switch mode := fi.Mode(); {
		case mode.IsDir():
			// fmt.Println("directory")
			filepath.Walk(imgpath, func(path string, f os.FileInfo, err error) error {

				submode := f.Mode()
				if submode.IsRegular() {
					fpath, _ := filepath.Abs(path)
					fileList = append(fileList, fpath)
				}

				return nil
			})
		case mode.IsRegular():
			// fmt.Println("file")
			fpath, _ := filepath.Abs(imgpath)
			fileList = append(fileList, fpath)
		}

		file.Close()

	}

	return fileList
}

func absdiff(a uint64, b uint64) uint64 {
	return uint64(math.Abs(float64(a) - float64(b)))
}
<commit_msg>Skip files starting with "."<commit_after>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/cheggaaa/pb"
	"io"
	"log"
	"math"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"strings"
)

var scratchDir string

var (
	subdivisions *int = flag.Int("subdivisions", 10, "Slices per axis")
	tolerance    *int = flag.Int("tolerance", 100, "Color delta tolerance, higher = more tolerant")
)

func init() {
	flag.Parse()

	if flag.NArg() < 1 {
		fmt.Println("usage: imgdedup [options] [<directories>/files]")
		flag.PrintDefaults()
		os.Exit(1)
	}
}

func init() {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	scratchDir = path.Join(usr.HomeDir, ".imgdedup")

	if _, err := os.Stat(scratchDir); err != nil {
		if os.IsNotExist(err) {
			os.Mkdir(scratchDir, 0700)
		} else {
			log.Fatal(err)
		}
	}
}

func main() {

	imgdata := make(map[string]*imageInfo)

	fileList := getFiles(flag.Args())

	bar := pb.StartNew(len(fileList))
	bar.Output = os.Stderr

	for _, imgpath := range fileList {
		bar.Increment()

		file, err := os.Open(imgpath)
		if err != nil {
			log.Fatal(err)
		}

		fExt := strings.ToLower(filepath.Ext(imgpath))
		if fExt == ".png" || fExt == ".jpg" || fExt == ".jpeg" || fExt == ".gif" {

			fi, err := file.Stat()
			if err != nil {
				log.Fatal(err)
			}

			h := md5.New()

			cacheUnit := imgpath + "|" + string(*subdivisions) + "|" + string(fi.Size()) + string(fi.ModTime().Unix())

			io.WriteString(h, cacheUnit)
			cachename := path.Join(scratchDir, fmt.Sprintf("%x", h.Sum(nil))+".tmp")

			imginfo, err := loadCache(cachename)

			if err != nil {

				imginfo, err = scanImg(file)
				if err != nil {
					log.Print(imgpath, " - ", err)
					continue
				}

				storeCache(cachename, imginfo)
			}

			imgdata[imgpath] = imginfo

			file.Close()

		} else {
			file.Close()
		}
	}

	bar.Finish()

	fileLength := len(fileList)

	for i := 0; i < fileLength-1; i++ {
		for j := i + 1; j < fileLength-1; j++ {

			filename1 := fileList[i]
			filename2 := fileList[j]

			imgdata1, ok1 := imgdata[filename1]
			imgdata2, ok2 := imgdata[filename2]

			if ok1 && ok2 {

				avgdata1 := imgdata1.Data
				avgdata2 := imgdata2.Data

				if filename1 == filename2 {
					continue
				}

				var xdiff uint64 = 0

				for rX := 0; rX < *subdivisions; rX++ {
					for rY := 0; rY < *subdivisions; rY++ {
						aa := avgdata1[rX][rY]
						bb := avgdata2[rX][rY]

						xdiff += absdiff(absdiff(absdiff(aa[0], bb[0]), absdiff(aa[1], bb[1])), absdiff(aa[2], bb[2]))
					}
				}

				if xdiff < uint64(*tolerance) {
					fmt.Println(filename1, filename2)
					fmt.Println(xdiff)
				}

			}

		}
	}

}

func getFiles(paths []string) []string {
	var fileList []string

	for _, imgpath := range paths {

		file, err := os.Open(imgpath)
		if err != nil {
			log.Fatal(err)
		}

		fi, err := file.Stat()
		if err != nil {
			log.Fatal(err)
		}

		switch mode := fi.Mode(); {
		case mode.IsDir():
			// fmt.Println("directory")
			filepath.Walk(imgpath, func(path string, f os.FileInfo, err error) error {

				submode := f.Mode()
				if submode.IsRegular() {
					fpath, _ := filepath.Abs(path)

					base := filepath.Base(fpath)
					if string(base[0]) == "." {
						return nil
					}

					fileList = append(fileList, fpath)
				}

				return nil
			})
		case mode.IsRegular():
			// fmt.Println("file")
			fpath, _ := filepath.Abs(imgpath)
			fileList = append(fileList, fpath)
		}

		file.Close()

	}

	return fileList
}

func absdiff(a uint64, b uint64) uint64 {
	return uint64(math.Abs(float64(a) - float64(b)))
}
<|endoftext|>


<commit_before>package main

import "math"
import "math/rand"
import "math/big"
import crand "crypto/rand"
import "fmt"
import "strings"
import "flag"

var players []*Player
var debug bool
var roundsNeeded int

func main() {
	seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(err)
	}
	rand.Seed(seed.Int64())

	flag.BoolVar(&debug, "v", false, "Provide more verbose output for debugging purposes")
	flag.IntVar(&roundsNeeded, "r", 4, "The number of rounds that must be won to win the game")
	single := flag.Bool("s", false, "Ignore -r and stop after running only a single round of the game.")
	flag.Parse()
	if *single {
		roundsNeeded = 1
	}

	players = make([]*Player, flag.NArg())
	var startPlayer = 0

	var deck = new(Deck)

	for i, _ := range players {
		players[i] = new(Player)
	}

	for won, _ := PlayerWon(); !won; won, _ = PlayerWon() {
		deck.Init()
		for i, _ := range players {
			players[i].Init(flag.Arg(i), i, len(players), startPlayer)
			cardDrawn := deck.Draw()
			players[i].Send("draw", cardDrawn)
			players[i].AddToHand(cardDrawn)
			Debug("*** Player", i, "drew", cardDrawn, "***")
			Debug("*** Player", i, "hand:", players[i].HandString(), "***")
		}

		_ = deck.Draw() //throw away a card to prevent perfect information

		var currentPlayer = startPlayer
		for !deck.IsEmpty() && (SurvivingPlayers() > 1) {
			var fullMove, action, cardStr, queryStr string
			var target, n int
			var card, cardDrawn Card

			if players[currentPlayer].lost {
				goto endTurn
			}

			SendAll("player", currentPlayer)
			players[currentPlayer].protected = false //protection expires on your turn

			cardDrawn = deck.Draw()
			players[currentPlayer].Send("draw", cardDrawn)
			players[currentPlayer].AddToHand(cardDrawn)
			if players[currentPlayer].HasInHand(Minister) &&
				(players[currentPlayer].HasInHand(Wizard) || players[currentPlayer].HasInHand(General)) {
				Debug("*** Player", currentPlayer, "drew", cardDrawn, "***")
				Debug("*** Player", currentPlayer, "was forced to discard the minister ***")
				_ = players[currentPlayer].RemoveFromHand(Minister)
				SendAll("played", currentPlayer, Minister)
				goto endTurn
			}

			Debug("*** Player", currentPlayer, "drew", cardDrawn, "***")
			Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")

			fullMove = players[currentPlayer].Recieve()
			Debug("*** Player", currentPlayer, "move:", fullMove, "***")

			n, _ = fmt.Sscan(fullMove, &action, &cardStr, &target, &queryStr)

			if !strings.EqualFold(action, "play") {
				//must be "forfeit" or some invalid action
				Debug("*** ERROR: Player", currentPlayer, "made an illegal play or forfeited ***")
				Out(currentPlayer)
				goto endTurn
			}

			if n < 2 {
				//malformed move
				Debug("*** ERROR: Player", currentPlayer, "passed too few arguments ***")
				Out(currentPlayer)
				goto endTurn
			}

			card = ParseCard(cardStr)
			if ok := players[currentPlayer].RemoveFromHand(card); !ok {
				//trying to play a card you don't have is illegal
				Debug("*** ERROR: Player", currentPlayer, "played a card they dont have:", card, "***")
				Out(currentPlayer)
				goto endTurn
			}
			Debug("*** Player", currentPlayer, "played", card, "***")
			Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")

			//check the right arguments have been given
			switch card {
			case Princess, Minister, Priestess:
				if n != 2 {
					Debug("*** ERROR: Player", currentPlayer, "passed the wrong number of arguments for the played card ***")
					Out(currentPlayer)
					goto endTurn
				}
			case General, Wizard, Knight, Clown:
				if n != 3 {
					Debug("*** ERROR: Player", currentPlayer, "passed the wrong number of arguments for the played card ***")
					Out(currentPlayer)
					goto endTurn
				}

			case Soldier:
				if n != 4 {
					Debug("*** ERROR: Player", currentPlayer, "passed the wrong number of arguments for the played card ***")
					Out(currentPlayer)
					goto endTurn
				}
				if (target >= len(players)) && (target < 0) {
					Debug("*** ERROR: Player", currentPlayer, "passed a bad target argument ***")
					Out(currentPlayer)
					goto endTurn
				}
			}

			SendAll("played", currentPlayer, card)

			//check target is okay
			switch card {
			case General, Wizard, Knight, Clown, Soldier:
				if (target >= len(players)) && (target < 0) {
					Debug("*** ERROR: Player", currentPlayer, "passed a bad target argument ***")
					Out(currentPlayer)
					goto endTurn
				}
				if players[target].lost {
					Debug("*** ERROR: Player", currentPlayer, "attempted to target a player who has already lost ***")
					Out(currentPlayer)
					goto endTurn
				}
				if players[target].protected {
					Debug("*** Player", currentPlayer, "tried to target Player", target, "who was protected by a priestess ***")
					Out(currentPlayer)
					goto endTurn
				}
				if target == currentPlayer {
					Debug("*** Player", currentPlayer, "tried to target themself ***")
					var availableTargets = 0
					for _, v := range players {
						if !(v.protected || v.lost) {
							availableTargets++
						}
					}
					if availableTargets <= 1 {
						Debug("*** Player", currentPlayer, "is permitted to target themself as there are no other valid targets left ***")
					} else {
						Out(currentPlayer)
						goto endTurn
					}
				}
			}

			switch card {
			case Princess:
				Out(currentPlayer)
			case Minister:
			case General:
				players[currentPlayer].Send("swap", players[target].hand[0])
				players[target].Send("swap", players[currentPlayer].hand[0])
				players[currentPlayer].hand[0], players[target].hand[0] = players[target].hand[0], players[currentPlayer].hand[0]
				Debug("*** Player", currentPlayer, "and Player", target, "swapped hands ***")
				Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")
				Debug("*** Player", target, "hand:", players[target].HandString(), "***")
			case Wizard:
				Debug("*** Player", target, "discards hand and draws a new card ***")
				SendAll("discard", target, players[target].hand[0])
				if players[target].hand[0] == Princess {
					Out(target)
					goto endTurn
				}
				if deck.IsEmpty() {
					Debug("*** Player", target, "can't draw as there are no cards left ***")
					Out(target)
					goto endTurn
				}
				players[target].hand[0] = deck.Draw()
				players[target].Send("draw", players[target].hand[0])
				Debug("*** Player", target, "hand:", players[target].HandString(), "***")
			case Priestess:
				players[currentPlayer].protected = true
			case Knight:
				Debug("*** Player", currentPlayer, "and Player", target, "compare hands ***")
				Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")
				Debug("*** Player", target, "hand:", players[target].HandString(), "***")
				players[target].Send("reveal", currentPlayer, players[currentPlayer].hand[0])
				players[currentPlayer].Send("reveal", target, players[target].hand[0])
				if players[currentPlayer].HandValue() > players[target].HandValue() {
					Out(target)
				} else if players[currentPlayer].HandValue() < players[target].HandValue() {
					Out(currentPlayer)
				}
			case Clown:
				Debug("*** Player", target, "reveals a", players[target].hand[0], "from their hand to Player", currentPlayer, "***")
				players[currentPlayer].Send("reveal", target, players[target].hand[0])
			case Soldier:
				Debug("*** Player", currentPlayer, "asked Player", target, "if they have a", queryStr, "in their hand ***")
				if players[target].HasInHand(ParseCard(queryStr)) {
					Out(target)
				}
			default:
				Debug("*** ERROR: Player", currentPlayer, "played an illegal card:", cardStr, "***")
				Out(currentPlayer)
			}

		endTurn:
			currentPlayer++
			if currentPlayer >= len(players) {
				currentPlayer = 0
			}
		}
		startPlayer++
		if startPlayer >= len(players) {
			startPlayer = 0
		}

		winningHandValue := 0
		for i, _ := range players {
			if players[i].HandValue() > winningHandValue {
				winningHandValue = players[i].HandValue()
			}
		}
		for i, _ := range players {
			if players[i].HandValue() == winningHandValue {
				players[i].roundsWon++
			}
			players[i].Forfeit()
		}
		fmt.Print("Round ended.\t")
		for i, p := range players {
			fmt.Print("Player ", i, " score: ", p.roundsWon, "\t")
		}
		fmt.Println()
	}
	_, winner := PlayerWon()
	fmt.Println("Player won:", winner, players[winner].name)
}

func SendAll(args ...interface{}) {
	for _, player := range players {
		if !player.lost {
			player.Send(args...)
		}
	}
}

func SurvivingPlayers() int {
	result := 0
	for _, player := range players {
		if !player.lost {
			result++
		}
	}
	return result
}

func PlayerWon() (bool, int) {
	for i, p := range players {
		if p.roundsWon >= roundsNeeded {
			return true, i
		}
	}
	return false, 0
}

func Out(playerNumber int) {
	players[playerNumber].Forfeit()
	SendAll("out", playerNumber, players[playerNumber].HandString())
	Debug("*** Player", playerNumber, "out:", players[playerNumber].HandString(), "***")
}

func Debug(args ...interface{}) {
	if debug {
		fmt.Println(args...)
	}
}
<commit_msg>Made targetting a Soldier with a Soldier illegal<commit_after>package main

import "math"
import "math/rand"
import "math/big"
import crand "crypto/rand"
import "fmt"
import "strings"
import "flag"

var players []*Player
var debug bool
var roundsNeeded int

func main() {
	seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
	if err != nil {
		panic(err)
	}
	rand.Seed(seed.Int64())

	flag.BoolVar(&debug, "v", false, "Provide more verbose output for debugging purposes")
	flag.IntVar(&roundsNeeded, "r", 4, "The number of rounds that must be won to win the game")
	single := flag.Bool("s", false, "Ignore -r and stop after running only a single round of the game.")
	flag.Parse()
	if *single {
		roundsNeeded = 1
	}

	players = make([]*Player, flag.NArg())
	var startPlayer = 0

	var deck = new(Deck)

	for i, _ := range players {
		players[i] = new(Player)
	}

	for won, _ := PlayerWon(); !won; won, _ = PlayerWon() {
		deck.Init()
		for i, _ := range players {
			players[i].Init(flag.Arg(i), i, len(players), startPlayer)
			cardDrawn := deck.Draw()
			players[i].Send("draw", cardDrawn)
			players[i].AddToHand(cardDrawn)
			Debug("*** Player", i, "drew", cardDrawn, "***")
			Debug("*** Player", i, "hand:", players[i].HandString(), "***")
		}

		_ = deck.Draw() //throw away a card to prevent perfect information

		var currentPlayer = startPlayer
		for !deck.IsEmpty() && (SurvivingPlayers() > 1) {
			var fullMove, action, cardStr, queryStr string
			var target, n int
			var card, cardDrawn Card

			if players[currentPlayer].lost {
				goto endTurn
			}

			SendAll("player", currentPlayer)
			players[currentPlayer].protected = false //protection expires on your turn

			cardDrawn = deck.Draw()
			players[currentPlayer].Send("draw", cardDrawn)
			players[currentPlayer].AddToHand(cardDrawn)
			if players[currentPlayer].HasInHand(Minister) &&
				(players[currentPlayer].HasInHand(Wizard) || players[currentPlayer].HasInHand(General)) {
				Debug("*** Player", currentPlayer, "drew", cardDrawn, "***")
				Debug("*** Player", currentPlayer, "was forced to discard the minister ***")
				_ = players[currentPlayer].RemoveFromHand(Minister)
				SendAll("played", currentPlayer, Minister)
				goto endTurn
			}

			Debug("*** Player", currentPlayer, "drew", cardDrawn, "***")
			Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")

			fullMove = players[currentPlayer].Recieve()
			Debug("*** Player", currentPlayer, "move:", fullMove, "***")

			n, _ = fmt.Sscan(fullMove, &action, &cardStr, &target, &queryStr)

			if !strings.EqualFold(action, "play") {
				//must be "forfeit" or some invalid action
				Debug("*** ERROR: Player", currentPlayer, "made an illegal play or forfeited ***")
				Out(currentPlayer)
				goto endTurn
			}

			if n < 2 {
				//malformed move
				Debug("*** ERROR: Player", currentPlayer, "passed too few arguments ***")
				Out(currentPlayer)
				goto endTurn
			}

			card = ParseCard(cardStr)
			if ok := players[currentPlayer].RemoveFromHand(card); !ok {
				//trying to play a card you don't have is illegal
				Debug("*** ERROR: Player", currentPlayer, "played a card they dont have:", card, "***")
				Out(currentPlayer)
				goto endTurn
			}
			Debug("*** Player", currentPlayer, "played", card, "***")
			Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")

			//check the right arguments have been given
			switch card {
			case Princess, Minister, Priestess:
				if n != 2 {
					Debug("*** ERROR: Player", currentPlayer, "passed the wrong number of arguments for the played card ***")
					Out(currentPlayer)
					goto endTurn
				}
			case General, Wizard, Knight, Clown:
				if n != 3 {
					Debug("*** ERROR: Player", currentPlayer, "passed the wrong number of arguments for the played card ***")
					Out(currentPlayer)
					goto endTurn
				}

			case Soldier:
				if n != 4 {
					Debug("*** ERROR: Player", currentPlayer, "passed the wrong number of arguments for the played card ***")
					Out(currentPlayer)
					goto endTurn
				}
				if (target >= len(players)) && (target < 0) {
					Debug("*** ERROR: Player", currentPlayer, "passed a bad target argument ***")
					Out(currentPlayer)
					goto endTurn
				}
			}

			SendAll("played", currentPlayer, card)

			//check target is okay
			switch card {
			case General, Wizard, Knight, Clown, Soldier:
				if (target >= len(players)) && (target < 0) {
					Debug("*** ERROR: Player", currentPlayer, "passed a bad target argument ***")
					Out(currentPlayer)
					goto endTurn
				}
				if players[target].lost {
					Debug("*** ERROR: Player", currentPlayer, "attempted to target a player who has already lost ***")
					Out(currentPlayer)
					goto endTurn
				}
				if players[target].protected {
					Debug("*** Player", currentPlayer, "tried to target Player", target, "who was protected by a priestess ***")
					Out(currentPlayer)
					goto endTurn
				}
				if target == currentPlayer {
					Debug("*** Player", currentPlayer, "tried to target themself ***")
					var availableTargets = 0
					for _, v := range players {
						if !(v.protected || v.lost) {
							availableTargets++
						}
					}
					if availableTargets <= 1 {
						Debug("*** Player", currentPlayer, "is permitted to target themself as there are no other valid targets left ***")
					} else {
						Out(currentPlayer)
						goto endTurn
					}
				}
			}

			switch card {
			case Princess:
				Out(currentPlayer)
			case Minister:
			case General:
				players[currentPlayer].Send("swap", players[target].hand[0])
				players[target].Send("swap", players[currentPlayer].hand[0])
				players[currentPlayer].hand[0], players[target].hand[0] = players[target].hand[0], players[currentPlayer].hand[0]
				Debug("*** Player", currentPlayer, "and Player", target, "swapped hands ***")
				Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")
				Debug("*** Player", target, "hand:", players[target].HandString(), "***")
			case Wizard:
				Debug("*** Player", target, "discards hand and draws a new card ***")
				SendAll("discard", target, players[target].hand[0])
				if players[target].hand[0] == Princess {
					Out(target)
					goto endTurn
				}
				if deck.IsEmpty() {
					Debug("*** Player", target, "can't draw as there are no cards left ***")
					Out(target)
					goto endTurn
				}
				players[target].hand[0] = deck.Draw()
				players[target].Send("draw", players[target].hand[0])
				Debug("*** Player", target, "hand:", players[target].HandString(), "***")
			case Priestess:
				players[currentPlayer].protected = true
			case Knight:
				Debug("*** Player", currentPlayer, "and Player", target, "compare hands ***")
				Debug("*** Player", currentPlayer, "hand:", players[currentPlayer].HandString(), "***")
				Debug("*** Player", target, "hand:", players[target].HandString(), "***")
				players[target].Send("reveal", currentPlayer, players[currentPlayer].hand[0])
				players[currentPlayer].Send("reveal", target, players[target].hand[0])
				if players[currentPlayer].HandValue() > players[target].HandValue() {
					Out(target)
				} else if players[currentPlayer].HandValue() < players[target].HandValue() {
					Out(currentPlayer)
				}
			case Clown:
				Debug("*** Player", target, "reveals a", players[target].hand[0], "from their hand to Player", currentPlayer, "***")
				players[currentPlayer].Send("reveal", target, players[target].hand[0])
			case Soldier:
				Debug("*** Player", currentPlayer, "asked Player", target, "if they have a", queryStr, "in their hand ***")
				queryCard := ParseCard(queryStr)
				if queryCard == Soldier {
					Debug("*** ERROR: Can't target a soldier with a soldier ***")
					Out(currentPlayer)
					goto endTurn
				}
				if players[target].HasInHand(queryCard) {
					Out(target)
				}
			default:
				Debug("*** ERROR: Player", currentPlayer, "played an illegal card:", cardStr, "***")
				Out(currentPlayer)
			}

		endTurn:
			currentPlayer++
			if currentPlayer >= len(players) {
				currentPlayer = 0
			}
		}
		startPlayer++
		if startPlayer >= len(players) {
			startPlayer = 0
		}

		winningHandValue := 0
		for i, _ := range players {
			if players[i].HandValue() > winningHandValue {
				winningHandValue = players[i].HandValue()
			}
		}
		for i, _ := range players {
			if players[i].HandValue() == winningHandValue {
				players[i].roundsWon++
			}
			players[i].Forfeit()
		}
		fmt.Print("Round ended.\t")
		for i, p := range players {
			fmt.Print("Player ", i, " score: ", p.roundsWon, "\t")
		}
		fmt.Println()
	}
	_, winner := PlayerWon()
	fmt.Println("Player won:", winner, players[winner].name)
}

func SendAll(args ...interface{}) {
	for _, player := range players {
		if !player.lost {
			player.Send(args...)
		}
	}
}

func SurvivingPlayers() int {
	result := 0
	for _, player := range players {
		if !player.lost {
			result++
		}
	}
	return result
}

func PlayerWon() (bool, int) {
	for i, p := range players {
		if p.roundsWon >= roundsNeeded {
			return true, i
		}
	}
	return false, 0
}

func Out(playerNumber int) {
	players[playerNumber].Forfeit()
	SendAll("out", playerNumber, players[playerNumber].HandString())
	Debug("*** Player", playerNumber, "out:", players[playerNumber].HandString(), "***")
}

func Debug(args ...interface{}) {
	if debug {
		fmt.Println(args...)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
)

const (
	ApiURL = "https://api.stackexchange.com/2.2/users"
	CQuery = "pagesize=100&order=desc&sort=reputation&site=stackoverflow"
)

type SOUsers struct {
	Items []struct {
		BadgeCounts struct {
			Bronze int `json:"bronze"`
			Silver int `json:"silver"`
			Gold   int `json:"gold"`
		} `json:"badge_counts"`
		AccountID               int    `json:"account_id"`
		IsEmployee              bool   `json:"is_employee"`
		LastModifiedDate        int    `json:"last_modified_date"`
		LastAccessDate          int    `json:"last_access_date"`
		Age                     int    `json:"age,omitempty"`
		ReputationChangeYear    int    `json:"reputation_change_year"`
		ReputationChangeQuarter int    `json:"reputation_change_quarter"`
		ReputationChangeMonth   int    `json:"reputation_change_month"`
		ReputationChangeWeek    int    `json:"reputation_change_week"`
		ReputationChangeDay     int    `json:"reputation_change_day"`
		Reputation              int    `json:"reputation"`
		CreationDate            int    `json:"creation_date"`
		UserType                string `json:"user_type"`
		UserID                  int    `json:"user_id"`
		AcceptRate              int    `json:"accept_rate,omitempty"`
		Location                string `json:"location,omitempty"`
		WebsiteURL              string `json:"website_url,omitempty"`
		Link                    string `json:"link"`
		ProfileImage            string `json:"profile_image"`
		DisplayName             string `json:"display_name"`
	} `json:"items"`
	HasMore        bool `json:"has_more"`
	QuotaMax       int  `json:"quota_max"`
	QuotaRemaining int  `json:"quota_remaining"`
}

func Decode(r io.Reader) (x *SOUsers, err error) {
	x = new(SOUsers)
	err = json.NewDecoder(r).Decode(x)
	if err != nil {
		fmt.Println(err)
	}
	return x, err
}

func Streamdata() (x *SOUsers, err error) {
	url := fmt.Sprintf("%s?page=%d&%s", ApiURL, 1, CQuery)

	fmt.Println(url)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Println("Error 1", err)
	}
	req.Header.Set("Accept-Encoding", "gzip")

	response, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Println("Error 2", err)
	}
	defer response.Body.Close()

	var reader io.ReadCloser

	switch response.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err = gzip.NewReader(response.Body)
		defer reader.Close()
	default:
		reader = response.Body
	}
	return Decode(reader)
}

func main() {

	users, _ := Streamdata()

	for _, user := range users.Items {
		fmt.Println(user.DisplayName)
		fmt.Println(user.Reputation)
		fmt.Println(user.Location)
	}

}
<commit_msg>looping by page<commit_after>package main

import (
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
)

const (
	MaxPages = 3
	ApiURL   = "https://api.stackexchange.com/2.2/users"
	CQuery   = "pagesize=100&order=desc&sort=reputation&site=stackoverflow"
)

type SOUsers struct {
	Items []struct {
		BadgeCounts struct {
			Bronze int `json:"bronze"`
			Silver int `json:"silver"`
			Gold   int `json:"gold"`
		} `json:"badge_counts"`
		AccountID               int    `json:"account_id"`
		IsEmployee              bool   `json:"is_employee"`
		LastModifiedDate        int    `json:"last_modified_date"`
		LastAccessDate          int    `json:"last_access_date"`
		Age                     int    `json:"age,omitempty"`
		ReputationChangeYear    int    `json:"reputation_change_year"`
		ReputationChangeQuarter int    `json:"reputation_change_quarter"`
		ReputationChangeMonth   int    `json:"reputation_change_month"`
		ReputationChangeWeek    int    `json:"reputation_change_week"`
		ReputationChangeDay     int    `json:"reputation_change_day"`
		Reputation              int    `json:"reputation"`
		CreationDate            int    `json:"creation_date"`
		UserType                string `json:"user_type"`
		UserID                  int    `json:"user_id"`
		AcceptRate              int    `json:"accept_rate,omitempty"`
		Location                string `json:"location,omitempty"`
		WebsiteURL              string `json:"website_url,omitempty"`
		Link                    string `json:"link"`
		ProfileImage            string `json:"profile_image"`
		DisplayName             string `json:"display_name"`
	} `json:"items"`
	HasMore        bool `json:"has_more"`
	QuotaMax       int  `json:"quota_max"`
	QuotaRemaining int  `json:"quota_remaining"`
}

func Decode(r io.Reader) (x *SOUsers, err error) {
	x = new(SOUsers)
	err = json.NewDecoder(r).Decode(x)
	if err != nil {
		fmt.Println(err)
	}
	return x, err
}

func Streamdata(page int) (x *SOUsers, err error) {
	url := fmt.Sprintf("%s?page=%d&%s", ApiURL, page, CQuery)

	fmt.Println(url)

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		fmt.Println("Error 1", err)
	}
	req.Header.Set("Accept-Encoding", "gzip")

	response, err := http.DefaultClient.Do(req)
	if err != nil {
		fmt.Println("Error 2", err)
	}
	defer response.Body.Close()

	var reader io.ReadCloser

	switch response.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err = gzip.NewReader(response.Body)
		defer reader.Close()
	default:
		reader = response.Body
	}
	return Decode(reader)
}

func main() {

	currentPage := 1

	for {
		users, _ := Streamdata(currentPage)

		for _, user := range users.Items {
			fmt.Println(user.DisplayName)
			fmt.Println(user.Reputation)
			fmt.Println(user.Location)
		}

		currentPage += 1
		if currentPage == MaxPages || !users.HasMore {
			break
		}
	}

}
<|endoftext|>