<commit_before>// This is a utility script for building a table of URLs and their associated
// response codes within a given sitemap. If the sitemap index contains links to
// other sitemaps, it will traverse through until it comes across actual sitemap
// URLs to make HEAD requests to and determine their HTTP response codes.
//
// Usage:
// $ go run main.go https://www.compass.com/sitemaps/index/
package main

import (
	"encoding/xml"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

type SitemapIndex struct {
	XMLName xml.Name `xml:"sitemapindex"`
	XMLNS   string   `xml:"xmlns,attr"`

	Sitemaps []Sitemap `xml:"sitemap"`
}

type Sitemap struct {
	XMLName xml.Name `xml:"sitemap"`

	Loc     string `xml:"loc"`
	LastMod string `xml:"lastmod,omitempty"`
}

type SitemapURLSet struct {
	XMLName xml.Name `xml:"urlset"`
	XMLNS   string   `xml:"xmlns,attr"`

	URLs []SitemapURL `xml:"url"`
}

type SitemapURL struct {
	XMLName xml.Name `xml:"url"`

	Loc        string  `xml:"loc"`
	LastMod    string  `xml:"lastmod,omitempty"`
	ChangeFreq string  `xml:"changefreq,omitempty"`
	Priority   float64 `xml:"priority,omitempty"`
}

func main() {
	if len(os.Args) < 2 {
		log.Fatalln("You must pass a sitemap index URL argument!")
	}

	loadSitemapIndex(os.Args[1])
}

func loadSitemapIndex(url string) {
	resp, err := http.Get(url)

	log.Printf("(%v) %v \n", resp.Status, url)

	if err != nil || resp.StatusCode > 200 {
		log.Printf("Unable to load sitemap index: %v\n", url)
		log.Fatal(err)
	}

	parseSitemapIndex(resp)
}

func parseSitemapIndex(resp *http.Response) {
	// Ensure that read operation cleans up after itself
	defer resp.Body.Close()

	// Load response into a byte array
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("Error reading sitemap index")
		log.Fatal(err)
	}

	// Parse XML from response byte array
	v := &SitemapIndex{}
	xmlErr := xml.Unmarshal(respBytes, &v)
	if xmlErr != nil {
		log.Println("Error parsing XML response")
		log.Fatal(xmlErr)
	}

	ch := make(chan string)

	for _, sitemap := range v.Sitemaps {
		go loadSitemap(sitemap.Loc, ch)
	}

	// Iterate over channel output
	for range v.Sitemaps {
		<-ch
	}

	log.Println("All sitemaps loaded")
}

func loadSitemap(url string, sitemapChan chan string) {
	resp, err := http.Get(url)

	log.Printf("(%v) %v \n", resp.Status, url)

	if err != nil || resp.StatusCode > 200 {
		log.Fatal(err)
	}

	parseSitemap(resp)

	// Notify channel of success
	sitemapChan <- url
}

func parseSitemap(resp *http.Response) {
	// Ensure that read operation cleans up after itself
	defer resp.Body.Close()

	// Load response into a byte array
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("Error reading sitemap")
		log.Fatal(err)
	}

	// Parse XML from response byte array
	v := &SitemapURLSet{}
	xmlErr := xml.Unmarshal(respBytes, &v)
	if xmlErr != nil {
		log.Println("Error parsing XML response")
		log.Fatal(xmlErr)
	}

	for _, url := range v.URLs {
		log.Println(url.Loc)
	}
}

// Build tree of URLs
// Issue a HEAD request for each URL, IGNORING redirects
// For each link in the sitemap, save parent sitemap path, sitemap link path, and HTTP status (200, 301, 404, 500, etc.) to CSV
<commit_msg>Think more about channel communication<commit_after>// This is a utility script for building a table of URLs and their associated
// response codes within a given sitemap. If the sitemap index contains links to
// other sitemaps, it will traverse through until it comes across actual sitemap
// URLs to make HEAD requests to and determine their HTTP response codes.
//
// Usage:
// $ go run main.go https://www.compass.com/sitemaps/index/
package main

import (
	"encoding/xml"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

type SitemapIndex struct {
	XMLName xml.Name `xml:"sitemapindex"`
	XMLNS   string   `xml:"xmlns,attr"`

	Sitemaps []Sitemap `xml:"sitemap"`
}

type Sitemap struct {
	XMLName xml.Name `xml:"sitemap"`

	Loc     string `xml:"loc"`
	LastMod string `xml:"lastmod,omitempty"`
}

type SitemapURLSet struct {
	XMLName xml.Name `xml:"urlset"`
	XMLNS   string   `xml:"xmlns,attr"`

	URLs []SitemapURL `xml:"url"`
}

type SitemapURL struct {
	XMLName xml.Name `xml:"url"`

	Loc        string  `xml:"loc"`
	LastMod    string  `xml:"lastmod,omitempty"`
	ChangeFreq string  `xml:"changefreq,omitempty"`
	Priority   float64 `xml:"priority,omitempty"`
}

type SitemapValidationResponse struct {
	Name    string
	Results []URLValidationResponse
}

type URLValidationResponse struct {
	URL        string
	StatusCode int
}

func main() {
	if len(os.Args) < 2 {
		log.Fatalln("You must pass a sitemap index URL argument!")
	}

	loadSitemapIndex(os.Args[1])
}

func loadSitemapIndex(url string) {
	resp, err := http.Get(url)

	log.Printf("(%v) %v \n", resp.Status, url)

	if err != nil || resp.StatusCode > 200 {
		log.Printf("Unable to load sitemap index: %v\n", url)
		log.Fatal(err)
	}

	parseSitemapIndex(resp)
}

func parseSitemapIndex(resp *http.Response) {
	// Ensure that read operation cleans up after itself
	defer resp.Body.Close()

	// Load response into a byte array
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("Error reading sitemap index")
		log.Fatal(err)
	}

	// Parse XML from response byte array
	v := &SitemapIndex{}
	xmlErr := xml.Unmarshal(respBytes, &v)
	if xmlErr != nil {
		log.Println("Error parsing XML response")
		log.Fatal(xmlErr)
	}

	ch := make(chan SitemapValidationResponse, len(v.Sitemaps))
	for _, sitemap := range v.Sitemaps {
		go loadSitemap(sitemap.Loc, ch)
	}

	// Iterate over channel output
	results := make([]SitemapValidationResponse, len(v.Sitemaps))
	for i, _ := range v.Sitemaps {
		results[i] = <-ch
	}

	log.Println("All sitemaps loaded")
	log.Print(results)
}

func loadSitemap(url string, results chan SitemapValidationResponse) {
	resp, err := http.Get(url)

	log.Printf("(%v) %v \n", resp.Status, url)

	if err != nil || resp.StatusCode > 200 {
		log.Fatal(err)
	}

	// ch := make(chan []ValidationResponse)
	// parseSitemap(resp)

	// Notify channel of success
	results <- SitemapValidationResponse{
		Name: url,
		Results: []URLValidationResponse
	}
}

func parseSitemap(resp *http.Response) {
	// Ensure that read operation cleans up after itself
	defer resp.Body.Close()

	// Load response into a byte array
	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("Error reading sitemap")
		log.Fatal(err)
	}

	// Build tree of URLs by parsing XML from response byte array
	v := &SitemapURLSet{}
	xmlErr := xml.Unmarshal(respBytes, &v)
	if xmlErr != nil {
		log.Println("Error parsing XML response")
		log.Fatal(xmlErr)
	}

	for _, url := range v.URLs {
		loadSitemapUrl(url.Loc)
	}
}

// func loadSitemapUrl(url string) {
// Issue a HEAD request for each URL, IGNORING redirects, spacing the requests out if at all possible
// resp, err := http.Head(url)
// }

// For each link in the sitemap, save parent sitemap path, sitemap link path, and HTTP status (200, 301, 404, 500, etc.) to CSV
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

type Configuration struct {
	hostname	string 	// Displayed Hostname
	host		string 	// Listened Host
	port		string 	// HTTP Port
	tls 		bool	// TLS enabled
	tlscert		string	// TLS Cert Path
	tlskey		string	// TLS Cert Key Path
	tlsport		string	// HTTPS Port
}

var configuration = Configuration{}

func init() {
	hostname := getEnvWithDefault("HOSTNAME", "ifconfig.io")

	host := getEnvWithDefault("HOST", "")
	port := getEnvWithDefault("PORT", "8080")

	tlsenabled := getEnvWithDefault("TLS", "0")
	tlsport := getEnvWithDefault("TLSPORT", "8443")
	tlscert := getEnvWithDefault("TLSCERT", "/opt/ifconfig/.cf/ifconfig.io.crt")
	tlskey := getEnvWithDefault("TLSKEY", "/opt/ifconfig/.cf/ifconfig.io.key")

	configuration = Configuration{
		hostname: hostname,
		host: host,
		port: port,
		tls: tlsenabled == "1",
		tlscert: tlscert,
		tlskey: tlskey,
		tlsport: tlsport,
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func testRemoteTCPPort(address string) bool {
	_, err := net.DialTimeout("tcp", address, 3*time.Second)
	if err != nil {
		return false
	}
	return true
}

func mainHandler(c *gin.Context) {
	// fields := strings.Split(c.Params.ByName("field"), ".")
	URLFields := strings.Split(strings.Trim(c.Request.URL.EscapedPath(), "/"), "/")
	fields := strings.Split(URLFields[0], ".")
	ip, err := net.ResolveTCPAddr("tcp", c.Request.RemoteAddr)
	if err != nil {
		c.Abort()
	}

	cfIP := net.ParseIP(c.Request.Header.Get("CF-Connecting-IP"))
	if cfIP != nil {
		ip.IP = cfIP
	}

	if fields[0] == "porttest" {
		if len(fields) >= 2 {
			if port, err := strconv.Atoi(fields[1]); err == nil && port > 0 && port <= 65535 {
				c.String(200, fmt.Sprintln(testRemoteTCPPort(ip.IP.String()+":"+fields[1])))
			} else {
				c.String(400, "Invalid Port Number")
			}
		} else {
			c.String(400, "Need Port")
		}
		return
	}

	//if strings.HasPrefix(fields[0], ".well-known/") {
	//	http.ServeFile(c.Writer, c.Request)
	//	return
	//}

	c.Set("ifconfig_hostname", configuration.hostname)

	c.Set("ip", ip.IP.String())
	c.Set("port", ip.Port)
	c.Set("ua", c.Request.UserAgent())
	c.Set("lang", c.Request.Header.Get("Accept-Language"))
	c.Set("encoding", c.Request.Header.Get("Accept-Encoding"))
	c.Set("method", c.Request.Method)
	c.Set("mime", c.Request.Header.Get("Accept"))
	c.Set("referer", c.Request.Header.Get("Referer"))
	c.Set("forwarded", c.Request.Header.Get("X-Forwarded-For"))
	c.Set("country_code", c.Request.Header.Get("CF-IPCountry"))

	ua := strings.Split(c.Request.UserAgent(), "/")

	// Only lookup hostname if the results are going to need it.
	// if stringInSlice(fields[0], []string{"all", "host"}) || (fields[0] == "" && ua[0] != "curl") {
	if stringInSlice(fields[0], []string{"host"}) || (fields[0] == "" && ua[0] != "curl") {
		hostnames, err := net.LookupAddr(ip.IP.String())
		if err != nil {
			c.Set("host", "")
		} else {
			c.Set("host", hostnames[0])
		}
	}

	wantsJSON := false
	if len(fields) >= 2 && fields[1] == "json" {
		wantsJSON = true
	}

	switch fields[0] {
	case "":
		//If the user is using curl, then we should just return the IP, else we show the home page.
		if ua[0] == "curl" {
			c.String(200, fmt.Sprintln(ip.IP))
		} else {
			c.HTML(200, "index.html", c.Keys)
		}
		return
	case "request":
		c.JSON(200, c.Request)
		return
	case "all":
		if wantsJSON {
			c.JSON(200, c.Keys)
		} else {
			c.String(200, "%v", c.Keys)
		}
		return
	case "headers":
		c.JSON(200, c.Request.Header)
		return
	}

	fieldResult, exists := c.Get(fields[0])
	if !exists {
		c.String(404, "Not Found")
		return
	}
	if wantsJSON {
		c.JSON(200, fieldResult)
	} else {
		c.String(200, fmt.Sprintln(fieldResult))
	}

}

func getEnvWithDefault(key string, defaultValue string) string {
	value := os.Getenv(key)
	if value == "" {
		return defaultValue
	}
	return value
}

func main() {
	r := gin.New()
	r.Use(gin.Recovery())
	r.LoadHTMLGlob("templates/*")

	for _, route := range []string{
		"ip", "ua", "port", "lang", "encoding", "method",
		"mime", "referer", "forwarded", "country_code",
		"all", "headers", "porttest",
	} {
		r.GET(fmt.Sprintf("/%s", route), mainHandler)
		r.GET(fmt.Sprintf("/%s.json", route), mainHandler)
	}
	r.GET("/", mainHandler)

	errc := make(chan error)
	go func(errc chan error) {
		for err := range errc {
			panic(err)
		}
	}(errc)

	go func(errc chan error) {
		errc <- r.Run(fmt.Sprintf("%s:%s", configuration.host, configuration.port))
	}(errc)

	if configuration.tls {
		go func(errc chan error) {
			errc <- r.RunTLS(
			fmt.Sprintf("%s:%s", configuration.host, configuration.tlsport),
			configuration.tlscert, configuration.tlskey)
		}(errc)
	}

	fmt.Println(<-errc)
}
<commit_msg>Add a header to change the which header we look for the remote IP<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

type Configuration struct {
	hostname string // Displayed Hostname
	host     string // Listened Host
	port     string // HTTP Port
	ipheader string // Header to overwrite the remote IP
	tls      bool   // TLS enabled
	tlscert  string // TLS Cert Path
	tlskey   string // TLS Cert Key Path
	tlsport  string // HTTPS Port
}

var configuration = Configuration{}

func init() {
	hostname := getEnvWithDefault("HOSTNAME", "ifconfig.io")

	host := getEnvWithDefault("HOST", "")
	port := getEnvWithDefault("PORT", "8080")

	// Most common alternative would be X-Forwarded-For
	ipheader := getEnvWithDefault("FORWARD_IP_HEADER", "CF-Connecting-IP")

	tlsenabled := getEnvWithDefault("TLS", "0")
	tlsport := getEnvWithDefault("TLSPORT", "8443")
	tlscert := getEnvWithDefault("TLSCERT", "/opt/ifconfig/.cf/ifconfig.io.crt")
	tlskey := getEnvWithDefault("TLSKEY", "/opt/ifconfig/.cf/ifconfig.io.key")

	configuration = Configuration{
		hostname: hostname,
		host:     host,
		port:     port,
		ipheader: ipheader,
		tls:      tlsenabled == "1",
		tlscert:  tlscert,
		tlskey:   tlskey,
		tlsport:  tlsport,
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func testRemoteTCPPort(address string) bool {
	_, err := net.DialTimeout("tcp", address, 3*time.Second)
	if err != nil {
		return false
	}
	return true
}

func mainHandler(c *gin.Context) {
	// fields := strings.Split(c.Params.ByName("field"), ".")
	URLFields := strings.Split(strings.Trim(c.Request.URL.EscapedPath(), "/"), "/")
	fields := strings.Split(URLFields[0], ".")
	ip, err := net.ResolveTCPAddr("tcp", c.Request.RemoteAddr)
	if err != nil {
		c.Abort()
	}

	header_ip := net.ParseIP(c.Request.Header.Get(configuration.ipheader))
	if header_ip != nil {
		ip.IP = header_ip
	}

	if fields[0] == "porttest" {
		if len(fields) >= 2 {
			if port, err := strconv.Atoi(fields[1]); err == nil && port > 0 && port <= 65535 {
				c.String(200, fmt.Sprintln(testRemoteTCPPort(ip.IP.String()+":"+fields[1])))
			} else {
				c.String(400, "Invalid Port Number")
			}
		} else {
			c.String(400, "Need Port")
		}
		return
	}

	//if strings.HasPrefix(fields[0], ".well-known/") {
	//	http.ServeFile(c.Writer, c.Request)
	//	return
	//}

	c.Set("ifconfig_hostname", configuration.hostname)

	c.Set("ip", ip.IP.String())
	c.Set("port", ip.Port)
	c.Set("ua", c.Request.UserAgent())
	c.Set("lang", c.Request.Header.Get("Accept-Language"))
	c.Set("encoding", c.Request.Header.Get("Accept-Encoding"))
	c.Set("method", c.Request.Method)
	c.Set("mime", c.Request.Header.Get("Accept"))
	c.Set("referer", c.Request.Header.Get("Referer"))
	c.Set("forwarded", c.Request.Header.Get("X-Forwarded-For"))
	c.Set("country_code", c.Request.Header.Get("CF-IPCountry"))

	ua := strings.Split(c.Request.UserAgent(), "/")

	// Only lookup hostname if the results are going to need it.
	// if stringInSlice(fields[0], []string{"all", "host"}) || (fields[0] == "" && ua[0] != "curl") {
	if stringInSlice(fields[0], []string{"host"}) || (fields[0] == "" && ua[0] != "curl") {
		hostnames, err := net.LookupAddr(ip.IP.String())
		if err != nil {
			c.Set("host", "")
		} else {
			c.Set("host", hostnames[0])
		}
	}

	wantsJSON := false
	if len(fields) >= 2 && fields[1] == "json" {
		wantsJSON = true
	}

	switch fields[0] {
	case "":
		//If the user is using curl, then we should just return the IP, else we show the home page.
		if ua[0] == "curl" {
			c.String(200, fmt.Sprintln(ip.IP))
		} else {
			c.HTML(200, "index.html", c.Keys)
		}
		return
	case "request":
		c.JSON(200, c.Request)
		return
	case "all":
		if wantsJSON {
			c.JSON(200, c.Keys)
		} else {
			c.String(200, "%v", c.Keys)
		}
		return
	case "headers":
		c.JSON(200, c.Request.Header)
		return
	}

	fieldResult, exists := c.Get(fields[0])
	if !exists {
		c.String(404, "Not Found")
		return
	}
	if wantsJSON {
		c.JSON(200, fieldResult)
	} else {
		c.String(200, fmt.Sprintln(fieldResult))
	}

}

func getEnvWithDefault(key string, defaultValue string) string {
	value := os.Getenv(key)
	if value == "" {
		return defaultValue
	}
	return value
}

func main() {
	r := gin.New()
	r.Use(gin.Recovery())
	r.LoadHTMLGlob("templates/*")

	for _, route := range []string{
		"ip", "ua", "port", "lang", "encoding", "method",
		"mime", "referer", "forwarded", "country_code",
		"all", "headers", "porttest",
	} {
		r.GET(fmt.Sprintf("/%s", route), mainHandler)
		r.GET(fmt.Sprintf("/%s.json", route), mainHandler)
	}
	r.GET("/", mainHandler)

	errc := make(chan error)
	go func(errc chan error) {
		for err := range errc {
			panic(err)
		}
	}(errc)

	go func(errc chan error) {
		errc <- r.Run(fmt.Sprintf("%s:%s", configuration.host, configuration.port))
	}(errc)

	if configuration.tls {
		go func(errc chan error) {
			errc <- r.RunTLS(
				fmt.Sprintf("%s:%s", configuration.host, configuration.tlsport),
				configuration.tlscert, configuration.tlskey)
		}(errc)
	}

	fmt.Println(<-errc)
}
<|endoftext|>


<commit_before>package main

/*
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>

int fadvise(const char *path, float r)
{
    int fd;
    struct stat st;
    off_t l;
    fd = open(path, O_RDONLY);
    if(fd == -1) {
        return -1;
    }

    if(fstat(fd, &st) == -1) {
        goto error;
    }

    l = (off_t)(st.st_size * r);

    if(posix_fadvise(fd, 0, l, POSIX_FADV_DONTNEED) != 0) {
        goto error;
    }

    close(fd);
    return 1;
error:
    close(fd);
    return -1;
}

int activePages(const char *path)
{
    int i, j, fd, pages, pagesize;
    struct stat st;
    void *m;
    char *pageinfo;

    fd = open(path, O_RDONLY);
    if(fd == -1) {
        return -1;
    }

    if(fstat(fd, &st) == -1) {
        goto error;
    }

    pagesize = getpagesize();
    pages = (st.st_size + pagesize - 1) / pagesize;
    pageinfo = calloc(sizeof(*pageinfo), pages);
    if(!pageinfo) {
        goto error;
    }

    m = mmap(NULL, st.st_size, PROT_NONE, MAP_SHARED, fd, 0);
    if(m == MAP_FAILED) {
        free(pageinfo);
        goto error;
    }

    if(mincore(m, st.st_size, pageinfo) == -1) {
        free(pageinfo);
        munmap(m, st.st_size);
        goto error;
    }

    i = 0;
    j = 0;
    for (i = 0; i < pages; i++) {
        if(pageinfo[i++] & 1) {
            j++;
        }
    }

    munmap(m, st.st_size);

    return j;
error:
    close(fd);
    return -1;
}
*/
import "C"
import (
	"errors"
	"flag"
	"fmt"
	"os"
	"runtime"
)

const CACHECTL_VERSION = "0.0.1"

func printCachectlVersion() {
	fmt.Printf(`cachectl %s
Compiler: %s %s
Copyright (C) 2014 Tatsuhiko Kubo <cubicdaiya@gmail.com>
`,
		CACHECTL_VERSION,
		runtime.Compiler,
		runtime.Version())
}

func printCacheStat(fpath string, fsize int64) {
	pagesize := os.Getpagesize()
	pagesizeKB := pagesize / 1024
	if fsize == 0 {
		fmt.Printf("%s 's pages in cache: %d/%d (%.1f%%) [filesize=%.1fK, pagesize=%dK]\n", fpath, 0, 0, 0.0, 0.0, pagesizeKB)
	}

	pages := (fsize + int64(pagesize) - 1) / int64(pagesize)
	pagesActive := C.activePages(C.CString(fpath))
	activeRate := 100.0 * (float64(pagesActive) / float64(pages))
	filesizeKB := float64(fsize) / 1024
	fmt.Printf("%s 's pages in cache: %d/%d (%.1f%%)  [filesize=%.1fK, pagesize=%dK]\n",
		fpath, pagesActive, pages, activeRate, filesizeKB, pagesizeKB)
}

func deleteCache(fpath string, fsize int64, rate float64) error {
	if rate < 0.0 || rate > 1.0 {
		return errors.New(fmt.Sprintf("%f: rate should be less than 1.0\n", rate))
	}

	return nil
}

func main() {

	// Parse flags
	version := flag.Bool("v", false, "show version")
	op := flag.String("op", "stat", "operation(stat, del)")
	fpath := flag.String("f", "", "target file path")
	rate := flag.Float64("r", 0.0, "rate of page cache deleted(0.0 <= r<= 1.0)")
	flag.Parse()

	if *version {
		printCachectlVersion()
		os.Exit(0)
	}

	if *fpath == "" {
		fmt.Println("target file path is empty.")
		os.Exit(0)
	}

	fi, err := os.Stat(*fpath)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	if !fi.Mode().IsRegular() {
		fmt.Printf("%s is not regular file\n", fi.Name())
		os.Exit(1)
	}

	if *op == "stat" {
		printCacheStat(*fpath, fi.Size())
	} else {
		fmt.Printf("Before deleting %s 's page cache\n", *fpath)
		printCacheStat(*fpath, fi.Size())

		deleteCache(*fpath, fi.Size(), *rate)

		fmt.Printf("After deleting %s 's page cache\n", *fpath)
		printCacheStat(*fpath, fi.Size())
	}

}
<commit_msg>Bugfixed: forgot to add logic for deleting cache.<commit_after>package main

/*
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/mman.h>

int fadvise(const char *path, float r)
{
    int fd;
    struct stat st;
    off_t l;
    fd = open(path, O_RDONLY);
    if(fd == -1) {
        return -1;
    }

    if(fstat(fd, &st) == -1) {
        goto error;
    }

    l = (off_t)(st.st_size * r);

    if(posix_fadvise(fd, 0, l, POSIX_FADV_DONTNEED) != 0) {
        goto error;
    }

    close(fd);
    return 1;
error:
    close(fd);
    return -1;
}

int activePages(const char *path)
{
    int i, j, fd, pages, pagesize;
    struct stat st;
    void *m;
    char *pageinfo;

    fd = open(path, O_RDONLY);
    if(fd == -1) {
        return -1;
    }

    if(fstat(fd, &st) == -1) {
        goto error;
    }

    pagesize = getpagesize();
    pages = (st.st_size + pagesize - 1) / pagesize;
    pageinfo = calloc(sizeof(*pageinfo), pages);
    if(!pageinfo) {
        goto error;
    }

    m = mmap(NULL, st.st_size, PROT_NONE, MAP_SHARED, fd, 0);
    if(m == MAP_FAILED) {
        free(pageinfo);
        goto error;
    }

    if(mincore(m, st.st_size, pageinfo) == -1) {
        free(pageinfo);
        munmap(m, st.st_size);
        goto error;
    }

    i = 0;
    j = 0;
    for (i = 0; i < pages; i++) {
        if(pageinfo[i++] & 1) {
            j++;
        }
    }

    munmap(m, st.st_size);

    return j;
error:
    close(fd);
    return -1;
}
*/
import "C"
import (
	"errors"
	"flag"
	"fmt"
	"os"
	"runtime"
)

const CACHECTL_VERSION = "0.0.1"

func printCachectlVersion() {
	fmt.Printf(`cachectl %s
Compiler: %s %s
Copyright (C) 2014 Tatsuhiko Kubo <cubicdaiya@gmail.com>
`,
		CACHECTL_VERSION,
		runtime.Compiler,
		runtime.Version())
}

func printCacheStat(fpath string, fsize int64) {
	pagesize := os.Getpagesize()
	pagesizeKB := pagesize / 1024
	if fsize == 0 {
		fmt.Printf("%s 's pages in cache: %d/%d (%.1f%%) [filesize=%.1fK, pagesize=%dK]\n", fpath, 0, 0, 0.0, 0.0, pagesizeKB)
	}

	pages := (fsize + int64(pagesize) - 1) / int64(pagesize)
	pagesActive := C.activePages(C.CString(fpath))
	activeRate := 100.0 * (float64(pagesActive) / float64(pages))
	filesizeKB := float64(fsize) / 1024
	fmt.Printf("%s 's pages in cache: %d/%d (%.1f%%)  [filesize=%.1fK, pagesize=%dK]\n",
		fpath, pagesActive, pages, activeRate, filesizeKB, pagesizeKB)
}

func deleteCache(fpath string, fsize int64, rate float64) error {
	if rate < 0.0 || rate > 1.0 {
		return errors.New(fmt.Sprintf("%f: rate should be less than 1.0\n", rate))
	}

	result := C.fadvise(fpath, rate)
	if result == -1 {
		return errors.New(fmt.Sprintf("failed to delete page cache for %s", fpath))
	}

	return nil
}

func main() {

	// Parse flags
	version := flag.Bool("v", false, "show version")
	op := flag.String("op", "stat", "operation(stat, del)")
	fpath := flag.String("f", "", "target file path")
	rate := flag.Float64("r", 1.0, "rate of page cache deleted(0.0 <= r<= 1.0)")
	flag.Parse()

	if *version {
		printCachectlVersion()
		os.Exit(0)
	}

	if *fpath == "" {
		fmt.Println("target file path is empty.")
		os.Exit(0)
	}

	fi, err := os.Stat(*fpath)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	if !fi.Mode().IsRegular() {
		fmt.Printf("%s is not regular file\n", fi.Name())
		os.Exit(1)
	}

	if *op == "stat" {
		printCacheStat(*fpath, fi.Size())
	} else {
		fmt.Printf("Before deleting %s 's page cache\n", *fpath)
		printCacheStat(*fpath, fi.Size())

		deleteCache(*fpath, fi.Size(), *rate)

		fmt.Printf("After deleting %s 's page cache\n", *fpath)
		printCacheStat(*fpath, fi.Size())
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/codegangsta/cli"
	"github.com/koding/klientctl/logging"
	"github.com/koding/klientctl/util"
)

// ExitingCommand is a function that returns an exit code
type ExitingCommand func(*cli.Context) int

// ExitingWithMessageCommand is a function which prints the given message to
// Stdout. Useful for printig a message to the user in a convenient single-use way.
type ExitingWithMessageCommand func(*cli.Context) (string, int)

// sudoRequiredFor is the default list of commands that require sudo.
// The actual handling of this list is done in the SudoRequired func.
var sudoRequiredFor = []string{
	"install",
	"uninstall",
	"start",
	"stop",
	"restart",
	"update",
}

// log is used as a global loggger, for commands like ListCommand that
// need refactoring to support instance based commands.
//
// TODO: Remove this after all commands have been refactored into structs. Ie, the
// cli rewrite.
var log logging.Logger

func main() {
	// The writer used for the logging output. Either a file, or /dev/null
	var logWriter io.Writer

	f, err := os.OpenFile(LogFilePath, os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		// Don't warn the user about failing to load the log file if we are installing.
		if len(os.Args) > 1 && os.Args[1] != "install" {
			// TODO: We may not want to log a logfile failure here, but it seems useful to
			// give us some idea that the logs aren't being written.
			fmt.Printf(
				"Warning: Unable to load log file. Please reinstall %s.\n",
				Name,
			)
		}
		// Rather than exit `kd` because we are unable to log, we simple disable logging
		// by writing to /dev/null. This also measn that even if we can't load the log
		// file, the log instance is valid and doesn't have to be checked for being
		// nil before every usage.
		logWriter = ioutil.Discard
	} else {
		// TODO: Does defer get triggered on os.Exit? I think main() needs to be moved
		// to an alternate func that we can actuall use defer. Eg, main() calls Main(),
		// or something.
		defer f.Close()
		logWriter = f
	}

	// Create our logger instance.
	log = logging.NewLogger("kd")
	log.SetHandler(logging.NewWriterHandler(logWriter))
	log.Infof("kd binary called with: %s", os.Args)

	// Check if the command the user is giving requires sudo.
	if err := AdminRequired(os.Args, sudoRequiredFor, util.NewPermissions()); err != nil {
		// In the event of an error, simply print the error to the user
		// and exit.
		fmt.Println("Error: this command requires sudo.")
		os.Exit(10)
	}

	app := cli.NewApp()
	app.Name = Name
	app.Version = fmt.Sprintf("%d", Version)

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "list",
			ShortName: "ls",
			Usage:     "List the available machines.",
			Action:    Exit(CheckUpdateFirst(ListCommand)),
			Subcommands: []cli.Command{
				cli.Command{
					Name:   "mounts",
					Usage:  "List the mounted machines.",
					Action: Exit(CheckUpdateFirst(MountsCommand)),
				},
			},
		},
		cli.Command{
			Name:        "mount",
			ShortName:   "m",
			Usage:       "Mount a remote folder to a local folder.",
			Description: cmdDescriptions["mount"],
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "remotepath, r",
					Usage: "Full path of remote folder in machine to mount.",
				},
				cli.BoolFlag{
					Name:  "noignore, n",
					Usage: "Disable ignoring of default remote folders.",
				},
				cli.BoolFlag{
					Name:  "noprefetch, p",
					Usage: "Disable prefetching of folder metadata on mount.",
				},
				cli.BoolFlag{
					Name:  "nowatch, w",
					Usage: "Disable watching for changes on remote machine.",
				},
			},
			Action: Exit(CheckUpdateFirst(MountCommand)),
		},
		cli.Command{
			Name:        "unmount",
			ShortName:   "u",
			Usage:       "Unmount previously mounted machine.",
			Description: cmdDescriptions["unmount"],
			Action:      Exit(CheckUpdateFirst(UnmountCommand)),
		},
		cli.Command{
			Name:        "ssh",
			ShortName:   "s",
			Usage:       "SSH into the machine.",
			Description: cmdDescriptions["ssh"],
			Action:      Exit(CheckUpdateFirst(SSHCommandFactory)),
		},
		cli.Command{
			Name:        "install",
			Usage:       fmt.Sprintf("Install the %s.", KlientName),
			Description: cmdDescriptions["install"],
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "kontrol, k",
					Usage: "Specify an alternate Kontrol",
				},
			},
			//HideHelp: true,
			Action: Exit(InstallCommandFactory),
		},
		cli.Command{
			Name:        "uninstall",
			Usage:       fmt.Sprintf("Uninstall the %s.", KlientName),
			Description: fmt.Sprintf("Uninstall the %s.", KlientName),
			Action:      ExitWithMessage(UninstallCommand),
		},
		cli.Command{
			Name:        "status",
			Usage:       fmt.Sprintf("Status of the %s.", KlientName),
			Description: fmt.Sprintf("Status the %s.", KlientName),
			Action:      Exit(StatusCommand),
		},
		cli.Command{
			Name:        "start",
			Usage:       fmt.Sprintf("Start the %s.", KlientName),
			Description: fmt.Sprintf("Start the %s.", KlientName),
			Action:      Exit(StartCommand),
		},
		cli.Command{
			Name:        "stop",
			Usage:       fmt.Sprintf("Stop the %s.", KlientName),
			Description: fmt.Sprintf("Stop the %s.", KlientName),
			Action:      Exit(StopCommand),
		},
		cli.Command{
			Name:        "restart",
			Usage:       fmt.Sprintf("Restart the %s.", KlientName),
			Description: fmt.Sprintf("Restart the %s.", KlientName),
			Action:      Exit(RestartCommand),
		},
		cli.Command{
			Name:        "update",
			Usage:       "Update to latest version.",
			Description: "Update to latest version.",
			Action:      Exit(UpdateCommand),
		},
		cli.Command{
			Name:            "run",
			Usage:           "Run command on remote or local machine.",
			Description:     cmdDescriptions["run"],
			Action:          Exit(RunCommandFactory),
			SkipFlagParsing: true,
		},
	}

	app.Run(os.Args)
}

// Exit is a wrapper around commands to return to proper error code.
func Exit(f ExitingCommand) func(*cli.Context) {
	return func(c *cli.Context) { os.Exit(f(c)) }
}

// ExitWithMessage takes a ExitingWithMessageCommand type and returns a
// codegansta/cli friendly command Action.
func ExitWithMessage(f ExitingWithMessageCommand) func(*cli.Context) {
	return func(c *cli.Context) {
		s, e := f(c)
		if s != "" {
			fmt.Println(s)
		}
		os.Exit(e)
	}
}
<commit_msg>logging: Removed no log file warning<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/codegangsta/cli"
	"github.com/koding/klientctl/logging"
	"github.com/koding/klientctl/util"
)

// ExitingCommand is a function that returns an exit code
type ExitingCommand func(*cli.Context) int

// ExitingWithMessageCommand is a function which prints the given message to
// Stdout. Useful for printig a message to the user in a convenient single-use way.
type ExitingWithMessageCommand func(*cli.Context) (string, int)

// sudoRequiredFor is the default list of commands that require sudo.
// The actual handling of this list is done in the SudoRequired func.
var sudoRequiredFor = []string{
	"install",
	"uninstall",
	"start",
	"stop",
	"restart",
	"update",
}

// log is used as a global loggger, for commands like ListCommand that
// need refactoring to support instance based commands.
//
// TODO: Remove this after all commands have been refactored into structs. Ie, the
// cli rewrite.
var log logging.Logger

func main() {
	// The writer used for the logging output. Either a file, or /dev/null
	var logWriter io.Writer

	f, err := os.OpenFile(LogFilePath, os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		// Rather than exit `kd` because we are unable to log, we simple disable logging
		// by writing to /dev/null. This also measn that even if we can't load the log
		// file, the log instance is valid and doesn't have to be checked for being
		// nil before every usage.
		logWriter = ioutil.Discard
	} else {
		// TODO: Does defer get triggered on os.Exit? I think main() needs to be moved
		// to an alternate func that we can actuall use defer. Eg, main() calls Main(),
		// or something.
		defer f.Close()
		logWriter = f
	}

	// Create our logger instance.
	log = logging.NewLogger("kd")
	log.SetHandler(logging.NewWriterHandler(logWriter))
	log.Infof("kd binary called with: %s", os.Args)

	// Check if the command the user is giving requires sudo.
	if err := AdminRequired(os.Args, sudoRequiredFor, util.NewPermissions()); err != nil {
		// In the event of an error, simply print the error to the user
		// and exit.
		fmt.Println("Error: this command requires sudo.")
		os.Exit(10)
	}

	app := cli.NewApp()
	app.Name = Name
	app.Version = fmt.Sprintf("%d", Version)

	app.Commands = []cli.Command{
		cli.Command{
			Name:      "list",
			ShortName: "ls",
			Usage:     "List the available machines.",
			Action:    Exit(CheckUpdateFirst(ListCommand)),
			Subcommands: []cli.Command{
				cli.Command{
					Name:   "mounts",
					Usage:  "List the mounted machines.",
					Action: Exit(CheckUpdateFirst(MountsCommand)),
				},
			},
		},
		cli.Command{
			Name:        "mount",
			ShortName:   "m",
			Usage:       "Mount a remote folder to a local folder.",
			Description: cmdDescriptions["mount"],
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "remotepath, r",
					Usage: "Full path of remote folder in machine to mount.",
				},
				cli.BoolFlag{
					Name:  "noignore, n",
					Usage: "Disable ignoring of default remote folders.",
				},
				cli.BoolFlag{
					Name:  "noprefetch, p",
					Usage: "Disable prefetching of folder metadata on mount.",
				},
				cli.BoolFlag{
					Name:  "nowatch, w",
					Usage: "Disable watching for changes on remote machine.",
				},
			},
			Action: Exit(CheckUpdateFirst(MountCommand)),
		},
		cli.Command{
			Name:        "unmount",
			ShortName:   "u",
			Usage:       "Unmount previously mounted machine.",
			Description: cmdDescriptions["unmount"],
			Action:      Exit(CheckUpdateFirst(UnmountCommand)),
		},
		cli.Command{
			Name:        "ssh",
			ShortName:   "s",
			Usage:       "SSH into the machine.",
			Description: cmdDescriptions["ssh"],
			Action:      Exit(CheckUpdateFirst(SSHCommandFactory)),
		},
		cli.Command{
			Name:        "install",
			Usage:       fmt.Sprintf("Install the %s.", KlientName),
			Description: cmdDescriptions["install"],
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "kontrol, k",
					Usage: "Specify an alternate Kontrol",
				},
			},
			//HideHelp: true,
			Action: Exit(InstallCommandFactory),
		},
		cli.Command{
			Name:        "uninstall",
			Usage:       fmt.Sprintf("Uninstall the %s.", KlientName),
			Description: fmt.Sprintf("Uninstall the %s.", KlientName),
			Action:      ExitWithMessage(UninstallCommand),
		},
		cli.Command{
			Name:        "status",
			Usage:       fmt.Sprintf("Status of the %s.", KlientName),
			Description: fmt.Sprintf("Status the %s.", KlientName),
			Action:      Exit(StatusCommand),
		},
		cli.Command{
			Name:        "start",
			Usage:       fmt.Sprintf("Start the %s.", KlientName),
			Description: fmt.Sprintf("Start the %s.", KlientName),
			Action:      Exit(StartCommand),
		},
		cli.Command{
			Name:        "stop",
			Usage:       fmt.Sprintf("Stop the %s.", KlientName),
			Description: fmt.Sprintf("Stop the %s.", KlientName),
			Action:      Exit(StopCommand),
		},
		cli.Command{
			Name:        "restart",
			Usage:       fmt.Sprintf("Restart the %s.", KlientName),
			Description: fmt.Sprintf("Restart the %s.", KlientName),
			Action:      Exit(RestartCommand),
		},
		cli.Command{
			Name:        "update",
			Usage:       "Update to latest version.",
			Description: "Update to latest version.",
			Action:      Exit(UpdateCommand),
		},
		cli.Command{
			Name:            "run",
			Usage:           "Run command on remote or local machine.",
			Description:     cmdDescriptions["run"],
			Action:          Exit(RunCommandFactory),
			SkipFlagParsing: true,
		},
	}

	app.Run(os.Args)
}

// Exit is a wrapper around commands to return to proper error code.
func Exit(f ExitingCommand) func(*cli.Context) {
	return func(c *cli.Context) { os.Exit(f(c)) }
}

// ExitWithMessage takes a ExitingWithMessageCommand type and returns a
// codegansta/cli friendly command Action.
func ExitWithMessage(f ExitingWithMessageCommand) func(*cli.Context) {
	return func(c *cli.Context) {
		s, e := f(c)
		if s != "" {
			fmt.Println(s)
		}
		os.Exit(e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"flag"

	"log"

	"path/filepath"

	"strconv"

	"github.com/gregjones/httpcache"
	"github.com/if1live/staticfilecache"
	"github.com/jhoonb/archivex"
)

var cmd string
var uri string
var cid string
var similarity float64
var workerCount int
var filter int
var output string

func clusterIdList() []int {
	cids := []int{}
	for _, c := range strings.Split(cid, ",") {
		clusterid, err := strconv.Atoi(c)
		if err != nil {
			continue
		}
		cids = append(cids, clusterid)
	}
	return cids
}

type fetchConfig struct {
	URL           string  `json:"url"`
	ClusterIDList []int   `json:"cluster_id_list"`
	Filter        int     `json:"filter"`
	Similarity    float64 `json:"similarity"`
}

func (m *fetchConfig) Marshal() []byte {
	data, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}
	var out bytes.Buffer
	json.Indent(&out, data, "", "  ")
	return out.Bytes()
}

func makeClusterFromFlag() *clusterList {
	htmltext := getHTMLText(uri)
	links := parseHTMLText(htmltext)
	for i, link := range links {
		links[i] = ApplyFilter(link, filter)
	}
	return newClusterList(links, similarity)
}

func init() {
	flag.StringVar(&cmd, "cmd", "help", "command")
	flag.StringVar(&uri, "uri", "", "target uri")
	flag.StringVar(&cid, "cid", "", "cluster id")
	flag.Float64Var(&similarity, "similarity", 0.9, "similarity")
	flag.IntVar(&workerCount, "worker", 8, "worker count")
	flag.IntVar(&filter, "filter", 0, "link filter")
	flag.StringVar(&output, "output", "download.zip", "output zip filename")
}

func main() {
	flag.Parse()

	switch cmd {
	case "show":
		mainForShow()
	case "fetch":
		mainForFetch()
	default:
		mainForHelp()
	}
}

func mainForShow() {
	cluster := makeClusterFromFlag()
	if cid == "" {
		cluster.Show()
	} else {
		links := []string{}
		for _, clusterid := range clusterIdList() {
			links = append(links, cluster.GetCluster(clusterid)...)
		}
		for i, link := range links {
			fmt.Printf("(%d/%d) %s\n", i+1, len(links), link)
		}
	}
}

func mainForHelp() {
	fmt.Println("help")
}

type fetchCommand struct {
	uri string
	idx int
}
type fetchResult struct {
	resp *http.Response
	uri  string
	idx  int
}

func workerFetch(id int, jobs <-chan *fetchCommand, results chan<- *fetchResult) {
	for req := range jobs {
		if req.uri == "" {
			return
		}
		uri := req.uri

		log.Printf("[worker %d] download %s\n", id, uri)
		cachedir := "./_cache_static"
		cache := staticfilecache.New(cachedir)
		tp := httpcache.NewTransport(cache)
		client := &http.Client{Transport: tp}
		resp, _ := client.Get(uri)

		results <- &fetchResult{
			resp: resp,
			uri:  uri,
			idx:  req.idx,
		}
	}
}

func mainForFetch() {
	cluster := makeClusterFromFlag()
	links := []string{}
	cids := clusterIdList()
	for _, clusterid := range cids {
		links = append(links, cluster.GetCluster(clusterid)...)
	}

	linkcount := len(links)
	if linkcount == 0 {
		log.Printf("No link found, cid=%d\n", cid)
		return
	}

	jobs := make(chan *fetchCommand, linkcount)
	fetchResults := make(chan *fetchResult, linkcount)
	zipFinishCh := make(chan bool, linkcount)

	zip := new(archivex.ZipFile)
	zip.Create(output)

	// 다운로드에 사용한 정보 저장하기
	config := &fetchConfig{
		URL:           uri,
		ClusterIDList: cids,
		Filter:        filter,
		Similarity:    similarity,
	}
	zip.Add("metadata.json", config.Marshal())

	go func(z *archivex.ZipFile, count int, results chan *fetchResult, finishCh chan bool) {
		for i := 1; i <= count; i++ {
			result := <-results
			log.Printf("complete (%d/%d)\n", i, linkcount)
			bytes, _ := ioutil.ReadAll(result.resp.Body)
			result.resp.Body.Close()

			basename := filepath.Base(result.uri)
			filename := MakeFilename(basename, result.idx)

			z.Add(filename, bytes)
		}

		finishCh <- true
	}(zip, linkcount, fetchResults, zipFinishCh)

	for w := 1; w <= workerCount; w++ {
		go workerFetch(w, jobs, fetchResults)
	}
	for i, link := range links {
		jobs <- &fetchCommand{
			uri: link,
			idx: i,
		}
	}
	// 종료 지점 넣어주기. 요청을 받은 fetch worker는 알아서 종료될것이다
	for i := 0; i < linkcount; i++ {
		jobs <- &fetchCommand{
			uri: "",
			idx: -1,
		}
	}
	// zip 파일쓰기 끝난거 확인하면 종료
	<-zipFinishCh
	zip.Close()
}
<commit_msg>fix typebug<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"

	"flag"

	"log"

	"path/filepath"

	"strconv"

	"github.com/gregjones/httpcache"
	"github.com/if1live/staticfilecache"
	"github.com/jhoonb/archivex"
)

var cmd string
var uri string
var cid string
var similarity float64
var workerCount int
var filter int
var output string

func clusterIdList() []int {
	cids := []int{}
	for _, c := range strings.Split(cid, ",") {
		clusterid, err := strconv.Atoi(c)
		if err != nil {
			continue
		}
		cids = append(cids, clusterid)
	}
	return cids
}

type fetchConfig struct {
	URL           string  `json:"url"`
	ClusterIDList []int   `json:"cluster_id_list"`
	Filter        int     `json:"filter"`
	Similarity    float64 `json:"similarity"`
}

func (m *fetchConfig) Marshal() []byte {
	data, err := json.Marshal(m)
	if err != nil {
		panic(err)
	}
	var out bytes.Buffer
	json.Indent(&out, data, "", "  ")
	return out.Bytes()
}

func makeClusterFromFlag() *clusterList {
	htmltext := getHTMLText(uri)
	links := parseHTMLText(htmltext)
	for i, link := range links {
		links[i] = ApplyFilter(link, filter)
	}
	return newClusterList(links, similarity)
}

func init() {
	flag.StringVar(&cmd, "cmd", "help", "command")
	flag.StringVar(&uri, "uri", "", "target uri")
	flag.StringVar(&cid, "cid", "", "cluster id")
	flag.Float64Var(&similarity, "similarity", 0.9, "similarity")
	flag.IntVar(&workerCount, "worker", 8, "worker count")
	flag.IntVar(&filter, "filter", 0, "link filter")
	flag.StringVar(&output, "output", "download.zip", "output zip filename")
}

func main() {
	flag.Parse()

	switch cmd {
	case "show":
		mainForShow()
	case "fetch":
		mainForFetch()
	default:
		mainForHelp()
	}
}

func mainForShow() {
	cluster := makeClusterFromFlag()
	if cid == "" {
		cluster.Show()
	} else {
		links := []string{}
		for _, clusterid := range clusterIdList() {
			links = append(links, cluster.GetCluster(clusterid)...)
		}
		for i, link := range links {
			fmt.Printf("(%d/%d) %s\n", i+1, len(links), link)
		}
	}
}

func mainForHelp() {
	fmt.Println("help")
}

type fetchCommand struct {
	uri string
	idx int
}
type fetchResult struct {
	resp *http.Response
	uri  string
	idx  int
}

func workerFetch(id int, jobs <-chan *fetchCommand, results chan<- *fetchResult) {
	for req := range jobs {
		if req.uri == "" {
			return
		}
		uri := req.uri

		log.Printf("[worker %d] download %s\n", id, uri)
		cachedir := "./_cache_static"
		cache := staticfilecache.New(cachedir)
		tp := httpcache.NewTransport(cache)
		client := &http.Client{Transport: tp}
		resp, _ := client.Get(uri)

		results <- &fetchResult{
			resp: resp,
			uri:  uri,
			idx:  req.idx,
		}
	}
}

func mainForFetch() {
	cluster := makeClusterFromFlag()
	links := []string{}
	cids := clusterIdList()
	for _, clusterid := range cids {
		links = append(links, cluster.GetCluster(clusterid)...)
	}

	linkcount := len(links)
	if linkcount == 0 {
		log.Printf("No link found, cid=%s\n", cid)
		return
	}

	jobs := make(chan *fetchCommand, linkcount)
	fetchResults := make(chan *fetchResult, linkcount)
	zipFinishCh := make(chan bool, linkcount)

	zip := new(archivex.ZipFile)
	zip.Create(output)

	// 다운로드에 사용한 정보 저장하기
	config := &fetchConfig{
		URL:           uri,
		ClusterIDList: cids,
		Filter:        filter,
		Similarity:    similarity,
	}
	zip.Add("metadata.json", config.Marshal())

	go func(z *archivex.ZipFile, count int, results chan *fetchResult, finishCh chan bool) {
		for i := 1; i <= count; i++ {
			result := <-results
			log.Printf("complete (%d/%d)\n", i, linkcount)
			bytes, _ := ioutil.ReadAll(result.resp.Body)
			result.resp.Body.Close()

			basename := filepath.Base(result.uri)
			filename := MakeFilename(basename, result.idx)

			z.Add(filename, bytes)
		}

		finishCh <- true
	}(zip, linkcount, fetchResults, zipFinishCh)

	for w := 1; w <= workerCount; w++ {
		go workerFetch(w, jobs, fetchResults)
	}
	for i, link := range links {
		jobs <- &fetchCommand{
			uri: link,
			idx: i,
		}
	}
	// 종료 지점 넣어주기. 요청을 받은 fetch worker는 알아서 종료될것이다
	for i := 0; i < linkcount; i++ {
		jobs <- &fetchCommand{
			uri: "",
			idx: -1,
		}
	}
	// zip 파일쓰기 끝난거 확인하면 종료
	<-zipFinishCh
	zip.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/davecgh/go-spew/spew"
	"github.com/jessevdk/go-flags"
	"github.com/nsf/termbox-go"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// command line flags variable
var opts struct {
	// display ascii instead of kana's
	Ascii bool `short:"a" long:"ascii" description:"Use ascii/alphanumeric characters instead of japanese kana's."`

	// enable logging
	Logging bool `short:"l" long:"log" description:"Enable logging debug messages to ~/.gomatrix-log."`
}

// array with half width kanas as Go runes
// source: http://en.wikipedia.org/wiki/Half-width_kana
var halfWidthKana = []rune{
	'｡', '｢', '｣', '､', '･', 'ｦ', 'ｧ', 'ｨ', 'ｩ', 'ｪ', 'ｫ', 'ｬ', 'ｭ', 'ｮ', 'ｯ',
	'ｰ', 'ｱ', 'ｲ', 'ｳ', 'ｴ', 'ｵ', 'ｶ', 'ｷ', 'ｸ', 'ｹ', 'ｺ', 'ｻ', 'ｼ', 'ｽ', 'ｾ', 'ｿ',
	'ﾀ', 'ﾁ', 'ﾂ', 'ﾃ', 'ﾄ', 'ﾅ', 'ﾆ', 'ﾇ', 'ﾈ', 'ﾉ', 'ﾊ', 'ﾋ', 'ﾌ', 'ﾍ', 'ﾎ', 'ﾏ',
	'ﾐ', 'ﾑ', 'ﾒ', 'ﾓ', 'ﾔ', 'ﾕ', 'ﾖ', 'ﾗ', 'ﾘ', 'ﾙ', 'ﾚ', 'ﾛ', 'ﾜ', 'ﾝ', 'ﾞ', 'ﾟ',
}

// just basic alphanumeric characters
var alphaNumerics = []rune{
	'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
	'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
	'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
}

// characters to be used, is being set to alphaNumerics or halfWidthKana depending on flags
var characters []rune

// streamDisplays by column number
var streamDisplaysByColumn = make(map[int]*StreamDisplay)

// struct sizes contains terminal sizes (in amount of characters)
type sizes struct {
	width  int
	height int
}

var curSizes sizes                   // current sizes
var curStreamsPerStreamDisplay = 0   // curent amount of streams per display allowed
var sizesUpdateCh = make(chan sizes) //channel used to notify StreamDisplayManager

// set the sizes and notify StreamDisplayManager
func setSizes(width int, height int) {
	s := sizes{
		width:  width,
		height: height,
	}
	curSizes = s
	curStreamsPerStreamDisplay = 1 + height/10
	sizesUpdateCh <- s
}

func main() {
	// parse flags
	args, err := flags.Parse(&opts)
	if err != nil {
		flagError := err.(*flags.Error)
		if flagError.Type == flags.ErrHelp {
			return
		}
		if flagError.Type == flags.ErrUnknownFlag {
			fmt.Println("Use --help to view all available options.")
			return
		}
		fmt.Printf("Error parsing flags: %s\n", err)
		return
	}
	if len(args) > 0 {
		// we don't accept too much arguments..
		fmt.Printf("Unknown argument '%s'.\n", args[0])
		return
	}

	// Juse a println for fun..
	fmt.Println("Opening connection to The Matrix.. Please stand by..")

	// setup logging with logfile /dev/null or ~/.gomatrix-log
	filename := "/dev/null"
	if opts.Logging {
		filename = os.Getenv("HOME") + "/.gomatrix-log"
	}
	logfile, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Printf("Could not open logfile. %s\n", err)
		os.Exit(1)
	}
	defer logfile.Close()
	log.SetOutput(logfile)
	log.Println("-------------")
	log.Println("Starting gomatrix. This logfile is for development/debug purposes.")

	characters = halfWidthKana
	if opts.Ascii {
		characters = alphaNumerics
	}

	// seed the rand package with time
	rand.Seed(time.Now().UnixNano())

	// initialize termbox
	err = termbox.Init()
	if err != nil {
		fmt.Println("Could not start termbox for gomatrix. View ~/.gomatrix-log for error messages.")
		log.Printf("Cannot start gomatrix, termbox.Init() gave an error:\n%s\n", err)
		os.Exit(1)
	}
	termbox.HideCursor()

	// StreamDisplay manager
	go func() {
		var lastWidth int

		for newSizes := range sizesUpdateCh {
			log.Printf("New width: %d\n", newSizes.width)
			diffWidth := newSizes.width - lastWidth

			if diffWidth == 0 {
				// same column size, wait for new information
				log.Println("Got resize over channel, but diffWidth = 0")
				continue
			}

			if diffWidth > 0 {
				log.Printf("Starting %d new SD's\n", diffWidth)
				for newColumn := lastWidth; newColumn < newSizes.width; newColumn++ {
					// create stream display
					sd := &StreamDisplay{
						column:    newColumn,
						stopCh:    make(chan bool, 1),
						streams:   make(map[*Stream]bool),
						newStream: make(chan bool, 1), // will only be filled at start and when a spawning stream has it's tail released
					}
					streamDisplaysByColumn[newColumn] = sd

					// start StreamDisplay in goroutine
					go sd.run()

					// create first new stream
					sd.newStream <- true
				}
				lastWidth = newSizes.width
			}

			if diffWidth < 0 {
				log.Printf("Closing %d SD's\n", diffWidth)
				for closeColumn := lastWidth - 1; closeColumn > newSizes.width; closeColumn-- {
					// get sd
					sd := streamDisplaysByColumn[closeColumn]

					// delete from map
					delete(streamDisplaysByColumn, closeColumn)

					// inform sd that it's being closed
					sd.stopCh <- true
				}
				lastWidth = newSizes.width
			}
		}
	}()

	// set initial sizes
	setSizes(termbox.Size())

	// flusher flushes the termbox every x miliseconds
	go func() {
		for {
			<-time.After(40 * time.Millisecond) //++ TODO: find out wether .After() or .Sleep() is better performance-wise
			termbox.Flush()
		}
	}()

	// make chan for tembox events and run poller to send events on chan
	eventChan := make(chan termbox.Event)
	go func() {
		for {
			event := termbox.PollEvent()
			eventChan <- event
		}
	}()

	// register signals to channel
	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGTSTP)
	signal.Notify(sigChan, syscall.SIGKILL)
	signal.Notify(sigChan, syscall.SIGQUIT)
	signal.Notify(sigChan, syscall.SIGTERM)

	// handle termbox events and unix signals
	func() { //++ TODO: dont use function literal. use labels instead.
		for {
			// select for either event or signal
			select {
			case event := <-eventChan:
				log.Printf("Have event: \n%s", spew.Sdump(event))
				// switch on event type
				switch event.Type {
				case termbox.EventKey: // actions depend on key
					switch event.Key {
					case termbox.KeyCtrlZ, termbox.KeyCtrlC:
						return
						//++ TODO: add more fun keys (slowmo? freeze? rampage?)
					}

					switch event.Ch {
					case 'q':
						return

					case 'c':
						termbox.Clear(termbox.ColorBlack, termbox.ColorBlack)

					case 'a':
						characters = alphaNumerics

					case 'k', 'j':
						characters = halfWidthKana
					}

				case termbox.EventResize: // set sizes
					setSizes(event.Width, event.Height)

				case termbox.EventError: // quit
					log.Fatalf("Quitting because of termbox error: \n%s\n", event.Err)
				}
			case signal := <-sigChan:
				log.Printf("Have signal: \n%s", spew.Sdump(signal))
				return
			}
		}
	}()

	// close down
	termbox.Close()
	log.Println("stopping gomatrix")
	fmt.Println("Thank you for connecting with Morpheus' Matrix API v4.2. Have a nice day!")
	os.Exit(0)
}
<commit_msg>Only on character per command: removing 'j' as alias for 'k'<commit_after>package main

import (
	"fmt"
	"github.com/davecgh/go-spew/spew"
	"github.com/jessevdk/go-flags"
	"github.com/nsf/termbox-go"
	"log"
	"math/rand"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// command line flags variable
var opts struct {
	// display ascii instead of kana's
	Ascii bool `short:"a" long:"ascii" description:"Use ascii/alphanumeric characters instead of japanese kana's."`

	// enable logging
	Logging bool `short:"l" long:"log" description:"Enable logging debug messages to ~/.gomatrix-log."`
}

// array with half width kanas as Go runes
// source: http://en.wikipedia.org/wiki/Half-width_kana
var halfWidthKana = []rune{
	'｡', '｢', '｣', '､', '･', 'ｦ', 'ｧ', 'ｨ', 'ｩ', 'ｪ', 'ｫ', 'ｬ', 'ｭ', 'ｮ', 'ｯ',
	'ｰ', 'ｱ', 'ｲ', 'ｳ', 'ｴ', 'ｵ', 'ｶ', 'ｷ', 'ｸ', 'ｹ', 'ｺ', 'ｻ', 'ｼ', 'ｽ', 'ｾ', 'ｿ',
	'ﾀ', 'ﾁ', 'ﾂ', 'ﾃ', 'ﾄ', 'ﾅ', 'ﾆ', 'ﾇ', 'ﾈ', 'ﾉ', 'ﾊ', 'ﾋ', 'ﾌ', 'ﾍ', 'ﾎ', 'ﾏ',
	'ﾐ', 'ﾑ', 'ﾒ', 'ﾓ', 'ﾔ', 'ﾕ', 'ﾖ', 'ﾗ', 'ﾘ', 'ﾙ', 'ﾚ', 'ﾛ', 'ﾜ', 'ﾝ', 'ﾞ', 'ﾟ',
}

// just basic alphanumeric characters
var alphaNumerics = []rune{
	'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
	'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
	'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
}

// characters to be used, is being set to alphaNumerics or halfWidthKana depending on flags
var characters []rune

// streamDisplays by column number
var streamDisplaysByColumn = make(map[int]*StreamDisplay)

// struct sizes contains terminal sizes (in amount of characters)
type sizes struct {
	width  int
	height int
}

var curSizes sizes                   // current sizes
var curStreamsPerStreamDisplay = 0   // curent amount of streams per display allowed
var sizesUpdateCh = make(chan sizes) //channel used to notify StreamDisplayManager

// set the sizes and notify StreamDisplayManager
func setSizes(width int, height int) {
	s := sizes{
		width:  width,
		height: height,
	}
	curSizes = s
	curStreamsPerStreamDisplay = 1 + height/10
	sizesUpdateCh <- s
}

func main() {
	// parse flags
	args, err := flags.Parse(&opts)
	if err != nil {
		flagError := err.(*flags.Error)
		if flagError.Type == flags.ErrHelp {
			return
		}
		if flagError.Type == flags.ErrUnknownFlag {
			fmt.Println("Use --help to view all available options.")
			return
		}
		fmt.Printf("Error parsing flags: %s\n", err)
		return
	}
	if len(args) > 0 {
		// we don't accept too much arguments..
		fmt.Printf("Unknown argument '%s'.\n", args[0])
		return
	}

	// Juse a println for fun..
	fmt.Println("Opening connection to The Matrix.. Please stand by..")

	// setup logging with logfile /dev/null or ~/.gomatrix-log
	filename := "/dev/null"
	if opts.Logging {
		filename = os.Getenv("HOME") + "/.gomatrix-log"
	}
	logfile, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Printf("Could not open logfile. %s\n", err)
		os.Exit(1)
	}
	defer logfile.Close()
	log.SetOutput(logfile)
	log.Println("-------------")
	log.Println("Starting gomatrix. This logfile is for development/debug purposes.")

	characters = halfWidthKana
	if opts.Ascii {
		characters = alphaNumerics
	}

	// seed the rand package with time
	rand.Seed(time.Now().UnixNano())

	// initialize termbox
	err = termbox.Init()
	if err != nil {
		fmt.Println("Could not start termbox for gomatrix. View ~/.gomatrix-log for error messages.")
		log.Printf("Cannot start gomatrix, termbox.Init() gave an error:\n%s\n", err)
		os.Exit(1)
	}
	termbox.HideCursor()

	// StreamDisplay manager
	go func() {
		var lastWidth int

		for newSizes := range sizesUpdateCh {
			log.Printf("New width: %d\n", newSizes.width)
			diffWidth := newSizes.width - lastWidth

			if diffWidth == 0 {
				// same column size, wait for new information
				log.Println("Got resize over channel, but diffWidth = 0")
				continue
			}

			if diffWidth > 0 {
				log.Printf("Starting %d new SD's\n", diffWidth)
				for newColumn := lastWidth; newColumn < newSizes.width; newColumn++ {
					// create stream display
					sd := &StreamDisplay{
						column:    newColumn,
						stopCh:    make(chan bool, 1),
						streams:   make(map[*Stream]bool),
						newStream: make(chan bool, 1), // will only be filled at start and when a spawning stream has it's tail released
					}
					streamDisplaysByColumn[newColumn] = sd

					// start StreamDisplay in goroutine
					go sd.run()

					// create first new stream
					sd.newStream <- true
				}
				lastWidth = newSizes.width
			}

			if diffWidth < 0 {
				log.Printf("Closing %d SD's\n", diffWidth)
				for closeColumn := lastWidth - 1; closeColumn > newSizes.width; closeColumn-- {
					// get sd
					sd := streamDisplaysByColumn[closeColumn]

					// delete from map
					delete(streamDisplaysByColumn, closeColumn)

					// inform sd that it's being closed
					sd.stopCh <- true
				}
				lastWidth = newSizes.width
			}
		}
	}()

	// set initial sizes
	setSizes(termbox.Size())

	// flusher flushes the termbox every x miliseconds
	go func() {
		for {
			<-time.After(40 * time.Millisecond) //++ TODO: find out wether .After() or .Sleep() is better performance-wise
			termbox.Flush()
		}
	}()

	// make chan for tembox events and run poller to send events on chan
	eventChan := make(chan termbox.Event)
	go func() {
		for {
			event := termbox.PollEvent()
			eventChan <- event
		}
	}()

	// register signals to channel
	sigChan := make(chan os.Signal)
	signal.Notify(sigChan, syscall.SIGTSTP)
	signal.Notify(sigChan, syscall.SIGKILL)
	signal.Notify(sigChan, syscall.SIGQUIT)
	signal.Notify(sigChan, syscall.SIGTERM)

	// handle termbox events and unix signals
	func() { //++ TODO: dont use function literal. use labels instead.
		for {
			// select for either event or signal
			select {
			case event := <-eventChan:
				log.Printf("Have event: \n%s", spew.Sdump(event))
				// switch on event type
				switch event.Type {
				case termbox.EventKey: // actions depend on key
					switch event.Key {
					case termbox.KeyCtrlZ, termbox.KeyCtrlC:
						return
						//++ TODO: add more fun keys (slowmo? freeze? rampage?)
					}

					switch event.Ch {
					case 'q':
						return

					case 'c':
						termbox.Clear(termbox.ColorBlack, termbox.ColorBlack)

					case 'a':
						characters = alphaNumerics

					case 'k':
						characters = halfWidthKana
					}

				case termbox.EventResize: // set sizes
					setSizes(event.Width, event.Height)

				case termbox.EventError: // quit
					log.Fatalf("Quitting because of termbox error: \n%s\n", event.Err)
				}
			case signal := <-sigChan:
				log.Printf("Have signal: \n%s", spew.Sdump(signal))
				return
			}
		}
	}()

	// close down
	termbox.Close()
	log.Println("stopping gomatrix")
	fmt.Println("Thank you for connecting with Morpheus' Matrix API v4.2. Have a nice day!")
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
)

var db *neoism.Database

func main() {
	fmt.Println(os.Args)
	app := cli.App("public-people-api-neo4j", "A public RESTful API for accessing People in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")

	app.Action = func() {
		runServer(*neoURL, *port)
	}

	app.Run(os.Args)
}

func runServer(neoURL string, port string) {
	var err error
	db, err = neoism.Connect(neoURL)
	if err != nil {
		panic(err)
	}

	r := mux.NewRouter()

	// Healthchecks and standards first
	r.HandleFunc("/__health", v1a.Handler("PeopleReadWriteNeo4j Healthchecks",
		"Checks for accessing neo4j", healthCheck()))
	r.HandleFunc("/ping", ping)

	// Then API specific ones:
	//r.HandleFunc("/people/{uuid}", peopleWrite).Methods("PUT")
	// TODO wonder if we should use a regex here since this won't match /people or /people/
	r.HandleFunc("/people/{uuid}", getPerson).Methods("GET")

	http.ListenAndServe(":"+port, handlers.CombinedLoggingHandler(os.Stdout, r))

}

func healthCheck() v1a.Check {
	return v1a.Check{
		BusinessImpact: "Unable to respond to Public People api requests",
		Checker:        checker,
	}
}

func checker() (string, error) {
	return "some message to return", nil
}

func ping(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func getPerson(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]
	if uuid == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	person := Person{
		PrefLabel: "someName",
		ID:        uuid,
		Memberships: []Membership{
			{Title: "213",
				Organisation: Organisation{
					ID: "org-123",
				},
				Roles: []Role{
					{ID: "role-123",
						ChangeEvents: []ChangeEvent{
							{StartedAt: time.Now()},
						},
					},
				},
			},
		},
	}

	// Organisation: Organisation{
	// {PrefLabel: "Marks and Sparks"},
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	json.NewEncoder(w).Encode(person)
	w.WriteHeader(http.StatusOK)
}
<commit_msg>Updated with models to give json structure<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
)

var db *neoism.Database

func main() {
	fmt.Println(os.Args)
	app := cli.App("public-people-api-neo4j", "A public RESTful API for accessing People in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	port := app.StringOpt("port", "8080", "Port to listen on")

	app.Action = func() {
		runServer(*neoURL, *port)
	}

	app.Run(os.Args)
}

func runServer(neoURL string, port string) {
	var err error
	db, err = neoism.Connect(neoURL)
	if err != nil {
		panic(err)
	}

	r := mux.NewRouter()

	// Healthchecks and standards first
	r.HandleFunc("/__health", v1a.Handler("PeopleReadWriteNeo4j Healthchecks",
		"Checks for accessing neo4j", healthCheck()))
	r.HandleFunc("/ping", ping)

	// Then API specific ones:
	// TODO wonder if we should use a regex here since this won't match /people or /people/
	r.HandleFunc("/people/{uuid}", getPerson).Methods("GET")

	http.ListenAndServe(":"+port, handlers.CombinedLoggingHandler(os.Stdout, r))

}

func healthCheck() v1a.Check {
	return v1a.Check{
		BusinessImpact: "Unable to respond to Public People api requests",
		Checker:        checker,
	}
}

func checker() (string, error) {
	return "some message to return", nil
}

func ping(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "pong")
}

func getPerson(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]
	if uuid == "" {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	person := Person{
		PrefLabel: "someName",
		ID:        uuid,
		Memberships: []Membership{
			{Title: "213",
				Organisation: Organisation{
					ID: "org-123",
				},
				Roles: []Role{
					{ID: "role-123",
						ChangeEvents: []ChangeEvent{
							{StartedAt: time.Now()},
						},
					},
				},
			},
		},
	}

	// Organisation: Organisation{
	// {PrefLabel: "Marks and Sparks"},
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	json.NewEncoder(w).Encode(person)
	w.WriteHeader(http.StatusOK)
}
<|endoftext|>


<commit_before>// Force12.io is a package that monitors demand for resource in a system and then scales and repurposes
// containers, based on agreed "quality of service" contracts, to best handle that demand within the constraints of your existing VM
// or physical infrastructure (for v1).
//
// Force12 is defined to optimize the use of existing physical and VM resources instantly. VMs cannot be scaled in real time (it takes
// several minutes) and new physical machines take even longer. However, containers can be started or stopped at sub second speeds,
// allowing your infrastructure to adapt itself in real time to meet system demands.
//
// Force12 is aimed at effectively using the resources you have right now - your existing VMs or physical servers - by using them as
// optimally as possible.
//
// The Force12 approach is analogous to the way that a router dynamically optimises the use of a physical network. A router is limited
// by the capacity of the lines physically connected to it. Adding additional capacity is a physical process and takes time. Routers
// therefore make decisions in real time about which packets will be prioritized on a particular line based on the packet's priority
// (defined by a "quality of service" contract).
//
// For example, at times of high bandwidth usage a router might prioritize VOIP traffic over web browsing in real time.
//
// Containers allow Force12 to make similar "instant" judgements on service prioritisation within your existing infrastructure. Routers
// make very simplistic judgments because they have limited time and cpu and they act at a per packet level. Force12 has the capability
// of making far more sophisticated judgements, although even fairly simple ones will still provide a significant new service.
//
// This prototype is a bare bones implementation of Force12.io that recognises only 1 demand type:
// randomised demand for a priority 1 service. Resources are allocated to meet this demand for priority 1, and spare resource can
// be used for a priority 2 service.
//
// These demand type examples have been chosen purely for simplicity of demonstration. In the future more demand types
// will be offered
//
// V1 - Force12.io reacts to increased demand by starting/stopping containers on the slaves already in play.
//
// This version of Force12 starts and stops containers on a Mesos cluser using Marathon as the scheduler
package main

import (
	"log"
	"os"
	"strconv"
	"time"

	"bitbucket.org/force12io/force12-scheduler/api"
	"bitbucket.org/force12io/force12-scheduler/compose"
	"bitbucket.org/force12io/force12-scheduler/consul"
	"bitbucket.org/force12io/force12-scheduler/demand"
	"bitbucket.org/force12io/force12-scheduler/marathon"
	"bitbucket.org/force12io/force12-scheduler/rng"
	"bitbucket.org/force12io/force12-scheduler/scheduler"
	"bitbucket.org/force12io/force12-scheduler/toy_scheduler"
)

const const_sleep = 100           // milliseconds
const const_sendstate_sleeps = 5  // number of sleeps before we send state on the API
const const_stopsleep = 250       // milliseconds pause between stopping and restarting containers
const const_p1demandstart int = 1 // The yaml file will automatically start one of each
const const_p2demandstart int = 1

var p1TaskName string = "priority1"
var p2TaskName string = "priority2"
var p1FamilyName string = "p1-family"
var p2FamilyName string = "p2-family"

var maximumContainers int

var tasks map[string]demand.Task

// handleDemandChange checks the new demand
func handleDemandChange(input demand.Input, s scheduler.Scheduler) error {
	var err error = nil
	var demandChanged bool

	demandChanged, err = update(input)
	if err != nil {
		log.Printf("Failed to get new demand. %v", err)
		return err
	}

	if demandChanged {
		// Ask the scheduler to make the changes
		for name, task := range tasks {
			err = s.StopStartNTasks(name, &task)
			if err != nil {
				log.Printf("Failed to start %s tasks. %v", name, err)
				break
			}
		}
	}

	return err
}

// update checks for changes in demand, returning true if demand changed
func update(input demand.Input) (bool, error) {
	var err error = nil
	var demandchange bool

	// TODO! Make this less tied to the p1 / p2 simple model
	var p1 demand.Task = tasks[p1TaskName]
	var p2 demand.Task = tasks[p2TaskName]

	// Save the old demand
	oldP1Demand := p1.Demand
	oldP2Demand := p2.Demand

	p1.Demand, err = input.GetDemand(p1TaskName)
	if err != nil {
		log.Printf("Failed to get new demand for task %s. %v", p1TaskName, err)
		return false, err
	}
	//log.Printf("container count %v\n", container_count)
	p2.Demand = maximumContainers - p1.Demand

	//Has the demand changed?
	demandchange = (p1.Demand != oldP1Demand) || (p2.Demand != oldP2Demand)

	// Update tsaks map
	tasks[p1TaskName] = p1
	tasks[p2TaskName] = p2

	// This is where we could decide whether this is a significant enough
	// demand change to do anything

	log.Println(tasks)

	return demandchange, err
}

func getEnvOrDefault(name string, defaultValue string) string {
	v := os.Getenv(name)
	if v == "" {
		v = defaultValue
	}

	return v
}

// For the simple prototype, Force12.io sits in a loop checking for demand changes every X milliseconds
// In phase 2 we'll add a reactive mode where appropriate.
//
// Note - we don't route messages from demandcheckers to demandhandlers using channels because we want new values
// to override old values. Queued history is of no importance here.
//
// Also for simplicity this first release is concurrency free (single threaded)
func main() {
	var err error
	// TODO! Make it so you can send in settings on the command line
	var demandModelType string = getEnvOrDefault("F12_DEMAND_MODEL", "RNG")
	var schedulerType string = getEnvOrDefault("F12_SCHEDULER", "COMPOSE")
	var sendstateString string = getEnvOrDefault("F12_SEND_STATE_TO_API", "true")
	var sendstate bool = (sendstateString == "true")
	var userID string = getEnvOrDefault("F12_USER_ID", "5k5gk")
	var demandIntervalMs int
	var demandDelta int
	demandDelta, _ = strconv.Atoi(getEnvOrDefault("F12_DEMAND_DELTA", "3"))
	demandIntervalMs, _ = strconv.Atoi(getEnvOrDefault("F12_DEMAND_CHANGE_INTERVAL_MS", "3000"))
	var demandInterval time.Duration = time.Duration(demandIntervalMs) * time.Millisecond
	maximumContainers, _ = strconv.Atoi(getEnvOrDefault("F12_MAXIMUM_CONTAINERS", "9"))
	p1TaskName = getEnvOrDefault("F12_PRIORITY1_TASK", p1TaskName)
	p2TaskName = getEnvOrDefault("F12_PRIORITY2_TASK", p2TaskName)
	// TODO!! FInd out what CLIENT/SERVER_FAMILY should default to
	p1FamilyName = getEnvOrDefault("F12_PRIORITY1_FAMILY", p1FamilyName)
	p2FamilyName = getEnvOrDefault("F12_PRIORITY2_FAMILY", p2FamilyName)

	log.Printf("Vary tasks %s and %s with delta %d up to max %d containers every %d s", p1TaskName, p2TaskName, demandDelta, maximumContainers, int(demandInterval.Seconds()))

	var di demand.Input
	var s scheduler.Scheduler

	switch demandModelType {
	case "CONSUL":
		log.Println("Getting demand metric from Consul")
		di = consul.NewDemandModel()
	case "RNG":
		log.Println("Random demand generation")
		di = rng.NewDemandModel(demandDelta, maximumContainers)
	default:
		log.Printf("Bad value for F12_DEMAND_MODEL: %s", demandModelType)
		return
	}

	switch schedulerType {
	case "COMPOSE":
		log.Println("Scheduling with Docker compose")
		s = compose.NewScheduler()
	case "ECS":
		log.Println("Scheduling with ECS not yet supported")
		return
	case "MESOS":
		log.Println("Scheduling with Mesos / Marathon")
		s = marathon.NewScheduler()
	case "TOY":
		log.Println("Scheduling with toy scheduler")
		s = toy_scheduler.NewScheduler()
	default:
		log.Printf("Bad value for F12_SCHEDULER: %s", schedulerType)
		return
	}

	// Initialise task types
	tasks = make(map[string]demand.Task)

	tasks[p1TaskName] = demand.Task{
		FamilyName: p1FamilyName,
		Demand:     const_p1demandstart,
		Requested:  0,
	}

	tasks[p2TaskName] = demand.Task{
		FamilyName: p2FamilyName,
		Demand:     const_p2demandstart,
		Requested:  0,
	}

	// Let the scheduler know about these task types. For the moment the actual container information is hard-coded
	for name, _ := range tasks {
		err = s.InitScheduler(name)
		if err != nil {
			log.Printf("Failed to start task %s: %v", name, err)
			return
		}
	}

	var sleepcount int = 0
	var sleep time.Duration = const_sleep * time.Millisecond
	lastDemandUpdate := time.Now()

	// Loop, continually checking for changes in demand that need to be scheduled
	// At the moment we plough on regardless in the face of errors, simply logging them out
	for {
		// Don't change demand more often than defined by demandInterval
		if time.Since(lastDemandUpdate) > demandInterval {
			err = handleDemandChange(di, s)
			if err != nil {
				log.Printf("Failed to handle demand change. %v", err)
			}
			lastDemandUpdate = time.Now()
		}

		time.Sleep(sleep)
		sleepcount++
		if sleepcount == const_sendstate_sleeps {
			sleepcount = 0

			//Periodically send state to the API if required
			if sendstate {
				err = api.SendState(userID, s, tasks)
				if err != nil {
					log.Printf("Failed to send state. %v", err)
				}
			}
		}
	}
}
<commit_msg>A little comment tidy<commit_after>// Force12.io is a package that monitors demand for resource in a system and then scales and repurposes
// containers, based on agreed "quality of service" contracts, to best handle that demand within the constraints of your existing VM
// or physical infrastructure (for v1).
//
// Force12 is defined to optimize the use of existing physical and VM resources instantly. VMs cannot be scaled in real time (it takes
// several minutes) and new physical machines take even longer. However, containers can be started or stopped at sub-second speeds,
// allowing your infrastructure to adapt itself in real time to meet system demands.
//
// Force12 is aimed at effectively using the resources you have right now - your existing VMs or physical servers - by using them as
// optimally as possible.
//
// The Force12 approach is analogous to the way that a router dynamically optimises the use of a physical network. A router is limited
// by the capacity of the lines physically connected to it. Adding additional capacity is a physical process and takes time. Routers
// therefore make decisions in real time about which packets will be prioritized on a particular line based on the packet's priority
// (defined by a "quality of service" contract).
//
// For example, at times of high bandwidth usage a router might prioritize VOIP traffic over web browsing in real time.
//
// Containers allow Force12 to make similar "instant" judgements on service prioritisation within your existing infrastructure. Routers
// make very simplistic judgments because they have limited time and cpu and they act at a per packet level. Force12 has the capability
// of making far more sophisticated judgements, although even fairly simple ones will still provide a significant new service.
//
// This prototype is a bare bones implementation of Force12.io that recognises only 1 demand type:
// randomised demand for a priority 1 service. Resources are allocated to meet this demand for priority 1, and spare resource can
// be used for a priority 2 service.
//
// These demand type examples have been chosen purely for simplicity of demonstration. In the future more demand types
// will be offered
package main

import (
	"log"
	"os"
	"strconv"
	"time"

	"bitbucket.org/force12io/force12-scheduler/api"
	"bitbucket.org/force12io/force12-scheduler/compose"
	"bitbucket.org/force12io/force12-scheduler/consul"
	"bitbucket.org/force12io/force12-scheduler/demand"
	"bitbucket.org/force12io/force12-scheduler/marathon"
	"bitbucket.org/force12io/force12-scheduler/rng"
	"bitbucket.org/force12io/force12-scheduler/scheduler"
	"bitbucket.org/force12io/force12-scheduler/toy_scheduler"
)

const const_sleep = 100           // milliseconds
const const_sendstate_sleeps = 5  // number of sleeps before we send state on the API
const const_stopsleep = 250       // milliseconds pause between stopping and restarting containers
const const_p1demandstart int = 1 // The yaml file will automatically start one of each
const const_p2demandstart int = 1

var p1TaskName string = "priority1"
var p2TaskName string = "priority2"
var p1FamilyName string = "p1-family"
var p2FamilyName string = "p2-family"

var maximumContainers int

var tasks map[string]demand.Task

// handleDemandChange checks the new demand
func handleDemandChange(input demand.Input, s scheduler.Scheduler) error {
	var err error = nil
	var demandChanged bool

	demandChanged, err = update(input)
	if err != nil {
		log.Printf("Failed to get new demand. %v", err)
		return err
	}

	if demandChanged {
		// Ask the scheduler to make the changes
		for name, task := range tasks {
			err = s.StopStartNTasks(name, &task)
			if err != nil {
				log.Printf("Failed to start %s tasks. %v", name, err)
				break
			}
		}
	}

	return err
}

// update checks for changes in demand, returning true if demand changed
func update(input demand.Input) (bool, error) {
	var err error = nil
	var demandchange bool

	// TODO! Make this less tied to the p1 / p2 simple model
	var p1 demand.Task = tasks[p1TaskName]
	var p2 demand.Task = tasks[p2TaskName]

	// Save the old demand
	oldP1Demand := p1.Demand
	oldP2Demand := p2.Demand

	p1.Demand, err = input.GetDemand(p1TaskName)
	if err != nil {
		log.Printf("Failed to get new demand for task %s. %v", p1TaskName, err)
		return false, err
	}
	//log.Printf("container count %v\n", container_count)
	p2.Demand = maximumContainers - p1.Demand

	//Has the demand changed?
	demandchange = (p1.Demand != oldP1Demand) || (p2.Demand != oldP2Demand)

	// Update tsaks map
	tasks[p1TaskName] = p1
	tasks[p2TaskName] = p2

	// This is where we could decide whether this is a significant enough
	// demand change to do anything

	log.Println(tasks)

	return demandchange, err
}

func getEnvOrDefault(name string, defaultValue string) string {
	v := os.Getenv(name)
	if v == "" {
		v = defaultValue
	}

	return v
}

// For the simple prototype, Force12.io sits in a loop checking for demand changes every X milliseconds
// In phase 2 we'll add a reactive mode where appropriate.
//
// Note - we don't route messages from demandcheckers to demandhandlers using channels because we want new values
// to override old values. Queued history is of no importance here.
//
// Also for simplicity this first release is concurrency free (single threaded)
func main() {
	var err error
	// TODO! Make it so you can send in settings on the command line
	var demandModelType string = getEnvOrDefault("F12_DEMAND_MODEL", "RNG")
	var schedulerType string = getEnvOrDefault("F12_SCHEDULER", "COMPOSE")
	var sendstateString string = getEnvOrDefault("F12_SEND_STATE_TO_API", "true")
	var sendstate bool = (sendstateString == "true")
	var userID string = getEnvOrDefault("F12_USER_ID", "5k5gk")
	var demandIntervalMs int
	var demandDelta int
	demandDelta, _ = strconv.Atoi(getEnvOrDefault("F12_DEMAND_DELTA", "3"))
	demandIntervalMs, _ = strconv.Atoi(getEnvOrDefault("F12_DEMAND_CHANGE_INTERVAL_MS", "3000"))
	var demandInterval time.Duration = time.Duration(demandIntervalMs) * time.Millisecond
	maximumContainers, _ = strconv.Atoi(getEnvOrDefault("F12_MAXIMUM_CONTAINERS", "9"))
	p1TaskName = getEnvOrDefault("F12_PRIORITY1_TASK", p1TaskName)
	p2TaskName = getEnvOrDefault("F12_PRIORITY2_TASK", p2TaskName)
	// TODO!! FInd out what CLIENT/SERVER_FAMILY should default to
	p1FamilyName = getEnvOrDefault("F12_PRIORITY1_FAMILY", p1FamilyName)
	p2FamilyName = getEnvOrDefault("F12_PRIORITY2_FAMILY", p2FamilyName)

	log.Printf("Vary tasks %s and %s with delta %d up to max %d containers every %d s", p1TaskName, p2TaskName, demandDelta, maximumContainers, int(demandInterval.Seconds()))

	var di demand.Input
	var s scheduler.Scheduler

	switch demandModelType {
	case "CONSUL":
		log.Println("Getting demand metric from Consul")
		di = consul.NewDemandModel()
	case "RNG":
		log.Println("Random demand generation")
		di = rng.NewDemandModel(demandDelta, maximumContainers)
	default:
		log.Printf("Bad value for F12_DEMAND_MODEL: %s", demandModelType)
		return
	}

	switch schedulerType {
	case "COMPOSE":
		log.Println("Scheduling with Docker compose")
		s = compose.NewScheduler()
	case "ECS":
		log.Println("Scheduling with ECS not yet supported")
		return
	case "MESOS":
		log.Println("Scheduling with Mesos / Marathon")
		s = marathon.NewScheduler()
	case "TOY":
		log.Println("Scheduling with toy scheduler")
		s = toy_scheduler.NewScheduler()
	default:
		log.Printf("Bad value for F12_SCHEDULER: %s", schedulerType)
		return
	}

	// Initialise task types
	tasks = make(map[string]demand.Task)

	tasks[p1TaskName] = demand.Task{
		FamilyName: p1FamilyName,
		Demand:     const_p1demandstart,
		Requested:  0,
	}

	tasks[p2TaskName] = demand.Task{
		FamilyName: p2FamilyName,
		Demand:     const_p2demandstart,
		Requested:  0,
	}

	// Let the scheduler know about these task types. For the moment the actual container information is hard-coded
	for name, _ := range tasks {
		err = s.InitScheduler(name)
		if err != nil {
			log.Printf("Failed to start task %s: %v", name, err)
			return
		}
	}

	var sleepcount int = 0
	var sleep time.Duration = const_sleep * time.Millisecond
	lastDemandUpdate := time.Now()

	// Loop, continually checking for changes in demand that need to be scheduled
	// At the moment we plough on regardless in the face of errors, simply logging them out
	for {
		// Don't change demand more often than defined by demandInterval
		if time.Since(lastDemandUpdate) > demandInterval {
			err = handleDemandChange(di, s)
			if err != nil {
				log.Printf("Failed to handle demand change. %v", err)
			}
			lastDemandUpdate = time.Now()
		}

		time.Sleep(sleep)
		sleepcount++
		if sleepcount == const_sendstate_sleeps {
			sleepcount = 0

			//Periodically send state to the API if required
			if sendstate {
				err = api.SendState(userID, s, tasks)
				if err != nil {
					log.Printf("Failed to send state. %v", err)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/chrisledet/rebasebot/config"
	"github.com/chrisledet/rebasebot/git"
	"github.com/chrisledet/rebasebot/github"
	_http "github.com/chrisledet/rebasebot/http"
)

var (
	BotConfig config.Config
)

func main() {
	http.HandleFunc("/", _http.Receive)
	http.HandleFunc("/status", _http.Status)

	port := "8080"
	configPath := "./rebasebot.json"

	if len(os.Getenv("PORT")) > 0 {
		port = os.Getenv("PORT")
	}

	if len(os.Getenv("CONFIG")) > 0 {
		configPath = os.Getenv("CONFIG")
	}

	go func() {
		log.Printf("config.load.started: %s\n", configPath)

		BotConfig, err := config.LoadFromPath(configPath)

		if err != nil {
			fmt.Printf("config.load.failed: %s\n", configPath)
			return
		}

		log.Printf("config.load.finished: %s\n", configPath)

		if len(BotConfig.TmpDir) > 0 {
			git.SetParentDir(BotConfig.TmpDir)
		}

		github.SetAuth(BotConfig.Username, BotConfig.Password)

		log.Printf("server.tmpdir.cleanup.started: %s\n", git.ParentDir())

		if err := os.RemoveAll(git.ParentDir()); err != nil {
			log.Fatalf("server.tmpdir.cleanup.failed: %s\n", err.Error())
		}

		log.Printf("server.tmpdir.cleanup.finished: %s\n", git.ParentDir())
	}()

	log.Printf("server.up: 0.0.0.0:%s\n", port)
	http.ListenAndServe(":"+port, nil)
}
<commit_msg>Use log over fmt?<commit_after>package main

import (
	"log"
	"net/http"
	"os"

	"github.com/chrisledet/rebasebot/config"
	"github.com/chrisledet/rebasebot/git"
	"github.com/chrisledet/rebasebot/github"
	_http "github.com/chrisledet/rebasebot/http"
)

var (
	BotConfig config.Config
)

func main() {
	http.HandleFunc("/", _http.Receive)
	http.HandleFunc("/status", _http.Status)

	port := "8080"
	configPath := "./rebasebot.json"

	if len(os.Getenv("PORT")) > 0 {
		port = os.Getenv("PORT")
	}

	if len(os.Getenv("CONFIG")) > 0 {
		configPath = os.Getenv("CONFIG")
	}

	go func() {
		log.Printf("config.load.started: %s\n", configPath)

		BotConfig, err := config.LoadFromPath(configPath)

		if err != nil {
			log.Printf("config.load.failed: %s\n", configPath)
			return
		}

		log.Printf("config.load.finished: %s\n", configPath)

		if len(BotConfig.TmpDir) > 0 {
			git.SetParentDir(BotConfig.TmpDir)
		}

		github.SetAuth(BotConfig.Username, BotConfig.Password)

		log.Printf("server.tmpdir.cleanup.started: %s\n", git.ParentDir())

		if err := os.RemoveAll(git.ParentDir()); err != nil {
			log.Fatalf("server.tmpdir.cleanup.failed: %s\n", err.Error())
		}

		log.Printf("server.tmpdir.cleanup.finished: %s\n", git.ParentDir())
	}()

	log.Printf("server.up: 0.0.0.0:%s\n", port)
	http.ListenAndServe(":"+port, nil)
}
<|endoftext|>


<commit_before>package floats

import (
	"math"
	"math/big"
)

// agm returns the arithmetic-geometric mean of a and b, to
// max(a.Prec, b.Prec) bits of precision. a and b must have
// the same precision.
func agm(a, b *big.Float) *big.Float {

	if a.Prec() != b.Prec() {
		panic("agm: different precisions")
	}

	prec := a.Prec()

	// do not overwrite a and b
	a2 := new(big.Float).Copy(a).SetPrec(prec + 64)
	b2 := new(big.Float).Copy(b).SetPrec(prec + 64)

	if a2.Cmp(b2) == -1 {
		a2, b2 = b2, a2
	}
	// a >= b

	// set lim to 2**(-prec)
	lim := new(big.Float)
	lim.SetMantExp(big.NewFloat(1).SetPrec(prec+64), -int(prec+1))

	half := big.NewFloat(0.5)
	t := new(big.Float)

	for t.Sub(a2, b2).Cmp(lim) != -1 {
		t.Copy(a2)
		a2.Add(a2, b2).Mul(a2, half)
		b2 = Sqrt(b2.Mul(b2, t))
	}

	return a2.SetPrec(prec)
}

var piCache *big.Float
var piCachePrec uint
var enablePiCache bool = true

func init() {
	if !enablePiCache {
		return
	}

	piCache, _, _ = new(big.Float).SetPrec(1024).Parse("3."+
		"14159265358979323846264338327950288419716939937510"+
		"58209749445923078164062862089986280348253421170679"+
		"82148086513282306647093844609550582231725359408128"+
		"48111745028410270193852110555964462294895493038196"+
		"44288109756659334461284756482337867831652712019091"+
		"45648566923460348610454326648213393607260249141273"+
		"72458700660631558817488152092096282925409171536444", 10)

	piCachePrec = 1024
}

// pi returns pi to prec bits of precision
func pi(prec uint) *big.Float {

	if prec <= piCachePrec && enablePiCache {
		return new(big.Float).Copy(piCache).SetPrec(prec)
	}

	precExt := prec + 64

	half := big.NewFloat(0.5)
	one := big.NewFloat(1)
	two := big.NewFloat(2)
	four := big.NewFloat(4)

	temp := new(big.Float)
	a := new(big.Float).SetPrec(precExt).SetInt64(1)         // a_0 = 1
	b := new(big.Float).Quo(one, Sqrt(two.SetPrec(precExt))) // b_0 = 1/sqrt(2)
	t := new(big.Float).SetPrec(precExt).SetFloat64(0.25)    // t_0 = 1/4
	p := new(big.Float).SetPrec(precExt).SetInt64(1)         // p_0 = 1

	steps := math.Log2(float64(precExt))

	a2 := new(big.Float)
	for i := 0; i < int(steps); i++ {
		a2.Add(a, b).Mul(a2, half) // a_{n+1} = (a_{n} + b_{n}) / 2
		b = Sqrt(temp.Mul(a, b))   // b_{n+1} = sqrt(a_{n} * b_{n})

		temp.Sub(a, a2).Mul(temp, temp) // temp = (a_{n} - a_{n+1})²
		t.Sub(t, temp.Mul(p, temp))     // t_{n+1} = t_{n} - p_{n} * temp

		p.Mul(two, p) // p_{n+1} = 2 * p_{n}

		a.Copy(a2)
	}

	temp.Add(a, b)
	temp.Mul(temp, temp)
	res := new(big.Float).Quo(temp, t.Mul(four, t)) // pi = (a_{n+1} + b_{n+1})² / (4t_{n+1})
	res.SetPrec(prec)

	if enablePiCache {
		piCache.Copy(res)
		piCachePrec = prec
	}

	return res
}

// returns an approximate (to precision dPrec) solution to
//    f(t) = 0
// using the Newton Method.
// fOverDf needs to be a fuction returning f(t)/f'(t).
// t must not be changed by fOverDf.
// guess is the initial guess (and it's not preserved).
func newton(fOverDf func(z *big.Float) *big.Float, guess *big.Float, dPrec uint) *big.Float {

	prec, guard := guess.Prec(), uint(64)
	guess.SetPrec(prec + guard)

	for prec < 2*dPrec {
		guess.Sub(guess, fOverDf(guess))
		prec *= 2
		guess.SetPrec(prec + guard)
	}

	return guess.SetPrec(dPrec)
}
<commit_msg>misc.go: clean up pi<commit_after>package floats

import "math/big"

// agm returns the arithmetic-geometric mean of a and b.
// a and b must have the same precision.
func agm(a, b *big.Float) *big.Float {

	if a.Prec() != b.Prec() {
		panic("agm: different precisions")
	}

	prec := a.Prec()

	// do not overwrite a and b
	a2 := new(big.Float).Copy(a).SetPrec(prec + 64)
	b2 := new(big.Float).Copy(b).SetPrec(prec + 64)

	if a2.Cmp(b2) == -1 {
		a2, b2 = b2, a2
	}
	// a2 >= b2

	// set lim to 2**(-prec)
	lim := new(big.Float)
	lim.SetMantExp(big.NewFloat(1).SetPrec(prec+64), -int(prec+1))

	half := big.NewFloat(0.5)
	t := new(big.Float)

	for t.Sub(a2, b2).Cmp(lim) != -1 {
		t.Copy(a2)
		a2.Add(a2, b2).Mul(a2, half)
		b2 = Sqrt(b2.Mul(b2, t))
	}

	return a2.SetPrec(prec)
}

var piCache *big.Float
var piCachePrec uint
var enablePiCache bool = true

func init() {
	if !enablePiCache {
		return
	}

	piCache, _, _ = new(big.Float).SetPrec(1024).Parse("3."+
		"14159265358979323846264338327950288419716939937510"+
		"58209749445923078164062862089986280348253421170679"+
		"82148086513282306647093844609550582231725359408128"+
		"48111745028410270193852110555964462294895493038196"+
		"44288109756659334461284756482337867831652712019091"+
		"45648566923460348610454326648213393607260249141273"+
		"72458700660631558817488152092096282925409171536444", 10)

	piCachePrec = 1024
}

// pi returns pi to prec bits of precision
func pi(prec uint) *big.Float {

	if prec <= piCachePrec && enablePiCache {
		return new(big.Float).Copy(piCache).SetPrec(prec)
	}

	// Following R. P. Brent, Multiple-precision zero-finding
	// methods and the complexity of elementary function evaluation,
	// in Analytic Computational Complexity, Academic Press,
	// New York, 1975, Section 8.

	half := big.NewFloat(0.5)
	two := big.NewFloat(2).SetPrec(prec + 64)

	// initialization
	a := big.NewFloat(1).SetPrec(prec + 64)    // a = 1
	b := new(big.Float).Mul(Sqrt(two), half)   // b = 1/√2
	t := big.NewFloat(0.25).SetPrec(prec + 64) // t = 1/4
	x := big.NewFloat(1).SetPrec(prec + 64)    // x = 1

	// limit is 2**(-prec)
	lim := new(big.Float)
	lim.SetMantExp(big.NewFloat(1).SetPrec(prec+64), -int(prec+1))

	// temp variables
	y := new(big.Float)
	for y.Sub(a, b).Cmp(lim) != -1 { // assume a > b
		y.Copy(a)
		a.Add(a, b).Mul(a, half) // a = (a+b)/2
		b = Sqrt(b.Mul(b, y))    // b = √(ab)

		y.Sub(a, y)           // y = a - y
		y.Mul(y, y).Mul(y, x) // y = x(a-y)²
		t.Sub(t, y)           // t = t - x(a-y)²
		x.Mul(x, two)         // x = 2x
	}

	a.Mul(a, a).Quo(a, t) // π = a² / t
	a.SetPrec(prec)

	if enablePiCache {
		piCache.Copy(a)
		piCachePrec = prec
	}

	return a
}

// returns an approximate (to precision dPrec) solution to
//    f(t) = 0
// using the Newton Method.
// fOverDf needs to be a fuction returning f(t)/f'(t).
// t must not be changed by fOverDf.
// guess is the initial guess (and it's not preserved).
func newton(fOverDf func(z *big.Float) *big.Float, guess *big.Float, dPrec uint) *big.Float {

	prec, guard := guess.Prec(), uint(64)
	guess.SetPrec(prec + guard)

	for prec < 2*dPrec {
		guess.Sub(guess, fOverDf(guess))
		prec *= 2
		guess.SetPrec(prec + guard)
	}

	return guess.SetPrec(dPrec)
}
<|endoftext|>


<commit_before>package main

import "github.com/howeyc/fsnotify"
import "log"
import "flag"
import "fmt"
import "os"
import "os/exec"
import "regexp"

var versionStr = "0.1.0"

var goCommandSet = map[string][]string{
	"test":    []string{"go", "test"},
	"install": []string{"go", "install"},
	"build":   []string{"go", "build"},
}

func main() {
	var helpFlag = flag.Bool("h", false, "help")
	var buildFlag = flag.Bool("b", true, "build, default behavior")
	var testFlag = flag.Bool("t", false, "test")
	var installFlag = flag.Bool("i", false, "install")
	var versionFlag = flag.Bool("v", false, "version")
	var xFlag = flag.Bool("x", false, "show verbose command")

	flag.Parse()
	args := flag.Args()

	if *helpFlag {
		fmt.Println("Usage: gomon [options] [dir] [-- command]")
		fmt.Println("   -b build")
		fmt.Println("   -t test")
		fmt.Println("   -i install")
		fmt.Println("   -x show verbose command")
		fmt.Println("   -h help")
		os.Exit(0)
	}
	if *versionFlag {
		fmt.Printf("gomon %s\n", versionStr)
		os.Exit(0)
	}

	var dirs = []string{}
	var cmds = []string{}

	var takeDir = true
	for _, a := range args {
		var exists, _ = FileExists(a)
		if a == "--" || (takeDir && !exists) {
			takeDir = false
			continue
		}
		if takeDir {
			dirs = append(dirs, a)
		} else {
			cmds = append(cmds, a)
		}
	}

	if len(cmds) == 0 {
		if *testFlag {
			cmds = goCommandSet["test"]
		} else if *buildFlag {
			cmds = goCommandSet["build"]
		} else if *installFlag {
			cmds = goCommandSet["install"]
		}
		if *xFlag && len(cmds) > 0 {
			cmds = append(cmds, "-x")
		}
	}

	if len(cmds) == 0 {
		fmt.Println("No command specified")
		os.Exit(2)
	}

	if len(dirs) == 0 {
		var cwd, err = os.Getwd()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		dirs = []string{cwd}
	}

	fmt.Println("Watching", dirs, "for", cmds)

	watcher, err := fsnotify.NewWatcher()

	if err != nil {
		log.Fatal(err)
	}

	for _, dir := range dirs {
		subfolders := Subfolders(dir)
		for _, f := range subfolders {
			err = watcher.WatchFlags(f, fsnotify.FSN_CREATE|fsnotify.FSN_MODIFY)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	var cmd *exec.Cmd

	for {
		select {
		case e := <-watcher.Event:
			matched, err := regexp.MatchString("\\.(go|c|h)$", e.Name)
			if err != nil {
				log.Println(err)
			}

			if !matched {
				continue
			}

			log.Println("Event:", e)

			if cmd != nil && cmd.ProcessState != nil && !cmd.ProcessState.Exited() {
				err := cmd.Process.Kill()
				if err != nil {
					log.Println(err)
				}
			}
			cmd = exec.Command(cmds[0], cmds[1:]...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			cmd.Start()
		case err := <-watcher.Error:
			log.Println("Error:", err)
		}
	}

	watcher.Close()
}
<commit_msg>Add gntp option<commit_after>package main

import "github.com/howeyc/fsnotify"
import "log"
import "flag"
import "fmt"
import "os"
import "os/exec"
import "regexp"

var versionStr = "0.1.0"

var goCommandSet = map[string][]string{
	"test":    []string{"go", "test"},
	"install": []string{"go", "install"},
	"build":   []string{"go", "build"},
}

func main() {
	var helpFlag = flag.Bool("h", false, "help")
	var buildFlag = flag.Bool("b", true, "build, default behavior")
	var testFlag = flag.Bool("t", false, "test")
	var installFlag = flag.Bool("i", false, "install")
	var versionFlag = flag.Bool("v", false, "version")
	var xFlag = flag.Bool("x", false, "show verbose command")
	var gntpServer = flag.String("gntp", "127.0.0.1:23053", "use Growler")

	flag.Parse()
	args := flag.Args()

	if *helpFlag {
		fmt.Println("Usage: gomon [options] [dir] [-- command]")
		fmt.Println("   -b build")
		fmt.Println("   -t test")
		fmt.Println("   -i install")
		fmt.Println("   -x show verbose command")
		fmt.Println("   -h help")
		os.Exit(0)
	}
	if *versionFlag {
		fmt.Printf("gomon %s\n", versionStr)
		os.Exit(0)
	}

	var dirs = []string{}
	var cmds = []string{}

	var takeDir = true
	for _, a := range args {
		var exists, _ = FileExists(a)
		if a == "--" || (takeDir && !exists) {
			takeDir = false
			continue
		}
		if takeDir {
			dirs = append(dirs, a)
		} else {
			cmds = append(cmds, a)
		}
	}

	if len(cmds) == 0 {
		if *testFlag {
			cmds = goCommandSet["test"]
		} else if *buildFlag {
			cmds = goCommandSet["build"]
		} else if *installFlag {
			cmds = goCommandSet["install"]
		}
		if *xFlag && len(cmds) > 0 {
			cmds = append(cmds, "-x")
		}
	}

	if len(cmds) == 0 {
		fmt.Println("No command specified")
		os.Exit(2)
	}

	if len(dirs) == 0 {
		var cwd, err = os.Getwd()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		dirs = []string{cwd}
	}

	fmt.Println("Watching", dirs, "for", cmds)

	watcher, err := fsnotify.NewWatcher()

	if err != nil {
		log.Fatal(err)
	}

	for _, dir := range dirs {
		subfolders := Subfolders(dir)
		for _, f := range subfolders {
			err = watcher.WatchFlags(f, fsnotify.FSN_CREATE|fsnotify.FSN_MODIFY)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	var cmd *exec.Cmd

	runCommand := func(cmd *exec.Cmd) {
		var err error
		err = cmd.Start()
		if err != nil {
			log.Println(err)
			notifyFail(err.Error(), "", gntpServer)
			return
		}
		err = cmd.Wait()
		if err != nil {
			log.Println(err)
			notifyFail(err.Error(), "", gntpServer)
			return
		}
	}

	for {
		select {
		case e := <-watcher.Event:
			matched, err := regexp.MatchString("\\.(go|c|h)$", e.Name)
			if err != nil {
				log.Println(err)
			}

			if !matched {
				continue
			}

			log.Println("Event:", e)

			if cmd != nil && cmd.ProcessState != nil && !cmd.ProcessState.Exited() {
				err := cmd.Process.Kill()
				if err != nil {
					log.Println(err)
				}
			}
			cmd = exec.Command(cmds[0], cmds[1:]...)
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr
			go runCommand(cmd)

		case err := <-watcher.Error:
			log.Println("Error:", err)
		}
	}

	watcher.Close()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	ci "github.com/ipfs/go-libp2p-crypto"
	peer "github.com/ipfs/go-libp2p-peer"
)

func main() {
	size := flag.Int("bitsize", 2048, "select the bitsize of the key to generate")
	typ := flag.String("type", "RSA", "select type of key to generate")

	flag.Parse()

	var atyp int
	switch *typ {
	case "RSA":
		atyp = ci.RSA
	default:
		fmt.Fprintln(os.Stderr, "unrecognized key type: ", *typ)
		os.Exit(1)
	}

	fmt.Fprintf(os.Stderr, "Generating a %d bit %s key...\n", *size, *typ)
	priv, pub, err := ci.GenerateKeyPair(atyp, *size)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	fmt.Fprintln(os.Stderr, "Success!")

	pid, err := peer.IDFromPublicKey(pub)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Fprintf(os.Stderr, "ID for generated key: %s\n", pid.Pretty())

	data, err := priv.Bytes()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	os.Stdout.Write(data)
}
<commit_msg>Update imports<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	ci "github.com/libp2p/go-libp2p-crypto"
	peer "github.com/libp2p/go-libp2p-peer"
)

func main() {
	size := flag.Int("bitsize", 2048, "select the bitsize of the key to generate")
	typ := flag.String("type", "RSA", "select type of key to generate")

	flag.Parse()

	var atyp int
	switch *typ {
	case "RSA":
		atyp = ci.RSA
	default:
		fmt.Fprintln(os.Stderr, "unrecognized key type: ", *typ)
		os.Exit(1)
	}

	fmt.Fprintf(os.Stderr, "Generating a %d bit %s key...\n", *size, *typ)
	priv, pub, err := ci.GenerateKeyPair(atyp, *size)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	fmt.Fprintln(os.Stderr, "Success!")

	pid, err := peer.IDFromPublicKey(pub)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	fmt.Fprintf(os.Stderr, "ID for generated key: %s\n", pid.Pretty())

	data, err := priv.Bytes()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	os.Stdout.Write(data)
}
<|endoftext|>