<commit_before>package main

import (
    "fmt"
	"encoding/csv"
	"os"
	"io"
	"path"
	"path/filepath"
	"log"
	"time"
	"strings"
	"sort"
	"strconv"
	"math"
	"github.com/garyburd/redigo/redis"
)

// Structure to hold filenames to be ordered by the time part
type Event struct {
	path string
	ts time.Time
}

type Events []Event

// Declare methods needed to sort implementation on events
//
func (this Events) Len() int {
    return len(this)
}
func (this Events) Less(i, j int) bool {
    return this[i].ts.Unix() < this[j].ts.Unix()
}
func (this Events) Swap(i, j int) {
    this[i], this[j] = this[j], this[i]
}

// Dum error checking function

func check(e error) {
    if e != nil {
        panic(e)
    }
}

// Function called to create a summary file for Key with same Timestamp
// and then delete all those eys from DB

func searchAndDestroy(key int64) {

	// Open output file for Key

		filename := fmt.Sprintf("/Volumes/BigBud/data/vf/results/%d.txt", key)
	   	file, err := os.Create(filename)
	   	check(err)
	   	defer file.Close()

		// Write Headers
	    _, err = file.WriteString("Cell, Count\n")

		// Issue a Sync to flush writes to stable storage.
	    file.Sync()

	// Get Connection from the pool. Only way to be thread-safe with Redis
		c := pool.Get()
		if c == nil {
				log.Fatal("Cannot get connection from Redis Pool")
		}
		defer c.Close()

	// Scan the database to find all entries starting with provided key
	// Write line with count computed for cell by Hyperloglog
	// Delete keys
/*
	if _, err := c.Do("SCAN", redis.Args{}.Add("id1").AddFlat(&p1)...); err != nil {
	   	panic(err)
	}

			s, err := redis.String(c.Do("scan ", key))

			count, err = redis.Uint64(cnx.Do("PFCOUNT", key))
			if err != nil {
				fmt.Println("Can't find key: " + key)			fmt.Println("Error: ")
			} else {
				fmt.Printf("Redis: %s = %d\n", key, count)
			}
*/

}

// Function listen for Timestamps ready to be summarized
// and then cleaned-up

func listenUp(channel chan int64, count int) {

	for key := range channel {
		searchAndDestroy(key)
	}
}

// Read all the files names in all directories from provided root
// Return an ordered list of file names to process

func processFile(inFile string) error {

// Get Connection from the pool. Only way to be thread-safe with Redis
	c := pool.Get()
	if c == nil {
			log.Fatal("Cannot get connection from Redis Pool")
	}
	defer c.Close()

//	println(">> Processing: ", inFile)
	file, err := os.Open(inFile) // For read access.

	if err != nil {
		if err == os.ErrNotExist {
			log.Fatal("No such file!")
		} else {
			log.Fatal(err)
		}
	}
	defer file.Close()

	r := csv.NewReader(file)

	// Read First Line
	row, err := r.Read()

	i := 0

	var count int64

	// Get ride of the unused variable error when printf is commented!
	var _ = count

	count = 0
	var pipeCount float64 = 0

	startTime := time.Now()
	fmt.Printf("%s - %s Procesing starts ", startTime.Format("2006/01/2 - 15:04:05"), inFile)

	for (err != io.EOF) && (len(row) >1) {

		// Could optimize in the case of 5mins files (always same round t)
		// But will break if process larger files

		utime, _ := strconv.ParseInt(row[1], 10, 64)

		t := time.Unix(utime, 0).In(time.UTC)

		// t, _ := time.Parse(time.RFC3339, row[0])

		// Could use Truncate to take lower interval
		tt := t.Truncate(5 * time.Minute)

		key := fmt.Sprintf("%d:%s", tt.Unix(), row[3])
		imsi := row[2]

//		fmt.Printf("     %d - Time=%s Key=%s Imsi=%s\n", i, tt.String(), key, imsi)
		// Add imsi value for that key (Time:Cell) value to Redis

		//Manage Pipeline of command
		if pipeCount == 0 {
			c.Send("Multi")
		} else { // we have a least one command in pipe
			if math.Mod(pipeCount, 20000) == 0.0 {
				pipeCount = 0
				_, err := c.Do("EXEC")
				// fmt.Print("exec")
				check (err)
				c.Send("Multi")
			}
		}
		 // just add one more command
		c.Send ("PFADD", key, imsi)
		pipeCount += 1

		// Next line
     	row, err = r.Read()
		i += 1
	}

	// Still stuff in pipe to finish?
	if pipeCount != 0 {
		_, err := c.Do("EXEC")
		check (err)
	}

	endTime := time.Now()
	duration := endTime.Sub(startTime)

 	if err != nil && err != io.EOF {
   		fmt.Println("Error:", err)
 	}
    fmt.Print(" lasted: ", duration)
	fmt.Printf(" for %d keys inserted\n", i)

	return nil
}

// Utility function to allocate pool

func newPool(server, password string) *redis.Pool {
    return &redis.Pool{
        MaxIdle: 3,
        IdleTimeout: 240 * time.Second,
        Dial: func () (redis.Conn, error) {
            c, err := redis.Dial("tcp", server)
            if err != nil {
                return nil, err
            }
			if password != "" {
            	if _, err := c.Do("AUTH", password); err != nil {
                	c.Close()
                	return nil, err
            	}
			}
            return c, err
        },
        TestOnBorrow: func(c redis.Conn, t time.Time) error {
            _, err := c.Do("PING")
            return err
        },
    }
}

// Global variables

var (
    pool *redis.Pool
    redisServer = "localhost:6379"
    redisPassword = ""
)

// Main function
func main() {

// Create Channel for inter routine communication
	channel := make (chan int64, 10)

// Open Connection to Redis Server
//	const proto = "tcp"
//	const port = ":6379"
//	c, err := redis.Dial(proto, port)

	pool = newPool(redisServer, redisPassword)

	if pool == nil {
			log.Fatal("Cannot Create Redis Pool")
	}

// Get Connection from the pool. Only way to be thread-safe with Redis
	c := pool.Get()
	if c == nil {
			log.Fatal("Cannot get connection from Redis Pool")
	}
	defer c.Close()

	// Create list of file to be processed
	eList := make(Events, 0, 100)

	// Launch 5 processing functions
	for i :=0; i < 5; i++ {
		go listenUp(channel, i)
	}

	// Recursive walk to get all the csv files
	filepath.Walk("/Volumes/BigBud/data/vf", func(aPath string, info os.FileInfo, err error) error {
		// fmt.Println(path)
		if (!info.IsDir() && strings.HasSuffix(info.Name(), ".csv")) {
			slice := strings.Split(path.Base(aPath), ".")
			sDate := slice[0] +":"+ slice[1]
			t, err := time.ParseInLocation("20060102:150405", sDate, time.UTC)
			if err != nil {
				fmt.Println(err)
				log.Fatal("Abort: Bad Filename with date: ", sDate)
			}
			e := Event{aPath, t}
			eList = append(eList, e)
		}
		return nil
	})

	// Order the events by date
	sort.Sort(eList)

	// Loop thru the files, ordered by time and file the unique counts
	// Breaks on 5 min boundaries to geterate cumulated result file and purge redis for those keys
	// Need to slow down ingestion process if too much data being inserted

	var count int = 0

	startTime := time.Now()
	fmt.Println("Let's get started: ", startTime)

	// Initialize first timestamp

	var pTime time.Time

	if eList.Len() > 0 {
		pTime = eList[0].ts
	}

	for _, v := range eList {

		if pTime != v.ts {
			fmt.Printf("Process time %s with key: %d\n", pTime, pTime.Unix())
			channel <- pTime.Unix()
			pTime = v.ts
		}

//		fmt.Printf ("Processing: %s - %s\n", v.ts, v.path)
		err := processFile(v.path)

		if err != nil {
			log.Fatal(err)
		}
		// For test only
		if count += 1; count > 10000 {
			break
		}
	}

// Close all and Cleanup
	close(channel)

	println("Total File List ", eList.Len())

	endTime := time.Now()
	duration := endTime.Sub(startTime)

    fmt.Println("Let's end: ", duration)

	os.Exit(0)
}<commit_msg>small optim and set for cells<commit_after>package main

import (
    "fmt"
	"encoding/csv"
	"os"
	"io"
	"path"
	"path/filepath"
	"log"
	"time"
	"strings"
	"sort"
	"strconv"
	"math"
	"github.com/garyburd/redigo/redis"
)

// Structure to hold filenames to be ordered by the time part
type Event struct {
	path string
	ts time.Time
}

type Events []Event

// Declare methods needed to sort implementation on events
//
func (this Events) Len() int {
    return len(this)
}
func (this Events) Less(i, j int) bool {
    return this[i].ts.Unix() < this[j].ts.Unix()
}
func (this Events) Swap(i, j int) {
    this[i], this[j] = this[j], this[i]
}

// Dum error checking function

func check(e error) {
    if e != nil {
        panic(e)
    }
}

// Function called to create a summary file for Key with same Timestamp
// and then delete all those eys from DB

func searchAndDestroy(key int64) {

	// Open output file for Key

		filename := fmt.Sprintf("/Volumes/BigBud/data/vf/results/%d.txt", key)
	   	file, err := os.Create(filename)
	   	check(err)
	   	defer file.Close()

		// Write Headers
	    _, err = file.WriteString("Cell, Count\n")

		// Issue a Sync to flush writes to stable storage.
	    file.Sync()

	// Get Connection from the pool. Only way to be thread-safe with Redis
		c := pool.Get()
		if c == nil {
				log.Fatal("Cannot get connection from Redis Pool")
		}
		defer c.Close()

	// Scan the database to find all entries starting with provided key
	// Write line with count computed for cell by Hyperloglog
	// Delete keys
/*
	if _, err := c.Do("SCAN", redis.Args{}.Add("id1").AddFlat(&p1)...); err != nil {
	   	panic(err)
	}

			s, err := redis.String(c.Do("scan ", key))

			count, err = redis.Uint64(cnx.Do("PFCOUNT", key))
			if err != nil {
				fmt.Println("Can't find key: " + key)			fmt.Println("Error: ")
			} else {
				fmt.Printf("Redis: %s = %d\n", key, count)
			}
*/

}

// Function listen for Timestamps ready to be summarized
// and then cleaned-up

func listenUp(channel chan int64, count int) {

	for key := range channel {
		searchAndDestroy(key)
	}
}

// Read all the files names in all directories from provided root
// Return an ordered list of file names to process

func processFile(inFile string) error {

// Get Connection from the pool. Only way to be thread-safe with Redis
	c := pool.Get()
	if c == nil {
			log.Fatal("Cannot get connection from Redis Pool")
	}
	defer c.Close()

//	println(">> Processing: ", inFile)
	file, err := os.Open(inFile) // For read access.

	if err != nil {
		if err == os.ErrNotExist {
			log.Fatal("No such file!")
		} else {
			log.Fatal(err)
		}
	}
	defer file.Close()

	r := csv.NewReader(file)

	// Read First Line
	row, err := r.Read()

	i := 0

	var count int64

	// Get ride of the unused variable error when printf is commented!
	var _ = count

	count = 0
	var pipeCount float64 = 0
	var tt time.Time
//	var set string
	startTime := time.Now()
	fmt.Printf("%s - %s Procesing starts ", startTime.Format("2006/01/2 - 15:04:05"), inFile)

	for (err != io.EOF) && (len(row) >1) {

		// Add imsi value for that key (Time:Cell) value to Redis
		// Create a set per key with all the cell for that period

		//Manage Pipeline of command

		if pipeCount == 0 {
		// Optimize in the case of 5mins files take the time of the first line (rounded)
		// This will break if files contain more than 5 mins splits
			utime, _ := strconv.ParseInt(row[1], 10, 64)
			t := time.Unix(utime, 0).In(time.UTC)
			tt = t.Truncate(5 * time.Minute)
//			set = fmt.Sprintf("set:%d", tt.Unix())
			// Init pipeline
			c.Send("Multi")
		} // we have a least one command in pipe

		key := fmt.Sprintf("%d:%s", tt.Unix(), row[3])
		imsi := row[2]

		if math.Mod(pipeCount, 5000) == 0.0 && pipeCount != 0 {
				pipeCount = 0
				_, err := c.Do("EXEC")
				check (err)
				c.Send("Multi")
		}

		 // just add one more command
		c.Send ("PFADD", key, imsi)
//		c.Send ("SADD", set, row[3])
		pipeCount += 1

		// Next line
     	row, err = r.Read()
		i += 1
	}

	// Still stuff in pipe to finish?
	if pipeCount != 0 {
		_, err := c.Do("EXEC")
		check (err)
	}

	endTime := time.Now()
	duration := endTime.Sub(startTime)

 	if err != nil && err != io.EOF {
   		fmt.Println("Error:", err)
 	}
    fmt.Print(" lasted: ", duration)
	fmt.Printf(" for %d keys inserted\n", i)

	return nil
}

// Utility function to allocate pool

func newPool(server, password string) *redis.Pool {
    return &redis.Pool{
        MaxIdle: 3,
        IdleTimeout: 240 * time.Second,
        Dial: func () (redis.Conn, error) {
            c, err := redis.Dial("tcp", server)
            if err != nil {
                return nil, err
            }
			if password != "" {
            	if _, err := c.Do("AUTH", password); err != nil {
                	c.Close()
                	return nil, err
            	}
			}
            return c, err
        },
        TestOnBorrow: func(c redis.Conn, t time.Time) error {
            _, err := c.Do("PING")
            return err
        },
    }
}

// Global variables

var (
    pool *redis.Pool
    redisServer = "localhost:6379"
    redisPassword = ""
)

// Main function
func main() {

// Create Channel for inter routine communication
	channel := make (chan int64, 10)

// Open Connection to Redis Server
//	const proto = "tcp"
//	const port = ":6379"
//	c, err := redis.Dial(proto, port)

	pool = newPool(redisServer, redisPassword)

	if pool == nil {
			log.Fatal("Cannot Create Redis Pool")
	}

// Get Connection from the pool. Only way to be thread-safe with Redis
	c := pool.Get()
	if c == nil {
			log.Fatal("Cannot get connection from Redis Pool")
	}
	defer c.Close()

	// Create list of file to be processed
	eList := make(Events, 0, 100)

	// Launch 5 processing functions
	for i :=0; i < 5; i++ {
		go listenUp(channel, i)
	}

	// Recursive walk to get all the csv files
	filepath.Walk("/Volumes/BigBud/data/vf", func(aPath string, info os.FileInfo, err error) error {
		// fmt.Println(path)
		if (!info.IsDir() && strings.HasSuffix(info.Name(), ".csv")) {
			slice := strings.Split(path.Base(aPath), ".")
			sDate := slice[0] +":"+ slice[1]
			t, err := time.ParseInLocation("20060102:150405", sDate, time.UTC)
			if err != nil {
				fmt.Println(err)
				log.Fatal("Abort: Bad Filename with date: ", sDate)
			}
			e := Event{aPath, t}
			eList = append(eList, e)
		}
		return nil
	})

	// Order the events by date
	sort.Sort(eList)

	// Loop thru the files, ordered by time and file the unique counts
	// Breaks on 5 min boundaries to geterate cumulated result file and purge redis for those keys
	// Need to slow down ingestion process if too much data being inserted

	var count int = 0

	startTime := time.Now()
	fmt.Println("Let's get started: ", startTime)

	// Initialize first timestamp

	var pTime time.Time

	if eList.Len() > 0 {
		pTime = eList[0].ts
	}

	for _, v := range eList {

		if pTime != v.ts {
			fmt.Printf("Process time %s with key: %d\n", pTime, pTime.Unix())
			channel <- pTime.Unix()
			pTime = v.ts
		}

//		fmt.Printf ("Processing: %s - %s\n", v.ts, v.path)
		err := processFile(v.path)

		if err != nil {
			log.Fatal(err)
		}
		// For test only
		if count += 1; count > 10000 {
			break
		}
	}

// Close all and Cleanup
	close(channel)

	println("Total File List ", eList.Len())

	endTime := time.Now()
	duration := endTime.Sub(startTime)

    fmt.Println("Let's end: ", duration)

	os.Exit(0)
}<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	"github.com/CiscoCloud/mantl-api/zookeeper"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const Name = "mantl-api"
const Version = "0.1.8"

var wg sync.WaitGroup

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret", "", "Deprecated. Use mesos-secret-path instead")
	rootCmd.PersistentFlags().String("mesos-secret-path", "/etc/sysconfig/mantl-api", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")
	rootCmd.PersistentFlags().Int("consul-refresh-interval", 10, "The number of seconds after which to check consul for package requests")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			syncRepo(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonUrl = NewDiscovery(client, "marathon", "", "http", "http://localhost:8080").discoveredUrl
	}
	marathonClient, err := marathon.NewMarathon(
		marathonUrl,
		viper.GetString("marathon-user"),
		viper.GetString("marathon-password"),
		viper.GetBool("marathon-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create marathon client: %v", err)
	}

	mesosUrl := viper.GetString("mesos")
	if mesosUrl == "" {
		mesosUrl = NewDiscovery(client, "mesos", "leader", "http", "http://localhost:5050").discoveredUrl
	}
	mesosClient, err := mesos.NewMesos(
		mesosUrl,
		viper.GetString("mesos-principal"),
		viper.GetString("mesos-secret-path"),
		viper.GetBool("mesos-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create mesos client: %v", err)
	}

	zkUrls := viper.GetString("zookeeper")
	if zkUrls == "" {
		zkUrls = NewDiscovery(client, "zookeeper", "", "", "localhost:2181").discoveredUrl
	}
	zkServers := strings.Split(zkUrls, ",")
	zk := zookeeper.NewZookeeper(zkServers)

	inst, err := install.NewInstall(client, marathonClient, mesosClient, zk)
	if err != nil {
		log.Fatalf("Could not create install client: %v", err)
	}

	// sync sources to consul
	syncRepo(inst, viper.GetBool("force-sync"))

	wg.Add(1)
	go inst.Watch(time.Duration(viper.GetInt("consul-refresh-interval")))
	go api.NewApi(Name, viper.GetString("listen"), inst, mesosClient, wg).Start()
	wg.Wait()
}

func consulClient() *consul.Client {
	consulConfig := consul.DefaultConfig()
	scheme, address, _, err := http.ParseUrl(viper.GetString("consul"))
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}
	consulConfig.Scheme = scheme
	consulConfig.Address = address

	log.Debugf("Using Consul at %s over %s", consulConfig.Address, consulConfig.Scheme)

	if viper.GetBool("consul-no-verify-ssl") {
		transport := cleanhttp.DefaultTransport()
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
		consulConfig.HttpClient.Transport = transport
	}

	client, err := consul.NewClient(consulConfig)
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}

	// abort if we cannot connect to consul
	err = testConsul(client)
	if err != nil {
		log.Fatalf("Could not connect to consul: %v", err)
	}

	return client
}

func testConsul(client *consul.Client) error {
	kv := client.KV()
	_, _, err := kv.Get("mantl-install", nil)
	return err
}

func syncRepo(inst *install.Install, force bool) {
	var err error
	if inst == nil {
		client := consulClient()
		inst, err = install.NewInstall(client, nil, nil, nil)
		if err != nil {
			log.Fatalf("Could not create install client: %v", err)
		}
	}

	defaultSources := []*install.Source{
		&install.Source{
			Name:       "mantl",
			Path:       "https://github.com/CiscoCloud/mantl-universe.git",
			SourceType: install.Git,
			Branch:     "version-0.7",
			Index:      0,
		},
	}

	sources := []*install.Source{}

	configuredSources := viper.GetStringMap("sources")

	if len(configuredSources) > 0 {
		for name, val := range configuredSources {
			source := &install.Source{Name: name, SourceType: install.FileSystem}
			sourceConfig := val.(map[string]interface{})

			if path, ok := sourceConfig["path"].(string); ok {
				source.Path = path
			}

			if index, ok := sourceConfig["index"].(int64); ok {
				source.Index = int(index)
			}

			if sourceType, ok := sourceConfig["type"].(string); ok {
				if strings.EqualFold(sourceType, "git") {
					source.SourceType = install.Git
				}
			}

			if branch, ok := sourceConfig["branch"].(string); ok {
				source.Branch = branch
			}

			if source.IsValid() {
				sources = append(sources, source)
			} else {
				log.Warnf("Invalid source configuration for %s", name)
			}
		}
	}

	if len(sources) == 0 {
		sources = defaultSources
	}

	if err := inst.SyncSources(sources, force); err != nil {
		log.Fatal(err)
	}
}

func readConfigFile() {
	// read configuration file if specified
	configFile := viper.GetString("config-file")
	if configFile != "" {
		configFile = os.ExpandEnv(configFile)
		if _, err := os.Stat(configFile); err == nil {
			viper.SetConfigFile(configFile)
			err = viper.ReadInConfig()
			if err != nil {
				log.Warnf("Could not read configuration file: %v", err)
			}
		} else {
			log.Warnf("Could not find configuration file: %s", configFile)
		}
	}
}

func setupLogging() {
	switch viper.GetString("log-level") {
	case "debug":
		log.SetLevel(log.DebugLevel)
	case "info":
		log.SetLevel(log.InfoLevel)
	case "warn":
		log.SetLevel(log.WarnLevel)
	case "error":
		log.SetLevel(log.ErrorLevel)
	case "fatal":
		log.SetLevel(log.FatalLevel)
	default:
		log.WithField("log-level", viper.GetString("log-level")).Warning("invalid log level. defaulting to info.")
		log.SetLevel(log.InfoLevel)
	}

	switch viper.GetString("log-format") {
	case "text":
		log.SetFormatter(new(log.TextFormatter))
	case "json":
		log.SetFormatter(new(log.JSONFormatter))
	default:
		log.WithField("log-format", viper.GetString("log-format")).Warning("invalid log format. defaulting to text.")
		log.SetFormatter(new(log.TextFormatter))
	}
}
<commit_msg>version 0.1.9<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/CiscoCloud/mantl-api/api"
	"github.com/CiscoCloud/mantl-api/install"
	"github.com/CiscoCloud/mantl-api/marathon"
	"github.com/CiscoCloud/mantl-api/mesos"
	"github.com/CiscoCloud/mantl-api/utils/http"
	"github.com/CiscoCloud/mantl-api/zookeeper"
	log "github.com/Sirupsen/logrus"
	consul "github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-cleanhttp"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
)

const Name = "mantl-api"
const Version = "0.1.9"

var wg sync.WaitGroup

func main() {
	rootCmd := &cobra.Command{
		Use:   "mantl-api",
		Short: "runs the mantl-api",
		Run: func(cmd *cobra.Command, args []string) {
			start()
		},
		PersistentPreRun: func(cmd *cobra.Command, args []string) {
			readConfigFile()
			setupLogging()
		},
	}

	rootCmd.PersistentFlags().String("log-level", "info", "one of debug, info, warn, error, or fatal")
	rootCmd.PersistentFlags().String("log-format", "text", "specify output (text or json)")
	rootCmd.PersistentFlags().String("consul", "http://localhost:8500", "Consul Api address")
	rootCmd.PersistentFlags().Bool("consul-no-verify-ssl", false, "Consul SSL verification")
	rootCmd.PersistentFlags().String("marathon", "", "Marathon Api address")
	rootCmd.PersistentFlags().String("marathon-user", "", "Marathon Api user")
	rootCmd.PersistentFlags().String("marathon-password", "", "Marathon Api password")
	rootCmd.PersistentFlags().Bool("marathon-no-verify-ssl", false, "Marathon SSL verification")
	rootCmd.PersistentFlags().String("mesos", "", "Mesos Api address")
	rootCmd.PersistentFlags().String("mesos-principal", "", "Mesos principal for framework authentication")
	rootCmd.PersistentFlags().String("mesos-secret", "", "Deprecated. Use mesos-secret-path instead")
	rootCmd.PersistentFlags().String("mesos-secret-path", "/etc/sysconfig/mantl-api", "Path to a file on host sytem that contains the mesos secret for framework authentication")
	rootCmd.PersistentFlags().Bool("mesos-no-verify-ssl", false, "Mesos SSL verification")
	rootCmd.PersistentFlags().String("listen", ":4001", "mantl-api listen address")
	rootCmd.PersistentFlags().String("zookeeper", "", "Comma-delimited list of zookeeper servers")
	rootCmd.PersistentFlags().Bool("force-sync", false, "Force a synchronization of all sources")
	rootCmd.PersistentFlags().String("config-file", "", "The path to a configuration file")
	rootCmd.PersistentFlags().Int("consul-refresh-interval", 10, "The number of seconds after which to check consul for package requests")

	for _, flags := range []*pflag.FlagSet{rootCmd.PersistentFlags()} {
		err := viper.BindPFlags(flags)
		if err != nil {
			log.WithField("error", err).Fatal("could not bind flags")
		}
	}

	viper.SetEnvPrefix("mantl_api")
	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
	viper.AutomaticEnv()

	syncCommand := &cobra.Command{
		Use:   "sync",
		Short: "Synchronize universe repositories",
		Long:  "Forces a synchronization of all configured sources",
		Run: func(cmd *cobra.Command, args []string) {
			syncRepo(nil, true)
		},
	}
	rootCmd.AddCommand(syncCommand)

	versionCommand := &cobra.Command{
		Use:   "version",
		Short: fmt.Sprintf("Print the version number of %s", Name),
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Printf("%s v%s\n", Name, Version)
		},
	}
	rootCmd.AddCommand(versionCommand)

	rootCmd.Execute()
}

func start() {
	log.Infof("Starting %s v%s", Name, Version)
	client := consulClient()

	marathonUrl := viper.GetString("marathon")
	if marathonUrl == "" {
		marathonUrl = NewDiscovery(client, "marathon", "", "http", "http://localhost:8080").discoveredUrl
	}
	marathonClient, err := marathon.NewMarathon(
		marathonUrl,
		viper.GetString("marathon-user"),
		viper.GetString("marathon-password"),
		viper.GetBool("marathon-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create marathon client: %v", err)
	}

	mesosUrl := viper.GetString("mesos")
	if mesosUrl == "" {
		mesosUrl = NewDiscovery(client, "mesos", "leader", "http", "http://localhost:5050").discoveredUrl
	}
	mesosClient, err := mesos.NewMesos(
		mesosUrl,
		viper.GetString("mesos-principal"),
		viper.GetString("mesos-secret-path"),
		viper.GetBool("mesos-no-verify-ssl"),
	)
	if err != nil {
		log.Fatalf("Could not create mesos client: %v", err)
	}

	zkUrls := viper.GetString("zookeeper")
	if zkUrls == "" {
		zkUrls = NewDiscovery(client, "zookeeper", "", "", "localhost:2181").discoveredUrl
	}
	zkServers := strings.Split(zkUrls, ",")
	zk := zookeeper.NewZookeeper(zkServers)

	inst, err := install.NewInstall(client, marathonClient, mesosClient, zk)
	if err != nil {
		log.Fatalf("Could not create install client: %v", err)
	}

	// sync sources to consul
	syncRepo(inst, viper.GetBool("force-sync"))

	wg.Add(1)
	go inst.Watch(time.Duration(viper.GetInt("consul-refresh-interval")))
	go api.NewApi(Name, viper.GetString("listen"), inst, mesosClient, wg).Start()
	wg.Wait()
}

func consulClient() *consul.Client {
	consulConfig := consul.DefaultConfig()
	scheme, address, _, err := http.ParseUrl(viper.GetString("consul"))
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}
	consulConfig.Scheme = scheme
	consulConfig.Address = address

	log.Debugf("Using Consul at %s over %s", consulConfig.Address, consulConfig.Scheme)

	if viper.GetBool("consul-no-verify-ssl") {
		transport := cleanhttp.DefaultTransport()
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
		consulConfig.HttpClient.Transport = transport
	}

	client, err := consul.NewClient(consulConfig)
	if err != nil {
		log.Fatalf("Could not create consul client: %v", err)
	}

	// abort if we cannot connect to consul
	err = testConsul(client)
	if err != nil {
		log.Fatalf("Could not connect to consul: %v", err)
	}

	return client
}

func testConsul(client *consul.Client) error {
	kv := client.KV()
	_, _, err := kv.Get("mantl-install", nil)
	return err
}

func syncRepo(inst *install.Install, force bool) {
	var err error
	if inst == nil {
		client := consulClient()
		inst, err = install.NewInstall(client, nil, nil, nil)
		if err != nil {
			log.Fatalf("Could not create install client: %v", err)
		}
	}

	defaultSources := []*install.Source{
		&install.Source{
			Name:       "mantl",
			Path:       "https://github.com/CiscoCloud/mantl-universe.git",
			SourceType: install.Git,
			Branch:     "version-0.7",
			Index:      0,
		},
	}

	sources := []*install.Source{}

	configuredSources := viper.GetStringMap("sources")

	if len(configuredSources) > 0 {
		for name, val := range configuredSources {
			source := &install.Source{Name: name, SourceType: install.FileSystem}
			sourceConfig := val.(map[string]interface{})

			if path, ok := sourceConfig["path"].(string); ok {
				source.Path = path
			}

			if index, ok := sourceConfig["index"].(int64); ok {
				source.Index = int(index)
			}

			if sourceType, ok := sourceConfig["type"].(string); ok {
				if strings.EqualFold(sourceType, "git") {
					source.SourceType = install.Git
				}
			}

			if branch, ok := sourceConfig["branch"].(string); ok {
				source.Branch = branch
			}

			if source.IsValid() {
				sources = append(sources, source)
			} else {
				log.Warnf("Invalid source configuration for %s", name)
			}
		}
	}

	if len(sources) == 0 {
		sources = defaultSources
	}

	if err := inst.SyncSources(sources, force); err != nil {
		log.Fatal(err)
	}
}

func readConfigFile() {
	// read configuration file if specified
	configFile := viper.GetString("config-file")
	if configFile != "" {
		configFile = os.ExpandEnv(configFile)
		if _, err := os.Stat(configFile); err == nil {
			viper.SetConfigFile(configFile)
			err = viper.ReadInConfig()
			if err != nil {
				log.Warnf("Could not read configuration file: %v", err)
			}
		} else {
			log.Warnf("Could not find configuration file: %s", configFile)
		}
	}
}

func setupLogging() {
	switch viper.GetString("log-level") {
	case "debug":
		log.SetLevel(log.DebugLevel)
	case "info":
		log.SetLevel(log.InfoLevel)
	case "warn":
		log.SetLevel(log.WarnLevel)
	case "error":
		log.SetLevel(log.ErrorLevel)
	case "fatal":
		log.SetLevel(log.FatalLevel)
	default:
		log.WithField("log-level", viper.GetString("log-level")).Warning("invalid log level. defaulting to info.")
		log.SetLevel(log.InfoLevel)
	}

	switch viper.GetString("log-format") {
	case "text":
		log.SetFormatter(new(log.TextFormatter))
	case "json":
		log.SetFormatter(new(log.JSONFormatter))
	default:
		log.WithField("log-format", viper.GetString("log-format")).Warning("invalid log format. defaulting to text.")
		log.SetFormatter(new(log.TextFormatter))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"github.com/boltdb/bolt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
)

var db *bolt.DB
var primaryBucketName []byte
var markov *Chain
var domain string

func init() {
	markov = buildMarkov()
	primaryBucketName = []byte("Links")
	domain = os.Getenv("DOMAIN")
}

type shortenerResponse struct {
	Key         string `json:",omitempty"`
	Destination string `json:",omitempty"`
	Error       string `json:",omitempty"`
}

func returnJSONFromStruct(s shortenerResponse) []byte {
	o, err := json.Marshal(s)
	if err != nil {
		log.Printf("error occurred marshalling notFoundError response: %v\n", err)
	}
	return o
}

func createBucket(name []byte) error {
	// Start a writable transaction.
	tx, err := db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()

	// Use the transaction...
	_, err = tx.CreateBucket(name)
	if err != nil {
		log.Println("Bucket already exists!")
	}

	// Commit the transaction and check for error.
	if err = tx.Commit(); err != nil {
		return err
	}
	return nil
}

func readKeyFromBucket(bucketName []byte, key []byte) []byte {
	var r []byte
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		r = b.Get(key)
		return nil
	})
	return r
}

func addKeyValueToBucket(bucketName []byte, key []byte, value []byte) {
	db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		err := b.Put(key, value)
		if err != nil {
			log.Printf("there was an error:\n\t%v", err)
		}
		return err
	})
}

// prints a list of all keys. Not used in any routes, but handy to keep around, should the need arise.
func keys() {
	db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket(primaryBucketName)
		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
			log.Printf("key=%s, value=%s\n", k, v)
		}

		return nil
	})
}

func apiAddValue(w http.ResponseWriter, r *http.Request) {
	url := r.URL.Query().Get("url")

	if strings.Contains(url, domain) {
		// If the url provided contains the domain name we're running, we respond with an error
		respondWithError(w, r, "Invalid URL Provided! You musn't shorten links to the service itself!")
		return
	}

	if len(url) == 0 {
		// If no url query param is provided, we respond with an error
		respondWithError(w, r, "Empty URL Provided!")
		return
	}

	var result shortenerResponse
	var v []byte
	m := []byte(" ")

	for len(m) != 0 {
		v = generateMarkovString(markov)
		m = readKeyFromBucket(primaryBucketName, v)
	}

	result.Key = string(v)
	o, err := json.Marshal(result)
	if err != nil {
		log.Printf("error encountered marshalling json: %v\n", err)
	}

	addKeyValueToBucket(primaryBucketName, v, []byte(url))
	w.Write(o)
}

func apiGetValue(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	k := []byte(vars["key"])
	if len(k) != 0 {
		rv := readKeyFromBucket(primaryBucketName, k)
		http.Redirect(w, r, string(rv), 301)
	} else {
		notFoundError(w, r)
	}
}

func respondWithError(w http.ResponseWriter, r *http.Request, e string) {
	w.Write(returnJSONFromStruct(
		shortenerResponse{
			Error: e,
		}))
}

func homepage(w http.ResponseWriter, r *http.Request) {
	indexPage, err := ioutil.ReadFile("index.html")
	if err != nil {
		log.Printf("error occurred reading indexPage: %v\n", err)
	}
	w.Write(indexPage)
}

func notFoundError(w http.ResponseWriter, r *http.Request) {
	response := shortenerResponse{
		Error: "Sorry, 404! :(",
	}
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(returnJSONFromStruct(response))
}

func main() {
	var err error
	// Open the db data file in your current directory.
	// It will be created if it doesn't exist.
	db, err = bolt.Open("whatever.db", 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	// ensure our default bucket exists
	err = createBucket(primaryBucketName)
	if err != nil {
		log.Fatal(err)
	}

	router := mux.NewRouter()
	router.HandleFunc("/", homepage)
	router.HandleFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {})
	// I know I need to actually serve the above files, but
	// I don't feel like it at the moment, sue me if you must.

	router.HandleFunc("/api/add", apiAddValue)
	router.HandleFunc("/{key}", apiGetValue)
	router.NotFoundHandler = http.HandlerFunc(notFoundError)

	http.Handle("/", router)
	http.ListenAndServe(":80", nil)
}
<commit_msg>Address issue where keys with encoded characters return no matches<commit_after>package main

import (
	"encoding/json"
	"github.com/boltdb/bolt"
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

var db *bolt.DB
var primaryBucketName []byte
var markov *Chain
var domain string

func init() {
	markov = buildMarkov()
	primaryBucketName = []byte("Links")
	domain = os.Getenv("DOMAIN")
}

type shortenerResponse struct {
	Key         string `json:",omitempty"`
	Destination string `json:",omitempty"`
	Error       string `json:",omitempty"`
}

func returnJSONFromStruct(s shortenerResponse) []byte {
	o, err := json.Marshal(s)
	if err != nil {
		log.Printf("error occurred marshalling notFoundError response: %v\n", err)
	}
	return o
}

func createBucket(name []byte) error {
	// Start a writable transaction.
	tx, err := db.Begin(true)
	if err != nil {
		return err
	}
	defer tx.Rollback()

	// Use the transaction...
	_, err = tx.CreateBucket(name)
	if err != nil {
		log.Println("Bucket already exists!")
	}

	// Commit the transaction and check for error.
	if err = tx.Commit(); err != nil {
		return err
	}
	return nil
}

func readKeyFromBucket(bucketName []byte, key []byte) []byte {
	var r []byte
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		r = b.Get(key)
		return nil
	})
	return r
}

func addKeyValueToBucket(bucketName []byte, key []byte, value []byte) {
	db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		err := b.Put(key, value)
		if err != nil {
			log.Printf("there was an error:\n\t%v", err)
		}
		return err
	})
}

// prints a list of all keys. Not used in any routes, but handy to keep around, should the need arise.
func keys() {
	db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket(primaryBucketName)
		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
			log.Printf("key=%s, value=%s\n", k, v)
		}

		return nil
	})
}

func apiAddValue(w http.ResponseWriter, r *http.Request) {
	url := r.URL.Query().Get("url")

	if strings.Contains(url, domain) {
		// If the url provided contains the domain name we're running, we respond with an error
		respondWithError(w, r, "Invalid URL Provided! You musn't shorten links to the service itself!")
		return
	}

	if len(url) == 0 {
		// If no url query param is provided, we respond with an error
		respondWithError(w, r, "Empty URL Provided!")
		return
	}

	var result shortenerResponse
	var v []byte
	m := []byte(" ")

	for len(m) != 0 {
		v = generateMarkovString(markov)
		m = readKeyFromBucket(primaryBucketName, v)
	}

	result.Key = string(v)
	o, err := json.Marshal(result)
	if err != nil {
		log.Printf("error encountered marshalling json: %v\n", err)
	}

	addKeyValueToBucket(primaryBucketName, v, []byte(url))
	w.Write(o)
}

func apiGetValue(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	k := url.QueryEscape(vars["key"])
	if len(k) != 0 {
		rv := readKeyFromBucket(primaryBucketName, []byte(k))
		http.Redirect(w, r, string(rv), 301)
	} else {
		notFoundError(w, r)
	}
}

func respondWithError(w http.ResponseWriter, r *http.Request, e string) {
	w.Write(returnJSONFromStruct(
		shortenerResponse{
			Error: e,
		}))
}

func homepage(w http.ResponseWriter, r *http.Request) {
	indexPage, err := ioutil.ReadFile("index.html")
	if err != nil {
		log.Printf("error occurred reading indexPage: %v\n", err)
	}
	w.Write(indexPage)
}

func notFoundError(w http.ResponseWriter, r *http.Request) {
	response := shortenerResponse{
		Error: "Sorry, 404! :(",
	}
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(returnJSONFromStruct(response))
}

func main() {
	var err error
	// Open the db data file in your current directory.
	// It will be created if it doesn't exist.
	db, err = bolt.Open("whatever.db", 0600, nil)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	// ensure our default bucket exists
	err = createBucket(primaryBucketName)
	if err != nil {
		log.Fatal(err)
	}

	router := mux.NewRouter()
	router.HandleFunc("/", homepage)
	router.HandleFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {})
	// I know I need to actually serve the above files, but
	// I don't feel like it at the moment, sue me if you must.

	router.HandleFunc("/api/add", apiAddValue)
	router.HandleFunc("/{key}", apiGetValue)
	router.NotFoundHandler = http.HandlerFunc(notFoundError)

	http.Handle("/", router)
	http.ListenAndServe(":80", nil)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"time"

	"golang.org/x/mobile/app"
	"golang.org/x/mobile/audio"
	"golang.org/x/mobile/f32"
	"golang.org/x/mobile/gl"
	"golang.org/x/mobile/gl/glutil"
)

const (
	numBeats  = 16
	numTracks = 8
)

var (
	program  gl.Program
	position gl.Attrib
	offset   gl.Uniform
	color    gl.Uniform
	buf      gl.Buffer

	index    int
	green    float32
	greenDec bool

	stopped bool
)

var (
	hits    [numBeats][numTracks]bool
	samples [numTracks]io.Closer
	players [numTracks]*audio.Player
)

func main() {
	app.Run(app.Callbacks{
		Start: start,
		Stop:  stop,
		Draw:  draw,
	})
}

func start() {
	var err error
	program, err = glutil.CreateProgram(vertexShader, fragmentShader)
	if err != nil {
		log.Printf("error creating GL program: %v", err)
		return
	}

	buf = gl.CreateBuffer()
	gl.BindBuffer(gl.ARRAY_BUFFER, buf)
	gl.BufferData(gl.ARRAY_BUFFER, rectData, gl.STATIC_DRAW)

	position = gl.GetAttribLocation(program, "position")
	color = gl.GetUniformLocation(program, "color")
	offset = gl.GetUniformLocation(program, "offset")

	for i := 0; i < numTracks; i++ {
		rc, err := app.Open(fmt.Sprintf("track%d.wav", i))
		if err != nil {
			log.Fatal(err)
		}
		samples[i] = rc
		p, err := audio.NewPlayer(rc, audio.Stereo16, 44100)
		if err != nil {
			log.Fatal(err)
		}
		players[i] = p
	}

	// hi hat
	hits[0][1] = true
	hits[2][1] = true
	hits[4][1] = true
	hits[6][1] = true
	hits[8][1] = true
	hits[10][1] = true
	hits[12][1] = true
	hits[14][1] = true

	// kick
	hits[5][2] = true
	hits[7][2] = true
	hits[11][2] = true
	hits[13][2] = true
	hits[14][2] = true
	hits[15][2] = true

	// bass
	hits[0][4] = true
	hits[3][4] = true
	hits[5][4] = true
	hits[6][4] = true
	hits[8][4] = true
	hits[11][4] = true
	hits[13][4] = true

	// bass2
	hits[2][6] = true
	hits[10][6] = true

	go func() {
		for {
			if stopped {
				stopped = false
				return
			}
			index = (index + 1) % numBeats
			for t := 0; t < numTracks; t++ {
				go func(t int) {
					if hits[index][t] {
						players[t].Play()
					}
				}(t)
			}
			// bpm=140
			time.Sleep(time.Minute / 140)
		}
	}()
}

func stop() {
	for _, p := range players {
		p.Destroy()
	}
	for _, s := range samples {
		s.Close()
	}
	gl.DeleteProgram(program)
	gl.DeleteBuffer(buf)
	stopped = true
}

var rectData = f32.Bytes(binary.LittleEndian,
	0, 0,
	0, 0.1,
	0.1, 0,
	0.1, 0.1,
)

func draw() {
	gl.ClearColor(0, 0, 0, 1)
	gl.Clear(gl.COLOR_BUFFER_BIT)
	gl.UseProgram(program)

	if greenDec {
		green -= 0.01
	} else {
		green += 0.01
	}
	if green <= 0.2 {
		greenDec = false
	}
	if green >= 0.5 {
		greenDec = true
	}

	for i := 0; i < numBeats; i++ {
		for j := 0; j < numTracks; j++ {
			var c float32
			switch {
			case hits[i][j]:
				c = 1
			case i == index:
				c = green
			default:
				c = 0
			}
			drawButton(c, float32(i)*1/numBeats, float32(j)*1/numTracks)
		}
	}
}

func drawButton(g, x, y float32) {
	gl.Uniform4f(color, 0.1, g, 0.4, 1) // color
	gl.Uniform2f(offset, x, y)          // position

	gl.BindBuffer(gl.ARRAY_BUFFER, buf)
	gl.EnableVertexAttribArray(position)
	gl.VertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0)
	gl.DrawArrays(gl.TRIANGLE_STRIP, 0, 4)
	gl.DisableVertexAttribArray(position)
}

const vertexShader = `#version 100
uniform vec2 offset;
attribute vec4 position;
void main() {
  // offset comes in with x/y values between 0 and 1.
  // position bounds are -1 to 1.
  vec4 offset4 = vec4(2.0*offset.x-1.0, 1.0-2.0*offset.y, 0, 0);
  gl_Position = position + offset4;
}`

const fragmentShader = `#version 100
precision mediump float;
uniform vec4 color;
void main() {
  gl_FragColor = color;
}`
<commit_msg>add todo to handle touch events.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"time"

	"golang.org/x/mobile/app"
	"golang.org/x/mobile/audio"
	"golang.org/x/mobile/f32"
	"golang.org/x/mobile/gl"
	"golang.org/x/mobile/gl/glutil"
)

const (
	numBeats  = 16
	numTracks = 8
)

var (
	program  gl.Program
	position gl.Attrib
	offset   gl.Uniform
	color    gl.Uniform
	buf      gl.Buffer

	index    int
	green    float32
	greenDec bool

	stopped bool
)

var (
	hits    [numBeats][numTracks]bool
	samples [numTracks]io.Closer
	players [numTracks]*audio.Player
)

func main() {
	// TODO(jbd): Handle touch to turn on/off the beats.
	app.Run(app.Callbacks{
		Start: start,
		Stop:  stop,
		Draw:  draw,
	})
}

func start() {
	var err error
	program, err = glutil.CreateProgram(vertexShader, fragmentShader)
	if err != nil {
		log.Printf("error creating GL program: %v", err)
		return
	}

	buf = gl.CreateBuffer()
	gl.BindBuffer(gl.ARRAY_BUFFER, buf)
	gl.BufferData(gl.ARRAY_BUFFER, rectData, gl.STATIC_DRAW)

	position = gl.GetAttribLocation(program, "position")
	color = gl.GetUniformLocation(program, "color")
	offset = gl.GetUniformLocation(program, "offset")

	for i := 0; i < numTracks; i++ {
		rc, err := app.Open(fmt.Sprintf("track%d.wav", i))
		if err != nil {
			log.Fatal(err)
		}
		samples[i] = rc
		p, err := audio.NewPlayer(rc, audio.Stereo16, 44100)
		if err != nil {
			log.Fatal(err)
		}
		players[i] = p
	}

	// hi hat
	hits[0][1] = true
	hits[2][1] = true
	hits[4][1] = true
	hits[6][1] = true
	hits[8][1] = true
	hits[10][1] = true
	hits[12][1] = true
	hits[14][1] = true

	// kick
	hits[5][2] = true
	hits[7][2] = true
	hits[11][2] = true
	hits[13][2] = true
	hits[14][2] = true
	hits[15][2] = true

	// bass
	hits[0][4] = true
	hits[3][4] = true
	hits[5][4] = true
	hits[6][4] = true
	hits[8][4] = true
	hits[11][4] = true
	hits[13][4] = true

	// bass2
	hits[2][6] = true
	hits[10][6] = true

	go func() {
		for {
			if stopped {
				stopped = false
				return
			}
			index = (index + 1) % numBeats
			for t := 0; t < numTracks; t++ {
				go func(t int) {
					if hits[index][t] {
						players[t].Play()
					}
				}(t)
			}
			// bpm=140
			time.Sleep(time.Minute / 140)
		}
	}()
}

func stop() {
	for _, p := range players {
		p.Destroy()
	}
	for _, s := range samples {
		s.Close()
	}
	gl.DeleteProgram(program)
	gl.DeleteBuffer(buf)
	stopped = true
}

var rectData = f32.Bytes(binary.LittleEndian,
	0, 0,
	0, 0.1,
	0.1, 0,
	0.1, 0.1,
)

func draw() {
	gl.ClearColor(0, 0, 0, 1)
	gl.Clear(gl.COLOR_BUFFER_BIT)
	gl.UseProgram(program)

	if greenDec {
		green -= 0.01
	} else {
		green += 0.01
	}
	if green <= 0.2 {
		greenDec = false
	}
	if green >= 0.5 {
		greenDec = true
	}

	for i := 0; i < numBeats; i++ {
		for j := 0; j < numTracks; j++ {
			var c float32
			switch {
			case hits[i][j]:
				c = 1
			case i == index:
				c = green
			default:
				c = 0
			}
			drawButton(c, float32(i)*1/numBeats, float32(j)*1/numTracks)
		}
	}
}

func drawButton(g, x, y float32) {
	gl.Uniform4f(color, 0.1, g, 0.4, 1) // color
	gl.Uniform2f(offset, x, y)          // position

	gl.BindBuffer(gl.ARRAY_BUFFER, buf)
	gl.EnableVertexAttribArray(position)
	gl.VertexAttribPointer(position, 2, gl.FLOAT, false, 0, 0)
	gl.DrawArrays(gl.TRIANGLE_STRIP, 0, 4)
	gl.DisableVertexAttribArray(position)
}

const vertexShader = `#version 100
uniform vec2 offset;
attribute vec4 position;
void main() {
  // offset comes in with x/y values between 0 and 1.
  // position bounds are -1 to 1.
  vec4 offset4 = vec4(2.0*offset.x-1.0, 1.0-2.0*offset.y, 0, 0);
  gl_Position = position + offset4;
}`

const fragmentShader = `#version 100
precision mediump float;
uniform vec4 color;
void main() {
  gl_FragColor = color;
}`
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"

	"github.com/flosch/pongo2"
)

type Page struct {
	Title              string
	CurrentTemperature float64
	TargetTemperature  float64
	IsHeating          string
}

func root(c web.C, w http.ResponseWriter, r *http.Request) {

	var page Page
	page.Title = "My Cooker"

	page.TargetTemperature = getTargetTemp()
	page.CurrentTemperature = getCurrentTemp()
	page.IsHeating = isHeating()

	tpl, err := pongo2.DefaultSet.FromFile("template.tpl")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	tpl.ExecuteWriter(pongo2.Context{"page": page}, w)
}

func setTarget(c web.C, w http.ResponseWriter, r *http.Request) {
	target := c.URLParams["target"]
	fmt.Println(target)

	t, err := strconv.ParseFloat(target, 32)
	if err != nil || t < 0 || t > 100 {
		// Invalid format or out of correct range
		http.Redirect(w, r, "/", 400)
		return
	}

	fmt.Println("Set " + target)
	setTargetTemp(target)
	http.Redirect(w, r, "/", 200)
}

func main() {
	goji.Get("/", root)
	goji.Get("/set/:target", setTarget)
	goji.Serve()
}
<commit_msg>avoid to show transition page<commit_after>package main

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web"

	"github.com/flosch/pongo2"
)

type Page struct {
	Title              string
	CurrentTemperature float64
	TargetTemperature  float64
	IsHeating          string
}

func root(c web.C, w http.ResponseWriter, r *http.Request) {

	var page Page
	page.Title = "My Cooker"

	page.TargetTemperature = getTargetTemp()
	page.CurrentTemperature = getCurrentTemp()
	page.IsHeating = isHeating()

	tpl, err := pongo2.DefaultSet.FromFile("template.tpl")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	tpl.ExecuteWriter(pongo2.Context{"page": page}, w)
}

func setTarget(c web.C, w http.ResponseWriter, r *http.Request) {
	target := c.URLParams["target"]
	fmt.Println(target)

	t, err := strconv.ParseFloat(target, 32)
	if err != nil || t < 0 || t > 100 {
		// Invalid format or out of correct range
		http.Redirect(w, r, "/", 400)
		return
	}

	fmt.Println("Set " + target)
	setTargetTemp(target)
	http.Redirect(w, r, "/", http.StatusMovedPermanently)	// workaround to suppress transition page.
}

func forceOff(c web.C, w http.ResponseWriter, r *http.Request) {
	setTargetTemp("0.0")
	http.Redirect(w, r, "/", http.StatusMovedPermanently)
}

func forceOn(c web.C, w http.ResponseWriter, r *http.Request) {
	setTargetTemp("100.0")
	http.Redirect(w, r, "/", http.StatusMovedPermanently)
}

func main() {
	goji.Get("/", root)
	goji.Get("/set/:target", setTarget)
	goji.Get("/force_on", forceOn)
	goji.Get("/force_off", forceOff)
	goji.Serve()
}
<|endoftext|>


<commit_before>package main

import (
	whisper "github.com/grobian/go-whisper"
	pickle "github.com/kisielk/og-rek"

	"encoding/json"
	"flag"
	"fmt"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

var config = struct {
	WhisperData string
}{
	WhisperData: "/var/lib/carbon/whisper",
}

type WhisperFetchResponse struct {
	Name      string    `json:"name"`
	StartTime int       `json:"startTime"`
	StopTime  int       `json:"stopTime"`
	StepTime  int       `json:"stepTime"`
	Values    []float64 `json:"values"`
	IsAbsent  []bool    `json:"isAbsent"`
}

type WhisperGlobResponse struct {
	Name  string   `json:"name"`
	Paths []string `json:"paths"`
}

var log Logger

func findHandler(wr http.ResponseWriter, req *http.Request) {
	//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
	//	http://localhost:8080/metrics/find/?query=test
	req.ParseForm()
	glob := req.FormValue("query")
	format := req.FormValue("format")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
			format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
			http.StatusBadRequest)
		return
	}

	if glob == "" {
		log.Warn("dropping invalid request (query=): %s", req.URL.RequestURI())
		http.Error(wr, "Bad request (no query)", http.StatusBadRequest)
		return
	}

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */
	lbrace := strings.Index(glob, "{")
	rbrace := -1
	if lbrace > -1 {
		rbrace = strings.Index(glob[lbrace:], "}")
		if rbrace > -1 {
			rbrace += lbrace
		}
	}
	files := make([]string, 0)
	if lbrace > -1 && rbrace > -1 {
		expansion := glob[lbrace+1 : rbrace]
		parts := strings.Split(expansion, ",")
		for _, sub := range parts {
			sglob := glob[:lbrace] + sub + glob[rbrace+1:]
			path := config.WhisperData + "/" + strings.Replace(sglob, ".", "/", -1) + "*"
			nfiles, err := filepath.Glob(path)
			if err == nil {
				files = append(files, nfiles...)
			}
		}
	} else {
		path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
		nfiles, err := filepath.Glob(path)
		if err == nil {
			files = append(files, nfiles...)
		}
	}

	leafs := make([]bool, len(files))
	for i, p := range files {
		p = p[len(config.WhisperData+"/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p)-4]
			leafs[i] = true
		} else {
			leafs[i] = false
		}
		files[i] = strings.Replace(p, "/", ".", -1)
	}

	if format == "json" {
		response := WhisperGlobResponse{
			Name:  glob,
			Paths: make([]string, 0),
		}
		for _, p := range files {
			response.Paths = append(response.Paths, p)
		}
		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for glob %s: %s", glob, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}

		for i, p := range files {
			m = make(map[string]interface{})
			m["metric_path"] = p
			// m["intervals"] = dunno how to do a tuple here
			m["isLeaf"] = leafs[i]
			metrics = append(metrics, m)
		}

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}
	log.Info("find: %d hits for %s", len(files), glob)
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
	//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
	//	http://localhost:8080/render/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
			format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
			http.StatusBadRequest)
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		// the FE/carbonzipper often requests metrics we don't have
		log.Debug("failed to %s", err)
		http.Error(wr, "Metric not found", http.StatusNotFound)
		return
	}

	i, err := strconv.Atoi(from)
	if err != nil {
		log.Debug("fromTime (%s) invalid: %s (in %s)",
			from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	fromTime := int(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		log.Debug("untilTime (%s) invalid: %s (in %s)",
			from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	untilTime := int(i)

	if w != nil {
		defer w.Close()
	} else {
		http.Error(wr, "Bad request (invalid from/until time)",
			http.StatusBadRequest)
		return
	}

	points, err := w.Fetch(fromTime, untilTime)
	if err != nil {
		log.Error("failed to fetch points from %s: %s", path, err)
		http.Error(wr, "Fetching data points failed",
			http.StatusInternalServerError)
		return
	}
	values := points.Values()

	if format == "json" {
		response := WhisperFetchResponse{
			Name:      metric,
			StartTime: points.FromTime(),
			StopTime:  points.UntilTime(),
			StepTime:  points.Step(),
			Values:    make([]float64, len(values)),
			IsAbsent:  make([]bool, len(values)),
		}

		for i, p := range values {
			if math.IsNaN(p) {
				response.Values[i] = 0
				response.IsAbsent[i] = true
			} else {
				response.Values[i] = p
				response.IsAbsent[i] = false
			}
		}

		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for %s: %s", path, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		//[{'start': 1396271100, 'step': 60, 'name': 'metric',
		//'values': [9.0, 19.0, None], 'end': 1396273140}
		var metrics []map[string]interface{}
		var m map[string]interface{}

		m = make(map[string]interface{})
		m["start"] = points.FromTime()
		m["step"] = points.Step()
		m["end"] = points.UntilTime()
		m["name"] = metric

		mv := make([]interface{}, len(values))
		for i, p := range values {
			if math.IsNaN(p) {
				mv[i] = nil
			} else {
				mv[i] = p
			}
		}

		m["values"] = mv
		metrics = append(metrics, m)

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}

	log.Info("served %d points for %s", len(values), metric)
	return
}

func main() {
	port := flag.Int("p", 8080, "port to bind to")
	verbose := flag.Bool("v", false, "enable verbose logging")
	debug := flag.Bool("vv", false, "enable more verbose (debug) logging")
	whisperdata := flag.String("w", config.WhisperData, "location where whisper files are stored")

	flag.Parse()

	loglevel := WARN
	if *verbose {
		loglevel = INFO
	}
	if *debug {
		loglevel = DEBUG
	}
	log = NewOutputLogger(loglevel)

	config.WhisperData = *whisperdata
	log.Info("reading whisper files from: %s", config.WhisperData)

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", fetchHandler)

	listen := fmt.Sprintf(":%d", *port)
	log.Info("listening on %s", listen)
	err := http.ListenAndServe(listen, nil)
	if err != nil {
		log.Fatal("%s", err)
	}
	log.Info("stopped")
}

// Simple wrapper to enable/disable lots of spam
type Logger interface {
	Info(format string, a ...interface{})
	Warn(format string, a ...interface{})
	Error(format string, a ...interface{})
	Fatal(format string, a ...interface{})
	Debug(format string, a ...interface{})
}

type LogLevel int

const (
	FATAL LogLevel = 0
	ERROR LogLevel = 1
	WARN  LogLevel = 2
	INFO  LogLevel = 3
	DEBUG LogLevel = 4
)

type outputLogger struct {
	level LogLevel
	out   *os.File
	err   *os.File
}

func NewOutputLogger(level LogLevel) *outputLogger {
	r := new(outputLogger)
	r.level = level
	r.out = os.Stdout
	r.err = os.Stderr

	return r
}

func (l *outputLogger) Debug(format string, a ...interface{}) {
	if l.level >= DEBUG {
		l.out.WriteString(fmt.Sprintf("DEBUG: "+format+"\n", a...))
	}
}

func (l *outputLogger) Info(format string, a ...interface{}) {
	if l.level >= INFO {
		l.out.WriteString(fmt.Sprintf("INFO: "+format+"\n", a...))
	}
}

func (l *outputLogger) Warn(format string, a ...interface{}) {
	if l.level >= WARN {
		l.out.WriteString(fmt.Sprintf("WARN: "+format+"\n", a...))
	}
}

func (l *outputLogger) Error(format string, a ...interface{}) {
	if l.level >= ERROR {
		l.err.WriteString(fmt.Sprintf("ERROR: "+format+"\n", a...))
	}
}

func (l *outputLogger) Fatal(format string, a ...interface{}) {
	if l.level >= FATAL {
		l.err.WriteString(fmt.Sprintf("ERROR: "+format+"\n", a...))
	}
	os.Exit(1)
}
<commit_msg>Remove useless make()<commit_after>package main

import (
	whisper "github.com/grobian/go-whisper"
	pickle "github.com/kisielk/og-rek"

	"encoding/json"
	"flag"
	"fmt"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
)

var config = struct {
	WhisperData string
}{
	WhisperData: "/var/lib/carbon/whisper",
}

type WhisperFetchResponse struct {
	Name      string    `json:"name"`
	StartTime int       `json:"startTime"`
	StopTime  int       `json:"stopTime"`
	StepTime  int       `json:"stepTime"`
	Values    []float64 `json:"values"`
	IsAbsent  []bool    `json:"isAbsent"`
}

type WhisperGlobResponse struct {
	Name  string   `json:"name"`
	Paths []string `json:"paths"`
}

var log Logger

func findHandler(wr http.ResponseWriter, req *http.Request) {
	//	GET /metrics/find/?local=1&format=pickle&query=general.hadoop.lhr4.ha201jobtracker-01.jobtracker.NonHeapMemoryUsage.committed HTTP/1.1
	//	http://localhost:8080/metrics/find/?query=test
	req.ParseForm()
	glob := req.FormValue("query")
	format := req.FormValue("format")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
			format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
			http.StatusBadRequest)
		return
	}

	if glob == "" {
		log.Warn("dropping invalid request (query=): %s", req.URL.RequestURI())
		http.Error(wr, "Bad request (no query)", http.StatusBadRequest)
		return
	}

	/* things to glob:
	 * - carbon.relays  -> carbon.relays
	 * - carbon.re      -> carbon.relays, carbon.rewhatever
	 * - carbon.[rz]    -> carbon.relays, carbon.zipper
	 * - carbon.{re,zi} -> carbon.relays, carbon.zipper
	 * - implicit * at the end of each query
	 * - match is either dir or .wsp file
	 * unfortunately, filepath.Glob doesn't handle the curly brace
	 * expansion for us */
	lbrace := strings.Index(glob, "{")
	rbrace := -1
	if lbrace > -1 {
		rbrace = strings.Index(glob[lbrace:], "}")
		if rbrace > -1 {
			rbrace += lbrace
		}
	}
	var files []string
	if lbrace > -1 && rbrace > -1 {
		expansion := glob[lbrace+1 : rbrace]
		parts := strings.Split(expansion, ",")
		for _, sub := range parts {
			sglob := glob[:lbrace] + sub + glob[rbrace+1:]
			path := config.WhisperData + "/" + strings.Replace(sglob, ".", "/", -1) + "*"
			nfiles, err := filepath.Glob(path)
			if err == nil {
				files = append(files, nfiles...)
			}
		}
	} else {
		path := config.WhisperData + "/" + strings.Replace(glob, ".", "/", -1) + "*"
		nfiles, err := filepath.Glob(path)
		if err == nil {
			files = append(files, nfiles...)
		}
	}

	leafs := make([]bool, len(files))
	for i, p := range files {
		p = p[len(config.WhisperData+"/"):]
		if strings.HasSuffix(p, ".wsp") {
			p = p[:len(p)-4]
			leafs[i] = true
		} else {
			leafs[i] = false
		}
		files[i] = strings.Replace(p, "/", ".", -1)
	}

	if format == "json" {
		response := WhisperGlobResponse{
			Name:  glob,
			Paths: make([]string, 0),
		}
		for _, p := range files {
			response.Paths = append(response.Paths, p)
		}
		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for glob %s: %s", glob, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		// [{'metric_path': 'metric', 'intervals': [(x,y)], 'isLeaf': True},]
		var metrics []map[string]interface{}
		var m map[string]interface{}

		for i, p := range files {
			m = make(map[string]interface{})
			m["metric_path"] = p
			// m["intervals"] = dunno how to do a tuple here
			m["isLeaf"] = leafs[i]
			metrics = append(metrics, m)
		}

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}
	log.Info("find: %d hits for %s", len(files), glob)
	return
}

func fetchHandler(wr http.ResponseWriter, req *http.Request) {
	//	GET /render/?target=general.me.1.percent_time_active.pfnredis&format=pickle&from=1396008021&until=1396022421 HTTP/1.1
	//	http://localhost:8080/render/?target=testmetric&format=json&from=1395961200&until=1395961800
	req.ParseForm()
	metric := req.FormValue("target")
	format := req.FormValue("format")
	from := req.FormValue("from")
	until := req.FormValue("until")

	if format != "json" && format != "pickle" {
		log.Warn("dropping invalid uri (format=%s): %s",
			format, req.URL.RequestURI())
		http.Error(wr, "Bad request (unsupported format)",
			http.StatusBadRequest)
		return
	}

	path := config.WhisperData + "/" + strings.Replace(metric, ".", "/", -1) + ".wsp"
	w, err := whisper.Open(path)
	if err != nil {
		// the FE/carbonzipper often requests metrics we don't have
		log.Debug("failed to %s", err)
		http.Error(wr, "Metric not found", http.StatusNotFound)
		return
	}

	i, err := strconv.Atoi(from)
	if err != nil {
		log.Debug("fromTime (%s) invalid: %s (in %s)",
			from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	fromTime := int(i)
	i, err = strconv.Atoi(until)
	if err != nil {
		log.Debug("untilTime (%s) invalid: %s (in %s)",
			from, err, req.URL.RequestURI)
		if w != nil {
			w.Close()
		}
		w = nil
	}
	untilTime := int(i)

	if w != nil {
		defer w.Close()
	} else {
		http.Error(wr, "Bad request (invalid from/until time)",
			http.StatusBadRequest)
		return
	}

	points, err := w.Fetch(fromTime, untilTime)
	if err != nil {
		log.Error("failed to fetch points from %s: %s", path, err)
		http.Error(wr, "Fetching data points failed",
			http.StatusInternalServerError)
		return
	}
	values := points.Values()

	if format == "json" {
		response := WhisperFetchResponse{
			Name:      metric,
			StartTime: points.FromTime(),
			StopTime:  points.UntilTime(),
			StepTime:  points.Step(),
			Values:    make([]float64, len(values)),
			IsAbsent:  make([]bool, len(values)),
		}

		for i, p := range values {
			if math.IsNaN(p) {
				response.Values[i] = 0
				response.IsAbsent[i] = true
			} else {
				response.Values[i] = p
				response.IsAbsent[i] = false
			}
		}

		b, err := json.Marshal(response)
		if err != nil {
			log.Error("failed to create JSON data for %s: %s", path, err)
			return
		}
		wr.Write(b)
	} else if format == "pickle" {
		//[{'start': 1396271100, 'step': 60, 'name': 'metric',
		//'values': [9.0, 19.0, None], 'end': 1396273140}
		var metrics []map[string]interface{}
		var m map[string]interface{}

		m = make(map[string]interface{})
		m["start"] = points.FromTime()
		m["step"] = points.Step()
		m["end"] = points.UntilTime()
		m["name"] = metric

		mv := make([]interface{}, len(values))
		for i, p := range values {
			if math.IsNaN(p) {
				mv[i] = nil
			} else {
				mv[i] = p
			}
		}

		m["values"] = mv
		metrics = append(metrics, m)

		wr.Header().Set("Content-Type", "application/pickle")
		pEnc := pickle.NewEncoder(wr)
		pEnc.Encode(metrics)
	}

	log.Info("served %d points for %s", len(values), metric)
	return
}

func main() {
	port := flag.Int("p", 8080, "port to bind to")
	verbose := flag.Bool("v", false, "enable verbose logging")
	debug := flag.Bool("vv", false, "enable more verbose (debug) logging")
	whisperdata := flag.String("w", config.WhisperData, "location where whisper files are stored")

	flag.Parse()

	loglevel := WARN
	if *verbose {
		loglevel = INFO
	}
	if *debug {
		loglevel = DEBUG
	}
	log = NewOutputLogger(loglevel)

	config.WhisperData = *whisperdata
	log.Info("reading whisper files from: %s", config.WhisperData)

	http.HandleFunc("/metrics/find/", findHandler)
	http.HandleFunc("/render/", fetchHandler)

	listen := fmt.Sprintf(":%d", *port)
	log.Info("listening on %s", listen)
	err := http.ListenAndServe(listen, nil)
	if err != nil {
		log.Fatal("%s", err)
	}
	log.Info("stopped")
}

// Simple wrapper to enable/disable lots of spam
type Logger interface {
	Info(format string, a ...interface{})
	Warn(format string, a ...interface{})
	Error(format string, a ...interface{})
	Fatal(format string, a ...interface{})
	Debug(format string, a ...interface{})
}

type LogLevel int

const (
	FATAL LogLevel = 0
	ERROR LogLevel = 1
	WARN  LogLevel = 2
	INFO  LogLevel = 3
	DEBUG LogLevel = 4
)

type outputLogger struct {
	level LogLevel
	out   *os.File
	err   *os.File
}

func NewOutputLogger(level LogLevel) *outputLogger {
	r := new(outputLogger)
	r.level = level
	r.out = os.Stdout
	r.err = os.Stderr

	return r
}

func (l *outputLogger) Debug(format string, a ...interface{}) {
	if l.level >= DEBUG {
		l.out.WriteString(fmt.Sprintf("DEBUG: "+format+"\n", a...))
	}
}

func (l *outputLogger) Info(format string, a ...interface{}) {
	if l.level >= INFO {
		l.out.WriteString(fmt.Sprintf("INFO: "+format+"\n", a...))
	}
}

func (l *outputLogger) Warn(format string, a ...interface{}) {
	if l.level >= WARN {
		l.out.WriteString(fmt.Sprintf("WARN: "+format+"\n", a...))
	}
}

func (l *outputLogger) Error(format string, a ...interface{}) {
	if l.level >= ERROR {
		l.err.WriteString(fmt.Sprintf("ERROR: "+format+"\n", a...))
	}
}

func (l *outputLogger) Fatal(format string, a ...interface{}) {
	if l.level >= FATAL {
		l.err.WriteString(fmt.Sprintf("ERROR: "+format+"\n", a...))
	}
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import "github.com/gizak/termui"

func main() {
	err := termui.Init()
	if err != nil {
		panic(err)
	}
	defer termui.Close()

	p := termui.NewPar(":PRESS q or Esc TO QUIT DEMO Hello World")
	p.Height = 3
	p.Width = 50
	p.TextFgColor = termui.ColorWhite
	p.BorderLabel = "Hello-World"
	p.BorderFg = termui.ColorCyan

	termui.Render(p)

	termui.Handle("/sys", func(e termui.Event) {
		k, ok := e.Data.(termui.EvtKbd)
		if ok && (k.KeyStr == "q" || k.KeyStr == "<escape>") {
			termui.StopLoop()
		}
	})
	termui.Loop()
}
<commit_msg>Ajout Courbe<commit_after>package main

import "github.com/gizak/termui"
import "math"
func main() {

	err := termui.Init()
	if err != nil {
		panic(err)
	}
	defer termui.Close()

	sinps := (func() []float64 {
		n := 220
		ps := make([]float64, n)
		for i := range ps {
			ps[i] = 1 + math.Sin(float64(i)/5)
		}
		return ps
	})()

	p := termui.NewPar(":PRESS q or Esc TO QUIT DEMO Hello World")
	p.Height = 3
	p.Width = 50
	p.TextFgColor = termui.ColorWhite
	p.BorderLabel = "Hello-World"
	p.BorderFg = termui.ColorCyan

	lc1 := termui.NewLineChart()
	lc1.BorderLabel = "dot-mode Line Chart"
	lc1.Mode = "dot"
	lc1.Data = sinps
	lc1.Width = 26
	lc1.Height = 12
	lc1.X = 51
	lc1.DotStyle = '+'
	lc1.AxesColor = termui.ColorWhite
	lc1.LineColor = termui.ColorYellow | termui.AttrBold
	termui.Render(p, lc1)

	termui.Handle("/sys", func(e termui.Event) {
		k, ok := e.Data.(termui.EvtKbd)
		if ok && (k.KeyStr == "q" || k.KeyStr == "<escape>") {
			termui.StopLoop()
		}
	})
	termui.Loop()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/codegangsta/martini"
	"github.com/martini-contrib/render"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	s "strings"
	"time"
)

var data = make(map[string]interface{})

const filename = "db.txt"

type Proj struct {
	Title       string
	Description string
}

func ScrapProj() {
	fmt.Println("scraping ..")
	doc, err := goquery.NewDocument("https://github.com/karan/Projects/blob/master/README.md")
	if err != nil {
		log.Fatal(err)
	}

	if !isFileExist(filename) {
		// if the file doesn't exist, create a new one
		w, err := os.Create(filename)
		if err != nil {
			panic(err)
		}
		defer w.Close()

		// write empty one first
		err = ioutil.WriteFile(filename, []byte(""), 0644)

		doc.Find(".markdown-body p").Slice(7, 95).Each(func(i int, s *goquery.Selection) {
			txt := s.Text() + "\n"
			f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0600)
			if err != nil {
				panic(err)
			}

			defer f.Close()

			if _, err = f.WriteString(txt); err != nil {
				panic(err)
			}
		})
	}
}

func ReadProjFromFile() []string {
	fmt.Println("reading from file ..")
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return s.Split(string(b), "\n")
}

func GetRandomProj(projs []string) Proj {
	i := random(0, len(projs))
	project := projs[i]

	title := s.Split(project, " - ")[0]
	desc := s.Split(project, " - ")[1]

	return Proj{title, desc}
}

func random(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}

func main() {

	// If the file doesn't exist, scrap it
	if !isFileExist(filename) {
		ScrapProj()
	}

	m := martini.Classic()
	m.Use(render.Renderer(render.Options{
		Layout: "layout",
	}))

	m.Get("/suggest", func(ren render.Render) {
		ren.HTML(200, "index", GetRandomProj(ReadProjFromFile()))
	})

	m.Get("/", func(ren render.Render) {
		ren.HTML(200, "index", nil)
	})

	m.Run()
}

// Check if file already exists or not
func isFileExist(filename string) bool {
	if _, err := os.Stat(filename); err == nil {
		return true
	}
	return false
}
<commit_msg>remove unused data<commit_after>package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"github.com/codegangsta/martini"
	"github.com/martini-contrib/render"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	s "strings"
	"time"
)

const filename = "db.txt"

type Proj struct {
	Title       string
	Description string
}

func ScrapProj() {
	fmt.Println("scraping ..")
	doc, err := goquery.NewDocument("https://github.com/karan/Projects/blob/master/README.md")
	if err != nil {
		log.Fatal(err)
	}

	if !isFileExist(filename) {
		// if the file doesn't exist, create a new one
		w, err := os.Create(filename)
		if err != nil {
			panic(err)
		}
		defer w.Close()

		// write empty one first
		err = ioutil.WriteFile(filename, []byte(""), 0644)

		doc.Find(".markdown-body p").Slice(7, 95).Each(func(i int, s *goquery.Selection) {
			txt := s.Text() + "\n"
			f, err := os.OpenFile(filename, os.O_APPEND|os.O_WRONLY, 0600)
			if err != nil {
				panic(err)
			}

			defer f.Close()

			if _, err = f.WriteString(txt); err != nil {
				panic(err)
			}
		})
	}
}

func ReadProjFromFile() []string {
	fmt.Println("reading from file ..")
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	return s.Split(string(b), "\n")
}

func GetRandomProj(projs []string) Proj {
	i := random(0, len(projs))
	project := projs[i]

	title := s.Split(project, " - ")[0]
	desc := s.Split(project, " - ")[1]

	return Proj{title, desc}
}

func random(min, max int) int {
	rand.Seed(time.Now().UTC().UnixNano())
	return rand.Intn(max-min) + min
}

func main() {

	// If the file doesn't exist, scrap it
	if !isFileExist(filename) {
		ScrapProj()
	}

	m := martini.Classic()
	m.Use(render.Renderer(render.Options{
		Layout: "layout",
	}))

	m.Get("/suggest", func(ren render.Render) {
		ren.HTML(200, "index", GetRandomProj(ReadProjFromFile()))
	})

	m.Get("/", func(ren render.Render) {
		ren.HTML(200, "index", nil)
	})

	m.Run()
}

// Check if file already exists or not
func isFileExist(filename string) bool {
	if _, err := os.Stat(filename); err == nil {
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "net"
    "log"
    "os"
    "os/signal"
    "syscall"
    "io"
)

// Signal handler catches SIGINT and SIGTERM and sends a "done" flag to the main loop
func signalHandler(signalChannel chan os.Signal, doneChannel chan bool){
    // block the goroutine until we get a signal
    signal := <-signalChannel
    log.Printf("Got signal %v, exiting...\n", signal)
    // Send the message to terminate the app
    doneChannel <- true
}

// Collects data from the socket and sends it to where it needs to go
func aggregateCollectorData(socket net.Listener, doneChannel, finishedChannel chan bool) {
    for {
        select {
        case <- doneChannel:
            log.Println("Ceasing to accept collector connections...")
            finishedChannel <- true
            return
        default:
            log.Println("Accepting a collector connection...")
            data, err := readDataFromClient(socket)
            if err != nil {
                log.Printf("Error: %v\n", err)
            } else {
                log.Println(data)
            }
        }
    }
}

// Accept a connection from the socket and read 512 bytes of data into a buffer
func readDataFromClient(socket net.Listener) ([]byte, error) {
    readBuffer := make([]byte, 512)

    fd, err := socket.Accept()
    if err != nil {
        return nil, fmt.Errorf("Failed to accept a connection: err: %v\n", err)
    }

    defer fd.Close()

    bytesRead, err := fd.Read(readBuffer)

    if err == io.EOF {
        return nil, fmt.Errorf("Received no data from client connection")
    }

    if err != nil {
        return nil, fmt.Errorf("Failed to read from the socket into the buffer: err: %v\n", err)
    }

    return readBuffer[:bytesRead], nil
}

// Connect to the socket as a client to unblock the Accept call
func mimicFinalClient(socketUrl string) {
    log.Println("Creating a mimic client to terminate socket accept thread")
    conn, err := net.Dial("unix", socketUrl)
    if err != nil {
        log.Fatalf("Failed to open the final client connection to Garnet")
    }
    conn.Close()
}

func main() {
    // Create a channel to pass to os.Notify for OS signal handling
    signalChannel := make(chan os.Signal, 1)
    signalDoneChannel := make(chan bool, 1)
    signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM)
    go signalHandler(signalChannel, signalDoneChannel)

    socket, err := net.Listen("unix", "/tmp/garnet.sock")
    if err != nil {
        log.Fatalf("Failed to create a new Unix socket: err: %v\n", err)
    }
    defer socket.Close()

    log.Printf("Opened a socket connection '/tmp/garnet.sock'\n")

    // Start the aggregation collector
    aggregationDoneChannel := make(chan bool, 1)
    cleanUpChannel := make(chan bool, 1)
    go aggregateCollectorData(socket, aggregationDoneChannel, cleanUpChannel)

    // Wait until we get a catchable signal before cleaning up
    <- signalDoneChannel
    aggregationDoneChannel <- true
    mimicFinalClient("/tmp/garnet.sock")
    <- cleanUpChannel
}
<commit_msg>Add comments surrounding waiting for a "done" message from a channel<commit_after>package main

import (
    "fmt"
    "net"
    "log"
    "os"
    "os/signal"
    "syscall"
    "io"
)

// Signal handler catches SIGINT and SIGTERM and sends a "done" flag to the main loop
func signalHandler(signalChannel chan os.Signal, doneChannel chan bool){
    // block the goroutine until we get a signal
    signal := <-signalChannel
    log.Printf("Got signal %v, exiting...\n", signal)
    // Send the message to terminate the app
    doneChannel <- true
}

// Collects data from the socket and sends it to where it needs to go
func aggregateCollectorData(socket net.Listener, doneChannel, finishedChannel chan bool) {
    for {
        select {
        case <- doneChannel:
            log.Println("Ceasing to accept collector connections...")
            finishedChannel <- true
            return
        default:
            log.Println("Accepting a collector connection...")
            data, err := readDataFromClient(socket)
            if err != nil {
                log.Printf("Error: %v\n", err)
            } else {
                log.Println(data)
            }
        }
    }
}

// Accept a connection from the socket and read 512 bytes of data into a buffer
func readDataFromClient(socket net.Listener) ([]byte, error) {
    readBuffer := make([]byte, 512)

    fd, err := socket.Accept()
    if err != nil {
        return nil, fmt.Errorf("Failed to accept a connection: err: %v\n", err)
    }

    defer fd.Close()

    bytesRead, err := fd.Read(readBuffer)

    if err == io.EOF {
        return nil, fmt.Errorf("Received no data from client connection")
    }

    if err != nil {
        return nil, fmt.Errorf("Failed to read from the socket into the buffer: err: %v\n", err)
    }

    return readBuffer[:bytesRead], nil
}

// Connect to the socket as a client to unblock the Accept call
func mimicFinalClient(socketUrl string) {
    log.Println("Creating a mimic client to terminate socket accept thread")
    conn, err := net.Dial("unix", socketUrl)
    if err != nil {
        log.Fatalf("Failed to open the final client connection to Garnet")
    }
    conn.Close()
}

func main() {
    // Create a channel to pass to os.Notify for OS signal handling
    signalChannel := make(chan os.Signal, 1)
    signalDoneChannel := make(chan bool, 1)
    signal.Notify(signalChannel, syscall.SIGINT, syscall.SIGTERM)
    go signalHandler(signalChannel, signalDoneChannel)

    socket, err := net.Listen("unix", "/tmp/garnet.sock")
    if err != nil {
        log.Fatalf("Failed to create a new Unix socket: err: %v\n", err)
    }
    defer socket.Close()

    log.Printf("Opened a socket connection '/tmp/garnet.sock'\n")

    // Start the aggregation collector
    aggregationDoneChannel := make(chan bool, 1)
    aggregationCleanUpChannel := make(chan bool, 1)
    go aggregateCollectorData(socket, aggregationDoneChannel, aggregationCleanUpChannel)

    // Wait until we get a catchable signal before cleaning up
    <- signalDoneChannel

    // Tell the collector aggregator to stop processing connections
    aggregationDoneChannel <- true
    mimicFinalClient("/tmp/garnet.sock")
    <- aggregationCleanUpChannel
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"

	"github.com/gwenn/gosqlite"
	"github.com/minio/go-homedir"
)

var (
	outputDir = "Databases"
	fileName = "72mb.sqlite"
	numRows = 100000 // 100000 makes a 72MB file (taking ~4.5 seconds) on my Linux desktop.  Adjust to suit your desired target file size
)

type oneRow struct {
	key_data int
	int_data int
	signed_data int
	float_data float32
	double_data float64
	decim_data string
	date_data string
	code_data string
	name_data string
	address_data string
}

func main() {
	// Determine full path to target file
	userHome, err := homedir.Dir()
	if err != nil {
		log.Printf("User home directory couldn't be determined: %s", "\n")
		return
	}
	fn := filepath.Join(userHome, outputDir, fileName)

	// If the database file already exists, nuke the file
	_, err = os.Stat(fn)
	if err == nil {
		// No error occurred when looking for an existing file, which means something is there.  For now, we're just
		// going to blindly kill the existing thing without any kind of better safeguard
		log.Printf("A SQLite database appears to be there already... removing it")
		os.Remove(fn)
	}

	// Create empty SQLite database
	log.Printf("Creating new SQLite database file '%s'\n", fn)
	sdb, err := sqlite.Open(fn, sqlite.OpenCreate | sqlite.OpenReadWrite)
	if err != nil {
		log.Printf("Couldn't open database: %s", err)
		return
	}
	defer sdb.Close()

	// Disable the journal
	err = sdb.Select("PRAGMA journal_mode=OFF", func(s *sqlite.Stmt) error {
		return nil
	})
	if err != nil {
		log.Printf("Error when disabling the journal: %s\n", err)
		return
	}

	// Turn off synchronous mode
	err = sdb.Exec("PRAGMA synchronous=OFF")
	if err != nil {
		log.Printf("Error when setting synchronous mode: %s\n", err)
		return
	}

	// Create the schema
	log.Println("Creating tables")
	tableNames := []string{"uniques", "updates", "hundred", "tenpct", "tiny"}
	var dbQuery string
	for _, tbl := range tableNames {
		dbQuery = fmt.Sprintf(`
			CREATE TABLE IF NOT EXISTS %s (
				'col_key'     INTEGER NOT NULL,
				'col_int'     INTEGER NOT NULL,
				'col_signed'  INTEGER NOT NULL,
				'col_float'   REAL NOT NULL,
				'col_double'  REAL NOT NULL,
				'col_decim'   NUMERIC NOT NULL,
				'col_date'    TEXT NOT NULL,
				'col_code'    TEXT NOT NULL,
				'col_name'    TEXT NOT NULL,
				'col_address' TEXT NOT NULL
			)`, tbl)
		err := sdb.Exec(dbQuery)
		if err != nil {
			log.Printf("Error when creating table '%s': %s\n", tbl, err)
			return
		}
	}

	// Launch a worker pool generating row data
	cpus := runtime.NumCPU()
	log.Printf("# of cpu's detected: %d.  Launching %d data generation workers\n", cpus, cpus)
	results := make(chan *oneRow, cpus * 5) // 5 seems ok, less than 5 seems slightly slower (not properly measured though!)
	for w := 0; w < cpus; w++ {
		go worker(results)
	}

	// Bulk insert row data (inside a single transaction per table)
	log.Println("Adding data")
	var r *oneRow
	for _, tbl := range tableNames {
		err = sdb.Begin()
		if err != nil {
			log.Printf("Error for Begin(): %s\n", err)
			return
		}

		// Prepare the insert statement
		dbQuery := sqlite.Mprintf(`
			INSERT into %w (col_key, col_int, col_signed, col_float, col_double, col_decim, col_date, col_code, col_name, col_address)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, tbl)
		stmt, err := sdb.Prepare(dbQuery)
		if err != nil {
			log.Printf("Error when preparing statement for inserts: %s\n", err)
			return
		}

		// Insert the data
		for i := 0; i < numRows; i++ {
			r = <- results
			err = stmt.Exec(r.key_data, r.int_data, r.signed_data, r.float_data, r.double_data, r.decim_data,
				r.date_data, r.code_data, r.name_data, r.address_data)
			if err != nil {
				log.Printf("Error when inserting data: %s\n", err)
				return
			}
		}

		// Clean up for this loop
		stmt.Finalize()

		// Commit the transaction
		sdb.Commit()
	}

	// TODO: Create indexes ?

	// Let the user know the program completed ok
	log.Printf("SQLite database generation completed")
}

// Generate a random string
func randomString(length int) string {
	const alphaNum = "abcdefghijklmnopqrstuvwxyz0123456789"
	randomString := make([]byte, length)
	for i := range randomString {
		randomString[i] = alphaNum[rand.Intn(len(alphaNum))]
	}
	return string(randomString)
}

// Goroutine which generates rows of test data
func worker(results chan <- *oneRow) {
	for {
		row := new(oneRow)
		row.key_data = rand.Int()
		row.int_data = rand.Int()
		row.signed_data = rand.Int()
		row.float_data = rand.Float32()
		row.double_data = rand.Float64()
		row.decim_data = fmt.Sprintf("%d.%d", rand.Intn(100000000000000000), rand.Intn(100))
		row.date_data = fmt.Sprintf("%d%d%d%d-%d%d-%d%d", rand.Intn(10), rand.Intn(10), rand.Intn(10),
			rand.Intn(10), rand.Intn(10), rand.Intn(10), rand.Intn(10), rand.Intn(10))
		row.code_data = randomString(10)
		row.name_data = randomString(20)
		addLen := rand.Intn(80)
		if addLen < 8 {
			addLen = 8
		}
		row.address_data = randomString(addLen)
		results <- row
	}
}
<commit_msg>Generates the file in the current directory<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"

	"github.com/gwenn/gosqlite"
)

var (
	fileName = "72mb.sqlite"
	numRows  = 100000 // 100000 makes a 72MB file (taking ~4.5 seconds) on my Linux desktop.  Adjust to suit your desired target file size
)

type oneRow struct {
	key_data     int
	int_data     int
	signed_data  int
	float_data   float32
	double_data  float64
	decim_data   string
	date_data    string
	code_data    string
	name_data    string
	address_data string
}

func main() {
	// Determine full path to target file
	d, err := os.Getwd()
	if err != nil {
		log.Fatalf(err.Error())
	}
	fn := filepath.Join(d, fileName)

	// If the database file already exists, nuke the file
	_, err = os.Stat(fn)
	if err == nil {
		// No error occurred when looking for an existing file, which means something is there.  For now, we're just
		// going to blindly kill the existing thing without any kind of better safeguard
		log.Printf("A SQLite database appears to be there already... removing it")
		os.Remove(fn)
	}

	// Create empty SQLite database
	log.Printf("Creating new SQLite database file '%s'\n", fn)
	sdb, err := sqlite.Open(fn, sqlite.OpenCreate|sqlite.OpenReadWrite)
	if err != nil {
		log.Printf("Couldn't open database: %s", err)
		return
	}
	defer sdb.Close()

	// Disable the journal
	err = sdb.Select("PRAGMA journal_mode=OFF", func(s *sqlite.Stmt) error {
		return nil
	})
	if err != nil {
		log.Printf("Error when disabling the journal: %s\n", err)
		return
	}

	// Turn off synchronous mode
	err = sdb.Exec("PRAGMA synchronous=OFF")
	if err != nil {
		log.Printf("Error when setting synchronous mode: %s\n", err)
		return
	}

	// Create the schema
	log.Println("Creating tables")
	tableNames := []string{"uniques", "updates", "hundred", "tenpct", "tiny"}
	var dbQuery string
	for _, tbl := range tableNames {
		dbQuery = fmt.Sprintf(`
			CREATE TABLE IF NOT EXISTS %s (
				'col_key'     INTEGER NOT NULL,
				'col_int'     INTEGER NOT NULL,
				'col_signed'  INTEGER NOT NULL,
				'col_float'   REAL NOT NULL,
				'col_double'  REAL NOT NULL,
				'col_decim'   NUMERIC NOT NULL,
				'col_date'    TEXT NOT NULL,
				'col_code'    TEXT NOT NULL,
				'col_name'    TEXT NOT NULL,
				'col_address' TEXT NOT NULL
			)`, tbl)
		err := sdb.Exec(dbQuery)
		if err != nil {
			log.Printf("Error when creating table '%s': %s\n", tbl, err)
			return
		}
	}

	// Launch a worker pool generating row data
	cpus := runtime.NumCPU()
	log.Printf("# of cpu's detected: %d.  Launching %d data generation workers\n", cpus, cpus)
	results := make(chan *oneRow, cpus*5) // 5 seems ok, less than 5 seems slightly slower (not properly measured though!)
	for w := 0; w < cpus; w++ {
		go worker(results)
	}

	// Bulk insert row data (inside a single transaction per table)
	log.Println("Adding data")
	var r *oneRow
	for _, tbl := range tableNames {
		err = sdb.Begin()
		if err != nil {
			log.Printf("Error for Begin(): %s\n", err)
			return
		}

		// Prepare the insert statement
		dbQuery := sqlite.Mprintf(`
			INSERT into %w (col_key, col_int, col_signed, col_float, col_double, col_decim, col_date, col_code, col_name, col_address)
			VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, tbl)
		stmt, err := sdb.Prepare(dbQuery)
		if err != nil {
			log.Printf("Error when preparing statement for inserts: %s\n", err)
			return
		}

		// Insert the data
		for i := 0; i < numRows; i++ {
			r = <-results
			err = stmt.Exec(r.key_data, r.int_data, r.signed_data, r.float_data, r.double_data, r.decim_data,
				r.date_data, r.code_data, r.name_data, r.address_data)
			if err != nil {
				log.Printf("Error when inserting data: %s\n", err)
				return
			}
		}

		// Clean up for this loop
		stmt.Finalize()

		// Commit the transaction
		sdb.Commit()
	}

	// TODO: Create indexes ?

	// Let the user know the program completed ok
	log.Printf("SQLite database generation completed")
}

// Generate a random string
func randomString(length int) string {
	const alphaNum = "abcdefghijklmnopqrstuvwxyz0123456789"
	randomString := make([]byte, length)
	for i := range randomString {
		randomString[i] = alphaNum[rand.Intn(len(alphaNum))]
	}
	return string(randomString)
}

// Goroutine which generates rows of test data
func worker(results chan<- *oneRow) {
	for {
		row := new(oneRow)
		row.key_data = rand.Int()
		row.int_data = rand.Int()
		row.signed_data = rand.Int()
		row.float_data = rand.Float32()
		row.double_data = rand.Float64()
		row.decim_data = fmt.Sprintf("%d.%d", rand.Intn(100000000000000000), rand.Intn(100))
		row.date_data = fmt.Sprintf("%d%d%d%d-%d%d-%d%d", rand.Intn(10), rand.Intn(10), rand.Intn(10),
			rand.Intn(10), rand.Intn(10), rand.Intn(10), rand.Intn(10), rand.Intn(10))
		row.code_data = randomString(10)
		row.name_data = randomString(20)
		addLen := rand.Intn(80)
		if addLen < 8 {
			addLen = 8
		}
		row.address_data = randomString(addLen)
		results <- row
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func in(slice []string, str string) bool {
	for _, s := range slice {
		if s == str {
			return true
		}
	}
	return false
}

var acceptedImageExt = []string{".jpg", ".jpeg"}
var images = []string{}
var dirThumbs = fmt.Sprintf("%s%s", os.Getenv("HOME"), "/.cache/sxiv")
var dirPath = "."

func main() {

	flag.Parse()

	directory := flag.Arg(0)
	dirPath, _ = filepath.Abs(directory)

	fmt.Println("lk is serving", dirPath, "from http://0.0.0.0:3000")

	filepath.Walk(dirPath, func(filePath string, info os.FileInfo, err error) error {
		if err == nil && in(acceptedImageExt, strings.ToLower(path.Ext(filePath))) {
			thumbnail := fmt.Sprintf("%s%s.jpg", dirThumbs, filePath)
			if _, err := os.Stat(thumbnail); os.IsNotExist(err) {
				fmt.Println("Missing thumbnail:", thumbnail)
				genthumb(filePath, thumbnail)
			}
			images = append(images, filePath)
		}
		return nil
	})

	http.Handle("/o/", http.StripPrefix("/o/", http.FileServer(http.Dir("/"))))
	http.Handle("/t/", http.StripPrefix("/t/", (http.FileServer(http.Dir(dirThumbs)))))
	http.HandleFunc("/", lk)
	http.ListenAndServe(":3000", nil)
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println("right", r.Method, r.URL.Path)
		h.ServeHTTP(w, r)
	})
}

func lk(w http.ResponseWriter, r *http.Request) {

	t, err := template.New("foo").Parse(`{{ range . }}

{{ end }}`)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	t.Execute(w, images)
}
<commit_msg>Only show files from dirPath<commit_after>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func in(slice []string, str string) bool {
	for _, s := range slice {
		if s == str {
			return true
		}
	}
	return false
}

var acceptedImageExt = []string{".jpg", ".jpeg"}
var images = []string{}
var dirThumbs = fmt.Sprintf("%s%s", os.Getenv("HOME"), "/.cache/sxiv")
var dirPath = "."

func main() {

	flag.Parse()

	directory := flag.Arg(0)
	dirPath, _ = filepath.Abs(directory)

	fmt.Println("lk is serving", dirPath, "from http://0.0.0.0:3000")

	filepath.Walk(dirPath, func(filePath string, info os.FileInfo, err error) error {
		if err == nil && in(acceptedImageExt, strings.ToLower(path.Ext(filePath))) {
			thumbnail := fmt.Sprintf("%s%s.jpg", dirThumbs, filePath)
			if _, err := os.Stat(thumbnail); os.IsNotExist(err) {
				fmt.Println("Missing thumbnail:", thumbnail)
				genthumb(filePath, thumbnail)
			}
			images = append(images, filePath)
		}
		return nil
	})

	// Don't allow path under dirPath to be viewed
	// http://www.reddit.com/r/golang/comments/2l59wk/web_based_jpg_viewer_for_sharing_images_on_a_lan/clrpbyo
	http.Handle("/o/", http.StripPrefix(path.Join("/o", dirPath), http.FileServer(http.Dir(dirPath))))
	http.Handle("/t/", http.StripPrefix(path.Join("/t", dirPath), http.FileServer(http.Dir(path.Join(dirThumbs, dirPath)))))

	http.HandleFunc("/", lk)
	http.ListenAndServe(":3000", nil)
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Println("right", r.Method, r.URL.Path)
		h.ServeHTTP(w, r)
	})
}

func lk(w http.ResponseWriter, r *http.Request) {

	t, err := template.New("foo").Parse(`{{ range . }}

{{ end }}`)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	t.Execute(w, images)
}
<|endoftext|>