<commit_before>package main

import (
	"os"
	"path"
	"strconv"
	"sync"

	"github.com/MJKWoolnough/minecraft"
)

func (c *Config) Name(_ struct{}, serverName *string) error {
	c.mu.RLock()
	defer c.mu.RUnlock()
	*serverName = c.ServerName
	return nil
}

func (c *Config) List(_ struct{}, list *[]Server) error {
	c.mu.RLock()
	defer c.mu.RUnlock()
	*list = make([]Server, 0, len(c.Servers))
	for _, s := range c.Servers {
		*list = append(*list, s)
	}
	return nil
}

func (c *Config) Save(s Server, _ *struct{}) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	defer c.save()
	ns, ok := c.Servers[s.ID]
	if !ok {
		return ErrNoServer
	}
	s.Path = ns.Path
	s.status = ns.status
	c.Servers[s.ID] = s
	return nil
}

func (c *Config) MapList(_ struct{}, list *[]Map) error {
	c.mu.RLock()
	defer c.mu.RUnlock()
	*list = make([]Map, 0, len(c.Maps))
	for _, m := range c.Maps {
		*list = append(*list, m)
	}
	return nil
}

type DefaultMap struct {
	Mode               int
	Name               string
	GameMode           int
	Seed               int64
	Structures, Cheats bool
}

func (c *Config) CreateDefaultMap(data DefaultMap, _ *struct{}) error {
	d, err := setupMapDir()
	if err != nil {
		return err
	}
	p, err := minecraft.NewFilePath(d)
	if err != nil {
		return err
	}
	l, err := minecraft.NewLevel(p)
	if err != nil {
		return err
	}
	l.GameMode(int32(data.GameMode))
	l.LevelName(data.Name)
	switch data.Mode {
	case 0:
		l.Generator(minecraft.DefaultGenerator)
	case 1:
		l.Generator(minecraft.FlatGenerator)
	case 2:
		l.Generator(minecraft.LargeBiomeGenerator)
	case 3:
		l.Generator(minecraft.AmplifiedGenerator)
	case 4:
		l.Generator(minecraft.CustomGenerator)
	}
	l.Seed(data.Seed)
	l.AllowCommands(data.Cheats)
	l.MapFeatures(data.Structures)
	l.Save()
	c.newMap(data.Name, d)
	f, err := os.Create(path.Join(d, "server.properties"))
	if err != nil {
		return err
	}
	defer f.Close()
	m := DefaultConfig()
	m["gamemode"] = strconv.Itoa(data.GameMode)
	if !data.Structures {
		m["generate-structures"] = "false"
	}
	if data.GameMode == 3 {
		m["hardcore"] = "true"
	}
	data["level-name"] = data.Name
	data["level-seed"] = strconv.FormatInt(data.Seed, 10)
	switch data.Mode {
	case 0:
		data["level-type"] = minecraft.DefaultGenerator
	case 1:
		data["level-type"] = minecraft.FlatGenerator
	case 2:
		data["level-type"] = minecraft.LargeBiomeGenerator
	case 3:
		data["level-type"] = minecraft.AmplifiedGenerator
	case 4:
		data["level-type"] = minecraft.CustomGenerator
	case 5:
		data["level-type"] = minecraft.DebugGenerator
	}
	if err := m.WriteTo(f); err != nil {
		return err
	}
	return nil
}

var mapDirLock sync.Mutex

func setupMapDir() (string, error) {
	mapDirLock.Lock()
	defer mapDirLock.Unlock()
	num := 0
	for {
		dir := path.Join(config.MapsDir, strconv.Itoa(num))
		_, err := os.Stat(dir)
		if os.IsNotExist(err) {
			err := os.MkdirAll(dir, 0777)
			if err != nil {
				return "", err
			}
			return dir, nil
		}
		num++
	}
}
<commit_msg>wrong var<commit_after>package main

import (
	"os"
	"path"
	"strconv"
	"sync"

	"github.com/MJKWoolnough/minecraft"
)

func (c *Config) Name(_ struct{}, serverName *string) error {
	c.mu.RLock()
	defer c.mu.RUnlock()
	*serverName = c.ServerName
	return nil
}

func (c *Config) List(_ struct{}, list *[]Server) error {
	c.mu.RLock()
	defer c.mu.RUnlock()
	*list = make([]Server, 0, len(c.Servers))
	for _, s := range c.Servers {
		*list = append(*list, s)
	}
	return nil
}

func (c *Config) Save(s Server, _ *struct{}) error {
	c.mu.Lock()
	defer c.mu.Unlock()
	defer c.save()
	ns, ok := c.Servers[s.ID]
	if !ok {
		return ErrNoServer
	}
	s.Path = ns.Path
	s.status = ns.status
	c.Servers[s.ID] = s
	return nil
}

func (c *Config) MapList(_ struct{}, list *[]Map) error {
	c.mu.RLock()
	defer c.mu.RUnlock()
	*list = make([]Map, 0, len(c.Maps))
	for _, m := range c.Maps {
		*list = append(*list, m)
	}
	return nil
}

type DefaultMap struct {
	Mode               int
	Name               string
	GameMode           int
	Seed               int64
	Structures, Cheats bool
}

func (c *Config) CreateDefaultMap(data DefaultMap, _ *struct{}) error {
	d, err := setupMapDir()
	if err != nil {
		return err
	}
	p, err := minecraft.NewFilePath(d)
	if err != nil {
		return err
	}
	l, err := minecraft.NewLevel(p)
	if err != nil {
		return err
	}
	l.GameMode(int32(data.GameMode))
	l.LevelName(data.Name)
	switch data.Mode {
	case 0:
		l.Generator(minecraft.DefaultGenerator)
	case 1:
		l.Generator(minecraft.FlatGenerator)
	case 2:
		l.Generator(minecraft.LargeBiomeGenerator)
	case 3:
		l.Generator(minecraft.AmplifiedGenerator)
	case 4:
		l.Generator(minecraft.CustomGenerator)
	}
	l.Seed(data.Seed)
	l.AllowCommands(data.Cheats)
	l.MapFeatures(data.Structures)
	l.Save()
	c.newMap(data.Name, d)
	f, err := os.Create(path.Join(d, "server.properties"))
	if err != nil {
		return err
	}
	defer f.Close()
	m := DefaultSettings()
	m["gamemode"] = strconv.Itoa(data.GameMode)
	if !data.Structures {
		m["generate-structures"] = "false"
	}
	if data.GameMode == 3 {
		m["hardcore"] = "true"
	}
	m["level-name"] = data.Name
	m["level-seed"] = strconv.FormatInt(data.Seed, 10)
	switch data.Mode {
	case 0:
		m["level-type"] = minecraft.DefaultGenerator
	case 1:
		m["level-type"] = minecraft.FlatGenerator
	case 2:
		m["level-type"] = minecraft.LargeBiomeGenerator
	case 3:
		m["level-type"] = minecraft.AmplifiedGenerator
	case 4:
		m["level-type"] = minecraft.CustomGenerator
	case 5:
		m["level-type"] = minecraft.DebugGenerator
	}
	if err := m.WriteTo(f); err != nil {
		return err
	}
	return nil
}

var mapDirLock sync.Mutex

func setupMapDir() (string, error) {
	mapDirLock.Lock()
	defer mapDirLock.Unlock()
	num := 0
	for {
		dir := path.Join(config.MapsDir, strconv.Itoa(num))
		_, err := os.Stat(dir)
		if os.IsNotExist(err) {
			err := os.MkdirAll(dir, 0777)
			if err != nil {
				return "", err
			}
			return dir, nil
		}
		num++
	}
}
<|endoftext|>


<commit_before>//This package simulates the ETL operations by reading from a txt file
//and writing to a file
package main

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"time"
)

type Product struct {
	PartNumber string
	UnitCost   float64
	UnitPrice  float64
}

type Order struct {
	CustomerNumber int
	PartNumber     string
	Quantity       int

	UnitCost  float64
	UnitPrice float64
}

func main() {
	start := time.Now()
	orders := extract()
	orders = transform(orders)
	load(orders)
	fmt.Println(time.Since(start))
}

func extract() []*Order {
	result := []*Order{}

	f, _ := os.Open("./orders.txt")
	defer f.Close()

	r := csv.NewReader(f)

	for record, err := r.Read(); err == nil; record, err = r.Read() {
		order := new(Order)
		order.CustomerNumber, _ = strconv.Atoi(record[0])
		order.PartNumber = record[1]
		order.Quantity, _ = strconv.Atoi(record[2])
		result = append(result, order)
	}
	return result
}

func transform(orders []*Order) []*Order {
	f, _ := os.Open("./productList.txt")
	defer f.Close()

	r := csv.NewReader(f)
	records, _ := r.ReadAll()
	productList := make(map[string]*Product)

	for _, record := range records {
		product := new(Product)
		product.PartNumber = record[0]
		product.UnitCost, _ = strconv.ParseFloat(record[1], 64)
		product.UnitPrice, _ = strconv.ParseFloat(record[2], 64)
		productList[product.PartNumber] = product
	}

	for idx, _ := range orders {
		time.Sleep(3 * time.Millisecond)
		o := orders[idx]
		o.UnitCost = productList[o.PartNumber].UnitCost
		o.UnitPrice = productList[o.PartNumber].UnitPrice
	}
	return orders
}

func load(orders []*Order) {
	f, _ := os.Create("./dest.txt")
	defer f.Close()

	fmt.Fprintf(f, "%20s%15s%12s%12s%15s%15s\n",
		"Part Number", "Quantity", "Unit Cost",
		"Unit Price", "Total Cost", "Total Price")

	for _, o := range orders {
		time.Sleep(1 * time.Millisecond)
		fmt.Fprintf(f, "%20s %15d %12.2f %12.2f %15.2f %15.2f\n",
			o.PartNumber, o.Quantity, o.UnitCost, o.UnitPrice, o.UnitCost*float64(o.Quantity), o.UnitPrice*float64(o.Quantity))
	}
}
<commit_msg>Concurrency implementation done<commit_after>//This package simulates the ETL operations by reading from a txt file
//and writing to a file
package main

import (
	"encoding/csv"
	"fmt"
	"os"
	"strconv"
	"sync"
	"time"
)

type Product struct {
	PartNumber string
	UnitCost   float64
	UnitPrice  float64
}

type Order struct {
	CustomerNumber int
	PartNumber     string
	Quantity       int

	UnitCost  float64
	UnitPrice float64
}

func main() {
	start := time.Now()

	extractChannel := make(chan *Order)
	transformChannel := make(chan *Order)
	doneChannel := make(chan bool)

	go extract(extractChannel)
	go transform(extractChannel, transformChannel)
	go load(transformChannel, doneChannel)

	<-doneChannel

	fmt.Println(time.Since(start))
}

func extract(ch chan *Order) {

	f, _ := os.Open("./orders.txt")
	defer f.Close()

	r := csv.NewReader(f)

	for record, err := r.Read(); err == nil; record, err = r.Read() {
		order := new(Order)
		order.CustomerNumber, _ = strconv.Atoi(record[0])
		order.PartNumber = record[1]
		order.Quantity, _ = strconv.Atoi(record[2])
		ch <- order
	}
	close(ch)
}

func transform(extractChannel, transformChannel chan *Order) {
	f, _ := os.Open("./productList.txt")
	defer f.Close()

	r := csv.NewReader(f)
	records, _ := r.ReadAll()
	productList := make(map[string]*Product)

	for _, record := range records {
		product := new(Product)
		product.PartNumber = record[0]
		product.UnitCost, _ = strconv.ParseFloat(record[1], 64)
		product.UnitPrice, _ = strconv.ParseFloat(record[2], 64)
		productList[product.PartNumber] = product
	}
	var syncWait sync.WaitGroup

	for o := range extractChannel {
		syncWait.Add(1)
		go func(o *Order) {
			//sleep to imitate database operation
			time.Sleep(3 * time.Millisecond)
			o.UnitCost = productList[o.PartNumber].UnitCost
			o.UnitPrice = productList[o.PartNumber].UnitPrice
			transformChannel <- o
			syncWait.Done()
		}(o)
	}

	syncWait.Wait()
	close(transformChannel)
}

func load(transformChannel chan *Order, doneChannel chan bool) {
	f, _ := os.Create("./dest.txt")
	defer f.Close()

	fmt.Fprintf(f, "%20s%15s%12s%12s%15s%15s\n",
		"Part Number", "Quantity", "Unit Cost",
		"Unit Price", "Total Cost", "Total Price")
	var syncWait sync.WaitGroup
	for o := range transformChannel {
		syncWait.Add(1)
		go func(o *Order) {
			//sleep to imitate database operation
			time.Sleep(1 * time.Millisecond)
			fmt.Fprintf(f, "%20s %15d %12.2f %12.2f %15.2f %15.2f\n",
				o.PartNumber, o.Quantity, o.UnitCost, o.UnitPrice, o.UnitCost*float64(o.Quantity), o.UnitPrice*float64(o.Quantity))
			syncWait.Done()
		}(o)
	}

	syncWait.Wait()
	doneChannel <- true
}
<|endoftext|>


<commit_before>// Copyright 2016,2017 Yaacov Zamir <kobi.zamir@gmail.com>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package server API REST server
package server

import (
	"encoding/base64"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"time"

	"github.com/spf13/viper"

	"github.com/MohawkTSDB/mohawk/src/alerts"
	"github.com/MohawkTSDB/mohawk/src/server/handlers"
	"github.com/MohawkTSDB/mohawk/src/server/middleware"
	"github.com/MohawkTSDB/mohawk/src/server/router"
	"github.com/MohawkTSDB/mohawk/src/storage"
	"github.com/MohawkTSDB/mohawk/src/storage/example"
	"github.com/MohawkTSDB/mohawk/src/storage/memory"
	"github.com/MohawkTSDB/mohawk/src/storage/mongo"
	"github.com/MohawkTSDB/mohawk/src/storage/sqlite"
)

// VER the server version
const VER = "0.28.2"

// defaults
const defaultAPI = "0.21.0"
const publicPath = "^/hawkular/metrics/status$"

// BackendName Mohawk active storage
var BackendName string

// GetStatus return a json status struct
func GetStatus(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	resTemplate := `{"MetricsService":"STARTED","Implementation-Version":"%s","MohawkVersion":"%s","MohawkBackend":"%s"}`
	res := fmt.Sprintf(resTemplate, defaultAPI, VER, BackendName)

	w.WriteHeader(200)
	fmt.Fprintln(w, res)
}

func printOptionsHelp() {
	fmt.Println("Storage options:")
	fmt.Println(sqlite.Storage{}.Help())
	fmt.Println(memory.Storage{}.Help())
	fmt.Println(mongo.Storage{}.Help())
}

// Serve run the REST API server
func Serve() error {
	var db storage.Storage
	var alertRules *alerts.AlertRules
	var routers http.HandlerFunc
	var authorizationKey string

	var backendQuery = viper.GetString("storage")
	var optionsQuery = viper.GetString("options")
	var verbose = viper.GetBool("verbose")
	var media = viper.GetString("media")
	var gzip = viper.GetBool("gzip")
	var bearerAuth = viper.GetString("bearer-auth")
	var basicAuth = viper.GetString("basic-auth")
	var alertsInterval = viper.GetInt("alerts-interval")
	var alertsServerURL = viper.GetString("alerts-server")
	var alertsServerMethod = viper.GetString("alerts-server-method")
	var alertsServerInsecure = viper.GetBool("alerts-server-insecure")
	var configAlerts = viper.ConfigFileUsed() != "" && viper.Get("alerts") != ""

	// if options is "help" print storage options help and exit
	if optionsQuery == "help" {
		printOptionsHelp()
		return nil
	}

	// Create and init the storage
	switch backendQuery {
	case "sqlite":
		db = &sqlite.Storage{}
	case "memory":
		db = &memory.Storage{}
	case "mongo":
		db = &mongo.Storage{}
	case "example":
		db = &example.Storage{}
	default:
		log.Fatal("Can't find storage:", backendQuery)
	}

	// parse options
	if options, err := url.ParseQuery(optionsQuery); err == nil {
		db.Open(options)
	} else {
		log.Fatal("Can't parse opetions:", optionsQuery)
	}

	// set global variables
	BackendName = db.Name()

	// Create alerts runner
	if configAlerts {
		// parse alert list from config yaml
		l := []*alerts.Alert{}
		viper.UnmarshalKey("alerts", &l)

		if len(l) > 0 {
			// creat and Init the alert handler
			alertRules = &alerts.AlertRules{
				Storage:        db,
				Verbose:        verbose,
				Alerts:         l,
				AlertsInterval: alertsInterval,
				ServerURL:      alertsServerURL,
				ServerMethod:   alertsServerMethod,
				ServerInsecure: alertsServerInsecure,
			}
			alertRules.Init()
		}
	}

	// h common variables to be used for the storage Handler functions
	// Storage the storage to use for metrics source
	h := handler.APIHhandler{
		Verbose: verbose,
		Storage: db,
		Alerts:  alertRules,
	}

	// Create the routers
	// Requests not handled by the routers will be forworded to BadRequest Handler
	rRoot := router.Router{
		Prefix: "/hawkular/metrics/",
	}
	// Root Routing table
	rRoot.Add("GET", "status", GetStatus)
	rRoot.Add("GET", "tenants", h.GetTenants)
	rRoot.Add("GET", "metrics", h.GetMetrics)

	// M (Global Metrics) Routing tables
	rM := router.Router{
		Prefix: "/hawkular/metrics/m/",
	}
	rM.Add("POST", "stats/query", h.PostMQuery)

	// Metrics Routing tables
	rGauges := router.Router{
		Prefix: "/hawkular/metrics/gauges/",
	}
	rGauges.Add("GET", ":id/raw", h.GetData)
	rGauges.Add("GET", ":id/stats", h.GetData)
	rGauges.Add("POST", "raw", h.PostData)
	rGauges.Add("POST", "raw/query", h.PostQuery)
	rGauges.Add("PUT", "tags", h.PutMultiTags)
	rGauges.Add("PUT", ":id/tags", h.PutTags)
	rGauges.Add("DELETE", ":id/raw", h.DeleteData)
	rGauges.Add("DELETE", ":id/tags/:tags", h.DeleteTags)

	// deprecated
	rGauges.Add("GET", ":id/data", h.GetData)
	rGauges.Add("POST", "data", h.PostData)
	rGauges.Add("POST", "stats/query", h.PostQuery)

	rCounters := router.Router{
		Prefix: "/hawkular/metrics/counters/",
	}
	rCounters.Add("GET", ":id/raw", h.GetData)
	rCounters.Add("GET", ":id/stats", h.GetData)
	rCounters.Add("POST", "raw", h.PostData)
	rCounters.Add("POST", "raw/query", h.PostQuery)
	rCounters.Add("PUT", ":id/tags", h.PutTags)

	// deprecated
	rCounters.Add("GET", ":id/data", h.GetData)
	rCounters.Add("POST", "data", h.PostData)
	rCounters.Add("POST", "stats/query", h.PostQuery)

	rAvailability := router.Router{
		Prefix: "/hawkular/metrics/availability/",
	}
	rAvailability.Add("GET", ":id/raw", h.GetData)
	rAvailability.Add("GET", ":id/stats", h.GetData)

	// Requests not handled by the routers will be forworded to BadRequest Handler
	rAlerts := router.Router{
		Prefix: "/hawkular/alerts/",
	}
	rAlerts.Add("GET", "status", h.GetAlertsStatus)
	rAlerts.Add("GET", "raw", h.GetAlerts)

	// Create the http handlers
	// logging handler
	logger := handler.Logger{}

	// set the authorization key
	if basicAuth != "" {
		authorizationKey = "Basic " + base64.StdEncoding.EncodeToString([]byte(basicAuth))
	}

	if bearerAuth != "" {
		authorizationKey = "Bearer " + bearerAuth
	}

	// add headers to response
	headers := handler.Headers{}

	// static a file server handler
	static := handler.Static{
		MediaPath: media,
	}

	// badrequest a BadRequest handler
	badrequest := handler.BadRequest{
		Verbose: verbose,
	}

	// concat all routers and add fallback handler
	if authorizationKey == "" {
		routers = handler.Append(
			&logger, &headers, &rM, &rGauges, &rCounters, &rAvailability, &rAlerts, &rRoot, &static, &badrequest)
	} else {
		// create an authentication handler
		authorization := handler.Authorization{
			// init the values:
			//    make the public path a regex once on init
			//    create the full Authorization header using the bearer token
			// this will prevent this values from re-calculate each http request
			PublicPathRegex: regexp.MustCompile(publicPath),
			Authorization:   authorizationKey,
			Verbose:         verbose,
		}

		routers = handler.Append(
			&logger, &authorization, &headers, &rM, &rGauges, &rCounters, &rAvailability, &rAlerts, &rRoot, &static, &badrequest)
	}

	// Create a list of middlwares
	decorators := []middleware.Decorator{}
	if gzip {
		decorators = append(decorators, middleware.GzipDecodeDecorator(), middleware.GzipEncodeDecorator())
	}

	// concat middlewars and routes (first logger until rRoot) with a fallback to BadRequest
	core := middleware.Append(routers, decorators...)

	// start serving http/s requests
	return RunServer(core)
}

// RunServer run the http/s server
func RunServer(core http.HandlerFunc) error {
	var port = viper.GetInt("port")
	var tls = viper.GetBool("tls")
	var cert = viper.GetString("cert")
	var key = viper.GetString("key")

	// Run the server
	srv := &http.Server{
		Addr:           fmt.Sprintf("0.0.0.0:%d", port),
		Handler:        core,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	if tls {
		log.Printf("Start server, listen on https://%+v", srv.Addr)
		log.Fatal(srv.ListenAndServeTLS(cert, key))
	} else {
		log.Printf("Start server, listen on http://%+v", srv.Addr)
		log.Fatal(srv.ListenAndServe())
	}

	return nil
}
<commit_msg>backend should be storage<commit_after>// Copyright 2016,2017 Yaacov Zamir <kobi.zamir@gmail.com>
// and other contributors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package server API REST server
package server

import (
	"encoding/base64"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"
	"time"

	"github.com/spf13/viper"

	"github.com/MohawkTSDB/mohawk/src/alerts"
	"github.com/MohawkTSDB/mohawk/src/server/handlers"
	"github.com/MohawkTSDB/mohawk/src/server/middleware"
	"github.com/MohawkTSDB/mohawk/src/server/router"
	"github.com/MohawkTSDB/mohawk/src/storage"
	"github.com/MohawkTSDB/mohawk/src/storage/example"
	"github.com/MohawkTSDB/mohawk/src/storage/memory"
	"github.com/MohawkTSDB/mohawk/src/storage/mongo"
	"github.com/MohawkTSDB/mohawk/src/storage/sqlite"
)

// VER the server version
const VER = "0.28.2"

// defaults
const defaultAPI = "0.21.0"
const publicPath = "^/hawkular/metrics/status$"

// BackendName Mohawk active storage
var BackendName string

// GetStatus return a json status struct
func GetStatus(w http.ResponseWriter, r *http.Request, argv map[string]string) {
	resTemplate := `{"MetricsService":"STARTED","Implementation-Version":"%s","MohawkVersion":"%s","MohawkStorage":"%s"}`
	res := fmt.Sprintf(resTemplate, defaultAPI, VER, BackendName)

	w.WriteHeader(200)
	fmt.Fprintln(w, res)
}

func printOptionsHelp() {
	fmt.Println("Storage options:")
	fmt.Println(sqlite.Storage{}.Help())
	fmt.Println(memory.Storage{}.Help())
	fmt.Println(mongo.Storage{}.Help())
}

// Serve run the REST API server
func Serve() error {
	var db storage.Storage
	var alertRules *alerts.AlertRules
	var routers http.HandlerFunc
	var authorizationKey string

	var backendQuery = viper.GetString("storage")
	var optionsQuery = viper.GetString("options")
	var verbose = viper.GetBool("verbose")
	var media = viper.GetString("media")
	var gzip = viper.GetBool("gzip")
	var bearerAuth = viper.GetString("bearer-auth")
	var basicAuth = viper.GetString("basic-auth")
	var alertsInterval = viper.GetInt("alerts-interval")
	var alertsServerURL = viper.GetString("alerts-server")
	var alertsServerMethod = viper.GetString("alerts-server-method")
	var alertsServerInsecure = viper.GetBool("alerts-server-insecure")
	var configAlerts = viper.ConfigFileUsed() != "" && viper.Get("alerts") != ""

	// if options is "help" print storage options help and exit
	if optionsQuery == "help" {
		printOptionsHelp()
		return nil
	}

	// Create and init the storage
	switch backendQuery {
	case "sqlite":
		db = &sqlite.Storage{}
	case "memory":
		db = &memory.Storage{}
	case "mongo":
		db = &mongo.Storage{}
	case "example":
		db = &example.Storage{}
	default:
		log.Fatal("Can't find storage:", backendQuery)
	}

	// parse options
	if options, err := url.ParseQuery(optionsQuery); err == nil {
		db.Open(options)
	} else {
		log.Fatal("Can't parse opetions:", optionsQuery)
	}

	// set global variables
	BackendName = db.Name()

	// Create alerts runner
	if configAlerts {
		// parse alert list from config yaml
		l := []*alerts.Alert{}
		viper.UnmarshalKey("alerts", &l)

		if len(l) > 0 {
			// creat and Init the alert handler
			alertRules = &alerts.AlertRules{
				Storage:        db,
				Verbose:        verbose,
				Alerts:         l,
				AlertsInterval: alertsInterval,
				ServerURL:      alertsServerURL,
				ServerMethod:   alertsServerMethod,
				ServerInsecure: alertsServerInsecure,
			}
			alertRules.Init()
		}
	}

	// h common variables to be used for the storage Handler functions
	// Storage the storage to use for metrics source
	h := handler.APIHhandler{
		Verbose: verbose,
		Storage: db,
		Alerts:  alertRules,
	}

	// Create the routers
	// Requests not handled by the routers will be forworded to BadRequest Handler
	rRoot := router.Router{
		Prefix: "/hawkular/metrics/",
	}
	// Root Routing table
	rRoot.Add("GET", "status", GetStatus)
	rRoot.Add("GET", "tenants", h.GetTenants)
	rRoot.Add("GET", "metrics", h.GetMetrics)

	// M (Global Metrics) Routing tables
	rM := router.Router{
		Prefix: "/hawkular/metrics/m/",
	}
	rM.Add("POST", "stats/query", h.PostMQuery)

	// Metrics Routing tables
	rGauges := router.Router{
		Prefix: "/hawkular/metrics/gauges/",
	}
	rGauges.Add("GET", ":id/raw", h.GetData)
	rGauges.Add("GET", ":id/stats", h.GetData)
	rGauges.Add("POST", "raw", h.PostData)
	rGauges.Add("POST", "raw/query", h.PostQuery)
	rGauges.Add("PUT", "tags", h.PutMultiTags)
	rGauges.Add("PUT", ":id/tags", h.PutTags)
	rGauges.Add("DELETE", ":id/raw", h.DeleteData)
	rGauges.Add("DELETE", ":id/tags/:tags", h.DeleteTags)

	// deprecated
	rGauges.Add("GET", ":id/data", h.GetData)
	rGauges.Add("POST", "data", h.PostData)
	rGauges.Add("POST", "stats/query", h.PostQuery)

	rCounters := router.Router{
		Prefix: "/hawkular/metrics/counters/",
	}
	rCounters.Add("GET", ":id/raw", h.GetData)
	rCounters.Add("GET", ":id/stats", h.GetData)
	rCounters.Add("POST", "raw", h.PostData)
	rCounters.Add("POST", "raw/query", h.PostQuery)
	rCounters.Add("PUT", ":id/tags", h.PutTags)

	// deprecated
	rCounters.Add("GET", ":id/data", h.GetData)
	rCounters.Add("POST", "data", h.PostData)
	rCounters.Add("POST", "stats/query", h.PostQuery)

	rAvailability := router.Router{
		Prefix: "/hawkular/metrics/availability/",
	}
	rAvailability.Add("GET", ":id/raw", h.GetData)
	rAvailability.Add("GET", ":id/stats", h.GetData)

	// Requests not handled by the routers will be forworded to BadRequest Handler
	rAlerts := router.Router{
		Prefix: "/hawkular/alerts/",
	}
	rAlerts.Add("GET", "status", h.GetAlertsStatus)
	rAlerts.Add("GET", "raw", h.GetAlerts)

	// Create the http handlers
	// logging handler
	logger := handler.Logger{}

	// set the authorization key
	if basicAuth != "" {
		authorizationKey = "Basic " + base64.StdEncoding.EncodeToString([]byte(basicAuth))
	}

	if bearerAuth != "" {
		authorizationKey = "Bearer " + bearerAuth
	}

	// add headers to response
	headers := handler.Headers{}

	// static a file server handler
	static := handler.Static{
		MediaPath: media,
	}

	// badrequest a BadRequest handler
	badrequest := handler.BadRequest{
		Verbose: verbose,
	}

	// concat all routers and add fallback handler
	if authorizationKey == "" {
		routers = handler.Append(
			&logger, &headers, &rM, &rGauges, &rCounters, &rAvailability, &rAlerts, &rRoot, &static, &badrequest)
	} else {
		// create an authentication handler
		authorization := handler.Authorization{
			// init the values:
			//    make the public path a regex once on init
			//    create the full Authorization header using the bearer token
			// this will prevent this values from re-calculate each http request
			PublicPathRegex: regexp.MustCompile(publicPath),
			Authorization:   authorizationKey,
			Verbose:         verbose,
		}

		routers = handler.Append(
			&logger, &authorization, &headers, &rM, &rGauges, &rCounters, &rAvailability, &rAlerts, &rRoot, &static, &badrequest)
	}

	// Create a list of middlwares
	decorators := []middleware.Decorator{}
	if gzip {
		decorators = append(decorators, middleware.GzipDecodeDecorator(), middleware.GzipEncodeDecorator())
	}

	// concat middlewars and routes (first logger until rRoot) with a fallback to BadRequest
	core := middleware.Append(routers, decorators...)

	// start serving http/s requests
	return RunServer(core)
}

// RunServer run the http/s server
func RunServer(core http.HandlerFunc) error {
	var port = viper.GetInt("port")
	var tls = viper.GetBool("tls")
	var cert = viper.GetString("cert")
	var key = viper.GetString("key")

	// Run the server
	srv := &http.Server{
		Addr:           fmt.Sprintf("0.0.0.0:%d", port),
		Handler:        core,
		ReadTimeout:    10 * time.Second,
		WriteTimeout:   10 * time.Second,
		MaxHeaderBytes: 1 << 20,
	}
	if tls {
		log.Printf("Start server, listen on https://%+v", srv.Addr)
		log.Fatal(srv.ListenAndServeTLS(cert, key))
	} else {
		log.Printf("Start server, listen on http://%+v", srv.Addr)
		log.Fatal(srv.ListenAndServe())
	}

	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"admin"
	"api/http"
	"cluster"
	log "code.google.com/p/log4go"
	"configuration"
	"coordinator"
	"datastore"
	"wal"
)

type Server struct {
	RaftServer     *coordinator.RaftServer
	Db             datastore.Datastore
	ProtobufServer *coordinator.ProtobufServer
	ClusterConfig  *cluster.ClusterConfiguration
	HttpApi        *http.HttpServer
	AdminServer    *admin.HttpServer
	Coordinator    coordinator.Coordinator
	Config         *configuration.Configuration
	RequestHandler *coordinator.ProtobufRequestHandler
	stopped        bool
	writeLog       *wal.WAL
}

func NewServer(config *configuration.Configuration) (*Server, error) {
	log.Info("Opening database at %s", config.DataDir)
	db, err := datastore.NewLevelDbDatastore(config.DataDir, config.LevelDbMaxOpenFiles)
	if err != nil {
		return nil, err
	}

	shardDb, err := datastore.NewLevelDbShardDatastore(config)
	if err != nil {
		return nil, err
	}

	newClient := func(connectString string) cluster.ServerConnection {
		return coordinator.NewProtobufClient(connectString, config.ProtobufTimeout.Duration)
	}
	writeLog, err := wal.NewWAL(config)
	if err != nil {
		return nil, err
	}

	clusterConfig := cluster.NewClusterConfiguration(config, writeLog, shardDb, newClient)
	raftServer := coordinator.NewRaftServer(config, clusterConfig)
	clusterConfig.SetShardCreator(raftServer)
	clusterConfig.CreateFutureShardsAutomaticallyBeforeTimeComes()

	coord := coordinator.NewCoordinatorImpl(db, raftServer, clusterConfig)
	requestHandler := coordinator.NewProtobufRequestHandler(db, coord, clusterConfig)
	protobufServer := coordinator.NewProtobufServer(config.ProtobufPortString(), requestHandler)

	raftServer.AssignCoordinator(coord)
	httpApi := http.NewHttpServer(config.ApiHttpPortString(), config.AdminAssetsDir, coord, coord, clusterConfig, raftServer)
	httpApi.EnableSsl(config.ApiHttpSslPortString(), config.ApiHttpCertPath)
	adminServer := admin.NewHttpServer(config.AdminAssetsDir, config.AdminHttpPortString())

	return &Server{
		RaftServer:     raftServer,
		Db:             db,
		ProtobufServer: protobufServer,
		ClusterConfig:  clusterConfig,
		HttpApi:        httpApi,
		Coordinator:    coord,
		AdminServer:    adminServer,
		Config:         config,
		RequestHandler: requestHandler,
		writeLog:       writeLog}, nil
}

func (self *Server) ListenAndServe() error {
	go self.ProtobufServer.ListenAndServe()

	err := self.RaftServer.ListenAndServe()
	if err != nil {
		return err
	}

	log.Info("Waiting for local server to be added")
	self.ClusterConfig.WaitForLocalServerLoaded()
	self.writeLog.SetServerId(self.ClusterConfig.ServerId())

	log.Info("Recovering from log...")
	err = self.ClusterConfig.RecoverFromWAL()
	if err != nil {
		return err
	}
	log.Info("recovered")

	err = self.Coordinator.(*coordinator.CoordinatorImpl).ConnectToProtobufServers(self.Config.ProtobufConnectionString())
	if err != nil {
		return err
	}
	log.Info("Starting admin interface on port %d", self.Config.AdminHttpPort)
	go self.AdminServer.ListenAndServe()
	log.Info("Starting Http Api server on port %d", self.Config.ApiHttpPort)
	self.HttpApi.ListenAndServe()
	return nil
}

func (self *Server) Stop() {
	if self.stopped {
		return
	}
	self.stopped = true
	self.RaftServer.Close()
	self.Db.Close()
	self.HttpApi.Close()
	self.ProtobufServer.Close()
	self.AdminServer.Close()
	// TODO: close admin server and protobuf client connections
	log.Info("Stopping server")
}
<commit_msg>close the write log on exit<commit_after>package server

import (
	"admin"
	"api/http"
	"cluster"
	log "code.google.com/p/log4go"
	"configuration"
	"coordinator"
	"datastore"
	"wal"
)

type Server struct {
	RaftServer     *coordinator.RaftServer
	Db             datastore.Datastore
	ProtobufServer *coordinator.ProtobufServer
	ClusterConfig  *cluster.ClusterConfiguration
	HttpApi        *http.HttpServer
	AdminServer    *admin.HttpServer
	Coordinator    coordinator.Coordinator
	Config         *configuration.Configuration
	RequestHandler *coordinator.ProtobufRequestHandler
	stopped        bool
	writeLog       *wal.WAL
}

func NewServer(config *configuration.Configuration) (*Server, error) {
	log.Info("Opening database at %s", config.DataDir)
	db, err := datastore.NewLevelDbDatastore(config.DataDir, config.LevelDbMaxOpenFiles)
	if err != nil {
		return nil, err
	}

	shardDb, err := datastore.NewLevelDbShardDatastore(config)
	if err != nil {
		return nil, err
	}

	newClient := func(connectString string) cluster.ServerConnection {
		return coordinator.NewProtobufClient(connectString, config.ProtobufTimeout.Duration)
	}
	writeLog, err := wal.NewWAL(config)
	if err != nil {
		return nil, err
	}

	clusterConfig := cluster.NewClusterConfiguration(config, writeLog, shardDb, newClient)
	raftServer := coordinator.NewRaftServer(config, clusterConfig)
	clusterConfig.SetShardCreator(raftServer)
	clusterConfig.CreateFutureShardsAutomaticallyBeforeTimeComes()

	coord := coordinator.NewCoordinatorImpl(db, raftServer, clusterConfig)
	requestHandler := coordinator.NewProtobufRequestHandler(db, coord, clusterConfig)
	protobufServer := coordinator.NewProtobufServer(config.ProtobufPortString(), requestHandler)

	raftServer.AssignCoordinator(coord)
	httpApi := http.NewHttpServer(config.ApiHttpPortString(), config.AdminAssetsDir, coord, coord, clusterConfig, raftServer)
	httpApi.EnableSsl(config.ApiHttpSslPortString(), config.ApiHttpCertPath)
	adminServer := admin.NewHttpServer(config.AdminAssetsDir, config.AdminHttpPortString())

	return &Server{
		RaftServer:     raftServer,
		Db:             db,
		ProtobufServer: protobufServer,
		ClusterConfig:  clusterConfig,
		HttpApi:        httpApi,
		Coordinator:    coord,
		AdminServer:    adminServer,
		Config:         config,
		RequestHandler: requestHandler,
		writeLog:       writeLog}, nil
}

func (self *Server) ListenAndServe() error {
	go self.ProtobufServer.ListenAndServe()

	err := self.RaftServer.ListenAndServe()
	if err != nil {
		return err
	}

	log.Info("Waiting for local server to be added")
	self.ClusterConfig.WaitForLocalServerLoaded()
	self.writeLog.SetServerId(self.ClusterConfig.ServerId())

	log.Info("Recovering from log...")
	err = self.ClusterConfig.RecoverFromWAL()
	if err != nil {
		return err
	}
	log.Info("recovered")

	err = self.Coordinator.(*coordinator.CoordinatorImpl).ConnectToProtobufServers(self.Config.ProtobufConnectionString())
	if err != nil {
		return err
	}
	log.Info("Starting admin interface on port %d", self.Config.AdminHttpPort)
	go self.AdminServer.ListenAndServe()
	log.Info("Starting Http Api server on port %d", self.Config.ApiHttpPort)
	self.HttpApi.ListenAndServe()
	return nil
}

func (self *Server) Stop() {
	if self.stopped {
		return
	}
	self.stopped = true
	self.RaftServer.Close()
	self.Db.Close()
	self.HttpApi.Close()
	self.ProtobufServer.Close()
	self.AdminServer.Close()
	self.writeLog.Close()
	// TODO: close admin server and protobuf client connections
	log.Info("Stopping server")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/user"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/nsf/termbox-go"
)

const (
	_  = iota
	KB = 1 << (10 * iota)
	MB
	GB
	TB
	PB
)

const usage = `Usage: jtop [options]

Options:
  -d, --delay    delay between updates
  -k, --kernel   show kernel threads
  -p, --pids     filter by PID (comma-separated list)
  -s, --sort     sort by the specified column
  -t, --tree     display process list as tree
  -u, --users    filter by User (comma-separated list)
      --verbose  show full command line with arguments
`

var (
	delayFlag   time.Duration
	kernelFlag  bool
	pidsFlag    string
	sortFlag    string
	treeFlag    bool
	usersFlag   string
	verboseFlag bool
)

func exit(message string) {
	fmt.Fprintln(os.Stderr, message)
	flag.Usage()
	os.Exit(1)
}

func signalSelf(sig syscall.Signal) {
	if err := syscall.Kill(os.Getpid(), sig); err != nil {
		panic(err)
	}
}

func validateDelayFlag() {
	if delayFlag <= 0 {
		exit("flag error: delay must be positive")
	}
}

func validatePidsFlag() {
	if pidsFlag == "" {
		return
	}

	pids := strings.Split(pidsFlag, ",")
	for _, value := range pids {
		if pid, err := strconv.ParseUint(value, 10, 64); err != nil {
			message := fmt.Sprintf("flag error: %s is not a valid PID", value)
			exit(message)
		} else {
			PidWhitelist = append(PidWhitelist, pid)
		}
	}
}

func validateSortFlag() {
	for _, column := range Columns {
		if sortFlag == column.Title {
			return
		}
	}
	message := fmt.Sprintf("flag error: %s is not a valid sort column", sortFlag)
	exit(message)
}

func validateUsersFlag() {
	if usersFlag == "" {
		return
	}

	users := strings.Split(usersFlag, ",")
	for _, username := range users {
		if user, err := user.Lookup(username); err != nil {
			message := fmt.Sprintf("flag error: user %s does not exist", username)
			exit(message)
		} else {
			UserWhitelist = append(UserWhitelist, user)
		}
	}
}

func validateFlags() {
	validateDelayFlag()
	validatePidsFlag()
	validateSortFlag()
	validateUsersFlag()
}

func init() {
	defaultDelay := time.Duration(1500 * time.Millisecond)
	flag.DurationVar(&delayFlag, "d", defaultDelay, "")
	flag.DurationVar(&delayFlag, "delay", defaultDelay, "")

	flag.BoolVar(&kernelFlag, "k", false, "")
	flag.BoolVar(&kernelFlag, "kernel", false, "")

	flag.StringVar(&pidsFlag, "p", "", "")
	flag.StringVar(&pidsFlag, "pids", "", "")

	defaultSort := CPUPercentColumn.Title
	flag.StringVar(&sortFlag, "s", defaultSort, "")
	flag.StringVar(&sortFlag, "sort", defaultSort, "")

	flag.BoolVar(&treeFlag, "t", false, "")
	flag.BoolVar(&treeFlag, "tree", false, "")

	flag.StringVar(&usersFlag, "u", "", "")
	flag.StringVar(&usersFlag, "users", "", "")

	flag.BoolVar(&verboseFlag, "verbose", false, "")

	flag.Usage = func() {
		fmt.Fprint(os.Stdout, usage)
	}
}

func termboxInit() {
	if err := termbox.Init(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
}

func main() {
	flag.Parse()
	validateFlags()

	termboxInit()
	defer termbox.Close()

	events := make(chan termbox.Event)
	go func() {
		for {
			events <- termbox.PollEvent()
		}
	}()

	ticker := time.Tick(delayFlag)
	monitor := NewMonitor()
	monitor.Update()
	ui := NewUI(monitor)

	for {
		ui.Draw()

		select {
		case <-ticker:
			monitor.Update()

		case ev := <-events:
			if ev.Type == termbox.EventKey {
				switch {
				case ev.Ch == 'q' || ev.Key == termbox.KeyCtrlC:
					return
				case ev.Ch == 'j' || ev.Key == termbox.KeyArrowDown:
					ui.HandleDown()
				case ev.Ch == 'k' || ev.Key == termbox.KeyArrowUp:
					ui.HandleUp()
				case ev.Ch == 'v':
					verboseFlag = !verboseFlag
				case ev.Key == termbox.KeyCtrlD:
					ui.HandleCtrlD()
				case ev.Key == termbox.KeyCtrlU:
					ui.HandleCtrlU()
				case ev.Key == termbox.KeyCtrlZ:
					termbox.Close()
					signalSelf(syscall.SIGTSTP)
					termboxInit()
				}
			} else if ev.Type == termbox.EventResize {
				ui.HandleResize()
			}
		}
	}
}
<commit_msg>Toggle tree view with 't'<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/user"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/nsf/termbox-go"
)

const (
	_  = iota
	KB = 1 << (10 * iota)
	MB
	GB
	TB
	PB
)

const usage = `Usage: jtop [options]

Options:
  -d, --delay    delay between updates
  -k, --kernel   show kernel threads
  -p, --pids     filter by PID (comma-separated list)
  -s, --sort     sort by the specified column
  -t, --tree     display process list as tree
  -u, --users    filter by User (comma-separated list)
      --verbose  show full command line with arguments
`

var (
	delayFlag   time.Duration
	kernelFlag  bool
	pidsFlag    string
	sortFlag    string
	treeFlag    bool
	usersFlag   string
	verboseFlag bool
)

func exit(message string) {
	fmt.Fprintln(os.Stderr, message)
	flag.Usage()
	os.Exit(1)
}

func signalSelf(sig syscall.Signal) {
	if err := syscall.Kill(os.Getpid(), sig); err != nil {
		panic(err)
	}
}

func validateDelayFlag() {
	if delayFlag <= 0 {
		exit("flag error: delay must be positive")
	}
}

func validatePidsFlag() {
	if pidsFlag == "" {
		return
	}

	pids := strings.Split(pidsFlag, ",")
	for _, value := range pids {
		if pid, err := strconv.ParseUint(value, 10, 64); err != nil {
			message := fmt.Sprintf("flag error: %s is not a valid PID", value)
			exit(message)
		} else {
			PidWhitelist = append(PidWhitelist, pid)
		}
	}
}

func validateSortFlag() {
	for _, column := range Columns {
		if sortFlag == column.Title {
			return
		}
	}
	message := fmt.Sprintf("flag error: %s is not a valid sort column", sortFlag)
	exit(message)
}

func validateUsersFlag() {
	if usersFlag == "" {
		return
	}

	users := strings.Split(usersFlag, ",")
	for _, username := range users {
		if user, err := user.Lookup(username); err != nil {
			message := fmt.Sprintf("flag error: user %s does not exist", username)
			exit(message)
		} else {
			UserWhitelist = append(UserWhitelist, user)
		}
	}
}

func validateFlags() {
	validateDelayFlag()
	validatePidsFlag()
	validateSortFlag()
	validateUsersFlag()
}

func init() {
	defaultDelay := time.Duration(1500 * time.Millisecond)
	flag.DurationVar(&delayFlag, "d", defaultDelay, "")
	flag.DurationVar(&delayFlag, "delay", defaultDelay, "")

	flag.BoolVar(&kernelFlag, "k", false, "")
	flag.BoolVar(&kernelFlag, "kernel", false, "")

	flag.StringVar(&pidsFlag, "p", "", "")
	flag.StringVar(&pidsFlag, "pids", "", "")

	defaultSort := CPUPercentColumn.Title
	flag.StringVar(&sortFlag, "s", defaultSort, "")
	flag.StringVar(&sortFlag, "sort", defaultSort, "")

	flag.BoolVar(&treeFlag, "t", false, "")
	flag.BoolVar(&treeFlag, "tree", false, "")

	flag.StringVar(&usersFlag, "u", "", "")
	flag.StringVar(&usersFlag, "users", "", "")

	flag.BoolVar(&verboseFlag, "verbose", false, "")

	flag.Usage = func() {
		fmt.Fprint(os.Stdout, usage)
	}
}

func termboxInit() {
	if err := termbox.Init(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(2)
	}
}

func main() {
	flag.Parse()
	validateFlags()

	termboxInit()
	defer termbox.Close()

	events := make(chan termbox.Event)
	go func() {
		for {
			events <- termbox.PollEvent()
		}
	}()

	ticker := time.Tick(delayFlag)
	monitor := NewMonitor()
	monitor.Update()
	ui := NewUI(monitor)

	for {
		ui.Draw()

		select {
		case <-ticker:
			monitor.Update()

		case ev := <-events:
			if ev.Type == termbox.EventKey {
				switch {
				case ev.Ch == 'q' || ev.Key == termbox.KeyCtrlC:
					return
				case ev.Ch == 'j' || ev.Key == termbox.KeyArrowDown:
					ui.HandleDown()
				case ev.Ch == 'k' || ev.Key == termbox.KeyArrowUp:
					ui.HandleUp()
				case ev.Ch == 't':
					treeFlag = !treeFlag
				case ev.Ch == 'v':
					verboseFlag = !verboseFlag
				case ev.Key == termbox.KeyCtrlD:
					ui.HandleCtrlD()
				case ev.Key == termbox.KeyCtrlU:
					ui.HandleCtrlU()
				case ev.Key == termbox.KeyCtrlZ:
					termbox.Close()
					signalSelf(syscall.SIGTSTP)
					termboxInit()
				}
			} else if ev.Type == termbox.EventResize {
				ui.HandleResize()
			}
		}
	}
}
<|endoftext|>


<commit_before>package kana

import (
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

var consonants []string = []string{"b", "d", "f", "g", "h", "j", "k", "l", "m", "p", "r", "s", "t", "w", "z"}

var hiragana_re = regexp.MustCompile(`ん([あいうえおなにぬねの])`)
var katakana_re = regexp.MustCompile(`ン([アイウエオナニヌネノ])`)

var kanaToRomajiTrie *Trie
var romajiToHiraganaTrie *Trie
var romajiToKatakanaTrie *Trie

func Initialize() {
	/*
		Build the Hiragana + Katakana trie.

		Because there is no overlap between the hiragana and katakana sets,
		they both use the same trie without conflict. Nice bonus!
	*/
	kanaToRomajiTrie = newTrie()
	romajiToHiraganaTrie = newTrie()
	romajiToKatakanaTrie = newTrie()

	tables := []string{HiraganaTable, KatakanaTable}
	for t, table := range tables {
		rows := strings.Split(table, "\n")
		colNames := strings.Split(string(rows[0]), "\t")[1:]
		for _, row := range rows[1:] {
			cols := strings.Split(string(row), "\t")
			rowName := cols[0]
			for i, kana := range cols[1:] {
				value := rowName + colNames[i]
				kanas := strings.Split(kana, "/")
				for _, singleKana := range kanas {
					if singleKana != "" {
						// add to tries
						kanaToRomajiTrie.insert(singleKana, value)
						if t == 0 {
							romajiToHiraganaTrie.insert(value, singleKana)
						} else if t == 1 {
							romajiToKatakanaTrie.insert(value, singleKana)
						}
					}
				}
			}
		}
	}
}

func KanaToRomaji(kana string) (romaji string) {

	// unfortunate hack to deal with double n's
	romaji = hiragana_re.ReplaceAllString(kana, "nn$1")
	romaji = katakana_re.ReplaceAllString(romaji, "nn$1")

	romaji = kanaToRomajiTrie.convert(romaji)

	// do some post-processing for the tsu and stripe characters
	// maybe a bit of a hacky solution - how can we improve?
	// (they act more like punctuation)
	tsus := []string{"っ", "ッ"}
	for _, tsu := range tsus {
		if strings.Index(romaji, tsu) > -1 {
			for _, c := range romaji {
				ch := string(c)
				if ch == tsu {
					i := strings.Index(romaji, ch)
					runeSize := len(ch)
					followingLetter, _ := utf8.DecodeRuneInString(romaji[i+runeSize:])
					followingLetterStr := string(followingLetter)
					if followingLetterStr != tsu {
						romaji = strings.Replace(romaji, tsu, followingLetterStr, 1)
					} else {
						romaji = strings.Replace(romaji, tsu, "", 1)
					}
				}
			}
		}
	}

	line := "ー"
	for i := strings.Index(romaji, line); i > -1; i = strings.Index(romaji, line) {
		if i > 0 {
			romaji = strings.Replace(romaji, line, "-", 1)
		} else {
			romaji = strings.Replace(romaji, line, "", 1)
		}
	}
	return romaji
}

func replace_tsus(romaji string, tsu string) (result string) {
	result = romaji
	for _, consonant := range consonants {
		result = strings.Replace(result, consonant+consonant, tsu+consonant, -1)
	}
	return result
}

func replace_ns(romaji string, n string) (result string) {
	result = romaji
	result = strings.Replace(result, "nn", n, -1)
	return result
}

func RomajiToHiragana(romaji string) (hiragana string) {
	romaji = strings.Replace(romaji, "-", "ー", -1)
	romaji = replace_tsus(romaji, "っ")
	romaji = replace_ns(romaji, "ん")
	hiragana = romajiToHiraganaTrie.convert(romaji)
	return hiragana
}

func RomajiToKatakana(romaji string) (katakana string) {
	romaji = strings.Replace(romaji, "-", "ー", -1)
	// convert double consonants to little tsus first
	romaji = replace_tsus(romaji, "ッ")
	romaji = replace_ns(romaji, "ン")
	katakana = romajiToKatakanaTrie.convert(romaji)
	return katakana
}

func IsLatin(s string) bool {
	isLatin := true
	runeForm := []rune(s)
	for _, r := range runeForm {
		isLatin = isLatin && unicode.IsOneOf([]*unicode.RangeTable{unicode.Latin, unicode.ASCII_Hex_Digit, unicode.White_Space, unicode.Hyphen}, r)
		if !isLatin {
			return isLatin
		}
	}
	return isLatin
}

func IsKana(s string) bool {
	isKana := true
	runeForm := []rune(s)
	for _, r := range runeForm {
		isKana = isKana && unicode.IsOneOf([]*unicode.RangeTable{unicode.Hiragana, unicode.Katakana, unicode.Hyphen, unicode.Diacritic}, r)
		if !isKana {
			return isKana
		}
	}
	return isKana
}

func IsKanji(s string) bool {
	isKanji := true
	runeForm := []rune(s)
	for _, r := range runeForm {
		isKanji = isKanji && unicode.IsOneOf([]*unicode.RangeTable{unicode.Ideographic}, r)
		if !isKanji {
			return isKanji
		}
	}
	return isKanji
}

func replaceAll(haystack string, needles []string, replacements []string) (replaced string) {
	replaced = haystack
	for i := range needles {
		replaced = strings.Replace(replaced, needles[i], replacements[i], -1)
	}
	return replaced
}

func NormalizeRomaji(s string) (romaji string) {
	// transform romaji input to one specific standard form,
	// which should be as close as possible to hiragana so that
	// this library gives correct output when transforming to
	// hiragana / katakana

	romaji = s
	romaji = strings.ToLower(romaji)
	romaji = replaceAll(
		romaji,
		[]string{"ā", "ē", "ī", "ō", "ū", "ee", "uu"},
		[]string{"a-", "ei", "ii", "oo", "u-", "ei", "u-"},
	)

	return romaji
}
<commit_msg>remove unnecessary []string in declaration<commit_after>package kana

import (
	"regexp"
	"strings"
	"unicode"
	"unicode/utf8"
)

var consonants = []string{"b", "d", "f", "g", "h", "j", "k", "l", "m", "p", "r", "s", "t", "w", "z"}

var hiragana_re = regexp.MustCompile(`ん([あいうえおなにぬねの])`)
var katakana_re = regexp.MustCompile(`ン([アイウエオナニヌネノ])`)

var kanaToRomajiTrie *Trie
var romajiToHiraganaTrie *Trie
var romajiToKatakanaTrie *Trie

func Initialize() {
	/*
		Build the Hiragana + Katakana trie.

		Because there is no overlap between the hiragana and katakana sets,
		they both use the same trie without conflict. Nice bonus!
	*/
	kanaToRomajiTrie = newTrie()
	romajiToHiraganaTrie = newTrie()
	romajiToKatakanaTrie = newTrie()

	tables := []string{HiraganaTable, KatakanaTable}
	for t, table := range tables {
		rows := strings.Split(table, "\n")
		colNames := strings.Split(string(rows[0]), "\t")[1:]
		for _, row := range rows[1:] {
			cols := strings.Split(string(row), "\t")
			rowName := cols[0]
			for i, kana := range cols[1:] {
				value := rowName + colNames[i]
				kanas := strings.Split(kana, "/")
				for _, singleKana := range kanas {
					if singleKana != "" {
						// add to tries
						kanaToRomajiTrie.insert(singleKana, value)
						if t == 0 {
							romajiToHiraganaTrie.insert(value, singleKana)
						} else if t == 1 {
							romajiToKatakanaTrie.insert(value, singleKana)
						}
					}
				}
			}
		}
	}
}

func KanaToRomaji(kana string) (romaji string) {

	// unfortunate hack to deal with double n's
	romaji = hiragana_re.ReplaceAllString(kana, "nn$1")
	romaji = katakana_re.ReplaceAllString(romaji, "nn$1")

	romaji = kanaToRomajiTrie.convert(romaji)

	// do some post-processing for the tsu and stripe characters
	// maybe a bit of a hacky solution - how can we improve?
	// (they act more like punctuation)
	tsus := []string{"っ", "ッ"}
	for _, tsu := range tsus {
		if strings.Index(romaji, tsu) > -1 {
			for _, c := range romaji {
				ch := string(c)
				if ch == tsu {
					i := strings.Index(romaji, ch)
					runeSize := len(ch)
					followingLetter, _ := utf8.DecodeRuneInString(romaji[i+runeSize:])
					followingLetterStr := string(followingLetter)
					if followingLetterStr != tsu {
						romaji = strings.Replace(romaji, tsu, followingLetterStr, 1)
					} else {
						romaji = strings.Replace(romaji, tsu, "", 1)
					}
				}
			}
		}
	}

	line := "ー"
	for i := strings.Index(romaji, line); i > -1; i = strings.Index(romaji, line) {
		if i > 0 {
			romaji = strings.Replace(romaji, line, "-", 1)
		} else {
			romaji = strings.Replace(romaji, line, "", 1)
		}
	}
	return romaji
}

func replace_tsus(romaji string, tsu string) (result string) {
	result = romaji
	for _, consonant := range consonants {
		result = strings.Replace(result, consonant+consonant, tsu+consonant, -1)
	}
	return result
}

func replace_ns(romaji string, n string) (result string) {
	result = romaji
	result = strings.Replace(result, "nn", n, -1)
	return result
}

func RomajiToHiragana(romaji string) (hiragana string) {
	romaji = strings.Replace(romaji, "-", "ー", -1)
	romaji = replace_tsus(romaji, "っ")
	romaji = replace_ns(romaji, "ん")
	hiragana = romajiToHiraganaTrie.convert(romaji)
	return hiragana
}

func RomajiToKatakana(romaji string) (katakana string) {
	romaji = strings.Replace(romaji, "-", "ー", -1)
	// convert double consonants to little tsus first
	romaji = replace_tsus(romaji, "ッ")
	romaji = replace_ns(romaji, "ン")
	katakana = romajiToKatakanaTrie.convert(romaji)
	return katakana
}

func IsLatin(s string) bool {
	isLatin := true
	runeForm := []rune(s)
	for _, r := range runeForm {
		isLatin = isLatin && unicode.IsOneOf([]*unicode.RangeTable{unicode.Latin, unicode.ASCII_Hex_Digit, unicode.White_Space, unicode.Hyphen}, r)
		if !isLatin {
			return isLatin
		}
	}
	return isLatin
}

func IsKana(s string) bool {
	isKana := true
	runeForm := []rune(s)
	for _, r := range runeForm {
		isKana = isKana && unicode.IsOneOf([]*unicode.RangeTable{unicode.Hiragana, unicode.Katakana, unicode.Hyphen, unicode.Diacritic}, r)
		if !isKana {
			return isKana
		}
	}
	return isKana
}

func IsKanji(s string) bool {
	isKanji := true
	runeForm := []rune(s)
	for _, r := range runeForm {
		isKanji = isKanji && unicode.IsOneOf([]*unicode.RangeTable{unicode.Ideographic}, r)
		if !isKanji {
			return isKanji
		}
	}
	return isKanji
}

func replaceAll(haystack string, needles []string, replacements []string) (replaced string) {
	replaced = haystack
	for i := range needles {
		replaced = strings.Replace(replaced, needles[i], replacements[i], -1)
	}
	return replaced
}

func NormalizeRomaji(s string) (romaji string) {
	// transform romaji input to one specific standard form,
	// which should be as close as possible to hiragana so that
	// this library gives correct output when transforming to
	// hiragana / katakana

	romaji = s
	romaji = strings.ToLower(romaji)
	romaji = replaceAll(
		romaji,
		[]string{"ā", "ē", "ī", "ō", "ū", "ee", "uu"},
		[]string{"a-", "ei", "ii", "oo", "u-", "ei", "u-"},
	)

	return romaji
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/schollz/versionedtext"
)

// Page is the basic struct
type Page struct {
	Name                    string
	Text                    versionedtext.VersionedText
	RenderedPage            string
	IsLocked                bool
	PassphraseToUnlock      string
	IsEncrypted             bool
	IsPrimedForSelfDestruct bool
}

func Open(name string) (p *Page) {
	name = strings.ToLower(name) // no case
	p = new(Page)
	p.Name = name
	p.Text = versionedtext.NewVersionedText("")
	p.Render()
	bJSON, err := ioutil.ReadFile(path.Join(pathToData, encodeToBase32(name)+".json"))
	if err != nil {
		return
	}
	err = json.Unmarshal(bJSON, &p)
	if err != nil {
		p = new(Page)
	}
	return p
}

func (p *Page) Update(newText string) error {
	p.Text.Update(newText)
	p.Render()
	return p.Save()
}

func (p *Page) Render() {
	if p.IsEncrypted {
		p.RenderedPage = "" + p.Text.GetCurrent() + ""
		return
	}

	// Convert [[page]] to [page](/page/view)
	r, _ := regexp.Compile("\\[\\[(.*?)\\]\\]")
	currentText := p.Text.GetCurrent()
	for _, s := range r.FindAllString(currentText, -1) {
		currentText = strings.Replace(currentText, s, "["+s[2:len(s)-2]+"](/"+s[2:len(s)-2]+"/view)", 1)
	}
	p.Text.Update(currentText)
	p.RenderedPage = MarkdownToHtml(p.Text.GetCurrent())
}

func (p *Page) Save() error {
	bJSON, err := json.MarshalIndent(p, "", " ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path.Join(pathToData, encodeToBase32(p.Name)+".json"), bJSON, 0755)
}

func (p *Page) Erase() error {
	return os.Remove(path.Join(pathToData, encodeToBase32(p.Name)+".json"))
}
<commit_msg>Normalize filenames when writing<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/schollz/versionedtext"
)

// Page is the basic struct
type Page struct {
	Name                    string
	Text                    versionedtext.VersionedText
	RenderedPage            string
	IsLocked                bool
	PassphraseToUnlock      string
	IsEncrypted             bool
	IsPrimedForSelfDestruct bool
}

func Open(name string) (p *Page) {
	p = new(Page)
	p.Name = name
	p.Text = versionedtext.NewVersionedText("")
	p.Render()
	bJSON, err := ioutil.ReadFile(path.Join(pathToData, encodeToBase32(strings.ToLower(name))+".json"))
	if err != nil {
		return
	}
	err = json.Unmarshal(bJSON, &p)
	if err != nil {
		p = new(Page)
	}
	return p
}

func (p *Page) Update(newText string) error {
	p.Text.Update(newText)
	p.Render()
	return p.Save()
}

func (p *Page) Render() {
	if p.IsEncrypted {
		p.RenderedPage = "" + p.Text.GetCurrent() + ""
		return
	}

	// Convert [[page]] to [page](/page/view)
	r, _ := regexp.Compile("\\[\\[(.*?)\\]\\]")
	currentText := p.Text.GetCurrent()
	for _, s := range r.FindAllString(currentText, -1) {
		currentText = strings.Replace(currentText, s, "["+s[2:len(s)-2]+"](/"+s[2:len(s)-2]+"/view)", 1)
	}
	p.Text.Update(currentText)
	p.RenderedPage = MarkdownToHtml(p.Text.GetCurrent())
}

func (p *Page) Save() error {
	bJSON, err := json.MarshalIndent(p, "", " ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path.Join(pathToData, encodeToBase32(strings.ToLower(p.Name))+".json"), bJSON, 0755)
}

func (p *Page) Erase() error {
	return os.Remove(path.Join(pathToData, encodeToBase32(p.Name)+".json"))
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"time"
)

// Page is the short to long url conversion object
type Page struct {
	ID        int8
	ShortURL  string
	LongURL   string
	CreatedAt time.Time
}

// DB Based
func (p *Page) save() error {
	_, err := db.Exec("INSERT INTO pages (shorturl, longurl) VALUES ($1, $2)", p.ShortURL, p.LongURL)
	return err
}

func queryPage(path string) (*Page, error) {
	var id int8
	var shorturl string
	var longurl string
	var createdAt string
	err := db.QueryRow("SELECT * FROM pages WHERE shorturl = $1", path).Scan(&id, &shorturl, &longurl, &createdAt)
	if err != nil {
		return nil, err
	}
	return &Page{ID: id, LongURL: longurl, ShortURL: shorturl}, nil
}

// File Store Based
func (p *Page) saveFile() error {
	filename := p.shortURLHASH()
	return ioutil.WriteFile("redirs/"+filename, []byte(p.LongURL), 0600)
}

func (p *Page) shortURLHASH() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(p.ShortURL)))
}

func loadPage(path string) (*Page, error) {
	filename := fmt.Sprintf("%x", md5.Sum([]byte(path)))
	longurl, err := ioutil.ReadFile("redirs/" + filename)
	if err != nil {
		return nil, err
	}
	return &Page{ShortURL: path, LongURL: string(longurl)}, nil
}
<commit_msg>Re-arrange<commit_after>package main

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"time"
)

// Page is the short to long url conversion object
type Page struct {
	ID        int8
	ShortURL  string
	LongURL   string
	CreatedAt time.Time
}

// DB Based
func queryPage(path string) (*Page, error) {
	var id int8
	var shorturl string
	var longurl string
	var createdAt string
	err := db.QueryRow("SELECT * FROM pages WHERE shorturl = $1", path).Scan(&id, &shorturl, &longurl, &createdAt)
	if err != nil {
		return nil, err
	}
	return &Page{ID: id, LongURL: longurl, ShortURL: shorturl}, nil
}

func (p *Page) save() error {
	_, err := db.Exec("INSERT INTO pages (shorturl, longurl) VALUES ($1, $2)", p.ShortURL, p.LongURL)
	return err
}

// File Store Based
func (p *Page) saveFile() error {
	filename := p.shortURLHASH()
	return ioutil.WriteFile("redirs/"+filename, []byte(p.LongURL), 0600)
}

func (p *Page) shortURLHASH() string {
	return fmt.Sprintf("%x", md5.Sum([]byte(p.ShortURL)))
}

func loadPage(path string) (*Page, error) {
	filename := fmt.Sprintf("%x", md5.Sum([]byte(path)))
	longurl, err := ioutil.ReadFile("redirs/" + filename)
	if err != nil {
		return nil, err
	}
	return &Page{ShortURL: path, LongURL: string(longurl)}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"text/template"
	"text/template/parse"

	"sourcegraph.com/sourcegraph/go-template-lint/tmplwalk"
)

var (
	verbose      = flag.Bool("v", false, "show verbose output")
	funcmapFile  = flag.String("f", "", "Go source file with FuncMap literal (commas separate multiple files)")
	tmplSetsFile = flag.String("t", "", "Go source file containing template set (a [][]string literal)")
	tmplDir      = flag.String("td", "", "base path of templates (prepended to template set filenames)")

	fset = token.NewFileSet()
)

func main() {
	log.SetFlags(0)

	flag.Parse()
	if *funcmapFile == "" {
		log.Fatal("-f is required (run with -h for usage info)")
	}
	if *tmplSetsFile == "" {
		log.Fatal("-t is required (run with -h for usage info)")
	}

	var allDefinedFuncs []string
	for _, f := range strings.Split(*funcmapFile, ",") {
		definedFuncs, err := parseFuncDefs(f)
		if err != nil {
			log.Fatalf("Error parsing FuncMap names from %s: %s", f, err)
		}
		if len(definedFuncs) == 0 {
			log.Fatalf("No func definitions in a FuncMap found in %s.", f)
		}
		allDefinedFuncs = append(allDefinedFuncs, definedFuncs...)
	}
	if *verbose {
		log.Printf("# Found %d template functions:", len(allDefinedFuncs))
		for _, name := range allDefinedFuncs {
			log.Printf("# - %s", name)
		}
	}

	tmplSets, err := parseTmplSet(*tmplSetsFile)
	if err != nil {
		log.Fatalf("Error parsing template sets (a [][]string literal) from %s: %s", *tmplSetsFile, err)
	}
	if len(tmplSets) == 0 {
		log.Fatal("No template files found.")
	}
	if *verbose {
		log.Printf("# Found %d template file sets:", len(tmplSets))
		for _, ts := range tmplSets {
			log.Printf("# - %s", strings.Join(ts, " "))
		}
	}

	invokedFuncs, err := findInvokedFuncs(tmplSets, allDefinedFuncs)
	if err != nil {
		log.Fatalf("Error parsing templates to find invoked template functions: %s", err)
	}
	if len(invokedFuncs) == 0 {
		log.Fatal("No func invocations in templates found.")
	}
	if *verbose {
		log.Printf("# Found %d functions invoked in templates:", len(invokedFuncs))
		for _, name := range invokedFuncs {
			log.Printf("# - %s", name)
		}
	}

	definedFuncMap := sliceToMap(allDefinedFuncs)
	invokedFuncMap := sliceToMap(invokedFuncs)
	fail := false

	unusedFuncs := subtract(definedFuncMap, invokedFuncMap)
	for _, f := range unusedFuncs {
		fmt.Println("unused template func", f)
		fail = true
	}

	addPredefs(definedFuncMap)
	undefinedFuncs := subtract(invokedFuncMap, definedFuncMap)
	for _, f := range undefinedFuncs {
		fmt.Println("undefined template func", f)
		fail = true
	}

	if fail {
		os.Exit(1)
	}
}

// parseFuncDefs extracts and returns function names from a Go source
// file containing a FuncMap literal.
func parseFuncDefs(filename string) ([]string, error) {
	f, err := parser.ParseFile(fset, filename, nil, parser.AllErrors)
	if err != nil {
		return nil, err
	}

	var funcNames []string
	ast.Walk(visitFn(func(n ast.Node) bool {
		switch n := n.(type) {
		case *ast.CompositeLit:
			if isFuncMap(n.Type) {
				for _, e := range n.Elts {
					kv := e.(*ast.KeyValueExpr)
					name, err := strconv.Unquote(kv.Key.(*ast.BasicLit).Value)
					if err != nil {
						log.Fatal(err)
					}
					funcNames = append(funcNames, name)
				}
			}
		}
		return true
	}), f)
	return funcNames, nil
}

func isFuncMap(x ast.Expr) bool {
	switch x := x.(type) {
	case *ast.Ident:
		return x.Name == "FuncMap"
	case *ast.SelectorExpr:
		return isFuncMap(x.Sel)
	}
	return false
}

func parseTmplSet(filename string) ([][]string, error) {
	f, err := parser.ParseFile(fset, filename, nil, parser.AllErrors)
	if err != nil {
		return nil, err
	}

	var tmplSets [][]string
	ast.Walk(visitFn(func(n ast.Node) bool {
		switch n := n.(type) {
		case *ast.CompositeLit:
			if isTmplSet(n.Type) {
				for _, e := range n.Elts {
					tmplSets = append(tmplSets, astStringSlice(e.(*ast.CompositeLit)))
				}
			}
		}
		return true
	}), f)
	return tmplSets, nil
}

func isTmplSet(x ast.Expr) bool {
	if sx, ok := x.(*ast.ArrayType); ok && sx.Len == nil {
		if sx2, ok := sx.Elt.(*ast.ArrayType); ok && sx2.Len == nil {
			if t, ok := sx2.Elt.(*ast.Ident); ok && t.Name == "string" {
				return true
			}
		}
	}
	return false
}

func astStringSlice(cl *ast.CompositeLit) []string {
	var ss []string
	for _, e := range cl.Elts {
		s, err := strconv.Unquote(e.(*ast.BasicLit).Value)
		if err != nil {
			log.Fatal(err)
		}
		ss = append(ss, s)
	}
	return ss
}

// visitFn is a wrapper for traversing nodes in the AST
type visitFn func(node ast.Node) (descend bool)

func (v visitFn) Visit(node ast.Node) ast.Visitor {
	descend := v(node)
	if descend {
		return v
	}
	return nil
}

// findInvokedFuncs returns a list of all functions (including
// predefined functions) invoked in Go templates in templateDir
// (recursively).
func findInvokedFuncs(tmplSets [][]string, definedFuncs []string) ([]string, error) {
	definedFuncMap := template.FuncMap{}
	for _, f := range definedFuncs {
		definedFuncMap[f] = func() interface{} { return nil }
	}

	invoked := map[string]struct{}{}
	for _, tmplSet := range tmplSets {
		tt := template.New("")
		tt.Funcs(definedFuncMap)
		_, err := tt.ParseFiles(joinTemplateDir(*tmplDir, tmplSet)...)
		if err != nil {
			return nil, fmt.Errorf("template set %v: %s", tmplSet, err)
		}
		for _, t := range tt.Templates() {
			if t.Tree == nil {
				log.Printf("No template root for %v", t.Name())
				continue
			}
			tmplwalk.Inspect(t.Tree.Root, func(n parse.Node) bool {
				switch n := n.(type) {
				case *parse.IdentifierNode:
					invoked[n.Ident] = struct{}{}
				}
				return true
			})
		}
	}

	var invokedList []string
	for f := range invoked {
		invokedList = append(invokedList, f)
	}
	sort.Strings(invokedList)
	return invokedList, nil
}

func joinTemplateDir(base string, files []string) []string {
	result := make([]string, len(files))
	for i := range files {
		result[i] = filepath.Join(base, files[i])
	}
	return result
}

var predefFuncs = []string{"and", "call", "html", "index", "js", "len", "not", "or", "print", "printf", "println", "urlquery", "eq", "ne", "lt", "le", "gt", "ge"}

// addPredefs adds predefined template functions to fm.
func addPredefs(fm map[string]struct{}) {
	for _, f := range predefFuncs {
		fm[f] = struct{}{}
	}
}

// subtract returns a list of keys in a that are not in b.
func subtract(a, b map[string]struct{}) []string {
	var d []string
	for k := range a {
		if _, inB := b[k]; !inB {
			d = append(d, k)
		}
	}
	sort.Strings(d)
	return d
}

func sliceToMap(ss []string) map[string]struct{} {
	m := make(map[string]struct{}, len(ss))
	for _, s := range ss {
		m[s] = struct{}{}
	}
	return m
}
<commit_msg>update: account for layout templates (string slices, not just slices of string slices).<commit_after>package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"text/template"
	"text/template/parse"

	"sourcegraph.com/sourcegraph/go-template-lint/tmplwalk"
)

var (
	verbose      = flag.Bool("v", false, "show verbose output")
	funcmapFile  = flag.String("f", "", "Go source file with FuncMap literal (commas separate multiple files)")
	tmplSetsFile = flag.String("t", "", "Go source file containing template set (a [][]string literal)")
	tmplDir      = flag.String("td", "", "base path of templates (prepended to template set filenames)")

	fset = token.NewFileSet()
)

func main() {
	log.SetFlags(0)

	flag.Parse()
	if *funcmapFile == "" {
		log.Fatal("-f is required (run with -h for usage info)")
	}
	if *tmplSetsFile == "" {
		log.Fatal("-t is required (run with -h for usage info)")
	}

	var allDefinedFuncs []string
	for _, f := range strings.Split(*funcmapFile, ",") {
		definedFuncs, err := parseFuncDefs(f)
		if err != nil {
			log.Fatalf("Error parsing FuncMap names from %s: %s", f, err)
		}
		if len(definedFuncs) == 0 {
			log.Fatalf("No func definitions in a FuncMap found in %s.", f)
		}
		allDefinedFuncs = append(allDefinedFuncs, definedFuncs...)
	}
	if *verbose {
		log.Printf("# Found %d template functions:", len(allDefinedFuncs))
		for _, name := range allDefinedFuncs {
			log.Printf("# - %s", name)
		}
	}

	tmplSets, err := parseTmplSet(*tmplSetsFile)
	if err != nil {
		log.Fatalf("Error parsing template sets (a [][]string literal) from %s: %s", *tmplSetsFile, err)
	}
	if len(tmplSets) == 0 {
		log.Fatal("No template files found.")
	}
	if *verbose {
		log.Printf("# Found %d template file sets:", len(tmplSets))
		for _, ts := range tmplSets {
			log.Printf("# - %s", strings.Join(ts, " "))
		}
	}

	invokedFuncs, err := findInvokedFuncs(tmplSets, allDefinedFuncs)
	if err != nil {
		log.Fatalf("Error parsing templates to find invoked template functions: %s", err)
	}
	if len(invokedFuncs) == 0 {
		log.Fatal("No func invocations in templates found.")
	}
	if *verbose {
		log.Printf("# Found %d functions invoked in templates:", len(invokedFuncs))
		for _, name := range invokedFuncs {
			log.Printf("# - %s", name)
		}
	}

	definedFuncMap := sliceToMap(allDefinedFuncs)
	invokedFuncMap := sliceToMap(invokedFuncs)
	fail := false

	unusedFuncs := subtract(definedFuncMap, invokedFuncMap)
	for _, f := range unusedFuncs {
		fmt.Println("unused template func", f)
		fail = true
	}

	addPredefs(definedFuncMap)
	undefinedFuncs := subtract(invokedFuncMap, definedFuncMap)
	for _, f := range undefinedFuncs {
		fmt.Println("undefined template func", f)
		fail = true
	}

	if fail {
		os.Exit(1)
	}
}

// parseFuncDefs extracts and returns function names from a Go source
// file containing a FuncMap literal.
func parseFuncDefs(filename string) ([]string, error) {
	f, err := parser.ParseFile(fset, filename, nil, parser.AllErrors)
	if err != nil {
		return nil, err
	}

	var funcNames []string
	ast.Walk(visitFn(func(n ast.Node) bool {
		switch n := n.(type) {
		case *ast.CompositeLit:
			if isFuncMap(n.Type) {
				for _, e := range n.Elts {
					kv := e.(*ast.KeyValueExpr)
					name, err := strconv.Unquote(kv.Key.(*ast.BasicLit).Value)
					if err != nil {
						log.Fatal(err)
					}
					funcNames = append(funcNames, name)
				}
			}
		}
		return true
	}), f)
	return funcNames, nil
}

func isFuncMap(x ast.Expr) bool {
	switch x := x.(type) {
	case *ast.Ident:
		return x.Name == "FuncMap"
	case *ast.SelectorExpr:
		return isFuncMap(x.Sel)
	}
	return false
}

func parseTmplSet(filename string) ([][]string, error) {
	f, err := parser.ParseFile(fset, filename, nil, parser.AllErrors)
	if err != nil {
		return nil, err
	}

	var tmplSets [][]string
	ast.Walk(visitFn(func(n ast.Node) bool {
		switch n := n.(type) {
		case *ast.CompositeLit:
			switch {
			case isTmplSet(n.Type):
				for _, e := range n.Elts {
					tmplSets = append(tmplSets, astStringSlice(e.(*ast.CompositeLit)))
				}
			case isLayoutSet(n.Type):
				tmplSets = append(tmplSets, astStringSlice(n))
			}
		}
		return true
	}), f)
	return tmplSets, nil
}

func isTmplSet(x ast.Expr) bool {
	if sx, ok := x.(*ast.ArrayType); ok && sx.Len == nil {
		if sx2, ok := sx.Elt.(*ast.ArrayType); ok && sx2.Len == nil {
			if t, ok := sx2.Elt.(*ast.Ident); ok && t.Name == "string" {
				return true
			}
		}
	}
	return false
}

func isLayoutSet(x ast.Expr) bool {
	if sx, ok := x.(*ast.ArrayType); ok && sx.Len == nil {
		if t, ok := sx.Elt.(*ast.Ident); ok && t.Name == "string" {
			return true
		}
	}
	return false
}

func astStringSlice(cl *ast.CompositeLit) []string {
	var ss []string
	for _, e := range cl.Elts {
		s, err := strconv.Unquote(e.(*ast.BasicLit).Value)
		if err != nil {
			log.Fatal(err)
		}
		ss = append(ss, s)
	}
	return ss
}

// visitFn is a wrapper for traversing nodes in the AST
type visitFn func(node ast.Node) (descend bool)

func (v visitFn) Visit(node ast.Node) ast.Visitor {
	descend := v(node)
	if descend {
		return v
	}
	return nil
}

// findInvokedFuncs returns a list of all functions (including
// predefined functions) invoked in Go templates in templateDir
// (recursively).
func findInvokedFuncs(tmplSets [][]string, definedFuncs []string) ([]string, error) {
	definedFuncMap := template.FuncMap{}
	for _, f := range definedFuncs {
		definedFuncMap[f] = func() interface{} { return nil }
	}

	invoked := map[string]struct{}{}
	for _, tmplSet := range tmplSets {
		tt := template.New("")
		tt.Funcs(definedFuncMap)
		_, err := tt.ParseFiles(joinTemplateDir(*tmplDir, tmplSet)...)
		if err != nil {
			return nil, fmt.Errorf("template set %v: %s", tmplSet, err)
		}
		for _, t := range tt.Templates() {
			if t.Tree == nil {
				log.Printf("No template root for %v", t.Name())
				continue
			}
			tmplwalk.Inspect(t.Tree.Root, func(n parse.Node) bool {
				switch n := n.(type) {
				case *parse.IdentifierNode:
					invoked[n.Ident] = struct{}{}
				}
				return true
			})
		}
	}

	var invokedList []string
	for f := range invoked {
		invokedList = append(invokedList, f)
	}
	sort.Strings(invokedList)
	return invokedList, nil
}

func joinTemplateDir(base string, files []string) []string {
	result := make([]string, len(files))
	for i := range files {
		result[i] = filepath.Join(base, files[i])
	}
	return result
}

var predefFuncs = []string{"and", "call", "html", "index", "js", "len", "not", "or", "print", "printf", "println", "urlquery", "eq", "ne", "lt", "le", "gt", "ge"}

// addPredefs adds predefined template functions to fm.
func addPredefs(fm map[string]struct{}) {
	for _, f := range predefFuncs {
		fm[f] = struct{}{}
	}
}

// subtract returns a list of keys in a that are not in b.
func subtract(a, b map[string]struct{}) []string {
	var d []string
	for k := range a {
		if _, inB := b[k]; !inB {
			d = append(d, k)
		}
	}
	sort.Strings(d)
	return d
}

func sliceToMap(ss []string) map[string]struct{} {
	m := make(map[string]struct{}, len(ss))
	for _, s := range ss {
		m[s] = struct{}{}
	}
	return m
}
<|endoftext|>


<commit_before>package main

import (
	"container/list"

	"github.com/sachaos/todoist/lib"
	"github.com/urfave/cli"
)

func List(sync todoist.Sync, c *cli.Context) error {
	colorList := ColorList()
	projectTree := ProjectTree(sync)
	var projectIds []int
	for _, project := range sync.Projects {
		projectIds = append(projectIds, project.GetID())
	}
	projectColorHash := GenerateColorHash(projectIds, colorList)

	itemQue := list.New()
	for _, item := range sync.Items {
		itemQue.PushBack(item)
	}

	tree, err := NewTree(itemQue)
	if err != nil {
		return err
	}

	defer writer.Flush()

	for _, node := range tree.Traverse() {
		item := node.Value.(todoist.Item)
		writer.Write([]string{
			IdFormat(item),
			PriorityFormat(item.Priority),
			DueDateFormat(item.DueDateTime(), item.AllDay),
			ProjectFormat(item.ProjectID, projectTree, projectColorHash, c),
			item.LabelsString(sync.Labels),
			ContentPrefix(node, c) + ContentFormat(item),
		})
	}

	return nil
}
<commit_msg>:+1: Reject checked task from list<commit_after>package main

import (
	"container/list"

	"github.com/sachaos/todoist/lib"
	"github.com/urfave/cli"
)

func List(sync todoist.Sync, c *cli.Context) error {
	colorList := ColorList()
	projectTree := ProjectTree(sync)
	var projectIds []int
	for _, project := range sync.Projects {
		projectIds = append(projectIds, project.GetID())
	}
	projectColorHash := GenerateColorHash(projectIds, colorList)

	itemQue := list.New()
	for _, item := range sync.Items {
		itemQue.PushBack(item)
	}

	tree, err := NewTree(itemQue)
	if err != nil {
		return err
	}

	defer writer.Flush()

	for _, node := range tree.Traverse() {
		item := node.Value.(todoist.Item)
		if item.Checked == 1 {
			continue
		}
		writer.Write([]string{
			IdFormat(item),
			PriorityFormat(item.Priority),
			DueDateFormat(item.DueDateTime(), item.AllDay),
			ProjectFormat(item.ProjectID, projectTree, projectColorHash, c),
			item.LabelsString(sync.Labels),
			ContentPrefix(node, c) + ContentFormat(item),
		})
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	//"fmt"
	//"time"
	//"sync"
	"log"
	//"strings"
	//"strconv"
	//"math/rand"
	"io/ioutil"
	"net/http"
	"os"
	//"bytes"
)

////////////////////////////////////////////////////////////////////////////////////////
// GetValuebyID
//
// Called to get the contents of an item in the Consul KV store, as identified by the item's unique ID Key
//
// input unique ID (Key) of target item (actually not used, it's hardcoded for now)
// output contents in json format or "" if error
//
func GetValuebyID(key string) string {
	// Code to get value from Consul
	// GET http://marathon.force12.io:8500/v1/kv/priority1-demand
	var demandvalue string = ""
	var str string
	var port string
	port = os.Getenv("MARATHON_CONSUL_PORT")
	str = os.Getenv("MARATHON_ADDRESS")
	str = str + port
	if port == "" {
		port = "8500"
	}
	if str == "" {
		str = "http://marathon.force12.io:" + port
	}

	str += "/v1/kv/priority1-demand"

	log.Println("GET demand: " + str)
	resp, err := http.Get(str)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil || resp == nil {
		// handle error
		log.Println("GET demand failed ")
	} else {
		body, err0 := ioutil.ReadAll(resp.Body)
		if err0 != nil {
			// handle error
			log.Println("GET demand failed read body ")
		} else {
			s := string(body[:])
			log.Println("demand json: " + s)
			demandvalue = s
		}
	}
	return demandvalue
}
<commit_msg>Make Consul address configurable in correct file<commit_after>package main

import (
	//"fmt"
	//"time"
	//"sync"
	"log"
	//"strings"
	//"strconv"
	//"math/rand"
	"io/ioutil"
	"net/http"
	"os"
	//"bytes"
)

////////////////////////////////////////////////////////////////////////////////////////
// GetValuebyID
//
// Called to get the contents of an item in the Consul KV store, as identified by the item's unique ID Key
//
// input unique ID (Key) of target item (actually not used, it's hardcoded for now)
// output contents in json format or "" if error
//
func GetValuebyID(key string) string {
	// Code to get value from Consul
	// GET http://marathon.force12.io:8500/v1/kv/priority1-demand
	var demandvalue string = ""
	var str string
	var port string
	port = os.Getenv("CONSUL_PORT")
	str = os.Getenv("CONSUL_ADDRESS")
	str = str + port
	if port == "" {
		port = "8500"
	}
	if str == "" {
		str = "http://marathon.force12.io:" + port
	}

	str += "/v1/kv/priority1-demand"

	log.Println("GET demand: " + str)
	resp, err := http.Get(str)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil || resp == nil {
		// handle error
		log.Println("GET demand failed ")
	} else {
		body, err0 := ioutil.ReadAll(resp.Body)
		if err0 != nil {
			// handle error
			log.Println("GET demand failed read body ")
		} else {
			s := string(body[:])
			log.Println("demand json: " + s)
			demandvalue = s
		}
	}
	return demandvalue
}
<|endoftext|>


<commit_before>package main

import (
	"compress/bzip2"
	"log"
	"net/http"

	"gopkg.in/freeeve/pgn.v1"
)

var SOURCES = []string{
	//"http://www.ficsgames.org/dl/ficsgamesdb_201708_chess_nomovetimes_1494519.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_201707_chess_nomovetimes_1494520.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_201706_chess_nomovetimes_1494521.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_201705_chess_nomovetimes_1494522.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_201704_chess_nomovetimes_1494523.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_201703_chess_nomovetimes_1494524.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_201702_chess_nomovetimes_1494525.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_201701_chess_nomovetimes_1494526.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2016_chess_nomovetimes_1495477.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2015_chess_nomovetimes_1495478.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2014_chess_nomovetimes_1495479.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2013_chess_nomovetimes_1494498.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2012_chess_nomovetimes_1494499.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2011_chess_nomovetimes_1494501.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2010_chess_nomovetimes_1494502.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2009_chess_nomovetimes_1494503.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2008_chess_nomovetimes_1494504.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2007_chess_nomovetimes_1494505.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2006_chess_nomovetimes_1494506.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2005_chess_nomovetimes_1494507.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2004_chess_nomovetimes_1494508.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2003_chess_nomovetimes_1494509.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2002_chess_nomovetimes_1494510.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2001_chess_nomovetimes_1494511.pgn.bz2",
	//"http://www.ficsgames.org/dl/ficsgamesdb_2000_chess_nomovetimes_1494513.pgn.bz2",
}

func YieldSourceURLs(done <-chan struct{}) <-chan string {
	out := make(chan string)
	go func() {
		defer close(out)
		for _, url := range SOURCES {
			select {
			case <-done:
				return
			default:
				out <- url
			}
		}
	}()
	return out
}

func YieldBoardFeaturesAndResult(urls <-chan string, done <-chan struct{}) <-chan *BoardFeaturesAndResult {
	features := make(chan *BoardFeaturesAndResult, CHAN_BUFFER)
	go func() {
		defer close(features)
		for url := range urls {
			log.Printf("Reading data from %v", url)
			response, err := http.Get(url)
			if err != nil {
				log.Printf("Error reading url %v: %v", url, err)
				continue
			}

			reader := bzip2.NewReader(response.Body)

			ps := pgn.NewPGNScanner(reader)
			for ps.Next() {
				game, err := ps.Scan()
				if err != nil {
					continue
				}

				result, err := parseResult(game.Tags["Result"])
				if err != nil {
					log.Printf("Error parsing result: %v\n", err)
					continue
				}

				board := pgn.NewBoard()
				for i, move := range game.Moves {
					if i == 0 {
						features <- parseBoard(board, result)
					}

					board.MakeMove(move)

					select {
					case <-done:
						return
					default:
						features <- parseBoard(board, result)
					}
				}
			}
		}
	}()
	return features
}
<commit_msg>Update sources<commit_after>package main

import (
	"compress/bzip2"
	"log"
	"net/http"

	"gopkg.in/freeeve/pgn.v1"
)

// http://www.ficsgames.org/download.html
var SOURCES = []string{
	"http://www.ficsgames.org/dl/ficsgamesdb_2016_chess_nomovetimes_1495633.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2015_chess_nomovetimes_1495634.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2014_chess_nomovetimes_1495635.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2013_chess_nomovetimes_1495484.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2012_chess_nomovetimes_1495631.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2011_chess_nomovetimes_1495632.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2010_chess_nomovetimes_1495636.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2009_chess_nomovetimes_1495637.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2008_chess_nomovetimes_1495638.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2007_chess_nomovetimes_1495639.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2006_chess_nomovetimes_1495640.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2005_chess_nomovetimes_1495641.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2004_chess_nomovetimes_1495643.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2003_chess_nomovetimes_1495644.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2002_chess_nomovetimes_1495645.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2001_chess_nomovetimes_1495646.pgn.bz2",
	"http://www.ficsgames.org/dl/ficsgamesdb_2000_chess_nomovetimes_1495647.pgn.bz2",
}

func YieldSourceURLs(done <-chan struct{}) <-chan string {
	out := make(chan string)
	go func() {
		defer close(out)
		for _, url := range SOURCES {
			select {
			case <-done:
				return
			default:
				out <- url
			}
		}
	}()
	return out
}

func YieldBoardFeaturesAndResult(urls <-chan string, done <-chan struct{}) <-chan *BoardFeaturesAndResult {
	features := make(chan *BoardFeaturesAndResult, CHAN_BUFFER)
	go func() {
		defer close(features)
		for url := range urls {
			log.Printf("Reading data from %v", url)
			response, err := http.Get(url)
			if err != nil {
				log.Printf("Error reading url %v: %v", url, err)
				continue
			}

			reader := bzip2.NewReader(response.Body)

			ps := pgn.NewPGNScanner(reader)
			for ps.Next() {
				game, err := ps.Scan()
				if err != nil {
					continue
				}

				result, err := parseResult(game.Tags["Result"])
				if err != nil {
					log.Printf("Error parsing result: %v\n", err)
					continue
				}

				board := pgn.NewBoard()
				for i, move := range game.Moves {
					if i == 0 {
						features <- parseBoard(board, result)
					}

					board.MakeMove(move)

					select {
					case <-done:
						return
					default:
						features <- parseBoard(board, result)
					}
				}
			}
		}
	}()
	return features
}
<|endoftext|>