<commit_before>package main

import (
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"strconv"
	"time"
)

var ds datastorer

var timeoutInMilliseconds = time.Millisecond * 100

func main() {
	if d, err := newDatastore(); err != nil {
		log.Fatalln(err)
	} else {
		ds = datastorer(d)
	}

	r := mux.NewRouter()
	kr := r.PathPrefix("/key/{key}").Subrouter()
	kr.Methods("GET").HandlerFunc(getHandler)
	kr.Methods("POST").HandlerFunc(postHandler)
	kr.Methods("DELETE").HandlerFunc(deleteHandler)
	kr.Methods("PUT").HandlerFunc(putHandler)

	if err := http.ListenAndServe(":8080", r); err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]

	if err := ds.validKey(k); err != nil {
		http.Error(w, k+": invalid key", http.StatusBadRequest)
		return
	}

	ch := make(chan []byte)
	go func() {
		ch <- ds.get(k)
	}()

	select {
	case v := <-ch:
		log.Println("get ["+k+"] in", timeSpent(t0), "ms")
		if v != nil {
			w.Write(v)
		} else {
			log.Println(k + ": not found")
			http.Error(w, k+": not found", http.StatusNotFound)
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func postHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]
	ttl, _ := strconv.Atoi(r.FormValue("ttl"))

	v, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, k+": can't get value", http.StatusBadRequest)
		return
	}

	ch := make(chan error)
	go func() {
		ch <- ds.set(k, v, ttl)
	}()

	select {
	case err := <-ch:
		log.Println("set ["+k+"] in", timeSpent(t0), "ms")
		if err == nil {
			w.WriteHeader(http.StatusCreated)
		} else {
			switch err {
			case OVERSIZED_BODY, EMPTY_BODY:
				http.Error(w, k+": value is invalid", http.StatusBadRequest)
			case INVALID_KEY:
				http.Error(w, k+": invalid key", http.StatusBadRequest)
			default:
				http.Error(w, k+": cache server error", http.StatusInternalServerError)
			}
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]

	ch := make(chan error)
	go func() {
		ch <- ds.delete(k)
	}()

	select {
	case err := <-ch:
		log.Println("delete ["+k+"] in", timeSpent(t0), "ms")
		if err == nil {
			w.WriteHeader(http.StatusOK)
		} else {
			switch err {
			case INVALID_KEY:
				http.Error(w, k+": invalid key", http.StatusBadRequest)
			default:
				http.Error(w, k+": cache server error", http.StatusInternalServerError)
			}
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func putHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]

	v, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, k+": can't get value", http.StatusBadRequest)
		return
	}

	ch := make(chan error)
	go func() {
		ch <- ds.append(k, v)
	}()

	select {
	case err := <-ch:
		log.Println("append ["+k+"] in", timeSpent(t0), "ms")
		if err == nil {
			w.WriteHeader(http.StatusOK)
		} else {
			switch err {
			case NOT_FOUND_ERROR:
				http.Error(w, k+": not found", http.StatusNotFound)
			case OVERSIZED_BODY, EMPTY_BODY:
				http.Error(w, k+": value is invalid", http.StatusBadRequest)
			case INVALID_KEY:
				http.Error(w, k+": invalid key", http.StatusBadRequest)
			default:
				http.Error(w, k+": cache server error", http.StatusInternalServerError)
			}
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func returnTimeout(w http.ResponseWriter, k string) {
	log.Println(k + ": timeout")
	http.Error(w, k+": timeout", http.StatusRequestTimeout)
}

func timeSpent(t0 int64) int64 {
	return int64(math.Floor(float64(time.Now().UnixNano()-t0)/1000000 + .5))
}
<commit_msg>func to map err to http status<commit_after>package main

import (
	"github.com/gorilla/mux"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"strconv"
	"time"
)

var ds datastorer

var timeoutInMilliseconds = time.Millisecond * 100

func main() {
	if d, err := newDatastore(); err != nil {
		log.Fatalln(err)
	} else {
		ds = datastorer(d)
	}

	r := mux.NewRouter()
	kr := r.PathPrefix("/key/{key}").Subrouter()
	kr.Methods("GET").HandlerFunc(getHandler)
	kr.Methods("POST").HandlerFunc(postHandler)
	kr.Methods("DELETE").HandlerFunc(deleteHandler)
	kr.Methods("PUT").HandlerFunc(putHandler)

	if err := http.ListenAndServe(":8080", r); err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func getHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]

	if err := ds.validKey(k); err != nil {
		http.Error(w, k+": invalid key", http.StatusBadRequest)
		return
	}

	ch := make(chan []byte)
	go func() {
		ch <- ds.get(k)
	}()

	select {
	case v := <-ch:
		log.Println("get ["+k+"] in", timeSpent(t0), "ms")
		if v != nil {
			w.Write(v)
		} else {
			log.Println(k + ": not found")
			http.Error(w, k+": not found", http.StatusNotFound)
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func postHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]
	ttl, _ := strconv.Atoi(r.FormValue("ttl"))

	v, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, k+": can't get value", http.StatusBadRequest)
		return
	}

	ch := make(chan error)
	go func() {
		ch <- ds.set(k, v, ttl)
	}()

	select {
	case err := <-ch:
		log.Println("set ["+k+"] in", timeSpent(t0), "ms")
		if err == nil {
			w.WriteHeader(http.StatusCreated)
		} else {
			errorToStatus(err, w)
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func deleteHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]

	ch := make(chan error)
	go func() {
		ch <- ds.delete(k)
	}()

	select {
	case err := <-ch:
		log.Println("delete ["+k+"] in", timeSpent(t0), "ms")
		if err == nil {
			w.WriteHeader(http.StatusOK)
		} else {
			errorToStatus(err, w)
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func putHandler(w http.ResponseWriter, r *http.Request) {
	t0 := time.Now().UnixNano()
	k := mux.Vars(r)["key"]

	v, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, k+": can't get value", http.StatusBadRequest)
		return
	}

	ch := make(chan error)
	go func() {
		ch <- ds.append(k, v)
	}()

	select {
	case err := <-ch:
		log.Println("append ["+k+"] in", timeSpent(t0), "ms")
		if err == nil {
			w.WriteHeader(http.StatusOK)
		} else {
			errorToStatus(err, w)
		}
	case <-time.After(timeoutInMilliseconds):
		returnTimeout(w, k)
	}
}

func returnTimeout(w http.ResponseWriter, k string) {
	log.Println(k + ": timeout")
	http.Error(w, k+": timeout", http.StatusRequestTimeout)
}

func timeSpent(t0 int64) int64 {
	return int64(math.Floor(float64(time.Now().UnixNano()-t0)/1000000 + .5))
}

func errorToStatus(err error, w http.ResponseWriter) {
	switch err {
	case NOT_FOUND_ERROR:
		http.Error(w, "key not found", http.StatusNotFound)
	case OVERSIZED_BODY, EMPTY_BODY:
		http.Error(w, "invalid value", http.StatusBadRequest)
	case INVALID_KEY:
		http.Error(w, "invalid key", http.StatusBadRequest)
	default:
		http.Error(w, "cache server error", http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/fsnotify/fsnotify"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sync"
	"time"
)

type featureFlag int

const (
	flgAutoIgnore featureFlag = 1 + iota
	// Particularly useful for VIM flury of events, see:
	//   https://stackoverflow.com/q/10300835/287374
	flgDebugOutput
)

func (flg featureFlag) String() string {
	switch flg {
	case flgAutoIgnore:
		return "flgAutoIgnore"
	case flgDebugOutput:
		return "flgDebugOutput"
	default:
		panic(fmt.Sprintf("unexpected flag, '%d'", int(flg)))
	}
}

type exitReason int

const (
	exCommandline exitReason = 1 + iota
	exWatcher
	exFsevent
)

type runDirective struct {
	Shell       string
	Command     string
	WatchTarget string
	InvertMatch *regexp.Regexp
	Features    map[featureFlag]bool
	LastRun     time.Time
	LastRunLk   sync.RWMutex
}

func (run *runDirective) Exec(msgStdout bool) error {
	run.LastRunLk.Lock()
	run.LastRun = time.Now()
	run.LastRunLk.Unlock()

	if msgStdout {
		fmt.Printf("RUNNING `%s`\n", run.Command)
	}

	// TODO(zacsh) find out a shell-agnostic way to run comands (eg: *bash*
	// specifically takes a "-c" flag)
	cmd := exec.Command(run.Shell, "-c", run.Command)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (run *runDirective) hasRunInlast(since time.Duration) bool {
	run.LastRunLk.RLock()
	last := run.LastRun
	run.LastRunLk.RUnlock()

	return time.Since(last) < since
}

func usage() string {
	return fmt.Sprintf(`Runs a command everytime some filesystem events happen.
  Usage:  COMMAND  [DIR_TO_WATCH  [FILE_IGNORE_PATTERN]]

  DIR_TO_WATCH defaults to the current working directory.
  FILE_IGNORE_PATTERN If provided, is used to match against the basename of the
    exact file whose event has been captured. If FILE_IGNORE_PATTERN expression
    matches said file, COMMAND will not be run.
    Valid arguments are those accepted by https://golang.org/pkg/regexp/#Compile
`)
}

func die(reason exitReason, e error) {
	var reasonStr string
	switch reason {
	case exCommandline:
		reasonStr = "usage"
	case exWatcher:
		reasonStr = "watcher"
	case exFsevent:
		reasonStr = "event"
	}

	fmt.Fprintf(os.Stderr, "%s error: %s\n", reasonStr, e)
	os.Exit(int(reason))
}

func (c *runDirective) debugStr() string {
	invertMatch := "n/a"
	if c.InvertMatch != nil {
		invertMatch = c.InvertMatch.String()
	}

	var features string
	for k, v := range c.Features {
		if v {
			var sep string
			if len(features) > 0 {
				sep = ", "
			}
			features = fmt.Sprintf("%s%s%s", features, sep, k.String())
		}
	}

	return fmt.Sprintf(`
  run.Command:            "%s"
  run.WatchTarget.Name(): "%s"
  run.InvertMatch:        "%s"
  run.Shell:              "%s"
  run.Features:           %s
  `, c.Command, c.WatchTarget, invertMatch, c.Shell, features)
}

func main() {
	magicFileRegexp := regexp.MustCompile(`^(\.\w.*sw[a-z]|4913)$`)

	run, perr := parseCli()
	if perr != nil {
		if perr.Stage == psHelp {
			fmt.Printf(usage())
			os.Exit(0)
		}

		die(exCommandline, perr)
	}

	watcher, e := fsnotify.NewWatcher()
	if e != nil {
		die(exCommandline, e)
	}
	defer watcher.Close()

	fmt.Printf("Watching `%s`\n", run.WatchTarget)

	if run.Features[flgDebugOutput] {
		fmt.Fprintf(
			os.Stderr,
			"[debug] not yet implemented, but here's what you asked for:\n%s\n",
			run.debugStr())
	}

	run.Exec(true /*msgStdout*/)

	done := make(chan bool)
	go func() {
		for {
			select {
			case e := <-watcher.Events:
				if run.Features[flgDebugOutput] {
					fmt.Fprintf(os.Stderr, "[debug] [%s] %s\n", e.Op.String(), e.Name)
				}

				if run.Features[flgAutoIgnore] {
					if magicFileRegexp.MatchString(filepath.Base(e.Name)) {
						continue
					}
				}

				if run.InvertMatch != nil &&
					run.InvertMatch.MatchString(filepath.Base(e.Name)) {
					continue
				}

				if run.hasRunInlast(2 * time.Second) {
					fmt.Fprintf(os.Stderr, ".")
					continue
				}

				fmt.Fprintf(os.Stderr, "\n")
				run.Exec(true /*msgStdout*/)
			case err := <-watcher.Errors:
				die(exFsevent, err)
			}
		}
	}()

	if err := watcher.Add(run.WatchTarget); err != nil {
		die(exWatcher, e)
	}
	<-done // hang main
}
<commit_msg>colorize: matching feature from original `runonchange` script<commit_after>package main

import (
	"fmt"
	"github.com/fatih/color"
	"github.com/fsnotify/fsnotify"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"sync"
	"time"
)

type featureFlag int

const (
	flgAutoIgnore featureFlag = 1 + iota
	// Particularly useful for VIM flury of events, see:
	//   https://stackoverflow.com/q/10300835/287374
	flgDebugOutput
)

func (flg featureFlag) String() string {
	switch flg {
	case flgAutoIgnore:
		return "flgAutoIgnore"
	case flgDebugOutput:
		return "flgDebugOutput"
	default:
		panic(fmt.Sprintf("unexpected flag, '%d'", int(flg)))
	}
}

type exitReason int

const (
	exCommandline exitReason = 1 + iota
	exWatcher
	exFsevent
)

type runDirective struct {
	Shell       string
	Command     string
	WatchTarget string
	InvertMatch *regexp.Regexp
	Features    map[featureFlag]bool
	LastRun     time.Time
	LastRunLk   sync.RWMutex
}

func (run *runDirective) Exec(msgStdout bool) error {
	run.LastRunLk.Lock()
	run.LastRun = time.Now()
	run.LastRunLk.Unlock()

	if msgStdout {
		fmt.Printf("%s `%s`\n",
			color.New(color.Bold, color.FgYellow).Sprintf("RUNNING"),
			color.New(color.Bold, color.FgRed).Sprintf(run.Command))
	}

	// TODO(zacsh) find out a shell-agnostic way to run comands (eg: *bash*
	// specifically takes a "-c" flag)
	cmd := exec.Command(run.Shell, "-c", run.Command)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

func (run *runDirective) hasRunInlast(since time.Duration) bool {
	run.LastRunLk.RLock()
	last := run.LastRun
	run.LastRunLk.RUnlock()

	return time.Since(last) < since
}

func usage() string {
	return fmt.Sprintf(`Runs a command everytime some filesystem events happen.
  Usage:  COMMAND  [DIR_TO_WATCH  [FILE_IGNORE_PATTERN]]

  DIR_TO_WATCH defaults to the current working directory.
  FILE_IGNORE_PATTERN If provided, is used to match against the basename of the
    exact file whose event has been captured. If FILE_IGNORE_PATTERN expression
    matches said file, COMMAND will not be run.
    Valid arguments are those accepted by https://golang.org/pkg/regexp/#Compile
`)
}

func die(reason exitReason, e error) {
	var reasonStr string
	switch reason {
	case exCommandline:
		reasonStr = "usage"
	case exWatcher:
		reasonStr = "watcher"
	case exFsevent:
		reasonStr = "event"
	}

	fmt.Fprintf(os.Stderr, "%s error: %s\n", reasonStr, e)
	os.Exit(int(reason))
}

func (c *runDirective) debugStr() string {
	invertMatch := "n/a"
	if c.InvertMatch != nil {
		invertMatch = c.InvertMatch.String()
	}

	var features string
	for k, v := range c.Features {
		if v {
			var sep string
			if len(features) > 0 {
				sep = ", "
			}
			features = fmt.Sprintf("%s%s%s", features, sep, k.String())
		}
	}

	return fmt.Sprintf(`
  run.Command:            "%s"
  run.WatchTarget.Name(): "%s"
  run.InvertMatch:        "%s"
  run.Shell:              "%s"
  run.Features:           %s
  `, c.Command, c.WatchTarget, invertMatch, c.Shell, features)
}

func main() {
	magicFileRegexp := regexp.MustCompile(`^(\.\w.*sw[a-z]|4913)$`)

	run, perr := parseCli()
	if perr != nil {
		if perr.Stage == psHelp {
			fmt.Printf(usage())
			os.Exit(0)
		}

		die(exCommandline, perr)
	}

	watcher, e := fsnotify.NewWatcher()
	if e != nil {
		die(exCommandline, e)
	}
	defer watcher.Close()

	fmt.Printf("Watching `%s`\n", run.WatchTarget)

	if run.Features[flgDebugOutput] {
		fmt.Fprintf(
			os.Stderr,
			"[debug] not yet implemented, but here's what you asked for:\n%s\n",
			run.debugStr())
	}

	run.Exec(true /*msgStdout*/)

	done := make(chan bool)
	go func() {
		for {
			select {
			case e := <-watcher.Events:
				if run.Features[flgDebugOutput] {
					fmt.Fprintf(os.Stderr, "[debug] [%s] %s\n", e.Op.String(), e.Name)
				}

				if run.Features[flgAutoIgnore] {
					if magicFileRegexp.MatchString(filepath.Base(e.Name)) {
						continue
					}
				}

				if run.InvertMatch != nil &&
					run.InvertMatch.MatchString(filepath.Base(e.Name)) {
					continue
				}

				if run.hasRunInlast(2 * time.Second) {
					fmt.Fprintf(os.Stderr, ".")
					continue
				}

				fmt.Fprintf(os.Stderr, "\n")
				run.Exec(true /*msgStdout*/)
			case err := <-watcher.Errors:
				die(exFsevent, err)
			}
		}
	}()

	if err := watcher.Add(run.WatchTarget); err != nil {
		die(exWatcher, e)
	}
	<-done // hang main
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"
	"strconv"
)

var idMap = map[string]map[string]int{}
var initialValue = 42
var incrementBy = 5

func getOrSetID(name, environment string, passedID int) (int, string) {
	// check if the environment is found, otherwise add it
	if _, ok := idMap[environment]; ok {
		// check if the name is found, otherwise add it
		if foundID, ok := idMap[environment][name]; ok {
			if passedID == 0 {
				idMap[environment][name] = foundID + incrementBy
			} else {
				idMap[environment][name] = passedID
			}
		} else {
			// add unfound name, return ID
			fmt.Printf("Name %s was not found in Environment %s, adding it\n", name, environment)
			if passedID == 0 {
				idMap[environment][name] = initialValue
			} else {
				idMap[environment][name] = passedID
			}
		}
	} else {
		// add unfound environment and name, return ID
		fmt.Printf("Environment %s was not found, adding it\n", environment)
		if passedID == 0 {
			idMap[environment] = map[string]int{name: initialValue}
		} else {
			idMap[environment] = map[string]int{name: passedID}
		}
	}
	return http.StatusOK, strconv.Itoa(idMap[environment][name])
}

func main() {
	router := gin.Default()

	router.GET("/list", func(context *gin.Context) {
		context.JSON(http.StatusOK, gin.H{"idMap": idMap})
	})

	router.GET("/getter/:environment/:name", func(context *gin.Context) {
		status, id := getOrSetID(context.Param("name"), context.Param("environment"), 0)
		context.String(status, id)
	})

	router.POST("/setter", func(context *gin.Context) {
		passedID, err := strconv.Atoi(context.PostForm("id"))
		if err != nil {
			context.String(http.StatusBadRequest, fmt.Sprintf("Error converting %s to an integer", context.PostForm("id")))
			return
		}
		if passedID == 0 {
			context.String(http.StatusBadRequest, "You can't set the ID to zero") // TODO rethink this limitation?
		} else {
			status, id := getOrSetID(context.PostForm("name"), context.PostForm("environment"), passedID)
			context.String(status, id)
		}
	})

	router.Run("localhost:8080")
}
<commit_msg>Add mutex<commit_after>package main

import (
	"fmt"
	"github.com/gin-gonic/gin"
	"net/http"
	"strconv"
	"sync"
)

// TODO setup auto API documentation, add auth, add persistent storage

var idMap = map[string]map[string]int{}
var initialValue = 42
var incrementBy = 5
var mutex = &sync.Mutex{}

func getOrSetID(name, environment string, passedID int) (int, string) {
	// check if the environment is found, otherwise add it
	if _, ok := idMap[environment]; ok {
		// check if the name is found, otherwise add it
		if foundID, ok := idMap[environment][name]; ok {
			if passedID == 0 {
				idMap[environment][name] = foundID + incrementBy
			} else {
				idMap[environment][name] = passedID
			}
		} else {
			// add unfound name, return ID
			fmt.Printf("Name %s was not found in Environment %s, adding it\n", name, environment)
			if passedID == 0 {
				idMap[environment][name] = initialValue
			} else {
				idMap[environment][name] = passedID
			}
		}
	} else {
		// add unfound environment and name, return ID
		fmt.Printf("Environment %s was not found, adding it\n", environment)
		if passedID == 0 {
			idMap[environment] = map[string]int{name: initialValue}
		} else {
			idMap[environment] = map[string]int{name: passedID}
		}
	}
	return http.StatusOK, strconv.Itoa(idMap[environment][name])
}

func main() {
	router := gin.Default()

	router.GET("/list", func(context *gin.Context) {
		mutex.Lock()
		context.JSON(http.StatusOK, gin.H{"idMap": idMap})
		mutex.Unlock()
	})

	router.GET("/getter/:environment/:name", func(context *gin.Context) {
		mutex.Lock()
		status, id := getOrSetID(context.Param("name"), context.Param("environment"), 0)
		mutex.Unlock()
		context.String(status, id)
	})

	router.POST("/setter", func(context *gin.Context) {
		passedID, err := strconv.Atoi(context.PostForm("id"))
		if err != nil {
			context.String(http.StatusBadRequest, fmt.Sprintf("Error converting %s to an integer", context.PostForm("id")))
			return
		}
		if passedID == 0 {
			context.String(http.StatusBadRequest, "You can't set the ID to zero") // TODO rethink this limitation?
		} else {
			mutex.Lock()
			status, id := getOrSetID(context.PostForm("name"), context.PostForm("environment"), passedID)
			mutex.Unlock()
			context.String(status, id)
		}
	})

	router.Run("localhost:8080")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/0xAX/notificator"
	"github.com/boltdb/bolt"
	"github.com/manishrjain/asanawarrior/asana"
	"github.com/manishrjain/asanawarrior/taskwarrior"
	"github.com/manishrjain/asanawarrior/x"
	"github.com/pkg/errors"
)

var duration = flag.Int("dur", 1, "How often to run sync, specified in minutes.")
var dbpath = flag.String("db", os.Getenv("HOME")+"/.task/asanawarrior.db",
	"File path for db which stores certain sync information.")
var notifyInterval = flag.Int("interval", 10,
	"Minimum duration in seconds between successive notifications. Set to zero for no notifications.")
var maxDeletes = flag.Int("deletes", 5,
	"If Asanawarrior sees more than these number of deletes, it's going to crash to"+
		" protect your Asana from mass deletion.")

var db *bolt.DB
var bucketName = []byte("aw")
var notify *notificator.Notificator

type Match struct {
	Xid    uint64
	Asana  x.WarriorTask
	TaskWr x.WarriorTask
}

type notification struct {
	Title string
	Text  string
}

var notifications = make(chan notification, 100)

// generateMatches matches all tasks from Asana to Taskwarrior, and stores non-matches as
// individual entries from each, without the other being present.
func generateMatches(atasks []x.WarriorTask, twtasks []x.WarriorTask) []*Match {
	amap := make(map[uint64]*Match)
	for _, at := range atasks {
		m := &Match{
			Xid:   at.Xid,
			Asana: at,
		}
		amap[at.Xid] = m
	}

	matches := make([]*Match, 0, 1000)
	// Iterate over task warrior tasks.
	for _, tw := range twtasks {
		if m, ok := amap[tw.Xid]; !ok {
			// Not in Asana.
			match := &Match{
				TaskWr: tw,
			}
			matches = append(matches, match)

		} else {
			// Also in Asana.
			m.TaskWr = tw
			delete(amap, tw.Xid)
			matches = append(matches, m)
		}
	}

	// Left over tasks from Asana not in task warrior.
	for _, m := range amap {
		matches = append(matches, m)
	}
	return matches
}

func approxAfter(t1, t2 time.Time) bool {
	return t1.Sub(t2) > time.Second
}

func asanaKey(xid uint64) []byte {
	return []byte(fmt.Sprintf("asana-%d", xid))
}

func taskwKey(uuid string) []byte {
	return []byte(fmt.Sprintf("taskw-%s", uuid))
}

func storeInDb(asanaTask, twTask x.WarriorTask) {
	if err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		if err := b.Put(asanaKey(asanaTask.Xid),
			[]byte(asanaTask.Modified.Format(time.RFC3339))); err != nil {
			return err
		}
		if err := b.Put(taskwKey(twTask.Uuid),
			[]byte(twTask.Modified.Format(time.RFC3339))); err != nil {
			return err
		}
		return nil

	}); err != nil {
		log.Fatalf("Write to db failed with error: %v", err)
	}
}

func getSyncTimestamps(xid uint64, uuid string) (time.Time, time.Time) {
	var at, tt time.Time
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		ats := string(b.Get(asanaKey(xid)))
		tts := string(b.Get(taskwKey(uuid)))
		var err error
		if at, err = time.Parse(time.RFC3339, ats); err != nil {
			log.Fatalf("Unable to find asana ts: %v %v", xid, uuid)
		}
		if tt, err = time.Parse(time.RFC3339, tts); err != nil {
			log.Fatalf("Unable to find taskwarrior ts: %v %v", xid, uuid)
		}
		return nil
	})
	return at, tt
}

func syncMatch(m *Match, deleteFromAsana *[]*Match) error {
	if m.Xid == 0 {
		// Task not present in Asana, but present in TW.

		if m.TaskWr.Xid > 0 {
			if m.TaskWr.Deleted {
				// Already deleted from TW. Do nothing.
				return nil
			}

			// This task used to have an Asana ID. But, we can't find the corresponding Asana task.
			// It can happen when Asana task was deleted.
			// If so, delete the task from TW as well.
			fmt.Printf("Delete from Taskwarrior: [%q]\n", m.TaskWr.Name)
			pushNotification("Delete", m.TaskWr.Name)

			if err := taskwarrior.Delete(m.TaskWr); err != nil {
				return errors.Wrap(err, "Delete from Taskwarrior")
			}
			return nil
		}

		// Create in Asana.
		fmt.Printf("Create in Asana: [%q]\n", m.TaskWr.Name)
		asanaUpdated, err := asana.AddNew(m.TaskWr)
		if err != nil {
			return errors.Wrap(err, "create asana addnew")
		}

		// Update TW with the Xid.
		if err := taskwarrior.OverwriteUuid(asanaUpdated, m.TaskWr.Uuid); err != nil {
			return errors.Wrap(err, "create asana overwriteuuid")
		}
		taskwUpdated, err := taskwarrior.GetTask(m.TaskWr.Uuid)
		if err != nil {
			return errors.Wrap(err, "create asana GetTask")
		}

		// Store Asana and Taskwarrior timestamps as of this sync.
		storeInDb(asanaUpdated, taskwUpdated)
		return nil
	}

	if m.TaskWr.Xid == 0 {
		// No Asana xid found in Taskwarrior. So, create it.

		fmt.Printf("Create in Taskwarrior: [%q]\n", m.Asana.Name)
		pushNotification("Create", m.Asana.Name)
		uuid, err := taskwarrior.AddNew(m.Asana)
		if err != nil {
			return errors.Wrap(err, "syncMatch create in taskwarrior")
		}
		if len(uuid) == 0 {
			log.Fatalf("Unable to parse UUID of new task: %+v", m.Asana)
		}
		updated, err := taskwarrior.GetTask(uuid)
		if err != nil {
			return err
		}

		// Store Asana and Taskwarrior timestamps as of this sync.
		storeInDb(m.Asana, updated)
		return nil
	}

	if m.Asana.Xid != m.TaskWr.Xid {
		log.Fatalf("Xids should be matched: %+v\n", m)
	}

	// Task is present in both Asana and TW.
	asanaTs, taskwTs := getSyncTimestamps(m.Asana.Xid, m.TaskWr.Uuid)

	if approxAfter(m.Asana.Modified, asanaTs) {
		// Asana was updated. Overwrite TW.
		fmt.Printf("Overwrite Taskwarrior: [%q] [time diff: %v]\n",
			m.Asana.Name, m.Asana.Modified.Sub(asanaTs))
		pushNotification("Update", m.Asana.Name)

		if err := taskwarrior.OverwriteUuid(m.Asana, m.TaskWr.Uuid); err != nil {
			return errors.Wrap(err, "Overwrite Taskwarrior")
		}
		updated, err := taskwarrior.GetTask(m.TaskWr.Uuid)
		if err != nil {
			return errors.Wrap(err, "Overwrite Taskwarrior GetTask")
		}
		storeInDb(m.Asana, updated)
		return nil
	}

	// If task has been marked as deleted since the last modification.
	if m.TaskWr.Deleted && approxAfter(m.TaskWr.Modified, taskwTs) {
		if deleteFromAsana != nil {
			*deleteFromAsana = append(*deleteFromAsana, m)
			return nil
		}

		fmt.Printf("Deleting task from Asana: [%q]\n", m.TaskWr.Name)
		pushNotification("Deleting from Asana", m.TaskWr.Name)
		if err := asana.Delete(m.Xid); err != nil {
			return errors.Wrap(err, "Delete task from Asana")
		}

		// Don't delete from boltdb, but update the timestamps,
		// so we don't reapply this deletion.
		// We don't need to retrieve Asana task back, because we won't be able to. It has been deleted.
		// If the task gets undeleted, Asana won't modify the timestamp. So, let's set it to zero
		// in our records, so if it comes back, we'll see it as an update.
		m.Asana.Modified = time.Time{}
		storeInDb(m.Asana, m.TaskWr)
		return nil
	}

	if approxAfter(m.TaskWr.Modified, taskwTs) {
		// TW was updated. Overwrite Asana.
		fmt.Printf("Overwrite Asana: [%q] [time diff: %v]\n",
			m.TaskWr.Name, m.TaskWr.Modified.Sub(taskwTs))

		if err := asana.UpdateTask(m.TaskWr, m.Asana); err != nil {
			return errors.Wrap(err, "syncMatch overwrite asana")
		}
		updated, err := asana.GetOneTask(m.Xid)
		if err != nil {
			return errors.Wrap(err, "syncMatch GetOneTask")
		}
		storeInDb(updated, m.TaskWr)
		return nil
	}
	return nil
}

func runSync() {
	atasks, err := asana.GetTasks()
	// atasks, err := asana.GetTasks(1)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%27s: %d active\n", "Asana results found", len(atasks))

	twtasks, err := taskwarrior.GetTasks()
	if err != nil {
		log.Fatal(err)
	}

	deleted := 0
	for _, t := range twtasks {
		if t.Deleted {
			deleted++
		}
	}
	fmt.Printf("%27s: %d active, %d deleted\n",
		"Taskwarrior results found", len(twtasks)-deleted, deleted)

	matches := generateMatches(atasks, twtasks)
	deletes := make([]*Match, 0, 10)
	for _, m := range matches {
		if err := syncMatch(m, &deletes); err != nil {
			log.Printf("syncMatch error: %v %+v", err, m)
		}
	}

	if len(deletes) > *maxDeletes {
		fmt.Printf(`
Task deletions requested from Asana : %d
Max allowed per sync                : %d
Most likely this is a mistake!
If so, please run 'rm -f ~/.task' to clean up Taskwarrior (or wherever else you store it's state).
If you genuinely want to delete all these tasks, then set the 'deletes' flag to a higher value.
Crashing to avoid mass deletes from Asana!
`, len(deletes), *maxDeletes)
		os.Exit(1)
	}
	for _, m := range deletes {
		if err := syncMatch(m, nil); err != nil {
			log.Printf("syncMatch error: %v %+v", err, m)
		}
	}

	fmt.Println("All synced up. DONE.")
}

func pushNotification(title, text string) {
	if notify == nil {
		return
	}
	n := notification{Title: title, Text: text}

	select {
	case notifications <- n:
	default:
		// Let it go.
	}

}

func processNotifications() {
	ni := time.Duration(*notifyInterval)
	ticker := time.NewTicker(ni * time.Second)
	l := make([]notification, 0, 10)
	for {
		select {
		case <-ticker.C:
			if len(l) == 0 {
				// pass

			} else if len(l) == 1 {
				n := l[0]
				notify.Push("Asanawarrior "+n.Title, n.Text, "", notificator.UR_NORMAL)

			} else {
				notify.Push("Asanawarrior "+l[0].Title,
					fmt.Sprintf("%q and %d more updates", l[0].Text, len(l)-1), "", notificator.UR_NORMAL)
			}
			l = l[:0]

		case n := <-notifications:
			if *notifyInterval > 0 {
				l = append(l, n)
			}
		}
	}
}

func main() {
	flag.Parse()
	fmt.Println("Asanawarrior v0.7 - Bringing the power of Taskwarrior to Asana")
	notify = notificator.New(notificator.Options{
		AppName: "Asanawarrior",
	})
	go processNotifications()

	var err error
	db, err = bolt.Open(*dbpath, 0600, nil)
	if err != nil {
		log.Fatalf("Unable to open bolt db at %v. Error: %v", *dbpath, err)
	}
	defer db.Close()
	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(bucketName)
		if err != nil {
			log.Fatalf("Unable to create bucket in bolt db.")
		}
		return nil
	})

	// Initiate a sync right away.
	fmt.Println()
	fmt.Println("Starting sync at", time.Now())
	runSync()

	// And then do it at regular intervals.
	ticker := time.NewTicker(time.Duration(*duration) * time.Minute)
	for t := range ticker.C {
		fmt.Println()
		fmt.Println("Starting sync at", t)
		runSync()
	}
}
<commit_msg>Update version and make things look better<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/0xAX/notificator"
	"github.com/boltdb/bolt"
	"github.com/manishrjain/asanawarrior/asana"
	"github.com/manishrjain/asanawarrior/taskwarrior"
	"github.com/manishrjain/asanawarrior/x"
	"github.com/pkg/errors"
)

var duration = flag.Int("dur", 1, "How often to run sync, specified in minutes.")
var dbpath = flag.String("db", os.Getenv("HOME")+"/.task/asanawarrior.db",
	"File path for db which stores certain sync information.")
var notifyInterval = flag.Int("interval", 10,
	"Minimum duration in seconds between successive notifications. Set to zero for no notifications.")
var maxDeletes = flag.Int("deletes", 5,
	"If Asanawarrior sees more than these number of deletes, it's going to crash to"+
		" protect your Asana from mass deletion.")

var db *bolt.DB
var bucketName = []byte("aw")
var notify *notificator.Notificator

type Match struct {
	Xid    uint64
	Asana  x.WarriorTask
	TaskWr x.WarriorTask
}

type notification struct {
	Title string
	Text  string
}

var notifications = make(chan notification, 100)

// generateMatches matches all tasks from Asana to Taskwarrior, and stores non-matches as
// individual entries from each, without the other being present.
func generateMatches(atasks []x.WarriorTask, twtasks []x.WarriorTask) []*Match {
	amap := make(map[uint64]*Match)
	for _, at := range atasks {
		m := &Match{
			Xid:   at.Xid,
			Asana: at,
		}
		amap[at.Xid] = m
	}

	matches := make([]*Match, 0, 1000)
	// Iterate over task warrior tasks.
	for _, tw := range twtasks {
		if m, ok := amap[tw.Xid]; !ok {
			// Not in Asana.
			match := &Match{
				TaskWr: tw,
			}
			matches = append(matches, match)

		} else {
			// Also in Asana.
			m.TaskWr = tw
			delete(amap, tw.Xid)
			matches = append(matches, m)
		}
	}

	// Left over tasks from Asana not in task warrior.
	for _, m := range amap {
		matches = append(matches, m)
	}
	return matches
}

func approxAfter(t1, t2 time.Time) bool {
	return t1.Sub(t2) > time.Second
}

func asanaKey(xid uint64) []byte {
	return []byte(fmt.Sprintf("asana-%d", xid))
}

func taskwKey(uuid string) []byte {
	return []byte(fmt.Sprintf("taskw-%s", uuid))
}

func storeInDb(asanaTask, twTask x.WarriorTask) {
	if err := db.Update(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		if err := b.Put(asanaKey(asanaTask.Xid),
			[]byte(asanaTask.Modified.Format(time.RFC3339))); err != nil {
			return err
		}
		if err := b.Put(taskwKey(twTask.Uuid),
			[]byte(twTask.Modified.Format(time.RFC3339))); err != nil {
			return err
		}
		return nil

	}); err != nil {
		log.Fatalf("Write to db failed with error: %v", err)
	}
}

func getSyncTimestamps(xid uint64, uuid string) (time.Time, time.Time) {
	var at, tt time.Time
	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(bucketName)
		ats := string(b.Get(asanaKey(xid)))
		tts := string(b.Get(taskwKey(uuid)))
		var err error
		if at, err = time.Parse(time.RFC3339, ats); err != nil {
			log.Fatalf("Unable to find asana ts: %v %v", xid, uuid)
		}
		if tt, err = time.Parse(time.RFC3339, tts); err != nil {
			log.Fatalf("Unable to find taskwarrior ts: %v %v", xid, uuid)
		}
		return nil
	})
	return at, tt
}

func syncMatch(m *Match, deleteFromAsana *[]*Match) error {
	if m.Xid == 0 {
		// Task not present in Asana, but present in TW.

		if m.TaskWr.Xid > 0 {
			if m.TaskWr.Deleted {
				// Already deleted from TW. Do nothing.
				return nil
			}

			// This task used to have an Asana ID. But, we can't find the corresponding Asana task.
			// It can happen when Asana task was deleted.
			// If so, delete the task from TW as well.
			fmt.Printf("Delete from Taskwarrior: [%q]\n", m.TaskWr.Name)
			pushNotification("Delete", m.TaskWr.Name)

			if err := taskwarrior.Delete(m.TaskWr); err != nil {
				return errors.Wrap(err, "Delete from Taskwarrior")
			}
			return nil
		}

		// Create in Asana.
		fmt.Printf("Create in Asana: [%q]\n", m.TaskWr.Name)
		asanaUpdated, err := asana.AddNew(m.TaskWr)
		if err != nil {
			return errors.Wrap(err, "create asana addnew")
		}

		// Update TW with the Xid.
		if err := taskwarrior.OverwriteUuid(asanaUpdated, m.TaskWr.Uuid); err != nil {
			return errors.Wrap(err, "create asana overwriteuuid")
		}
		taskwUpdated, err := taskwarrior.GetTask(m.TaskWr.Uuid)
		if err != nil {
			return errors.Wrap(err, "create asana GetTask")
		}

		// Store Asana and Taskwarrior timestamps as of this sync.
		storeInDb(asanaUpdated, taskwUpdated)
		return nil
	}

	if m.TaskWr.Xid == 0 {
		// No Asana xid found in Taskwarrior. So, create it.

		fmt.Printf("Create in Taskwarrior: [%q]\n", m.Asana.Name)
		pushNotification("Create", m.Asana.Name)
		uuid, err := taskwarrior.AddNew(m.Asana)
		if err != nil {
			return errors.Wrap(err, "syncMatch create in taskwarrior")
		}
		if len(uuid) == 0 {
			log.Fatalf("Unable to parse UUID of new task: %+v", m.Asana)
		}
		updated, err := taskwarrior.GetTask(uuid)
		if err != nil {
			return err
		}

		// Store Asana and Taskwarrior timestamps as of this sync.
		storeInDb(m.Asana, updated)
		return nil
	}

	if m.Asana.Xid != m.TaskWr.Xid {
		log.Fatalf("Xids should be matched: %+v\n", m)
	}

	// Task is present in both Asana and TW.
	asanaTs, taskwTs := getSyncTimestamps(m.Asana.Xid, m.TaskWr.Uuid)

	if approxAfter(m.Asana.Modified, asanaTs) {
		// Asana was updated. Overwrite TW.
		fmt.Printf("Overwrite Taskwarrior: [%q] [time diff: %v]\n",
			m.Asana.Name, m.Asana.Modified.Sub(asanaTs))
		pushNotification("Update", m.Asana.Name)

		if err := taskwarrior.OverwriteUuid(m.Asana, m.TaskWr.Uuid); err != nil {
			return errors.Wrap(err, "Overwrite Taskwarrior")
		}
		updated, err := taskwarrior.GetTask(m.TaskWr.Uuid)
		if err != nil {
			return errors.Wrap(err, "Overwrite Taskwarrior GetTask")
		}
		storeInDb(m.Asana, updated)
		return nil
	}

	// If task has been marked as deleted since the last modification.
	if m.TaskWr.Deleted && approxAfter(m.TaskWr.Modified, taskwTs) {
		if deleteFromAsana != nil {
			*deleteFromAsana = append(*deleteFromAsana, m)
			return nil
		}

		fmt.Printf("Deleting task from Asana: [%q]\n", m.TaskWr.Name)
		pushNotification("Deleting from Asana", m.TaskWr.Name)
		if err := asana.Delete(m.Xid); err != nil {
			return errors.Wrap(err, "Delete task from Asana")
		}

		// Don't delete from boltdb, but update the timestamps,
		// so we don't reapply this deletion.
		// We don't need to retrieve Asana task back, because we won't be able to. It has been deleted.
		// If the task gets undeleted, Asana won't modify the timestamp. So, let's set it to zero
		// in our records, so if it comes back, we'll see it as an update.
		m.Asana.Modified = time.Time{}
		storeInDb(m.Asana, m.TaskWr)
		return nil
	}

	if approxAfter(m.TaskWr.Modified, taskwTs) {
		// TW was updated. Overwrite Asana.
		fmt.Printf("Overwrite Asana: [%q] [time diff: %v]\n",
			m.TaskWr.Name, m.TaskWr.Modified.Sub(taskwTs))

		if err := asana.UpdateTask(m.TaskWr, m.Asana); err != nil {
			return errors.Wrap(err, "syncMatch overwrite asana")
		}
		updated, err := asana.GetOneTask(m.Xid)
		if err != nil {
			return errors.Wrap(err, "syncMatch GetOneTask")
		}
		storeInDb(updated, m.TaskWr)
		return nil
	}
	return nil
}

func runSync() {
	atasks, err := asana.GetTasks()
	// atasks, err := asana.GetTasks(1)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Printf("%27s: %d active\n", "Asana results found", len(atasks))

	twtasks, err := taskwarrior.GetTasks()
	if err != nil {
		log.Fatal(err)
	}

	deleted := 0
	for _, t := range twtasks {
		if t.Deleted {
			deleted++
		}
	}
	fmt.Printf("%27s: %d active, %d deleted\n",
		"Taskwarrior results found", len(twtasks)-deleted, deleted)

	matches := generateMatches(atasks, twtasks)
	deletes := make([]*Match, 0, 10)
	for _, m := range matches {
		if err := syncMatch(m, &deletes); err != nil {
			log.Printf("syncMatch error: %v %+v", err, m)
		}
	}

	if len(deletes) > *maxDeletes {
		fmt.Printf(`
==========================================
Task deletions requested from Asana : %d
Max allowed per sync                : %d
Most likely this is a mistake!
If so, please run 'rm -f ~/.task/*' to clean up Taskwarrior (or wherever else you store it's state).
If you genuinely want to delete all these tasks, then set the 'deletes' flag to a higher value.
Crashing to avoid mass deletes from Asana!
==========================================
`, len(deletes), *maxDeletes)
		os.Exit(1)
	}
	for _, m := range deletes {
		if err := syncMatch(m, nil); err != nil {
			log.Printf("syncMatch error: %v %+v", err, m)
		}
	}

	fmt.Println("All synced up. DONE.")
}

func pushNotification(title, text string) {
	if notify == nil {
		return
	}
	n := notification{Title: title, Text: text}

	select {
	case notifications <- n:
	default:
		// Let it go.
	}

}

func processNotifications() {
	ni := time.Duration(*notifyInterval)
	ticker := time.NewTicker(ni * time.Second)
	l := make([]notification, 0, 10)
	for {
		select {
		case <-ticker.C:
			if len(l) == 0 {
				// pass

			} else if len(l) == 1 {
				n := l[0]
				notify.Push("Asanawarrior "+n.Title, n.Text, "", notificator.UR_NORMAL)

			} else {
				notify.Push("Asanawarrior "+l[0].Title,
					fmt.Sprintf("%q and %d more updates", l[0].Text, len(l)-1), "", notificator.UR_NORMAL)
			}
			l = l[:0]

		case n := <-notifications:
			if *notifyInterval > 0 {
				l = append(l, n)
			}
		}
	}
}

func main() {
	flag.Parse()
	fmt.Println("Asanawarrior v0.8 - Bringing the power of Taskwarrior to Asana")
	notify = notificator.New(notificator.Options{
		AppName: "Asanawarrior",
	})
	go processNotifications()

	var err error
	db, err = bolt.Open(*dbpath, 0600, nil)
	if err != nil {
		log.Fatalf("Unable to open bolt db at %v. Error: %v", *dbpath, err)
	}
	defer db.Close()
	db.Update(func(tx *bolt.Tx) error {
		_, err := tx.CreateBucketIfNotExists(bucketName)
		if err != nil {
			log.Fatalf("Unable to create bucket in bolt db.")
		}
		return nil
	})

	// Initiate a sync right away.
	fmt.Println()
	fmt.Println("Starting sync at", time.Now())
	runSync()

	// And then do it at regular intervals.
	ticker := time.NewTicker(time.Duration(*duration) * time.Minute)
	for t := range ticker.C {
		fmt.Println()
		fmt.Println("Starting sync at", t)
		runSync()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/evolsnow/robot/conn"
	"golang.org/x/net/websocket"
	"log"
	"net/http"
	"strings"
	"time"
	"unicode"
)

func main() {
	conn.Pool = conn.NewPool("127.0.0.1:6379", "", 1)
	if !conn.Ping("127.0.0.1:6379", "") {
		log.Fatal("connect to redis server failed")
	}

	samaritan := newRobot("164760320:AAEE0sKLgCwHGYJ0Iqz7o-GYH4jVTQZAZho", "samaritan")
	jarvis := newRobot("176820788:AAH26vgFIk7oWKibd7P8XHHZX2t2_2Jvke8", "jarvis")
	//	samaritan.bot.Debug = true
	go samaritan.run()
	go jarvis.run()

	http.Handle("/websocket", websocket.Handler(socketHandler))
	log.Fatal(http.ListenAndServe("localhost:8000", nil))
	//used for 104
	//go http.ListenAndServeTLS("0.0.0.0:8443", "server.crt", "server.key", nil)
}

func socketHandler(ws *websocket.Conn) {
	queue, wait := 0, 0
	go func() {
		for {
			time.Sleep(time.Second)
			fmt.Println(queue)
		}
	}()
	for {
		var in, response string
		var ret []string
		sf := func(c rune) bool {
			return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
		}
		if err := websocket.Message.Receive(ws, &in); err != nil {
			log.Println(err)
			return
		}
		fmt.Printf("Received: %s\n", in)
		queue++
		wait++
		zh := false
		for _, r := range in {
			if unicode.Is(unicode.Scripts["Han"], r) {
				in = strings.TrimSpace(in)
				zh = true
				break
			}
		}
		if zh {
			response = tlAI(in)
			// Separate into fields with func.
			ret = strings.FieldsFunc(response, sf)

		} else {
			response = mitAI(in)
			ret = strings.FieldsFunc(response, sf)
		}
		//		for i := range ret {
		//
		//			websocket.Message.Send(ws, ret[i])
		//			time.Sleep(time.Second)
		//		}
		//		websocket.Message.Send(ws, "")
		for {
			if queue > 1 {
				//wait
			} else {
				go func(ws *websocket.Conn, ret []string) {
					defer func() {
						queue--
					}()
					for i := range ret {
						if wait > 0 {
							defer func() {
								wait--
								queue--
							}()
							log.Println("new comming")
							return
						}
						websocket.Message.Send(ws, ret[i])
						time.Sleep(time.Second)
					}
					websocket.Message.Send(ws, "")
					return
				}(ws, ret)
			}
			break
		}
	}
}
<commit_msg>split<commit_after>package main

import (
	"fmt"
	"github.com/evolsnow/robot/conn"
	"golang.org/x/net/websocket"
	"log"
	"net/http"
	"strings"
	"time"
	"unicode"
)

func main() {
	conn.Pool = conn.NewPool("127.0.0.1:6379", "", 1)
	if !conn.Ping("127.0.0.1:6379", "") {
		log.Fatal("connect to redis server failed")
	}

	samaritan := newRobot("164760320:AAEE0sKLgCwHGYJ0Iqz7o-GYH4jVTQZAZho", "samaritan")
	jarvis := newRobot("176820788:AAH26vgFIk7oWKibd7P8XHHZX2t2_2Jvke8", "jarvis")
	//	samaritan.bot.Debug = true
	go samaritan.run()
	go jarvis.run()

	http.Handle("/websocket", websocket.Handler(socketHandler))
	log.Fatal(http.ListenAndServe("localhost:8000", nil))
	//used for 104
	//go http.ListenAndServeTLS("0.0.0.0:8443", "server.crt", "server.key", nil)
}

func socketHandler(ws *websocket.Conn) {
	working, wait := 0, 0
	go func() {
		for {
			time.Sleep(time.Second)
			fmt.Println(wait)
		}
	}()
	for {
		var in, response string
		var ret []string
		sf := func(c rune) bool {
			return c == ',' || c == '，' || c == ';' || c == '。' || c == '.' || c == '？' || c == '?'
		}
		if err := websocket.Message.Receive(ws, &in); err != nil {
			log.Println(err)
			return
		}
		fmt.Printf("Received: %s\n", in)
		wait++
		zh := false
		for _, r := range in {
			if unicode.Is(unicode.Scripts["Han"], r) {
				in = strings.TrimSpace(in)
				zh = true
				break
			}
		}
		if zh {
			response = tlAI(in)
			// Separate into fields with func.
			ret = strings.FieldsFunc(response, sf)

		} else {
			response = mitAI(in)
			ret = strings.FieldsFunc(response, sf)
		}
		//		for i := range ret {
		//
		//			websocket.Message.Send(ws, ret[i])
		//			time.Sleep(time.Second)
		//		}
		//		websocket.Message.Send(ws, "")
		for {
			if working > 0 {
				//wait
			} else {
				go func(ws *websocket.Conn, ret []string) {
					working++
					wait--
					for i := range ret {
						if wait > 0 {
							working--
							log.Println("new comming")
							return
						}
						websocket.Message.Send(ws, ret[i])
						time.Sleep(time.Second)
					}
					websocket.Message.Send(ws, "")
					working--
					return
				}(ws, ret)
			}
			break
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/Financial-Times/organisations-rw-neo4j/organisations"
	log "github.com/Sirupsen/logrus"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"net/http"
)

func main() {
	log.SetLevel(log.InfoLevel)
	log.Println("Application started with args %s", os.Args)
	app := cli.App("organisations-rw-neo4j", "A RESTful API for managing Organisations in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	// neoURL := app.StringOpt("neo-url", "http://ftper58827-law1b-eu-t:8080/db/data", "neo4j endpoint URL")
	port := app.IntOpt("port", 8080, "Port to listen on")
	env := app.StringOpt("env", "local", "environment this app is running in")
	batchSize := app.IntOpt("batchSize", 1024, "Maximum number of statements to execute per batch")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.organisations.rw.neo4j.ftaps58938-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")

	app.Action = func() {
		db, err := neoism.Connect(neoURL)
		db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
		if err != nil {
			log.Errorf("Could not connect to neo4j, error=[%s]\n", err)
		}
		batchRunner := neoutils.NewBatchCypherRunner(neoutils.StringerDb{db}, *batchSize)
		organisationsDriver := organisations.NewCypherOrganisationService(batchRunner, db)
		organisationsDriver.Initialise()

		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)

		engs := map[string]baseftrwapp.Service{
			"organisations": organisationsDriver,
		}

		var checks []v1a.Check
		for _, e := range engs {
			checks = append(checks, makeCheck(e, batchRunner))
		}

		baseftrwapp.RunServer(engs,
			v1a.Handler("ft-organisations_rw_neo4j ServiceModule", "Writes 'organisations' to Neo4j, usually as part of a bulk upload done on a schedule", checks...),
			*port, "organisations-rw-neo4j", *env)
	}
	log.SetLevel(log.InfoLevel)
	log.Println("Application started with args %s", os.Args)

	app.Run(os.Args)
}

func makeCheck(service baseftrwapp.Service, cr neoutils.CypherRunner) v1a.Check {
	return v1a.Check{
		BusinessImpact:   "Cannot read/write organisations via this writer",
		Name:             "Check connectivity to Neo4j - neoUrl is a parameter in hieradata for this service",
		PanicGuide:       "TODO - write panic guide",
		Severity:         1,
		TechnicalSummary: fmt.Sprintf("Cannot connect to Neo4j instance %s with at least one organisation loaded in it", cr),
		Checker:          func() (string, error) { return "", service.Check() },
	}
}
<commit_msg>Fixing breaking build.<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/Financial-Times/base-ft-rw-app-go/baseftrwapp"
	"github.com/Financial-Times/go-fthealth/v1a"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	"github.com/Financial-Times/organisations-rw-neo4j/organisations"
	log "github.com/Sirupsen/logrus"
	"github.com/jawher/mow.cli"
	"github.com/jmcvetta/neoism"
	"net/http"
)

func main() {
	log.SetLevel(log.InfoLevel)
	log.Println("Application started with args %s", os.Args)
	app := cli.App("organisations-rw-neo4j", "A RESTful API for managing Organisations in neo4j")
	neoURL := app.StringOpt("neo-url", "http://localhost:7474/db/data", "neo4j endpoint URL")
	// neoURL := app.StringOpt("neo-url", "http://ftper58827-law1b-eu-t:8080/db/data", "neo4j endpoint URL")
	port := app.IntOpt("port", 8080, "Port to listen on")
	env := app.StringOpt("env", "local", "environment this app is running in")
	batchSize := app.IntOpt("batchSize", 1024, "Maximum number of statements to execute per batch")
	graphiteTCPAddress := app.StringOpt("graphiteTCPAddress", "",
		"Graphite TCP address, e.g. graphite.ft.com:2003. Leave as default if you do NOT want to output to graphite (e.g. if running locally)")
	graphitePrefix := app.StringOpt("graphitePrefix", "",
		"Prefix to use. Should start with content, include the environment, and the host name. e.g. content.test.organisations.rw.neo4j.ftaps58938-law1a-eu-t")
	logMetrics := app.BoolOpt("logMetrics", false, "Whether to log metrics. Set to true if running locally and you want metrics output")

	app.Action = func() {
		db, err := neoism.Connect(*neoURL)
		db.Session.Client = &http.Client{Transport: &http.Transport{MaxIdleConnsPerHost: 100}}
		if err != nil {
			log.Errorf("Could not connect to neo4j, error=[%s]\n", err)
		}
		batchRunner := neoutils.NewBatchCypherRunner(neoutils.StringerDb{db}, *batchSize)
		organisationsDriver := organisations.NewCypherOrganisationService(batchRunner, db)
		organisationsDriver.Initialise()

		baseftrwapp.OutputMetricsIfRequired(*graphiteTCPAddress, *graphitePrefix, *logMetrics)

		engs := map[string]baseftrwapp.Service{
			"organisations": organisationsDriver,
		}

		var checks []v1a.Check
		for _, e := range engs {
			checks = append(checks, makeCheck(e, batchRunner))
		}

		baseftrwapp.RunServer(engs,
			v1a.Handler("ft-organisations_rw_neo4j ServiceModule", "Writes 'organisations' to Neo4j, usually as part of a bulk upload done on a schedule", checks...),
			*port, "organisations-rw-neo4j", *env)
	}
	log.SetLevel(log.InfoLevel)
	log.Println("Application started with args %s", os.Args)

	app.Run(os.Args)
}

func makeCheck(service baseftrwapp.Service, cr neoutils.CypherRunner) v1a.Check {
	return v1a.Check{
		BusinessImpact:   "Cannot read/write organisations via this writer",
		Name:             "Check connectivity to Neo4j - neoUrl is a parameter in hieradata for this service",
		PanicGuide:       "TODO - write panic guide",
		Severity:         1,
		TechnicalSummary: fmt.Sprintf("Cannot connect to Neo4j instance %s with at least one organisation loaded in it", cr),
		Checker:          func() (string, error) { return "", service.Check() },
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
)

var weatherUrl = fmt.Sprintf("http://api.wunderground.com/api/%s/%%s/q/%s/%s.json",
	os.Getenv("WUNDERGROUND_API_KEY"),
	os.Getenv("TWEATHER_COUNTRY"),
	os.Getenv("TWEATHER_CITY"),
)

type Weather struct {
	Observation struct {
		Condition      string `json:"weather"`
		Temperature    int    `json:"temp_c"`
		TemperatureMod string `json:"feelslike_c"`
		Wind           int    `json:"wind_kph"`
		Visibility     string `json:"visibility_km"`
	} `json:"current_observation"`
	Forecast struct {
		Simple struct {
			Day []struct {
				High struct {
					Celsius string
				}
				Low struct {
					Celsius string
				}
				Precipitation int
			} `json:"forecastday"`
		} `json:"simpleforecast"`
	}
}

func (w *Weather) String() string {
	o := &w.Observation
	f := &w.Forecast.Simple.Day[0]
	return fmt.Sprintf("%s at %dºC (%sºC)\nWind %dkm/h — Visibility %skm\n▼%sºC ▲%sºC ☂%d%%",
		o.Condition, o.Temperature, o.TemperatureMod,
		o.Wind, o.Visibility,
		f.Low.Celsius, f.High.Celsius, f.Precipitation,
	)
}

func weatherFrom(features []string) (*Weather, error) {
	url := fmt.Sprintf(weatherUrl, strings.Join(features, "/"))
	rv, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer rv.Body.Close()
	if rv.StatusCode != http.StatusOK {
		return nil, errors.New(rv.Status)
	}
	weather := new(Weather)
	err = json.NewDecoder(rv.Body).Decode(weather)
	if err != nil {
		return nil, err
	}
	return weather, nil
}

func main() {
	weather, err := weatherFrom([]string{"conditions", "forecast"})
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(weather)
}
<commit_msg>Tweet the weather.<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"

	"github.com/ChimeraCoder/anaconda"
)

var weatherUrl = fmt.Sprintf("http://api.wunderground.com/api/%s/%%s/q/%s/%s.json",
	os.Getenv("WUNDERGROUND_API_KEY"),
	os.Getenv("TWEATHER_COUNTRY"),
	os.Getenv("TWEATHER_CITY"),
)

type Weather struct {
	Observation struct {
		Condition      string `json:"weather"`
		Temperature    int    `json:"temp_c"`
		TemperatureMod string `json:"feelslike_c"`
		Wind           int    `json:"wind_kph"`
		Visibility     string `json:"visibility_km"`
	} `json:"current_observation"`
	Forecast struct {
		Simple struct {
			Day []struct {
				High struct {
					Celsius string
				}
				Low struct {
					Celsius string
				}
				Precipitation int
			} `json:"forecastday"`
		} `json:"simpleforecast"`
	}
}

func (w *Weather) String() string {
	o := &w.Observation
	f := &w.Forecast.Simple.Day[0]
	return fmt.Sprintf("%s at %dºC (%sºC)\nWind %dkm/h — Visibility %skm\n▼%sºC ▲%sºC ☂%d%%",
		o.Condition, o.Temperature, o.TemperatureMod,
		o.Wind, o.Visibility,
		f.Low.Celsius, f.High.Celsius, f.Precipitation,
	)
}

func weatherFrom(features []string) (*Weather, error) {
	url := fmt.Sprintf(weatherUrl, strings.Join(features, "/"))
	rv, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer rv.Body.Close()
	if rv.StatusCode != http.StatusOK {
		return nil, errors.New(rv.Status)
	}
	weather := new(Weather)
	err = json.NewDecoder(rv.Body).Decode(weather)
	if err != nil {
		return nil, err
	}
	return weather, nil
}

func main() {
	anaconda.SetConsumerKey(os.Getenv("TWITTER_CONSUMER_KEY"))
	anaconda.SetConsumerSecret(os.Getenv("TWITTER_CONSUMER_SECRET"))
	weather, err := weatherFrom([]string{"conditions", "forecast"})
	if err != nil {
		log.Fatal(err)
	}
	api := anaconda.NewTwitterApi(os.Getenv("TWITTER_ACCESS_TOKEN"), os.Getenv("TWITTER_ACCESS_TOKEN_SECRET"))
	_, err = api.PostTweet(weather.String(), nil)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"encoding/binary"
	"log"
	"net/http"
	"reflect"
	"time"
)

// Block is the building blocks of a block chain
type Block struct {
	index     uint64
	timestamp int64
	prevHash  []byte
	data      []byte
	hash      []byte
}

const (
	MsgTypeQueryLatest    = 0
	MsgTypeQueryAll       = 1
	MsgTypeRespBlockchain = 2
)

const (
	HashLen = 32
	IntSize = 8
)

// CalculateHash calculates the hash of a block
func CalculateHash(index uint64, ts int64, prevHash []byte, data []byte) []byte {
	if len(prevHash) != HashLen {
		log.Fatal("CalculateHash(): Previous hash is not 32 bytes")
	}

	h := sha256.New()

	b := make([]byte, IntSize)
	binary.LittleEndian.PutUint64(b, index)
	h.Write(b)

	// uint64 conversion doesn't change the sign bit, only the way it's interpreted
	t := make([]byte, IntSize)
	binary.LittleEndian.PutUint64(t, uint64(ts))
	h.Write(t)

	h.Write(prevHash)
	h.Write(data)

	log.Printf("Calculated hash: %x\n", h.Sum(nil))
	return h.Sum(nil)
}

// CalculateHashForBlock calculates the has for the block
func CalculateHashForBlock(block *Block) []byte {
	return CalculateHash(block.index, block.timestamp, block.prevHash, block.data)
}

// GenerateNextBlock generates next block in the chain
func GenerateNextBlock(blockData []byte) *Block {
	prevBlock := GetLatestBlock()
	nextIndex := prevBlock.index + 1
	nextTimestamp := time.Now().UTC().Unix()
	nextHash := CalculateHash(nextIndex, nextTimestamp, prevBlock.prevHash, blockData)
	return &Block{nextIndex, nextTimestamp, prevBlock.prevHash, blockData, nextHash}
}

// GetGensisBlock generates and returns the genesis block
func GetGensisBlock() *Block {
	var e = make([]byte, HashLen)
	timeNow := time.Now().UTC().Unix()
	genesisHash := CalculateHash(0, timeNow, e, e)
	return &Block{0, timeNow, e, e, genesisHash}
}

// GetLatestBlock retrieves the last block in the chain
func GetLatestBlock() *Block {
	// returns last element in slice
	return &blockchain[len(blockchain)-1]
}

// IsValidNewBlock checks the integrity of the newest block
func IsValidNewBlock(newBlock, prevBlock *Block) bool {
	if prevBlock.index+1 != newBlock.index {
		log.Println("IsValidNewBlock(): invalid index")
		return false
	} else if !reflect.DeepEqual(prevBlock.hash, newBlock.prevHash) {
		log.Println("IsValidNewBlock(): invalid previous hash")
		return false
	} else if !reflect.DeepEqual(CalculateHashForBlock(newBlock), newBlock.hash) {
		log.Println("invalid hash: %x is not %x", CalculateHashForBlock(newBlock), newBlock.hash)
		return false
	}
	return true
}

// IsValidChain checks if the chain received is valid
func IsValidChain(newBlockchain []Block) { // TODO: pass by ref?

}

// ReplaceChain checks and see if the current chain stored on the node needs to be replaced with
// a more up to date version (which is validated).
func ReplaceChain(newBlockchain []Block) { // TODO: pass by ref?
	if IsValidChain(newBlockchain) && (len(newBlockchain) > blockchain.length) {
		log.Println("Received blockchain is valid. Replacing current blockchain with the received blockchain")
		blockchain = newBlockchain
		// broadcast(responseLatestMsg())
	} else {
		log.Println("Received blockchain invalid")
	}
}

func InitHttpServer() {
	s := &http.Server {
		Addr: ":" + HttpPort,
		Handler:
	}
}

var blockchain []Block

func main() {
	//var sockets

	blockchain = append(blockchain, *GetGensisBlock())

}
<commit_msg>in progress<commit_after>package main

import (
	"crypto/sha256"
	"encoding/binary"
	"log"
	"net/http"
	"reflect"
	"strconv"
	"time"

	"github.com/gorilla/mux"
	"github.com/gorilla/websocket"
)

// Block is the building blocks of a block chain
type Block struct {
	index     uint64
	timestamp int64
	prevHash  []byte
	data      []byte
	hash      []byte
}

const (
	MsgTypeQueryLatest    = 0
	MsgTypeQueryAll       = 1
	MsgTypeRespBlockchain = 2
)

const (
	HashLen = 32
	IntSize = 8
)

// CalculateHash calculates the hash of a block
func CalculateHash(index uint64, ts int64, prevHash []byte, data []byte) []byte {
	if len(prevHash) != HashLen {
		log.Fatal("CalculateHash(): Previous hash is not 32 bytes")
	}

	h := sha256.New()

	b := make([]byte, IntSize)
	binary.LittleEndian.PutUint64(b, index)
	h.Write(b)

	// uint64 conversion doesn't change the sign bit, only the way it's interpreted
	t := make([]byte, IntSize)
	binary.LittleEndian.PutUint64(t, uint64(ts))
	h.Write(t)

	h.Write(prevHash)
	h.Write(data)

	log.Printf("Calculated hash: %x\n", h.Sum(nil))
	return h.Sum(nil)
}

// CalculateHashForBlock calculates the has for the block
func CalculateHashForBlock(block *Block) []byte {
	return CalculateHash(block.index, block.timestamp, block.prevHash, block.data)
}

// GenerateNextBlock generates next block in the chain
func GenerateNextBlock(blockData []byte) *Block {
	prevBlock := GetLatestBlock()
	nextIndex := prevBlock.index + 1
	nextTimestamp := time.Now().UTC().Unix()
	nextHash := CalculateHash(nextIndex, nextTimestamp, prevBlock.prevHash, blockData)
	return &Block{nextIndex, nextTimestamp, prevBlock.prevHash, blockData, nextHash}
}

// GetGensisBlock generates and returns the genesis block
func GetGensisBlock() *Block {
	var e = make([]byte, HashLen)
	timeNow := time.Now().UTC().Unix()
	genesisHash := CalculateHash(0, timeNow, e, e)
	return &Block{0, timeNow, e, e, genesisHash}
}

// GetLatestBlock retrieves the last block in the chain
func GetLatestBlock() *Block {
	// returns last element in slice
	return &blockchain[len(blockchain)-1]
}

// IsValidNewBlock checks the integrity of the newest block
func IsValidNewBlock(newBlock, prevBlock *Block) bool {
	if prevBlock.index+1 != newBlock.index {
		log.Println("IsValidNewBlock(): invalid index")
		return false
	} else if !reflect.DeepEqual(prevBlock.hash, newBlock.prevHash) {
		log.Println("IsValidNewBlock(): invalid previous hash")
		return false
	} else if !reflect.DeepEqual(CalculateHashForBlock(newBlock), newBlock.hash) {
		log.Println("invalid hash: %x is not %x", CalculateHashForBlock(newBlock), newBlock.hash)
		return false
	}
	return true
}

// IsValidChain checks if the chain received is valid
func IsValidChain(newBlockchain []Block) bool { // TODO: pass by ref?
	// newBlockchain is in JSON format
	// Check if the genesis block matches
	//genesisBlockRaw := []byte(newBlockchain[0])
	//var genesisBlock Block
	//err := json.Unmarshal(genesisBlockRaw, &genesisBlock)
	//if err != nil {
	//panic(err)
	//}
	if !reflect.DeepEqual(newBlockchain[0], blockchain[0]) {
		return false
	}

	var tempBlocks []Block
	tempBlocks = append(tempBlocks, newBlockchain[0])
	for i := 1; i < len(newBlockchain); i++ {
		if IsValidNewBlock(&newBlockchain[i], &tempBlocks[i-1]) {
			tempBlocks = append(tempBlocks, newBlockchain[i])
		} else {
			return false
		}
	}

	// All blocks are valid in the new blockchain
	return true
}

// ReplaceChain checks and see if the current chain stored on the node needs to be replaced with
// a more up to date version (which is validated).
func ReplaceChain(newBlockchain []Block) { // TODO: pass by ref?
	if IsValidChain(newBlockchain) && (len(newBlockchain) > len(blockchain)) {
		log.Println("Received blockchain is valid. Replacing current blockchain with the received blockchain")
		blockchain = newBlockchain
		// broadcast(responseLatestMsg())
	} else {
		log.Println("Received blockchain invalid")
	}
}

// InitHTTPServer initializes the http server and REST api
// TODO: refactor http portion to a new file
func InitHTTPServer() {
	// need to pass these into command line arguments
	HttpPort := 3001

	// setup REST interface
	router = mux.NewRouter().StrictSlash(true)

	router.HandleFunc("/blocks", GetBlocks).Methods("GET")
	router.HandleFunc("/mineBlock", MineBlock).Methods("POST")
	router.HandleFunc("/peers", GetPeers).Methods("GET")
	router.HandleFunc("/addPeer", AddPeer).Methods("POST")

	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(HttpPort), router))
}

// GetBlocks handles the /getBlocks REST request
func GetBlocks(w http.ResponseWriter, r *http.Request) {

}

// MineBlock handles the /mineBlock REST post
func MineBlock(w http.ResponseWriter, r *http.Request) {

}

// GetPeers handles the /peers REST request
func GetPeers(w http.ResponseWriter, r *http.Request) {

}

// AddPeer handles the /addPeer REST post
func AddPeer(w http.ResponseWriter, r *http.Request) {

}

// InitP2PServer sets up the websocket to other nodes
func InitP2PServer() {
	// need to pass these into command line arguments
	//P2pPort := 6001

}

var blockchain []Block
var router *mux.Router
var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

func main() {
	//var sockets

	// generate genesis block
	blockchain = append(blockchain, *GetGensisBlock())

}
<|endoftext|>


<commit_before>// +build darwin linux
package main
import (
	"flag"
	"os"
	"fmt"
	"time"
	"errors"
	"strings"
	"strconv"
	"io/ioutil"
	"path/filepath"
)

var VERSION = [...]int{1, 0, 3}

func getVersion() string {
	versionStr := fmt.Sprintf("v%v", VERSION[0])
	for i := 1; i < len(VERSION); i++ {
		versionStr = fmt.Sprintf("%v.%v", versionStr, VERSION[i])
	}
	return versionStr
}

var minUptime = flag.Duration("minuptime", 0, "will not exit 0 before uptime >= <minuptime>")
var locks = flag.String("lock", "/tmp/lockfiles/", "where to look for lockfiles")
var lockDur = flag.Duration("duration", time.Minute * 10, "duration for which lock files are considered valid")
var sleepDur = flag.Duration("sleep", 0, "duration to sleep at the end of script before exit 0")

func exists(path string) (bool, error) {
    _, err := os.Stat(path)
    if err == nil { return true, nil }
    if os.IsNotExist(err) { return false, nil }
    return true, err
}

func lockIsStale(lockFile string) (bool, error) {
	info, err := os.Stat(lockFile)
	if err != nil {
		return false, err
	}
	lockAge := time.Now().Sub(info.ModTime())
	if lockAge > *lockDur {
		return true, nil
	} else {
		return false, nil
	}
}

func main() {
	flag.Parse()

	logging("/var/log/dozy", fmt.Sprint(" `( ◔ ౪◔)´  dozy ", getVersion()))
	Info.Println(fmt.Sprintf("minimum uptime: %v, locks valid for %vm, lock: %v", *minUptime, *lockDur, *locks))

	uptime_str, err := ioutil.ReadFile("/proc/uptime")
	if err != nil {
		panic(err)
	}
	uptime_secnds, err := strconv.Atoi(strings.Split(string(uptime_str), ".")[0])
	if err != nil {
		panic(err)
	}
	uptime := time.Duration(int(uptime_secnds)) * time.Second

	if uptime < *minUptime {
		panic(fmt.Sprintf("uptime not Ok (%v < %v)", uptime, *minUptime))
	} else {
		Info.Println(fmt.Sprintf("uptime is Ok (%v > %v)", uptime, *minUptime))
	}

	if locksPlaceExists, _ := exists(*locks); !locksPlaceExists {
		panic("specified locks location doesn't exist")
	}

	fh, err := os.Open(*locks)
	if err != nil {
		Error.Println(err)
	}
	defer fh.Close()
	locksFh, err := fh.Stat()
	if err != nil {
		panic(err)
		return
	}

	if locksFh.IsDir() {
		// locks is a directory
		Info.Println("input is a directory, scanning")
		fileList := []string{}
		err := filepath.Walk(*locks, func(path string, f os.FileInfo, err error) error {
			if ! f.IsDir() {
				fileList = append(fileList, path)
			}
			return nil
		})
		if err != nil {
			panic(err)
		}
		if (len(fileList) == 0) {
			Info.Println("no locks found, killing containers")
			err := firstDegree()
			if err != nil {
				panic(err)
			}
		} else {
			counter := 0
			for i := range (fileList) {
				isStale, err := lockIsStale(fileList[i])
				if err != nil {
					panic(err)
				}
				if isStale {
					Error.Println(fmt.Sprintf("found stale lock: %v, continuing..", fileList[i]))
				} else {
					panic("valid lock found.")
				}
				counter++
			}
			Info.Println(fmt.Sprintf("%v lock(s) checked", counter))
		}
	} else if locksFh.Mode().IsRegular() {
			// locks is a file
			Info.Println("lock is a file, checking...")
			isStale, err := lockIsStale(*locks)
			if err != nil {
				panic(err)
			}
			if isStale {
				Error.Println("WARNING: stale lock, killing containers and shutting down")
				firstDegree()
			} else {
				panic("valid lock found, exiting.")
			}
	} else {
		panic(errors.New(fmt.Sprintf("fucked up lock(s). -lock=%s", *locks)))
	}
	Info.Println("shutting down...")
	time.Sleep(*sleepDur)
}
<commit_msg>bumpped up version number<commit_after>// +build darwin linux
package main
import (
	"flag"
	"os"
	"fmt"
	"time"
	"errors"
	"strings"
	"strconv"
	"io/ioutil"
	"path/filepath"
)

var VERSION = [...]int{1, 0, 4}

func getVersion() string {
	versionStr := fmt.Sprintf("v%v", VERSION[0])
	for i := 1; i < len(VERSION); i++ {
		versionStr = fmt.Sprintf("%v.%v", versionStr, VERSION[i])
	}
	return versionStr
}

var minUptime = flag.Duration("minuptime", 0, "will not exit 0 before uptime >= <minuptime>")
var locks = flag.String("lock", "/tmp/lockfiles/", "where to look for lockfiles")
var lockDur = flag.Duration("duration", time.Minute * 10, "duration for which lock files are considered valid")
var sleepDur = flag.Duration("sleep", 0, "duration to sleep at the end of script before exit 0")

func exists(path string) (bool, error) {
    _, err := os.Stat(path)
    if err == nil { return true, nil }
    if os.IsNotExist(err) { return false, nil }
    return true, err
}

func lockIsStale(lockFile string) (bool, error) {
	info, err := os.Stat(lockFile)
	if err != nil {
		return false, err
	}
	lockAge := time.Now().Sub(info.ModTime())
	if lockAge > *lockDur {
		return true, nil
	} else {
		return false, nil
	}
}

func main() {
	flag.Parse()

	logging("/var/log/dozy", fmt.Sprint(" `( ◔ ౪◔)´  dozy ", getVersion()))
	Info.Println(fmt.Sprintf("minimum uptime: %v, locks valid for %vm, lock: %v", *minUptime, *lockDur, *locks))

	uptime_str, err := ioutil.ReadFile("/proc/uptime")
	if err != nil {
		panic(err)
	}
	uptime_secnds, err := strconv.Atoi(strings.Split(string(uptime_str), ".")[0])
	if err != nil {
		panic(err)
	}
	uptime := time.Duration(int(uptime_secnds)) * time.Second

	if uptime < *minUptime {
		panic(fmt.Sprintf("uptime not Ok (%v < %v)", uptime, *minUptime))
	} else {
		Info.Println(fmt.Sprintf("uptime is Ok (%v > %v)", uptime, *minUptime))
	}

	if locksPlaceExists, _ := exists(*locks); !locksPlaceExists {
		panic("specified locks location doesn't exist")
	}

	fh, err := os.Open(*locks)
	if err != nil {
		Error.Println(err)
	}
	defer fh.Close()
	locksFh, err := fh.Stat()
	if err != nil {
		panic(err)
		return
	}

	if locksFh.IsDir() {
		// locks is a directory
		Info.Println("input is a directory, scanning")
		fileList := []string{}
		err := filepath.Walk(*locks, func(path string, f os.FileInfo, err error) error {
			if ! f.IsDir() {
				fileList = append(fileList, path)
			}
			return nil
		})
		if err != nil {
			panic(err)
		}
		if (len(fileList) == 0) {
			Info.Println("no locks found, killing containers")
			err := firstDegree()
			if err != nil {
				panic(err)
			}
		} else {
			counter := 0
			for i := range (fileList) {
				isStale, err := lockIsStale(fileList[i])
				if err != nil {
					panic(err)
				}
				if isStale {
					Error.Println(fmt.Sprintf("found stale lock: %v, continuing..", fileList[i]))
				} else {
					panic("valid lock found.")
				}
				counter++
			}
			Info.Println(fmt.Sprintf("%v lock(s) checked", counter))
		}
	} else if locksFh.Mode().IsRegular() {
			// locks is a file
			Info.Println("lock is a file, checking...")
			isStale, err := lockIsStale(*locks)
			if err != nil {
				panic(err)
			}
			if isStale {
				Error.Println("WARNING: stale lock, killing containers and shutting down")
				firstDegree()
			} else {
				panic("valid lock found, exiting.")
			}
	} else {
		panic(errors.New(fmt.Sprintf("fucked up lock(s). -lock=%s", *locks)))
	}
	Info.Println("shutting down...")
	time.Sleep(*sleepDur)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/drone/drone-plugin-go/plugin"
)

type Slack struct {
	Webhook   string `json:"webhook_url"`
	Channel   string `json:"channel"`
	Recipient string `json:"recipient"`
	Username  string `json:"username"`
}

func main() {
	repo := plugin.Repo{}
	build := plugin.Build{}
	system := plugin.System{}
	vargs := Slack{}

	plugin.Param("build", &build)
	plugin.Param("repo", &repo)
	plugin.Param("system", &system)
	plugin.Param("vargs", &vargs)

	// parse the parameters
	if err := plugin.Parse(); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	// create the Slack client
	client := Client{}
	client.Url = vargs.Webhook

	// generate the Slack message
	msg := Message{}
	msg.Username = vargs.Username

	if len(vargs.Recipient) != 0 {
		if strings.HasPrefix(vargs.Recipient, "@") {
			msg.Channel = vargs.Recipient
		} else {
			msg.Channel = "@" + vargs.Recipient
		}
	} else {
		if strings.HasPrefix(vargs.Channel, "#") {
			msg.Channel = vargs.Channel
		} else {
			msg.Channel = "#" + vargs.Channel
		}
	}

	attach := msg.NewAttachment()
	attach.Text = GetMessage(&repo, &build, &system)
	attach.Fallback = GetFallback(&repo, &build)
	attach.Color = GetColor(&build)
	attach.MrkdwnIn = []string{"text", "fallback"}

	// sends the message
	if err := client.SendMessage(&msg); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func GetMessage(repo *plugin.Repo, build *plugin.Build, sys *plugin.System) string {
	return fmt.Sprintf("*%s* <%s|%s/%s#%s> (%s) by %s",
		build.Status,
		fmt.Sprintf("%s/%s/%v", sys.Link, repo.FullName, build.Number),
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func GetFallback(repo *plugin.Repo, build *plugin.Build) string {
	return fmt.Sprintf("%s %s/%s#%s (%s) by %s",
		build.Status,
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func GetColor(build *plugin.Build) string {
	switch build.Status {
	case plugin.StateSuccess:
		return "good"
	case plugin.StateFailure, plugin.StateError, plugin.StateKilled:
		return "danger"
	default:
		return "warning"
	}
}
<commit_msg>added prepend func<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	"github.com/drone/drone-plugin-go/plugin"
)

type Slack struct {
	Webhook   string `json:"webhook_url"`
	Channel   string `json:"channel"`
	Recipient string `json:"recipient"`
	Username  string `json:"username"`
}

func main() {
	repo := plugin.Repo{}
	build := plugin.Build{}
	system := plugin.System{}
	vargs := Slack{}

	plugin.Param("build", &build)
	plugin.Param("repo", &repo)
	plugin.Param("system", &system)
	plugin.Param("vargs", &vargs)

	// parse the parameters
	if err := plugin.Parse(); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	// create the Slack client
	client := Client{}
	client.Url = vargs.Webhook

	// generate the Slack message
	msg := Message{}
	msg.Username = vargs.Username
	msg.Channel = vargs.Recipient

	if len(vargs.Recipient) != 0 {
		msg.Channel = Prepend("@", vargs.Recipient)
	} else {
		msg.Channel = Prepend("#", vargs.Channel)
	}

	attach := msg.NewAttachment()
	attach.Text = GetMessage(&repo, &build, &system)
	attach.Fallback = GetFallback(&repo, &build)
	attach.Color = GetColor(&build)
	attach.MrkdwnIn = []string{"text", "fallback"}

	// sends the message
	if err := client.SendMessage(&msg); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func Prepend(prefix, s string) string {
	if !strings.HasPrefix(s, prefix) {
		return prefix + s
	}
	return s
}

func GetMessage(repo *plugin.Repo, build *plugin.Build, sys *plugin.System) string {
	return fmt.Sprintf("*%s* <%s|%s/%s#%s> (%s) by %s",
		build.Status,
		fmt.Sprintf("%s/%s/%v", sys.Link, repo.FullName, build.Number),
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func GetFallback(repo *plugin.Repo, build *plugin.Build) string {
	return fmt.Sprintf("%s %s/%s#%s (%s) by %s",
		build.Status,
		repo.Owner,
		repo.Name,
		build.Commit[:8],
		build.Branch,
		build.Author,
	)
}

func GetColor(build *plugin.Build) string {
	switch build.Status {
	case plugin.StateSuccess:
		return "good"
	case plugin.StateFailure, plugin.StateError, plugin.StateKilled:
		return "danger"
	default:
		return "warning"
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"./twitch"
	"net/http"
	"io/ioutil"
	"os"
	"log"
)

func main() {
	fmt.Println("Booting the server...")

	// Configure a sample route
	http.HandleFunc("/users", myTwitchHandler)
	http.HandleFunc("/channel", myChannelHandler)

	// Run your server
	http.ListenAndServe(":8080", nil)
}

func myTwitchHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Recieved the following request:", r.Body)

	twitch.DoSomething()

	// YOUR ROUTES LOGIC GOES HERE
	//
	// Feel free to structure your routing however you see fit, this is just an example to get you started.

	client := &http.Client{}

	// resp, err := client.Get("https://api.twitch.tv/kraken/users?login=dallas,dallasnchains")

	request, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/users?login=dallas,dallasnchains", nil)
	if err != nil {
		fmt.Println(err.Error())
	 	os.Exit(1)
	}
	request.Header.Set("Client-ID", "ow8er2yjqzvbwzvfx832rrb5ucy933")
	request.Header.Set("Accept", "application/vnd.twitchtv.v5+json")

	resp, err := client.Do(request)

    responseData, err := ioutil.ReadAll(resp.Body)
    if err != nil {
		log.Fatal(err)
	}
    fmt.Println(string(responseData))

}

func myChannelHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Recieved the following request:", r.Body)

	// YOUR ROUTES LOGIC GOES HERE
	//
	// Feel free to structure your routing however you see fit, this is just an example to get you started.

	client := &http.Client{}

	// resp, err := client.Get("https://api.twitch.tv/kraken/users?login=dallas,dallasnchains")

	request, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/channels/44322889", nil)
	if err != nil {
		fmt.Println(err.Error())
	 	os.Exit(1)
	}
	request.Header.Set("Client-ID", "ow8er2yjqzvbwzvfx832rrb5ucy933")
	request.Header.Set("Accept", "application/vnd.twitchtv.v5+json")

	resp, err := client.Do(request)

    responseData, err := ioutil.ReadAll(resp.Body)
    if err != nil {
		log.Fatal(err)
	}
    fmt.Println(string(responseData))

}
<commit_msg>Streams Endpoint now available<commit_after>package main

import (
	"fmt"
	"./twitch"
	"net/http"
	"io/ioutil"
	"os"
	"log"
)

func main() {
	fmt.Println("Booting the server...")

	// Configure a sample route
	http.HandleFunc("/users", myTwitchHandler)
	http.HandleFunc("/channel", myChannelHandler)
	http.HandleFunc("/streams", myStreamHandler)

	// Run your server
	http.ListenAndServe(":8080", nil)
}

func myTwitchHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Recieved the following request:", r.Body)

	twitch.DoSomething()

	// YOUR ROUTES LOGIC GOES HERE
	//
	// Feel free to structure your routing however you see fit, this is just an example to get you started.

	client := &http.Client{}

	// resp, err := client.Get("https://api.twitch.tv/kraken/users?login=dallas,dallasnchains")

	request, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/users?login=dallas,dallasnchains", nil)
	if err != nil {
		fmt.Println(err.Error())
	 	os.Exit(1)
	}
	request.Header.Set("Client-ID", "ow8er2yjqzvbwzvfx832rrb5ucy933")
	request.Header.Set("Accept", "application/vnd.twitchtv.v5+json")

	resp, err := client.Do(request)

    responseData, err := ioutil.ReadAll(resp.Body)
    if err != nil {
		log.Fatal(err)
	}
    fmt.Println(string(responseData))

}

func myChannelHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Recieved the following request:", r.Body)

	// YOUR ROUTES LOGIC GOES HERE
	//
	// Feel free to structure your routing however you see fit, this is just an example to get you started.

	client := &http.Client{}

	// resp, err := client.Get("https://api.twitch.tv/kraken/users?login=dallas,dallasnchains")

	request, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/channels/44322889", nil)
	if err != nil {
		fmt.Println(err.Error())
	 	os.Exit(1)
	}
	request.Header.Set("Client-ID", "ow8er2yjqzvbwzvfx832rrb5ucy933")
	request.Header.Set("Accept", "application/vnd.twitchtv.v5+json")

	resp, err := client.Do(request)

    responseData, err := ioutil.ReadAll(resp.Body)
    if err != nil {
		log.Fatal(err)
	}
    fmt.Println(string(responseData))

}

func myStreamHandler(w http.ResponseWriter, r *http.Request) {
	fmt.Println("Recieved the following request:", r.Body)

	// YOUR ROUTES LOGIC GOES HERE
	//
	// Feel free to structure your routing however you see fit, this is just an example to get you started.

	client := &http.Client{}

	// resp, err := client.Get("https://api.twitch.tv/kraken/users?login=dallas,dallasnchains")

	request, err := http.NewRequest("GET", "https://api.twitch.tv/kraken/streams/44322889", nil)
	if err != nil {
		fmt.Println(err.Error())
	 	os.Exit(1)
	}
	request.Header.Set("Client-ID", "ow8er2yjqzvbwzvfx832rrb5ucy933")
	request.Header.Set("Accept", "application/vnd.twitchtv.v5+json")

	resp, err := client.Do(request)

    responseData, err := ioutil.ReadAll(resp.Body)
    if err != nil {
		log.Fatal(err)
	}
    fmt.Println(string(responseData))

}
<|endoftext|>


<commit_before>package main

/*
 Get Mesh terms for PMIDs
 1 - Read pmids from stdin
     i.e. "bunzip2 -c pmid_distinct_random_sorted.txt.bz2 |..."
 2 - Create 1 + n files:
   pubmed.mesh.gz - id on one line followed by indented mesh terms, one per line
    pubmed_xml_0.gz
    pubmed_xml_....n.gz
    XML downloaded, in 5000 record chunks

Find largest PMID: MAX PMID: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&reldate=60&datetype=pdat&retmax=1

NCBI: large jobs:
"In order not to overload the E-utility servers, NCBI recommends that users post no more than three URL requests per second and limit large jobs to either weekends or between 9:00 PM and 5:00 AM Eastern time during weekdays. "
  - https://www.ncbi.nlm.nih.gov/books/NBK25497/

*/

import (
	"bufio"
	//"bytes"
	"compress/gzip"
	"crypto/tls"
	//"encoding/xml"
	//"fmt"
	//	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"github.com/gnewton/gopubmed"
	//	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

var recordsPerFile = 50000

//////////
var recordsPerHttpRequest = 50
var recordsPerHttpRequestAfterHours = 150

var timeMin float64 = 2.0
var timeMax float64 = 8.0

var timeMinAfterHours float64 = 0.25
var timeMaxAfterHours float64 = 1.5

////////

var readFromStdin = false
var writeMesh = false
var defaultIdFile = "ids.txt"

const baseXmlFileName = "pubmed_xml_"

var meshFile = "pubmed.mesh.gz"
var inputFileName = ""
var verbose = false

func init() {
	//gopubmed.Debug = true
	flag.StringVar(&inputFileName, "f", inputFileName, "Name of input file with one pmid per line, if used")
	flag.StringVar(&meshFile, "M", meshFile, "File to write pmids and mesh terms")

	flag.BoolVar(&readFromStdin, "c", readFromStdin, "Read pmids from stdin, one per line")
	flag.IntVar(&recordsPerFile, "n", recordsPerFile, "Number of records per output file")
	flag.IntVar(&recordsPerHttpRequest, "t", recordsPerHttpRequest, "Number of records per http request to pubmed")
	flag.IntVar(&recordsPerHttpRequestAfterHours, "T", recordsPerHttpRequestAfterHours, "Number of records per http request to pubmed, after hours")
	flag.BoolVar(&verbose, "v", verbose, "verbose output")

	flag.Parse()
	if inputFileName == "" && !readFromStdin {
		fmt.Println("Either set -c for stdin or -f to read from a file\n")
		flag.Usage()
		os.Exit(1)
	}
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {

	log.SetFlags(log.LstdFlags | log.Lshortfile)

	tr := &http.Transport{
		ResponseHeaderTimeout: time.Second * 500,
		DisableKeepAlives:     false,
		DisableCompression:    false,
	}

	var wXml *gzip.Writer = nil
	var ww *bufio.Writer = nil
	var xFile *os.File = nil

	meshFile, err2 := os.Create(meshFile)
	if err2 != nil {
		return
	}
	defer meshFile.Close()
	wwMesh := bufio.NewWriter(meshFile)
	wMesh := gzip.NewWriter(wwMesh)

	//w := bufio.NewWriter(file)

	numIdsPerUrl := findNumIdsPerUrl()
	pmids := make([]string, numIdsPerUrl)

	urlFetcher := gopubmed.Fetcher{
		Ssl: true,
		Transport: &http.Transport{
			ResponseHeaderTimeout: time.Second * 500,
			DisableKeepAlives:     false,
			DisableCompression:    false,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		},
	}

	allCount := 0
	count := 0

	reader, err := makeReader()
	if err != nil {
		log.Fatal(err)
	}
	first := true
	chunkCount := 0
	for {
		thisNumIdsPerUrl := findNumIdsPerUrl()
		if numIdsPerUrl != thisNumIdsPerUrl {
			numIdsPerUrl = thisNumIdsPerUrl
			pmids = make([]string, numIdsPerUrl)
		}
		line, err := reader.ReadString('\n')
		if err != nil {
			// You may check here if err == io.EOF
			break
		}
		line = strings.TrimSpace(line)
		err = lineChecker(line)
		if err != nil {
			log.Fatal(err)
		}
		//mylog(line)
		pmids[count] = line

		if wXml == nil {
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
		}

		count = count + 1
		// Collected enough pmids: get their XML from NIH
		if count == numIdsPerUrl {
			getPubmedRecords(&urlFetcher, first, wMesh, wXml, tr, pmids)
			checkTime()
			first = false
			count = 0
			zeroArray(pmids)
		} else {

		}
		allCount += 1
		chunkCount += 1
		// Start new xml file: close old one: open new one
		if chunkCount > recordsPerFile {
			fmt.Fprintln(wXml, endPubmedArticleSet)
			wXml.Flush()
			wXml.Close()
			ww.Flush()
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
			chunkCount = 0
			first = true
		}
		if allCount%500 == 0 {
			mylog("Count: " + strconv.Itoa(allCount))
		}
	}
	if count != 0 {
		getPubmedRecords(&urlFetcher, first, wMesh, wXml, tr, pmids)
	}
	fmt.Fprintln(wXml, endPubmedArticleSet)

	wXml.Flush()
	wXml.Close()
	ww.Flush()

	wMesh.Flush()
	wwMesh.Flush()
	wMesh.Close()

	xFile.Close()
}

func zeroArray(a []string) {
	for i := 0; i < len(a); i++ {
		a[i] = ""
	}
}

func getPubmedRecords(urlFetcher *gopubmed.Fetcher, first bool, meshWriter *gzip.Writer, xmlWriter *gzip.Writer, transport *http.Transport, pmids []string) {
	preUrlTime := time.Now()

	articles, raw, err := urlFetcher.GetArticlesAndRaw(pmids)
	if err != nil {
		log.Fatal(err)
	}
	s := string(raw[:len(raw)])
	mylog("Records returned: " + strconv.Itoa(len(articles)))

	for i := 0; i < len(articles); i++ {
		pubmedArticle := articles[i]
		if pubmedArticle.MedlineCitation != nil && pubmedArticle.MedlineCitation.MeshHeadingList != nil && pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading != nil {
			fmt.Fprint(meshWriter, articles[i].MedlineCitation.PMID.Text)

			log.Println("foo")
			for j := 0; j < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading); j++ {
				fmt.Fprint(meshWriter, "|")
				fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.Attr_UI)
				fmt.Fprint(meshWriter, "::"+pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.Text)

				if len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName) > 0 {
					fmt.Fprint(meshWriter, "=")
					for q := 0; q < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName); q++ {
						if q != 0 {
							fmt.Fprint(meshWriter, "&")
						}

						fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName[q].Attr_UI)
						fmt.Fprint(meshWriter, "::"+pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName[q].Text)
					}
				}
			}
			fmt.Fprintln(meshWriter, "")
		}
	}
	meshWriter.Flush()
	if !first {
		s = removeFirstNLines(s, 3)
		// s = strings.Replace(s, startXml, "", -1)
		// s = strings.Replace(s, docType, "", -1)
		// s = strings.Replace(s, startPubmedArticleSet, "", -1)
	}
	s = strings.Replace(s, endPubmedArticleSet, "<!-- breakset -->", -1)

	xmlWriter.Write([]byte(s))
	postUrlTime := time.Now()
	mylog("Total request time:" + postUrlTime.Sub(preUrlTime).String())
}

func makeXmlWriter(fileCount int, startPmid string) (*gzip.Writer, *bufio.Writer, *os.File) {
	//xmlFile, err := os.Create("./" + baseXmlFileName + strconv.Itoa(fileCount) + "_" + strconv.Itoa(fileCount+recordsPerFile) + ".gz")
	xmlFile, err := os.Create("./" + baseXmlFileName + startPmid + ".gz")
	if err != nil {
		return nil, nil, nil
	}
	ww := bufio.NewWriter(xmlFile)
	return gzip.NewWriter(ww), ww, xmlFile
}

func findNumIdsPerUrl() int {
	if afterHours() {
		return recordsPerHttpRequestAfterHours
	}
	return recordsPerHttpRequest
}

func makeDelayTime() float64 {
	if afterHours() {
		mylog("  NCBI after hours")
		return timeMinAfterHours + rand.Float64()*timeMaxAfterHours
	} else {
		return timeMin + rand.Float64()*timeMax
	}
}

func afterHours() bool {
	now := time.Now().UTC()
	hour, _, _ := now.Clock()
	//return hour < 8 || hour > 18
	// hours in UTC; need US East coast (location of NCBI)
	return hour < 12 || hour > 22
}

func checkTime() {
	now := time.Now()
	rand.Seed(int64(now.Nanosecond()/9999999 + now.Second()*100 + now.Hour()))
	sleepSeconds := makeDelayTime()

	duration := (time.Duration)(sleepSeconds)

	time.Sleep(duration * time.Second)
}

func makeReader() (*bufio.Reader, error) {
	var inputFile *os.File
	if readFromStdin {
		mylog("Reading from stdin")
		inputFile = os.Stdin
	} else {
		if inputFileName == "" {
			return nil, errors.New("Empty input file name")
		}
		var err error
		inputFile, err = os.Open(inputFileName)
		if err != nil {
			return nil, err
		}
	}
	reader := bufio.NewReader(inputFile)
	return reader, nil
}

func lineChecker(l string) error {
	if len(l) == 0 {
		return errors.New("Error: Empty line")
	}
	var n int
	var err error
	if n, err = strconv.Atoi(l); err != nil {
		return errors.New("Error: Expecting pmid (integer); found[" + l + "]")
	}
	if n <= 0 {
		return errors.New("Error: pmids are positive integers; found [" + l + "]")
	}
	return nil
}

func removeFirstNLines(s string, n int) string {
	for i := 0; i < 3; i++ {
		n := strings.Index(s, "\n")
		s = s[n+1:]
	}
	return s
}

func mylog(m string) {
	if verbose {
		log.Println(m)
	}
}
<commit_msg>Fixed variable shadowing issue for meshFile/meshFileName<commit_after>package main

/*
 Get Mesh terms for PMIDs
 1 - Read pmids from stdin
     i.e. "bunzip2 -c pmid_distinct_random_sorted.txt.bz2 |..."
 2 - Create 1 + n files:
   pubmed.mesh.gz - id on one line followed by indented mesh terms, one per line
    pubmed_xml_0.gz
    pubmed_xml_....n.gz
    XML downloaded, in 5000 record chunks

Find largest PMID: MAX PMID: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&reldate=60&datetype=pdat&retmax=1

NCBI: large jobs:
"In order not to overload the E-utility servers, NCBI recommends that users post no more than three URL requests per second and limit large jobs to either weekends or between 9:00 PM and 5:00 AM Eastern time during weekdays. "
  - https://www.ncbi.nlm.nih.gov/books/NBK25497/

*/

import (
	"bufio"
	//"bytes"
	"compress/gzip"
	"crypto/tls"
	//"encoding/xml"
	//"fmt"
	//	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"github.com/gnewton/gopubmed"
	//	"io"
	"log"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"
)

var recordsPerFile = 50000

//////////
var recordsPerHttpRequest = 50
var recordsPerHttpRequestAfterHours = 150

var timeMin float64 = 2.0
var timeMax float64 = 8.0

var timeMinAfterHours float64 = 0.25
var timeMaxAfterHours float64 = 1.5

////////

var readFromStdin = false
var writeMesh = false
var defaultIdFile = "ids.txt"

const baseXmlFileName = "pubmed_xml_"

var meshFileName = "pubmed.mesh.gz"
var inputFileName = ""
var verbose = false

func init() {
	//gopubmed.Debug = true
	flag.StringVar(&inputFileName, "f", inputFileName, "Name of input file with one pmid per line, if used")
	flag.StringVar(&meshFileName, "M", meshFileName, "File to write pmids and mesh terms")

	flag.BoolVar(&readFromStdin, "c", readFromStdin, "Read pmids from stdin, one per line")
	flag.IntVar(&recordsPerFile, "n", recordsPerFile, "Number of records per output file")
	flag.IntVar(&recordsPerHttpRequest, "t", recordsPerHttpRequest, "Number of records per http request to pubmed")
	flag.IntVar(&recordsPerHttpRequestAfterHours, "T", recordsPerHttpRequestAfterHours, "Number of records per http request to pubmed, after hours")
	flag.BoolVar(&verbose, "v", verbose, "verbose output")

	flag.Parse()
	if inputFileName == "" && !readFromStdin {
		fmt.Println("Either set -c for stdin or -f to read from a file\n")
		flag.Usage()
		os.Exit(1)
	}
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {

	log.SetFlags(log.LstdFlags | log.Lshortfile)

	tr := &http.Transport{
		ResponseHeaderTimeout: time.Second * 500,
		DisableKeepAlives:     false,
		DisableCompression:    false,
	}

	var wXml *gzip.Writer = nil
	var ww *bufio.Writer = nil
	var xFile *os.File = nil

	mylog("Opening meshFile file: " + meshFileName)
	meshFile, err2 := os.Create(meshFileName)
	if err2 != nil {
		return
	}

	defer meshFile.Close()

	wwMesh := bufio.NewWriter(meshFile)
	wMesh := gzip.NewWriter(wwMesh)

	//w := bufio.NewWriter(file)

	numIdsPerUrl := findNumIdsPerUrl()
	pmids := make([]string, numIdsPerUrl)

	urlFetcher := gopubmed.Fetcher{
		Ssl: true,
		Transport: &http.Transport{
			ResponseHeaderTimeout: time.Second * 500,
			DisableKeepAlives:     false,
			DisableCompression:    false,
			TLSClientConfig:       &tls.Config{InsecureSkipVerify: true},
		},
	}

	allCount := 0
	count := 0

	reader, err := makeReader()
	if err != nil {
		log.Fatal(err)
	}
	first := true
	chunkCount := 0
	for {
		thisNumIdsPerUrl := findNumIdsPerUrl()
		if numIdsPerUrl != thisNumIdsPerUrl {
			numIdsPerUrl = thisNumIdsPerUrl
			pmids = make([]string, numIdsPerUrl)
		}
		line, err := reader.ReadString('\n')
		if err != nil {
			// You may check here if err == io.EOF
			break
		}
		line = strings.TrimSpace(line)
		err = lineChecker(line)
		if err != nil {
			log.Fatal(err)
		}
		//mylog(line)
		pmids[count] = line

		if wXml == nil {
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
		}

		count = count + 1
		// Collected enough pmids: get their XML from NIH
		if count == numIdsPerUrl {
			getPubmedRecords(&urlFetcher, first, wMesh, wXml, tr, pmids)
			checkTime()
			first = false
			count = 0
			zeroArray(pmids)
		} else {

		}
		allCount += 1
		chunkCount += 1
		// Start new xml file: close old one: open new one
		if chunkCount > recordsPerFile {
			fmt.Fprintln(wXml, endPubmedArticleSet)
			wXml.Flush()
			wXml.Close()
			ww.Flush()
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
			chunkCount = 0
			first = true
		}
		if allCount%500 == 0 {
			mylog("Count: " + strconv.Itoa(allCount))
		}
	}
	if count != 0 {
		getPubmedRecords(&urlFetcher, first, wMesh, wXml, tr, pmids)
	}
	fmt.Fprintln(wXml, endPubmedArticleSet)

	wXml.Flush()
	wXml.Close()
	ww.Flush()

	wMesh.Flush()
	wwMesh.Flush()
	wMesh.Close()

	xFile.Close()
}

func zeroArray(a []string) {
	for i := 0; i < len(a); i++ {
		a[i] = ""
	}
}

func getPubmedRecords(urlFetcher *gopubmed.Fetcher, first bool, meshWriter *gzip.Writer, xmlWriter *gzip.Writer, transport *http.Transport, pmids []string) {
	preUrlTime := time.Now()

	articles, raw, err := urlFetcher.GetArticlesAndRaw(pmids)
	if err != nil {
		log.Fatal(err)
	}
	s := string(raw[:len(raw)])
	mylog("Records returned: " + strconv.Itoa(len(articles)))

	for i := 0; i < len(articles); i++ {
		pubmedArticle := articles[i]
		if pubmedArticle.MedlineCitation != nil && pubmedArticle.MedlineCitation.MeshHeadingList != nil && pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading != nil {

			fmt.Fprint(meshWriter, articles[i].MedlineCitation.PMID.Text)

			for j := 0; j < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading); j++ {
				fmt.Fprint(meshWriter, "|")
				fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.Attr_UI)
				fmt.Fprint(meshWriter, "::"+pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.Text)

				if len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName) > 0 {
					fmt.Fprint(meshWriter, "=")
					for q := 0; q < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName); q++ {
						if q != 0 {
							fmt.Fprint(meshWriter, "&")
						}

						fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName[q].Attr_UI)
						fmt.Fprint(meshWriter, "::"+pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].QualifierName[q].Text)
					}
				}
			}
			fmt.Fprintln(meshWriter, "")
		}
	}
	meshWriter.Flush()
	if !first {
		s = removeFirstNLines(s, 3)
		// s = strings.Replace(s, startXml, "", -1)
		// s = strings.Replace(s, docType, "", -1)
		// s = strings.Replace(s, startPubmedArticleSet, "", -1)
	}
	s = strings.Replace(s, endPubmedArticleSet, "<!-- breakset -->", -1)

	xmlWriter.Write([]byte(s))
	postUrlTime := time.Now()
	mylog("Total request time:" + postUrlTime.Sub(preUrlTime).String())
}

func makeXmlWriter(fileCount int, startPmid string) (*gzip.Writer, *bufio.Writer, *os.File) {
	//xmlFile, err := os.Create("./" + baseXmlFileName + strconv.Itoa(fileCount) + "_" + strconv.Itoa(fileCount+recordsPerFile) + ".gz")
	xmlFileName := "./" + baseXmlFileName + startPmid + ".gz"
	xmlFile, err := os.Create(xmlFileName)
	if err != nil {
		return nil, nil, nil
	}
	mylog("Opening XML record file: " + xmlFileName)
	ww := bufio.NewWriter(xmlFile)
	return gzip.NewWriter(ww), ww, xmlFile
}

func findNumIdsPerUrl() int {
	if afterHours() {
		return recordsPerHttpRequestAfterHours
	}
	return recordsPerHttpRequest
}

func makeDelayTime() float64 {
	if afterHours() {
		mylog("  NCBI after hours")
		return timeMinAfterHours + rand.Float64()*timeMaxAfterHours
	} else {
		return timeMin + rand.Float64()*timeMax
	}
}

func afterHours() bool {
	now := time.Now().UTC()
	hour, _, _ := now.Clock()
	//return hour < 8 || hour > 18
	// hours in UTC; need US East coast (location of NCBI)
	return hour < 12 || hour > 22
}

func checkTime() {
	now := time.Now()
	rand.Seed(int64(now.Nanosecond()/9999999 + now.Second()*100 + now.Hour()))
	sleepSeconds := makeDelayTime()

	duration := (time.Duration)(sleepSeconds)

	time.Sleep(duration * time.Second)
}

func makeReader() (*bufio.Reader, error) {
	var inputFile *os.File
	if readFromStdin {
		mylog("Reading from stdin")
		inputFile = os.Stdin
	} else {
		if inputFileName == "" {
			return nil, errors.New("Empty input file name")
		}
		var err error
		inputFile, err = os.Open(inputFileName)
		if err != nil {
			return nil, err
		}
	}
	reader := bufio.NewReader(inputFile)
	return reader, nil
}

func lineChecker(l string) error {
	if len(l) == 0 {
		return errors.New("Error: Empty line")
	}
	var n int
	var err error
	if n, err = strconv.Atoi(l); err != nil {
		return errors.New("Error: Expecting pmid (integer); found[" + l + "]")
	}
	if n <= 0 {
		return errors.New("Error: pmids are positive integers; found [" + l + "]")
	}
	return nil
}

func removeFirstNLines(s string, n int) string {
	for i := 0; i < 3; i++ {
		n := strings.Index(s, "\n")
		s = s[n+1:]
	}
	return s
}

func mylog(m string) {
	if verbose {
		log.Println(m)
	}
}
<|endoftext|>