<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

type dependency struct {
	ImportPath string
	Rev        string // VCS-specific commit ID.
}

type result struct {
	importPath string
	commits    []string
}

var (
	godepsJSON = flag.String("godeps", os.Getenv("PWD")+"/Godeps/Godeps.json", "path to Godeps.json")
	godeps     struct {
		ImportPath string
		Deps       []dependency
	}
	temporaryDir, git string
)

func init() {
	flag.StringVar(&temporaryDir, "temp", "", "temporary path for cloning the repositories")
	flag.Parse()
}

func main() {
	f, err := ioutil.ReadFile(*godepsJSON)

	if err != nil {
		fmt.Fprintf(os.Stderr, "opening %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if err := json.Unmarshal(f, &godeps); err != nil {
		fmt.Fprintf(os.Stderr, "unmarshalling %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if len(temporaryDir) == 0 {
		createTemporaryDir()
		defer os.RemoveAll(temporaryDir)
	}

	lookGITPath()
	results := processDependencies(godeps.Deps)

	for _, dep := range godeps.Deps {
		if commits := results[dep.ImportPath]; len(commits) > 1 {
			fmt.Println(dep.ImportPath)

			for _, commit := range commits {
				fmt.Println("  " + commit)
			}
		}
	}
}

func createTemporaryDir() {
	var err error
	temporaryDir, err = ioutil.TempDir("/tmp", "godeps-check")

	if err != nil {
		fmt.Fprintf(os.Stderr, "creating temporary dir: %s", err)
		os.Exit(1)
	}

	if err := os.Chmod(temporaryDir, 0777); err != nil {
		fmt.Fprintf(os.Stderr, "chmod %s: %s", temporaryDir, err)
		os.Exit(1)
	}
}

func lookGITPath() {
	var err error
	git, err = exec.LookPath("git")

	if err != nil {
		fmt.Fprintf(os.Stderr, "looking up git path: %s", err)
		os.Exit(1)
	}
}

func processDependencies(deps []dependency) map[string][]string {
	results := make(map[string][]string)
	ch := make(chan result, len(deps))

	for _, d := range deps {
		go func(d dependency, ch chan result) {
			ch <- result{d.ImportPath, processDependency(d)}
		}(d, ch)
	}

	done := 0

alldone:
	for {
		select {
		case r := <-ch:
			results[r.importPath] = r.commits
			done++

			if done == len(deps) {
				break alldone
			}
		}
	}

	return results
}

func processDependency(d dependency) []string {
	importPath := d.ImportPath
	parts := strings.Split(importPath, "/")

	if len(parts) > 3 {
		importPath = strings.Join(parts[0:3], "/")
	}

	if !strings.Contains(parts[0], ".") {
		fmt.Fprintf(os.Stderr, "skipping %s: not go gettable\n", d.ImportPath)
		return nil
	}

	projectDir := path.Join(temporaryDir, importPath)

	if err := os.MkdirAll(projectDir, 0777); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return nil
	}

	if err := clone("https://"+importPath, projectDir); err != nil {
		return nil
	}

	return diff(projectDir, d.Rev)
}

func clone(url, projectDir string) error {
	cmd := exec.Command(git, "clone", url, projectDir)
	output, err := cmd.CombinedOutput()
	content := string(output)

	fmt.Fprint(os.Stderr, strings.Join(cmd.Args, " "))

	if err != nil {
		fmt.Fprintf(os.Stderr, ": %s\n", err)
	} else {
		fmt.Fprintln(os.Stderr, "")
	}

	fmt.Fprintln(os.Stderr, "  "+strings.Replace(content, "\n", "\n  ", -1))

	return err
}

func diff(path, revision string) []string {
	cmd := exec.Command(git, "-C", path, "log", "--pretty=oneline", revision+"..master")
	output, err := cmd.CombinedOutput()
	lines := strings.Split(string(output), "\n")

	if err != nil {
		return append(lines, err.Error()+"\n")
	}

	for i := range lines {
		if len(lines[i]) == 0 {
			continue
		}

		revision := strings.Split(lines[i], " ")[0]
		lines[i] = strings.Replace(lines[i], revision, revision[0:7], -1)
	}

	if len(lines) > 10 {
		size := len(lines) - 10

		lines = lines[0:9]
		lines = append(lines, fmt.Sprintf("[%d commits not shown]\n", size))
	}

	return lines
}
<commit_msg>git log --pretty=%h %s<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strings"
)

type dependency struct {
	ImportPath string
	Rev        string // VCS-specific commit ID.
}

type result struct {
	importPath string
	commits    []string
}

var (
	godepsJSON = flag.String("godeps", os.Getenv("PWD")+"/Godeps/Godeps.json", "path to Godeps.json")
	godeps     struct {
		ImportPath string
		Deps       []dependency
	}
	temporaryDir, git string
)

func init() {
	flag.StringVar(&temporaryDir, "temp", "", "temporary path for cloning the repositories")
	flag.Parse()
}

func main() {
	f, err := ioutil.ReadFile(*godepsJSON)

	if err != nil {
		fmt.Fprintf(os.Stderr, "opening %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if err := json.Unmarshal(f, &godeps); err != nil {
		fmt.Fprintf(os.Stderr, "unmarshalling %s: %s", *godepsJSON, err)
		os.Exit(1)
	}

	if len(temporaryDir) == 0 {
		createTemporaryDir()
		defer os.RemoveAll(temporaryDir)
	}

	lookGITPath()
	results := processDependencies(godeps.Deps)

	for _, dep := range godeps.Deps {
		if commits := results[dep.ImportPath]; len(commits) > 1 {
			fmt.Println(dep.ImportPath)

			for _, commit := range commits {
				fmt.Println("  " + commit)
			}
		}
	}
}

func createTemporaryDir() {
	var err error
	temporaryDir, err = ioutil.TempDir("/tmp", "godeps-check")

	if err != nil {
		fmt.Fprintf(os.Stderr, "creating temporary dir: %s", err)
		os.Exit(1)
	}

	if err := os.Chmod(temporaryDir, 0777); err != nil {
		fmt.Fprintf(os.Stderr, "chmod %s: %s", temporaryDir, err)
		os.Exit(1)
	}
}

func lookGITPath() {
	var err error
	git, err = exec.LookPath("git")

	if err != nil {
		fmt.Fprintf(os.Stderr, "looking up git path: %s", err)
		os.Exit(1)
	}
}

func processDependencies(deps []dependency) map[string][]string {
	results := make(map[string][]string)
	ch := make(chan result, len(deps))

	for _, d := range deps {
		go func(d dependency, ch chan result) {
			ch <- result{d.ImportPath, processDependency(d)}
		}(d, ch)
	}

	done := 0

alldone:
	for {
		select {
		case r := <-ch:
			results[r.importPath] = r.commits
			done++

			if done == len(deps) {
				break alldone
			}
		}
	}

	return results
}

func processDependency(d dependency) []string {
	importPath := d.ImportPath
	parts := strings.Split(importPath, "/")

	if len(parts) > 3 {
		importPath = strings.Join(parts[0:3], "/")
	}

	if !strings.Contains(parts[0], ".") {
		fmt.Fprintf(os.Stderr, "skipping %s: not go gettable\n", d.ImportPath)
		return nil
	}

	projectDir := path.Join(temporaryDir, importPath)

	if err := os.MkdirAll(projectDir, 0777); err != nil {
		fmt.Fprintln(os.Stderr, err)
		return nil
	}

	if err := clone("https://"+importPath, projectDir); err != nil {
		return nil
	}

	return diff(projectDir, d.Rev)
}

func clone(url, projectDir string) error {
	cmd := exec.Command(git, "clone", url, projectDir)
	output, err := cmd.CombinedOutput()
	content := string(output)

	fmt.Fprint(os.Stderr, strings.Join(cmd.Args, " "))

	if err != nil {
		fmt.Fprintf(os.Stderr, ": %s\n", err)
	} else {
		fmt.Fprintln(os.Stderr, "")
	}

	fmt.Fprintln(os.Stderr, "  "+strings.Replace(content, "\n", "\n  ", -1))

	return err
}

func diff(path, revision string) []string {
	cmd := exec.Command(git, "-C", path, "log", "--pretty=%h %s", revision+"..master")
	output, err := cmd.CombinedOutput()
	lines := strings.Split(string(output), "\n")

	if err != nil {
		return append(lines, err.Error()+"\n")
	}

	if len(lines) > 10 {
		size := len(lines) - 10

		lines = lines[0:9]
		lines = append(lines, fmt.Sprintf("[%d commits not shown]\n", size))
	}

	return lines
}
<|endoftext|>


<commit_before>// Copyright 2014-2015 Chadev. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"

	"github.com/danryan/hal"
	_ "github.com/danryan/hal/adapter/irc"
	_ "github.com/danryan/hal/adapter/shell"
	_ "github.com/danryan/hal/store/memory"
	_ "github.com/danryan/hal/store/redis"
)

// VERSION contians the current verison number and revison if need be
const VERSION = "2015-05-26"

// handler is an interface for objects to implement in order to respond to messages.
type handler interface {
	Handle(res *hal.Response) error
}

var helpMessages = make(map[string]string)

var pingHandler = hear(`ping`, "ping", "Causes Ash to reply with PONG", func(res *hal.Response) error {
	return res.Send("PONG")
})

var fooHandler = hear(`foo`, "foo", "Causes Ash to reply with a BAR", func(res *hal.Response) error {
	return res.Send("BAR")
})

var synHandler = hear(`SYN`, "SYN", "Causes Ash to reply with ACK", func(res *hal.Response) error {
	return res.Send("ACK")
})

var selfHandler = hear(`who are you`, "self", "", func(res *hal.Response) error {
	return res.Send("I'm Ash, the friendly #chadev bot.  I can preform a variety of tasks, and I am learning new tricks all the time.  I am open source, and pull requests are welcome!")
})

var quitHandler = hear(`(.*)+/quit(.*)+`, "quit", "", func(res *hal.Response) error {
	name := res.UserName()
	return res.Send(fmt.Sprintf("No!  Bad %s!", name))
})

var helpHandler = hear(`help`, "help", "Displays this message", func(res *hal.Response) error {
	helpMsg := []string{
		"HAL Chadev IRC Edition build: " + VERSION + "\n",
		"Supported commands:\n",
	}

	for command, message := range helpMessages {
		if command != "" && message != "" {
			helpMsg = append(helpMsg, command+" - "+message+"\n")
		}
	}

	var text string
	for _, msg := range helpMsg {
		text = text + msg
	}

	text = uploadHelpMsg(text)
	res.Send(fmt.Sprintf("My usage information can be found at %s", text))

	return nil
})

func hear(pattern string, command string, message string, fn func(res *hal.Response) error) handler {
	addHelpMessage(command, message)
	return hal.Hear("^(?i)Ash "+pattern, fn)
}

func addHelpMessage(command string, message string) {
	helpMessages[command] = message
}

func main() {
	os.Exit(run())
}

func run() int {
	robot, err := hal.NewRobot()
	if err != nil {
		hal.Logger.Error(err)
		return 1
	}

	robot.Handle(
		fooHandler,
		tableFlipHandler,
		eventHandler,
		synHandler,
		helpHandler,
		pingHandler,
		sourceHandler,
		issueHandler,
		cageMeHandler,
		whoisHandler,
		whoamHandler,
		isHandler,
		selfHandler,
		quitHandler,
		fizzBuzzHandler,
		noteStoreHandler,
		noteGetHandler,
		noteRemoveHandler,
		chadevCountHandler,
		chadevListAllHandler,
		chadevInfoHandler,
		fatherHandler,
		partyHandler,
		whoBackHandler,
		whatAreHandler,
		musicHandler,
		lunchHandler,
		talkHandler,
		addTalkHandler,
		devTalkLinkHandler,
		isAliveHandler,
		groupListHandler,
		groupDetailsHandler,
		groupRSVPHandler,
		devlunchRSVPHandler,
		karmaHandler,
		karmaStatsHandler,
	)

	if err := robot.Run(); err != nil {
		hal.Logger.Error(err)
		return 1
	}
	return 0
}
<commit_msg>Version bump<commit_after>// Copyright 2014-2015 Chadev. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"

	"github.com/danryan/hal"
	_ "github.com/danryan/hal/adapter/irc"
	_ "github.com/danryan/hal/adapter/shell"
	_ "github.com/danryan/hal/store/memory"
	_ "github.com/danryan/hal/store/redis"
)

// VERSION contians the current verison number and revison if need be
const VERSION = "2015-06-09"

// handler is an interface for objects to implement in order to respond to messages.
type handler interface {
	Handle(res *hal.Response) error
}

var helpMessages = make(map[string]string)

var pingHandler = hear(`ping`, "ping", "Causes Ash to reply with PONG", func(res *hal.Response) error {
	return res.Send("PONG")
})

var fooHandler = hear(`foo`, "foo", "Causes Ash to reply with a BAR", func(res *hal.Response) error {
	return res.Send("BAR")
})

var synHandler = hear(`SYN`, "SYN", "Causes Ash to reply with ACK", func(res *hal.Response) error {
	return res.Send("ACK")
})

var selfHandler = hear(`who are you`, "self", "", func(res *hal.Response) error {
	return res.Send("I'm Ash, the friendly #chadev bot.  I can preform a variety of tasks, and I am learning new tricks all the time.  I am open source, and pull requests are welcome!")
})

var quitHandler = hear(`(.*)+/quit(.*)+`, "quit", "", func(res *hal.Response) error {
	name := res.UserName()
	return res.Send(fmt.Sprintf("No!  Bad %s!", name))
})

var helpHandler = hear(`help`, "help", "Displays this message", func(res *hal.Response) error {
	helpMsg := []string{
		"HAL Chadev IRC Edition build: " + VERSION + "\n",
		"Supported commands:\n",
	}

	for command, message := range helpMessages {
		if command != "" && message != "" {
			helpMsg = append(helpMsg, command+" - "+message+"\n")
		}
	}

	var text string
	for _, msg := range helpMsg {
		text = text + msg
	}

	text = uploadHelpMsg(text)
	res.Send(fmt.Sprintf("My usage information can be found at %s", text))

	return nil
})

func hear(pattern string, command string, message string, fn func(res *hal.Response) error) handler {
	addHelpMessage(command, message)
	return hal.Hear("^(?i)Ash "+pattern, fn)
}

func addHelpMessage(command string, message string) {
	helpMessages[command] = message
}

func main() {
	os.Exit(run())
}

func run() int {
	robot, err := hal.NewRobot()
	if err != nil {
		hal.Logger.Error(err)
		return 1
	}

	robot.Handle(
		fooHandler,
		tableFlipHandler,
		eventHandler,
		synHandler,
		helpHandler,
		pingHandler,
		sourceHandler,
		issueHandler,
		cageMeHandler,
		whoisHandler,
		whoamHandler,
		isHandler,
		selfHandler,
		quitHandler,
		fizzBuzzHandler,
		noteStoreHandler,
		noteGetHandler,
		noteRemoveHandler,
		chadevCountHandler,
		chadevListAllHandler,
		chadevInfoHandler,
		fatherHandler,
		partyHandler,
		whoBackHandler,
		whatAreHandler,
		musicHandler,
		lunchHandler,
		talkHandler,
		addTalkHandler,
		devTalkLinkHandler,
		isAliveHandler,
		groupListHandler,
		groupDetailsHandler,
		groupRSVPHandler,
		devlunchRSVPHandler,
		karmaHandler,
		karmaStatsHandler,
	)

	if err := robot.Run(); err != nil {
		hal.Logger.Error(err)
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>package main

/*
 Get Mesh terms for PMIDs
 1 - Read pmids from stdin
     i.e. "bunzip2 -c pmid_distinct_random_sorted.txt.bz2 |..."
 2 - Create 1 + n files:
   pubmed.mesh.gz - id on one line followed by indented mesh terms, one per line
    pubmed_xml_0.gz
    pubmed_xml_....n.gz
    XML downloaded, in 5000 record chunks

NCBI: large jobs:
"In order not to overload the E-utility servers, NCBI recommends that users post no more than three URL requests per second and limit large jobs to either weekends or between 9:00 PM and 5:00 AM Eastern time during weekdays. "
  - https://www.ncbi.nlm.nih.gov/books/NBK25497/

*/

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/xml"
	//"fmt"
	"io"
	"math/rand"
	"net/http"
	"os"
	//"strconv"
	"strings"
	"flag"
	"time"
	"log"
	"fmt"
)

var recordsPerFile = 50000
var recordsPerHttpRequest = 50
var recordsPerHttpRequestAfterHours = 150
var readFromStdin = false
var defaultIdFile = "ids.txt"

var baseXmlFileName = "pubmed_xml_"

func init() {
	flag.BoolVar(&readFromStdin, "c", readFromStdin, "Read pmids from stdin, one per line")
	flag.IntVar(&recordsPerFile, "n", recordsPerFile, "Number of records per output file")
	flag.IntVar(&recordsPerHttpRequest, "t", recordsPerHttpRequest, "Number of records per http request to pubmed")
	flag.IntVar(&recordsPerHttpRequestAfterHours, "T", recordsPerHttpRequestAfterHours, "Number of records per http request to pubmed, after hours")
// const recordsPerFile = 50000
// const NUM_IDS_PER_URL = 50w
// afterHoursMultiplyer = 1
}

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	tr := &http.Transport{
		ResponseHeaderTimeout: time.Second * 500,
		DisableKeepAlives:     false,
		DisableCompression:    false,
	}

	//fileCount := 0
	/*
		xmlFile, err := os.Create("./" + baseXmlFileName + strconv.Itoa(fileCount) + ".gz")
		if err != nil {
			return
		}
		defer xmlFile.Close()
		ww := bufio.NewWriter(xmlFile)
		//wXml := gzip.NewWriter(ww)
	*/
	var wXml *gzip.Writer = nil
	var ww *bufio.Writer = nil
	var xFile *os.File = nil

	meshFile, err2 := os.Create("./pubmed.mesh.gz")
	if err2 != nil {
		return
	}
	defer meshFile.Close()
	wwMesh := bufio.NewWriter(meshFile)
	wMesh := gzip.NewWriter(wwMesh)

	//w := bufio.NewWriter(file)

	var pmids []string = make([]string, recordsPerHttpRequest)

	allCount := 0
	count := 0
	reader := bufio.NewReader(os.Stdin)
	first := true
	chunkCount := 0
	for {
		numIdsPerUrl := findNumIdsPerUrl()
		line, err := reader.ReadString('\n')
		if err != nil {
			// You may check here if err == io.EOF
			break
		}
		line = strings.TrimSpace(line)
		//log.Println(line)
		pmids[count] = line

		if wXml == nil {
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
		}

		count = count + 1
		// Collected enough pmids: get their XML from NIH
		if count == numIdsPerUrl {
			getMesh(first, wMesh, wXml, tr, pmids)
			checkTime()
			first = false
			count = 0
			zeroArray(pmids)
		} else {

		}
		allCount += 1
		chunkCount += 1
		// Start new xml file: close old one: open new one
		if chunkCount > recordsPerFile {
			fmt.Fprintln(wXml, endPubmedArticleSet)
			wXml.Flush()
			wXml.Close()
			ww.Flush()
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
			chunkCount = 0
			first = true
		}
		if allCount%500 == 0 {
			log.Println(allCount)
		}
	}
	if count != 0 {
		getMesh(first, wMesh, wXml, tr, pmids)
	}
	fmt.Fprintln(wXml, endPubmedArticleSet)

	wXml.Flush()
	wXml.Close()
	ww.Flush()

	wMesh.Flush()
	wMesh.Close()
	wwMesh.Flush()
	xFile.Close()
}

func zeroArray(a []string) {
	for i := 0; i < len(a); i++ {
		a[i] = ""
	}
}

func makeUrl(baseUrl string, pmids []string) string {
	url := baseUrl
	for i := 0; i < len(pmids); i++ {
		if pmids[i] != "" {
			if i != 0 {
				url += ","
			}
			url += pmids[i]
		}
	}
	//return "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&rettype=xml&id=15718680,15718682,119703,157186,1571868,11970375"
	return url
}

const baseUrl = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&rettype=xml&id="

func getMesh(first bool, meshWriter io.Writer, xmlWriter *gzip.Writer, transport *http.Transport, pmids []string) {
	preUrlTime := time.Now()
	url := makeUrl(baseUrl, pmids)

	log.Println(url)
	//log.Println(url)
	//log.Println("\n\n")
	client := &http.Client{Transport: transport}
	req, err := http.NewRequest("GET", url, nil)
	req.Close = true
	resp, err := client.Do(req)

	//resp, err := client.Get(url)
	if err != nil {
		log.Println("Error opening url:", url, "   error=", err)
		return
	}
	defer resp.Body.Close()
	//log.Println(err)

	//log.Println("\n\n")
	//log.Println(resp.Body)

	buf := new(bytes.Buffer)
	buf.ReadFrom(resp.Body)
	s := buf.String()

	v := ArticleSet{}
	err = xml.Unmarshal([]byte(s), &v)

	//log.Println(v);
	// This should be inside a go routine listening on a channel
	for i := 0; i < len(v.ArticleList); i++ {
		//log.Println(i)
		fmt.Fprint(meshWriter, v.ArticleList[i].MedlineCitation.PMID)
		pubmedArticle := v.ArticleList[i]
		//log.Println(pubmedArticle.MedlineCitation.Article.Abstract.AbstractText)
		for j := 0; j < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading); j++ {
			//fmt.Fprintln(meshWriter, " ", article.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.DescriptorName)
			fmt.Fprint(meshWriter, "|")
			fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.DescriptorName)
		}
		fmt.Fprintln(meshWriter, "")
	}
	if !first {
		s = strings.Replace(s, startXml, "", -1)
		s = strings.Replace(s, docType, "", -1)
		s = strings.Replace(s, startPubmedArticleSet, "", -1)
	}
	s = strings.Replace(s, endPubmedArticleSet, "<!-- breakset -->", -1)

	xmlWriter.Write([]byte(s))
	postUrlTime := time.Now()
	log.Println("Total request time:", postUrlTime.Sub(preUrlTime))
}

func makeXmlWriter(fileCount int, startPmid string) (*gzip.Writer, *bufio.Writer, *os.File) {
	//xmlFile, err := os.Create("./" + baseXmlFileName + strconv.Itoa(fileCount) + "_" + strconv.Itoa(fileCount+recordsPerFile) + ".gz")
	xmlFile, err := os.Create("./" + baseXmlFileName + startPmid + ".gz")
	if err != nil {
		return nil, nil, nil
	}
	ww := bufio.NewWriter(xmlFile)
	return gzip.NewWriter(ww), ww, xmlFile
}

func afterHours() bool {
	now := time.Now()
	hour, _, _ := now.Clock()
	return hour < 8 || hour > 18
}

func findNumIdsPerUrl() int {
	if afterHours() {
		return recordsPerHttpRequestAfterHours
	}
	return recordsPerHttpRequest
}

func checkTime() {
	now := time.Now()
	rand.Seed(int64(now.Nanosecond()/9999999 + now.Second()*100 + now.Hour()))
	sleepSeconds := 1 + rand.Intn(2)

	/*
		if !afterHours() {
			sleepSeconds = 1 + rand.Intn(2)
		}
	*/

	duration := (time.Duration)(sleepSeconds)
	log.Println("Start sleep")
	t0 := time.Now()

	time.Sleep(duration * time.Second)
	t1 := time.Now()
	log.Println("End sleep:", t1.Sub(t0))
}
<commit_msg>Added command line for reading from stdin or file of ids<commit_after>package main

/*
 Get Mesh terms for PMIDs
 1 - Read pmids from stdin
     i.e. "bunzip2 -c pmid_distinct_random_sorted.txt.bz2 |..."
 2 - Create 1 + n files:
   pubmed.mesh.gz - id on one line followed by indented mesh terms, one per line
    pubmed_xml_0.gz
    pubmed_xml_....n.gz
    XML downloaded, in 5000 record chunks

NCBI: large jobs:
"In order not to overload the E-utility servers, NCBI recommends that users post no more than three URL requests per second and limit large jobs to either weekends or between 9:00 PM and 5:00 AM Eastern time during weekdays. "
  - https://www.ncbi.nlm.nih.gov/books/NBK25497/

*/

import (
	"bufio"
	"bytes"
	"compress/gzip"
	"encoding/xml"
	//"fmt"
	"io"
	"math/rand"
	"net/http"
	"os"
	"strconv"
	"strings"
	"flag"
	"time"
	"errors"
	"log"
	"fmt"
)

var recordsPerFile = 50000
var recordsPerHttpRequest = 50
var recordsPerHttpRequestAfterHours = 150
var readFromStdin = false
var writeMesh = false
var defaultIdFile = "ids.txt"

var baseXmlFileName = "pubmed_xml_"
var meshFile = "pubmed.mesh.gz"
var inputFileName = ""

func init() {
	flag.StringVar(&inputFileName, "f", inputFileName, "Name of input file with one pmid per line, if used")
	flag.StringVar(&meshFile, "M", meshFile, "File to write pmids and mesh terms")

	flag.BoolVar(&readFromStdin, "c", readFromStdin, "Read pmids from stdin, one per line")
	flag.IntVar(&recordsPerFile, "n", recordsPerFile, "Number of records per output file")
	flag.IntVar(&recordsPerHttpRequest, "t", recordsPerHttpRequest, "Number of records per http request to pubmed")
	flag.IntVar(&recordsPerHttpRequestAfterHours, "T", recordsPerHttpRequestAfterHours, "Number of records per http request to pubmed, after hours")

	flag.Parse()
	if inputFileName == "" && !readFromStdin{
		log.Println("Either set -c for stdin or -f to read from a file")
		flag.Usage()
		os.Exit(1)
	}

}

func main() {

	log.SetFlags(log.LstdFlags | log.Lshortfile)

	tr := &http.Transport{
		ResponseHeaderTimeout: time.Second * 500,
		DisableKeepAlives:     false,
		DisableCompression:    false,
	}

	var wXml *gzip.Writer = nil
	var ww *bufio.Writer = nil
	var xFile *os.File = nil

	meshFile, err2 := os.Create(meshFile)
	if err2 != nil {
		return
	}
	defer meshFile.Close()
	wwMesh := bufio.NewWriter(meshFile)
	wMesh := gzip.NewWriter(wwMesh)

	//w := bufio.NewWriter(file)

	var pmids []string = make([]string, recordsPerHttpRequest)

	allCount := 0
	count := 0
	//reader := bufio.NewReader(os.Stdin)
	reader, err := makeReader()
	if err != nil{
		log.Fatal(err)
	}
	first := true
	chunkCount := 0
	for {
		numIdsPerUrl := findNumIdsPerUrl()
		line, err := reader.ReadString('\n')
		if err != nil {
			// You may check here if err == io.EOF
			break
		}
		line = strings.TrimSpace(line)
		err = lineChecker(line)
		if err != nil{
			log.Fatal(err)
		}
		//log.Println(line)
		pmids[count] = line

		if wXml == nil {
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
		}

		count = count + 1
		// Collected enough pmids: get their XML from NIH
		if count == numIdsPerUrl {
			getPubmedRecords(first, wMesh, wXml, tr, pmids)
			checkTime()
			first = false
			count = 0
			zeroArray(pmids)
		} else {

		}
		allCount += 1
		chunkCount += 1
		// Start new xml file: close old one: open new one
		if chunkCount > recordsPerFile {
			fmt.Fprintln(wXml, endPubmedArticleSet)
			wXml.Flush()
			wXml.Close()
			ww.Flush()
			wXml, ww, xFile = makeXmlWriter(allCount, pmids[0])
			chunkCount = 0
			first = true
		}
		if allCount%500 == 0 {
			log.Println(allCount)
		}
	}
	if count != 0 {
		getPubmedRecords(first, wMesh, wXml, tr, pmids)
	}
	fmt.Fprintln(wXml, endPubmedArticleSet)

	wXml.Flush()
	wXml.Close()
	ww.Flush()

	wMesh.Flush()
	wMesh.Close()
	wwMesh.Flush()
	xFile.Close()
}

func zeroArray(a []string) {
	for i := 0; i < len(a); i++ {
		a[i] = ""
	}
}

func makeUrl(baseUrl string, pmids []string) string {
	url := baseUrl
	for i := 0; i < len(pmids); i++ {
		if pmids[i] != "" {
			if i != 0 {
				url += ","
			}
			url += pmids[i]
		}
	}
	//return "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&rettype=xml&id=15718680,15718682,119703,157186,1571868,11970375"
	return url
}

const baseUrl = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&rettype=xml&id="

func getPubmedRecords(first bool, meshWriter io.Writer, xmlWriter *gzip.Writer, transport *http.Transport, pmids []string) {
	preUrlTime := time.Now()
	url := makeUrl(baseUrl, pmids)

	log.Println(url)
	client := &http.Client{Transport: transport}
	req, err := http.NewRequest("GET", url, nil)
	req.Close = true
	resp, err := client.Do(req)

	if err != nil {
		log.Println("Error opening url:", url, "   error=", err)
		return
	}
	defer resp.Body.Close()

	buf := new(bytes.Buffer)
	buf.ReadFrom(resp.Body)
	s := buf.String()

	v := ArticleSet{}
	err = xml.Unmarshal(buf.Bytes(), &v)

	//log.Println(v);
	// This should be inside a go routine listening on a channel
	for i := 0; i < len(v.ArticleList); i++ {
		//log.Println(i)
		fmt.Fprint(meshWriter, v.ArticleList[i].MedlineCitation.PMID)
		pubmedArticle := v.ArticleList[i]
		//log.Println(pubmedArticle.MedlineCitation.Article.Abstract.AbstractText)
		for j := 0; j < len(pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading); j++ {
			//fmt.Fprintln(meshWriter, " ", article.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.DescriptorName)
			fmt.Fprint(meshWriter, "|")
			fmt.Fprint(meshWriter, pubmedArticle.MedlineCitation.MeshHeadingList.MeshHeading[j].DescriptorName.DescriptorName)
		}
		fmt.Fprintln(meshWriter, "")
	}
	if !first {
		s = strings.Replace(s, startXml, "", -1)
		s = strings.Replace(s, docType, "", -1)
		s = strings.Replace(s, startPubmedArticleSet, "", -1)
	}
	s = strings.Replace(s, endPubmedArticleSet, "<!-- breakset -->", -1)

	xmlWriter.Write([]byte(s))
	postUrlTime := time.Now()
	log.Println("Total request time:", postUrlTime.Sub(preUrlTime))
}

func makeXmlWriter(fileCount int, startPmid string) (*gzip.Writer, *bufio.Writer, *os.File) {
	//xmlFile, err := os.Create("./" + baseXmlFileName + strconv.Itoa(fileCount) + "_" + strconv.Itoa(fileCount+recordsPerFile) + ".gz")
	xmlFile, err := os.Create("./" + baseXmlFileName + startPmid + ".gz")
	if err != nil {
		return nil, nil, nil
	}
	ww := bufio.NewWriter(xmlFile)
	return gzip.NewWriter(ww), ww, xmlFile
}

func afterHours() bool {
	now := time.Now()
	hour, _, _ := now.Clock()
	return hour < 8 || hour > 18
}

func findNumIdsPerUrl() int {
	if afterHours() {
		return recordsPerHttpRequestAfterHours
	}
	return recordsPerHttpRequest
}

func checkTime() {
	now := time.Now()
	rand.Seed(int64(now.Nanosecond()/9999999 + now.Second()*100 + now.Hour()))
	sleepSeconds := 1 + rand.Intn(2)

	/*
		if !afterHours() {
			sleepSeconds = 1 + rand.Intn(2)
		}
	*/

	duration := (time.Duration)(sleepSeconds)
	log.Println("Start sleep")
	t0 := time.Now()

	time.Sleep(duration * time.Second)
	t1 := time.Now()
	log.Println("End sleep:", t1.Sub(t0))
}

func makeReader()(*bufio.Reader, error){
	var inputFile *os.File
	if readFromStdin{
		log.Println("*************************************************")
		inputFile = os.Stdin
	}else{
		if inputFileName == ""{
			return nil, errors.New("Empty input file name")
		}
		var err error
		inputFile, err = os.Open(inputFileName)
		if err != nil{
			return nil, err
		}
	}
	reader := bufio.NewReader(inputFile)
	return reader, nil
}

func lineChecker(l string)error{
	if len(l) == 0{
		return errors.New("Error: Empty line")
	}
	var n int
	var err error
	if n, err = strconv.Atoi(l); err != nil {
		return errors.New("Error: Expecting pmid (integer); found[" + l + "]")
	}
	if n <=0{
		return errors.New("Error: pmids are positive integers; found [" + l + "]")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/jideji/servicelauncher/config"
	"github.com/jideji/servicelauncher/service"
	"os"
)

func main() {
	if len(os.Args) < 2 {
		showHelp()
	}

	action := os.Args[1]
	var serviceName string
	if len(os.Args) >= 3 {
		serviceName = os.Args[2]
	}

	services := config.LoadServices()

	if len(serviceName) == 0 {
		for _, service := range services {
			doAction(service, action)
		}
		return
	}

	service := services[serviceName]
	if service == nil {
		println(fmt.Sprintf("No service named '%s' found.", serviceName))
		os.Exit(10)
	}

	doAction(service, action)
}

func doAction(service *service.Service, action string) {
	running := service.IsRunning()

	if action == "status" {
		if running {
			pid, _ := service.Pid()
			fmt.Printf("Service '%s' is running with pid %d.\n", service.Name, pid)
		} else {
			fmt.Printf("Service '%s' is not running.\n", service.Name)
		}
		return
	}

	if action == "stop" || action == "restart" {
		if running {
			pid, err := service.Pid()
			if err != nil {
				panic(err)
			}
			fmt.Printf("Killing process %d.\n", pid)
			service.Stop()
			running = false
		} else {
			fmt.Println("Not running.")
		}
	}

	if action == "start" || action == "restart" {
		if running {
			println(fmt.Sprintf("Service '%s' already running. Try restart.", service.Name))
			os.Exit(11)
		}
		p, err := service.Start()
		if err != nil {
			panic(err)
		}
		fmt.Printf("Service '%s' started with pid %d.\n", service.Name, p.Pid)
	}
}

func showHelp() {
	fmt.Fprint(os.Stderr, "SYNTAX:\n")
	fmt.Fprintf(os.Stderr, "\t%s <action> [<service name>]\n", os.Args[0])
	fmt.Fprint(os.Stderr, "Actions:\n")
	fmt.Fprint(os.Stderr, "\tstart, stop, restart, status\n")
	fmt.Fprint(os.Stderr, "Examples:\n")
	fmt.Fprintf(os.Stderr, "\t%s start httpserver\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t%s status\n", os.Args[0])
	os.Exit(1)
}
<commit_msg>Added support for specifying multiple services.<commit_after>package main

import (
	"fmt"
	"github.com/jideji/servicelauncher/config"
	"github.com/jideji/servicelauncher/service"
	"os"
)

func main() {
	if len(os.Args) < 2 {
		showHelp()
	}

	action := os.Args[1]

	services := config.LoadServices()

	selected := resolveServices(services, os.Args[2:]...)

	for _, service := range selected {
		doAction(service, action)
	}
}

func resolveServices(services map[string]*service.Service, names ...string) []*service.Service {
	var selected []*service.Service
	if len(names) > 0 {
		for _, name := range names {
			service, ok := services[name]
			if !ok {
				println(fmt.Sprintf("No service named '%s' found.", name))
				os.Exit(10)
			}
			selected = append(selected, service)
		}
	} else {
		for _, service := range services {
			selected = append(selected, service)
		}
	}

	return selected
}

func doAction(service *service.Service, action string) {
	running := service.IsRunning()

	if action == "status" {
		if running {
			pid, _ := service.Pid()
			fmt.Printf("Service '%s' is running with pid %d.\n", service.Name, pid)
		} else {
			fmt.Printf("Service '%s' is not running.\n", service.Name)
		}
		return
	}

	if action == "stop" || action == "restart" {
		if running {
			pid, err := service.Pid()
			if err != nil {
				panic(err)
			}
			fmt.Printf("Killing process %d.\n", pid)
			service.Stop()
			running = false
		} else {
			fmt.Println("Not running.")
		}
	}

	if action == "start" || action == "restart" {
		if running {
			println(fmt.Sprintf("Service '%s' already running. Try restart.", service.Name))
			os.Exit(11)
		}
		p, err := service.Start()
		if err != nil {
			panic(err)
		}
		fmt.Printf("Service '%s' started with pid %d.\n", service.Name, p.Pid)
	}
}

func showHelp() {
	fmt.Fprint(os.Stderr, "SYNTAX:\n")
	fmt.Fprintf(os.Stderr, "\t%s <action> [<service name>]\n", os.Args[0])
	fmt.Fprint(os.Stderr, "Actions:\n")
	fmt.Fprint(os.Stderr, "\tstart, stop, restart, status\n")
	fmt.Fprint(os.Stderr, "Examples:\n")
	fmt.Fprintf(os.Stderr, "\t%s start httpserver\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\t%s status\n", os.Args[0])
	os.Exit(1)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/codegangsta/cli"
	"github.com/lmas/ApolloStats/src"
)

var (
	g_addr     string
	g_database string
	g_debug    bool
)

func main() {
	app := cli.NewApp()
	app.Version = apollostats.VERSION
	app.Usage = "Run a web server, serving stats for Apollo."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "addr, a",
			Value:       "127.0.0.1:8000",
			Usage:       "serve web pages on this address",
			EnvVar:      "APOLLOSTATS_ADDR",
			Destination: &g_addr,
		},
		cli.StringFlag{
			Name:        "database, d",
			Value:       "user:password@/database",
			Usage:       "database authentication string",
			EnvVar:      "APOLLOSTATS_DBAUTH",
			Destination: &g_database,
		},
		cli.BoolFlag{
			Name:        "debug",
			Usage:       "run in debug mode",
			Destination: &g_debug,
		},
	}
	app.Commands = []cli.Command{
		{Name: "run", Usage: "Run the web server", Action: run_server},
		{Name: "update", Usage: "Update to the latest version", Action: run_update},
	}
	app.Run(os.Args)
}

func run_server(c *cli.Context) {
	db, e := apollostats.OpenDB(g_database)
	if e != nil {
		log.Fatal(fmt.Sprintf("Failed to connect to the database:\n%s\n", e.Error()))
	}

	i := apollostats.Instance{
		Debug: g_debug,
		DB:    db,
	}

	i.Init()
	i.Serve(g_addr)
}

func run_update(c *cli.Context) {
}
<commit_msg>Removes the update command, it won't be needed.<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/codegangsta/cli"
	"github.com/lmas/ApolloStats/src"
)

var (
	g_addr     string
	g_database string
	g_debug    bool
)

func main() {
	app := cli.NewApp()
	app.Version = apollostats.VERSION
	app.Usage = "Run a web server, serving stats for Apollo."
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:        "addr, a",
			Value:       "127.0.0.1:8000",
			Usage:       "serve web pages on this address",
			EnvVar:      "APOLLOSTATS_ADDR",
			Destination: &g_addr,
		},
		cli.StringFlag{
			Name:        "database, d",
			Value:       "user:password@/database",
			Usage:       "database authentication string",
			EnvVar:      "APOLLOSTATS_DBAUTH",
			Destination: &g_database,
		},
		cli.BoolFlag{
			Name:        "debug",
			Usage:       "run in debug mode",
			Destination: &g_debug,
		},
	}
	app.Commands = []cli.Command{
		{Name: "run", Usage: "Run the web server", Action: run_server},
	}
	app.Run(os.Args)
}

func run_server(c *cli.Context) {
	db, e := apollostats.OpenDB(g_database)
	if e != nil {
		log.Fatal(fmt.Sprintf("Failed to connect to the database:\n%s\n", e.Error()))
	}

	i := apollostats.Instance{
		Debug: g_debug,
		DB:    db,
	}

	i.Init()
	i.Serve(g_addr)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"
)

func main() {
	testNet()
}

func testNet() {
	addrs, _ := net.InterfaceAddrs()
	for _, addr := range addrs {
		log.Println(addr)
	}

	interfaces, _ := net.Interfaces()
	for _, interf := range interfaces {
		log.Println(interf)
	}
}
func init() {
	log.Println("init1")
}
func init() {
	log.Println("init2")
}
<commit_msg>dump request<commit_after>package main

import (
	// "bufio"
	"fmt"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	// "strings"
)

func main() {
	//testNet()
	testDumpRequest()
}

func testNet() {
	addrs, _ := net.InterfaceAddrs()
	for _, addr := range addrs {
		log.Println(addr)
	}

	interfaces, _ := net.Interfaces()
	for _, interf := range interfaces {
		log.Println(interf)
	}
}
func testDumpRequest() {
	// s := "POST /v2/api/?login HTTP/1.1\r\nHost: www.baidu.com\r\nConnection: keep-alive\r\nContent-Length: 21\r\nUser-Agent: Mozilla/5.0 (Windows NT 5.1)\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\nkey1=name1&key2=name2"
	// req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(s)))
	// if err != nil {
	// 	fmt.Println(err)
	// 	return
	// }
	req, _ := http.NewRequest("GET", "http://www.baidu.com", nil)
	req.ContentLength = 100
	// defer req.Body.Close()
	dump, err := httputil.DumpRequestOut(req, true)
	fmt.Println(string(dump))
	if err != nil {
		fmt.Println(err)
		return
	}
}
func init() {
	log.Println("init1")
}
func init() {
	log.Println("init2")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/koding/logging"
)

var (
	listen   = flag.String("listen", ":8080", "Address to listen for requests on")
	path     = flag.String("path", "./", "Path the server is started from")
	header   = flag.String("header", "", "Header sent with every response")
	redirect = flag.String("redirect", "", "Respond to unknown requests with this file. Used to serve Single Page Applications.")
)

func main() {
	flag.Parse()

	var headers map[string]string
	if *header != "" {
		s := strings.Split(*header, ":")
		headers = map[string]string{
			s[0]: s[1],
		}
	}

	server := HTTPServer{
		http.Server{
			Addr: *listen,
			Handler: HTTPHandler{
				realHandler: http.FileServer(SinglePageFileSystem{
					backendSystem: http.Dir(*path),
					redirectTo:    *redirect,
				}),
				headers: headers,
			},
		},
	}

	err := server.ListenAndServe()
	if err != nil {
		logging.Error(fmt.Sprint(err))
	}
}

type HTTPHandler struct {
	realHandler http.Handler
	headers     map[string]string
}

func (h HTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	logging.Info("%s %s", r.Method, r.URL)

	responseHeaders := w.Header()
	for k, v := range h.headers {
		responseHeaders.Set(k, v)
	}

	h.realHandler.ServeHTTP(w, r)
}

type HTTPServer struct {
	http.Server
}

func (h *HTTPServer) ListenAndServe() error {
	logging.Info(
		"Starting server on %s at %s",
		colorize(logging.CYAN, h.Server.Addr),
		colorize(logging.CYAN, *path),
	)

	for k, v := range h.Handler.(HTTPHandler).headers {
		logging.Info("Adding Header: %s:%s",
			colorize(logging.CYAN, k),
			colorize(logging.CYAN, v),
		)
	}

	return h.Server.ListenAndServe()
}

func colorize(color logging.Color, s string) string {
	return fmt.Sprintf("\033[%dm%s\033[0m", color, s)
}

// A http.FileSystem for serving single page applications by redirecting all
// unknown paths to a given path.
type SinglePageFileSystem struct {
	backendSystem http.Dir
	redirectTo    string
}

func (spa SinglePageFileSystem) Open(name string) (http.File, error) {
	var localPath string
	basePath := string(spa.backendSystem)
	reqPath := filepath.Join(basePath, name)

	if _, err := os.Stat(reqPath); os.IsNotExist(err) && spa.redirectTo != "" {
		localPath = spa.redirectTo
		logging.Info("[redirecting] (%s) -> (%s)\n",
			colorize(logging.CYAN, name),
			colorize(logging.CYAN, localPath),
		)
	} else {
		localPath = name
	}

	return spa.backendSystem.Open(localPath)
}
<commit_msg>Added Reverse Proxy<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/koding/logging"
)

var (
	listen    = flag.String("listen", ":8080", "Address to listen for requests on")
	servePath = flag.String("path", "./", "Path the server is started from")
	header    = flag.String("header", "", "Header sent with every response")
	redirect  = flag.String("redirect", "", "Respond to unknown requests with this file. Used to serve Single Page Applications.")

	proxyFrom = flag.String("proxy-from", "", "Proxy requests to this path to the the path specified by proxy-to.")
	proxyTo   = flag.String("proxy-to", "", "Proxy requests from proxy-from to this address")
)

func main() {
	flag.Parse()

	logging.Info("Starting server on %s at %s", emp(*listen), emp(*servePath))

	var headers map[string]string
	if *header != "" {
		s := strings.Split(*header, ":")
		headers = map[string]string{
			s[0]: s[1],
		}
	}

	for k, v := range headers {
		logging.Info("Adding Header: %s:%s", emp(k), emp(v))
	}

	mux := http.NewServeMux()

	if *redirect != "" {
		logging.Info("Redirecting 404s to: %s", emp(*redirect))
	}

	mux.Handle("/", HeaderHandler{
		Handler: http.FileServer(SinglePageFileSystem{
			backendSystem: http.Dir(*servePath),
			redirectTo:    *redirect,
		}),
		headers: headers,
	})

	if *proxyFrom != "" && *proxyTo != "" {
		// Handle Redirects
		logging.Info("Proxying: (%s) -> (%s)",
			emp(*proxyFrom),
			emp(*proxyTo),
		)

		proxyToUrl, err := url.Parse(*proxyTo)
		if err != nil {
			logging.Error("Bad -proxy-to URL:", err)
			return
		}

		basePath, err := url.Parse(*proxyFrom)
		if err != nil {
			logging.Error("Bad -proxy-from URL:", err)
			return
		}

		director := httputil.NewSingleHostReverseProxy(proxyToUrl).Director
		proxy := httputil.ReverseProxy{
			Director: func(r *http.Request) {
				// Remove Base Path
				oldPath := *r.URL
				r.URL.Path = strings.TrimPrefix(oldPath.Path, basePath.Path)

				logging.Info("[proxying] (%s) -> (%s)", emp(oldPath.String()), emp(r.URL.String()))
				director(r)
			},
			Transport: &ProxyRoundTripper{
				BasePath: basePath.Path,
			},
		}

		mux.Handle(*proxyFrom, &proxy)
	}

	server := http.Server{
		Addr:    *listen,
		Handler: LoggingHandler{mux},
	}

	err := server.ListenAndServe()
	if err != nil {
		logging.Error(fmt.Sprint(err))
	}
}

type HeaderHandler struct {
	headers map[string]string
	http.Handler
}

func (h HeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	responseHeaders := w.Header()
	for k, v := range h.headers {
		responseHeaders.Set(k, v)
	}

	h.Handler.ServeHTTP(w, r)
}

type LoggingHandler struct {
	http.Handler
}

func (h LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	logging.Info("%s %s", r.Method, r.URL)

	h.Handler.ServeHTTP(w, r)
}

func colorize(color logging.Color, s string) string {
	return fmt.Sprintf("\033[%dm%s\033[0m", color, s)
}

func emp(s string) string {
	return colorize(logging.CYAN, s)
}

// A http.FileSystem for serving single page applications by redirecting all
// unknown paths to a given path.
type SinglePageFileSystem struct {
	backendSystem http.Dir
	redirectTo    string
}

func (spa SinglePageFileSystem) Open(name string) (http.File, error) {
	var localPath string
	basePath := string(spa.backendSystem)
	reqPath := filepath.Join(basePath, name)

	if _, err := os.Stat(reqPath); os.IsNotExist(err) && spa.redirectTo != "" {
		localPath = spa.redirectTo
		logging.Info("[redirecting] (%s) -> (%s)\n", emp(name), emp(localPath))
	} else {
		localPath = name
	}

	return spa.backendSystem.Open(localPath)
}

type ProxyRoundTripper struct {
	BasePath string
}

func (t *ProxyRoundTripper) RoundTrip(r *http.Request) (*http.Response, error) {
	// Round Trip
	resp, err := http.DefaultTransport.RoundTrip(r)
	if err != nil {
		return nil, err
	}

	// Rewrite Cookie Paths
	cookies := resp.Cookies()
	for i := range cookies {
		cookie := cookies[i]
		cookie.Path = path.Join(t.BasePath, cookie.Path)
	}

	// Remove Cookies
	resp.Header.Del("Set-Cookie")

	// Set Cookies
	for _, cookie := range cookies {
		resp.Header.Add("Set-Cookie", cookie.String())
	}

	return resp, nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"log/syslog"
	"net/http"
	"strconv"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
)

// ~150k list limit

func maxLen(s string) int {
	if len(s) > 80 {
		return 80
	}
	return len(s)
}

func main() {
	m := martini.Classic()
	m.Use(render.Renderer())

	if err := dbCheck(); err != nil {
		log.Fatal("Error: could not connect to mongodb")
	}

	mainLogger, err := syslog.New(syslog.LOG_ERR, "")
	if err != nil {
		log.Fatal("Error: could not start syslog")
	}

	// Site
	m.Get("/", func(r render.Render) {
		count, err := getIndexInfo()
		if err != nil {
			mainLogger.Err("Error: db query went bad: " + err.Error())
			r.HTML(500, "index", nil) // make a new tmpl for this
		}
		r.HTML(200, "index", TempIndex{ListCount: strconv.Itoa(count)})
	})

	m.Get("/api", func(r render.Render) {
		r.HTML(200, "api", nil)
	})
	m.Get("/faq", func(r render.Render) {
		r.HTML(200, "faq", nil)
	})

	m.Get("/list/:label", func(params martini.Params, r render.Render) {
		label := params["label"][:maxLen(params["label"])]
		tmplList, err := getListValues(label)
		if err != nil {
			mainLogger.Err("Error: db query went bad: " + err.Error())
			r.HTML(500, "index", nil) // make a new tmpl for this
		}
		r.HTML(200, "list", TempList{Label: label, Todos: tmplList})
	})

	// API
	m.Get("/api/get/:label", func(params martini.Params) string {
		label := params["label"][:maxLen(params["label"])]
		todo := params["todo"][:maxLen(params["todo"])]
		return apiGet(label, todo, mainLogger)
	})

	m.Get("/api/remove/:label/:id", func(params martini.Params) string {
		label := params["label"][:maxLen(params["label"])]
		id := params["id"][:maxLen(params["id"])]
		return apiRemove(label, id, mainLogger)
	})

	m.Get("/api/add/:label/:todo", func(params martini.Params) string {
		label := params["label"][:maxLen(params["label"])]
		todo := params["todo"][:maxLen(params["todo"])]
		return apiAdd(label, todo, mainLogger)
	})

	m.Get("/api/count", func() string {
		return apiCount(mainLogger)
	})

	m.Use(func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "https://gogobbles.com"+r.RequestURI, http.StatusMovedPermanently)
	})

	m.Run()
}
<commit_msg>doesn't work<commit_after>package main

import (
	"log"
	"log/syslog"
	"strconv"

	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
)

// ~150k list limit

func maxLen(s string) int {
	if len(s) > 80 {
		return 80
	}
	return len(s)
}

func main() {
	m := martini.Classic()
	m.Use(render.Renderer())

	if err := dbCheck(); err != nil {
		log.Fatal("Error: could not connect to mongodb")
	}

	mainLogger, err := syslog.New(syslog.LOG_ERR, "")
	if err != nil {
		log.Fatal("Error: could not start syslog")
	}

	// Site
	m.Get("/", func(r render.Render) {
		count, err := getIndexInfo()
		if err != nil {
			mainLogger.Err("Error: db query went bad: " + err.Error())
			r.HTML(500, "index", nil) // make a new tmpl for this
		}
		r.HTML(200, "index", TempIndex{ListCount: strconv.Itoa(count)})
	})

	m.Get("/api", func(r render.Render) {
		r.HTML(200, "api", nil)
	})
	m.Get("/faq", func(r render.Render) {
		r.HTML(200, "faq", nil)
	})

	m.Get("/list/:label", func(params martini.Params, r render.Render) {
		label := params["label"][:maxLen(params["label"])]
		tmplList, err := getListValues(label)
		if err != nil {
			mainLogger.Err("Error: db query went bad: " + err.Error())
			r.HTML(500, "index", nil) // make a new tmpl for this
		}
		r.HTML(200, "list", TempList{Label: label, Todos: tmplList})
	})

	// API
	m.Get("/api/get/:label", func(params martini.Params) string {
		label := params["label"][:maxLen(params["label"])]
		todo := params["todo"][:maxLen(params["todo"])]
		return apiGet(label, todo, mainLogger)
	})

	m.Get("/api/remove/:label/:id", func(params martini.Params) string {
		label := params["label"][:maxLen(params["label"])]
		id := params["id"][:maxLen(params["id"])]
		return apiRemove(label, id, mainLogger)
	})

	m.Get("/api/add/:label/:todo", func(params martini.Params) string {
		label := params["label"][:maxLen(params["label"])]
		todo := params["todo"][:maxLen(params["todo"])]
		return apiAdd(label, todo, mainLogger)
	})

	m.Get("/api/count", func() string {
		return apiCount(mainLogger)
	})

	// m.Use(func(w http.ResponseWriter, r *http.Request) {
	// http.Redirect(w, r, "https://gogobbles.com"+r.RequestURI, http.StatusMovedPermanently)
	// })

	m.Run()
}
<|endoftext|>


<commit_before>//Collect stats from Strava and write out csv files for python analytics
package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"github.com/strava/go.strava"
	"io/ioutil"
	"os"
	"time"
)

type configuration struct {
	APIKey string `json:"apiKey"`
	Clubs  []struct {
		ID   int64  `json:"id"`
		Name string `json:"name"`
	} `json:"clubs"`
}

func main() {

	var csvHeader = []string{"Date", "Id", "athleteId", "athleteName", "distance", "movingTime", "wallTime",
		"totalElevationGain", "achievementCount", "averageSpeed", "maxSpeed", "runName"}

	data, err := ioutil.ReadFile("secrets.json")
	if err != nil {
		panic(err)
	}

	var conf configuration

	err = json.Unmarshal(data, &conf)
	if err != nil {
		panic(err)
	}

	client := strava.NewClient(conf.APIKey)

	//Get info about me.
	//athlete, err := service.Get().Do()
	if err != nil {
		panic(err)
	}
	//id := athlete.Id

	for _, club := range conf.Clubs {
		clubName := club.Name
		id := club.ID
		fmt.Printf("Fetching info for %s (id number %d)\n", clubName, id)
		clubInfo, err := strava.NewClubsService(client).ListMembers(id).Do()
		if err != nil {
			panic(err)
		}

		//A map since we're going to be looking up membership frequently
		clubMembers := make(map[string]bool, len(clubInfo))
		for _, x := range clubInfo {
			name := x.FirstName + " " + x.LastName
			clubMembers[name] = true
		}
		//Get mine and friends activities. No need to use page here since we're capped at 200 results.
		service := strava.NewCurrentAthleteService(client)
		friendActivities, err := service.ListFriendsActivities().PerPage(200).Do()
		if err != nil {
			panic(err)
		}

		//Get the club activities. There's going to be overlap, but this should get us the best spread
		// of data around Strava's API limits. It's easy enough to deal with duplicate data in CSV files.
		clubService := strava.NewClubsService(client)
		clubActivities, err := clubService.ListActivities(id).PerPage(200).Do()
		if err != nil {
			panic(err)
		}

		everything := [2][]*strava.ActivitySummary{friendActivities, clubActivities}

		t := time.Now().Format(time.RFC3339)
		csvFileName := t + "_" + clubName + ".csv"
		csvFile, err := os.Create(csvFileName)
		if err != nil {
			panic(err)
		}
		defer csvFile.Close()
		writer := csv.NewWriter(csvFile)

		writer.Write(csvHeader)

		for _, activities := range everything {
			for _, x := range activities {
				name := x.Athlete.FirstName + " " + x.Athlete.LastName
				if clubMembers[name] && x.Type == strava.ActivityTypes.Run {
					var line = []string{fmt.Sprintf("%v", x.StartDate), fmt.Sprintf("%d", x.Id),
						fmt.Sprintf("%d", x.Athlete.Id), name, fmt.Sprintf("%f", x.Distance),
						fmt.Sprintf("%v", x.MovingTime), fmt.Sprintf("%v", x.ElapsedTime),
						fmt.Sprintf("%f", x.TotalElevationGain), fmt.Sprintf("%d", x.AchievementCount),
						fmt.Sprintf("%f", x.AverageSpeed), fmt.Sprintf("%f", x.MaximunSpeed), x.Name}
					writer.Write(line)
				}
			}
		}

		writer.Flush()
		fmt.Println("Results written to", csvFileName)
	}
}
<commit_msg>write out two seperate files for better data manipulation<commit_after>//Collect stats from Strava and write out csv files for python analytics
package main

import (
	"encoding/csv"
	"encoding/json"
	"fmt"
	"github.com/strava/go.strava"
	"io/ioutil"
	"os"
	"time"
)

type configuration struct {
	APIKey string `json:"apiKey"`
	Clubs  []struct {
		ID   int64  `json:"id"`
		Name string `json:"name"`
	} `json:"clubs"`
}

func main() {

	var csvHeader = []string{"Date", "Id", "athleteId", "athleteName", "distance", "movingTime", "wallTime",
		"totalElevationGain", "achievementCount", "averageSpeed", "maxSpeed", "runName"}

	data, err := ioutil.ReadFile("secrets.json")
	if err != nil {
		panic(err)
	}

	var conf configuration

	err = json.Unmarshal(data, &conf)
	if err != nil {
		panic(err)
	}

	client := strava.NewClient(conf.APIKey)

	//Get info about me.
	//athlete, err := service.Get().Do()
	if err != nil {
		panic(err)
	}
	//id := athlete.Id

	for _, club := range conf.Clubs {
		clubName := club.Name
		id := club.ID
		fmt.Printf("Fetching info for %s (id number %d)\n", clubName, id)
		clubInfo, err := strava.NewClubsService(client).ListMembers(id).Do()
		if err != nil {
			panic(err)
		}

		//A map since we're going to be looking up membership frequently
		clubMembers := make(map[string]bool, len(clubInfo))
		for _, x := range clubInfo {
			name := x.FirstName + " " + x.LastName
			clubMembers[name] = true
		}
		//Get mine and friends activities. No need to use page here since we're capped at 200 results.
		service := strava.NewCurrentAthleteService(client)
		friendActivities, err := service.ListFriendsActivities().PerPage(200).Do()
		if err != nil {
			panic(err)
		}

		//Get the club activities. There's going to be overlap, but this should get us the best spread
		// of data around Strava's API limits. It's easy enough to deal with duplicate data in CSV files.
		clubService := strava.NewClubsService(client)
		clubActivities, err := clubService.ListActivities(id).PerPage(200).Do()
		if err != nil {
			panic(err)
		}

		everything := [2][]*strava.ActivitySummary{friendActivities, clubActivities}

		t := time.Now().Format(time.RFC3339)

		for y, activities := range everything {
			name := ""
			if y == 0 {
				name = "friends"
			} else {
				name = "entire_club"
			}
			csvFileName := t + "_" + clubName + "_" + name + ".csv"
			csvFile, err := os.Create(csvFileName)
			if err != nil {
				panic(err)
			}
			defer csvFile.Close()
			writer := csv.NewWriter(csvFile)

			writer.Write(csvHeader)

			for _, x := range activities {
				name := x.Athlete.FirstName + " " + x.Athlete.LastName
				if clubMembers[name] && x.Type == strava.ActivityTypes.Run {
					var line = []string{fmt.Sprintf("%v", x.StartDate), fmt.Sprintf("%d", x.Id),
						fmt.Sprintf("%d", x.Athlete.Id), name, fmt.Sprintf("%f", x.Distance),
						fmt.Sprintf("%v", x.MovingTime), fmt.Sprintf("%v", x.ElapsedTime),
						fmt.Sprintf("%f", x.TotalElevationGain), fmt.Sprintf("%d", x.AchievementCount),
						fmt.Sprintf("%f", x.AverageSpeed), fmt.Sprintf("%f", x.MaximunSpeed), x.Name}
					writer.Write(line)
				}
			}
			writer.Flush()
			fmt.Println("Results written to", csvFileName)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"regexp"
	"runtime/debug"
	"strings"
	"time"
)

type Fortune struct {
	wr   http.ResponseWriter
	rq   *http.Request
	deck *Deck
}

func init() {
	debug := flag.Bool("d", false, "debug")
	flag.Parse()

	if !*debug {
		log.SetOutput(ioutil.Discard)
	}
}

func main() {
	var fortune Fortune

	fmt.Println("Listening on http://localhost:8080")

	err := http.ListenAndServe(":8080", &fortune)
	if err != nil {
		fmt.Println(err)
	}
}

func (f *Fortune) ServeHTTP(wr http.ResponseWriter, rq *http.Request) {
	defer func() {
		obj := recover()
		if obj != nil {
			msg := fmt.Sprintf("
Error: %v\nStack: %v
", obj, string(debug.Stack()))
			io.WriteString(wr, msg)
			fmt.Println(msg)
		}
	}()

	f.wr = wr
	f.rq = rq

	path := rq.URL.Path
	contentType := "text/html"
	root := "."

	switch {
	case strings.HasPrefix(path, "/playing-cards/"):
		contentType = "image/png"
		wr.Header().Set("cache-control", "public")

	case strings.HasPrefix(path, "/js/"):
		contentType = "application/javascript"

	case strings.HasPrefix(path, "/css/"):
		contentType = "text/css"

	case strings.HasPrefix(path, "/html/"):
		contentType = "text/html"

	case path == "/":
		contentType = "text/html"
		path = "/html/main.html"
		fmt.Printf("%s: Visitor from %s\n", time.Now(), rq.RemoteAddr)

	case path == "/init":
		f.init()
		path = ""

	case path == "/deal":
		f.deal()
		path = ""

	case path == "/fortune":
		f.fortune()
		path = ""
	}

	if len(path) > 0 {
		wr.Header().Set("Content-Type", contentType)
		data, err := ioutil.ReadFile(root + path)
		if err == nil {
			wr.Write(data)
		} else {
			fmt.Fprint(wr, err)
		}
	}
}

func (f *Fortune) init() {
	f.deck = &Deck{}
	f.deck.init()
	f.deck.shuffle()
	f.deck.Cards = f.deck.Cards[:21]

	type Response struct {
		Cards []*Card
		Error string
	}

	response := &Response{
		Cards: f.deck.Cards,
	}
	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) deal() {
	type RequestCard struct {
		Image string
	}
	type Request struct {
		Cards []RequestCard
		Row   int
		Count int
	}
	type Response struct {
		Row1  []*Card
		Row2  []*Card
		Row3  []*Card
		Card  string
		Error string
	}

	response := &Response{}
	reqData, err := ioutil.ReadAll(f.rq.Body)
	if err != nil {
		response.Error = err.Error()
	} else {
		request := &Request{}
		err = json.Unmarshal(reqData, request)
		if err != nil {
			response.Error = err.Error()
		}
		f.deck = &Deck{}
		for _, card := range request.Cards {
			f.deck.Cards = append(f.deck.Cards, &Card{Image: card.Image})
		}
		if len(request.Cards) == 21 {
			if request.Row == 0 {
				response.Row1 = f.deck.Cards[:7]
				response.Row2 = f.deck.Cards[7:14]
				response.Row3 = f.deck.Cards[14:]
			} else {
				f.deck.placeMiddle(request.Row)
				f.deck.deal()
				response.Row1 = f.deck.Row1
				response.Row2 = f.deck.Row2
				response.Row3 = f.deck.Row3
			}
		} else {
			response.Error += "\nDeck should have 21 cards."
		}
		log.Printf("request: %v\n", request)
		if request.Count == 3 {
			response.Card = f.deck.Row2[3].Image
			log.Printf("memorized card: %s\n", response.Card)
		}
	}

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) fortune() {
	words := map[string]string{
		"2C.png":  "law",
		"2D.png":  "wealth",
		"2H.png":  "love",
		"2S.png":  "passion",
		"3C.png":  "rule",
		"3D.png":  "rich",
		"3H.png":  "like",
		"3S.png":  "interest",
		"4C.png":  "command",
		"4D.png":  "gold",
		"4H.png":  "nice",
		"4S.png":  "positive",
		"5C.png":  "advise",
		"5D.png":  "money",
		"5H.png":  "related",
		"5S.png":  "real",
		"6C.png":  "statement",
		"6D.png":  "fortune",
		"6H.png":  "good",
		"6S.png":  "growing",
		"7C.png":  "court",
		"7D.png":  "well",
		"7H.png":  "sweet",
		"7S.png":  "study",
		"8C.png":  "action",
		"8D.png":  "cash",
		"8H.png":  "protect",
		"8S.png":  "understand",
		"9C.png":  "act",
		"9D.png":  "stock",
		"9H.png":  "live",
		"9S.png":  "hobby",
		"10C.png": "order",
		"10D.png": "value",
		"10H.png": "friend",
		"10S.png": "knowledge",
		"JC.png":  "judge",
		"JD.png":  "banker",
		"JH.png":  "husband",
		"JS.png":  "student",
		"QC.png":  "queen",
		"QD.png":  "actress",
		"QH.png":  "wife",
		"QS.png":  "nurse",
		"KC.png":  "congressman",
		"KD.png":  "ceo",
		"KH.png":  "lover",
		"KS.png":  "researcher",
		"AC.png":  "country",
		"AD.png":  "thesaurus",
		"AH.png":  "family",
		"AS.png":  "president",
	}
	type Request struct {
		Card string
	}
	type Response struct {
		Tweet string
		Error string
	}

	response := &Response{}

	request := &Request{}
	reqData, err := ioutil.ReadAll(f.rq.Body)
	err = json.Unmarshal(reqData, request)
	if err != nil {
		response.Error = err.Error()
	}

	key, _ := words[request.Card]
	url := "https://twitter.com/search?q=" + key
	resp, err := http.Get(url)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	resp.Body.Close()

	search := regexp.MustCompile(`<p class="TweetTextSize .*>.*</p>`)
	tweets := search.FindStringSubmatch(string(body))

	tweet := "Unable to fetch tweets."
	if len(tweets) > 0 {
		tweet = tweets[0]
	}
	response.Tweet = tweet
	fmt.Printf("Visitor=%s word=%s fortune=%s\n", f.rq.RemoteAddr, key, tweet)

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}
<commit_msg>realtime search<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"regexp"
	"runtime/debug"
	"strings"
	"time"
)

type Fortune struct {
	wr   http.ResponseWriter
	rq   *http.Request
	deck *Deck
}

func init() {
	debug := flag.Bool("d", false, "debug")
	flag.Parse()

	if !*debug {
		log.SetOutput(ioutil.Discard)
	}
}

func main() {
	var fortune Fortune

	fmt.Println("Listening on http://localhost:8080")

	err := http.ListenAndServe(":8080", &fortune)
	if err != nil {
		fmt.Println(err)
	}
}

func (f *Fortune) ServeHTTP(wr http.ResponseWriter, rq *http.Request) {
	defer func() {
		obj := recover()
		if obj != nil {
			msg := fmt.Sprintf("
Error: %v\nStack: %v
", obj, string(debug.Stack()))
			io.WriteString(wr, msg)
			fmt.Println(msg)
		}
	}()

	f.wr = wr
	f.rq = rq

	path := rq.URL.Path
	contentType := "text/html"
	root := "."

	switch {
	case strings.HasPrefix(path, "/playing-cards/"):
		contentType = "image/png"
		wr.Header().Set("cache-control", "public")

	case strings.HasPrefix(path, "/js/"):
		contentType = "application/javascript"

	case strings.HasPrefix(path, "/css/"):
		contentType = "text/css"

	case strings.HasPrefix(path, "/html/"):
		contentType = "text/html"

	case path == "/":
		contentType = "text/html"
		path = "/html/main.html"
		fmt.Printf("%s: Visitor from %s\n", time.Now(), rq.RemoteAddr)

	case path == "/init":
		f.init()
		path = ""

	case path == "/deal":
		f.deal()
		path = ""

	case path == "/fortune":
		f.fortune()
		path = ""
	}

	if len(path) > 0 {
		wr.Header().Set("Content-Type", contentType)
		data, err := ioutil.ReadFile(root + path)
		if err == nil {
			wr.Write(data)
		} else {
			fmt.Fprint(wr, err)
		}
	}
}

func (f *Fortune) init() {
	f.deck = &Deck{}
	f.deck.init()
	f.deck.shuffle()
	f.deck.Cards = f.deck.Cards[:21]

	type Response struct {
		Cards []*Card
		Error string
	}

	response := &Response{
		Cards: f.deck.Cards,
	}
	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) deal() {
	type RequestCard struct {
		Image string
	}
	type Request struct {
		Cards []RequestCard
		Row   int
		Count int
	}
	type Response struct {
		Row1  []*Card
		Row2  []*Card
		Row3  []*Card
		Card  string
		Error string
	}

	response := &Response{}
	reqData, err := ioutil.ReadAll(f.rq.Body)
	if err != nil {
		response.Error = err.Error()
	} else {
		request := &Request{}
		err = json.Unmarshal(reqData, request)
		if err != nil {
			response.Error = err.Error()
		}
		f.deck = &Deck{}
		for _, card := range request.Cards {
			f.deck.Cards = append(f.deck.Cards, &Card{Image: card.Image})
		}
		if len(request.Cards) == 21 {
			if request.Row == 0 {
				response.Row1 = f.deck.Cards[:7]
				response.Row2 = f.deck.Cards[7:14]
				response.Row3 = f.deck.Cards[14:]
			} else {
				f.deck.placeMiddle(request.Row)
				f.deck.deal()
				response.Row1 = f.deck.Row1
				response.Row2 = f.deck.Row2
				response.Row3 = f.deck.Row3
			}
		} else {
			response.Error += "\nDeck should have 21 cards."
		}
		log.Printf("request: %v\n", request)
		if request.Count == 3 {
			response.Card = f.deck.Row2[3].Image
			log.Printf("memorized card: %s\n", response.Card)
		}
	}

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}

func (f *Fortune) fortune() {
	words := map[string]string{
		"2C.png":  "law",
		"2D.png":  "wealth",
		"2H.png":  "love",
		"2S.png":  "passion",
		"3C.png":  "rule",
		"3D.png":  "rich",
		"3H.png":  "like",
		"3S.png":  "interest",
		"4C.png":  "command",
		"4D.png":  "gold",
		"4H.png":  "nice",
		"4S.png":  "positive",
		"5C.png":  "advise",
		"5D.png":  "money",
		"5H.png":  "related",
		"5S.png":  "real",
		"6C.png":  "statement",
		"6D.png":  "fortune",
		"6H.png":  "good",
		"6S.png":  "growing",
		"7C.png":  "court",
		"7D.png":  "well",
		"7H.png":  "sweet",
		"7S.png":  "study",
		"8C.png":  "action",
		"8D.png":  "cash",
		"8H.png":  "protect",
		"8S.png":  "understand",
		"9C.png":  "act",
		"9D.png":  "stock",
		"9H.png":  "live",
		"9S.png":  "hobby",
		"10C.png": "order",
		"10D.png": "value",
		"10H.png": "friend",
		"10S.png": "knowledge",
		"JC.png":  "judge",
		"JD.png":  "banker",
		"JH.png":  "husband",
		"JS.png":  "student",
		"QC.png":  "queen",
		"QD.png":  "actress",
		"QH.png":  "wife",
		"QS.png":  "nurse",
		"KC.png":  "congressman",
		"KD.png":  "ceo",
		"KH.png":  "lover",
		"KS.png":  "researcher",
		"AC.png":  "country",
		"AD.png":  "thesaurus",
		"AH.png":  "family",
		"AS.png":  "president",
	}
	type Request struct {
		Card string
	}
	type Response struct {
		Tweet string
		Error string
	}

	response := &Response{}

	request := &Request{}
	reqData, err := ioutil.ReadAll(f.rq.Body)
	err = json.Unmarshal(reqData, request)
	if err != nil {
		response.Error = err.Error()
	}

	key, _ := words[request.Card]
	url := "https://twitter.com/search?f=realtime&q=" + key
	resp, err := http.Get(url)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		response.Error = "Error: " + err.Error()
	}
	resp.Body.Close()

	search := regexp.MustCompile(`<p class="TweetTextSize .*>.*</p>`)
	tweets := search.FindAllString(string(body), -1)

	tweet := "Unable to fetch tweets."
	if len(tweets) > 0 {
		index := rand.Intn(len(tweets))
		tweet = tweets[index]
	}
	response.Tweet = tweet
	fmt.Printf("Visitor=%s word=%s fortune=%s\n", f.rq.RemoteAddr, key, tweet)

	data, err := json.Marshal(response)
	if err != nil {
		response.Error = err.Error()
	}
	f.wr.Header().Set("Content-Type", "application/json")
	f.wr.Write(data)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/Dataman-Cloud/swan/src/config"
	"github.com/Dataman-Cloud/swan/src/node"
	"github.com/Dataman-Cloud/swan/src/version"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
)

func setupLogger(logLevel string) {
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		level = logrus.DebugLevel
	}
	logrus.SetLevel(level)

	logrus.SetOutput(os.Stderr)

	logrus.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: "2006-01-02 15:04:05",
		FullTimestamp:   true,
	})
}

func main() {
	app := cli.NewApp()
	app.Name = "swan"
	app.Usage = "A general purpose mesos framework"
	app.Version = version.Version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen-addr",
			Usage:  "listener address for agent",
			EnvVar: "SWAN_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "advertise-addr",
			Usage:  "advertise address for agent",
			EnvVar: "SWAN_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-listen-addr",
			Usage:  "swan raft serverlistener address",
			EnvVar: "SWAN_RAFT_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-advertise-addr",
			Usage:  "swan raft advertise address",
			EnvVar: "SWAN_RAFT_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "join-addrs",
			Usage:  "the addrs new node join to. Splited by ','",
			EnvVar: "SWAN_JOIN_ADDRS",
		},
		cli.StringFlag{
			Name:   "janitor-advertise-ip",
			Usage:  "janitor proxy advertise ip",
			EnvVar: "SWAN_JANITOR_ADVERTISE_IP",
		},

		cli.StringFlag{
			Name:   "zk-path",
			Usage:  "zookeeper mesos paths. eg. zk://host1:port1,host2:port2,.../path",
			EnvVar: "SWAN_MESOS_ZKPATH",
		},
		cli.StringFlag{
			Name:   "log-level,l",
			Usage:  "customize debug level [debug|info|error]",
			EnvVar: "SWAN_LOG_LEVEL",
		},
		cli.StringFlag{
			Name:   "mode",
			Usage:  "server mode, manager|agent|mixed ",
			EnvVar: "SWAN_MODE",
		},
		cli.StringFlag{
			Name:   "data-dir,d",
			Usage:  "swan data store dir",
			EnvVar: "SWAN_DATA_DIR",
		},
		cli.StringFlag{
			Name:   "domain",
			Usage:  "domain which resolve to proxies. eg. swan.com, which make any task can be access from path likes 0.appname.username.cluster.swan.com",
			EnvVar: "SWAN_DOMAIN",
		},
		//cli.StringFlag{
		//Name:   "janitor-listen-ip",
		//Usage:  "janitor proxy listener ip",
		//EnvVar: "SWAN_JANITOR_LISTEN_IP",
		//},

		//cli.StringFlag{
		//Name:   "dns-listen-addr",
		//Usage:  "dns proxy listener address",
		//EnvVar: "SWAN_DNS_LISTEN_ADDR",
		//},
	}
	app.Action = func(c *cli.Context) error {
		config, err := config.NewConfig(c)
		if err != nil {
			logrus.Errorf("load config failed. Error: %s", err)
			return err
		}

		setupLogger(config.LogLevel)

		node, err := node.NewNode(config)
		if err != nil {
			logrus.Error("Node initialization failed")
			return err
		}

		if err := node.Start(context.Background()); err != nil {
			logrus.Errorf("start node failed. Error: %s", err.Error())
			return err
		}

		return nil
	}

	app.Run(os.Args)
}
<commit_msg>Print usage when the program runs without any command-line args<commit_after>package main

import (
	"os"

	"github.com/Dataman-Cloud/swan/src/config"
	"github.com/Dataman-Cloud/swan/src/node"
	"github.com/Dataman-Cloud/swan/src/version"

	"github.com/Sirupsen/logrus"
	"github.com/urfave/cli"
	"golang.org/x/net/context"
)

func setupLogger(logLevel string) {
	level, err := logrus.ParseLevel(logLevel)
	if err != nil {
		level = logrus.DebugLevel
	}
	logrus.SetLevel(level)

	logrus.SetOutput(os.Stderr)

	logrus.SetFormatter(&logrus.TextFormatter{
		TimestampFormat: "2006-01-02 15:04:05",
		FullTimestamp:   true,
	})
}

func main() {
	app := cli.NewApp()
	app.Name = "swan"
	app.Usage = "A general purpose mesos framework"
	app.Version = version.Version

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "listen-addr",
			Usage:  "listener address for agent",
			EnvVar: "SWAN_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "advertise-addr",
			Usage:  "advertise address for agent",
			EnvVar: "SWAN_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-listen-addr",
			Usage:  "swan raft serverlistener address",
			EnvVar: "SWAN_RAFT_LISTEN_ADDR",
		},
		cli.StringFlag{
			Name:   "raft-advertise-addr",
			Usage:  "swan raft advertise address",
			EnvVar: "SWAN_RAFT_ADVERTISE_ADDR",
		},
		cli.StringFlag{
			Name:   "join-addrs",
			Usage:  "the addrs new node join to. Splited by ','",
			EnvVar: "SWAN_JOIN_ADDRS",
		},
		cli.StringFlag{
			Name:   "janitor-advertise-ip",
			Usage:  "janitor proxy advertise ip",
			EnvVar: "SWAN_JANITOR_ADVERTISE_IP",
		},

		cli.StringFlag{
			Name:   "zk-path",
			Usage:  "zookeeper mesos paths. eg. zk://host1:port1,host2:port2,.../path",
			EnvVar: "SWAN_MESOS_ZKPATH",
		},
		cli.StringFlag{
			Name:   "log-level,l",
			Usage:  "customize debug level [debug|info|error]",
			EnvVar: "SWAN_LOG_LEVEL",
		},
		cli.StringFlag{
			Name:   "mode",
			Usage:  "server mode, manager|agent|mixed ",
			EnvVar: "SWAN_MODE",
		},
		cli.StringFlag{
			Name:   "data-dir,d",
			Usage:  "swan data store dir",
			EnvVar: "SWAN_DATA_DIR",
		},
		cli.StringFlag{
			Name:   "domain",
			Usage:  "domain which resolve to proxies. eg. swan.com, which make any task can be access from path likes 0.appname.username.cluster.swan.com",
			EnvVar: "SWAN_DOMAIN",
		},
	}
	app.Action = func(c *cli.Context) error {
		if c.NArg() == 0 {
			return cli.ShowAppHelp(c)
		}

		config, err := config.NewConfig(c)
		if err != nil {
			logrus.Errorf("load config failed. Error: %s", err)
			return err
		}

		setupLogger(config.LogLevel)

		node, err := node.NewNode(config)
		if err != nil {
			logrus.Error("Node initialization failed")
			return err
		}

		if err := node.Start(context.Background()); err != nil {
			logrus.Errorf("start node failed. Error: %s", err.Error())
			return err
		}

		return nil
	}

	if err := app.Run(os.Args); err != nil {
		logrus.Errorf("%s", err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/montanaflynn/stats"
)

func worker(requests int, image string, completeCh chan time.Duration) {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		panic(err)
	}

	for i := 0; i < requests; i++ {
		start := time.Now()

		container, err := client.CreateContainer(docker.CreateContainerOptions{
			Config: &docker.Config{
				Image: image,
			}})
		if err != nil {
			panic(err)
		}

		err = client.StartContainer(container.ID, nil)
		if err != nil {
			panic(err)
		}

		completeCh <- time.Since(start)
	}
}

func session(requests, concurrency int, image string, completeCh chan time.Duration) {
	var wg sync.WaitGroup

	n := requests / concurrency

	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		go func() {
			worker(n, image, completeCh)
			wg.Done()
		}()
	}
	wg.Wait()
}

func bench(requests, concurrency int, image string) {
	start := time.Now()

	timings := make([]float64, requests)
	completeCh := make(chan time.Duration)
	doneCh := make(chan struct{})
	current := 0
	go func() {
		for timing := range completeCh {
			timings = append(timings, timing.Seconds())
			current++
			percent := float64(current) / float64(requests) * 100
			fmt.Printf("[%3.f%%] %d/%d containers started\n", percent, current, requests)
		}
		doneCh <- struct{}{}
	}()
	session(requests, concurrency, image, completeCh)
	close(completeCh)
	<-doneCh

	total := time.Since(start)
	mean, _ := stats.Mean(timings)
	p90th, _ := stats.Percentile(timings, 90)
	p99th, _ := stats.Percentile(timings, 99)

	fmt.Printf("\n")
	fmt.Printf("Time taken for tests: %s\n", total.String())
	fmt.Printf("Time per container: %vms [mean] | %vms [90th] | %vms [99th]\n", int(mean*1000), int(p90th*1000), int(p99th*1000))
}

func main() {
	app := cli.NewApp()
	app.Name = "swarm-bench"
	app.Usage = "Swarm Benchmarking Tool"
	app.Version = "0.1.0"
	app.Author = ""
	app.Email = ""
	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "concurrency, c",
			Value: 1,
			Usage: "Number of multiple requests to perform at a time. Default is one request at a time.",
		},
		cli.IntFlag{
			Name:  "requests, n",
			Value: 1,
			Usage: "Number of containers to start for the benchmarking session. The default is to just start a single container.",
		},
		cli.StringFlag{
			Name:  "image, i",
			Usage: "Image to use for benchmarking.",
		},
	}

	app.Action = func(c *cli.Context) {
		if c.String("image") == "" {
			cli.ShowAppHelp(c)
			os.Exit(1)
		}
		bench(c.Int("requests"), c.Int("concurrency"), c.String("image"))
	}

	app.Run(os.Args)
}
<commit_msg>Use a buffered channel for displaying so we don't slow down the workers.<commit_after>package main

import (
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	docker "github.com/fsouza/go-dockerclient"
	"github.com/montanaflynn/stats"
)

func worker(requests int, image string, completeCh chan time.Duration) {
	client, err := docker.NewClientFromEnv()
	if err != nil {
		panic(err)
	}

	for i := 0; i < requests; i++ {
		start := time.Now()

		container, err := client.CreateContainer(docker.CreateContainerOptions{
			Config: &docker.Config{
				Image: image,
			}})
		if err != nil {
			panic(err)
		}

		err = client.StartContainer(container.ID, nil)
		if err != nil {
			panic(err)
		}

		completeCh <- time.Since(start)
	}
}

func session(requests, concurrency int, image string, completeCh chan time.Duration) {
	var wg sync.WaitGroup

	n := requests / concurrency

	for i := 0; i < concurrency; i++ {
		wg.Add(1)
		go func() {
			worker(n, image, completeCh)
			wg.Done()
		}()
	}
	wg.Wait()
}

func bench(requests, concurrency int, image string) {
	start := time.Now()

	timings := make([]float64, requests)
	// Create a buffered channel so our display goroutine can't slow down the workers.
	completeCh := make(chan time.Duration, requests)
	doneCh := make(chan struct{})
	current := 0
	go func() {
		for timing := range completeCh {
			timings = append(timings, timing.Seconds())
			current++
			percent := float64(current) / float64(requests) * 100
			fmt.Printf("[%3.f%%] %d/%d containers started\n", percent, current, requests)
		}
		doneCh <- struct{}{}
	}()
	session(requests, concurrency, image, completeCh)
	close(completeCh)
	<-doneCh

	total := time.Since(start)
	mean, _ := stats.Mean(timings)
	p90th, _ := stats.Percentile(timings, 90)
	p99th, _ := stats.Percentile(timings, 99)

	fmt.Printf("\n")
	fmt.Printf("Time taken for tests: %s\n", total.String())
	fmt.Printf("Time per container: %vms [mean] | %vms [90th] | %vms [99th]\n", int(mean*1000), int(p90th*1000), int(p99th*1000))
}

func main() {
	app := cli.NewApp()
	app.Name = "swarm-bench"
	app.Usage = "Swarm Benchmarking Tool"
	app.Version = "0.1.0"
	app.Author = ""
	app.Email = ""
	app.Flags = []cli.Flag{
		cli.IntFlag{
			Name:  "concurrency, c",
			Value: 1,
			Usage: "Number of multiple requests to perform at a time. Default is one request at a time.",
		},
		cli.IntFlag{
			Name:  "requests, n",
			Value: 1,
			Usage: "Number of containers to start for the benchmarking session. The default is to just start a single container.",
		},
		cli.StringFlag{
			Name:  "image, i",
			Usage: "Image to use for benchmarking.",
		},
	}

	app.Action = func(c *cli.Context) {
		if c.String("image") == "" {
			cli.ShowAppHelp(c)
			os.Exit(1)
		}
		bench(c.Int("requests"), c.Int("concurrency"), c.String("image"))
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/mount"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcscaching"
	"golang.org/x/net/context"
)

var fBucketName = flag.String("bucket", "", "Name of GCS bucket to mount.")
var fMountPoint = flag.String("mount_point", "", "File system location.")

var fMountOptions = mount.OptionFlag(
	"o",
	"Additional system-specific mount options. Be careful!")

// TODO(jacobsa): Kill this in favor of "-o ro".
var fReadOnly = flag.Bool("read_only", false, "Mount in read-only mode.")

var fTempDir = flag.String(
	"temp_dir", "",
	"The temporary directory in which to store local copies of GCS objects. "+
		"If empty, the system default (probably /tmp) will be used.")

var fTempDirLimit = flag.Int64(
	"temp_dir_bytes", 1<<31,
	"A desired limit on the number of bytes used in --temp_dir. May be exceeded "+
		"for dirty files that have not been flushed or closed.")

var fGCSChunkSize = flag.Uint64(
	"gcs_chunk_size", 1<<24,
	"If set to a non-zero value N, split up GCS objects into multiple chunks of "+
		"size at most N when reading, and do not read or cache unnecessary chunks.")

var fImplicitDirs = flag.Bool(
	"implicit_dirs",
	false,
	"Implicitly define directories based on their content. See "+
		"docs/semantics.md.")

var fSupportNlink = flag.Bool(
	"support_nlink",
	false,
	"Return meaningful values for nlink from fstat(2). See docs/semantics.md.")

var fStatCacheTTL = flag.Duration(
	"stat_cache_ttl",
	time.Minute,
	"How long to cache StatObject results from GCS.")

var fTypeCacheTTL = flag.Duration(
	"type_cache_ttl",
	time.Minute,
	"How long to cache name -> file/dir type mappings in directory inodes.")

func getBucketName() string {
	s := *fBucketName
	if s == "" {
		fmt.Println("You must set --bucket.")
		os.Exit(1)
	}

	return s
}

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			log.Println("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				log.Printf("Failed to unmount in response to SIGINT: %v", err)
			} else {
				log.Printf("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func getBucket() (b gcs.Bucket) {
	// Set up a GCS connection.
	log.Println("Initializing GCS connection.")
	conn, err := getConn()
	if err != nil {
		log.Fatal("Couldn't get GCS connection: ", err)
	}

	// Extract the appropriate bucket.
	b = conn.GetBucket(getBucketName())

	// Enable cached StatObject results, if appropriate.
	if *fStatCacheTTL != 0 {
		const cacheCapacity = 4096
		b = gcscaching.NewFastStatBucket(
			*fStatCacheTTL,
			gcscaching.NewStatCache(cacheCapacity),
			timeutil.RealClock(),
			b)
	}

	return
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up flags.
	flag.Parse()

	// Check --mount_point.
	if *fMountPoint == "" {
		log.Fatalf("You must set --mount_point.")
	}

	mountPoint := *fMountPoint

	// Sanity check: make sure the temporary directory exists and is writable
	// currently. This gives a better user experience than harder to debug EIO
	// errors when reading files in the future.
	if *fTempDir != "" {
		f, err := fsutil.AnonymousFile(*fTempDir)
		f.Close()

		if err != nil {
			log.Fatalf(
				"Error writing to temporary directory (%q); are you sure it exists "+
					"with the correct permissions?",
				err.Error())
		}
	}

	// Create a file system server.
	bucket := getBucket()
	serverCfg := &fs.ServerConfig{
		Clock:               timeutil.RealClock(),
		Bucket:              bucket,
		TempDir:             *fTempDir,
		TempDirLimit:        *fTempDirLimit,
		GCSChunkSize:        *fGCSChunkSize,
		ImplicitDirectories: *fImplicitDirs,
		SupportNlink:        *fSupportNlink,
		DirTypeCacheTTL:     *fTypeCacheTTL,
	}

	server, err := fs.NewServer(serverCfg)
	if err != nil {
		log.Fatal("fs.NewServer:", err)
	}

	// Mount the file system.
	mountCfg := &fuse.MountConfig{
		FSName:   bucket.Name(),
		ReadOnly: *fReadOnly,
		Options:  fMountOptions,
	}

	mountedFS, err := fuse.Mount(mountPoint, server, mountCfg)
	if err != nil {
		log.Fatal("Mount:", err)
	}

	log.Println("File system has been successfully mounted.")

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mountedFS.Dir())

	// Wait for it to be unmounted.
	if err := mountedFS.Join(context.Background()); err != nil {
		log.Fatal("MountedFileSystem.Join:", err)
	}

	log.Println("Successfully exiting.")
}
<commit_msg>Removed the --read_only flag. Use "-o ro" instead.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/googlecloudplatform/gcsfuse/fs"
	"github.com/googlecloudplatform/gcsfuse/mount"
	"github.com/googlecloudplatform/gcsfuse/timeutil"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fsutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcscaching"
	"golang.org/x/net/context"
)

var fBucketName = flag.String("bucket", "", "Name of GCS bucket to mount.")
var fMountPoint = flag.String("mount_point", "", "File system location.")

var fMountOptions = mount.OptionFlag(
	"o",
	"Additional system-specific mount options. Be careful!")

var fTempDir = flag.String(
	"temp_dir", "",
	"The temporary directory in which to store local copies of GCS objects. "+
		"If empty, the system default (probably /tmp) will be used.")

var fTempDirLimit = flag.Int64(
	"temp_dir_bytes", 1<<31,
	"A desired limit on the number of bytes used in --temp_dir. May be exceeded "+
		"for dirty files that have not been flushed or closed.")

var fGCSChunkSize = flag.Uint64(
	"gcs_chunk_size", 1<<24,
	"If set to a non-zero value N, split up GCS objects into multiple chunks of "+
		"size at most N when reading, and do not read or cache unnecessary chunks.")

var fImplicitDirs = flag.Bool(
	"implicit_dirs",
	false,
	"Implicitly define directories based on their content. See "+
		"docs/semantics.md.")

var fSupportNlink = flag.Bool(
	"support_nlink",
	false,
	"Return meaningful values for nlink from fstat(2). See docs/semantics.md.")

var fStatCacheTTL = flag.Duration(
	"stat_cache_ttl",
	time.Minute,
	"How long to cache StatObject results from GCS.")

var fTypeCacheTTL = flag.Duration(
	"type_cache_ttl",
	time.Minute,
	"How long to cache name -> file/dir type mappings in directory inodes.")

func getBucketName() string {
	s := *fBucketName
	if s == "" {
		fmt.Println("You must set --bucket.")
		os.Exit(1)
	}

	return s
}

func registerSIGINTHandler(mountPoint string) {
	// Register for SIGINT.
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)

	// Start a goroutine that will unmount when the signal is received.
	go func() {
		for {
			<-signalChan
			log.Println("Received SIGINT, attempting to unmount...")

			err := fuse.Unmount(mountPoint)
			if err != nil {
				log.Printf("Failed to unmount in response to SIGINT: %v", err)
			} else {
				log.Printf("Successfully unmounted in response to SIGINT.")
				return
			}
		}
	}()
}

func getBucket() (b gcs.Bucket) {
	// Set up a GCS connection.
	log.Println("Initializing GCS connection.")
	conn, err := getConn()
	if err != nil {
		log.Fatal("Couldn't get GCS connection: ", err)
	}

	// Extract the appropriate bucket.
	b = conn.GetBucket(getBucketName())

	// Enable cached StatObject results, if appropriate.
	if *fStatCacheTTL != 0 {
		const cacheCapacity = 4096
		b = gcscaching.NewFastStatBucket(
			*fStatCacheTTL,
			gcscaching.NewStatCache(cacheCapacity),
			timeutil.RealClock(),
			b)
	}

	return
}

func main() {
	// Make logging output better.
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Set up flags.
	flag.Parse()

	// Check --mount_point.
	if *fMountPoint == "" {
		log.Fatalf("You must set --mount_point.")
	}

	mountPoint := *fMountPoint

	// Sanity check: make sure the temporary directory exists and is writable
	// currently. This gives a better user experience than harder to debug EIO
	// errors when reading files in the future.
	if *fTempDir != "" {
		f, err := fsutil.AnonymousFile(*fTempDir)
		f.Close()

		if err != nil {
			log.Fatalf(
				"Error writing to temporary directory (%q); are you sure it exists "+
					"with the correct permissions?",
				err.Error())
		}
	}

	// Create a file system server.
	bucket := getBucket()
	serverCfg := &fs.ServerConfig{
		Clock:               timeutil.RealClock(),
		Bucket:              bucket,
		TempDir:             *fTempDir,
		TempDirLimit:        *fTempDirLimit,
		GCSChunkSize:        *fGCSChunkSize,
		ImplicitDirectories: *fImplicitDirs,
		SupportNlink:        *fSupportNlink,
		DirTypeCacheTTL:     *fTypeCacheTTL,
	}

	server, err := fs.NewServer(serverCfg)
	if err != nil {
		log.Fatal("fs.NewServer:", err)
	}

	// Mount the file system.
	mountCfg := &fuse.MountConfig{
		FSName:  bucket.Name(),
		Options: fMountOptions,
	}

	mountedFS, err := fuse.Mount(mountPoint, server, mountCfg)
	if err != nil {
		log.Fatal("Mount:", err)
	}

	log.Println("File system has been successfully mounted.")

	// Let the user unmount with Ctrl-C (SIGINT).
	registerSIGINTHandler(mountedFS.Dir())

	// Wait for it to be unmounted.
	if err := mountedFS.Join(context.Background()); err != nil {
		log.Fatal("MountedFileSystem.Join:", err)
	}

	log.Println("Successfully exiting.")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"

	"github.com/urfave/negroni"
	"github.com/xoriath/alexandria-go/fetch"
	"github.com/xoriath/alexandria-go/index"

	"net/http"
)

var (
	mainIndex = flag.String("main-index", "http://content.alexandria.atmel.com/meta/index.xml",
		"Provide the name of the main index to use. Leave undefined to download from the content server.")
	fetchKeywords = flag.Bool("fetch-keywords", true,
		"Fetch the keyword indexes.")
	preparedKeywordStore = flag.String("prepared-keyword-store", "",
		"Point initally to a already populated keyword store.")
	keywordStorePrefix = flag.String("keyword-store-prefix", "keywords",
		"Prefix of the keyword store data base.")
	keywordStoreExtension = flag.String("keyword-store-extension", "db",
		"Extension of the keyword store data base.")
	webhelpRedirectPattern = flag.String("webhelp-redirect-pattern", "/content/webhelp/{{.Book}}/index.html?{{.Topic}}",
		"Redirect pattern for webhelp lookups. 2 replacement parameters, first is Book GUID and second is Topic GUID.")
	contentRedirectPattern = flag.String("content-redirect-pattern", "/content/{{.ResourceType}}/{{.Id}}-{{.Language}}-{{.Version}}",
		"Redirect pattern for content lookups.")
	f1FragmentPattern = flag.String("f1-fragment-pattern", "http://content.alexandria.atmel.com/meta/f1/{{.Id}}-{{.Language}}-{{.Version}}.xml",
		"Pattern for the F1 fragments")
	contentServerBase = flag.String("content-server-base", "http://s3.amazonaws.com/atmel-studio-doc/",
		"The content server")
)

func main() {

	flag.Parse()
	books, err := fetch.MainIndex(*mainIndex)
	if err != nil {
		panic(err)
	} else {
		log.Println("Read main index,", len(books.Books), "books are available")
	}

	var store *index.Store
	if *preparedKeywordStore != "" {
		store = index.OldStore(*preparedKeywordStore, *f1FragmentPattern)
	} else {
		store = index.NewStore(*keywordStorePrefix, *keywordStoreExtension, *f1FragmentPattern)
	}

	if *fetchKeywords {
		go func() {
			store = fetch.F1Indexes(books, store)

			keywordStatistics := store.GetStatistics()
			log.Println("Index store using ", store.FileName, "with", keywordStatistics.KeywordCount, "keywords covering", keywordStatistics.NumberOfFiles, "files")
		}()
	}

	mux := createRoutes(books, store, *mainIndex, *webhelpRedirectPattern)
	n := negroni.Classic()
	logger := negroni.NewLogger()

	n.Use(logger)
	n.UseHandler(mux)

	serverAddress := ":3001"
	log.Println("Server running, listening on", serverAddress)
	log.Fatal(http.ListenAndServe(serverAddress, n))
}
<commit_msg>Lets wait for the indexes to be fetched before we boot the server<commit_after>package main

import (
	"flag"
	"log"

	"github.com/urfave/negroni"
	"github.com/xoriath/alexandria-go/fetch"
	"github.com/xoriath/alexandria-go/index"

	"net/http"
)

var (
	mainIndex = flag.String("main-index", "http://content.alexandria.atmel.com/meta/index.xml",
		"Provide the name of the main index to use. Leave undefined to download from the content server.")
	fetchKeywords = flag.Bool("fetch-keywords", true,
		"Fetch the keyword indexes.")
	preparedKeywordStore = flag.String("prepared-keyword-store", "",
		"Point initally to a already populated keyword store.")
	keywordStorePrefix = flag.String("keyword-store-prefix", "keywords",
		"Prefix of the keyword store data base.")
	keywordStoreExtension = flag.String("keyword-store-extension", "db",
		"Extension of the keyword store data base.")
	webhelpRedirectPattern = flag.String("webhelp-redirect-pattern", "/content/webhelp/{{.Book}}/index.html?{{.Topic}}",
		"Redirect pattern for webhelp lookups. 2 replacement parameters, first is Book GUID and second is Topic GUID.")
	contentRedirectPattern = flag.String("content-redirect-pattern", "/content/{{.ResourceType}}/{{.Id}}-{{.Language}}-{{.Version}}",
		"Redirect pattern for content lookups.")
	f1FragmentPattern = flag.String("f1-fragment-pattern", "http://content.alexandria.atmel.com/meta/f1/{{.Id}}-{{.Language}}-{{.Version}}.xml",
		"Pattern for the F1 fragments")
	contentServerBase = flag.String("content-server-base", "http://s3.amazonaws.com/atmel-studio-doc/",
		"The content server")
)

func main() {

	flag.Parse()
	books, err := fetch.MainIndex(*mainIndex)
	if err != nil {
		panic(err)
	} else {
		log.Println("Read main index,", len(books.Books), "books are available")
	}

	var store *index.Store
	if *preparedKeywordStore != "" {
		store = index.OldStore(*preparedKeywordStore, *f1FragmentPattern)
	} else {
		store = index.NewStore(*keywordStorePrefix, *keywordStoreExtension, *f1FragmentPattern)
	}

	if *fetchKeywords {
		func() {
			store = fetch.F1Indexes(books, store)

			keywordStatistics := store.GetStatistics()
			log.Println("Index store using ", store.FileName, "with", keywordStatistics.KeywordCount, "keywords covering", keywordStatistics.NumberOfFiles, "files")
		}()
	}

	mux := createRoutes(books, store, *mainIndex, *webhelpRedirectPattern)
	n := negroni.Classic()
	logger := negroni.NewLogger()

	n.Use(logger)
	n.UseHandler(mux)

	serverAddress := ":3001"
	log.Println("Server running, listening on", serverAddress)
	log.Fatal(http.ListenAndServe(serverAddress, n))
}
<|endoftext|>


<commit_before>package main

import (
	// "database/sql"
	"encoding/json"
	"fmt"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"net/http"
	"os"

	"github.com/gorilla/mux"
	"github.com/nu7hatch/gouuid"
)

var db *sqlx.DB

type User struct {
	Id       string `json:"id" db:"id"`
	Email    string `json:"email" db:"email"`
	Password string `json:"password" db:"password"`
}

type Notification struct {
	Id      string `json:"id" db:"id"`
	Message string `json:"message" db:"message"`
	UserId  string `json:"user_id" db:"user_id"`
}

type Token struct {
	Id     string `json:"id" db:"id"`
	Token  string `json:"token" db:"token"`
	UserId string `json:"user_id" db:"user_id"`
	Status bool   `json:"status" db:"status"`
}

func DBConnection() *sqlx.DB {
	dbURL := os.Getenv("DATABASE_URL")

	if dbURL == "" {
		log.Fatal("$DATABASE_URL environment variable must be set")
	} else {
		fmt.Println("Connected to database: " + dbURL)
	}

	db, err := sqlx.Connect("postgres", dbURL)

	if err != nil {
		log.Fatal(err)
	}

	return db
}

func main() {
	port := os.Getenv("PORT")

	if port == "" {
		log.Fatal("$PORT environment variable must be set")
	} else {
		fmt.Println("Running on port: " + port)
	}

	db = DBConnection()
	router := mux.NewRouter().StrictSlash(true)

	router.HandleFunc("/", Index)
	router.HandleFunc("/user/{id:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/notifications", ListUserNotifications).Methods("GET")
	router.HandleFunc("/notifications", CreateNotification).Methods("POST")
	log.Fatal(http.ListenAndServe(":"+port, router))
}

func Index(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "OK")
}

func ListUserNotifications(w http.ResponseWriter, r *http.Request) {
	// Get all notifications for user
	vars := mux.Vars(r)
	id, _ := vars["id"]
	query := `SELECT id, message, user_id FROM notification WHERE user_id = $1`

	fmt.Printf("User ID %v", id)

	var notifications []*Notification

	db.Select(&notifications, query, id)
	fmt.Printf("Notifications: %v", notifications)
	w.Header().Set("Content-Type", "application/json")

	jsonString, _ := json.Marshal(notifications)

	w.Write(jsonString)
}

func CreateNotification(w http.ResponseWriter, r *http.Request) {
	// TODO
	decoder := json.NewDecoder(r.Body)

	var notification Notification

	err := decoder.Decode(&notification)
	if err != nil {
		log.Fatal(err)
	}

	u, err := uuid.NewV4()
	if err != nil {
		log.Fatal(err)
	}

	notification.Id = u.String()
	query := "INSERT INTO notification (id, message, user_id) VALUES (:id, :message, :user_id)"
	result, err := db.NamedExec(query, &notification)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Result: %v", result)
	jsonString, _ := json.Marshal(notification)

	w.Write(jsonString)

}
<commit_msg>add user login<commit_after>package main

import (
	// "database/sql"
	"encoding/json"
	"fmt"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	"log"
	"net/http"
	"os"

	"github.com/gorilla/mux"
	"github.com/nu7hatch/gouuid"
)

var db *sqlx.DB

type User struct {
	Id       string `json:"id" db:"id"`
	Email    string `json:"email" db:"email"`
	Password string `json:"password" db:"password"`
}

type Notification struct {
	Id      string `json:"id" db:"id"`
	Message string `json:"message" db:"message"`
	UserId  string `json:"user_id" db:"user_id"`
}

type Token struct {
	Id     string `json:"id" db:"id"`
	Token  string `json:"token" db:"token"`
	UserId string `json:"user_id" db:"user_id"`
	Status bool   `json:"status" db:"status"`
}

func DBConnection() *sqlx.DB {
	dbURL := os.Getenv("DATABASE_URL")

	if dbURL == "" {
		log.Fatal("$DATABASE_URL environment variable must be set")
	} else {
		fmt.Println("Connected to database: " + dbURL)
	}

	db, err := sqlx.Connect("postgres", dbURL)

	if err != nil {
		log.Fatal(err)
	}

	return db
}

func main() {
	port := os.Getenv("PORT")

	if port == "" {
		log.Fatal("$PORT environment variable must be set")
	} else {
		fmt.Println("Running on port: " + port)
	}

	db = DBConnection()
	router := mux.NewRouter().StrictSlash(true)

	router.HandleFunc("/", Index)
	router.HandleFunc("/user/{id:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/notifications", ListUserNotifications).Methods("GET")
	router.HandleFunc("/notifications", CreateNotification).Methods("POST")
	router.HandleFunc("/login", UserLogin).Methods("POST")
	log.Fatal(http.ListenAndServe(":"+port, router))
}

/// UserLogin will register a user with the service so that they can receive notifications
func UserLogin(w http.ResponseWriter, r *http.Request) {
	decoder := json.NewDecoder(r.Body)

	var user User

	err := decoder.Decode(&user)

	if err != nil {
		log.Fatal(err)
	}

	id, err := uuid.NewV4()
	if err != nil {
		log.Fatal(err)
	}

	user.Id = id.String()
	query := "INSERT INTO \"user\" (id, email, password) VALUES (:id, :email, :password)"
	_, err = db.NamedExec(query, &user)
	if err != nil {
		log.Fatal(err)
	}

	jsonString, _ := json.Marshal(user)

	w.Write(jsonString)
}

func Index(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "OK")
}

func ListUserNotifications(w http.ResponseWriter, r *http.Request) {
	// Get all notifications for user
	vars := mux.Vars(r)
	id, _ := vars["id"]
	query := `SELECT id, message, user_id FROM notification WHERE user_id = $1`

	fmt.Printf("User ID %v", id)

	var notifications []*Notification

	db.Select(&notifications, query, id)
	fmt.Printf("Notifications: %v", notifications)
	w.Header().Set("Content-Type", "application/json")

	jsonString, _ := json.Marshal(notifications)

	w.Write(jsonString)
}

func CreateNotification(w http.ResponseWriter, r *http.Request) {
	// TODO
	decoder := json.NewDecoder(r.Body)

	var notification Notification

	err := decoder.Decode(&notification)
	if err != nil {
		log.Fatal(err)
	}

	u, err := uuid.NewV4()
	if err != nil {
		log.Fatal(err)
	}

	notification.Id = u.String()
	query := "INSERT INTO notification (id, message, user_id) VALUES (:id, :message, :user_id)"
	_, err = db.NamedExec(query, &notification)
	if err != nil {
		log.Fatal(err)
	}

	jsonString, _ := json.Marshal(notification)

	w.Write(jsonString)

}
<|endoftext|>