<commit_before>/*
 * Copyright (c) 2014 Mark Samman <https://github.com/marksamman/gotorrent>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

package main

import (
	"bytes"
	"fmt"
	"log"
	"math/rand"
	"os"
	"strconv"
	"time"
)

type Client struct {
	PeerId []byte
}

var client Client

func main() {
	if len(os.Args) <= 1 {
		fmt.Println("usage: gotorrent file")
		return
	}

	// Seed rand
	rand.Seed(time.Now().UnixNano())

	// Generate a 20 byte peerId
	var peerId bytes.Buffer
	for i := 0; i < 20; i++ {
		peerId.WriteByte(byte(rand.Intn(255)))
	}
	client.PeerId = peerId.Bytes()

	// Open torrent file
	torrent := Torrent{}
	err := torrent.open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	// Start a TCP listener on a port in the range 6881-6889
	port := 6881 + rand.Intn(8)
	go TCPListener(port)

	fmt.Println("Name:", torrent.getName())
	fmt.Println("Announce URL:", torrent.getAnnounceURL())
	fmt.Println("Comment:", torrent.getComment())
	fmt.Printf("Total size: %.2f MB\n", float64(torrent.getTotalSize())/1024/1024)

	params := make(map[string]string)
	params["event"] = "started"
	params["port"] = strconv.Itoa(port)

	httpResponse, err := torrent.sendTrackerRequest(params)
	if err != nil {
		log.Fatal(err)
	}
	defer httpResponse.Body.Close()

	if httpResponse.StatusCode != 200 {
		log.Fatalf("bad response from tracker: %s", httpResponse.Status)
	}

	resp := BencodeDecode(httpResponse.Body)
	torrent.parsePeers(resp["peers"])

	fmt.Println("Connecting to peers...")
	for _, peer := range torrent.Peers {
		go func(peer Peer) {
			peer.connect()
		}(peer)
	}
	time.Sleep(time.Minute)
}
<commit_msg>Torrent client identifier<commit_after>/*
 * Copyright (c) 2014 Mark Samman <https://github.com/marksamman/gotorrent>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

package main

import (
	"bytes"
	"fmt"
	"log"
	"math/rand"
	"os"
	"strconv"
	"time"
)

type Client struct {
	PeerId []byte
}

var client Client

func main() {
	if len(os.Args) <= 1 {
		fmt.Println("usage: gotorrent file")
		return
	}

	// Seed rand
	rand.Seed(time.Now().UnixNano())

	// Generate a 20 byte peerId
	var peerId bytes.Buffer
	peerId.WriteString("-GO10000")
	for i := 0; i < 12; i++ {
		peerId.WriteByte(byte(rand.Intn(255)))
	}
	client.PeerId = peerId.Bytes()

	// Open torrent file
	torrent := Torrent{}
	err := torrent.open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}

	// Start a TCP listener on a port in the range 6881-6889
	port := 6881 + rand.Intn(8)
	go TCPListener(port)

	fmt.Println("Name:", torrent.getName())
	fmt.Println("Announce URL:", torrent.getAnnounceURL())
	fmt.Println("Comment:", torrent.getComment())
	fmt.Printf("Total size: %.2f MB\n", float64(torrent.getTotalSize())/1024/1024)

	params := make(map[string]string)
	params["event"] = "started"
	params["port"] = strconv.Itoa(port)

	httpResponse, err := torrent.sendTrackerRequest(params)
	if err != nil {
		log.Fatal(err)
	}
	defer httpResponse.Body.Close()

	if httpResponse.StatusCode != 200 {
		log.Fatalf("bad response from tracker: %s", httpResponse.Status)
	}

	resp := BencodeDecode(httpResponse.Body)
	torrent.parsePeers(resp["peers"])

	fmt.Println("Connecting to peers...")
	for _, peer := range torrent.Peers {
		go func(peer Peer) {
			peer.connect()
		}(peer)
	}
	time.Sleep(time.Minute)
}
<|endoftext|>


<commit_before>package main

import "fmt"

/*
	c	matches any literal character c
	.	matches any single character
	^	matches the beginning of the input string
	$	matches the end of the input string
	*	matches zero or more occurrences of the previous character
*/

/* match: search for regexp anywhere in text */
func match(regexp, text []byte) bool {
	if regexp[0] == '^' {
		return matchhere(regexp[1:], text)
	}
	for i := range text {
		if matchhere(regexp, text[i:]) {
			return true
		}
	}
	return false
}

/* matchhere: search for regexp at beginning of text */
func matchhere(regexp, text []byte) bool {
	if len(regexp) == 0 {
		return true
	}
	if len(regexp) > 1 && regexp[1] == '*' {
		return matchstar(regexp[0], regexp[2:], text)
	}
	if regexp[0] == '$' && len(regexp) == 1 {
		return len(text) == 0
	}
	if len(text) != 0 && (regexp[0] == '.' || regexp[0] == text[0]) {
		return matchhere(regexp[1:], text[1:])
	}
	return false
}

/* matchstar: search for c*regexp at beginning of text */
func matchstar(c byte, regexp, text []byte) bool {
	i := 0
	for {
		if matchhere(regexp, text[i:]) {
			return true
		}
		if i == len(text) || (text[i] != c && c != '.') {
			break
		}
		i++
	}
	return false
}

// assumes input is correct
func main() {
	regexp := "da*b"
	text := "bcccb"
	fmt.Println(match([]byte(regexp), []byte(text)))
}
<commit_msg>matcher gonna branch!<commit_after>package main

import "fmt"

/*
	c	matches any literal character c
	.	matches any single character
	^	matches the beginning of the input string
	$	matches the end of the input string
	*	matches zero or more occurrences of the previous character
*/

/* match: search for regexp anywhere in text */
func match(regexp, text []byte) bool {
	if regexp[0] == '^' {
		return matchhere(regexp[1:], text)
	}
	for i := range text {
		if matchhere(regexp, text[i:]) {
			return true
		}
	}
	return false
}

/* matchhere: search for regexp at beginning of text */
func matchhere(regexp, text []byte) bool {
	if len(regexp) == 0 {
		return true
	}
	if len(regexp) > 1 && regexp[1] == '*' {
		return matchstar(regexp[0], regexp[2:], text)
	}
	if regexp[0] == '$' && len(regexp) == 1 {
		return len(text) == 0
	}
	if len(text) != 0 && (regexp[0] == '.' || regexp[0] == text[0]) {
		return matchhere(regexp[1:], text[1:])
	}
	return false
}

/* matchstar: search for c*regexp at beginning of text */
func matchstar(c byte, regexp, text []byte) bool {
	i := 0
	for {
		if matchhere(regexp, text[i:]) {
			return true
		}
		if i == len(text) || (text[i] != c && c != '.') {
			break
		}
		i++
	}
	return false
}

// assumes input is correct
func main() {
	regexp := "abc$"
	text := "abcb"
	fmt.Println(match([]byte(regexp), []byte(text)))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"path"
	"time"

	"github.com/getlantern/systray"
	quic "github.com/lucas-clemente/quic-go"
	"github.com/nange/easypool"
	"github.com/nange/easyss/utils"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

func init() {
	log.SetFormatter(&log.JSONFormatter{TimestampFormat: "2006-01-02 15:04:05.000"})
	log.SetLevel(log.InfoLevel)
}

func PrintVersion() {
	const version = "RC2"
	fmt.Println("easyss version", version)
}

type Easyss struct {
	config *Config
	quic   struct {
		localSess quic.Session
		sessChan  chan sessOpts
	}
	pac struct {
		ch   chan PACStatus
		url  string
		gurl string
	}
	tcpPool easypool.Pool
}

func New(config *Config) (*Easyss, error) {
	ss := &Easyss{config: config}
	if !config.ServerModel {
		ss.pac.ch = make(chan PACStatus)
		ss.pac.url = fmt.Sprintf("http://localhost:%d%s", ss.config.LocalPort+1, pacpath)
		ss.pac.gurl = fmt.Sprintf("http://localhost:%d%s?global=true", ss.config.LocalPort+1, pacpath)
	}
	if config.EnableQuic {
		ss.quic.sessChan = make(chan sessOpts, 10)
	}

	return ss, nil
}

func (ss *Easyss) InitTcpPool() error {
	factory := func() (net.Conn, error) {
		return net.Dial("tcp", fmt.Sprintf("%s:%d", ss.config.Server, ss.config.ServerPort))
	}
	pconfig := &easypool.PoolConfig{
		InitialCap:  10,
		MaxCap:      50,
		MaxIdle:     10,
		Idletime:    3 * time.Minute,
		MaxLifetime: 15 * time.Minute,
		Factory:     factory,
	}
	tcppool, err := easypool.NewHeapPool(pconfig)
	if err != nil {
		return err
	}
	ss.tcpPool = tcppool
	return nil
}

func main() {
	var configFile string
	var printVer, debug, godaemon bool
	var cmdConfig Config

	flag.BoolVar(&printVer, "version", false, "print version")
	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.StringVar(&cmdConfig.Server, "s", "", "server address")
	flag.StringVar(&cmdConfig.Password, "k", "", "password")
	flag.IntVar(&cmdConfig.ServerPort, "p", 0, "server port")
	flag.IntVar(&cmdConfig.Timeout, "t", 300, "timeout in seconds")
	flag.IntVar(&cmdConfig.LocalPort, "l", 0, "local socks5 proxy port")
	flag.StringVar(&cmdConfig.Method, "m", "", "encryption method, default: aes-256-gcm")
	flag.BoolVar(&cmdConfig.EnableQuic, "quic", false, "enable quic if set this value to be true")
	flag.BoolVar(&debug, "d", false, "print debug message")
	flag.BoolVar(&cmdConfig.ServerModel, "server", false, "server model")
	flag.BoolVar(&godaemon, "daemon", true, "run app as a non-daemon with -daemon=false")

	flag.Parse()

	if printVer {
		PrintVersion()
		os.Exit(0)
	}
	daemon(godaemon)

	if debug {
		log.SetLevel(log.DebugLevel)
	}

	exists, err := utils.FileExists(configFile)
	if !exists || err != nil {
		log.Debugf("config file err:%v", err)

		binDir := path.Dir(os.Args[0])
		configFile = path.Join(binDir, "config.json")

		log.Debugf("config file not found, try config file %s", configFile)
	}

	config, err := ParseConfig(configFile)
	if err != nil {
		config = &cmdConfig
		if !os.IsNotExist(errors.Cause(err)) {
			log.Errorf("error reading %s: %+v", configFile, err)
			os.Exit(1)
		}
	} else {
		UpdateConfig(config, &cmdConfig)
	}

	if config.Method == "" {
		config.Method = "aes-256-gcm"
	}

	ss, err := New(config)
	if err != nil {
		log.Fatalf("init Easyss err:%+v", err)
	}
	if config.ServerModel {
		if config.ServerPort == 0 || config.Password == "" {
			log.Fatalln("server port and password should not empty")
		}

		ss.Remote()
	} else {
		if config.Password == "" || config.Server == "" || config.ServerPort == 0 {
			log.Fatalln("server address, server port and password should not empty")
		}

		systray.Run(ss.trayReady, ss.trayExit) // system tray management
	}

}
<commit_msg>if not server model, we don't startup as daemon<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"os"
	"path"
	"time"

	"github.com/getlantern/systray"
	quic "github.com/lucas-clemente/quic-go"
	"github.com/nange/easypool"
	"github.com/nange/easyss/utils"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

func init() {
	log.SetFormatter(&log.JSONFormatter{TimestampFormat: "2006-01-02 15:04:05.000"})
	log.SetLevel(log.InfoLevel)
}

func PrintVersion() {
	const version = "RC2"
	fmt.Println("easyss version", version)
}

type Easyss struct {
	config *Config
	quic   struct {
		localSess quic.Session
		sessChan  chan sessOpts
	}
	pac struct {
		ch   chan PACStatus
		url  string
		gurl string
	}
	tcpPool easypool.Pool
}

func New(config *Config) (*Easyss, error) {
	ss := &Easyss{config: config}
	if !config.ServerModel {
		ss.pac.ch = make(chan PACStatus)
		ss.pac.url = fmt.Sprintf("http://localhost:%d%s", ss.config.LocalPort+1, pacpath)
		ss.pac.gurl = fmt.Sprintf("http://localhost:%d%s?global=true", ss.config.LocalPort+1, pacpath)
	}
	if config.EnableQuic {
		ss.quic.sessChan = make(chan sessOpts, 10)
	}

	return ss, nil
}

func (ss *Easyss) InitTcpPool() error {
	factory := func() (net.Conn, error) {
		return net.Dial("tcp", fmt.Sprintf("%s:%d", ss.config.Server, ss.config.ServerPort))
	}
	pconfig := &easypool.PoolConfig{
		InitialCap:  10,
		MaxCap:      50,
		MaxIdle:     10,
		Idletime:    3 * time.Minute,
		MaxLifetime: 15 * time.Minute,
		Factory:     factory,
	}
	tcppool, err := easypool.NewHeapPool(pconfig)
	if err != nil {
		return err
	}
	ss.tcpPool = tcppool
	return nil
}

func main() {
	var configFile string
	var printVer, debug, godaemon bool
	var cmdConfig Config

	flag.BoolVar(&printVer, "version", false, "print version")
	flag.StringVar(&configFile, "c", "config.json", "specify config file")
	flag.StringVar(&cmdConfig.Server, "s", "", "server address")
	flag.StringVar(&cmdConfig.Password, "k", "", "password")
	flag.IntVar(&cmdConfig.ServerPort, "p", 0, "server port")
	flag.IntVar(&cmdConfig.Timeout, "t", 300, "timeout in seconds")
	flag.IntVar(&cmdConfig.LocalPort, "l", 0, "local socks5 proxy port")
	flag.StringVar(&cmdConfig.Method, "m", "", "encryption method, default: aes-256-gcm")
	flag.BoolVar(&cmdConfig.EnableQuic, "quic", false, "enable quic if set this value to be true")
	flag.BoolVar(&debug, "d", false, "print debug message")
	flag.BoolVar(&cmdConfig.ServerModel, "server", false, "server model")
	flag.BoolVar(&godaemon, "daemon", true, "run app as a non-daemon with -daemon=false")

	flag.Parse()

	if printVer {
		PrintVersion()
		os.Exit(0)
	}
	// if not server model, we don't startup as daemon
	if !cmdConfig.ServerModel {
		daemon(godaemon)
	}

	if debug {
		log.SetLevel(log.DebugLevel)
	}

	exists, err := utils.FileExists(configFile)
	if !exists || err != nil {
		log.Debugf("config file err:%v", err)

		binDir := path.Dir(os.Args[0])
		configFile = path.Join(binDir, "config.json")

		log.Debugf("config file not found, try config file %s", configFile)
	}

	config, err := ParseConfig(configFile)
	if err != nil {
		config = &cmdConfig
		if !os.IsNotExist(errors.Cause(err)) {
			log.Errorf("error reading %s: %+v", configFile, err)
			os.Exit(1)
		}
	} else {
		UpdateConfig(config, &cmdConfig)
	}

	if config.Method == "" {
		config.Method = "aes-256-gcm"
	}

	ss, err := New(config)
	if err != nil {
		log.Fatalf("init Easyss err:%+v", err)
	}
	if config.ServerModel {
		if config.ServerPort == 0 || config.Password == "" {
			log.Fatalln("server port and password should not empty")
		}

		ss.Remote()
	} else {
		if config.Password == "" || config.Server == "" || config.ServerPort == 0 {
			log.Fatalln("server address, server port and password should not empty")
		}

		systray.Run(ss.trayReady, ss.trayExit) // system tray management
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/alext/tablecloth"
	"gopkg.in/mgo.v2"
)

var (
	eventPort  = getenvDefault("PORT", "3097")
	mgoNodes   = getenvDefault("EVENT_STORE_MONGO_NODES", "localhost")
	mgoSession *mgo.Session
)

func ConnectToMongo(hostname string) (*mgo.Session, error) {
	session, err := mgo.DialWithTimeout(hostname, 200*time.Millisecond)
	if err != nil {
		return nil, err
	}

	// Queries return "read tcp 127.0.0.1:27017: i/o timeout" unless
	// the session socket timeout is increased.
	session.SetSocketTimeout(1 * time.Second)

	session.SetMode(mgo.Strong, true)

	return session, nil
}

func getenvDefault(key string, defaultVal string) string {
	val := os.Getenv(key)
	if val == "" {
		val = defaultVal
	}

	return val
}

func main() {
	if wd := os.Getenv("GOVUK_APP_ROOT"); wd != "" {
		tablecloth.WorkingDir = wd
	}

	mgoSession, err := ConnectToMongo(mgoNodes)

	publicMux := http.NewServeMux()
	publicMux.HandleFunc("/e", ReportHandler(mgoSession))
	publicMux.HandleFunc("/healthcheck", HealthcheckHandler(mgoSession))

	log.Println("event-store: listening for events on " + eventPort)

	err = tablecloth.ListenAndServe(fmt.Sprintf(":%v", eventPort), publicMux, "reports")

	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Abort on errors connecting to mongo.<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/alext/tablecloth"
	"gopkg.in/mgo.v2"
)

var (
	eventPort  = getenvDefault("PORT", "3097")
	mgoNodes   = getenvDefault("EVENT_STORE_MONGO_NODES", "localhost")
	mgoSession *mgo.Session
)

func ConnectToMongo(hostname string) (*mgo.Session, error) {
	session, err := mgo.DialWithTimeout(hostname, 200*time.Millisecond)
	if err != nil {
		return nil, err
	}

	// Queries return "read tcp 127.0.0.1:27017: i/o timeout" unless
	// the session socket timeout is increased.
	session.SetSocketTimeout(1 * time.Second)

	session.SetMode(mgo.Strong, true)

	return session, nil
}

func getenvDefault(key string, defaultVal string) string {
	val := os.Getenv(key)
	if val == "" {
		val = defaultVal
	}

	return val
}

func main() {
	if wd := os.Getenv("GOVUK_APP_ROOT"); wd != "" {
		tablecloth.WorkingDir = wd
	}

	mgoSession, err := ConnectToMongo(mgoNodes)
	if err != nil {
		log.Fatal("Error connectiong to mongo : ", err)
	}

	publicMux := http.NewServeMux()
	publicMux.HandleFunc("/e", ReportHandler(mgoSession))
	publicMux.HandleFunc("/healthcheck", HealthcheckHandler(mgoSession))

	log.Println("event-store: listening for events on " + eventPort)

	err = tablecloth.ListenAndServe(fmt.Sprintf(":%v", eventPort), publicMux, "reports")

	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"

	"github.com/gorilla/mux"
	"github.com/voxelbrain/goptions"
)

var (
	options = struct {
		Port int           `goptions:"-p, --port, description='Port to bind webserver to'"`
		Help goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Port: 5000,
	}
)

func main() {
	goptions.ParseAndFail(&options)

	r := mux.NewRouter()
	r.HandleFunc("/{server}/{id}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		mh, err := MatchHistory(path.Join(vars["server"], vars["id"]))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		json.NewEncoder(w).Encode(mh)
	})

	log.Printf("Starting webserver...")
	err := http.ListenAndServe(fmt.Sprintf("localhost:%d", options.Port), r)
	if err != nil {
		log.Fatalf("Could not start webserver: %s", err)
	}
}
<commit_msg>Bind to global port, dummy<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"path"

	"github.com/gorilla/mux"
	"github.com/voxelbrain/goptions"
)

var (
	options = struct {
		Port int           `goptions:"-p, --port, description='Port to bind webserver to'"`
		Help goptions.Help `goptions:"-h, --help, description='Show this help'"`
	}{
		Port: 5000,
	}
)

func main() {
	goptions.ParseAndFail(&options)

	r := mux.NewRouter()
	r.HandleFunc("/{server}/{id}", func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		mh, err := MatchHistory(path.Join(vars["server"], vars["id"]))
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		json.NewEncoder(w).Encode(mh)
	})

	addr := fmt.Sprintf("0.0.0.0:%d", options.Port)
	log.Printf("Starting webserver on %s...", addr)
	err := http.ListenAndServe(addr, r)
	if err != nil {
		log.Fatalf("Could not start webserver: %s", err)
	}
}
<|endoftext|>


<commit_before>package main

// about using johnniedoe/contrib/gzip:
// johnniedoe's fork fixes a critical issue for which .String resulted in
// an ERR_DECODING_FAILED. This is an actual pull request on the contrib
// repo, but apparently, gin is dead.

import (
	"encoding/gob"
	"fmt"

	"git.zxq.co/ripple/rippleapi/app"
	"git.zxq.co/ripple/schiavolib"
	"git.zxq.co/x/rs"
	"github.com/fatih/structs"
	"github.com/getsentry/raven-go"
	"github.com/gin-gonic/contrib/sessions"
	"github.com/gin-gonic/gin"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	"github.com/johnniedoe/contrib/gzip"
	"github.com/thehowl/conf"
	"github.com/thehowl/qsql"
	"gopkg.in/mailgun/mailgun-go.v1"
)

// version is the version of hanayo
const version = "0.6b"

var (
	config struct {
		ListenTo string `description:"ip:port from which to take requests."`
		Unix     bool   `description:"Whether ListenTo is an unix socket."`

		DSN string `description:"MySQL server DSN"`

		CookieSecret string

		RedisEnable         bool
		RedisMaxConnections int
		RedisNetwork        string
		RedisAddress        string
		RedisPassword       string

		AvatarURL     string
		BaseURL       string
		DiscordServer string

		API       string
		BanchoAPI string
		APISecret string

		IP_API string

		Offline          bool   `description:"If this is true, files will be served from the local server instead of the CDN."`
		MainRippleFolder string `description:"Folder where all the non-go projects are contained, such as old-frontend, lets, ci-system."`
		AvatarsFolder    string `description:"location folder of avatars"`

		MailgunDomain        string
		MailgunPrivateAPIKey string
		MailgunPublicAPIKey  string
		MailgunFrom          string

		RecaptchaSite    string
		RecaptchaPrivate string

		DiscordOAuthID     string
		DiscordOAuthSecret string
		DonorBotURL        string
		DonorBotSecret     string

		SentryDSN string
	}
	configMap map[string]interface{}
	db        *sqlx.DB
	qb        *qsql.DB
	mg        mailgun.Mailgun
)

func main() {
	fmt.Println("hanayo v" + version)

	err := conf.Load(&config, "hanayo.conf")
	switch err {
	case nil:
		// carry on
	case conf.ErrNoFile:
		conf.Export(config, "hanayo.conf")
		fmt.Println("The configuration file was not found. We created one for you.")
		return
	default:
		panic(err)
	}

	var configDefaults = map[*string]string{
		&config.ListenTo:         ":45221",
		&config.CookieSecret:     rs.String(46),
		&config.AvatarURL:        "https://a.ripple.moe",
		&config.BaseURL:          "https://ripple.moe",
		&config.BanchoAPI:        "https://c.ripple.moe",
		&config.API:              "http://localhost:40001/api/v1/",
		&config.APISecret:        "Potato",
		&config.IP_API:           "https://ip.zxq.co",
		&config.DiscordServer:    "#",
		&config.MainRippleFolder: "/home/ripple/ripple",
		&config.MailgunFrom:      `"Ripple" <noreply@ripple.moe>`,
	}
	for key, value := range configDefaults {
		if *key == "" {
			*key = value
		}
	}

	configMap = structs.Map(config)

	// initialise db
	db, err = sqlx.Open("mysql", config.DSN)
	if err != nil {
		panic(err)
	}
	qb = qsql.New(db.DB)
	if err != nil {
		panic(err)
	}

	// initialise mailgun
	mg = mailgun.NewMailgun(
		config.MailgunDomain,
		config.MailgunPrivateAPIKey,
		config.MailgunPublicAPIKey,
	)

	if gin.Mode() == gin.DebugMode {
		fmt.Println("Development environment detected. Starting fsnotify on template folder...")
		err := reloader()
		if err != nil {
			fmt.Println(err)
		}
	}

	schiavo.Prefix = "hanayo"
	schiavo.Bunker.Send(fmt.Sprintf("STARTUATO, mode: %s", gin.Mode()))

	// even if it's not release, we say that it's release
	// so that gin doesn't spam
	gin.SetMode(gin.ReleaseMode)

	gobRegisters := []interface{}{
		[]message{},
		errorMessage{},
		infoMessage{},
		neutralMessage{},
		warningMessage{},
		successMessage{},
	}
	for _, el := range gobRegisters {
		gob.Register(el)
	}

	fmt.Println("Importing templates...")
	loadTemplates("")

	fmt.Println("Setting up rate limiter...")
	setUpLimiter()

	fmt.Println("Exporting configuration...")

	conf.Export(config, "hanayo.conf")

	httpLoop()
}

func httpLoop() {
	for {
		e := generateEngine()
		fmt.Println("Starting webserver...")
		if !startuato(e) {
			break
		}
	}
}

func generateEngine() *gin.Engine {
	fmt.Println("Starting session system...")
	var store sessions.Store
	if config.RedisMaxConnections != 0 {
		var err error
		store, err = sessions.NewRedisStore(
			config.RedisMaxConnections,
			config.RedisNetwork,
			config.RedisAddress,
			config.RedisPassword,
			[]byte(config.CookieSecret),
		)
		if err != nil {
			fmt.Println(err)
			store = sessions.NewCookieStore([]byte(config.CookieSecret))
		}
	} else {
		store = sessions.NewCookieStore([]byte(config.CookieSecret))
	}

	r := gin.Default()

	// sentry
	if config.SentryDSN != "" {
		ravenClient, err := raven.New(config.SentryDSN)
		if err != nil {
			fmt.Println(err)
		} else {
			r.Use(app.Recovery(ravenClient, false))
		}
	}

	r.Use(
		gzip.Gzip(gzip.DefaultCompression),
		checkRedirect,
		sessions.Sessions("session", store),
		sessionInitializer(),
		rateLimiter(false),
		twoFALock,
	)

	r.Static("/static", "static")
	r.StaticFile("/favicon.ico", "static/favicon.ico")

	r.POST("/login", loginSubmit)
	r.GET("/logout", logout)

	r.GET("/register", register)
	r.POST("/register", registerSubmit)
	r.GET("/register/verify", verifyAccount)
	r.GET("/register/welcome", welcome)

	r.GET("/u/:user", userProfile)

	r.POST("/pwreset", passwordReset)
	r.GET("/pwreset/continue", passwordResetContinue)
	r.POST("/pwreset/continue", passwordResetContinueSubmit)

	r.GET("/2fa_gateway", tfaGateway)
	r.GET("/2fa_gateway/clear", clear2fa)
	r.GET("/2fa_gateway/verify", verify2fa)

	r.GET("/irc/generate", ircGenToken)

	r.GET("/settings/password", changePassword)
	r.POST("/settings/password", changePasswordSubmit)
	r.POST("/settings/userpage/parse", parseBBCode)
	r.POST("/settings/avatar", avatarSubmit)
	r.POST("/settings/2fa/disable", disable2fa)
	r.GET("/settings/discord/finish", discordFinish)

	loadSimplePages(r)

	r.NoRoute(notFound)

	return r
}

const alwaysRespondText = `Ooops! Looks like something went really wrong while trying to process your request.
Perhaps report this to a Ripple developer?
Retrying doing again what you were trying to do might work, too.`
<commit_msg>⬆ v1.0.0 ⬆<commit_after>package main

// about using johnniedoe/contrib/gzip:
// johnniedoe's fork fixes a critical issue for which .String resulted in
// an ERR_DECODING_FAILED. This is an actual pull request on the contrib
// repo, but apparently, gin is dead.

import (
	"encoding/gob"
	"fmt"

	"git.zxq.co/ripple/rippleapi/app"
	"git.zxq.co/ripple/schiavolib"
	"git.zxq.co/x/rs"
	"github.com/fatih/structs"
	"github.com/getsentry/raven-go"
	"github.com/gin-gonic/contrib/sessions"
	"github.com/gin-gonic/gin"
	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	"github.com/johnniedoe/contrib/gzip"
	"github.com/thehowl/conf"
	"github.com/thehowl/qsql"
	"gopkg.in/mailgun/mailgun-go.v1"
)

// version is the version of hanayo
const version = "v1.0.0"

var (
	config struct {
		ListenTo string `description:"ip:port from which to take requests."`
		Unix     bool   `description:"Whether ListenTo is an unix socket."`

		DSN string `description:"MySQL server DSN"`

		CookieSecret string

		RedisEnable         bool
		RedisMaxConnections int
		RedisNetwork        string
		RedisAddress        string
		RedisPassword       string

		AvatarURL     string
		BaseURL       string
		DiscordServer string

		API       string
		BanchoAPI string
		APISecret string

		IP_API string

		Offline          bool   `description:"If this is true, files will be served from the local server instead of the CDN."`
		MainRippleFolder string `description:"Folder where all the non-go projects are contained, such as old-frontend, lets, ci-system."`
		AvatarsFolder    string `description:"location folder of avatars"`

		MailgunDomain        string
		MailgunPrivateAPIKey string
		MailgunPublicAPIKey  string
		MailgunFrom          string

		RecaptchaSite    string
		RecaptchaPrivate string

		DiscordOAuthID     string
		DiscordOAuthSecret string
		DonorBotURL        string
		DonorBotSecret     string

		SentryDSN string
	}
	configMap map[string]interface{}
	db        *sqlx.DB
	qb        *qsql.DB
	mg        mailgun.Mailgun
)

func main() {
	fmt.Println("hanayo v" + version)

	err := conf.Load(&config, "hanayo.conf")
	switch err {
	case nil:
		// carry on
	case conf.ErrNoFile:
		conf.Export(config, "hanayo.conf")
		fmt.Println("The configuration file was not found. We created one for you.")
		return
	default:
		panic(err)
	}

	var configDefaults = map[*string]string{
		&config.ListenTo:         ":45221",
		&config.CookieSecret:     rs.String(46),
		&config.AvatarURL:        "https://a.ripple.moe",
		&config.BaseURL:          "https://ripple.moe",
		&config.BanchoAPI:        "https://c.ripple.moe",
		&config.API:              "http://localhost:40001/api/v1/",
		&config.APISecret:        "Potato",
		&config.IP_API:           "https://ip.zxq.co",
		&config.DiscordServer:    "#",
		&config.MainRippleFolder: "/home/ripple/ripple",
		&config.MailgunFrom:      `"Ripple" <noreply@ripple.moe>`,
	}
	for key, value := range configDefaults {
		if *key == "" {
			*key = value
		}
	}

	configMap = structs.Map(config)

	// initialise db
	db, err = sqlx.Open("mysql", config.DSN)
	if err != nil {
		panic(err)
	}
	qb = qsql.New(db.DB)
	if err != nil {
		panic(err)
	}

	// initialise mailgun
	mg = mailgun.NewMailgun(
		config.MailgunDomain,
		config.MailgunPrivateAPIKey,
		config.MailgunPublicAPIKey,
	)

	if gin.Mode() == gin.DebugMode {
		fmt.Println("Development environment detected. Starting fsnotify on template folder...")
		err := reloader()
		if err != nil {
			fmt.Println(err)
		}
	}

	schiavo.Prefix = "hanayo"
	schiavo.Bunker.Send(fmt.Sprintf("STARTUATO, mode: %s", gin.Mode()))

	// even if it's not release, we say that it's release
	// so that gin doesn't spam
	gin.SetMode(gin.ReleaseMode)

	gobRegisters := []interface{}{
		[]message{},
		errorMessage{},
		infoMessage{},
		neutralMessage{},
		warningMessage{},
		successMessage{},
	}
	for _, el := range gobRegisters {
		gob.Register(el)
	}

	fmt.Println("Importing templates...")
	loadTemplates("")

	fmt.Println("Setting up rate limiter...")
	setUpLimiter()

	fmt.Println("Exporting configuration...")

	conf.Export(config, "hanayo.conf")

	httpLoop()
}

func httpLoop() {
	for {
		e := generateEngine()
		fmt.Println("Starting webserver...")
		if !startuato(e) {
			break
		}
	}
}

func generateEngine() *gin.Engine {
	fmt.Println("Starting session system...")
	var store sessions.Store
	if config.RedisMaxConnections != 0 {
		var err error
		store, err = sessions.NewRedisStore(
			config.RedisMaxConnections,
			config.RedisNetwork,
			config.RedisAddress,
			config.RedisPassword,
			[]byte(config.CookieSecret),
		)
		if err != nil {
			fmt.Println(err)
			store = sessions.NewCookieStore([]byte(config.CookieSecret))
		}
	} else {
		store = sessions.NewCookieStore([]byte(config.CookieSecret))
	}

	r := gin.Default()

	// sentry
	if config.SentryDSN != "" {
		ravenClient, err := raven.New(config.SentryDSN)
		if err != nil {
			fmt.Println(err)
		} else {
			r.Use(app.Recovery(ravenClient, false))
		}
	}

	r.Use(
		gzip.Gzip(gzip.DefaultCompression),
		checkRedirect,
		sessions.Sessions("session", store),
		sessionInitializer(),
		rateLimiter(false),
		twoFALock,
	)

	r.Static("/static", "static")
	r.StaticFile("/favicon.ico", "static/favicon.ico")

	r.POST("/login", loginSubmit)
	r.GET("/logout", logout)

	r.GET("/register", register)
	r.POST("/register", registerSubmit)
	r.GET("/register/verify", verifyAccount)
	r.GET("/register/welcome", welcome)

	r.GET("/u/:user", userProfile)

	r.POST("/pwreset", passwordReset)
	r.GET("/pwreset/continue", passwordResetContinue)
	r.POST("/pwreset/continue", passwordResetContinueSubmit)

	r.GET("/2fa_gateway", tfaGateway)
	r.GET("/2fa_gateway/clear", clear2fa)
	r.GET("/2fa_gateway/verify", verify2fa)

	r.GET("/irc/generate", ircGenToken)

	r.GET("/settings/password", changePassword)
	r.POST("/settings/password", changePasswordSubmit)
	r.POST("/settings/userpage/parse", parseBBCode)
	r.POST("/settings/avatar", avatarSubmit)
	r.POST("/settings/2fa/disable", disable2fa)
	r.GET("/settings/discord/finish", discordFinish)

	loadSimplePages(r)

	r.NoRoute(notFound)

	return r
}

const alwaysRespondText = `Ooops! Looks like something went really wrong while trying to process your request.
Perhaps report this to a Ripple developer?
Retrying doing again what you were trying to do might work, too.`
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/intervention-engine/fhir/models"
	_ "github.com/lib/pq"
	"github.com/synthetichealth/bulkfhirloader/bulkloader"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var debug *bool

// WorkerChannel coordinates the processing of FHIR bundles between several workers.
type WorkerChannel struct {
	bundleChannel chan (string)
}

// visit visits all the FHIR bundles in a specified path, adding each bundle to the
// bundleChannel that feeds the workers.
func (wc *WorkerChannel) visit(path string, f os.FileInfo, err error) error {

	if *debug {
		log.Printf("Visited: %s\n", path)
	}

	if !f.IsDir() && strings.HasSuffix(path, ".json") {

		// push bundle onto channel
		wc.bundleChannel <- path
		return nil
	}

	if *debug {
		log.Println("Processed directory path or non-json file....")
	}
	return nil
}

func main() {
	// required command line flags
	fhirBundlePath := flag.String("p", "", "Path to fhir bundles to upload")
	mongoServer := flag.String("mongo", "localhost:27017", "MongoDB server url, format: host:27017")
	mongoDBName := flag.String("dbname", "fhir", "MongoDB database name, e.g. 'fhir'")
	pgurl := flag.String("pgurl", "", "Postgres connection string, format: postgresql://username:password@host/dbname?sslmode=disable")

	// optional flags (with sensible defaults)
	numWorkers := flag.Int("workers", 8, "Number of concurrent workers to use")
	reset := flag.Bool("reset", false, "Reset the FHIR collections in Mongo and reset the synth_ma statistics")
	debug = flag.Bool("debug", false, "Display additional debug output")

	flag.Parse()

	if *fhirBundlePath == "" {
		fmt.Println("You must specify a path to the fhir bundles to upload")
		return
	}

	if *pgurl == "" {
		fmt.Println("You must specify a Postgres connection string")
		return
	}

	var err error

	// setup the MongoDB connection
	mongoSession, err := mgo.Dial(*mongoServer)
	if err != nil {
		log.Fatal(err)
	}
	defer mongoSession.Close()

	// setup the Postgres connection
	pgDB, err := sql.Open("postgres", *pgurl)
	if err != nil {
		log.Fatal("Failed to connect to Postgres")
	}

	// ping the Postgres db to ensure we connected successfully
	if err = pgDB.Ping(); err != nil {
		log.Fatal(err)
	}
	defer pgDB.Close()

	// optionally reset the data in postgres and mongo (if starting a clean upload)
	if *reset {
		bulkloader.ClearFactTables(pgDB)
		bulkloader.ClearMongoCollections(mongoSession, *mongoDBName)
	}

	// query Postgres for a list of the current subdivisions and diseases we track
	log.Println("Getting latest subdivision and disease information from Postgres...")

	cousubs, err := getCousubs(pgDB)
	if err != nil {
		if *debug {
			log.Println(err)
		}
		log.Fatal("Failed to get subdivision list from Postgres")
	}

	diseases, err := getDiseases(pgDB)
	if err != nil {
		if *debug {
			log.Println(err)
		}
		log.Fatal("Failed to get disease list from Postgres")
	}

	// create a new WorkerChannel to coordinate workers
	log.Printf("Reading FHIR bundles in %s\n", *fhirBundlePath)

	start := time.Now()
	workerChannel := new(WorkerChannel)
	workerChannel.bundleChannel = make(chan string, 256)

	var wg sync.WaitGroup
	var counter uint64 // total number of FHIR bundles processed

	// spawn workers
	for i := 0; i < *numWorkers; i++ {
		wg.Add(1)
		go worker(&wg, workerChannel.bundleChannel, mongoSession, *mongoDBName, cousubs, diseases, &counter)
	}

	err = filepath.Walk(*fhirBundlePath, workerChannel.visit)
	if err != nil {
		log.Println("An error occured while reading-in FHIR bundles:")
		log.Fatal(err)
	}

	// close the channel when done
	close(workerChannel.bundleChannel)

	// wait for all workers to shut down properly
	wg.Wait()
	log.Printf("%d FHIR bundles read in %f seconds\n", counter, getSecondsSince(start))

	// process the statistics for the uploaded bundles
	bulkloader.CalculatePopulationFacts(mongoSession, *mongoDBName, pgDB)
	log.Printf("Time elapsed: %f seconds\n", getSecondsSince(start))

	bulkloader.CalculateDiseaseFacts(mongoSession, *mongoDBName, pgDB)
	log.Printf("Time elapsed: %f seconds\n", getSecondsSince(start))

	bulkloader.CalculateConditionFacts(mongoSession, *mongoDBName, pgDB)
	log.Printf("Time elapsed: %f seconds\n", getSecondsSince(start))
}

// getCousubs queries the Postgres database for the latest list of subdivision in the
// synth_ma.synth_cousub_dim table.
func getCousubs(db *sql.DB) (*bulkloader.CousubMap, error) {

	rows, err := db.Query(`
		SELECT case when right(cd.cs_name, 5) = ' Town' then substring(cd.cs_name, 1, length(cd.cs_name)-5)
			else cs_name
			end
			, cd.ct_fips
			, cd.cs_fips
		FROM synth_ma.synth_cousub_dim cd`)

	if err != nil {
		return nil, err
	}
	defer rows.Close()

	cousubs := make(bulkloader.CousubMap)

	for rows.Next() {
		var csName, ctFips, csFips string
		var cousub bulkloader.Cousub

		err := rows.Scan(&csName, &ctFips, &csFips)
		if err != nil {
			return nil, err
		}
		cousub.CountyIDFips = ctFips
		cousub.SubCountyIDFips = csFips
		cousubs[csName] = cousub
	}
	return &cousubs, nil
}

// getDiseases queries the Postgres database for the latest list of diseases in the
// synth_ma.synth_condition_dim table.
func getDiseases(db *sql.DB) (*bulkloader.DiseaseMap, error) {

	rows, err := db.Query(`
		SELECT cd.condition_id, coalesce(cd.disease_id, -999), cd.code_system, cd.code
		FROM synth_ma.synth_condition_dim cd`)

	if err != nil {
		return nil, err
	}
	defer rows.Close()

	diseases := make(bulkloader.DiseaseMap)

	for rows.Next() {
		var conditionID, diseaseID int
		var system, code string
		var disease bulkloader.Disease

		err := rows.Scan(&conditionID, &diseaseID, &system, &code)
		if err != nil {
			return nil, err
		}
		disease.ConditionID = conditionID
		disease.DiseaseID = diseaseID
		key := bulkloader.DiseaseKey{
			CodeSystem:  system,
			CodeSysCode: code,
		}
		diseases[key] = disease
	}
	return &diseases, nil
}

// worker uses a WorkerChannel to process all of the resources in a single FHIR bundle, specified by the path to that bundle's JSON file.
func worker(wg *sync.WaitGroup, bundles <-chan string, mongoSession *mgo.Session, dbName string, cousubs *bulkloader.CousubMap, diseases *bulkloader.DiseaseMap, counter *uint64) {
	defer wg.Done()

	for {
		select {
		case path, ok := <-bundles:
			if !ok {
				return
			}

			jsonFile, err := os.Open(path)
			if err != nil && *debug {
				log.Println("Error opening JSON file:\n", err)
				continue
			}

			jsonData, err := ioutil.ReadAll(jsonFile)
			jsonFile.Close()
			if err != nil && *debug {
				log.Println("Error reading JSON data:\n", err)
				continue
			}

			var bundle models.Bundle
			json.Unmarshal(jsonData, &bundle)

			refMap := make(map[string]models.Reference)

			entries := make([]*models.BundleEntryComponent, len(bundle.Entry))
			for i := range bundle.Entry {
				entries[i] = &bundle.Entry[i]
			}

			for _, entry := range entries {
				// Create a new BSON ID and add it to the reference map
				id := bson.NewObjectId().Hex()
				refMap[entry.FullUrl] = models.Reference{
					Reference:    reflect.TypeOf(entry.Resource).Elem().Name() + "/" + id,
					Type:         reflect.TypeOf(entry.Resource).Elem().Name(),
					ReferencedID: id,
					External:     new(bool),
				}
				// Update the resource's UUID to the new BSON ID that was just generated
				bulkloader.SetID(entry.Resource, id)
			}

			// Update all the references to the entries (to reflect newly assigned IDs)
			bulkloader.UpdateAllReferences(entries, refMap)

			resources := make([]interface{}, len(entries))
			for i := range entries {
				resources[i] = entries[i].Resource
			}

			atomic.AddUint64(counter, 1)
			bulkloader.UploadResources(resources, mongoSession, dbName, *cousubs, *diseases)
		} // close the select
	} // close the for
}

func getSecondsSince(start time.Time) float64 {
	return time.Now().Sub(start).Seconds()
}
<commit_msg>Added filepath.Walk error check<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/intervention-engine/fhir/models"
	_ "github.com/lib/pq"
	"github.com/synthetichealth/bulkfhirloader/bulkloader"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var debug *bool

// WorkerChannel coordinates the processing of FHIR bundles between several workers.
type WorkerChannel struct {
	bundleChannel chan (string)
}

// visit visits all the FHIR bundles in a specified path, adding each bundle to the
// bundleChannel that feeds the workers.
func (wc *WorkerChannel) visit(path string, f os.FileInfo, err error) error {

	if err != nil {
		return err
	}

	if *debug {
		log.Printf("Visited: %s\n", path)
	}

	if !f.IsDir() && strings.HasSuffix(path, ".json") {

		// push bundle onto channel
		wc.bundleChannel <- path
		return nil
	}

	if *debug {
		log.Println("Processed directory path or non-json file....")
	}
	return nil
}

func main() {
	// required command line flags
	fhirBundlePath := flag.String("p", "", "Path to fhir bundles to upload")
	mongoServer := flag.String("mongo", "localhost:27017", "MongoDB server url, format: host:27017")
	mongoDBName := flag.String("dbname", "fhir", "MongoDB database name, e.g. 'fhir'")
	pgurl := flag.String("pgurl", "", "Postgres connection string, format: postgresql://username:password@host/dbname?sslmode=disable")

	// optional flags (with sensible defaults)
	numWorkers := flag.Int("workers", 8, "Number of concurrent workers to use")
	reset := flag.Bool("reset", false, "Reset the FHIR collections in Mongo and reset the synth_ma statistics")
	debug = flag.Bool("debug", false, "Display additional debug output")

	flag.Parse()

	if *fhirBundlePath == "" {
		fmt.Println("You must specify a path to the fhir bundles to upload")
		return
	}

	if *pgurl == "" {
		fmt.Println("You must specify a Postgres connection string")
		return
	}

	var err error

	// setup the MongoDB connection
	mongoSession, err := mgo.Dial(*mongoServer)
	if err != nil {
		log.Fatal(err)
	}
	defer mongoSession.Close()

	// setup the Postgres connection
	pgDB, err := sql.Open("postgres", *pgurl)
	if err != nil {
		log.Fatal("Failed to connect to Postgres")
	}

	// ping the Postgres db to ensure we connected successfully
	if err = pgDB.Ping(); err != nil {
		log.Fatal(err)
	}
	defer pgDB.Close()

	// optionally reset the data in postgres and mongo (if starting a clean upload)
	if *reset {
		bulkloader.ClearFactTables(pgDB)
		bulkloader.ClearMongoCollections(mongoSession, *mongoDBName)
	}

	// query Postgres for a list of the current subdivisions and diseases we track
	log.Println("Getting latest subdivision and disease information from Postgres...")

	cousubs, err := getCousubs(pgDB)
	if err != nil {
		if *debug {
			log.Println(err)
		}
		log.Fatal("Failed to get subdivision list from Postgres")
	}

	diseases, err := getDiseases(pgDB)
	if err != nil {
		if *debug {
			log.Println(err)
		}
		log.Fatal("Failed to get disease list from Postgres")
	}

	// create a new WorkerChannel to coordinate workers
	log.Printf("Reading FHIR bundles in %s\n", *fhirBundlePath)

	start := time.Now()
	workerChannel := new(WorkerChannel)
	workerChannel.bundleChannel = make(chan string, 256)

	var wg sync.WaitGroup
	var counter uint64 // total number of FHIR bundles processed

	// spawn workers
	for i := 0; i < *numWorkers; i++ {
		wg.Add(1)
		go worker(&wg, workerChannel.bundleChannel, mongoSession, *mongoDBName, cousubs, diseases, &counter)
	}

	err = filepath.Walk(*fhirBundlePath, workerChannel.visit)
	if err != nil {
		log.Println("An error occured while reading-in FHIR bundles:")
		log.Fatal(err)
	}

	// close the channel when done
	close(workerChannel.bundleChannel)

	// wait for all workers to shut down properly
	wg.Wait()
	log.Printf("%d FHIR bundles read in %f seconds\n", counter, getSecondsSince(start))

	// process the statistics for the uploaded bundles
	bulkloader.CalculatePopulationFacts(mongoSession, *mongoDBName, pgDB)
	log.Printf("Time elapsed: %f seconds\n", getSecondsSince(start))

	bulkloader.CalculateDiseaseFacts(mongoSession, *mongoDBName, pgDB)
	log.Printf("Time elapsed: %f seconds\n", getSecondsSince(start))

	bulkloader.CalculateConditionFacts(mongoSession, *mongoDBName, pgDB)
	log.Printf("Time elapsed: %f seconds\n", getSecondsSince(start))
}

// getCousubs queries the Postgres database for the latest list of subdivision in the
// synth_ma.synth_cousub_dim table.
func getCousubs(db *sql.DB) (*bulkloader.CousubMap, error) {

	rows, err := db.Query(`
		SELECT case when right(cd.cs_name, 5) = ' Town' then substring(cd.cs_name, 1, length(cd.cs_name)-5)
			else cs_name
			end
			, cd.ct_fips
			, cd.cs_fips
		FROM synth_ma.synth_cousub_dim cd`)

	if err != nil {
		return nil, err
	}
	defer rows.Close()

	cousubs := make(bulkloader.CousubMap)

	for rows.Next() {
		var csName, ctFips, csFips string
		var cousub bulkloader.Cousub

		err := rows.Scan(&csName, &ctFips, &csFips)
		if err != nil {
			return nil, err
		}
		cousub.CountyIDFips = ctFips
		cousub.SubCountyIDFips = csFips
		cousubs[csName] = cousub
	}
	return &cousubs, nil
}

// getDiseases queries the Postgres database for the latest list of diseases in the
// synth_ma.synth_condition_dim table.
func getDiseases(db *sql.DB) (*bulkloader.DiseaseMap, error) {

	rows, err := db.Query(`
		SELECT cd.condition_id, coalesce(cd.disease_id, -999), cd.code_system, cd.code
		FROM synth_ma.synth_condition_dim cd`)

	if err != nil {
		return nil, err
	}
	defer rows.Close()

	diseases := make(bulkloader.DiseaseMap)

	for rows.Next() {
		var conditionID, diseaseID int
		var system, code string
		var disease bulkloader.Disease

		err := rows.Scan(&conditionID, &diseaseID, &system, &code)
		if err != nil {
			return nil, err
		}
		disease.ConditionID = conditionID
		disease.DiseaseID = diseaseID
		key := bulkloader.DiseaseKey{
			CodeSystem:  system,
			CodeSysCode: code,
		}
		diseases[key] = disease
	}
	return &diseases, nil
}

// worker uses a WorkerChannel to process all of the resources in a single FHIR bundle, specified by the path to that bundle's JSON file.
func worker(wg *sync.WaitGroup, bundles <-chan string, mongoSession *mgo.Session, dbName string, cousubs *bulkloader.CousubMap, diseases *bulkloader.DiseaseMap, counter *uint64) {
	defer wg.Done()

	for {
		select {
		case path, ok := <-bundles:
			if !ok {
				return
			}

			jsonFile, err := os.Open(path)
			if err != nil && *debug {
				log.Println("Error opening JSON file:\n", err)
				continue
			}

			jsonData, err := ioutil.ReadAll(jsonFile)
			jsonFile.Close()
			if err != nil && *debug {
				log.Println("Error reading JSON data:\n", err)
				continue
			}

			var bundle models.Bundle
			json.Unmarshal(jsonData, &bundle)

			refMap := make(map[string]models.Reference)

			entries := make([]*models.BundleEntryComponent, len(bundle.Entry))
			for i := range bundle.Entry {
				entries[i] = &bundle.Entry[i]
			}

			for _, entry := range entries {
				// Create a new BSON ID and add it to the reference map
				id := bson.NewObjectId().Hex()
				refMap[entry.FullUrl] = models.Reference{
					Reference:    reflect.TypeOf(entry.Resource).Elem().Name() + "/" + id,
					Type:         reflect.TypeOf(entry.Resource).Elem().Name(),
					ReferencedID: id,
					External:     new(bool),
				}
				// Update the resource's UUID to the new BSON ID that was just generated
				bulkloader.SetID(entry.Resource, id)
			}

			// Update all the references to the entries (to reflect newly assigned IDs)
			bulkloader.UpdateAllReferences(entries, refMap)

			resources := make([]interface{}, len(entries))
			for i := range entries {
				resources[i] = entries[i].Resource
			}

			atomic.AddUint64(counter, 1)
			bulkloader.UploadResources(resources, mongoSession, dbName, *cousubs, *diseases)
		} // close the select
	} // close the for
}

func getSecondsSince(start time.Time) float64 {
	return time.Now().Sub(start).Seconds()
}
<|endoftext|>


<commit_before>// Copyright 2015 Dominique Feyer <dfeyer@ttree.ch>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/dfeyer/flow-debugproxy/config"

	"github.com/dfeyer/flow-debugproxy/errorhandler"
	"github.com/dfeyer/flow-debugproxy/logger"
	"github.com/dfeyer/flow-debugproxy/pathmapperfactory"
	"github.com/dfeyer/flow-debugproxy/pathmapping"
	"github.com/dfeyer/flow-debugproxy/xdebugproxy"

	// Register available path mapper
	_ "github.com/dfeyer/flow-debugproxy/dummypathmapper"
	_ "github.com/dfeyer/flow-debugproxy/flowpathmapper"

	"github.com/codegangsta/cli"

	"net"
	"os"
	"strings"
)

func main() {
	app := cli.NewApp()
	app.Name = "flow-debugproxy"
	app.Usage = "Flow Framework xDebug proxy"
	app.Author = "Dominique Feyer"
	app.Email = "dominique@neos.io"
	app.Version = "0.9.0"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "xdebug, l",
			Value: "127.0.0.1:9000",
			Usage: "Listen address IP and port number",
		},
		cli.StringFlag{
			Name:  "ide, I",
			Value: "127.0.0.1:9010",
			Usage: "Bind address IP and port number",
		},
		cli.StringFlag{
			Name:  "context, c",
			Value: "Development",
			Usage: "The context to run as",
		},
		cli.StringFlag{
			Name:  "localroot, r",
			Value: "",
			Usage: "Local project root for remote debugging",
		},
		cli.StringFlag{
			Name:  "framework",
			Value: "flow",
			Usage: "Framework support, currently on Flow framework (flow) or Dummy (dummy) is supported",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Verbose",
		},
		cli.BoolFlag{
			Name:  "vv",
			Usage: "Very verbose",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Show debug output",
		},
	}

	app.Action = func(cli *cli.Context) {
		c := &config.Config{
			Context:     cli.String("context"),
			Framework:   cli.String("framework"),
			LocalRoot:   strings.TrimRight(cli.String("localroot"), "/"),
			Verbose:     cli.Bool("verbose") || cli.Bool("vv"),
			VeryVerbose: cli.Bool("vv"),
			Debug:       cli.Bool("debug"),
		}

		log := &logger.Logger{
			Config: c,
		}

		laddr, raddr, listener := setupNetworkConnection(cli.String("xdebug"), cli.String("ide"), log)

		log.Info("special version [wy/ft]\n")
		log.Info("Debugger from %v\nIDE      from %v\n", laddr, raddr)

		pathMapping := &pathmapping.PathMapping{}
		pathMapper, err := pathmapperfactory.Create(c, pathMapping, log)
		errorhandler.PanicHandling(err, log)

		for {
			conn, err := listener.AcceptTCP()
			if err != nil {
				log.Warn("Failed to accept connection '%s'\n", err)
				continue
			}

			proxy := &xdebugproxy.Proxy{
				Lconn:      conn,
				Raddr:      raddr,
				PathMapper: pathMapper,
				Config:     c,
			}
			go proxy.Start()
		}
	}

	app.Run(os.Args)
}

func setupNetworkConnection(xdebugAddr string, ideAddr string, log *logger.Logger) (*net.TCPAddr, *net.TCPAddr, *net.TCPListener) {
	laddr, err := net.ResolveTCPAddr("tcp", xdebugAddr)
	errorhandler.PanicHandling(err, log)

	raddr, err := net.ResolveTCPAddr("tcp", ideAddr)
	errorhandler.PanicHandling(err, log)

	listener, err := net.ListenTCP("tcp", laddr)
	errorhandler.PanicHandling(err, log)

	return laddr, raddr, listener
}
<commit_msg>TASK: Remove unused CLI output<commit_after>// Copyright 2015 Dominique Feyer <dfeyer@ttree.ch>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"github.com/dfeyer/flow-debugproxy/config"

	"github.com/dfeyer/flow-debugproxy/errorhandler"
	"github.com/dfeyer/flow-debugproxy/logger"
	"github.com/dfeyer/flow-debugproxy/pathmapperfactory"
	"github.com/dfeyer/flow-debugproxy/pathmapping"
	"github.com/dfeyer/flow-debugproxy/xdebugproxy"

	// Register available path mapper
	_ "github.com/dfeyer/flow-debugproxy/dummypathmapper"
	_ "github.com/dfeyer/flow-debugproxy/flowpathmapper"

	"github.com/codegangsta/cli"

	"net"
	"os"
	"strings"
)

func main() {
	app := cli.NewApp()
	app.Name = "flow-debugproxy"
	app.Usage = "Flow Framework xDebug proxy"
	app.Author = "Dominique Feyer"
	app.Email = "dominique@neos.io"
	app.Version = "0.9.0"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:  "xdebug, l",
			Value: "127.0.0.1:9000",
			Usage: "Listen address IP and port number",
		},
		cli.StringFlag{
			Name:  "ide, I",
			Value: "127.0.0.1:9010",
			Usage: "Bind address IP and port number",
		},
		cli.StringFlag{
			Name:  "context, c",
			Value: "Development",
			Usage: "The context to run as",
		},
		cli.StringFlag{
			Name:  "localroot, r",
			Value: "",
			Usage: "Local project root for remote debugging",
		},
		cli.StringFlag{
			Name:  "framework",
			Value: "flow",
			Usage: "Framework support, currently on Flow framework (flow) or Dummy (dummy) is supported",
		},
		cli.BoolFlag{
			Name:  "verbose",
			Usage: "Verbose",
		},
		cli.BoolFlag{
			Name:  "vv",
			Usage: "Very verbose",
		},
		cli.BoolFlag{
			Name:  "debug",
			Usage: "Show debug output",
		},
	}

	app.Action = func(cli *cli.Context) {
		c := &config.Config{
			Context:     cli.String("context"),
			Framework:   cli.String("framework"),
			LocalRoot:   strings.TrimRight(cli.String("localroot"), "/"),
			Verbose:     cli.Bool("verbose") || cli.Bool("vv"),
			VeryVerbose: cli.Bool("vv"),
			Debug:       cli.Bool("debug"),
		}

		log := &logger.Logger{
			Config: c,
		}

		laddr, raddr, listener := setupNetworkConnection(cli.String("xdebug"), cli.String("ide"), log)

		log.Info("Debugger from %v\nIDE      from %v\n", laddr, raddr)

		pathMapping := &pathmapping.PathMapping{}
		pathMapper, err := pathmapperfactory.Create(c, pathMapping, log)
		errorhandler.PanicHandling(err, log)

		for {
			conn, err := listener.AcceptTCP()
			if err != nil {
				log.Warn("Failed to accept connection '%s'\n", err)
				continue
			}

			proxy := &xdebugproxy.Proxy{
				Lconn:      conn,
				Raddr:      raddr,
				PathMapper: pathMapper,
				Config:     c,
			}
			go proxy.Start()
		}
	}

	app.Run(os.Args)
}

func setupNetworkConnection(xdebugAddr string, ideAddr string, log *logger.Logger) (*net.TCPAddr, *net.TCPAddr, *net.TCPListener) {
	laddr, err := net.ResolveTCPAddr("tcp", xdebugAddr)
	errorhandler.PanicHandling(err, log)

	raddr, err := net.ResolveTCPAddr("tcp", ideAddr)
	errorhandler.PanicHandling(err, log)

	listener, err := net.ListenTCP("tcp", laddr)
	errorhandler.PanicHandling(err, log)

	return laddr, raddr, listener
}
<|endoftext|>


<commit_before>package qbs

import (
	"fmt"
	"reflect"
	"strings"
	"time"
)

type base struct {
	Dialect Dialect
}

func (d base) substituteMarkers(query string) string {
	return query
}

func (d base) quote(s string) string {
	sep := "."
	a := []string{}
	c := strings.Split(s, sep)
	for _, v := range c {
		a = append(a, fmt.Sprintf("`%s`", v))
	}
	return strings.Join(a, sep)
}

func (d base) parseBool(value reflect.Value) bool {
	return value.Bool()
}

func (d base) setModelValue(driverValue, fieldValue reflect.Value) error {
	switch fieldValue.Type().Kind() {
	case reflect.Bool:
		fieldValue.SetBool(d.Dialect.parseBool(driverValue.Elem()))
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fieldValue.SetInt(driverValue.Elem().Int())
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		// reading uint from int value causes panic
		switch driverValue.Elem().Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			fieldValue.SetUint(uint64(driverValue.Elem().Int()))
		default:
			fieldValue.SetUint(driverValue.Elem().Uint())
		}
	case reflect.Float32, reflect.Float64:
		fieldValue.SetFloat(driverValue.Elem().Float())
	case reflect.String:
		fieldValue.SetString(string(driverValue.Elem().Bytes()))
	case reflect.Slice:
		if reflect.TypeOf(driverValue.Interface()).Elem().Kind() == reflect.Uint8 {
			fieldValue.SetBytes(driverValue.Elem().Bytes())
		}
	case reflect.Struct:
		if _, ok := fieldValue.Interface().(time.Time); ok {
			fieldValue.Set(driverValue.Elem())
		}
	}
	return nil
}

func (d base) querySql(criteria *criteria) (string, []interface{}) {
	query := make([]string, 0, 20)
	args := make([]interface{}, 0, 20)

	table := d.Dialect.quote(criteria.model.table)
	columns := []string{}
	tables := []string{table}
	hasJoin := len(criteria.model.refs) > 0
	for _, v := range criteria.model.fields {
		colName := d.Dialect.quote(v.name)
		if hasJoin {
			colName = d.Dialect.quote(criteria.model.table) + "." + colName
		}
		columns = append(columns, colName)
	}
	for k, v := range criteria.model.refs {
		tableAlias := toSnake(k)
		quotedTableAlias := d.Dialect.quote(tableAlias)
		quotedParentTable := d.Dialect.quote(v.model.table)
		leftKey := table + "." + d.Dialect.quote(v.refKey)
		parentPrimary := quotedTableAlias + "." + d.Dialect.quote(v.model.pk.name)
		joinClause := fmt.Sprintf("LEFT JOIN %v AS %v ON %v = %v", quotedParentTable, quotedTableAlias, leftKey, parentPrimary)
		tables = append(tables, joinClause)
		for _, f := range v.model.fields {
			alias := tableAlias + "___" + f.name
			columns = append(columns, d.Dialect.quote(tableAlias+"."+f.name)+" AS "+alias)
		}
	}
	query = append(query, "SELECT", strings.Join(columns, ", "), "FROM", strings.Join(tables, " "))

	if criteria.condition != nil {
		cexpr, cargs := criteria.condition.Merge()
		query = append(query, "WHERE", cexpr)
		args = append(args, cargs...)
	}
	orderByLen := len(criteria.orderBys)
	if orderByLen > 0 {
		query = append(query, "ORDER BY")
		for i, order := range criteria.orderBys {
			query = append(query, order.path)
			if order.desc {
				query = append(query, "DESC")
			}
			if i < orderByLen-1 {
				query = append(query, ",")
			}
		}
	}

	if x := criteria.limit; x > 0 {
		query = append(query, "LIMIT ?")
		args = append(args, criteria.limit)
	}
	if x := criteria.offset; x > 0 {
		query = append(query, "OFFSET ?")
		args = append(args, criteria.offset)
	}
	return d.Dialect.substituteMarkers(strings.Join(query, " ")), args
}

func (d base) insert(q *Qbs) (int64, error) {
	sql, args := d.Dialect.insertSql(q.criteria)
	result, err := q.Exec(sql, args...)
	if err != nil {
		return -1, err
	}
	id, err := result.LastInsertId()
	if err != nil {
		return -1, err
	}
	return id, nil
}

func (d base) insertSql(criteria *criteria) (string, []interface{}) {
	columns, values := criteria.model.columnsAndValues(false)
	quotedColumns := make([]string, 0, len(columns))
	markers := make([]string, 0, len(columns))
	for _, c := range columns {
		quotedColumns = append(quotedColumns, d.Dialect.quote(c))
		markers = append(markers, "?")
	}
	sql := fmt.Sprintf(
		"INSERT INTO %v (%v) VALUES (%v)",
		d.Dialect.quote(criteria.model.table),
		strings.Join(quotedColumns, ", "),
		strings.Join(markers, ", "),
	)
	return sql, values
}

func (d base) update(q *Qbs) (int64, error) {
	sql, args := d.Dialect.updateSql(q.criteria)
	result, err := q.Exec(sql, args...)
	if err != nil {
		return 0, err
	}
	affected, err := result.RowsAffected()
	return affected, err
}

func (d base) updateSql(criteria *criteria) (string, []interface{}) {
	columns, values := criteria.model.columnsAndValues(true)
	pairs := make([]string, 0, len(columns))
	for _, column := range columns {
		pairs = append(pairs, fmt.Sprintf("%v = ?", d.Dialect.quote(column)))
	}
	conditionSql, args := criteria.condition.Merge()
	sql := fmt.Sprintf(
		"UPDATE %v SET %v WHERE %v",
		d.Dialect.quote(criteria.model.table),
		strings.Join(pairs, ", "),
		conditionSql,
	)
	values = append(values, args...)
	return sql, values
}

func (d base) delete(q *Qbs) (int64, error) {
	sql, args := d.Dialect.deleteSql(q.criteria)
	result, err := q.Exec(sql, args...)
	affected, err := result.RowsAffected()
	if err != nil {
		return 0, err
	}
	return affected, err
}

func (d base) deleteSql(criteria *criteria) (string, []interface{}) {
	conditionSql, args := criteria.condition.Merge()
	sql := "DELETE FROM " + d.Dialect.quote(criteria.model.table) + " WHERE " + conditionSql
	return sql, args
}

func (d base) createTableSql(model *model, ifNotExists bool) string {
	a := []string{"CREATE TABLE "}
	if ifNotExists {
		a = append(a, "IF NOT EXISTS ")
	}
	a = append(a, d.Dialect.quote(model.table), " ( ")
	for i, field := range model.fields {
		b := []string{
			d.Dialect.quote(field.name),
		}
		if field.pk {
			_, ok := field.value.(string)
			b = append(b, d.Dialect.primaryKeySql(ok, field.size()))
		} else {
			b = append(b, d.Dialect.sqlType(field.value, field.size()))
			if field.notNull() {
				b = append(b, "NOT NULL")
			}
			if x := field.dfault(); x != "" {
				b = append(b, "DEFAULT "+x)
			}
		}
		a = append(a, strings.Join(b, " "))
		if i < len(model.fields)-1 {
			a = append(a, ", ")
		}
	}
	for _, v := range model.refs {
		if v.foreignKey {
			a = append(a, ", FOREIGN KEY (", d.Dialect.quote(v.refKey), ") REFERENCES ")
			a = append(a, d.Dialect.quote(v.model.table), " (", d.Dialect.quote(v.model.pk.name), ") ON DELETE CASCADE")
		}
	}
	a = append(a, " )")
	return strings.Join(a, "")
}

func (d base) dropTableSql(table string) string {
	a := []string{"DROP TABLE IF EXISTS"}
	a = append(a, d.Dialect.quote(table))
	return strings.Join(a, " ")
}

func (d base) addColumnSql(table, column string, typ interface{}, size int) string {
	return fmt.Sprintf(
		"ALTER TABLE %v ADD COLUMN %v %v",
		d.Dialect.quote(table),
		d.Dialect.quote(column),
		d.Dialect.sqlType(typ, size),
	)
}

func (d base) createIndexSql(name, table string, unique bool, columns ...string) string {
	a := []string{"CREATE"}
	if unique {
		a = append(a, "UNIQUE")
	}
	quotedColumns := make([]string, 0, len(columns))
	for _, c := range columns {
		quotedColumns = append(quotedColumns, d.Dialect.quote(c))
	}
	a = append(a, fmt.Sprintf(
		"INDEX %v ON %v (%v)",
		d.Dialect.quote(name),
		d.Dialect.quote(table),
		strings.Join(quotedColumns, ", "),
	))
	return strings.Join(a, " ")
}

func (d base) columnsInTable(mg *Migration, table interface{}) map[string]bool {
	tn := tableName(table)
	columns := make(map[string]bool)
	query := "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"
	query = mg.Dialect.substituteMarkers(query)
	rows, err := mg.Db.Query(query, mg.DbName, tn)
	defer rows.Close()
	if err != nil {
		panic(err)
	}
	for rows.Next() {
		column := ""
		err := rows.Scan(&column)
		if err == nil {
			columns[column] = true
		}
	}
	return columns
}

func (d base) catchMigrationError(err error) bool {
	return false
}
<commit_msg>bug fix: delete error should not be ignored.<commit_after>package qbs

import (
	"fmt"
	"reflect"
	"strings"
	"time"
)

type base struct {
	Dialect Dialect
}

func (d base) substituteMarkers(query string) string {
	return query
}

func (d base) quote(s string) string {
	sep := "."
	a := []string{}
	c := strings.Split(s, sep)
	for _, v := range c {
		a = append(a, fmt.Sprintf("`%s`", v))
	}
	return strings.Join(a, sep)
}

func (d base) parseBool(value reflect.Value) bool {
	return value.Bool()
}

func (d base) setModelValue(driverValue, fieldValue reflect.Value) error {
	switch fieldValue.Type().Kind() {
	case reflect.Bool:
		fieldValue.SetBool(d.Dialect.parseBool(driverValue.Elem()))
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		fieldValue.SetInt(driverValue.Elem().Int())
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		// reading uint from int value causes panic
		switch driverValue.Elem().Kind() {
		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
			fieldValue.SetUint(uint64(driverValue.Elem().Int()))
		default:
			fieldValue.SetUint(driverValue.Elem().Uint())
		}
	case reflect.Float32, reflect.Float64:
		fieldValue.SetFloat(driverValue.Elem().Float())
	case reflect.String:
		fieldValue.SetString(string(driverValue.Elem().Bytes()))
	case reflect.Slice:
		if reflect.TypeOf(driverValue.Interface()).Elem().Kind() == reflect.Uint8 {
			fieldValue.SetBytes(driverValue.Elem().Bytes())
		}
	case reflect.Struct:
		if _, ok := fieldValue.Interface().(time.Time); ok {
			fieldValue.Set(driverValue.Elem())
		}
	}
	return nil
}

func (d base) querySql(criteria *criteria) (string, []interface{}) {
	query := make([]string, 0, 20)
	args := make([]interface{}, 0, 20)

	table := d.Dialect.quote(criteria.model.table)
	columns := []string{}
	tables := []string{table}
	hasJoin := len(criteria.model.refs) > 0
	for _, v := range criteria.model.fields {
		colName := d.Dialect.quote(v.name)
		if hasJoin {
			colName = d.Dialect.quote(criteria.model.table) + "." + colName
		}
		columns = append(columns, colName)
	}
	for k, v := range criteria.model.refs {
		tableAlias := toSnake(k)
		quotedTableAlias := d.Dialect.quote(tableAlias)
		quotedParentTable := d.Dialect.quote(v.model.table)
		leftKey := table + "." + d.Dialect.quote(v.refKey)
		parentPrimary := quotedTableAlias + "." + d.Dialect.quote(v.model.pk.name)
		joinClause := fmt.Sprintf("LEFT JOIN %v AS %v ON %v = %v", quotedParentTable, quotedTableAlias, leftKey, parentPrimary)
		tables = append(tables, joinClause)
		for _, f := range v.model.fields {
			alias := tableAlias + "___" + f.name
			columns = append(columns, d.Dialect.quote(tableAlias+"."+f.name)+" AS "+alias)
		}
	}
	query = append(query, "SELECT", strings.Join(columns, ", "), "FROM", strings.Join(tables, " "))

	if criteria.condition != nil {
		cexpr, cargs := criteria.condition.Merge()
		query = append(query, "WHERE", cexpr)
		args = append(args, cargs...)
	}
	orderByLen := len(criteria.orderBys)
	if orderByLen > 0 {
		query = append(query, "ORDER BY")
		for i, order := range criteria.orderBys {
			query = append(query, order.path)
			if order.desc {
				query = append(query, "DESC")
			}
			if i < orderByLen-1 {
				query = append(query, ",")
			}
		}
	}

	if x := criteria.limit; x > 0 {
		query = append(query, "LIMIT ?")
		args = append(args, criteria.limit)
	}
	if x := criteria.offset; x > 0 {
		query = append(query, "OFFSET ?")
		args = append(args, criteria.offset)
	}
	return d.Dialect.substituteMarkers(strings.Join(query, " ")), args
}

func (d base) insert(q *Qbs) (int64, error) {
	sql, args := d.Dialect.insertSql(q.criteria)
	result, err := q.Exec(sql, args...)
	if err != nil {
		return -1, err
	}
	id, err := result.LastInsertId()
	if err != nil {
		return -1, err
	}
	return id, nil
}

func (d base) insertSql(criteria *criteria) (string, []interface{}) {
	columns, values := criteria.model.columnsAndValues(false)
	quotedColumns := make([]string, 0, len(columns))
	markers := make([]string, 0, len(columns))
	for _, c := range columns {
		quotedColumns = append(quotedColumns, d.Dialect.quote(c))
		markers = append(markers, "?")
	}
	sql := fmt.Sprintf(
		"INSERT INTO %v (%v) VALUES (%v)",
		d.Dialect.quote(criteria.model.table),
		strings.Join(quotedColumns, ", "),
		strings.Join(markers, ", "),
	)
	return sql, values
}

func (d base) update(q *Qbs) (int64, error) {
	sql, args := d.Dialect.updateSql(q.criteria)
	result, err := q.Exec(sql, args...)
	if err != nil {
		return 0, err
	}
	affected, err := result.RowsAffected()
	return affected, err
}

func (d base) updateSql(criteria *criteria) (string, []interface{}) {
	columns, values := criteria.model.columnsAndValues(true)
	pairs := make([]string, 0, len(columns))
	for _, column := range columns {
		pairs = append(pairs, fmt.Sprintf("%v = ?", d.Dialect.quote(column)))
	}
	conditionSql, args := criteria.condition.Merge()
	sql := fmt.Sprintf(
		"UPDATE %v SET %v WHERE %v",
		d.Dialect.quote(criteria.model.table),
		strings.Join(pairs, ", "),
		conditionSql,
	)
	values = append(values, args...)
	return sql, values
}

func (d base) delete(q *Qbs) (int64, error) {
	sql, args := d.Dialect.deleteSql(q.criteria)
	result, err := q.Exec(sql, args...)
	if err != nil {
		return 0, err
	}
	affected, err := result.RowsAffected()
	if err != nil {
		return 0, err
	}
	return affected, err
}

func (d base) deleteSql(criteria *criteria) (string, []interface{}) {
	conditionSql, args := criteria.condition.Merge()
	sql := "DELETE FROM " + d.Dialect.quote(criteria.model.table) + " WHERE " + conditionSql
	return sql, args
}

func (d base) createTableSql(model *model, ifNotExists bool) string {
	a := []string{"CREATE TABLE "}
	if ifNotExists {
		a = append(a, "IF NOT EXISTS ")
	}
	a = append(a, d.Dialect.quote(model.table), " ( ")
	for i, field := range model.fields {
		b := []string{
			d.Dialect.quote(field.name),
		}
		if field.pk {
			_, ok := field.value.(string)
			b = append(b, d.Dialect.primaryKeySql(ok, field.size()))
		} else {
			b = append(b, d.Dialect.sqlType(field.value, field.size()))
			if field.notNull() {
				b = append(b, "NOT NULL")
			}
			if x := field.dfault(); x != "" {
				b = append(b, "DEFAULT "+x)
			}
		}
		a = append(a, strings.Join(b, " "))
		if i < len(model.fields)-1 {
			a = append(a, ", ")
		}
	}
	for _, v := range model.refs {
		if v.foreignKey {
			a = append(a, ", FOREIGN KEY (", d.Dialect.quote(v.refKey), ") REFERENCES ")
			a = append(a, d.Dialect.quote(v.model.table), " (", d.Dialect.quote(v.model.pk.name), ") ON DELETE CASCADE")
		}
	}
	a = append(a, " )")
	return strings.Join(a, "")
}

func (d base) dropTableSql(table string) string {
	a := []string{"DROP TABLE IF EXISTS"}
	a = append(a, d.Dialect.quote(table))
	return strings.Join(a, " ")
}

func (d base) addColumnSql(table, column string, typ interface{}, size int) string {
	return fmt.Sprintf(
		"ALTER TABLE %v ADD COLUMN %v %v",
		d.Dialect.quote(table),
		d.Dialect.quote(column),
		d.Dialect.sqlType(typ, size),
	)
}

func (d base) createIndexSql(name, table string, unique bool, columns ...string) string {
	a := []string{"CREATE"}
	if unique {
		a = append(a, "UNIQUE")
	}
	quotedColumns := make([]string, 0, len(columns))
	for _, c := range columns {
		quotedColumns = append(quotedColumns, d.Dialect.quote(c))
	}
	a = append(a, fmt.Sprintf(
		"INDEX %v ON %v (%v)",
		d.Dialect.quote(name),
		d.Dialect.quote(table),
		strings.Join(quotedColumns, ", "),
	))
	return strings.Join(a, " ")
}

func (d base) columnsInTable(mg *Migration, table interface{}) map[string]bool {
	tn := tableName(table)
	columns := make(map[string]bool)
	query := "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"
	query = mg.Dialect.substituteMarkers(query)
	rows, err := mg.Db.Query(query, mg.DbName, tn)
	defer rows.Close()
	if err != nil {
		panic(err)
	}
	for rows.Next() {
		column := ""
		err := rows.Scan(&column)
		if err == nil {
			columns[column] = true
		}
	}
	return columns
}

func (d base) catchMigrationError(err error) bool {
	return false
}
<|endoftext|>


<commit_before>// crypt.go
package jcrypt

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
)

var confdir string
var keypath string
var crypt Crypt

const sep = string(os.PathSeparator)

type Crypt struct {
	key []byte
}

func init() {
	//make sure confdir is available and secure
	usr, err := user.Current()
	if err != nil {
		fmt.Println(err)
	}
	confdir = usr.HomeDir + sep + ".dwego"
	keypath = confdir + sep + "keyfile"
	err = os.Mkdir(confdir, 0700)
	if os.IsExist(err) {
		//always (try to) make sure confdir permissions are secure
		err := os.Chmod(confdir, 0700)
		if err != nil {
			fmt.Println(err)
		}
	}
	err = crypt.LoadKey(keypath)
	if err != nil && !os.IsExist(err) {
		crypt.NewKey()
		crypt.SaveKey(keypath)
	}
	//Try these one at a time, reloading program between, to test homedir key file.
	//crypt.testsave("test")
	//crypt.testload("test")
}

//NewKey sets Crypt.key to a randomized 32 byte key to be used in encryption.
func (c *Crypt) NewKey() {
	alphanum := "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, 32)
	rand.Read(bytes)
	for i, b := range bytes {
		bytes[i] = alphanum[b%byte(len(alphanum))]
	}
	c.key = bytes
}

//ObjectToFile converts a data object to json then saves it to file with Crypt.SaveFile.
func (c *Crypt) ObjectToFile(path string, m interface{}) {
	b, err := json.Marshal(m)
	if err != nil {
		fmt.Println(err)
		return
	}
	c.SaveFile(path, b)
}

//FileToObject loads a file, saved with Crypt.ObjectToFile, into the given object variable.
func (c *Crypt) FileToObject(path string, m interface{}) {
	b, err := c.LoadFile(path)
	if err != nil {
		fmt.Println(err)
		return
	}
	err = json.Unmarshal(b, m)
	if err != nil {
		fmt.Println(err)
	}
}

//SaveFile encodes & encrypts byte data then writes it to file.
func (c *Crypt) SaveFile(path string, b []byte) error {
	return ioutil.WriteFile(path, c.encrypt(b), 0644)
}

//LoadFile decrypts & decodes a file that has been saved with Crypt.SaveFile.
func (c *Crypt) LoadFile(path string) (b []byte, e error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		e = err
	} else {
		b = c.decrypt(b)
	}
	return
}

//SaveKey encodes current Crypt.key in Base64 and saves it to file.
func (c *Crypt) SaveKey(path string) error {
	s := encodeBase64(c.key)
	return ioutil.WriteFile(path, []byte(s), 0600)
}

//LoadKey decodes a Base64 encoded key file & sets Crypt.key.
func (c *Crypt) LoadKey(path string) (e error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		e = err
	} else {
		c.key = decodeBase64(string(b))
	}
	return
}

//encrypt encodes bytes in base64 then encrypts data with AES.
func (c *Crypt) encrypt(text []byte) []byte {
	block, err := aes.NewCipher(c.key)
	if err != nil {
		panic(err)
	}
	b := encodeBase64(text)
	ciphertext := make([]byte, aes.BlockSize+len(b))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}
	cfb := cipher.NewCFBEncrypter(block, iv)
	cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
	return ciphertext
}

//decrypt data that has been encrypted with Crypt.encrypt.
func (c *Crypt) decrypt(text []byte) []byte {
	block, err := aes.NewCipher(c.key)
	if err != nil {
		panic(err)
	}
	if len(text) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := text[:aes.BlockSize]
	text = text[aes.BlockSize:]
	cfb := cipher.NewCFBDecrypter(block, iv)
	cfb.XORKeyStream(text, text)
	return decodeBase64(string(text))
}

func (c *Crypt) testsave(path string) {
	type ColorGroup struct {
		ID     int
		Name   string
		Colors []string
	}
	group := ColorGroup{
		ID:     1,
		Name:   "Reds",
		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
	}
	c.ObjectToFile(path, &group)
}

func (c *Crypt) testload(path string) {
	type ColorGroup struct {
		ID     int
		Name   string
		Colors []string
	}
	var ngroup ColorGroup
	c.FileToObject(path, &ngroup)
	fmt.Println(ngroup.Name)
}

func encodeBase64(b []byte) string {
	return base64.StdEncoding.EncodeToString(b)
}

func decodeBase64(s string) []byte {
	data, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		panic(err)
	}
	return data
}
<commit_msg>Updated NewKey method.<commit_after>// crypt.go
package jcrypt

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/user"
)

var confdir string
var keypath string
var crypt Crypt

const sep = string(os.PathSeparator)

type Crypt struct {
	key []byte
}

func init() {
	//make sure confdir is available and secure
	usr, err := user.Current()
	if err != nil {
		fmt.Println(err)
	}
	confdir = usr.HomeDir + sep + ".dwego"
	keypath = confdir + sep + "keyfile"
	err = os.Mkdir(confdir, 0700)
	if os.IsExist(err) {
		//always (try to) make sure confdir permissions are secure
		err := os.Chmod(confdir, 0700)
		if err != nil {
			fmt.Println(err)
		}
	}
	err = crypt.LoadKey(keypath)
	if err != nil && !os.IsExist(err) {
		crypt.NewKey()
		crypt.SaveKey(keypath)
	}
	//Try these one at a time, reloading program between, to test homedir key file.
	//crypt.testsave("test")
	//crypt.testload("test")
}

//NewKey sets Crypt.key to a randomized 32 byte key to be used in encryption.
func (c *Crypt) NewKey() {
	b := make([]byte, 32)
	_, err := rand.Read(b)
	if err != nil {
		panic(err)
	}
	c.key = b
}

//ObjectToFile converts a data object to json then saves it to file with Crypt.SaveFile.
func (c *Crypt) ObjectToFile(path string, m interface{}) {
	b, err := json.Marshal(m)
	if err != nil {
		fmt.Println(err)
		return
	}
	c.SaveFile(path, b)
}

//FileToObject loads a file, saved with Crypt.ObjectToFile, into the given object variable.
func (c *Crypt) FileToObject(path string, m interface{}) {
	b, err := c.LoadFile(path)
	if err != nil {
		fmt.Println(err)
		return
	}
	err = json.Unmarshal(b, m)
	if err != nil {
		fmt.Println(err)
	}
}

//SaveFile encodes & encrypts byte data then writes it to file.
func (c *Crypt) SaveFile(path string, b []byte) error {
	return ioutil.WriteFile(path, c.encrypt(b), 0644)
}

//LoadFile decrypts & decodes a file that has been saved with Crypt.SaveFile.
func (c *Crypt) LoadFile(path string) (b []byte, e error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		e = err
	} else {
		b = c.decrypt(b)
	}
	return
}

//SaveKey encodes current Crypt.key in Base64 and saves it to file.
func (c *Crypt) SaveKey(path string) error {
	s := encodeBase64(c.key)
	return ioutil.WriteFile(path, []byte(s), 0600)
}

//LoadKey decodes a Base64 encoded key file & sets Crypt.key.
func (c *Crypt) LoadKey(path string) (e error) {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		e = err
	} else {
		c.key = decodeBase64(string(b))
	}
	return
}

//encrypt encodes bytes in base64 then encrypts data with AES.
func (c *Crypt) encrypt(text []byte) []byte {
	block, err := aes.NewCipher(c.key)
	if err != nil {
		panic(err)
	}
	b := encodeBase64(text)
	ciphertext := make([]byte, aes.BlockSize+len(b))
	iv := ciphertext[:aes.BlockSize]
	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
		panic(err)
	}
	cfb := cipher.NewCFBEncrypter(block, iv)
	cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
	return ciphertext
}

//decrypt data that has been encrypted with Crypt.encrypt.
func (c *Crypt) decrypt(text []byte) []byte {
	block, err := aes.NewCipher(c.key)
	if err != nil {
		panic(err)
	}
	if len(text) < aes.BlockSize {
		panic("ciphertext too short")
	}
	iv := text[:aes.BlockSize]
	text = text[aes.BlockSize:]
	cfb := cipher.NewCFBDecrypter(block, iv)
	cfb.XORKeyStream(text, text)
	return decodeBase64(string(text))
}

func (c *Crypt) testsave(path string) {
	type ColorGroup struct {
		ID     int
		Name   string
		Colors []string
	}
	group := ColorGroup{
		ID:     1,
		Name:   "Reds",
		Colors: []string{"Crimson", "Red", "Ruby", "Maroon"},
	}
	c.ObjectToFile(path, &group)
}

func (c *Crypt) testload(path string) {
	type ColorGroup struct {
		ID     int
		Name   string
		Colors []string
	}
	var ngroup ColorGroup
	c.FileToObject(path, &ngroup)
	fmt.Println(ngroup.Name)
}

func encodeBase64(b []byte) string {
	return base64.StdEncoding.EncodeToString(b)
}

func decodeBase64(s string) []byte {
	data, err := base64.StdEncoding.DecodeString(s)
	if err != nil {
		panic(err)
	}
	return data
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"    // Working at 2002271f2160a4d243f0308af0827893e2868157
	"github.com/darkhelmet/twitterstream" // Working at 4051c41877496d38d54647c35897e768fd34385f
	"io/ioutil"
	"log"
	"net/url"
	"strings"
)

func main() {
	log.Println("Twitterd Started")
	tfg := GetCFG()
	b, e := ioutil.ReadFile("./twittercfg")
	if e != nil {
		log.Fatal("Could not read the ./twittercfg file. %s", tfg.Username)
	}
	twittertemp := string(b)
	twitterbits := strings.Split(twittertemp, "\n")
	if len(twitterbits) != 5 {
		log.Fatal("Not enought things in twitter cfg, Needs to be (seperated by \\n) username, consumerKey, consumerSecret, accessToken, accessSecret")
	}
	Client := twitterstream.NewClient(tfg.ConsumerKey, tfg.ConsumerSecret, tfg.AccessToken, tfg.AccessSecret)
	Conn, e := Client.Track(fmt.Sprintf("@%s", tfg.Username))
	// Streaming API is setup now, now just setup the general purpose one now
	anaconda.SetConsumerKey(tfg.ConsumerKey)
	anaconda.SetConsumerSecret(tfg.ConsumerSecret)
	api := anaconda.NewTwitterApi(tfg.AccessToken, tfg.AccessSecret)

	if e != nil {
		log.Fatal("could not open a streaming connection to get mentions :(")
	}
	for {
		t, e := Conn.Next()
		if e == nil {
			log.Println("TWEET: %s\n", t.Text)
			log.Println("OWNER @%s\n", strings.ToLower(tfg.Username))
			if strings.HasPrefix(strings.ToLower(t.Text), fmt.Sprintf("@%s", strings.ToLower(tfg.Username))) {
				v := url.Values{} // I dont even know
				t, e := api.PostTweet(fmt.Sprintf("@%s pong", t.User.ScreenName), v)
				if e == nil {
					fmt.Println(t)
				} else {
					fmt.Println(e)
				}
			} else {
				log.Println("Does not start with @<user> ignoring")
			}
		}
	}

}
<commit_msg>Removed old code for configuration<commit_after>package main

import (
	"fmt"
	"github.com/ChimeraCoder/anaconda"    // Working at 2002271f2160a4d243f0308af0827893e2868157
	"github.com/darkhelmet/twitterstream" // Working at 4051c41877496d38d54647c35897e768fd34385f
	"log"
	"net/url"
	"strings"
)

func main() {
	log.Println("Twitterd Started")
	tfg := GetCFG()
	Client := twitterstream.NewClient(tfg.ConsumerKey, tfg.ConsumerSecret, tfg.AccessToken, tfg.AccessSecret)
	Conn, e := Client.Track(fmt.Sprintf("@%s", tfg.Username))
	// Streaming API is setup now, now just setup the general purpose one now
	anaconda.SetConsumerKey(tfg.ConsumerKey)
	anaconda.SetConsumerSecret(tfg.ConsumerSecret)
	api := anaconda.NewTwitterApi(tfg.AccessToken, tfg.AccessSecret)

	if e != nil {
		log.Fatal("could not open a streaming connection to get mentions :(")
	}
	for {
		t, e := Conn.Next()
		if e == nil {
			log.Println("TWEET: %s\n", t.Text)
			log.Println("OWNER @%s\n", strings.ToLower(tfg.Username))
			if strings.HasPrefix(strings.ToLower(t.Text), fmt.Sprintf("@%s", strings.ToLower(tfg.Username))) {
				v := url.Values{} // I dont even know
				t, e := api.PostTweet(fmt.Sprintf("@%s pong", t.User.ScreenName), v)
				if e == nil {
					fmt.Println(t)
				} else {
					fmt.Println(e)
				}
			} else {
				log.Println("Does not start with @<user> ignoring")
			}
		}
	}

}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/christophwitzko/github-release-download/release"
	"github.com/julienschmidt/httprouter"
	"log"
	"net/http"
	"os"
	"os/signal"
	"time"
)

var bindAddress string = ":5000"
var client *release.GithubClient

func init() {
	if ba := os.Getenv("BIND_ADDRESS"); ba != "" {
		bindAddress = ba
	}
	client = release.NewClient(os.Getenv("GITHUB_TOKEN"))
}

func logger(router http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s - %s %s", r.RemoteAddr, r.Method, r.URL.EscapedPath())
		router.ServeHTTP(w, r)
	})
}

func doRedirect(w http.ResponseWriter, r *http.Request, url string, err error) {
	if err != nil {
		log.Println(err)
		http.Error(w, "server error", http.StatusInternalServerError)
		return
	}
	if url == "" {
		http.NotFound(w, r)
		return
	}
	http.Redirect(w, r, url, 302)
}

func getLatestDownload(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
	defer cancel()
	if len(ps) == 3 {
		url, err := client.GetLatestDownloadUrl(ctx, "go-" + ps[0].Value, ps[0].Value, ps[1].Value, ps[2].Value)
		doRedirect(w, r, url, err)
		return
	}
	url, err := client.GetLatestDownloadUrl(ctx, ps[0].Value, ps[1].Value, ps[2].Value, ps[3].Value)
	doRedirect(w, r, url, err)
}

func getMatchingDownload(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
	defer cancel()
	url, err := client.GetMatchingDownloadUrl(ctx, ps[0].Value, ps[1].Value, ps[2].Value, ps[3].Value, ps[4].Value)
	doRedirect(w, r, url, err)
}

func getVersions(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
	defer cancel()
	if ps[0].Value == "_" && ps[1].Value == "go" {
		versions, err := client.GetGoVersions(ctx)
		if err != nil {
			log.Println(err)
			http.Error(w, "server error", http.StatusInternalServerError)
			return
		}
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		json.NewEncoder(w).Encode(versions)
		return
	}
	http.NotFound(w, r)
}

func usage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	if ps[0].Value != "_usage" {
		http.NotFound(w, r)
		return
	}
	rl, _, err := client.Client.RateLimits(r.Context())
	if err != nil {
		log.Println(err)
		http.Error(w, "server error", http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	json.NewEncoder(w).Encode(rl.Core)
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	fmt.Fprintln(w, "https://get-release.xyz/:owner/:repo/:os/:arch{/:constraint}")
}

func main() {
	router := httprouter.New()
	router.GET("/", index)
	router.GET("/:owner", usage)
	router.GET("/:owner/:repo", getVersions)
	router.GET("/:owner/:repo/:os", getLatestDownload)
	router.GET("/:owner/:repo/:os/:arch", getLatestDownload)
	router.GET("/:owner/:repo/:os/:arch/:constraint", getMatchingDownload)
	server := http.Server{
		Addr:    bindAddress,
		Handler: logger(router),
	}
	go func() {
		log.Printf("starting server on port %s...", bindAddress)
		if err := server.ListenAndServe(); err != nil {
			log.Fatal(err)
		}
	}()
	defer server.Shutdown(nil)
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	<-c
	log.Println("server stopped")
}
<commit_msg>feat: set cache control header<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/christophwitzko/github-release-download/release"
	"github.com/julienschmidt/httprouter"
	"log"
	"net/http"
	"os"
	"os/signal"
	"time"
)

var bindAddress string = ":5000"
var client *release.GithubClient

func init() {
	if ba := os.Getenv("BIND_ADDRESS"); ba != "" {
		bindAddress = ba
	}
	client = release.NewClient(os.Getenv("GITHUB_TOKEN"))
}

func logger(router http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s - %s %s", r.RemoteAddr, r.Method, r.URL.EscapedPath())
		router.ServeHTTP(w, r)
	})
}

func doRedirect(w http.ResponseWriter, r *http.Request, url string, err error) {
	if err != nil {
		log.Println(err)
		http.Error(w, "server error", http.StatusInternalServerError)
		return
	}
	if url == "" {
		http.NotFound(w, r)
		return
	}
	w.Header().Set("Cache-Control", "max-age=300")
	http.Redirect(w, r, url, 302)
}

func getLatestDownload(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
	defer cancel()
	if len(ps) == 3 {
		url, err := client.GetLatestDownloadUrl(ctx, "go-" + ps[0].Value, ps[0].Value, ps[1].Value, ps[2].Value)
		doRedirect(w, r, url, err)
		return
	}
	url, err := client.GetLatestDownloadUrl(ctx, ps[0].Value, ps[1].Value, ps[2].Value, ps[3].Value)
	doRedirect(w, r, url, err)
}

func getMatchingDownload(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
	defer cancel()
	url, err := client.GetMatchingDownloadUrl(ctx, ps[0].Value, ps[1].Value, ps[2].Value, ps[3].Value, ps[4].Value)
	doRedirect(w, r, url, err)
}

func getVersions(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second)
	defer cancel()
	if ps[0].Value == "_" && ps[1].Value == "go" {
		versions, err := client.GetGoVersions(ctx)
		if err != nil {
			log.Println(err)
			http.Error(w, "server error", http.StatusInternalServerError)
			return
		}
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		json.NewEncoder(w).Encode(versions)
		return
	}
	http.NotFound(w, r)
}

func usage(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	if ps[0].Value != "_usage" {
		http.NotFound(w, r)
		return
	}
	rl, _, err := client.Client.RateLimits(r.Context())
	if err != nil {
		log.Println(err)
		http.Error(w, "server error", http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	json.NewEncoder(w).Encode(rl.Core)
}

func index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	fmt.Fprintln(w, "https://get-release.xyz/:owner/:repo/:os/:arch{/:constraint}")
}

func main() {
	router := httprouter.New()
	router.GET("/", index)
	router.GET("/:owner", usage)
	router.GET("/:owner/:repo", getVersions)
	router.GET("/:owner/:repo/:os", getLatestDownload)
	router.GET("/:owner/:repo/:os/:arch", getLatestDownload)
	router.GET("/:owner/:repo/:os/:arch/:constraint", getMatchingDownload)
	server := http.Server{
		Addr:    bindAddress,
		Handler: logger(router),
	}
	go func() {
		log.Printf("starting server on port %s...", bindAddress)
		if err := server.ListenAndServe(); err != nil {
			log.Fatal(err)
		}
	}()
	defer server.Shutdown(nil)
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill)
	<-c
	log.Println("server stopped")
}
<|endoftext|>


<commit_before>package main // import "github.com/tutumcloud/weave-daemon"

import (
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tutumcloud/go-tutum/tutum"
	"github.com/tutumcloud/weave-daemon/nodes"
)

func AttachContainer(c *docker.Client, container_id string) error {
	inspect, err := c.InspectContainer(container_id)

	if err != nil {
		log.Printf("%s: exception when inspecting the container", err)
	}

	cidr := ""
	//log.Println(inspect)
	env_vars := inspect.Config.Env

	for i := range env_vars {
		if strings.HasPrefix(env_vars[i], "TUTUM_IP_ADDRESS=") {
			cidr = env_vars[i][len("TUTUM_IP_ADDRESS="):]
			break
		}
	}

	if cidr != "" {
		tries := 0
		for tries < 3 {

			log.Printf("%s: adding to weave with IP %s", container_id, cidr)
			cmd := exec.Command("/weave", "--local", "attach", cidr, container_id)

			stdout, err := cmd.StdoutPipe()
			if err != nil {
				log.Fatal(err)
			}

			stderr, err := cmd.StderrPipe()
			if err != nil {
				log.Fatal(err)
			}

			if err := cmd.Start(); err != nil {
				log.Fatal(err)
			}

			if err := cmd.Wait(); err != nil {
				log.Printf("%s: %s %s", container_id, stdout, stderr)
				tries++
				time.Sleep(1)
			} else {
				break
			}
		}
	} else {
		log.Printf("%s: cannot find the IP address to add to weave", container_id)
	}

	if err != nil {
		return err
	}

	return nil
}

func ContainerAttachThread(c *docker.Client) {

	listener := make(chan *docker.APIEvents)

	containers, err := c.ListContainers(docker.ListContainersOptions{All: false, Size: true, Limit: 0, Since: "", Before: ""})
	if err != nil {
		log.Fatal(err)
	}

	for _, container := range containers {
		//log.Println(container.ID, container.Names)
		err := AttachContainer(c, container.ID)
		if err != nil {
			log.Println(err)
		}

	}
	err = c.AddEventListener(listener)
	if err != nil {
		log.Fatal(err)
	}

	defer func() {

		err = c.RemoveEventListener(listener)
		if err != nil {
			log.Fatal(err)
		}

	}()

	timeout := time.After(1 * time.Second)

	for {
		select {
		case msg := <-listener:
			//DEBUG
			//log.Print(msg.Status + " " + msg.ID + " " + msg.From)
			if msg.Status == "start" && !strings.HasPrefix(msg.From, "weaveworks/weave") {
				AttachContainer(c, msg.ID)
				//DEBUG
				//fmt.Println("attached")
			}
		case <-timeout:
			break
		}
	}
}

func discovering(ch chan string) {
	c := make(chan tutum.Event)
	nodes.DiscoverPeers(ch)
	go tutum.TutumEvents(c)
	for {
		events := <-c
		nodes.EventHandler(events)
	}
}

func main() {

	//Init client

	//BOOT2DOCKER NEW TLS CLIENT
	/*endpoint := "tcp://192.168.59.103:2376"
	path := os.Getenv("DOCKER_CERT_PATH")
	ca := fmt.Sprintf("%s/ca.pem", path)
	cert := fmt.Sprintf("%s/cert.pem", path)
	key := fmt.Sprintf("%s/key.pem", path)
	client, err := docker.NewTLSClient(endpoint, cert, key, ca)*/
	log.Println("Running main.go")

	endpoint := "unix:///var/run/docker.sock"
	client, err := docker.NewClient(endpoint)

	if err != nil {
		log.Fatal(err)
	}

	node, err := tutum.GetNode(nodes.Tutum_Node_Api_Uri)
	if err != nil {
		log.Fatal(err)
	}
	nodes.Tutum_Node_Public_Ip = node.Public_ip
	log.Printf("This node IP is %s", nodes.Tutum_Node_Public_Ip)
	if os.Getenv("TUTUM_AUTH") != "" {
		log.Println("Detected Tutum API access - starting peer discovery thread")
		log.Println("NODE")
		ch := make(chan string)
		go discovering(ch)
		log.Printf("%s", <-ch)
	}
	log.Println("CONTAINER")
	ContainerAttachThread(client)
}
<commit_msg>add Exit flag<commit_after>package main // import "github.com/tutumcloud/weave-daemon"

import (
	"log"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/tutumcloud/go-tutum/tutum"
	"github.com/tutumcloud/weave-daemon/nodes"
)

func AttachContainer(c *docker.Client, container_id string) error {
	inspect, err := c.InspectContainer(container_id)

	if err != nil {
		log.Printf("%s: exception when inspecting the container", err)
	}

	cidr := ""
	//log.Println(inspect)
	env_vars := inspect.Config.Env

	for i := range env_vars {
		if strings.HasPrefix(env_vars[i], "TUTUM_IP_ADDRESS=") {
			cidr = env_vars[i][len("TUTUM_IP_ADDRESS="):]
			break
		}
	}

	if cidr != "" {
		tries := 0
		for tries < 3 {

			log.Printf("%s: adding to weave with IP %s", container_id, cidr)
			cmd := exec.Command("/weave", "--local", "attach", cidr, container_id)

			stdout, err := cmd.StdoutPipe()
			if err != nil {
				log.Fatal(err)
			}

			stderr, err := cmd.StderrPipe()
			if err != nil {
				log.Fatal(err)
			}

			if err := cmd.Start(); err != nil {
				log.Fatal(err)
			}

			if err := cmd.Wait(); err != nil {
				log.Printf("%s: %s %s", container_id, stdout, stderr)
				tries++
				time.Sleep(1)
			} else {
				break
			}
		}
	} else {
		log.Printf("%s: cannot find the IP address to add to weave", container_id)
	}

	if err != nil {
		return err
	}

	return nil
}

func ContainerAttachThread(c *docker.Client) {

	listener := make(chan *docker.APIEvents)

	containers, err := c.ListContainers(docker.ListContainersOptions{All: false, Size: true, Limit: 0, Since: "", Before: ""})
	if err != nil {
		log.Fatal(err)
	}

	for _, container := range containers {
		//log.Println(container.ID, container.Names)
		err := AttachContainer(c, container.ID)
		if err != nil {
			log.Println(err)
		}

	}
	err = c.AddEventListener(listener)
	if err != nil {
		log.Fatal(err)
	}

	defer func() {

		err = c.RemoveEventListener(listener)
		if err != nil {
			log.Fatal(err)
		}

	}()

	timeout := time.After(1 * time.Second)

	for {
		select {
		case msg := <-listener:
			//DEBUG
			//log.Print(msg.Status + " " + msg.ID + " " + msg.From)
			if msg.Status == "start" && !strings.HasPrefix(msg.From, "weaveworks/weave") {
				AttachContainer(c, msg.ID)
				//DEBUG
				//fmt.Println("attached")
			}
		case <-timeout:
			break
		}
	}
}

func discovering(ch chan string) {
	c := make(chan tutum.Event)
	nodes.DiscoverPeers(ch)
	go tutum.TutumEvents(c)
	for {
		events := <-c
		nodes.EventHandler(events)
	}
}

func main() {

	//Init client

	//BOOT2DOCKER NEW TLS CLIENT
	/*endpoint := "tcp://192.168.59.103:2376"
	path := os.Getenv("DOCKER_CERT_PATH")
	ca := fmt.Sprintf("%s/ca.pem", path)
	cert := fmt.Sprintf("%s/cert.pem", path)
	key := fmt.Sprintf("%s/key.pem", path)
	client, err := docker.NewTLSClient(endpoint, cert, key, ca)*/
	log.Println("Running main.go")

	endpoint := "unix:///var/run/docker.sock"
	client, err := docker.NewClient(endpoint)

	if err != nil {
		log.Fatal(err)
	}

	node, err := tutum.GetNode(nodes.Tutum_Node_Api_Uri)
	if err != nil {
		log.Fatal(err)
	}
	nodes.Tutum_Node_Public_Ip = node.Public_ip
	log.Printf("This node IP is %s", nodes.Tutum_Node_Public_Ip)
	if os.Getenv("TUTUM_AUTH") != "" {
		log.Println("Detected Tutum API access - starting peer discovery thread")
		ch := make(chan string)
		go discovering(ch)
		log.Printf("%s", <-ch)
	}
	log.Println("CONTAINER")
	ContainerAttachThread(client)
	log.Println("EXIT")
}
<|endoftext|>