<commit_before>package fronted

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/tlsdialer"
)

var (
	poolCh      = make(chan *x509.CertPool, 1)
	candidateCh = make(chan *Masquerade)
	masqCh      = make(chan *Masquerade, 1)
)

func Configure(pool *x509.CertPool, masquerades map[string][]*Masquerade) {
	if masquerades == nil || len(masquerades) == 0 {
		log.Errorf("No masquerades!!")
	}

	go func() {
		poolCh <- pool
		for _, arr := range masquerades {
			shuffle(arr)
			for _, m := range arr {
				candidateCh <- m
			}
		}
	}()
}

func shuffle(slc []*Masquerade) {
	n := len(slc)
	for i := 0; i < n; i++ {
		// choose index uniformly in [i, n-1]
		r := i + rand.Intn(n-i)
		slc[r], slc[i] = slc[i], slc[r]
	}
}

func getCertPool() *x509.CertPool {
	pool := <-poolCh
	if len(poolCh) == 0 {
		poolCh <- pool
	}
	return pool
}

type direct struct {
	tlsConfigs      map[string]*tls.Config
	tlsConfigsMutex sync.Mutex
}

func NewDirect() *direct {
	d := &direct{
		tlsConfigs: make(map[string]*tls.Config),
	}
	d.fillMasquerades()
	return d
}

func (d *direct) fillMasquerades() {
	// We loop through the candidate masquerades to find ones that succeed, adding
	// them to a channel of verified masquerades.
	for i := 0; i < 40; i++ {
		go func() {
			m := <-candidateCh
			log.Debugf("Dialing to %v", m)

			// We do the full TLS connection here because in practice the domains at a given IP
			// address can change frequently on CDNs, so the certificate may not match what
			// we expect.
			conn, err := d.dialServerWith(m)
			if err != nil {
				log.Debugf("Could not dial to %v, %v", m.IpAddress, err)
				// Don't re-add this candidate if it's any certificate error, as that
				// will just keep failing and will waste connections. We can't access the underlying
				// error at this point so just look for "certificate".
				if strings.Contains(err.Error(), "certificate") {
					log.Debugf("Continuing on certificate error")
					return
				}
			} else {
				log.Debugf("Got successful connection to: %v", m)
				if err := conn.Close(); err != nil {
					log.Debugf("Could not close body %v", err)
				}
				masqCh <- m
			}
			// We now requeue the masquerade onto the candidate channel
			// so we don't churn through all the candidates. We do this regardless
			// of success or failure because the user being offline could easily cause
			// failures for masquerades that would otherwise succeed.
			go func() {
				candidateCh <- m
			}()
		}()
	}
}

func (d *direct) getMasquerade() *Masquerade {
	// We don't automatically requeue this masquerade into the valid channel because masquerade mappings can change
	// quite frequently to the point that a masquerade that was valid at the beginning of a run may no longer
	// be valid. In particular the certificate served at a given IP address may no longer match the domain
	// we have on record as it matching.
	return <-masqCh
}

// directTransport is a wrapper struct enabling us to modify the protocol of outgoing
// requests to make them all HTTP instead of potentially HTTPS, which breaks our particular
// implemenation of direct domain fronting.
type directTransport struct {
	http.Transport
}

func (ddf *directTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	// The connection is already encrypted by domain fronting.  We need to rewrite URLs starting
	// with "https://" to "http://", lest we get an error for doubling up on TLS.

	// The RoundTrip interface requires that we not modify the memory in the request, so we just
	// create a copy.
	norm := new(http.Request)
	*norm = *req // includes shallow copies of maps, but okay
	norm.URL = new(url.URL)
	*norm.URL = *req.URL
	norm.URL.Scheme = "http"
	return ddf.Transport.RoundTrip(norm)
}

// NewHttpClient creates a new http.Client that does direct domain fronting.
func (d *direct) NewDirectHttpClient() *http.Client {
	trans := &directTransport{}
	trans.Dial = d.Dial
	trans.TLSHandshakeTimeout = 40 * time.Second
	trans.DisableKeepAlives = true
	return &http.Client{
		Transport: trans,
	}
}

// Dial persistently dials masquerades until one succeeds.
func (d *direct) Dial(network, addr string) (net.Conn, error) {
	for i := 0; i < 20; i++ {
		log.Debugf("Dialing %s with direct domain fronter", addr)
		conn, err := d.dialServerWith(d.getMasquerade())
		if err != nil {
			log.Debugf("Error dialing? %v", err)
			continue
		} else {
			log.Debugf("Got connection to %v!", conn.RemoteAddr().String())
		}
		return conn, err
	}
	return nil, errors.New("Could not dial any masquerade?")
}

func (d *direct) dialServerWith(masquerade *Masquerade) (net.Conn, error) {
	tlsConfig := d.tlsConfig(masquerade)
	dialTimeout := 30 * time.Second
	sendServerNameExtension := false

	cwt, err := tlsdialer.DialForTimings(
		&net.Dialer{
			Timeout: dialTimeout,
		},
		"tcp",
		masquerade.IpAddress+":443",
		sendServerNameExtension, // SNI or no
		tlsConfig)

	if err != nil && masquerade != nil {
		err = fmt.Errorf("Unable to dial masquerade %s: %s", masquerade.Domain, err)
	}
	return cwt.Conn, err
}

// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *direct) tlsConfig(m *Masquerade) *tls.Config {
	d.tlsConfigsMutex.Lock()
	defer d.tlsConfigsMutex.Unlock()

	tlsConfig := d.tlsConfigs[m.Domain]
	if tlsConfig == nil {
		tlsConfig = &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(1000),
			InsecureSkipVerify: false,
			ServerName:         m.Domain,
			RootCAs:            getCertPool(),
		}
		d.tlsConfigs[m.Domain] = tlsConfig
	}

	return tlsConfig
}
<commit_msg>Changed to just iterate over connections<commit_after>package fronted

import (
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/tlsdialer"
)

var (
	poolCh      = make(chan *x509.CertPool, 1)
	candidateCh = make(chan *Masquerade, 1)
	masqCh      = make(chan *Masquerade, 1)
)

func Configure(pool *x509.CertPool, masquerades map[string][]*Masquerade) {
	if masquerades == nil || len(masquerades) == 0 {
		log.Errorf("No masquerades!!")
	}

	go func() {

		poolCh <- pool
		size := 0
		for _, arr := range masquerades {
			shuffle(arr)
			size += len(arr)
		}

		// Make an unblocke channel the same size as our group
		// of masquerades and push all of them into it.
		candidateCh = make(chan *Masquerade, size)
		for _, arr := range masquerades {
			for _, m := range arr {
				candidateCh <- m
			}
		}
	}()
}

func shuffle(slc []*Masquerade) {
	n := len(slc)
	for i := 0; i < n; i++ {
		// choose index uniformly in [i, n-1]
		r := i + rand.Intn(n-i)
		slc[r], slc[i] = slc[i], slc[r]
	}
}

func getCertPool() *x509.CertPool {
	pool := <-poolCh
	if len(poolCh) == 0 {
		poolCh <- pool
	}
	return pool
}

type direct struct {
	tlsConfigs      map[string]*tls.Config
	tlsConfigsMutex sync.Mutex
}

func NewDirect() *direct {
	d := &direct{
		tlsConfigs: make(map[string]*tls.Config),
	}
	return d
}

// directTransport is a wrapper struct enabling us to modify the protocol of outgoing
// requests to make them all HTTP instead of potentially HTTPS, which breaks our particular
// implemenation of direct domain fronting.
type directTransport struct {
	http.Transport
}

func (ddf *directTransport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
	// The connection is already encrypted by domain fronting.  We need to rewrite URLs starting
	// with "https://" to "http://", lest we get an error for doubling up on TLS.

	// The RoundTrip interface requires that we not modify the memory in the request, so we just
	// create a copy.
	norm := new(http.Request)
	*norm = *req // includes shallow copies of maps, but okay
	norm.URL = new(url.URL)
	*norm.URL = *req.URL
	norm.URL.Scheme = "http"
	return ddf.Transport.RoundTrip(norm)
}

// NewHttpClient creates a new http.Client that does direct domain fronting.
func (d *direct) NewDirectHttpClient() *http.Client {
	trans := &directTransport{}
	trans.Dial = d.Dial
	trans.TLSHandshakeTimeout = 40 * time.Second
	trans.DisableKeepAlives = true
	return &http.Client{
		Transport: trans,
	}
}

// Dial persistently dials masquerades until one succeeds.
func (d *direct) Dial(network, addr string) (net.Conn, error) {
	for i := 0; i < 40; i++ {
		m := <-candidateCh
		log.Debugf("Dialing to %v", m)

		// We do the full TLS connection here because in practice the domains at a given IP
		// address can change frequently on CDNs, so the certificate may not match what
		// we expect.
		conn, err := d.dialServerWith(m)
		if err != nil {
			log.Debugf("Could not dial to %v, %v", m.IpAddress, err)
			// Don't re-add this candidate if it's any certificate error, as that
			// will just keep failing and will waste connections. We can't access the underlying
			// error at this point so just look for "certificate".
			if strings.Contains(err.Error(), "certificate") {
				log.Debugf("Continuing on certificate error")
			} else {
				candidateCh <- m
			}
		} else {
			log.Debugf("Got successful connection to: %v", m)
			candidateCh <- m
			return conn, nil
		}
	}
	return nil, errors.New("Could not dial any masquerade?")
}

func (d *direct) dialServerWith(masquerade *Masquerade) (net.Conn, error) {
	tlsConfig := d.tlsConfig(masquerade)
	dialTimeout := 30 * time.Second
	sendServerNameExtension := false

	cwt, err := tlsdialer.DialForTimings(
		&net.Dialer{
			Timeout: dialTimeout,
		},
		"tcp",
		masquerade.IpAddress+":443",
		sendServerNameExtension, // SNI or no
		tlsConfig)

	if err != nil && masquerade != nil {
		err = fmt.Errorf("Unable to dial masquerade %s: %s", masquerade.Domain, err)
	}
	return cwt.Conn, err
}

// tlsConfig builds a tls.Config for dialing the upstream host. Constructed
// tls.Configs are cached on a per-masquerade basis to enable client session
// caching and reduce the amount of PEM certificate parsing.
func (d *direct) tlsConfig(m *Masquerade) *tls.Config {
	d.tlsConfigsMutex.Lock()
	defer d.tlsConfigsMutex.Unlock()

	tlsConfig := d.tlsConfigs[m.Domain]
	if tlsConfig == nil {
		tlsConfig = &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(1000),
			InsecureSkipVerify: false,
			ServerName:         m.Domain,
			RootCAs:            getCertPool(),
		}
		d.tlsConfigs[m.Domain] = tlsConfig
	}

	return tlsConfig
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/gempir/justlog/config"
)

func (s *Server) authenticateAdmin(w http.ResponseWriter, r *http.Request) bool {
	apiKey := r.Header.Get("X-Api-Key")

	if apiKey == "" || apiKey != s.cfg.AdminAPIKey {
		http.Error(w, "No I don't think so.", http.StatusForbidden)
		return false
	}

	return true
}

type channelConfigsDeleteRequest struct {
	MessageTypes bool `json:"messageTypes,omitempty"`
}

type channelConfigsRequest struct {
	config.ChannelConfig
}

func (s *Server) writeChannelConfigs(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost && r.Method != http.MethodDelete {
		http.Error(w, "We'll see, we'll see. The winner gets tea.", http.StatusMethodNotAllowed)
		return
	}

	channelID := strings.TrimPrefix(r.URL.String(), "/admin/channelConfigs/")

	if _, ok := s.cfg.ChannelConfigs[channelID]; !ok {
		http.Error(w, "Uhhhhhh... unkown channel", http.StatusBadRequest)
		return
	}

	if r.Method == http.MethodDelete {
		var request channelConfigsDeleteRequest

		err := json.NewDecoder(r.Body).Decode(&request)
		if err != nil {
			http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
			return
		}

		if request.MessageTypes {
			s.cfg.ResetMessageTypes(channelID)
			s.bot.UpdateMessageTypesToLog()
			writeJSON("Doubters? Reset "+channelID+" messageTypes", http.StatusOK, w, r)
			return
		}

	}

	var request channelConfigsRequest

	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
		return
	}

	s.cfg.SetMessageTypes(channelID, request.MessageTypes)
	s.bot.UpdateMessageTypesToLog()

	writeJSON("Doubters? Updated "+channelID+" messageTypes to "+fmt.Sprintf("%v", request.MessageTypes), http.StatusOK, w, r)
}

type channelsDeleteRequest struct {
	Channels []string `json:"channels"`
}

type channelConfigsJoinRequest struct {
	Channels []string `json:"channels"`
}

func (s *Server) writeChannels(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost && r.Method != http.MethodDelete {
		http.Error(w, "We'll see, we'll see. The winner gets tea.", http.StatusMethodNotAllowed)
		return
	}

	if r.Method == http.MethodDelete {
		var request channelsDeleteRequest

		err := json.NewDecoder(r.Body).Decode(&request)
		if err != nil {
			http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
			return
		}

		s.cfg.RemoveChannels(request.Channels...)
		data, err := s.helixClient.GetUsersByUserIds(request.Channels)
		if err != nil {
			http.Error(w, "Failed to get channel names to leave, config might be already updated", http.StatusInternalServerError)
			return
		}
		for _, userData := range data {
			s.bot.Depart(userData.Login)
		}

		writeJSON(fmt.Sprintf("Doubters? Removed channels %v", request.Channels), http.StatusOK, w, r)
		return
	}

	var request channelConfigsJoinRequest

	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
		return
	}

	s.cfg.AddChannels(request.Channels...)
	data, err := s.helixClient.GetUsersByUserIds(request.Channels)
	if err != nil {
		http.Error(w, "Failed to get channel names to join, config might be already updated", http.StatusInternalServerError)
		return
	}
	for _, userData := range data {
		s.bot.Join(userData.Login)
	}

	writeJSON(fmt.Sprintf("Doubters? Joined channels %v", request.Channels), http.StatusOK, w, r)
}
<commit_msg>end request when bad<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/gempir/justlog/config"
)

func (s *Server) authenticateAdmin(w http.ResponseWriter, r *http.Request) bool {
	apiKey := r.Header.Get("X-Api-Key")

	if apiKey == "" || apiKey != s.cfg.AdminAPIKey {
		http.Error(w, "No I don't think so.", http.StatusForbidden)
		return false
	}

	return true
}

type channelConfigsDeleteRequest struct {
	MessageTypes bool `json:"messageTypes,omitempty"`
}

type channelConfigsRequest struct {
	config.ChannelConfig
}

func (s *Server) writeChannelConfigs(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost && r.Method != http.MethodDelete {
		http.Error(w, "We'll see, we'll see. The winner gets tea.", http.StatusMethodNotAllowed)
		return
	}

	channelID := strings.TrimPrefix(r.URL.String(), "/admin/channelConfigs/")

	if _, ok := s.cfg.ChannelConfigs[channelID]; !ok {
		http.Error(w, "Uhhhhhh... unkown channel", http.StatusBadRequest)
		return
	}

	if r.Method == http.MethodDelete {
		var request channelConfigsDeleteRequest

		err := json.NewDecoder(r.Body).Decode(&request)
		if err != nil {
			http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
			return
		}

		if request.MessageTypes {
			s.cfg.ResetMessageTypes(channelID)
			s.bot.UpdateMessageTypesToLog()
			writeJSON("Doubters? Reset "+channelID+" messageTypes", http.StatusOK, w, r)
			return
		}

		http.Error(w, "Uhhhhhh...", http.StatusBadRequest)
		return
	}

	var request channelConfigsRequest

	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
		return
	}

	s.cfg.SetMessageTypes(channelID, request.MessageTypes)
	s.bot.UpdateMessageTypesToLog()

	writeJSON("Doubters? Updated "+channelID+" messageTypes to "+fmt.Sprintf("%v", request.MessageTypes), http.StatusOK, w, r)
}

type channelsDeleteRequest struct {
	Channels []string `json:"channels"`
}

type channelConfigsJoinRequest struct {
	Channels []string `json:"channels"`
}

func (s *Server) writeChannels(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost && r.Method != http.MethodDelete {
		http.Error(w, "We'll see, we'll see. The winner gets tea.", http.StatusMethodNotAllowed)
		return
	}

	if r.Method == http.MethodDelete {
		var request channelsDeleteRequest

		err := json.NewDecoder(r.Body).Decode(&request)
		if err != nil {
			http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
			return
		}

		s.cfg.RemoveChannels(request.Channels...)
		data, err := s.helixClient.GetUsersByUserIds(request.Channels)
		if err != nil {
			http.Error(w, "Failed to get channel names to leave, config might be already updated", http.StatusInternalServerError)
			return
		}
		for _, userData := range data {
			s.bot.Depart(userData.Login)
		}

		writeJSON(fmt.Sprintf("Doubters? Removed channels %v", request.Channels), http.StatusOK, w, r)
		return
	}

	var request channelConfigsJoinRequest

	err := json.NewDecoder(r.Body).Decode(&request)
	if err != nil {
		http.Error(w, "ANYWAYS: "+err.Error(), http.StatusBadRequest)
		return
	}

	s.cfg.AddChannels(request.Channels...)
	data, err := s.helixClient.GetUsersByUserIds(request.Channels)
	if err != nil {
		http.Error(w, "Failed to get channel names to join, config might be already updated", http.StatusInternalServerError)
		return
	}
	for _, userData := range data {
		s.bot.Join(userData.Login)
	}

	writeJSON(fmt.Sprintf("Doubters? Joined channels %v", request.Channels), http.StatusOK, w, r)
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/muesli/polly/api/config"

	"github.com/lib/pq"
	"github.com/muesli/cache2go"
	uuid "github.com/nu7hatch/gouuid"
)

var (
	pgDB   *sql.DB
	pgConn config.PostgreSQLConnection

	proposalsCache = cache2go.Cache("track")
	usersCache     = cache2go.Cache("user")

	// ErrInvalidID is the error returned when encountering an invalid database ID
	ErrInvalidID = errors.New("Invalid id")
)

// SetupPostgres sets the db configuration
func SetupPostgres(pc config.PostgreSQLConnection) {
	pgConn = pc
}

// GetDatabase connects to the database on first run and returns the existing
// connection on further calls
func GetDatabase() *sql.DB {
	if pgDB == nil {
		var err error
		pgDB, err = sql.Open("postgres", pgConn.Marshal())
		if err != nil {
			panic(err)
		}

		tables := []string{
			`CREATE TABLE IF NOT EXISTS users
				(
				  id          	bigserial 	PRIMARY KEY,
				  username    	text      	NOT NULL,
				  password		text		NOT NULL,
				  about       	text,
				  email       	text		NOT NULL,
				  activated   	bool		DEFAULT false,
				  authtoken   	text[]     	NOT NULL,
				  CONSTRAINT  	uk_username	UNIQUE (username),
				  CONSTRAINT  	uk_email 	UNIQUE (email)
				)`,
			`CREATE TABLE IF NOT EXISTS proposals
				(
				  id          	bigserial 	PRIMARY KEY,
				  userid      	bigserial 	NOT NULL,
				  title       	text      	NOT NULL,
				  description	text      	NOT NULL,
				  activities	text      	NOT NULL,
				  contact		text      	NOT NULL,
				  recipient		text		NOT NULL,
				  recipient2	text		NOT NULL,
				  value			int			NOT NULL,
				  starts		timestamp	NOT NULL,
				  votes	      	int       	DEFAULT 0,
				  moderated     bool        DEFAULT false,
				  CONSTRAINT  	fk_user		FOREIGN KEY (userid) REFERENCES users (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE
				)`,
			`CREATE TABLE IF NOT EXISTS votes
				(
				  id          	bigserial			PRIMARY KEY,
				  userid    	bigserial			NOT NULL,
				  proposalid   	bigserial			NOT NULL,
				  vote			bool				NOT NULL,
				  CONSTRAINT  	uk_user_proposal	UNIQUE (userid, proposalid),
				  CONSTRAINT  	fk_user				FOREIGN KEY (userid) REFERENCES users (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE,
				  CONSTRAINT  	fk_proposal			FOREIGN KEY (proposalid) REFERENCES proposals (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE
				)`,
			`CREATE TABLE IF NOT EXISTS mailman
				(
				  mailbox      	text				PRIMARY KEY,
				  lastseen    	bigserial			NOT NULL
				)`,
		}

		// FIXME: add IF NOT EXISTS to CREATE INDEX statements (coming in v9.5)
		// See: http://www.postgresql.org/docs/devel/static/sql-createindex.html
		indexes := []string{
			`CREATE INDEX idx_users_email ON users(email)`,
			`CREATE INDEX idx_users_authtoken ON users(authtoken)`,
			`CREATE INDEX idx_proposals_moderated ON proposals(moderated)`,
			`CREATE INDEX idx_proposals_value ON proposals(value)`,
			`CREATE INDEX idx_proposals_userid ON proposals(userid)`,
			`CREATE INDEX idx_proposals_starts ON proposals(starts)`,
			`CREATE INDEX idx_votes_userid ON votes(userid)`,
			`CREATE INDEX idx_votes_proposalid ON votes(proposalid)`,
		}

		for _, v := range tables {
			fmt.Println("Creating table:", v)
			_, err = pgDB.Exec(v)
			if err != nil {
				panic(err)
			}
		}
		for _, v := range indexes {
			fmt.Println("Creating index:", v)
			_, err = pgDB.Exec(v)
			if err != nil && strings.Index(err.Error(), "already exists") < 0 {
				fmt.Println("Error:", err)
			}
		}

	}

	return pgDB
}

// WipeDatabase drops all database tables - use carefully!
func WipeDatabase() {
	// Commented out to prevent accidental usage

	/*
		drops := []string{
			`DROP TABLE votes`,
			`DROP TABLE proposals`,
			`DROP TABLE users`,
		}

		for _, v := range drops {
			fmt.Println("Dropping table:", v)
			_, err := pgDB.Exec(v)
			if err != nil {
				panic(err)
			}
		}
	*/
}

func init() {
	fmt.Println("db.init")
	initCaches()

	negativeInf := time.Time{}
	positiveInf, _ := time.Parse("2006", "3000")

	pq.EnableInfinityTs(negativeInf, positiveInf)
}

// UUID returns a new unique identifier
func UUID() (string, error) {
	u, err := uuid.NewV4()
	if err != nil {
		return "", err
	}

	uuid := strings.Join(strings.Split(u.String(), "-"), "")
	return uuid, nil
}

func initCaches() {
	usersCache.SetAddedItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Now in users-cache:", item.Key().(string), item.Data().(*DbUser).Username)
	})
	usersCache.SetAboutToDeleteItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Deleting from users-cache:", item.Key().(string), item.Data().(*DbUser).Username, item.CreatedOn())
	})
	usersCache.SetDataLoader(func(key interface{}, args ...interface{}) *cache2go.CacheItem {
		if len(args) == 1 {
			if context, ok := args[0].(*PollyContext); ok {
				user, err := context.LoadUserByID(key.(int64))
				if err != nil {
					fmt.Println("usersCache ERROR for key", key, ":", err)
					return nil
				}

				entry := cache2go.NewCacheItem(key, 10*time.Minute, &user)
				return entry
			}
		}
		fmt.Println("Got no APIContext passed in")
		return nil
	})

	proposalsCache.SetAddedItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Now in proposals-cache:", item.Key().(string), item.Data().(*DbProposal).Title)
	})
	proposalsCache.SetAboutToDeleteItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Deleting from proposals-cache:", item.Key().(string), item.Data().(*DbProposal).Title, item.CreatedOn())
	})
	proposalsCache.SetDataLoader(func(key interface{}, args ...interface{}) *cache2go.CacheItem {
		if len(args) == 1 {
			if context, ok := args[0].(*PollyContext); ok {
				proposal, err := context.LoadProposalByID(key.(int64))
				if err != nil {
					fmt.Println("proposalsCache ERROR for key", key, ":", err)
					return nil
				}

				entry := cache2go.NewCacheItem(key, 10*time.Minute, &proposal)
				return entry
			}
		}
		fmt.Println("Got no APIContext passed in")
		return nil
	})
}
<commit_msg>Allow secondary recipient to be null<commit_after>package db

import (
	"database/sql"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/muesli/polly/api/config"

	"github.com/lib/pq"
	"github.com/muesli/cache2go"
	uuid "github.com/nu7hatch/gouuid"
)

var (
	pgDB   *sql.DB
	pgConn config.PostgreSQLConnection

	proposalsCache = cache2go.Cache("track")
	usersCache     = cache2go.Cache("user")

	// ErrInvalidID is the error returned when encountering an invalid database ID
	ErrInvalidID = errors.New("Invalid id")
)

// SetupPostgres sets the db configuration
func SetupPostgres(pc config.PostgreSQLConnection) {
	pgConn = pc
}

// GetDatabase connects to the database on first run and returns the existing
// connection on further calls
func GetDatabase() *sql.DB {
	if pgDB == nil {
		var err error
		pgDB, err = sql.Open("postgres", pgConn.Marshal())
		if err != nil {
			panic(err)
		}

		tables := []string{
			`CREATE TABLE IF NOT EXISTS users
				(
				  id          	bigserial 	PRIMARY KEY,
				  username    	text      	NOT NULL,
				  password		text		NOT NULL,
				  about       	text,
				  email       	text		NOT NULL,
				  activated   	bool		DEFAULT false,
				  authtoken   	text[]     	NOT NULL,
				  CONSTRAINT  	uk_username	UNIQUE (username),
				  CONSTRAINT  	uk_email 	UNIQUE (email)
				)`,
			`CREATE TABLE IF NOT EXISTS proposals
				(
				  id          	bigserial 	PRIMARY KEY,
				  userid      	bigserial 	NOT NULL,
				  title       	text      	NOT NULL,
				  description	text      	NOT NULL,
				  activities	text      	NOT NULL,
				  contact		text      	NOT NULL,
				  recipient		text		NOT NULL,
				  recipient2	text,
				  value			int			NOT NULL,
				  starts		timestamp	NOT NULL,
				  votes	      	int       	DEFAULT 0,
				  moderated     bool        DEFAULT false,
				  CONSTRAINT  	fk_user		FOREIGN KEY (userid) REFERENCES users (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE
				)`,
			`CREATE TABLE IF NOT EXISTS votes
				(
				  id          	bigserial			PRIMARY KEY,
				  userid    	bigserial			NOT NULL,
				  proposalid   	bigserial			NOT NULL,
				  vote			bool				NOT NULL,
				  CONSTRAINT  	uk_user_proposal	UNIQUE (userid, proposalid),
				  CONSTRAINT  	fk_user				FOREIGN KEY (userid) REFERENCES users (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE,
				  CONSTRAINT  	fk_proposal			FOREIGN KEY (proposalid) REFERENCES proposals (id) MATCH SIMPLE ON UPDATE CASCADE ON DELETE CASCADE
				)`,
			`CREATE TABLE IF NOT EXISTS mailman
				(
				  mailbox      	text				PRIMARY KEY,
				  lastseen    	bigserial			NOT NULL
				)`,
		}

		// FIXME: add IF NOT EXISTS to CREATE INDEX statements (coming in v9.5)
		// See: http://www.postgresql.org/docs/devel/static/sql-createindex.html
		indexes := []string{
			`CREATE INDEX idx_users_email ON users(email)`,
			`CREATE INDEX idx_users_authtoken ON users(authtoken)`,
			`CREATE INDEX idx_proposals_moderated ON proposals(moderated)`,
			`CREATE INDEX idx_proposals_value ON proposals(value)`,
			`CREATE INDEX idx_proposals_userid ON proposals(userid)`,
			`CREATE INDEX idx_proposals_starts ON proposals(starts)`,
			`CREATE INDEX idx_votes_userid ON votes(userid)`,
			`CREATE INDEX idx_votes_proposalid ON votes(proposalid)`,
		}

		for _, v := range tables {
			fmt.Println("Creating table:", v)
			_, err = pgDB.Exec(v)
			if err != nil {
				panic(err)
			}
		}
		for _, v := range indexes {
			fmt.Println("Creating index:", v)
			_, err = pgDB.Exec(v)
			if err != nil && strings.Index(err.Error(), "already exists") < 0 {
				fmt.Println("Error:", err)
			}
		}

	}

	return pgDB
}

// WipeDatabase drops all database tables - use carefully!
func WipeDatabase() {
	// Commented out to prevent accidental usage

	/*
		drops := []string{
			`DROP TABLE votes`,
			`DROP TABLE proposals`,
			`DROP TABLE users`,
		}

		for _, v := range drops {
			fmt.Println("Dropping table:", v)
			_, err := pgDB.Exec(v)
			if err != nil {
				panic(err)
			}
		}
	*/
}

func init() {
	fmt.Println("db.init")
	initCaches()

	negativeInf := time.Time{}
	positiveInf, _ := time.Parse("2006", "3000")

	pq.EnableInfinityTs(negativeInf, positiveInf)
}

// UUID returns a new unique identifier
func UUID() (string, error) {
	u, err := uuid.NewV4()
	if err != nil {
		return "", err
	}

	uuid := strings.Join(strings.Split(u.String(), "-"), "")
	return uuid, nil
}

func initCaches() {
	usersCache.SetAddedItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Now in users-cache:", item.Key().(string), item.Data().(*DbUser).Username)
	})
	usersCache.SetAboutToDeleteItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Deleting from users-cache:", item.Key().(string), item.Data().(*DbUser).Username, item.CreatedOn())
	})
	usersCache.SetDataLoader(func(key interface{}, args ...interface{}) *cache2go.CacheItem {
		if len(args) == 1 {
			if context, ok := args[0].(*PollyContext); ok {
				user, err := context.LoadUserByID(key.(int64))
				if err != nil {
					fmt.Println("usersCache ERROR for key", key, ":", err)
					return nil
				}

				entry := cache2go.NewCacheItem(key, 10*time.Minute, &user)
				return entry
			}
		}
		fmt.Println("Got no APIContext passed in")
		return nil
	})

	proposalsCache.SetAddedItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Now in proposals-cache:", item.Key().(string), item.Data().(*DbProposal).Title)
	})
	proposalsCache.SetAboutToDeleteItemCallback(func(item *cache2go.CacheItem) {
		// fmt.Println("Deleting from proposals-cache:", item.Key().(string), item.Data().(*DbProposal).Title, item.CreatedOn())
	})
	proposalsCache.SetDataLoader(func(key interface{}, args ...interface{}) *cache2go.CacheItem {
		if len(args) == 1 {
			if context, ok := args[0].(*PollyContext); ok {
				proposal, err := context.LoadProposalByID(key.(int64))
				if err != nil {
					fmt.Println("proposalsCache ERROR for key", key, ":", err)
					return nil
				}

				entry := cache2go.NewCacheItem(key, 10*time.Minute, &proposal)
				return entry
			}
		}
		fmt.Println("Got no APIContext passed in")
		return nil
	})
}
<|endoftext|>


<commit_before>// Project Gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package api

import (
	"net/http"
	"encoding/json"
	"github.com/supme/gonder/models"
	"encoding/base64"
	"io/ioutil"
	"time"
	"os"
	"encoding/csv"
	"path"
	"fmt"
	"github.com/tealeg/xlsx"
)

type Recipient struct {
	Id   int64 `json:"recid"`
	Name string `json:"name"`
	Email string `json:"email"`
	Result string `json:"result"`
}

type Recipients struct {
	Total	    int `json:"total"`
	Records		[]Recipient `json:"records"`
}

type RecipientParam struct  {
	Key string `json:"key"`
	Value string `json:"value"`
}

type RecipientParams struct {
	Total	    int `json:"total"`
	Records		[]RecipientParam `json:"records"`
}

func recipients(w http.ResponseWriter, r *http.Request)  {
	var err error
	var js []byte

	if err = r.ParseForm(); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	if r.Form["content"][0] == "recipients" {
		switch r.Form["cmd"][0] {
		case "get-records":
			if auth.Right("get-recipients") && auth.CampaignRightString(r.Form["campaign"][0]) {
				rs, err := getRecipients( r.Form["campaign"][0], r.Form["offset"][0], r.Form["limit"][0])
				js, err = json.Marshal(rs)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			}  else {
				js = []byte(`{"status": "error", "message": "Forbidden get recipients"}`)
			}
			break

		case "upload":
			if auth.Right("upload-recipients") && auth.CampaignRightString(r.Form["campaign"][0]) {
				content, err := base64.StdEncoding.DecodeString(r.FormValue("base64"))
				if err != nil {
					js = []byte(`{"status": "error", "message": "Base64 decode"}`)
				}
				file := models.FromRootDir("tmp/" + time.Now().String())
				err = ioutil.WriteFile(file, content, 0644)
				if err != nil {
					js = []byte(`{"status": "error", "message": "Write file"}`)
				}

				if path.Ext(r.Form["name"][0]) == ".csv" {
					fmt.Println("this csv file")
					err = recipientCsv(r.FormValue("campaign"), file)
					if err != nil {
						js = []byte(`{"status": "error", "message": "Add recipients csv"}`)
					}
				} else if path.Ext(r.Form["name"][0]) == ".xlsx" {
					fmt.Println("this xlsx file")
					err = recipientXlsx(r.FormValue("campaign"), file)
					if err != nil {
						js = []byte(`{"status": "error", "message": "Add recipients xlsx"}`)
					}
				} else {
					fmt.Println("this other file")
					js = []byte(`{"status": "error", "message": "This not csv or xlsx file"}`)
				}

			} else {
				js = []byte(`{"status": "error", "message": "Forbidden upload recipients"}`)
			}
			break

		case "deleteAll":
			if auth.Right("delete-recipients") && auth.CampaignRightString(r.Form["campaign"][0]) {
				err = delRecipients(r.Form["campaign"][0])
				if err != nil {
					js = []byte(`{"status": "error", "message": "Can't delete all recipients"}`)
				}
			} else {
				js = []byte(`{"status": "error", "message": "Forbidden delete recipients"}`)
			}
		}
	}

	if r.Form["content"][0] == "parameters" {
		switch r.Form["cmd"][0] {
		case "get-records":
			rId, err := getRecipientCampaign(r.Form["recipient"][0])
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			if auth.Right("get-recipient-parameters") && auth.CampaignRight(rId) {
				ps, err := getRecipientParams( r.Form["recipient"][0], r.Form["offset"][0], r.Form["limit"][0])
				js, err = json.Marshal(ps)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			}  else {
				js = []byte(`{"status": "error", "message": "Forbidden get recipient parameters"}`)
			}

			break
		}
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(js)
}

func getRecipientCampaign(recipientId string) (int64, error){
	var id int64
	err := models.Db.QueryRow("SELECT `campaign_id` FROM `recipient` WHERE `id`=?", recipientId).Scan(&id)
	return id, err
}

//ToDo check right errors
func getRecipients(campaign, offset, limit string) (Recipients, error) {
	var err error
	var r Recipient
	var rs Recipients
	rs.Records = []Recipient{}
	query, err := models.Db.Query("SELECT `id`, `name`, `email`, `status` FROM `recipient` WHERE `removed`!=1 AND `campaign_id`=?  LIMIT ? OFFSET ?", campaign, limit, offset)
	if err != nil {
		return rs, err
	}
	defer query.Close()
	for query.Next() {
		err = query.Scan(&r.Id, &r.Name, &r.Email, &r.Result)

		rs.Records = append(rs.Records, r)
	}
	err = models.Db.QueryRow("SELECT COUNT(*) FROM `recipient` WHERE `removed`!=1 AND `campaign_id`=?", campaign).Scan(&rs.Total)
	return rs, nil

}

//ToDo check right errors
func getRecipientParams(recipient, offset, limit string) (RecipientParams, error) {
	var err error
	var p RecipientParam
	var ps RecipientParams
	ps.Records = []RecipientParam{}
	query, err := models.Db.Query("SELECT `key`, `value` FROM `parameter` WHERE `recipient_id`=? LIMIT ? OFFSET ?", recipient, limit, offset)
	if err != nil {
		return ps, err
	}
	defer query.Close()
	for query.Next() {
		err = query.Scan(&p.Key, &p.Value)
		ps.Records = append(ps.Records, p)
	}
	err = models.Db.QueryRow("SELECT COUNT(*) FROM `parameter` WHERE `recipient_id`=?", recipient).Scan(&ps.Total)
	return ps, err
}

func delRecipients(campaignId string) error {
	_, err := models.Db.Exec("UPDATE `recipient` SET `removed`=1 WHERE `campaign_id`=?", campaignId)
	return err
}

// ToDo optimize this
func recipientCsv(campaignId string, file string) error {

	title := make(map[int]string)

	csvfile, err := os.Open(file)
	if err != nil {
		return err
	}

	reader := csv.NewReader(csvfile)
	reader.FieldsPerRecord = -1
	rawCSVdata, err := reader.ReadAll()
	if err != nil {
		return err
	}
	for k, v := range rawCSVdata {
		if k == 0 {
			for i, t := range v {
				title[i] = t
			}
		} else {
			email := ""
			name := ""
			data := make(map[string]string)
			for i, t := range v {
				if i == 0 {
					email = t
				} else if i == 1 {
					name = t
				} else {
					data[title[i]] = t
				}
			}

			res, err := models.Db.Exec("INSERT INTO recipient (`campaign_id`, `email`, `name`) VALUES (?, ?, ?)", campaignId, email, name)
			if err != nil {
				return err
			}
			id, err := res.LastInsertId()
			if err != nil {
				return err
			}
			for i, t := range data {
				_, err := models.Db.Exec("INSERT INTO parameter (`recipient_id`, `key`, `value`) VALUES (?, ?, ?)", id, i, t)
				if err != nil {
					return err
				}
			}
		}
	}

	csvfile.Close()

	os.Remove(file)

	return err
}

func recipientXlsx(campaignId string, file string) error {

	title := make(map[int]string)
	var email, name string

	xlFile, err := xlsx.OpenFile(file)
	if err != nil {
		return err
	}

	if xlFile.Sheets[0] != nil {
		for k, v := range xlFile.Sheets[0].Rows {
			if k == 0 {
				for i, cell := range v.Cells {
					t, err := cell.String()
					if err != nil {
						apilog.Println(err)
					}
					title[i] = t
				}
			} else {
				email = ""
				name = ""
				data := make(map[string]string)
				for i, cell := range v.Cells {
					t, err := cell.String()
					if err != nil {
						apilog.Println(err)
					}
					if i == 0 {
						email = t
					} else if i == 1 {
						name = t
					} else {
						data[title[i]] = t
					}
				}

				res, err := models.Db.Exec("INSERT INTO recipient (`campaign_id`, `email`, `name`) VALUES (?, ?, ?)", campaignId, email, name)
				if err != nil {
					return err
				}
				id, err := res.LastInsertId()
				if err != nil {
					return err
				}
				for i, t := range data {
					_, err := models.Db.Exec("INSERT INTO parameter (`recipient_id`, `key`, `value`) VALUES (?, ?, ?)", id, i, t)
					if err != nil {
						return err
					}
				}

			}
		}
	}

	os.Remove(file)
	return err
}<commit_msg>bug fix upload recipients (memory leak)<commit_after>// Project Gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package api

import (
	"net/http"
	"encoding/json"
	"github.com/supme/gonder/models"
	"encoding/base64"
	"io/ioutil"
	"time"
	"os"
	"encoding/csv"
	"path"
	"fmt"
	"github.com/tealeg/xlsx"
)

type Recipient struct {
	Id   int64 `json:"recid"`
	Name string `json:"name"`
	Email string `json:"email"`
	Result string `json:"result"`
}

type Recipients struct {
	Total	    int `json:"total"`
	Records		[]Recipient `json:"records"`
}

type RecipientParam struct  {
	Key string `json:"key"`
	Value string `json:"value"`
}

type RecipientParams struct {
	Total	    int `json:"total"`
	Records		[]RecipientParam `json:"records"`
}

func recipients(w http.ResponseWriter, r *http.Request)  {
	var err error
	var js []byte

	if err = r.ParseForm(); err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	if r.Form["content"][0] == "recipients" {
		switch r.Form["cmd"][0] {
		case "get-records":
			if auth.Right("get-recipients") && auth.CampaignRightString(r.Form["campaign"][0]) {
				rs, err := getRecipients( r.Form["campaign"][0], r.Form["offset"][0], r.Form["limit"][0])
				js, err = json.Marshal(rs)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			}  else {
				js = []byte(`{"status": "error", "message": "Forbidden get recipients"}`)
			}
			break

		case "upload":
			if auth.Right("upload-recipients") && auth.CampaignRightString(r.Form["campaign"][0]) {
				content, err := base64.StdEncoding.DecodeString(r.FormValue("base64"))
				if err != nil {
					js = []byte(`{"status": "error", "message": "Base64 decode"}`)
				}
				file := models.FromRootDir("tmp/" + time.Now().String())
				err = ioutil.WriteFile(file, content, 0644)
				if err != nil {
					js = []byte(`{"status": "error", "message": "Write file"}`)
				}

				if path.Ext(r.Form["name"][0]) == ".csv" {
					fmt.Println("this csv file")
					err = recipientCsv(r.FormValue("campaign"), file)
					if err != nil {
						js = []byte(`{"status": "error", "message": "Add recipients csv"}`)
					}
				} else if path.Ext(r.Form["name"][0]) == ".xlsx" {
					fmt.Println("this xlsx file")
					err = recipientXlsx(r.FormValue("campaign"), file)
					if err != nil {
						js = []byte(`{"status": "error", "message": "Add recipients xlsx"}`)
					}
				} else {
					fmt.Println("this other file")
					js = []byte(`{"status": "error", "message": "This not csv or xlsx file"}`)
				}

			} else {
				js = []byte(`{"status": "error", "message": "Forbidden upload recipients"}`)
			}
			break

		case "deleteAll":
			if auth.Right("delete-recipients") && auth.CampaignRightString(r.Form["campaign"][0]) {
				err = delRecipients(r.Form["campaign"][0])
				if err != nil {
					js = []byte(`{"status": "error", "message": "Can't delete all recipients"}`)
				}
			} else {
				js = []byte(`{"status": "error", "message": "Forbidden delete recipients"}`)
			}
		}
	}

	if r.Form["content"][0] == "parameters" {
		switch r.Form["cmd"][0] {
		case "get-records":
			rId, err := getRecipientCampaign(r.Form["recipient"][0])
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
				return
			}
			if auth.Right("get-recipient-parameters") && auth.CampaignRight(rId) {
				ps, err := getRecipientParams( r.Form["recipient"][0], r.Form["offset"][0], r.Form["limit"][0])
				js, err = json.Marshal(ps)
				if err != nil {
					http.Error(w, err.Error(), http.StatusInternalServerError)
					return
				}
			}  else {
				js = []byte(`{"status": "error", "message": "Forbidden get recipient parameters"}`)
			}

			break
		}
	}

	w.Header().Set("Content-Type", "application/json")
	w.Write(js)
}

func getRecipientCampaign(recipientId string) (int64, error){
	var id int64
	err := models.Db.QueryRow("SELECT `campaign_id` FROM `recipient` WHERE `id`=?", recipientId).Scan(&id)
	return id, err
}

//ToDo check right errors
func getRecipients(campaign, offset, limit string) (Recipients, error) {
	var err error
	var r Recipient
	var rs Recipients
	rs.Records = []Recipient{}
	query, err := models.Db.Query("SELECT `id`, `name`, `email`, `status` FROM `recipient` WHERE `removed`!=1 AND `campaign_id`=?  LIMIT ? OFFSET ?", campaign, limit, offset)
	if err != nil {
		return rs, err
	}
	defer query.Close()
	for query.Next() {
		err = query.Scan(&r.Id, &r.Name, &r.Email, &r.Result)

		rs.Records = append(rs.Records, r)
	}
	err = models.Db.QueryRow("SELECT COUNT(*) FROM `recipient` WHERE `removed`!=1 AND `campaign_id`=?", campaign).Scan(&rs.Total)
	return rs, nil

}

//ToDo check right errors
func getRecipientParams(recipient, offset, limit string) (RecipientParams, error) {
	var err error
	var p RecipientParam
	var ps RecipientParams
	ps.Records = []RecipientParam{}
	query, err := models.Db.Query("SELECT `key`, `value` FROM `parameter` WHERE `recipient_id`=? LIMIT ? OFFSET ?", recipient, limit, offset)
	if err != nil {
		return ps, err
	}
	defer query.Close()
	for query.Next() {
		err = query.Scan(&p.Key, &p.Value)
		ps.Records = append(ps.Records, p)
	}
	err = models.Db.QueryRow("SELECT COUNT(*) FROM `parameter` WHERE `recipient_id`=?", recipient).Scan(&ps.Total)
	return ps, err
}

func delRecipients(campaignId string) error {
	_, err := models.Db.Exec("UPDATE `recipient` SET `removed`=1 WHERE `campaign_id`=?", campaignId)
	return err
}

// ToDo optimize this
func recipientCsv(campaignId string, file string) error {

	title := make(map[int]string)
	data := make(map[string]string)
	var email, name string

	csvfile, err := os.Open(file)
	if err != nil {
		return err
	}

	reader := csv.NewReader(csvfile)
	reader.FieldsPerRecord = -1
	rawCSVdata, err := reader.ReadAll()
	if err != nil {
		return err
	}
	for k, v := range rawCSVdata {
		if k == 0 {
			for i, t := range v {
				title[i] = t
			}
		} else {
			email = ""
			name = ""
			data = map[string]string{}
			for i, t := range v {
				if i == 0 {
					email = t
				} else if i == 1 {
					name = t
				} else {
					data[title[i]] = t
				}
			}

			res, err := models.Db.Exec("INSERT INTO recipient (`campaign_id`, `email`, `name`) VALUES (?, ?, ?)", campaignId, email, name)
			if err != nil {
				return err
			}
			id, err := res.LastInsertId()
			if err != nil {
				return err
			}
			for i, t := range data {
				_, err := models.Db.Exec("INSERT INTO parameter (`recipient_id`, `key`, `value`) VALUES (?, ?, ?)", id, i, t)
				if err != nil {
					return err
				}
			}
		}
	}

	csvfile.Close()

	os.Remove(file)

	return err
}

func recipientXlsx(campaignId string, file string) error {

	title := make(map[int]string)
	data := make(map[string]string)
	var email, name string

	xlFile, err := xlsx.OpenFile(file)
	if err != nil {
		return err
	}

	if xlFile.Sheets[0] != nil {
		for k, v := range xlFile.Sheets[0].Rows {
			if k == 0 {
				for i, cell := range v.Cells {
					t, err := cell.String()
					if err != nil {
						apilog.Println(err)
					}
					title[i] = t
				}
			} else {
				email = ""
				name = ""
				data = map[string]string{}
				for i, cell := range v.Cells {
					t, err := cell.String()
					if err != nil {
						apilog.Println(err)
					}
					if i == 0 {
						email = t
					} else if i == 1 {
						name = t
					} else {
						data[title[i]] = t
					}
				}

				res, err := models.Db.Exec("INSERT INTO recipient (`campaign_id`, `email`, `name`) VALUES (?, ?, ?)", campaignId, email, name)
				if err != nil {
					return err
				}
				id, err := res.LastInsertId()
				if err != nil {
					return err
				}
				for i, t := range data {
					_, err := models.Db.Exec("INSERT INTO parameter (`recipient_id`, `key`, `value`) VALUES (?, ?, ?)", id, i, t)
					if err != nil {
						return err
					}
				}

			}
		}
	}

	os.Remove(file)
	return err
}<|endoftext|>


<commit_before>package api

func (c *Sys) Health() (*HealthResponse, error) {
	r := c.c.NewRequest("GET", "/v1/sys/health")
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var result HealthResponse
	err = resp.DecodeJSON(&result)
	return &result, err
}

type HealthResponse struct {
	Initialized   bool   `json:"initialized"`
	Sealed        bool   `json:"sealed"`
	Standby       bool   `json:"standby"`
	ServerTimeUTC int64  `json:"server_time_utc"`
	Version       string `json:"version"`
	ClusterName   string `json:"cluster_name,omitempty"`
	ClusterID     string `json:"cluster_id,omitempty"`
}
<commit_msg>Have sys health api always return even in an error case (#3087)<commit_after>package api

func (c *Sys) Health() (*HealthResponse, error) {
	r := c.c.NewRequest("GET", "/v1/sys/health")
	// If the code is 400 or above it will automatically turn into an error,
	// but the sys/health API defaults to returning 5xx when not sealed or
	// inited, so we force this code to be something else so we parse correctly
	r.Params.Add("sealedcode", "299")
	r.Params.Add("uninitcode", "299")
	resp, err := c.c.RawRequest(r)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var result HealthResponse
	err = resp.DecodeJSON(&result)
	return &result, err
}

type HealthResponse struct {
	Initialized   bool   `json:"initialized"`
	Sealed        bool   `json:"sealed"`
	Standby       bool   `json:"standby"`
	ServerTimeUTC int64  `json:"server_time_utc"`
	Version       string `json:"version"`
	ClusterName   string `json:"cluster_name,omitempty"`
	ClusterID     string `json:"cluster_id,omitempty"`
}
<|endoftext|>


<commit_before>// Copyright 2015 Yury Kozyrev. All rights reserved.
// Proprietary license.

// Package translator
package translator
import (
	"net/http"
	"time"
	"github.com/facebookgo/httpcontrol"
	"sync"
	"log"
//	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/translation_middleware"
	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/particular"
	"regexp"
)

// Translator interface
type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	AddParticular(p particular.IParticularBackend)
}

var (
	client *http.Client
	once = &sync.Once{}
)

func initClient() (*http.Client) {
	once.Do(func() {
		transport := &httpcontrol.Transport{
			RequestTimeout: time.Minute,
			MaxTries: 3,
			MaxIdleConnsPerHost: 10,
		}

		client = &http.Client{
			Transport: transport,
		}
	})

	return client
}

type IBackendResponse interface {
	GetText() string
	GetSource() string
	GetLang() string
}

type ITranslateBackend interface {
	TranslateOne(text string, language string) (data IBackendResponse)
	GetName() string
}

type TranslateAdapter struct {
	client *http.Client
	translateBackend []ITranslateBackend
	translateParticular []particular.IParticularBackend
//	middleware []translation_middleware.ITranslationMiddleware
	landChan chan string

}

func NewTranslateAdapter(back []ITranslateBackend) *TranslateAdapter {
	return &TranslateAdapter{client:initClient(), translateBackend:back, landChan:make(chan string, 1)}
}
func (t *TranslateAdapter) AddParticular(p particular.IParticularBackend){
	p.SetClient(t.client)
	t.translateParticular = append(t.translateParticular, p)
}
func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	container := t.newContainer()
	container.Original = text

	langs = array_uniq(langs)
	processor := NewEmojiProcessor()
	text = processor.Process(text)

	responseChan := make(chan *RawTransData, len(langs))
	responseChanParticular := make(chan *RawParticularData, len(langs))

	go t.doRequestsTranslators(text, langs, responseChan)
//	go func(){
//		lang := <-t.landChan
//		t.doRequestsParticular(text, lang, langs, responseChanParticular)
//	}()
	for resp := range responseChan {
		log.Println(resp)
		if "google" == resp.Name {
			container.Source = resp.Source
			t.landChan <- resp.Source
		}
		resp.Translation = processor.Restore(resp.Translation)
		container.RawTranslations[resp.Name][resp.Lang] = resp.Translation
		container.RawTransData = append(container.RawTransData, resp)
	}
	container.Translations = container.RawTranslations["google"]

	for resp := range responseChanParticular {
		container.RawParticularData = append(container.RawParticularData, resp)
	}
	return container
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTransData){

	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend{
			wg.Add(1)
			go func(text, lang string, backend ITranslateBackend){
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateOne(text, lang)

				raw := &RawTransData{
					Source:resp.GetSource(),
					Lang:resp.GetLang(),
					Name:backend.GetName(),
					Translation:resp.GetText(),
					Time: time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) doRequestsParticular(text, source string, languages []string, c chan<- *RawParticularData){

	bag := regexp.MustCompile(`([\w\x7f-\xff])+`).FindAllString(text, -1)

	wg := &sync.WaitGroup{}
	for _, lang := range languages {
		for _, back := range t.translateParticular{
			wg.Add(1)
			raw := &RawParticularData{
				Original: text,
				Source:source,
				Lang:lang,
				Name: back.GetName(),
				ParticularBag: make([]*ParticularItem, len(bag)),
			}
			go func(raw *RawParticularData, backend particular.IParticularBackend){
				defer wg.Done()
				t := time.Now()

				partChan := make(chan *ParticularItem, len(bag))
				wgSingle := &sync.WaitGroup{}
				for index, word := range bag {
					wgSingle.Add(1)
					item := &ParticularItem{Order: index, Original: word}
					go func(item *ParticularItem,raw *RawParticularData, backend particular.IParticularBackend, c chan<- *ParticularItem){
						defer wgSingle.Done()
						t1 := time.Now()
						resp := backend.TranslateOne(item.Original, raw.Source, raw.Lang)
						item.Translations = resp.GetMeanings()
						item.Url = resp.GetUrl()
						item.Time = time.Since(t1) / time.Millisecond
						c <- item
					}(item, raw, backend, partChan)
				}
				wgSingle.Wait()
				close(partChan)

				for item := range partChan {
					raw.ParticularBag[item.Order] = item
				}

				raw.Time = time.Since(t) / time.Millisecond

				c <- raw
			}(raw, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) newContainer() *TranslationContainer{
	raw := make(map[string]TranslationBag, len(t.translateBackend))
	for _, back := range t.translateBackend {
		raw[back.GetName()] = TranslationBag{}
	}
	return &TranslationContainer{
		Translations:TranslationBag{},
		RawTranslations:raw,
	}
}

func array_uniq(langs []string) []string {
	set := make(map[string]struct{})
	for _, val := range langs {
		set[val] = struct{}{}
	}
	var languages []string
	for lang, _ := range set{
		languages = append(languages, lang)
	}
	return languages
}<commit_msg>particular disable<commit_after>// Copyright 2015 Yury Kozyrev. All rights reserved.
// Proprietary license.

// Package translator
package translator
import (
	"net/http"
	"time"
	"github.com/facebookgo/httpcontrol"
	"sync"
	"log"
//	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/translation_middleware"
	"github.com/urakozz/transpoint.io/src/infrastrucrute/translator/particular"
	"regexp"
)

// Translator interface
type Translator interface {
	Translate(text string, languages []string) *TranslationContainer
	AddParticular(p particular.IParticularBackend)
}

var (
	client *http.Client
	once = &sync.Once{}
)

func initClient() (*http.Client) {
	once.Do(func() {
		transport := &httpcontrol.Transport{
			RequestTimeout: time.Minute,
			MaxTries: 3,
			MaxIdleConnsPerHost: 10,
		}

		client = &http.Client{
			Transport: transport,
		}
	})

	return client
}

type IBackendResponse interface {
	GetText() string
	GetSource() string
	GetLang() string
}

type ITranslateBackend interface {
	TranslateOne(text string, language string) (data IBackendResponse)
	GetName() string
}

type TranslateAdapter struct {
	client *http.Client
	translateBackend []ITranslateBackend
	translateParticular []particular.IParticularBackend
//	middleware []translation_middleware.ITranslationMiddleware
	landChan chan string

}

func NewTranslateAdapter(back []ITranslateBackend) *TranslateAdapter {
	return &TranslateAdapter{client:initClient(), translateBackend:back, landChan:make(chan string, 1)}
}
func (t *TranslateAdapter) AddParticular(p particular.IParticularBackend){
	p.SetClient(t.client)
	t.translateParticular = append(t.translateParticular, p)
}
func (t *TranslateAdapter) Translate(text string, langs []string) *TranslationContainer {
	container := t.newContainer()
	container.Original = text

	langs = array_uniq(langs)
	processor := NewEmojiProcessor()
	text = processor.Process(text)

	responseChan := make(chan *RawTransData, len(langs))
	//responseChanParticular := make(chan *RawParticularData, len(langs))

	go t.doRequestsTranslators(text, langs, responseChan)
//	go func(){
//		lang := <-t.landChan
//		t.doRequestsParticular(text, lang, langs, responseChanParticular)
//	}()
	for resp := range responseChan {
		log.Println(resp)
		if "google" == resp.Name {
			container.Source = resp.Source
			t.landChan <- resp.Source
		}
		resp.Translation = processor.Restore(resp.Translation)
		container.RawTranslations[resp.Name][resp.Lang] = resp.Translation
		container.RawTransData = append(container.RawTransData, resp)
	}
	container.Translations = container.RawTranslations["google"]

//	for resp := range responseChanParticular {
//		container.RawParticularData = append(container.RawParticularData, resp)
//	}
	return container
}

func (t *TranslateAdapter) doRequestsTranslators(text string, languages []string, c chan<- *RawTransData){

	wg := &sync.WaitGroup{}
	for _, v := range languages {
		for _, back := range t.translateBackend{
			wg.Add(1)
			go func(text, lang string, backend ITranslateBackend){
				defer wg.Done()
				t := time.Now()
				resp := backend.TranslateOne(text, lang)

				raw := &RawTransData{
					Source:resp.GetSource(),
					Lang:resp.GetLang(),
					Name:backend.GetName(),
					Translation:resp.GetText(),
					Time: time.Since(t) / time.Millisecond,
				}
				c <- raw
			}(text, v, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) doRequestsParticular(text, source string, languages []string, c chan<- *RawParticularData){

	bag := regexp.MustCompile(`([\w\x7f-\xff])+`).FindAllString(text, -1)

	wg := &sync.WaitGroup{}
	for _, lang := range languages {
		for _, back := range t.translateParticular{
			wg.Add(1)
			raw := &RawParticularData{
				Original: text,
				Source:source,
				Lang:lang,
				Name: back.GetName(),
				ParticularBag: make([]*ParticularItem, len(bag)),
			}
			go func(raw *RawParticularData, backend particular.IParticularBackend){
				defer wg.Done()
				t := time.Now()

				partChan := make(chan *ParticularItem, len(bag))
				wgSingle := &sync.WaitGroup{}
				for index, word := range bag {
					wgSingle.Add(1)
					item := &ParticularItem{Order: index, Original: word}
					go func(item *ParticularItem,raw *RawParticularData, backend particular.IParticularBackend, c chan<- *ParticularItem){
						defer wgSingle.Done()
						t1 := time.Now()
						resp := backend.TranslateOne(item.Original, raw.Source, raw.Lang)
						item.Translations = resp.GetMeanings()
						item.Url = resp.GetUrl()
						item.Time = time.Since(t1) / time.Millisecond
						c <- item
					}(item, raw, backend, partChan)
				}
				wgSingle.Wait()
				close(partChan)

				for item := range partChan {
					raw.ParticularBag[item.Order] = item
				}

				raw.Time = time.Since(t) / time.Millisecond

				c <- raw
			}(raw, back)
		}

	}
	wg.Wait()
	close(c)
}

func (t *TranslateAdapter) newContainer() *TranslationContainer{
	raw := make(map[string]TranslationBag, len(t.translateBackend))
	for _, back := range t.translateBackend {
		raw[back.GetName()] = TranslationBag{}
	}
	return &TranslationContainer{
		Translations:TranslationBag{},
		RawTranslations:raw,
	}
}

func array_uniq(langs []string) []string {
	set := make(map[string]struct{})
	for _, val := range langs {
		set[val] = struct{}{}
	}
	var languages []string
	for lang, _ := range set{
		languages = append(languages, lang)
	}
	return languages
}<|endoftext|>


<commit_before>// +build appengine

package handler

import (
	"net/http"
	"time"

	"github.com/juju/errors"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

func init() {
	AddMiddleware(ContextMiddleware)
	AddMiddleware(ContextTimeout)
	AddMiddleware(LoggerMiddleware)
	AddMiddleware(ClientErrorMiddleware)
}

// ContextMiddleware bootstraps the context to be able to contact with App Engine.
func ContextMiddleware(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	return next(appengine.WithContext(ctx, r))
}

// ContextTimeout sets a timeout in the context.
func ContextTimeout(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	secs := 55 * time.Second
	if r.Header.Get("X-AppEngine-QueueName") != "" || r.Header.Get("X-AppEngine-Cron") != "" {
		secs = (9*60 + 55) * time.Second
	}

	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(secs))

	done := make(chan error, 1)
	go func() {
		done <- next(ctx)
	}()

	select {
	case err := <-done:
		cancel()
		return err

	case <-time.After(secs):
		log.Criticalf(ctx, "request timeout")
		cancel()
		return nil
	}

	panic("should not reach here")
}

// ContextTimeoutFailFast sets a shorter timeout in the context of interactive requests and allows
// the previous middleware to log or answer request timeouts. Be sure you process the
// "request timeout" error or it will panic when finishing the middleware stack. It is not activated
// by default; but you can clear the middleware stack and put it instead of ContextTimeout().
func ContextTimeoutFailFast(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	secs := 15 * time.Second
	if r.Header.Get("X-AppEngine-QueueName") != "" || r.Header.Get("X-AppEngine-Cron") != "" {
		secs = (9*60 + 55) * time.Second
	}

	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(secs))

	done := make(chan error, 1)
	go func() {
		done <- next(ctx)
	}()

	select {
	case err := <-done:
		cancel()
		return err

	case <-time.After(secs):
		cancel()
		return errors.New("request timeout")
	}

	panic("should not reach here")
}

// LoggerMiddleware logs to Cloud Logging the error.
func LoggerMiddleware(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	if err := next(ctx); err != nil {
		log.Errorf(ctx, "%s", errors.ErrorStack(err))
	}

	return nil
}

// ClientErrorMiddleware rejects the request with a 500 status code when an error occurs.
func ClientErrorMiddleware(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	err := next(ctx)
	if err != nil {
		w.(*appResponseWriter).Reset()
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")

		if appengine.IsDevAppServer() {
			http.Error(w, errors.ErrorStack(err), http.StatusInternalServerError)
		} else {
			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		}
	}

	return err
}
<commit_msg>Fix typo.<commit_after>// +build appengine

package handler

import (
	"net/http"
	"time"

	"github.com/juju/errors"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

func init() {
	AddMiddleware(ContextMiddleware)
	AddMiddleware(ContextTimeout)
	AddMiddleware(LoggerMiddleware)
	AddMiddleware(ClientErrorMiddleware)
}

// ContextMiddleware bootstraps the context to be able to contact with App Engine.
func ContextMiddleware(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	return next(appengine.WithContext(ctx, r))
}

// ContextTimeout sets a timeout in the context.
func ContextTimeout(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	secs := 55 * time.Second
	if r.Header.Get("X-AppEngine-QueueName") != "" || r.Header.Get("X-AppEngine-Cron") != "" {
		secs = (9*60 + 55) * time.Second
	}

	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(secs))

	done := make(chan error, 1)
	go func() {
		done <- next(ctx)
	}()

	select {
	case err := <-done:
		cancel()
		return err

	case <-time.After(secs):
		log.Criticalf(ctx, "request timeout")
		cancel()
		return nil
	}

	panic("should not reach here")
}

// ContextTimeoutFailFast sets a shorter timeout in the context of interactive requests and allows
// the previous middleware to log or answer request timeouts. Be sure you process the
// "request timeout" error or it will panic when finishing the middleware stack. It is not activated
// by default; but you can clear the middleware stack and put it instead of ContextTimeout().
func ContextTimeoutFailFast(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	secs := 15 * time.Second
	if r.Header.Get("X-AppEngine-QueueName") != "" || r.Header.Get("X-AppEngine-Cron") != "" {
		secs = (9*60 + 55) * time.Second
	}

	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(secs))

	done := make(chan error, 1)
	go func() {
		done <- next(ctx)
	}()

	select {
	case err := <-done:
		cancel()
		return err

	case <-time.After(secs):
		cancel()
		return errors.New("request timeout")
	}

	panic("should not reach here")
}

// LoggerMiddleware logs to Cloud Logging the error.
func LoggerMiddleware(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	if err := next(ctx); err != nil {
		log.Errorf(ctx, "%s", errors.ErrorStack(err))
	}

	return nil
}

// ClientErrorMiddleware rejects the request with a 500 status code when an error occurs.
func ClientErrorMiddleware(ctx context.Context, w http.ResponseWriter, r *http.Request, next NextMiddlewareFn) error {
	err := next(ctx)
	if err != nil {
		w.(*AppResponseWriter).Reset()
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")

		if appengine.IsDevAppServer() {
			http.Error(w, errors.ErrorStack(err), http.StatusInternalServerError)
		} else {
			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
		}
	}

	return err
}
<|endoftext|>


<commit_before>package influxdb_output

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"log"
	"os"
	"time"

	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
	"github.com/influxdata/influxdb-client-go/v2/api"
	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/mitchellh/mapstructure"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

const (
	defaultURL               = "http://localhost:8086"
	defaultBatchSize         = 1000
	defaultFlushTimer        = 10 * time.Second
	defaultHealthCheckPeriod = 30 * time.Second

	numWorkers = 1
)

func init() {
	outputs.Register("influxdb", func() outputs.Output {
		return &InfluxDBOutput{
			Cfg:       &Config{},
			eventChan: make(chan *collector.EventMsg),
		}
	})
}

type InfluxDBOutput struct {
	Cfg       *Config
	client    influxdb2.Client
	writer    api.WriteAPI
	metrics   []prometheus.Collector
	logger    *log.Logger
	cancelFn  context.CancelFunc
	eventChan chan *collector.EventMsg
}
type Config struct {
	URL               string        `mapstructure:"url,omitempty"`
	Org               string        `mapstructure:"org,omitempty"`
	Bucket            string        `mapstructure:"bucket,omitempty"`
	Token             string        `mapstructure:"token,omitempty"`
	BatchSize         uint          `mapstructure:"batch_size,omitempty"`
	FlushTimer        time.Duration `mapstructure:"flush_timer,omitempty"`
	UseGzip           bool          `mapstructure:"use_gzip,omitempty"`
	EnableTLS         bool          `mapstructure:"enable_tls,omitempty"`
	HealthCheckPeriod time.Duration `mapstructure:"health_check_period,omitempty"`
	Debug             bool          `mapstructure:"debug,omitempty"`
}

func (k *InfluxDBOutput) String() string {
	b, err := json.Marshal(k)
	if err != nil {
		return ""
	}
	return string(b)
}
func (i *InfluxDBOutput) Init(ctx context.Context, cfg map[string]interface{}, logger *log.Logger) error {
	ctx, i.cancelFn = context.WithCancel(ctx)
	err := mapstructure.Decode(cfg, i.Cfg)
	if err != nil {
		return err
	}
	if i.Cfg.URL == "" {
		i.Cfg.URL = defaultURL
	}
	if i.Cfg.BatchSize == 0 {
		i.Cfg.BatchSize = defaultBatchSize
	}
	if i.Cfg.FlushTimer == 0 {
		i.Cfg.FlushTimer = defaultFlushTimer
	}
	if i.Cfg.HealthCheckPeriod == 0 {
		i.Cfg.HealthCheckPeriod = defaultHealthCheckPeriod
	}
	i.logger = log.New(os.Stderr, "influxdb_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		i.logger.SetOutput(logger.Writer())
		i.logger.SetFlags(logger.Flags())
	}
	opts := influxdb2.DefaultOptions().
		SetUseGZip(i.Cfg.UseGzip).
		SetBatchSize(i.Cfg.BatchSize).
		SetFlushInterval(uint(i.Cfg.FlushTimer.Milliseconds()))
	if i.Cfg.EnableTLS {
		opts.SetTLSConfig(&tls.Config{
			InsecureSkipVerify: true,
		})
	}
	if i.Cfg.Debug {
		opts.SetLogLevel(3)
	}
	i.client = influxdb2.NewClientWithOptions(i.Cfg.URL, i.Cfg.Token, opts)
	// start influx health check
	go i.healthCheck(ctx)
	i.writer = i.client.WriteAPI(i.Cfg.Org, i.Cfg.Bucket)
	i.logger.Printf("initialized influxdb write API: %s", i.String())
	// start influxdb error logs
	go func() {
		select {
		case <-ctx.Done():
			return
		case err := <-i.writer.Errors():
			i.logger.Printf("writeAPI error: %v", err)
		}
	}()
	for k := 0; k < numWorkers; k++ {
		go i.worker(ctx, k)
	}
	go func() {
		<-ctx.Done()
		i.Close()
	}()
	return nil
}

func (i *InfluxDBOutput) Write(ctx context.Context, rsp proto.Message, meta outputs.Meta) {
	if rsp == nil {
		return
	}
	switch rsp := rsp.(type) {
	case *gnmi.SubscribeResponse:
		measName := "default"
		if subName, ok := meta["subscription-name"]; ok {
			measName = subName
		}
		events, err := collector.ResponseToEventMsgs(measName, rsp, meta)
		if err != nil {
			i.logger.Printf("failed to convert message to event: %v", err)
			return
		}
		for _, ev := range events {
			select {
			case <-ctx.Done():
				return
			case i.eventChan <- ev:
			}
		}
	}
}

func (i *InfluxDBOutput) Close() error {
	i.logger.Printf("flushing data...")
	i.writer.Flush()
	i.logger.Printf("closing client...")
	i.client.Close()
	i.cancelFn()
	close(i.eventChan)
	i.logger.Printf("closed.")
	return nil
}
func (i *InfluxDBOutput) Metrics() []prometheus.Collector { return i.metrics }

func (i *InfluxDBOutput) healthCheck(ctx context.Context) {
	i.health(ctx)
	ticker := time.NewTicker(i.Cfg.HealthCheckPeriod)
	for {
		select {
		case <-ticker.C:
			i.health(ctx)
		case <-ctx.Done():
			return
		}
	}
}

func (i *InfluxDBOutput) health(ctx context.Context) {
	res, err := i.client.Health(ctx)
	if err != nil {
		i.logger.Printf("failed health check: %v", err)
		return
	}
	if res != nil {
		b, err := json.Marshal(res)
		if err != nil {
			i.logger.Printf("failed to marshal health check result: %v", err)
			i.logger.Printf("health check result: %+v", res)
			return
		}
		i.logger.Printf("health check result: %s", string(b))
		return
	}
	i.logger.Print("health check result is nil")
}

func (i *InfluxDBOutput) worker(ctx context.Context, idx int) {
	for {
		select {
		case <-ctx.Done():
			i.logger.Printf("worker-%d terminating...", idx)
			return
		case ev := <-i.eventChan:
			i.writer.WritePoint(influxdb2.NewPoint(ev.Name, ev.Tags, ev.Values, time.Unix(0, ev.Timestamp)))
		}
	}
}
<commit_msg>add logging msgs to influxdb output worker<commit_after>package influxdb_output

import (
	"context"
	"crypto/tls"
	"encoding/json"
	"log"
	"os"
	"time"

	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
	"github.com/influxdata/influxdb-client-go/v2/api"
	"github.com/karimra/gnmic/collector"
	"github.com/karimra/gnmic/outputs"
	"github.com/mitchellh/mapstructure"
	"github.com/openconfig/gnmi/proto/gnmi"
	"github.com/prometheus/client_golang/prometheus"
	"google.golang.org/protobuf/proto"
)

const (
	defaultURL               = "http://localhost:8086"
	defaultBatchSize         = 1000
	defaultFlushTimer        = 10 * time.Second
	defaultHealthCheckPeriod = 30 * time.Second

	numWorkers = 1
)

func init() {
	outputs.Register("influxdb", func() outputs.Output {
		return &InfluxDBOutput{
			Cfg:       &Config{},
			eventChan: make(chan *collector.EventMsg),
		}
	})
}

type InfluxDBOutput struct {
	Cfg       *Config
	client    influxdb2.Client
	writer    api.WriteAPI
	metrics   []prometheus.Collector
	logger    *log.Logger
	cancelFn  context.CancelFunc
	eventChan chan *collector.EventMsg
}
type Config struct {
	URL               string        `mapstructure:"url,omitempty"`
	Org               string        `mapstructure:"org,omitempty"`
	Bucket            string        `mapstructure:"bucket,omitempty"`
	Token             string        `mapstructure:"token,omitempty"`
	BatchSize         uint          `mapstructure:"batch_size,omitempty"`
	FlushTimer        time.Duration `mapstructure:"flush_timer,omitempty"`
	UseGzip           bool          `mapstructure:"use_gzip,omitempty"`
	EnableTLS         bool          `mapstructure:"enable_tls,omitempty"`
	HealthCheckPeriod time.Duration `mapstructure:"health_check_period,omitempty"`
	Debug             bool          `mapstructure:"debug,omitempty"`
}

func (k *InfluxDBOutput) String() string {
	b, err := json.Marshal(k)
	if err != nil {
		return ""
	}
	return string(b)
}
func (i *InfluxDBOutput) Init(ctx context.Context, cfg map[string]interface{}, logger *log.Logger) error {
	ctx, i.cancelFn = context.WithCancel(ctx)
	err := mapstructure.Decode(cfg, i.Cfg)
	if err != nil {
		return err
	}
	if i.Cfg.URL == "" {
		i.Cfg.URL = defaultURL
	}
	if i.Cfg.BatchSize == 0 {
		i.Cfg.BatchSize = defaultBatchSize
	}
	if i.Cfg.FlushTimer == 0 {
		i.Cfg.FlushTimer = defaultFlushTimer
	}
	if i.Cfg.HealthCheckPeriod == 0 {
		i.Cfg.HealthCheckPeriod = defaultHealthCheckPeriod
	}
	i.logger = log.New(os.Stderr, "influxdb_output ", log.LstdFlags|log.Lmicroseconds)
	if logger != nil {
		i.logger.SetOutput(logger.Writer())
		i.logger.SetFlags(logger.Flags())
	}
	opts := influxdb2.DefaultOptions().
		SetUseGZip(i.Cfg.UseGzip).
		SetBatchSize(i.Cfg.BatchSize).
		SetFlushInterval(uint(i.Cfg.FlushTimer.Milliseconds()))
	if i.Cfg.EnableTLS {
		opts.SetTLSConfig(&tls.Config{
			InsecureSkipVerify: true,
		})
	}
	if i.Cfg.Debug {
		opts.SetLogLevel(3)
	}
	i.client = influxdb2.NewClientWithOptions(i.Cfg.URL, i.Cfg.Token, opts)
	// start influx health check
	go i.healthCheck(ctx)
	i.writer = i.client.WriteAPI(i.Cfg.Org, i.Cfg.Bucket)
	i.logger.Printf("initialized influxdb write API: %s", i.String())
	// start influxdb error logs
	go func() {
		select {
		case <-ctx.Done():
			return
		case err := <-i.writer.Errors():
			i.logger.Printf("writeAPI error: %v", err)
		}
	}()
	for k := 0; k < numWorkers; k++ {
		go i.worker(ctx, k)
	}
	go func() {
		<-ctx.Done()
		i.Close()
	}()
	return nil
}

func (i *InfluxDBOutput) Write(ctx context.Context, rsp proto.Message, meta outputs.Meta) {
	if rsp == nil {
		return
	}
	switch rsp := rsp.(type) {
	case *gnmi.SubscribeResponse:
		measName := "default"
		if subName, ok := meta["subscription-name"]; ok {
			measName = subName
		}
		events, err := collector.ResponseToEventMsgs(measName, rsp, meta)
		if err != nil {
			i.logger.Printf("failed to convert message to event: %v", err)
			return
		}
		for _, ev := range events {
			select {
			case <-ctx.Done():
				return
			case i.eventChan <- ev:
			}
		}
	}
}

func (i *InfluxDBOutput) Close() error {
	i.logger.Printf("flushing data...")
	i.writer.Flush()
	i.logger.Printf("closing client...")
	i.client.Close()
	i.cancelFn()
	close(i.eventChan)
	i.logger.Printf("closed.")
	return nil
}
func (i *InfluxDBOutput) Metrics() []prometheus.Collector { return i.metrics }

func (i *InfluxDBOutput) healthCheck(ctx context.Context) {
	i.health(ctx)
	ticker := time.NewTicker(i.Cfg.HealthCheckPeriod)
	for {
		select {
		case <-ticker.C:
			i.health(ctx)
		case <-ctx.Done():
			return
		}
	}
}

func (i *InfluxDBOutput) health(ctx context.Context) {
	res, err := i.client.Health(ctx)
	if err != nil {
		i.logger.Printf("failed health check: %v", err)
		return
	}
	if res != nil {
		b, err := json.Marshal(res)
		if err != nil {
			i.logger.Printf("failed to marshal health check result: %v", err)
			i.logger.Printf("health check result: %+v", res)
			return
		}
		i.logger.Printf("health check result: %s", string(b))
		return
	}
	i.logger.Print("health check result is nil")
}

func (i *InfluxDBOutput) worker(ctx context.Context, idx int) {
	i.logger.Printf("starting worker-%d", idx)
	for {
		select {
		case <-ctx.Done():
			if ctx.Err() != nil {
				i.logger.Printf("worker-%d err=%v", idx, ctx.Err())
			}
			i.logger.Printf("worker-%d terminating...", idx)
			return
		case ev := <-i.eventChan:
			i.writer.WritePoint(influxdb2.NewPoint(ev.Name, ev.Tags, ev.Values, time.Unix(0, ev.Timestamp)))
		}
	}
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"os"
	"sort"
)

type TerminalLogger struct {
	// fieldOut is used to do log.WithFields correctly
	fieldOut string
}

func (t TerminalLogger) log(args ...interface{}) {
	fmt.Print(args...)
	fmt.Print(t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) logf(fmtString string, args ...interface{}) {
	fmt.Printf(fmtString, args...)
	fmt.Print(t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) err(args ...interface{}) {
	fmt.Fprint(os.Stderr, args...)
	fmt.Fprint(os.Stderr, t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) errf(fmtString string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, fmtString, args...)
	fmt.Print(t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) Debug(args ...interface{}) {
	if isDebug() {
		t.log(args...)
	}
}

func (t TerminalLogger) Debugf(fmtString string, args ...interface{}) {
	if isDebug() {
		t.logf(fmtString, args...)
	}
}

func (t TerminalLogger) Error(args ...interface{}) {
	t.err(args...)
}

func (t TerminalLogger) Errorf(fmtString string, args ...interface{}) {
	t.errf(fmtString, args...)
}

func (t TerminalLogger) Errorln(args ...interface{}) {
	t.err(args...)
}

func (t TerminalLogger) Info(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Infof(fmtString string, args ...interface{}) {
	t.logf(fmtString, args...)
}

func (t TerminalLogger) Infoln(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Fatal(args ...interface{}) {
	t.err(args...)
	os.Exit(1)
}

func (t TerminalLogger) Fatalf(fmtString string, args ...interface{}) {
	t.errf(fmtString, args...)
	os.Exit(1)
}

func (t TerminalLogger) Print(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Printf(fmtString string, args ...interface{}) {
	t.logf(fmtString, args...)
}

func (t TerminalLogger) Warn(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Warnf(fmtString string, args ...interface{}) {
	t.logf(fmtString, args...)
}

func (t TerminalLogger) WithFields(fields Fields) Logger {
	// When the user calls WithFields, we make a string which gets appended
	// to the output of the final [Info|Warn|Error] call for the
	// descriptive fields.  Because WithFields returns the proper Logger
	// (with the fieldOut string set correctly), the logrus syntax will
	// still work.
	kvpairs := []string{}

	// Why the string slice song and dance?  Because Go's map iteration
	// order is random, we will get inconsistent results if we don't sort
	// the fields (or their resulting string K/V pairs, like we have here).
	// Otherwise, we couldn't test this reliably.
	for k, v := range fields {
		kvpairs = append(kvpairs, fmt.Sprintf("%s=%v", k, v))
	}

	sort.Strings(kvpairs)

	// TODO:
	// 1. Is this thread-safe?
	// 2. Add more tabs?
	t.fieldOut = "\t\t"

	for _, s := range kvpairs {
		// TODO: Is %v the correct format string here?
		t.fieldOut = fmt.Sprintf("%s %s", t.fieldOut, s)
	}

	return t
}
<commit_msg>bugfix: logger.error was writing to stdout<commit_after>package log

import (
	"fmt"
	"os"
	"sort"
)

type TerminalLogger struct {
	// fieldOut is used to do log.WithFields correctly
	fieldOut string
}

func (t TerminalLogger) log(args ...interface{}) {
	fmt.Print(args...)
	fmt.Print(t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) logf(fmtString string, args ...interface{}) {
	fmt.Printf(fmtString, args...)
	fmt.Print(t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) err(args ...interface{}) {
	fmt.Fprint(os.Stderr, args...)
	fmt.Fprint(os.Stderr, t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) errf(fmtString string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, fmtString, args...)
	fmt.Fprint(os.Stderr, t.fieldOut, "\n")
	t.fieldOut = ""
}

func (t TerminalLogger) Debug(args ...interface{}) {
	if isDebug() {
		t.log(args...)
	}
}

func (t TerminalLogger) Debugf(fmtString string, args ...interface{}) {
	if isDebug() {
		t.logf(fmtString, args...)
	}
}

func (t TerminalLogger) Error(args ...interface{}) {
	t.err(args...)
}

func (t TerminalLogger) Errorf(fmtString string, args ...interface{}) {
	t.errf(fmtString, args...)
}

func (t TerminalLogger) Errorln(args ...interface{}) {
	t.err(args...)
}

func (t TerminalLogger) Info(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Infof(fmtString string, args ...interface{}) {
	t.logf(fmtString, args...)
}

func (t TerminalLogger) Infoln(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Fatal(args ...interface{}) {
	t.err(args...)
	os.Exit(1)
}

func (t TerminalLogger) Fatalf(fmtString string, args ...interface{}) {
	t.errf(fmtString, args...)
	os.Exit(1)
}

func (t TerminalLogger) Print(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Printf(fmtString string, args ...interface{}) {
	t.logf(fmtString, args...)
}

func (t TerminalLogger) Warn(args ...interface{}) {
	t.log(args...)
}

func (t TerminalLogger) Warnf(fmtString string, args ...interface{}) {
	t.logf(fmtString, args...)
}

func (t TerminalLogger) WithFields(fields Fields) Logger {
	// When the user calls WithFields, we make a string which gets appended
	// to the output of the final [Info|Warn|Error] call for the
	// descriptive fields.  Because WithFields returns the proper Logger
	// (with the fieldOut string set correctly), the logrus syntax will
	// still work.
	kvpairs := []string{}

	// Why the string slice song and dance?  Because Go's map iteration
	// order is random, we will get inconsistent results if we don't sort
	// the fields (or their resulting string K/V pairs, like we have here).
	// Otherwise, we couldn't test this reliably.
	for k, v := range fields {
		kvpairs = append(kvpairs, fmt.Sprintf("%s=%v", k, v))
	}

	sort.Strings(kvpairs)

	// TODO:
	// 1. Is this thread-safe?
	// 2. Add more tabs?
	t.fieldOut = "\t\t"

	for _, s := range kvpairs {
		// TODO: Is %v the correct format string here?
		t.fieldOut = fmt.Sprintf("%s %s", t.fieldOut, s)
	}

	return t
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"fmt"
	"os"
	"path/filepath"

	"github.com/techjanitor/pram-post/config"
	e "github.com/techjanitor/pram-post/errors"
	u "github.com/techjanitor/pram-post/utils"
)

type DeletePostModel struct {
	Thread uint
	Id     uint
	Ib     uint
	Name   string
}

type PostImage struct {
	Id    uint
	File  string
	Thumb string
}

// Status will return info
func (i *DeletePostModel) Status() (err error) {

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	// get thread ib and title
	err = db.QueryRow("SELECT ib_id, thread_title FROM threads WHERE thread_id = ? LIMIT 1", i.Thread).Scan(&i.Ib, &i.Name)
	if err == sql.ErrNoRows {
		return e.ErrNotFound
	} else if err != nil {
		return
	}

	return

}

// Delete will remove the entry
func (i *DeletePostModel) Delete() (err error) {

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	image := PostImage{}

	img := true

	// check if post has an image
	err = db.QueryRow(`SELECT image_id,image_file,image_thumbnail FROM posts
    INNER JOIN images on posts.post_id = images.post_id
    WHERE posts.thread_id = ? AND posts.post_num = ? LIMIT 1`, i.Thread, i.Id).Scan(&image.Id, &image.File, &image.Thumb)
	if err == sql.ErrNoRows {
		img = false
	} else if err != nil {
		return
	}

	// delete image file
	if img {

		// filename must exist to prevent deleting the directory ;D
		if image.Thumb == "" {
			return
		}

		if image.File == "" {
			return
		}

		// delete from google cloud storage
		u.DeleteGCS(fmt.Sprintf("src/%s", image.File))
		if err != nil {
			return
		}

		u.DeleteGCS(fmt.Sprintf("thumb/%s", image.Thumb))
		if err != nil {
			return
		}

		os.RemoveAll(filepath.Join(config.Settings.General.ImageDir, image.File))
		os.RemoveAll(filepath.Join(config.Settings.General.ThumbnailDir, image.Thumb))

	}

	// delete thread from database
	ps1, err := db.Prepare("DELETE FROM posts WHERE thread_id= ? AND post_num = ? LIMIT 1")
	if err != nil {
		return
	}
	defer ps1.Close()

	_, err = ps1.Exec(i.Thread, i.Id)
	if err != nil {
		return
	}

	var lasttime string

	// get last post time
	err = db.QueryRow("SELECT post_time FROM posts WHERE thread_id = ? ORDER BY post_id DESC LIMIT 1", i.Thread).Scan(&lasttime)
	if err != nil {
		return
	}

	// update last post time in thread
	ps1, err := db.Prepare("UPDATE threads SET thread_last_post= ? WHERE thread_id= ?")
	if err != nil {
		return
	}
	defer ps1.Close()

	_, err = ps1.Exec(lasttime, i.Thread)
	if err != nil {
		return
	}

	return

}
<commit_msg>add delete post<commit_after>package models

import (
	"database/sql"
	"fmt"
	"os"
	"path/filepath"

	"github.com/techjanitor/pram-post/config"
	e "github.com/techjanitor/pram-post/errors"
	u "github.com/techjanitor/pram-post/utils"
)

type DeletePostModel struct {
	Thread uint
	Id     uint
	Ib     uint
	Name   string
}

type PostImage struct {
	Id    uint
	File  string
	Thumb string
}

// Status will return info
func (i *DeletePostModel) Status() (err error) {

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	// get thread ib and title
	err = db.QueryRow("SELECT ib_id, thread_title FROM threads WHERE thread_id = ? LIMIT 1", i.Thread).Scan(&i.Ib, &i.Name)
	if err == sql.ErrNoRows {
		return e.ErrNotFound
	} else if err != nil {
		return
	}

	return

}

// Delete will remove the entry
func (i *DeletePostModel) Delete() (err error) {

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	image := PostImage{}

	img := true

	// check if post has an image
	err = db.QueryRow(`SELECT image_id,image_file,image_thumbnail FROM posts
    INNER JOIN images on posts.post_id = images.post_id
    WHERE posts.thread_id = ? AND posts.post_num = ? LIMIT 1`, i.Thread, i.Id).Scan(&image.Id, &image.File, &image.Thumb)
	if err == sql.ErrNoRows {
		img = false
	} else if err != nil {
		return
	}

	// delete image file
	if img {

		// filename must exist to prevent deleting the directory ;D
		if image.Thumb == "" {
			return
		}

		if image.File == "" {
			return
		}

		// delete from google cloud storage
		u.DeleteGCS(fmt.Sprintf("src/%s", image.File))
		if err != nil {
			return
		}

		u.DeleteGCS(fmt.Sprintf("thumb/%s", image.Thumb))
		if err != nil {
			return
		}

		os.RemoveAll(filepath.Join(config.Settings.General.ImageDir, image.File))
		os.RemoveAll(filepath.Join(config.Settings.General.ThumbnailDir, image.Thumb))

	}

	// delete thread from database
	ps1, err := db.Prepare("DELETE FROM posts WHERE thread_id= ? AND post_num = ? LIMIT 1")
	if err != nil {
		return
	}
	defer ps1.Close()

	_, err = ps1.Exec(i.Thread, i.Id)
	if err != nil {
		return
	}

	var lasttime string

	// get last post time
	err = db.QueryRow("SELECT post_time FROM posts WHERE thread_id = ? ORDER BY post_id DESC LIMIT 1", i.Thread).Scan(&lasttime)
	if err != nil {
		return
	}

	// update last post time in thread
	ps2, err := db.Prepare("UPDATE threads SET thread_last_post= ? WHERE thread_id= ?")
	if err != nil {
		return
	}
	defer ps1.Close()

	_, err = ps2.Exec(lasttime, i.Thread)
	if err != nil {
		return
	}

	return

}
<|endoftext|>