<commit_before>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/binary"
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/fiorix/go-redis/redis"
	"github.com/fiorix/go-web/httpxtra"
	_ "github.com/mattn/go-sqlite3"
)

type Settings struct {
	XMLName      xml.Name `xml:"Server"`
	Debug        bool     `xml:"debug,attr"`
	XHeaders     bool     `xml:"xheaders,attr"`
	Addr         string   `xml:"addr,attr"`
	DocumentRoot string
	IPDB         struct {
		File      string `xml:",attr"`
		CacheSize string `xml:",attr"`
	}
	Limit struct {
		MaxRequests int `xml:",attr"`
		Expire      int `xml:",attr"`
	}
	Redis []string `xml:"Redis>Addr"`
}

var conf *Settings

func main() {
	cf := flag.String("config", "freegeoip.conf", "set config file")
	flag.Parse()
	if buf, err := ioutil.ReadFile(*cf); err != nil {
		log.Fatal(err)
	} else {
		conf = &Settings{}
		if err := xml.Unmarshal(buf, conf); err != nil {
			log.Fatal(err)
		}
	}
	http.Handle("/", http.FileServer(http.Dir(conf.DocumentRoot)))
	h := GeoipHandler()
	http.HandleFunc("/csv/", h)
	http.HandleFunc("/xml/", h)
	http.HandleFunc("/json/", h)
	server := http.Server{
		Addr: conf.Addr,
		Handler: httpxtra.Handler{
			Logger:   logger,
			XHeaders: conf.XHeaders,
		},
		ReadTimeout:  15 * time.Second,
		WriteTimeout: 15 * time.Second,
	}
	log.Printf("FreeGeoIP server starting on %s (xheaders=%t)",
		conf.Addr, conf.XHeaders)
	log.Fatal(httpxtra.ListenAndServe(server))
}

func logger(r *http.Request, created time.Time, status, bytes int) {
	//fmt.Println(httpxtra.ApacheCommonLog(r, created, status, bytes))
	log.Printf("HTTP %d %s %s (%s) :: %s",
		status,
		r.Method,
		r.URL.Path,
		r.RemoteAddr,
		time.Since(created))
}

// GeoipHandler handles GET on /csv, /xml and /json.
func GeoipHandler() http.HandlerFunc {
	db, err := sql.Open("sqlite3", conf.IPDB.File)
	if err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec("PRAGMA cache_size=" + conf.IPDB.CacheSize)
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := db.Prepare(query)
	if err != nil {
		log.Fatal(err)
	}
	//defer stmt.Close()
	rc := redis.New(conf.Redis...)
	return func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "GET":
			w.Header().Set("Access-Control-Allow-Origin", "*")
		case "OPTIONS":
			w.Header().Set("Access-Control-Allow-Origin", "*")
			w.Header().Set("Content-Type", "text/plain")
			w.Header().Set("Access-Control-Allow-Methods", "GET")
			w.Header().Set("Access-Control-Allow-Headers", "X-Requested-With")
			w.WriteHeader(200)
			return
		default:
			w.Header().Set("Allow", "GET, OPTIONS")
			http.Error(w, http.StatusText(405), 405)
			return
		}
		// GET continuing...
		var (
			ip, ipkey string
			err       error
			ok        bool
		)
		if ip, _, err = net.SplitHostPort(r.RemoteAddr); err != nil {
			ipkey = r.RemoteAddr // Support for XHeaders
		} else {
			ipkey = ip
		}
		// Check quota
		if ok, err = HasQuota(rc, &ipkey); err != nil {
			// Redis down?
			if conf.Debug {
				log.Println("Redis error:", err.Error())
			}
			http.Error(w, http.StatusText(503), 503)
			return
		} else if !ok {
			// Over quota, soz :(
			http.Error(w, http.StatusText(403), 403)
			return
		}
		// Parse URL (e.g. /csv/ip, /xml/)
		a := strings.SplitN(r.URL.Path, "/", 3)
		if len(a) == 3 && a[2] != "" {
			addrs, err := net.LookupHost(a[2])
			if err != nil {
				// DNS lookup failed, assume host not found.
				http.Error(w, http.StatusText(404), 404)
				return
			}
			ip = addrs[0]
		} else {
			ip = ipkey
		}
		// Query the db
		geoip, err := GeoipLookup(stmt, ip)
		if err != nil {
			http.NotFound(w, r)
			return
		}
		switch a[1][0] {
		case 'c': // csv
			w.Header().Set("Content-Type", "application/csv")
			fmt.Fprintf(w, `"%s","%s","%s","%s","%s","%s",`+
				`"%s","%0.4f","%0.4f","%s","%s"`+"\r\n",
				geoip.Ip,
				geoip.CountryCode, geoip.CountryName,
				geoip.RegionCode, geoip.RegionName,
				geoip.CityName, geoip.ZipCode,
				geoip.Latitude, geoip.Longitude,
				geoip.MetroCode, geoip.AreaCode)
		case 'j': // json
			resp, err := json.Marshal(geoip)
			if err != nil {
				if conf.Debug {
					log.Println("JSON error:", err.Error())
				}
				http.NotFound(w, r)
				return
			}
			callback := r.FormValue("callback")
			if callback != "" {
				w.Header().Set("Content-Type", "text/javascript")
				fmt.Fprintf(w, "%s(%s);", callback, resp)
			} else {
				w.Header().Set("Content-Type", "application/json")
				fmt.Fprintf(w, "%s", resp)
			}
		case 'x': // xml
			w.Header().Set("Content-Type", "application/xml")
			resp, err := xml.MarshalIndent(geoip, "", " ")
			if err != nil {
				if conf.Debug {
					log.Println("XML error:", err.Error())
				}
				http.Error(w, http.StatusText(500), 500)
				return
			}
			fmt.Fprintf(w, xml.Header+"%s\n", resp)
		}
	}
}

func HasQuota(rc *redis.Client, ipkey *string) (bool, error) {
	if ns, err := rc.Get(*ipkey); err != nil {
		return false, err
	} else if ns == "" {
		if err := rc.Set(*ipkey, "1"); err != nil {
			return false, err
		}
		rc.Expire(*ipkey, conf.Limit.Expire)
	} else if n, _ := strconv.Atoi(ns); n < conf.Limit.MaxRequests {
		rc.Incr(*ipkey)
	} else {
		return false, nil
	}
	return true, nil
}

const query = `SELECT
  city_location.country_code,
  country_blocks.country_name,
  city_location.region_code,
  region_names.region_name,
  city_location.city_name,
  city_location.postal_code,
  city_location.latitude,
  city_location.longitude,
  city_location.metro_code,
  city_location.area_code
FROM city_blocks
  NATURAL JOIN city_location
  INNER JOIN country_blocks ON
    city_location.country_code = country_blocks.country_code
  LEFT OUTER JOIN region_names ON
    city_location.country_code = region_names.country_code
    AND
    city_location.region_code = region_names.region_code
WHERE city_blocks.ip_start <= ?
ORDER BY city_blocks.ip_start DESC LIMIT 1`

func GeoipLookup(stmt *sql.Stmt, ip string) (*GeoIP, error) {
	IP := net.ParseIP(ip)
	var reserved bool
	for _, net := range reservedIPs {
		if net.Contains(IP) {
			reserved = true
			break
		}
	}
	geoip := GeoIP{Ip: ip}
	if reserved {
		geoip.CountryCode = "RD"
		geoip.CountryName = "Reserved"
	} else {
		var uintIP uint32
		binary.Read(bytes.NewBuffer(IP.To4()),
			binary.BigEndian, &uintIP)
		if err := stmt.QueryRow(uintIP).Scan(
			&geoip.CountryCode,
			&geoip.CountryName,
			&geoip.RegionCode,
			&geoip.RegionName,
			&geoip.CityName,
			&geoip.ZipCode,
			&geoip.Latitude,
			&geoip.Longitude,
			&geoip.MetroCode,
			&geoip.AreaCode,
		); err != nil {
			return nil, err
		}
	}
	return &geoip, nil
}

type GeoIP struct {
	XMLName     xml.Name `json:"-" xml:"Response"`
	Ip          string   `json:"ip"`
	CountryCode string   `json:"country_code"`
	CountryName string   `json:"country_name"`
	RegionCode  string   `json:"region_code"`
	RegionName  string   `json:"region_name"`
	CityName    string   `json:"city" xml:"City"`
	ZipCode     string   `json:"zipcode"`
	Latitude    float32  `json:"latitude"`
	Longitude   float32  `json:"longitude"`
	MetroCode   string   `json:"metro_code"`
	AreaCode    string   `json:"areacode"`
}

// http://en.wikipedia.org/wiki/Reserved_IP_addresses
var reservedIPs = []net.IPNet{
	{net.IPv4(0, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(10, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(100, 64, 0, 0), net.IPv4Mask(255, 192, 0, 0)},
	{net.IPv4(127, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(169, 254, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(172, 16, 0, 0), net.IPv4Mask(255, 240, 0, 0)},
	{net.IPv4(192, 0, 0, 0), net.IPv4Mask(255, 255, 255, 248)},
	{net.IPv4(192, 0, 2, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 88, 99, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 168, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(198, 18, 0, 0), net.IPv4Mask(255, 254, 0, 0)},
	{net.IPv4(198, 51, 100, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(203, 0, 113, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(224, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(240, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(255, 255, 255, 255), net.IPv4Mask(255, 255, 255, 255)},
}
<commit_msg>Optimizations<commit_after>// Copyright 2013 Alexandre Fiori
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

package main

import (
	"bytes"
	"database/sql"
	"encoding/binary"
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/fiorix/go-redis/redis"
	"github.com/fiorix/go-web/httpxtra"
	_ "github.com/mattn/go-sqlite3"
)

type Settings struct {
	XMLName      xml.Name `xml:"Server"`
	Debug        bool     `xml:"debug,attr"`
	XHeaders     bool     `xml:"xheaders,attr"`
	Addr         string   `xml:"addr,attr"`
	DocumentRoot string
	IPDB         struct {
		File      string `xml:",attr"`
		CacheSize string `xml:",attr"`
	}
	Limit struct {
		MaxRequests int `xml:",attr"`
		Expire      int `xml:",attr"`
	}
	Redis []string `xml:"Redis>Addr"`
}

var conf *Settings

func main() {
	cf := flag.String("config", "freegeoip.conf", "set config file")
	flag.Parse()
	if buf, err := ioutil.ReadFile(*cf); err != nil {
		log.Fatal(err)
	} else {
		conf = &Settings{}
		if err := xml.Unmarshal(buf, conf); err != nil {
			log.Fatal(err)
		}
	}
	http.Handle("/", http.FileServer(http.Dir(conf.DocumentRoot)))
	h := GeoipHandler()
	http.HandleFunc("/csv/", h)
	http.HandleFunc("/xml/", h)
	http.HandleFunc("/json/", h)
	server := http.Server{
		Addr: conf.Addr,
		Handler: httpxtra.Handler{
			Logger:   logger,
			XHeaders: conf.XHeaders,
		},
		ReadTimeout:  15 * time.Second,
		WriteTimeout: 15 * time.Second,
	}
	log.Printf("FreeGeoIP server starting on %s (xheaders=%t)",
		conf.Addr, conf.XHeaders)
	log.Fatal(httpxtra.ListenAndServe(server))
}

func logger(r *http.Request, created time.Time, status, bytes int) {
	//fmt.Println(httpxtra.ApacheCommonLog(r, created, status, bytes))
	log.Printf("HTTP %d %s %s (%s) :: %s",
		status,
		r.Method,
		r.URL.Path,
		r.RemoteAddr,
		time.Since(created))
}

// GeoipHandler handles GET on /csv, /xml and /json.
func GeoipHandler() http.HandlerFunc {
	db, err := sql.Open("sqlite3", conf.IPDB.File)
	if err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec("PRAGMA cache_size=" + conf.IPDB.CacheSize)
	if err != nil {
		log.Fatal(err)
	}
	stmt, err := db.Prepare(query)
	if err != nil {
		log.Fatal(err)
	}
	//defer stmt.Close()
	rc := redis.New(conf.Redis...)
	return func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "GET":
			w.Header().Set("Access-Control-Allow-Origin", "*")
		case "OPTIONS":
			w.Header().Set("Access-Control-Allow-Origin", "*")
			w.Header().Set("Content-Type", "text/plain")
			w.Header().Set("Access-Control-Allow-Methods", "GET")
			w.Header().Set("Access-Control-Allow-Headers", "X-Requested-With")
			w.WriteHeader(200)
			return
		default:
			w.Header().Set("Allow", "GET, OPTIONS")
			http.Error(w, http.StatusText(405), 405)
			return
		}
		// GET continuing...
		var (
			ip, ipkey string
			err       error
			ok        bool
		)
		if ip, _, err = net.SplitHostPort(r.RemoteAddr); err != nil {
			ipkey = r.RemoteAddr // Support for XHeaders
		} else {
			ipkey = ip
		}
		// Check quota
		if ok, err = HasQuota(rc, &ipkey); err != nil {
			// Redis down?
			if conf.Debug {
				log.Println("Redis error:", err.Error())
			}
			http.Error(w, http.StatusText(503), 503)
			return
		} else if !ok {
			// Over quota, soz :(
			http.Error(w, http.StatusText(403), 403)
			return
		}
		// Parse URL (e.g. /csv/ip, /xml/)
		a := strings.SplitN(r.URL.Path, "/", 3)
		if len(a) == 3 && a[2] != "" {
			addrs, err := net.LookupHost(a[2])
			if err != nil {
				// DNS lookup failed, assume host not found.
				http.Error(w, http.StatusText(404), 404)
				return
			}
			ip = addrs[0]
		} else {
			ip = ipkey
		}
		// Query the db
		geoip, err := GeoipLookup(stmt, ip)
		if err != nil {
			http.NotFound(w, r)
			return
		}
		switch a[1][0] {
		case 'c': // csv
			w.Header().Set("Content-Type", "application/csv")
			fmt.Fprintf(w, `"%s","%s","%s","%s","%s","%s",`+
				`"%s","%0.4f","%0.4f","%s","%s"`+"\r\n",
				geoip.Ip,
				geoip.CountryCode, geoip.CountryName,
				geoip.RegionCode, geoip.RegionName,
				geoip.CityName, geoip.ZipCode,
				geoip.Latitude, geoip.Longitude,
				geoip.MetroCode, geoip.AreaCode)
		case 'j': // json
			resp, err := json.Marshal(geoip)
			if err != nil {
				if conf.Debug {
					log.Println("JSON error:", err.Error())
				}
				http.NotFound(w, r)
				return
			}
			if cb := r.FormValue("callback"); cb != "" {
				w.Header().Set("Content-Type", "text/javascript")
				fmt.Fprintf(w, "%s(%s);", cb, resp)
			} else {
				w.Header().Set("Content-Type", "application/json")
				w.Write(resp)
			}
		case 'x': // xml
			w.Header().Set("Content-Type", "application/xml")
			resp, err := xml.MarshalIndent(geoip, "", " ")
			if err != nil {
				if conf.Debug {
					log.Println("XML error:", err.Error())
				}
				http.Error(w, http.StatusText(500), 500)
				return
			}
			fmt.Fprintf(w, xml.Header+"%s\n", resp)
		}
	}
}

func HasQuota(rc *redis.Client, ipkey *string) (bool, error) {
	if ns, err := rc.Get(*ipkey); err != nil {
		return false, err
	} else if ns == "" {
		if err := rc.Set(*ipkey, "1"); err != nil {
			return false, err
		}
		rc.Expire(*ipkey, conf.Limit.Expire)
	} else if n, _ := strconv.Atoi(ns); n < conf.Limit.MaxRequests {
		rc.Incr(*ipkey)
	} else {
		return false, nil
	}
	return true, nil
}

const query = `SELECT
  city_location.country_code,
  country_blocks.country_name,
  city_location.region_code,
  region_names.region_name,
  city_location.city_name,
  city_location.postal_code,
  city_location.latitude,
  city_location.longitude,
  city_location.metro_code,
  city_location.area_code
FROM city_blocks
  NATURAL JOIN city_location
  INNER JOIN country_blocks ON
    city_location.country_code = country_blocks.country_code
  LEFT OUTER JOIN region_names ON
    city_location.country_code = region_names.country_code
    AND
    city_location.region_code = region_names.region_code
WHERE city_blocks.ip_start <= ?
ORDER BY city_blocks.ip_start DESC LIMIT 1`

func GeoipLookup(stmt *sql.Stmt, ip string) (*GeoIP, error) {
	IP := net.ParseIP(ip)
	var reserved bool
	for _, net := range reservedIPs {
		if net.Contains(IP) {
			reserved = true
			break
		}
	}
	geoip := GeoIP{Ip: ip}
	if reserved {
		geoip.CountryCode = "RD"
		geoip.CountryName = "Reserved"
	} else {
		var uintIP uint32
		binary.Read(bytes.NewBuffer(IP.To4()),
			binary.BigEndian, &uintIP)
		if err := stmt.QueryRow(uintIP).Scan(
			&geoip.CountryCode,
			&geoip.CountryName,
			&geoip.RegionCode,
			&geoip.RegionName,
			&geoip.CityName,
			&geoip.ZipCode,
			&geoip.Latitude,
			&geoip.Longitude,
			&geoip.MetroCode,
			&geoip.AreaCode,
		); err != nil {
			return nil, err
		}
	}
	return &geoip, nil
}

type GeoIP struct {
	XMLName     xml.Name `json:"-" xml:"Response"`
	Ip          string   `json:"ip"`
	CountryCode string   `json:"country_code"`
	CountryName string   `json:"country_name"`
	RegionCode  string   `json:"region_code"`
	RegionName  string   `json:"region_name"`
	CityName    string   `json:"city" xml:"City"`
	ZipCode     string   `json:"zipcode"`
	Latitude    float32  `json:"latitude"`
	Longitude   float32  `json:"longitude"`
	MetroCode   string   `json:"metro_code"`
	AreaCode    string   `json:"areacode"`
}

// http://en.wikipedia.org/wiki/Reserved_IP_addresses
var reservedIPs = []net.IPNet{
	{net.IPv4(0, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(10, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(100, 64, 0, 0), net.IPv4Mask(255, 192, 0, 0)},
	{net.IPv4(127, 0, 0, 0), net.IPv4Mask(255, 0, 0, 0)},
	{net.IPv4(169, 254, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(172, 16, 0, 0), net.IPv4Mask(255, 240, 0, 0)},
	{net.IPv4(192, 0, 0, 0), net.IPv4Mask(255, 255, 255, 248)},
	{net.IPv4(192, 0, 2, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 88, 99, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(192, 168, 0, 0), net.IPv4Mask(255, 255, 0, 0)},
	{net.IPv4(198, 18, 0, 0), net.IPv4Mask(255, 254, 0, 0)},
	{net.IPv4(198, 51, 100, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(203, 0, 113, 0), net.IPv4Mask(255, 255, 255, 0)},
	{net.IPv4(224, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(240, 0, 0, 0), net.IPv4Mask(240, 0, 0, 0)},
	{net.IPv4(255, 255, 255, 255), net.IPv4Mask(255, 255, 255, 255)},
}
<|endoftext|>


<commit_before>// Copyright 2014 The mqrouter Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dbrouter

import (
	"context"
	"fmt"
	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/log"
	"github.com/shawnfeng/sutil/slog/slog"
	stat "github.com/shawnfeng/sutil/stat"
	"github.com/shawnfeng/sutil/stime"
	"gopkg.in/mgo.v2"
)

const (
	spanLogKeyCluster = "cluster"
	spanLogKeyTable   = "table"
)

type Router struct {
	configer  Configer
	instances *InstanceManager
	report    *stat.StatReport
}

type dbConfigChange struct {
	dbInstanceChange map[string][]string
	dbGroups         []string
}

func NewRouter(data []byte) (*Router, error) {
	// TODO config type由哪里决定
	var dbChangeChan = make(chan dbConfigChange)
	configer, err := NewConfiger(CONFIG_TYPE_ETCD, data, dbChangeChan)
	if err != nil {
		return nil, err
	}

	factory := func(configer Configer) func(ctx context.Context, key, group string) (in Instancer, err error) {
		return func(ctx context.Context, key, group string) (in Instancer, err error) {
			return Factory(ctx, key, group, configer)
		}
	}(configer)

	return &Router{
		configer:  configer,
		instances: NewInstanceManager(factory, dbChangeChan, configer.GetGroups(context.TODO())),
		report:    stat.NewStat(),
	}, nil
}

func (m *Router) StatInfo() []*stat.QueryStat {
	return m.report.StatInfo()
}

func (m *Router) sqlPrepare(ctx context.Context, cluster, table string) (db *DB, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.sqlPrepare")
	defer span.Finish()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		err = fmt.Errorf("db instance not find: instance:%s", instance)
		return
	}

	dbsql, ok := in.(*Sql)
	if !ok {
		err = fmt.Errorf("db instance type error: instance:%s, dbtype:%s", instance, in.GetType())
		return
	}

	db = dbsql.getDB()
	return
}

func (m *Router) SqlExec(ctx context.Context, cluster string, query func(*DB, []interface{}) error, tables ...string) error {
	fun := "Router.SqlExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.SqlExec")
	defer span.Finish()

	st := stime.NewTimeStat()

	if len(tables) <= 0 {
		return fmt.Errorf("tables is empty")
	}
	table := tables[0]

	span.LogFields(
		log.String(spanLogKeyCluster, cluster),
		log.String(spanLogKeyTable, table))

	db, err := m.sqlPrepare(ctx, cluster, table)
	if err != nil {
		return err
	}

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef(ctx, "%s cls:%s table:%s dur:%d", fun, cluster, table, dur)
	}()

	var tmptables []interface{}
	for _, item := range tables {
		tmptables = append(tmptables, item)
	}

	return query(db, tmptables)
}

func (m *Router) ormPrepare(ctx context.Context, cluster, table string) (db *GormDB, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.ormPrepare")
	defer span.Finish()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		err = fmt.Errorf("db instance not find: instance:%s", instance)
		return
	}

	dbsql, ok := in.(*Sql)
	if !ok {
		err = fmt.Errorf("db instance type error: instance:%s, dbtype:%s", instance, in.GetType())
		return
	}

	db = dbsql.getGormDB()
	return
}

func (m *Router) OrmExec(ctx context.Context, cluster string, query func(*GormDB, []interface{}) error, tables ...string) error {
	fun := "Router.OrmExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.OrmExec")
	defer span.Finish()

	st := stime.NewTimeStat()

	if len(tables) <= 0 {
		return fmt.Errorf("tables is empty")
	}

	table := tables[0]
	span.LogFields(
		log.String(spanLogKeyCluster, cluster),
		log.String(spanLogKeyTable, table))

	db, err := m.ormPrepare(ctx, cluster, table)
	if err != nil {
		return err
	}

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef(ctx, "%s cls:%s table:%s dur:%d", fun, cluster, table, dur)
	}()

	var tmptables []interface{}
	for _, item := range tables {
		tmptables = append(tmptables, item)
	}

	return query(db, tmptables)
}

func (m *Router) MongoExecEventual(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, eventual, cluster, table, query)
}

func (m *Router) MongoExecMonotonic(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, monotonic, cluster, table, query)
}

func (m *Router) MongoExecStrong(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, strong, cluster, table, query)
}

func (m *Router) mongoPrepare(ctx context.Context, consistency mode, cluster, table string) (coll *mgo.Collection, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.mongoPrepare")
	defer span.Finish()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		err = fmt.Errorf("db instance not find: cluster:%s table:%s", cluster, table)
		return
	}

	db, ok := in.(*dbMongo)
	if !ok {
		err = fmt.Errorf("db instance type error: cluster:%s table:%s type:%s", cluster, table, in.GetType())
		return
	}

	sess, err := db.getSession(consistency)
	if err != nil {
		return
	}

	if sess == nil {
		err = fmt.Errorf("db instance session empty: cluster:%s table:%s type:%s", cluster, table, in.GetType())
		return
	}

	sessionCopy := sess.Copy()
	defer sessionCopy.Close()
	coll = sessionCopy.DB("").C(table)
	return
}

func (m *Router) mongoExec(ctx context.Context, consistency mode, cluster, table string, query func(*mgo.Collection) error) error {
	fun := "Router.mongoExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.mongoExec")
	defer span.Finish()
	span.LogFields(
		log.String(spanLogKeyCluster, cluster),
		log.String(spanLogKeyTable, table))

	st := stime.NewTimeStat()

	c, err := m.mongoPrepare(ctx, consistency, cluster, table)
	if err != nil {
		return err
	}

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef(ctx, "%s const:%d cls:%s table:%s dur:%d", fun, consistency, cluster, table, dur)
	}()

	return query(c)
}
<commit_msg>bugfix: close sessionCopy outside<commit_after>// Copyright 2014 The mqrouter Author. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dbrouter

import (
	"context"
	"fmt"
	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/log"
	"github.com/shawnfeng/sutil/slog/slog"
	stat "github.com/shawnfeng/sutil/stat"
	"github.com/shawnfeng/sutil/stime"
	"gopkg.in/mgo.v2"
)

const (
	spanLogKeyCluster = "cluster"
	spanLogKeyTable   = "table"
)

type Router struct {
	configer  Configer
	instances *InstanceManager
	report    *stat.StatReport
}

type dbConfigChange struct {
	dbInstanceChange map[string][]string
	dbGroups         []string
}

func NewRouter(data []byte) (*Router, error) {
	// TODO config type由哪里决定
	var dbChangeChan = make(chan dbConfigChange)
	configer, err := NewConfiger(CONFIG_TYPE_ETCD, data, dbChangeChan)
	if err != nil {
		return nil, err
	}

	factory := func(configer Configer) func(ctx context.Context, key, group string) (in Instancer, err error) {
		return func(ctx context.Context, key, group string) (in Instancer, err error) {
			return Factory(ctx, key, group, configer)
		}
	}(configer)

	return &Router{
		configer:  configer,
		instances: NewInstanceManager(factory, dbChangeChan, configer.GetGroups(context.TODO())),
		report:    stat.NewStat(),
	}, nil
}

func (m *Router) StatInfo() []*stat.QueryStat {
	return m.report.StatInfo()
}

func (m *Router) sqlPrepare(ctx context.Context, cluster, table string) (db *DB, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.sqlPrepare")
	defer span.Finish()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		err = fmt.Errorf("db instance not find: instance:%s", instance)
		return
	}

	dbsql, ok := in.(*Sql)
	if !ok {
		err = fmt.Errorf("db instance type error: instance:%s, dbtype:%s", instance, in.GetType())
		return
	}

	db = dbsql.getDB()
	return
}

func (m *Router) SqlExec(ctx context.Context, cluster string, query func(*DB, []interface{}) error, tables ...string) error {
	fun := "Router.SqlExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.SqlExec")
	defer span.Finish()

	st := stime.NewTimeStat()

	if len(tables) <= 0 {
		return fmt.Errorf("tables is empty")
	}
	table := tables[0]

	span.LogFields(
		log.String(spanLogKeyCluster, cluster),
		log.String(spanLogKeyTable, table))

	db, err := m.sqlPrepare(ctx, cluster, table)
	if err != nil {
		return err
	}

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef(ctx, "%s cls:%s table:%s dur:%d", fun, cluster, table, dur)
	}()

	var tmptables []interface{}
	for _, item := range tables {
		tmptables = append(tmptables, item)
	}

	return query(db, tmptables)
}

func (m *Router) ormPrepare(ctx context.Context, cluster, table string) (db *GormDB, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.ormPrepare")
	defer span.Finish()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		err = fmt.Errorf("db instance not find: instance:%s", instance)
		return
	}

	dbsql, ok := in.(*Sql)
	if !ok {
		err = fmt.Errorf("db instance type error: instance:%s, dbtype:%s", instance, in.GetType())
		return
	}

	db = dbsql.getGormDB()
	return
}

func (m *Router) OrmExec(ctx context.Context, cluster string, query func(*GormDB, []interface{}) error, tables ...string) error {
	fun := "Router.OrmExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.OrmExec")
	defer span.Finish()

	st := stime.NewTimeStat()

	if len(tables) <= 0 {
		return fmt.Errorf("tables is empty")
	}

	table := tables[0]
	span.LogFields(
		log.String(spanLogKeyCluster, cluster),
		log.String(spanLogKeyTable, table))

	db, err := m.ormPrepare(ctx, cluster, table)
	if err != nil {
		return err
	}

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef(ctx, "%s cls:%s table:%s dur:%d", fun, cluster, table, dur)
	}()

	var tmptables []interface{}
	for _, item := range tables {
		tmptables = append(tmptables, item)
	}

	return query(db, tmptables)
}

func (m *Router) MongoExecEventual(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, eventual, cluster, table, query)
}

func (m *Router) MongoExecMonotonic(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, monotonic, cluster, table, query)
}

func (m *Router) MongoExecStrong(ctx context.Context, cluster, table string, query func(*mgo.Collection) error) error {
	return m.mongoExec(ctx, strong, cluster, table, query)
}

func (m *Router) mongoPrepare(ctx context.Context, consistency mode, cluster, table string) (sess *mgo.Session, err error) {
	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.mongoPrepare")
	defer span.Finish()

	instance := m.configer.GetInstance(ctx, cluster, table)
	in := m.instances.Get(ctx, generateKey(instance))
	if in == nil {
		err = fmt.Errorf("db instance not find: cluster:%s table:%s", cluster, table)
		return
	}

	db, ok := in.(*dbMongo)
	if !ok {
		err = fmt.Errorf("db instance type error: cluster:%s table:%s type:%s", cluster, table, in.GetType())
		return
	}

	ss, err := db.getSession(consistency)
	if err != nil {
		return
	}

	if ss == nil {
		err = fmt.Errorf("db instance session empty: cluster:%s table:%s type:%s", cluster, table, in.GetType())
		return
	}

	sess = ss.Copy()
	return
}

func (m *Router) mongoExec(ctx context.Context, consistency mode, cluster, table string, query func(*mgo.Collection) error) error {
	fun := "Router.mongoExec -->"

	span, ctx := opentracing.StartSpanFromContext(ctx, "dbrouter.mongoExec")
	defer span.Finish()
	span.LogFields(
		log.String(spanLogKeyCluster, cluster),
		log.String(spanLogKeyTable, table))

	st := stime.NewTimeStat()

	sess, err := m.mongoPrepare(ctx, consistency, cluster, table)
	if err != nil {
		return err
	}
	defer sess.Close()
	coll := sess.DB("").C(table)

	defer func() {
		dur := st.Duration()
		m.report.IncQuery(cluster, table, st.Duration())
		slog.Tracef(ctx, "%s const:%d cls:%s table:%s dur:%d", fun, consistency, cluster, table, dur)
	}()

	return query(coll)
}
<|endoftext|>


<commit_before>package main

import (
    "database/sql"
    "encoding/json"
    "fmt"
    "io/ioutil"
    "regexp"
    "strings"
    "time"

    _ "github.com/lib/pq"
    _ "github.com/ziutek/mymysql/godrv"
)

// evo_categories
type Category struct {
    id        int64
    parent_id int64
    name      string
    url       string
}

// evo_items__item
type Post struct {
    id        int64
    id_sqlite int64
    date      time.Time
    body      string
    title     string
    url       string
}

// evo_comments
type Comment struct {
    postId        int64
    postId_sqlite int64
    authorId      sql.NullInt64
    author        sql.NullString
    authorEmail   sql.NullString
    authorUrl     sql.NullString
    authorIp      sql.NullString
    date          time.Time
    content       string
}

// evo_users
type User struct {
    id        int64
    id_sqlite int64
    login     string
    firstname string
    lastname  string
    nickname  string
    email     string
    url       string
    ip        string
}

type DbConfig map[string]string

func xferPosts(sconn, mconn *sql.DB) (posts []*Post, err error) {
    asterisk := regexp.MustCompile(`(\n\[\*\](.*)\n)`)
    links := regexp.MustCompile(`silavinimui:\r\n`)
    rows, err := mconn.Query(`select post_ID, post_datecreated, post_content,
                                     post_title, post_urltitle
                              from evo_items__item
                              where post_creator_user_ID=?`, 15)
    //posts := make([]*Post, 0, 10)
    for rows.Next() {
        var p Post
        err = rows.Scan(&p.id, &p.date, &p.body, &p.title, &p.url)
        if err != nil {
            fmt.Printf("err: %s\n" + err.Error())
        }
        posts = append(posts, &p)
    }
    fmt.Printf("#posts: %d\n", len(posts))
    xaction, err := sconn.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    for _, p := range posts {
        stmt, err := xaction.Prepare(`insert into post
                                      (author_id, title, date, url, body)
                                      values($1, $2, $3, $4, $5)
                                      returning id`)
        if err != nil {
            fmt.Println(err)
            return posts, err
        }
        defer stmt.Close()
        if p.url == "arrr" {
            p.body = asterisk.ReplaceAllString(p.body, "$1\n")
        }
        if p.url == "uzreferenduma-lt-the-good-the-bad-and-the-ugly" {
            prefix := links.FindStringIndex(p.body)
            startLinks := prefix[1]
            p.body = p.body[:startLinks] + fixupLinks(p.body[startLinks:startLinks+357]) + p.body[startLinks+357:]
        }
        newBody := fixupBody(p.body)
        rows := stmt.QueryRow(1, p.title, p.date.Unix(), p.url, newBody)
        rows.Scan(&p.id_sqlite)
        //fmt.Printf("%+v\n", p)
        //fmt.Printf("%q | %q\n", p.title, p.url)
    }
    xaction.Commit()
    return posts, err
}

func fixupLinks(olinks string) (nlinks string) {
    lst := strings.Split(olinks, "\n")
    nlst := make([]string, 0, len(lst))
    for _, line := range lst {
        s := strings.TrimSpace(line)
        if s == "" {
            continue
        }
        nlst = append(nlst, fmt.Sprintf(" - %s\n", s))
    }
    nlinks = "\n" + strings.Join(nlst, "\n") + "\n"
    return
}

func fixupBody(obody string) (nbody string) {
    nbody = fixupPre(obody)
    nbody = fixupTt(nbody)
    nbody = fixupOl(nbody)
    nbody = fixupImgLinks(nbody)
    nbody = strings.Replace(nbody, "pasistatyi", "pasistatyti", -1)
    nbody = strings.Replace(nbody, "sąngražinės", "sangrąžinės", -1)
    return
}

func fixupImgLinks(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    for _, line := range ilines {
        newline := strings.Replace(line, "http://blog.stent.lt/media/blogs/rtfb", "", -1)
        newline = strings.Replace(newline, "h3", "h4", -1)
        olines = append(olines, newline)
    }
    nbody = strings.Join(olines, "\n")
    return
}

func fixupPre(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    inPre := false
    for _, line := range ilines {
        if strings.Contains(line, "
") {
            inPre = true
            line = strings.Replace(line, "
", "", -1)
        }
        if strings.Contains(line, "
") {
            inPre = false
            line = strings.Replace(line, "
", "", -1)
        }
        if inPre {
            olines = append(olines, "    "+line)
        } else {
            olines = append(olines, line)
        }
    }
    nbody = strings.Join(olines, "\n")
    return
}

func fixupTt(obody string) (nbody string) {
    nbody = strings.Replace(obody, "<tt>", "`", -1)
    nbody = strings.Replace(nbody, "</tt>", "`", -1)
    return
}

func fixupOl(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    inList := false
    for _, line := range ilines {
        if strings.Contains(line, "<ol") {
            inList = true
        }
        if strings.Contains(line, "
") {
            inList = false
            s := strings.Replace(strings.TrimSpace(line), "
", "1. ", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            olines = append(olines, s)
            continue
        }
        if inList && strings.TrimSpace(line) == "" {
            continue
        } else if inList {
            s := strings.Replace(strings.TrimSpace(line), "
", "1. ", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            olines = append(olines, s)
        } else {
            olines = append(olines, line)
        }
    }
    nbody = strings.Join(olines, "\n")
    return
}

func xferComments(sconn, mconn *sql.DB, posts []*Post) {
    comms := make([]*Comment, 0, 10)
    for _, p := range posts {
        rows, err := mconn.Query(`select comment_post_ID, comment_author_ID,
                                         comment_author, comment_author_email,
                                         comment_author_url, comment_author_IP,
                                         comment_date, comment_content,
                                         comment_ID
                                  from evo_comments
                                  where comment_post_ID=?`, p.id)
        for rows.Next() {
            var c Comment
            var cid int
            err = rows.Scan(&c.postId, &c.authorId, &c.author, &c.authorEmail,
                &c.authorUrl, &c.authorIp, &c.date, &c.content, &cid)
            if err != nil {
                fmt.Printf("err: %s\n" + err.Error())
            }
            if strings.Contains(c.content, "Honesty is the rarest wealth anyone can possess") ||
                strings.Contains(c.content, "I do that you are going to be elaborating more on this issue") {
                fmt.Printf("skipping spam comment, id=%d\n", cid)
                continue
            }
            c.postId_sqlite = p.id_sqlite
            comms = append(comms, &c)
        }
    }
    fmt.Printf("#comms: %d\n", len(comms))
    xaction, err := sconn.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    for _, c := range comms {
        //fmt.Printf("%+v\n", c)
        authorId, err := getCommenterId(xaction, mconn, c)
        if err == sql.ErrNoRows {
            insertCommenter, _ := xaction.Prepare(`insert into commenter
                                                   (name, email, www, ip)
                                                   values ($1, $2, $3, $4)
                                                   returning id`)
            defer insertCommenter.Close()
            ip := ""
            if c.authorIp.Valid {
                ip = c.authorIp.String
            }
            rows := insertCommenter.QueryRow(c.author, c.authorEmail,
                c.authorUrl, ip)
            err = rows.Scan(&authorId)
            if err != nil {
                fmt.Println("Failed to insert commenter: " + err.Error())
            }
        } else if err != nil {
            fmt.Println("err: " + err.Error())
            continue
        }
        stmt, err := xaction.Prepare(`insert into comment
                                      (commenter_id, post_id, timestamp, body)
                                      values($1, $2, $3, $4)`)
        defer stmt.Close()
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
        _, err = stmt.Exec(authorId, c.postId_sqlite, c.date.Unix(), c.content)
        if c.authorId.Int64 == 10 {
            //fmt.Printf("%+v\n", c)
        }
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
    }
    xaction.Commit()
}

func getCommenterId(xaction *sql.Tx, mconn *sql.DB, comment *Comment) (id int64, err error) {
    if comment.authorId.Valid {
        query, err := mconn.Prepare(`select user_nickname, user_email,
                                              user_url, user_ip
                                       from evo_users
                                       where user_ID=?`)
        defer query.Close()
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
        err = query.QueryRow(comment.authorId.Int64).Scan(&comment.author,
            &comment.authorEmail,
            &comment.authorUrl,
            &comment.authorIp)
    }
    query, _ := xaction.Prepare(`select c.id from commenter as c
                                 where c.email like $1`)
    defer query.Close()
    err = query.QueryRow(comment.authorEmail.String).Scan(&id)
    return
}

func xferTags(sconn, mconn *sql.DB, posts []*Post) {
    for _, p := range posts {
        rows, err := mconn.Query(`select t.tag_name
                                  from evo_items__tag as t,
                                       evo_items__itemtag as it
                                  where t.tag_ID=it.itag_tag_ID
                                    and it.itag_itm_ID=?`, p.id)
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
        for rows.Next() {
            var tag string
            err = rows.Scan(&tag)
            if err != nil {
                fmt.Printf("err: %s\n" + err.Error())
            }
            fixedTag := strings.Replace(tag, " ", "-", -1)
            row := sconn.QueryRow(`select id from tag where url=$1`, fixedTag)
            var tagId int64
            err = row.Scan(&tagId)
            if err != nil {
                if err == sql.ErrNoRows {
                    stmt, err := sconn.Prepare(`insert into tag
                                                (name, url)
                                                values($1, $2)
                                                returning id`)
                    if err != nil {
                        fmt.Println(err)
                        continue
                    }
                    defer stmt.Close()
                    rows := stmt.QueryRow(strings.Title(tag), fixedTag)
                    err = rows.Scan(&tagId)
                } else {
                    fmt.Printf("err: %s\n", err.Error())
                }
            }
            stmt, err := sconn.Prepare(`insert into tagmap
                                        (tag_id, post_id)
                                        values($1, $2)`)
            if err != nil {
                fmt.Println(err)
                continue
            }
            defer stmt.Close()
            _, err = stmt.Exec(tagId, p.id_sqlite)
            if err != nil {
                fmt.Printf("err inserting tagmap: %s\n", err.Error())
            }
        }
    }
}

func readConf(path string) (db, uname, passwd string) {
    b, err := ioutil.ReadFile(path)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    var config DbConfig
    err = json.Unmarshal(b, &config)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    return config["db_name"], config["uname"], config["passwd"]
}

func importLegacyDb(sqliteFile, dbConf string) {
    db, uname, passwd := readConf(dbConf)
    mconn, err := sql.Open("mymysql", fmt.Sprintf("%s/%s/%s", db, uname, passwd))
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    defer mconn.Close()
    sconn, err := sql.Open("postgres", sqliteFile)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    defer sconn.Close()
    row := mconn.QueryRow(`select blog_shortname, blog_name, blog_owner_user_ID
                           from evo_blogs where blog_ID=?`, 19)
    shortname := ""
    name := ""
    uid := 0
    err = row.Scan(&shortname, &name, &uid)
    if err != nil {
        fmt.Printf("err: " + err.Error())
    } else {
        fmt.Printf("shortname: %q, name: %q, id=%d\n", shortname, name, uid)
    }
    rows, err := mconn.Query(`select cat_ID, cat_parent_ID,
                                     cat_name, cat_urlname
                              from evo_categories
                              where cat_blog_ID=?`, 19)
    cat := make([]*Category, 0, 10)
    var id int64
    var parent_id sql.NullInt64
    var url string
    for rows.Next() {
        err = rows.Scan(&id, &parent_id, &name, &url)
        if err != nil {
            fmt.Printf("err: %s\n" + err.Error())
        }
        cat = append(cat, &Category{id, parent_id.Int64, name, url})
    }
    fmt.Printf("#categories: %d\n", len(cat))
    posts, err := xferPosts(sconn, mconn)
    if err != nil {
        fmt.Printf("err: " + err.Error())
    }
    xferComments(sconn, mconn, posts)
    xferTags(sconn, mconn, posts)
}
<commit_msg>Improve error reporting<commit_after>package main

import (
    "database/sql"
    "encoding/json"
    "fmt"
    "io/ioutil"
    "regexp"
    "strings"
    "time"

    _ "github.com/lib/pq"
    _ "github.com/ziutek/mymysql/godrv"
)

// evo_categories
type Category struct {
    id        int64
    parent_id int64
    name      string
    url       string
}

// evo_items__item
type Post struct {
    id        int64
    id_sqlite int64
    date      time.Time
    body      string
    title     string
    url       string
}

// evo_comments
type Comment struct {
    postId        int64
    postId_sqlite int64
    authorId      sql.NullInt64
    author        sql.NullString
    authorEmail   sql.NullString
    authorUrl     sql.NullString
    authorIp      sql.NullString
    date          time.Time
    content       string
}

// evo_users
type User struct {
    id        int64
    id_sqlite int64
    login     string
    firstname string
    lastname  string
    nickname  string
    email     string
    url       string
    ip        string
}

type DbConfig map[string]string

func xferPosts(sconn, mconn *sql.DB) (posts []*Post, err error) {
    asterisk := regexp.MustCompile(`(\n\[\*\](.*)\n)`)
    links := regexp.MustCompile(`silavinimui:\r\n`)
    rows, err := mconn.Query(`select post_ID, post_datecreated, post_content,
                                     post_title, post_urltitle
                              from evo_items__item
                              where post_creator_user_ID=?`, 15)
    //posts := make([]*Post, 0, 10)
    for rows.Next() {
        var p Post
        err = rows.Scan(&p.id, &p.date, &p.body, &p.title, &p.url)
        if err != nil {
            fmt.Printf("err: %s\n" + err.Error())
        }
        posts = append(posts, &p)
    }
    fmt.Printf("#posts: %d\n", len(posts))
    xaction, err := sconn.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    for _, p := range posts {
        stmt, err := xaction.Prepare(`insert into post
                                      (author_id, title, date, url, body)
                                      values($1, $2, $3, $4, $5)
                                      returning id`)
        if err != nil {
            fmt.Println(err)
            return posts, err
        }
        defer stmt.Close()
        if p.url == "arrr" {
            p.body = asterisk.ReplaceAllString(p.body, "$1\n")
        }
        if p.url == "uzreferenduma-lt-the-good-the-bad-and-the-ugly" {
            prefix := links.FindStringIndex(p.body)
            startLinks := prefix[1]
            p.body = p.body[:startLinks] + fixupLinks(p.body[startLinks:startLinks+357]) + p.body[startLinks+357:]
        }
        newBody := fixupBody(p.body)
        rows := stmt.QueryRow(1, p.title, p.date.Unix(), p.url, newBody)
        err = rows.Scan(&p.id_sqlite)
        if err != nil {
            fmt.Println(err)
            return posts, err
        }
        //fmt.Printf("%+v\n", p)
        //fmt.Printf("%q | %q\n", p.title, p.url)
    }
    xaction.Commit()
    return posts, err
}

func fixupLinks(olinks string) (nlinks string) {
    lst := strings.Split(olinks, "\n")
    nlst := make([]string, 0, len(lst))
    for _, line := range lst {
        s := strings.TrimSpace(line)
        if s == "" {
            continue
        }
        nlst = append(nlst, fmt.Sprintf(" - %s\n", s))
    }
    nlinks = "\n" + strings.Join(nlst, "\n") + "\n"
    return
}

func fixupBody(obody string) (nbody string) {
    nbody = fixupPre(obody)
    nbody = fixupTt(nbody)
    nbody = fixupOl(nbody)
    nbody = fixupImgLinks(nbody)
    nbody = strings.Replace(nbody, "pasistatyi", "pasistatyti", -1)
    nbody = strings.Replace(nbody, "sąngražinės", "sangrąžinės", -1)
    return
}

func fixupImgLinks(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    for _, line := range ilines {
        newline := strings.Replace(line, "http://blog.stent.lt/media/blogs/rtfb", "", -1)
        newline = strings.Replace(newline, "h3", "h4", -1)
        olines = append(olines, newline)
    }
    nbody = strings.Join(olines, "\n")
    return
}

func fixupPre(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    inPre := false
    for _, line := range ilines {
        if strings.Contains(line, "
") {
            inPre = true
            line = strings.Replace(line, "
", "", -1)
        }
        if strings.Contains(line, "
") {
            inPre = false
            line = strings.Replace(line, "
", "", -1)
        }
        if inPre {
            olines = append(olines, "    "+line)
        } else {
            olines = append(olines, line)
        }
    }
    nbody = strings.Join(olines, "\n")
    return
}

func fixupTt(obody string) (nbody string) {
    nbody = strings.Replace(obody, "<tt>", "`", -1)
    nbody = strings.Replace(nbody, "</tt>", "`", -1)
    return
}

func fixupOl(obody string) (nbody string) {
    ilines := strings.Split(obody, "\n")
    olines := make([]string, 0, len(ilines))
    inList := false
    for _, line := range ilines {
        if strings.Contains(line, "<ol") {
            inList = true
        }
        if strings.Contains(line, "
") {
            inList = false
            s := strings.Replace(strings.TrimSpace(line), "
", "1. ", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            olines = append(olines, s)
            continue
        }
        if inList && strings.TrimSpace(line) == "" {
            continue
        } else if inList {
            s := strings.Replace(strings.TrimSpace(line), "
", "1. ", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            s = strings.Replace(s, "
", "", -1)
            olines = append(olines, s)
        } else {
            olines = append(olines, line)
        }
    }
    nbody = strings.Join(olines, "\n")
    return
}

func xferComments(sconn, mconn *sql.DB, posts []*Post) {
    comms := make([]*Comment, 0, 10)
    for _, p := range posts {
        rows, err := mconn.Query(`select comment_post_ID, comment_author_ID,
                                         comment_author, comment_author_email,
                                         comment_author_url, comment_author_IP,
                                         comment_date, comment_content,
                                         comment_ID
                                  from evo_comments
                                  where comment_post_ID=?`, p.id)
        for rows.Next() {
            var c Comment
            var cid int
            err = rows.Scan(&c.postId, &c.authorId, &c.author, &c.authorEmail,
                &c.authorUrl, &c.authorIp, &c.date, &c.content, &cid)
            if err != nil {
                fmt.Printf("err: %s\n" + err.Error())
            }
            if strings.Contains(c.content, "Honesty is the rarest wealth anyone can possess") ||
                strings.Contains(c.content, "I do that you are going to be elaborating more on this issue") {
                fmt.Printf("skipping spam comment, id=%d\n", cid)
                continue
            }
            c.postId_sqlite = p.id_sqlite
            comms = append(comms, &c)
        }
    }
    fmt.Printf("#comms: %d\n", len(comms))
    xaction, err := sconn.Begin()
    if err != nil {
        fmt.Println(err)
        return
    }
    for _, c := range comms {
        //fmt.Printf("%+v\n", c)
        authorId, err := getCommenterId(xaction, mconn, c)
        if err == sql.ErrNoRows {
            insertCommenter, _ := xaction.Prepare(`insert into commenter
                                                   (name, email, www, ip)
                                                   values ($1, $2, $3, $4)
                                                   returning id`)
            defer insertCommenter.Close()
            ip := ""
            if c.authorIp.Valid {
                ip = c.authorIp.String
            }
            rows := insertCommenter.QueryRow(c.author, c.authorEmail,
                c.authorUrl, ip)
            err = rows.Scan(&authorId)
            if err != nil {
                fmt.Println("Failed to insert commenter: " + err.Error())
                continue
            }
        } else if err != nil {
            fmt.Println("err: " + err.Error())
            continue
        }
        stmt, err := xaction.Prepare(`insert into comment
                                      (commenter_id, post_id, timestamp, body)
                                      values($1, $2, $3, $4)`)
        defer stmt.Close()
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
        _, err = stmt.Exec(authorId, c.postId_sqlite, c.date.Unix(), c.content)
        if c.authorId.Int64 == 10 {
            //fmt.Printf("%+v\n", c)
        }
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
    }
    xaction.Commit()
}

func getCommenterId(xaction *sql.Tx, mconn *sql.DB, comment *Comment) (id int64, err error) {
    if comment.authorId.Valid {
        query, err := mconn.Prepare(`select user_nickname, user_email,
                                              user_url, user_ip
                                       from evo_users
                                       where user_ID=?`)
        defer query.Close()
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
        err = query.QueryRow(comment.authorId.Int64).Scan(&comment.author,
            &comment.authorEmail,
            &comment.authorUrl,
            &comment.authorIp)
    }
    query, err := xaction.Prepare(`select c.id from commenter as c
                                   where c.email like $1`)
    defer query.Close()
    if err != nil {
        fmt.Println(err)
    }
    err = query.QueryRow(comment.authorEmail.String).Scan(&id)
    return
}

func xferTags(sconn, mconn *sql.DB, posts []*Post) {
    for _, p := range posts {
        rows, err := mconn.Query(`select t.tag_name
                                  from evo_items__tag as t,
                                       evo_items__itemtag as it
                                  where t.tag_ID=it.itag_tag_ID
                                    and it.itag_itm_ID=?`, p.id)
        if err != nil {
            fmt.Printf("err: %s\n", err.Error())
        }
        for rows.Next() {
            var tag string
            err = rows.Scan(&tag)
            if err != nil {
                fmt.Printf("err: %s\n" + err.Error())
            }
            fixedTag := strings.Replace(tag, " ", "-", -1)
            row := sconn.QueryRow(`select id from tag where url=$1`, fixedTag)
            var tagId int64
            err = row.Scan(&tagId)
            if err != nil {
                if err == sql.ErrNoRows {
                    stmt, err := sconn.Prepare(`insert into tag
                                                (name, url)
                                                values($1, $2)
                                                returning id`)
                    if err != nil {
                        fmt.Println(err)
                        continue
                    }
                    defer stmt.Close()
                    rows := stmt.QueryRow(strings.Title(tag), fixedTag)
                    err = rows.Scan(&tagId)
                } else {
                    fmt.Printf("err: %s\n", err.Error())
                }
            }
            stmt, err := sconn.Prepare(`insert into tagmap
                                        (tag_id, post_id)
                                        values($1, $2)`)
            if err != nil {
                fmt.Println(err)
                continue
            }
            defer stmt.Close()
            _, err = stmt.Exec(tagId, p.id_sqlite)
            if err != nil {
                fmt.Printf("err inserting tagmap: %s\n", err.Error())
            }
        }
    }
}

func readConf(path string) (db, uname, passwd string) {
    b, err := ioutil.ReadFile(path)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    var config DbConfig
    err = json.Unmarshal(b, &config)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    return config["db_name"], config["uname"], config["passwd"]
}

func importLegacyDb(sqliteFile, dbConf string) {
    db, uname, passwd := readConf(dbConf)
    mconn, err := sql.Open("mymysql", fmt.Sprintf("%s/%s/%s", db, uname, passwd))
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    defer mconn.Close()
    sconn, err := sql.Open("postgres", sqliteFile)
    if err != nil {
        fmt.Println(err.Error())
        return
    }
    defer sconn.Close()
    row := mconn.QueryRow(`select blog_shortname, blog_name, blog_owner_user_ID
                           from evo_blogs where blog_ID=?`, 19)
    shortname := ""
    name := ""
    uid := 0
    err = row.Scan(&shortname, &name, &uid)
    if err != nil {
        fmt.Printf("err: " + err.Error())
    } else {
        fmt.Printf("shortname: %q, name: %q, id=%d\n", shortname, name, uid)
    }
    rows, err := mconn.Query(`select cat_ID, cat_parent_ID,
                                     cat_name, cat_urlname
                              from evo_categories
                              where cat_blog_ID=?`, 19)
    cat := make([]*Category, 0, 10)
    var id int64
    var parent_id sql.NullInt64
    var url string
    for rows.Next() {
        err = rows.Scan(&id, &parent_id, &name, &url)
        if err != nil {
            fmt.Printf("err: %s\n" + err.Error())
        }
        cat = append(cat, &Category{id, parent_id.Int64, name, url})
    }
    fmt.Printf("#categories: %d\n", len(cat))
    posts, err := xferPosts(sconn, mconn)
    if err != nil {
        fmt.Printf("err: " + err.Error())
    }
    xferComments(sconn, mconn, posts)
    xferTags(sconn, mconn, posts)
}
<|endoftext|>


<commit_before>/*
This file represents the server for sensors
It opens communication links (for each sensor) through http protocol
Port # 8001 for Gyroscope Sensor,
Port # 8002 for Accelerator Sensor,
Port # 8003 for Temperature Sensor.
If the sensor client sends data through appropriate port,
each http handler is initiated, decoding data to appropriate JSON file.
Those datas are then logged to each sensor's log file
(log/Accel.log , log/Temp.log , log/Gyro.log)
*/
package main

/*
'models' package	: Stores basic sensor information in srtuct form
'net/http' package	: To serve http connection and handle requests
'os' package		: To open files for logging
'strings' package	: To join strings for file path
'sync' package		: To use 'WaitGroup' to hold main thread while goroutines are working
*/
import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"

	"github.com/mingrammer/go-codelab/models"
)

/*
String constants that are used in sensor_server.go
logDir	: Directory name to store logs
tempLog	: File name to store temperature sensor log
accelLog: File name to store accelerator sensor log
gyroLog	: File name to store gyroscope sensro log
*/
const (
	logDir   = "log"
	tempLog  = "Temp.log"
	accelLog = "Accel.log"
	gyroLog  = "Gyro.log"
)

/*
This logContent struct is to store data (string) would be written in log file
content 	: Actual string data, that would be logged in file
location	: Indicator that where the 'content' should be stored (or written)
*/
type logContent struct {
	content    string
	location   string
	sensorName string
}

/*
Three structs below are to implement ServeHTTP method
Each handler stores the pointer to data logging channel
Also, channel is bidirectional in these handlers, which only can store data in channel
*/

// GyroHandler : Gyroscopte sensor handler to implement ServeHTTP method
type GyroHandler struct {
	buf chan<- logContent
}

// AccelHandler : Accelerator sensro handler to implement ServeHTTP method
type AccelHandler struct {
	buf chan<- logContent
}

// TempHandler 	: Temperature sensor handler to implement ServeHTTP method
type TempHandler struct {
	buf chan<- logContent
}

/*
These methods are to handle request from each port.
Body of request (which is in JSON format) is decoded and allocated to TempSensor variable
and string, of which data would be saved in log fileis sent to logging channel
Note that BOdy of http request CAN NOT BE unmarshalled, because body of request is in array of bytes.
*/
func (m *TempHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.TempSensor

	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}
	defer req.Body.Close()

	fmt.Println(data.ReceivingOutputString())

	m.buf <- logContent{content: fmt.Sprintf("%s", data), location: tempLog, sensorName: data.Name}
}

func (m *GyroHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.GyroSensor

	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}
	defer req.Body.Close()

	fmt.Println(data.ReceivingOutputString())

	m.buf <- logContent{content: fmt.Sprintf("%s", data), location: gyroLog, sensorName: data.Name}
}

func (m *AccelHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.AccelSensor

	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}
	defer req.Body.Close()

	fmt.Println(data.ReceivingOutputString())

	m.buf <- logContent{content: fmt.Sprintf("%s", data), location: accelLog, sensorName: data.Name}
}

/*
This method loggs the content of sensor data
This method waits incoming data from 'logContent' channel at range block,
where ServeHTTP mehthod sends log data
When data is detected, for/range block immediately processes data
It checks the location, where the data should be stored,
and opens file of desired location (by joining string constants)
Note that channel used in this method is also BIDIRECTIONAL,
You only can pop the data from channel.
*/
func fileLogger(m <-chan logContent) {
	/*
		When program is initialized, fileLogger checks if 'log' dir exists.
		If it does not, it creates directory for the first time.
		If there is problem with creating it, it throws panic and aborts the application.
	*/
	dir, _ := os.Open("log")
	dirInfo, _ := dir.Stat()

	if dirInfo == nil {
		err := os.Mkdir("log", os.ModePerm)

		if err != nil {
			log.Fatal("Error creating directory 'log'\n", err)
		}
	}
	dir.Close()

	// This part continuously wait for incoming data through channel

	for i := range m {
		joinee := []string{logDir, i.location}
		filePath := strings.Join(joinee, "/")

		fileHandle, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)

		if err != nil {
			log.Fatal("Error Opening File\n", err)
		}

		logger := log.New(fileHandle, "", log.LstdFlags)

		logger.Printf("[%s Data Received]\n%s\n", i.sensorName, i.content)

		defer fileHandle.Close()
	}
}

/*
This method is for main thread. It starts go application.
It first creates handler to serve http serve for each sensors
sync.WaitGroup is to hold main thread while go routines are still working
Main thread indicates WaitGroup to wait 4 routines to stop.
After goroutines we need is created, main thread has WaitGroup to wait goroutines.
*/
func main() {
	var wg sync.WaitGroup

	wg.Add(4)

	logBuf := make(chan logContent)
	gyroHander := &GyroHandler{buf: logBuf}
	accelHandler := &AccelHandler{buf: logBuf}
	tempHandler := &TempHandler{buf: logBuf}

	go http.ListenAndServe(":8001", gyroHander)
	go http.ListenAndServe(":8002", accelHandler)
	go http.ListenAndServe(":8003", tempHandler)
	go fileLogger(logBuf)

	wg.Wait()
}
<commit_msg>Use better variable names<commit_after>/*
This file represents the server for sensors
It opens communication links (for each sensor) through http protocol
Port # 8001 for Gyroscope Sensor,
Port # 8002 for Accelerator Sensor,
Port # 8003 for Temperature Sensor.
If the sensor client sends data through appropriate port,
each http handler is initiated, decoding data to appropriate JSON file.
Those datas are then logged to each sensor's log file
(log/Accel.log , log/Temp.log , log/Gyro.log)
*/
package main

/*
'models' package	: Stores basic sensor information in srtuct form
'net/http' package	: To serve http connection and handle requests
'os' package		: To open files for logging
'strings' package	: To join strings for file path
'sync' package		: To use 'WaitGroup' to hold main thread while goroutines are working
*/
import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"

	"github.com/mingrammer/go-codelab/models"
)

/*
String constants that are used in sensor_server.go
logDir	: Directory name to store logs
tempLog	: File name to store temperature sensor log
accelLog: File name to store accelerator sensor log
gyroLog	: File name to store gyroscope sensro log
*/
const (
	logDir   = "log"
	tempLog  = "Temp.log"
	accelLog = "Accel.log"
	gyroLog  = "Gyro.log"
)

/*
This logContent struct is to store data (string) would be written in log file
content 	: Actual string data, that would be logged in file
location	: Indicator that where the 'content' should be stored (or written)
*/
type logContent struct {
	content    string
	location   string
	sensorName string
}

/*
Three structs below are to implement ServeHTTP method
Each handler stores the pointer to data logging channel
Also, channel is bidirectional in these handlers, which only can store data in channel
*/

// GyroHandler : Gyroscopte sensor handler to implement ServeHTTP method
type GyroHandler struct {
	buf chan<- logContent
}

// AccelHandler : Accelerator sensro handler to implement ServeHTTP method
type AccelHandler struct {
	buf chan<- logContent
}

// TempHandler 	: Temperature sensor handler to implement ServeHTTP method
type TempHandler struct {
	buf chan<- logContent
}

/*
These methods are to handle request from each port.
Body of request (which is in JSON format) is decoded and allocated to TempSensor variable
and string, of which data would be saved in log fileis sent to logging channel
Note that BOdy of http request CAN NOT BE unmarshalled, because body of request is in array of bytes.
*/
func (m *TempHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.TempSensor

	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}
	defer req.Body.Close()

	fmt.Println(data.ReceivingOutputString())

	m.buf <- logContent{content: fmt.Sprintf("%s", data), location: tempLog, sensorName: data.Name}
}

func (m *GyroHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.GyroSensor

	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}
	defer req.Body.Close()

	fmt.Println(data.ReceivingOutputString())

	m.buf <- logContent{content: fmt.Sprintf("%s", data), location: gyroLog, sensorName: data.Name}
}

func (m *AccelHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var data models.AccelSensor

	decoder := json.NewDecoder(req.Body)
	err := decoder.Decode(&data)
	if err != nil {
		fmt.Println("Something wrong")
	}
	defer req.Body.Close()

	fmt.Println(data.ReceivingOutputString())

	m.buf <- logContent{content: fmt.Sprintf("%s", data), location: accelLog, sensorName: data.Name}
}

/*
This method loggs the content of sensor data
This method waits incoming data from 'logContent' channel at range block,
where ServeHTTP mehthod sends log data
When data is detected, for/range block immediately processes data
It checks the location, where the data should be stored,
and opens file of desired location (by joining string constants)
Note that channel used in this method is also BIDIRECTIONAL,
You only can pop the data from channel.
*/
func fileLogger(logStream <-chan logContent) {
	/*
		When program is initialized, fileLogger checks if 'log' dir exists.
		If it does not, it creates directory for the first time.
		If there is problem with creating it, it throws panic and aborts the application.
	*/
	dir, _ := os.Open("log")
	dirInfo, _ := dir.Stat()

	if dirInfo == nil {
		err := os.Mkdir("log", os.ModePerm)

		if err != nil {
			log.Fatal("Error creating directory 'log'\n", err)
		}
	}
	dir.Close()

	// This part continuously wait for incoming data through channel
	for logData := range logStream {
		joinee := []string{logDir, logData.location}
		filePath := strings.Join(joinee, "/")

		fileHandle, err := os.OpenFile(filePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)

		if err != nil {
			log.Fatal("Error Opening File\n", err)
		}

		logger := log.New(fileHandle, "", log.LstdFlags)

		logger.Printf("[%s Data Received]\n%s\n", logData.sensorName, logData.content)

		defer fileHandle.Close()
	}
}

/*
This method is for main thread. It starts go application.
It first creates handler to serve http serve for each sensors
sync.WaitGroup is to hold main thread while go routines are still working
Main thread indicates WaitGroup to wait 4 routines to stop.
After goroutines we need is created, main thread has WaitGroup to wait goroutines.
*/
func main() {
	var wg sync.WaitGroup

	wg.Add(4)

	logBuf := make(chan logContent)
	gyroHander := &GyroHandler{buf: logBuf}
	accelHandler := &AccelHandler{buf: logBuf}
	tempHandler := &TempHandler{buf: logBuf}

	go http.ListenAndServe(":8001", gyroHander)
	go http.ListenAndServe(":8002", accelHandler)
	go http.ListenAndServe(":8003", tempHandler)
	go fileLogger(logBuf)

	wg.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"

	"github.com/codegangsta/cli"
)

func exitOnError(err error) {
	log.SetFlags(0)
	if err != nil {
		log.Fatalln(err)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "pgreport"
	app.Version = "0.1"
	app.Usage = "Generate reports from PostgreSQL"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "dbname, db",
			Value:  "postgres",
			Usage:  "database",
			EnvVar: "DB_NAME",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "host name",
			EnvVar: "DB_HOST",
		},
		cli.StringFlag{
			Name:   "port",
			Value:  "5432",
			Usage:  "port",
			EnvVar: "DB_PORT",
		},
		cli.StringFlag{
			Name:   "username, user",
			Value:  "postgres",
			Usage:  "username",
			EnvVar: "DB_USER",
		},
		cli.BoolFlag{
			Name:  "ssl",
			Usage: "require ssl mode",
		},
		cli.StringFlag{
			Name:   "pass, pw",
			Value:  "",
			Usage:  "password",
			EnvVar: "DB_PASS",
		},
	}

	app.Action = func(c *cli.Context) {
		query := c.Args().First()
		if query == "" {
			cli.ShowAppHelp(c)
			os.Exit(1)
		}

		connStr := parseConnStr(c)
		err := export(query, connStr, NewCsvEncoder())
		exitOnError(err)
	}

	app.Run(os.Args)
}
<commit_msg>Add local commands for different formats<commit_after>package main

import (
	"log"
	"os"
	"strings"

	"github.com/codegangsta/cli"
)

func changeHelpTemplateArgs(args string) {
	cli.CommandHelpTemplate = strings.Replace(cli.CommandHelpTemplate, "[arguments...]", args, -1)
}

func parseQuery(c *cli.Context, command string) string {
	query := c.Args().First()
	if query == "" {
		cli.ShowCommandHelp(c, command)
		os.Exit(1)
	}
	return query
}

func exitOnError(err error) {
	log.SetFlags(0)
	if err != nil {
		log.Fatalln(err)
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "pgreport"
	app.Version = "0.1"
	app.Usage = "Generate reports from PostgreSQL"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "dbname, db",
			Value:  "postgres",
			Usage:  "database",
			EnvVar: "DB_NAME",
		},
		cli.StringFlag{
			Name:   "host",
			Value:  "localhost",
			Usage:  "host name",
			EnvVar: "DB_HOST",
		},
		cli.StringFlag{
			Name:   "port",
			Value:  "5432",
			Usage:  "port",
			EnvVar: "DB_PORT",
		},
		cli.StringFlag{
			Name:   "username, user",
			Value:  "postgres",
			Usage:  "username",
			EnvVar: "DB_USER",
		},
		cli.BoolFlag{
			Name:  "ssl",
			Usage: "require ssl mode",
		},
		cli.StringFlag{
			Name:   "pass, pw",
			Value:  "",
			Usage:  "password",
			EnvVar: "DB_PASS",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:  "jsonlines",
			Usage: "Export newline-delimited JSON objects",
			Action: func(c *cli.Context) {
				changeHelpTemplateArgs("<query>")
				query := parseQuery(c, "jsonlines")
				connStr := parseConnStr(c)
				err := export(query, connStr, NewJsonEncoder())
				exitOnError(err)
			},
		},
		{
			Name:  "csv",
			Usage: "Export CSV",
			Action: func(c *cli.Context) {
				changeHelpTemplateArgs("<query>")
				query := parseQuery(c, "csv")
				connStr := parseConnStr(c)
				err := export(query, connStr, NewCsvEncoder())
				exitOnError(err)
			},
		},
		{
			Name:  "xml",
			Usage: "Export XML",
			Action: func(c *cli.Context) {
				changeHelpTemplateArgs("<query>")
				query := parseQuery(c, "xml")
				connStr := parseConnStr(c)
				err := export(query, connStr, NewXmlEncoder())
				exitOnError(err)
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package simplestore

import (
	"bufio"
	"encoding/gob"
	"os"
)

type entry struct {
	Key   string
	Value interface{}
}

// Store is a basic key value store which writes to a gob file on local disk on save
type Store struct {
	filePath string
	entries  []*entry
}

/*type storedData struct {
	registrations []string
	entries       []*entry
}*/

// New instantiates a new store with the given filepath either creating or updating the given file
func New(filePath string, types []interface{}) *Store {
	s := Store{entries: []*entry{}, filePath: filePath}
	if len(types) > 0 {
		s.registerTypes(types)
	}
	s.loadEntries()
	return &s
}

func (s *Store) registerTypes(types []interface{}) {
	for _, t := range types {
		gob.Register(t)
	}
}

// Get retrieves the value for a given key or returns nil if the key does not exist
func (s *Store) Get(key string) interface{} {
	for _, v := range s.entries {
		if key == v.Key {
			return v.Value
		}
	}

	return nil
}

// Set sets the value of the given key in the store - only accepts string values
func (s *Store) Set(key string, value interface{}) {
	for i, _ := range s.entries {
		v := s.entries[i]
		if key == v.Key {
			v.Value = value
			return
		}

	}

	s.entries = append(s.entries, &entry{Key: key, Value: value})
}

func (s *Store) Remove(key string) {
	for i, v := range s.entries {
		if key == v.Key {
			s.entries = append(s.entries[:i], s.entries[i+1:]...)
		}
	}
}

// Save writes the current state of the store to the file on disk
func (s *Store) Save() error {
	f := s.getFile()
	w := bufio.NewWriter(f)
	enc := gob.NewEncoder(w)

	err := enc.Encode(&s.entries)
	if err != nil {
		return err
	}
	w.Flush()
	f.Close()
	return nil
}

func (s *Store) loadEntries() error {
	r := bufio.NewReader(s.getFile())
	dec := gob.NewDecoder(r)
	err := dec.Decode(&s.entries)
	if err != nil {
		if err.Error() == "EOF" {
			s.entries = []*entry{}
			return nil
		}
		return err
	}

	return nil
}

func (s *Store) getFile() *os.File {
	f, err := os.OpenFile(s.filePath, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		panic(err.Error())
	}

	return f
}
<commit_msg>Adding exists method for key checking<commit_after>package simplestore

import (
	"bufio"
	"encoding/gob"
	"os"
)

type entry struct {
	Key   string
	Value interface{}
}

// Store is a basic key value store which writes to a gob file on local disk on save
type Store struct {
	filePath string
	entries  []*entry
}

/*type storedData struct {
	registrations []string
	entries       []*entry
}*/

// New instantiates a new store with the given filepath either creating or updating the given file
func New(filePath string, types []interface{}) *Store {
	s := Store{entries: []*entry{}, filePath: filePath}
	if len(types) > 0 {
		s.registerTypes(types)
	}
	s.loadEntries()
	return &s
}

func (s *Store) registerTypes(types []interface{}) {
	for _, t := range types {
		gob.Register(t)
	}
}

// Get retrieves the value for a given key or returns nil if the key does not exist
func (s *Store) Get(key string) interface{} {
	for _, v := range s.entries {
		if key == v.Key {
			return v.Value
		}
	}

	return nil
}

// Set sets the value of the given key in the store - only accepts string values
func (s *Store) Set(key string, value interface{}) {
	for i, _ := range s.entries {
		v := s.entries[i]
		if key == v.Key {
			v.Value = value
			return
		}

	}

	s.entries = append(s.entries, &entry{Key: key, Value: value})
}

func (s *Store) Exists(key string) bool {
	for i, _ := range s.entries {
		v := s.entries[i]
		if key == v.Key {
			return true
		}
	}
	return false
}

func (s *Store) Remove(key string) {
	for i, v := range s.entries {
		if key == v.Key {
			s.entries = append(s.entries[:i], s.entries[i+1:]...)
		}
	}
}

// Save writes the current state of the store to the file on disk
func (s *Store) Save() error {
	f := s.getFile()
	w := bufio.NewWriter(f)
	enc := gob.NewEncoder(w)

	err := enc.Encode(&s.entries)
	if err != nil {
		return err
	}
	w.Flush()
	f.Close()
	return nil
}

func (s *Store) loadEntries() error {
	r := bufio.NewReader(s.getFile())
	dec := gob.NewDecoder(r)
	err := dec.Decode(&s.entries)
	if err != nil {
		if err.Error() == "EOF" {
			s.entries = []*entry{}
			return nil
		}
		return err
	}

	return nil
}

func (s *Store) getFile() *os.File {
	f, err := os.OpenFile(s.filePath, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		panic(err.Error())
	}

	return f
}
<|endoftext|>


<commit_before>/*
  Copyright (c) 2012 José Carlos Nieto, http://xiam.menteslibres.org/

  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
  "Software"), to deal in the Software without restriction, including
  without limitation the rights to use, copy, modify, merge, publish,
  distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to
  the following conditions:

  The above copyright notice and this permission notice shall be
  included in all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"crypto"
	"flag"
	"fmt"
	"github.com/gosexy/canvas"
	"github.com/gosexy/checksum"
	"io"
	"log"
	"os"
	"path"
	"strings"
)

const PS = string(os.PathSeparator)

var flagFrom = flag.String("from", "", "Photos source directory.")
var flagDest = flag.String("to", "", "Photos destination directory.")
var flagMove = flag.Bool("move", false, "Delete original file after copying.")
var flagDryRun = flag.Bool("dry-run", false, "Just prints what would be done without actually doing it.")

func verifyDirectory(name string) error {
	stat, err := os.Stat(name)
	if err != nil {
		return err
	}
	if stat.IsDir() == false {
		return fmt.Errorf("%s: is not a directory.", *flagFrom)
	}
	return nil
}

func Copy(src string, dst string) error {
	var err error

	input, err := os.Open(src)
	if err != nil {
		return err
	}
	defer input.Close()

	output, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer output.Close()

	_, err = io.Copy(output, input)

	return err
}

func Move(src string, dst string) error {
	var err error

	err = os.Rename(src, dst)

	if err != nil {

		err = Copy(src, dst)

		if err != nil {
			return err
		}

		return os.Remove(src)
	}

	return nil
}

func Import(name string, dest string) {

	img := canvas.New()

	if img.Open(name) == true {

		defer img.Destroy()

		meta := img.Metadata()

		if meta["exif:DateTimeOriginal"] != "" {

			datetime := strings.Split(meta["exif:DateTimeOriginal"], " ")

			date := strings.Split(datetime[0], ":")
			time := strings.Split(datetime[1], ":")

			hash := checksum.File(name, crypto.SHA1)

			rename := dest + PS + strings.Join(date, PS) + PS + strings.Join(time, "") + strings.ToUpper(hash[0:5]) + strings.ToLower(path.Ext(name))

			_, err := os.Stat(rename)

			if err != nil {
				if *flagDryRun == false {
					err = os.MkdirAll(path.Dir(rename), os.ModeDir|0750)
					if err != nil {
						panic(err)
					}
				}
				err = nil
				if *flagMove == true {
					log.Printf("Moving file: %s -> %s\n", name, rename)
					if *flagDryRun == false {
						err = Move(name, rename)
					}
				} else {
					log.Printf("Copying file: %s -> %s\n", name, rename)
					if *flagDryRun == false {
						err = Copy(name, rename)
					}
				}
				if err != nil {
					panic(err)
				}
			} else {
				log.Printf("Skipping file: %s\n", rename)
			}

		}

	}

}

func Scandir(dirname string, dest string) error {

	var err error

	stat, err := os.Stat(dirname)

	if err != nil {
		return err
	}

	if stat.IsDir() == false {
		return fmt.Errorf("Not a directory.")
	}

	dh, err := os.Open(dirname)

	if err != nil {
		return err
	}

	defer dh.Close()

	files, err := dh.Readdir(-1)

	if err != nil {
		return err
	}

	for _, file := range files {

		name := dirname + PS + file.Name()

		if file.IsDir() == true {
			Scandir(name, dest)
		} else {
			Import(name, dest)
		}

	}

	return nil
}

func main() {

	flag.Parse()

	if *flagFrom == "" || *flagDest == "" {
		fmt.Printf("Sample usage: phosphor -from /run/media/me/usb/DCIM -to ~/Photos -dry-run")
		flag.PrintDefaults()
	} else {
		var err error

		err = verifyDirectory(*flagFrom)
		if err != nil {
			log.Println(err.Error())
			return
		}

		err = verifyDirectory(*flagDest)
		if err != nil {
			log.Println(err.Error())
			return
		}

		Scandir(*flagFrom, *flagDest)
	}
}
<commit_msg>Tiny typo.<commit_after>/*
  Copyright (c) 2012 José Carlos Nieto, http://xiam.menteslibres.org/

  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
  "Software"), to deal in the Software without restriction, including
  without limitation the rights to use, copy, modify, merge, publish,
  distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to
  the following conditions:

  The above copyright notice and this permission notice shall be
  included in all copies or substantial portions of the Software.

  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"crypto"
	"flag"
	"fmt"
	"github.com/gosexy/canvas"
	"github.com/gosexy/checksum"
	"io"
	"log"
	"os"
	"path"
	"strings"
)

const PS = string(os.PathSeparator)

var flagFrom = flag.String("from", "", "Photos source directory.")
var flagDest = flag.String("to", "", "Photos destination directory.")
var flagMove = flag.Bool("move", false, "Delete original file after copying.")
var flagDryRun = flag.Bool("dry-run", false, "Just prints what would be done without actually doing it.")

func verifyDirectory(name string) error {
	stat, err := os.Stat(name)
	if err != nil {
		return err
	}
	if stat.IsDir() == false {
		return fmt.Errorf("%s: is not a directory.", *flagFrom)
	}
	return nil
}

func Copy(src string, dst string) error {
	var err error

	input, err := os.Open(src)
	if err != nil {
		return err
	}
	defer input.Close()

	output, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer output.Close()

	_, err = io.Copy(output, input)

	return err
}

func Move(src string, dst string) error {
	var err error

	err = os.Rename(src, dst)

	if err != nil {

		err = Copy(src, dst)

		if err != nil {
			return err
		}

		return os.Remove(src)
	}

	return nil
}

func Import(name string, dest string) {

	img := canvas.New()

	if img.Open(name) == true {

		defer img.Destroy()

		meta := img.Metadata()

		if meta["exif:DateTimeOriginal"] != "" {

			datetime := strings.Split(meta["exif:DateTimeOriginal"], " ")

			date := strings.Split(datetime[0], ":")
			time := strings.Split(datetime[1], ":")

			hash := checksum.File(name, crypto.SHA1)

			rename := dest + PS + strings.Join(date, PS) + PS + strings.Join(time, "") + strings.ToUpper(hash[0:5]) + strings.ToLower(path.Ext(name))

			_, err := os.Stat(rename)

			if err != nil {
				if *flagDryRun == false {
					err = os.MkdirAll(path.Dir(rename), os.ModeDir|0750)
					if err != nil {
						panic(err)
					}
				}
				err = nil
				if *flagMove == true {
					log.Printf("Moving file: %s -> %s\n", name, rename)
					if *flagDryRun == false {
						err = Move(name, rename)
					}
				} else {
					log.Printf("Copying file: %s -> %s\n", name, rename)
					if *flagDryRun == false {
						err = Copy(name, rename)
					}
				}
				if err != nil {
					panic(err)
				}
			} else {
				log.Printf("Skipping file: %s\n", rename)
			}

		}

	}

}

func Scandir(dirname string, dest string) error {

	var err error

	stat, err := os.Stat(dirname)

	if err != nil {
		return err
	}

	if stat.IsDir() == false {
		return fmt.Errorf("Not a directory.")
	}

	dh, err := os.Open(dirname)

	if err != nil {
		return err
	}

	defer dh.Close()

	files, err := dh.Readdir(-1)

	if err != nil {
		return err
	}

	for _, file := range files {

		name := dirname + PS + file.Name()

		if file.IsDir() == true {
			Scandir(name, dest)
		} else {
			Import(name, dest)
		}

	}

	return nil
}

func main() {

	flag.Parse()

	if *flagFrom == "" || *flagDest == "" {
		fmt.Printf("Sample usage: phosphor -from /run/media/me/usb/DCIM -to ~/Photos -dry-run\n")
		flag.PrintDefaults()
	} else {
		var err error

		err = verifyDirectory(*flagFrom)
		if err != nil {
			log.Println(err.Error())
			return
		}

		err = verifyDirectory(*flagDest)
		if err != nil {
			log.Println(err.Error())
			return
		}

		Scandir(*flagFrom, *flagDest)
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"

type Element struct {
  next *Element
  list *List
  Value interface{}
}

type List struct {
  root Element
  len int
}

func (e *Element) Next() *Element{
  if n := e.next; e.list != nil && n != &e.list.root{
    return n
  }
  return nil
}

func (l *List) init() *List{
  l.root.next = &l.root
  l.len = 0
  return l
}

func (l *List) insert(e, at *Element) *List {
  n := at.Next()
  at.next = e
  e.next = n
  e.list = l
  l.len = l.len + 1
  return l
}

func main() {
  l := new(List).init()
  fmt.Println(l)
}<commit_msg>Proper working singly linked list<commit_after>package main

import "fmt"

type Element struct {
  next *Element
  list *List
  Value interface{}
}

type List struct {
  root Element
  len int
}

func (e *Element) Next() *Element{
  if n := e.next; e.list != nil && n != &e.list.root{
    return n
  }
  return nil
}

func (l *List) init() *List{
  l.root.next = &l.root
  l.len = 0
  return l
}

func (l *List) insert(e, at *Element) *Element {
  n := at.Next()
  at.next = e
  e.next = n
  e.list = l
  l.len = l.len + 1
  return e
}

func (l *List) insertValue(v interface{}, at *Element) *Element {
  return l.insert(&Element{Value: v}, at)
}

func New() *List { return new(List).init() }

func main() {
  l := New()
  e := l.insertValue(1, l.root.next)
  e1 := l.insertValue(3, e)
  e2 := l.insertValue(2, e1)
  fmt.Println(e2)
  fmt.Println(l.root.next.next.next.Value)
}<|endoftext|>


<commit_before>package errors

import (
	"encoding/json"
	"net/http"
)

type Error struct {
	Id     string `json:"id"`
	Code   int32  `json:"code"`
	Detail string `json:"detail"`
	Status string `json:"status"`
}

func (e *Error) Error() string {
	b, _ := json.Marshal(e)
	return string(b)
}

func New(id, detail string, code int32) error {
	return &Error{
		Id:     id,
		Code:   code,
		Detail: detail,
		Status: http.StatusText(int(code)),
	}
}

func Parse(err string) *Error {
	e := new(Error)
	errr := json.Unmarshal([]byte(err), e)
	if errr != nil {
		e.Detail = err
	}
	return e
}

func BadRequest(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   400,
		Detail: detail,
		Status: http.StatusText(400),
	}
}

func Unauthorized(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   401,
		Detail: detail,
		Status: http.StatusText(401),
	}
}

func Forbidden(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   403,
		Detail: detail,
		Status: http.StatusText(403),
	}
}

func NotFound(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   404,
		Detail: detail,
		Status: http.StatusText(404),
	}
}

func InternalServerError(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   500,
		Detail: detail,
		Status: http.StatusText(500),
	}
}
<commit_msg>Add error comment<commit_after>package errors

import (
	"encoding/json"
	"net/http"
)

// Errors provide a way to return detailed information
// for an RPC request error. The error is normally
// JSON encoded.
type Error struct {
	Id     string `json:"id"`
	Code   int32  `json:"code"`
	Detail string `json:"detail"`
	Status string `json:"status"`
}

func (e *Error) Error() string {
	b, _ := json.Marshal(e)
	return string(b)
}

func New(id, detail string, code int32) error {
	return &Error{
		Id:     id,
		Code:   code,
		Detail: detail,
		Status: http.StatusText(int(code)),
	}
}

func Parse(err string) *Error {
	e := new(Error)
	errr := json.Unmarshal([]byte(err), e)
	if errr != nil {
		e.Detail = err
	}
	return e
}

func BadRequest(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   400,
		Detail: detail,
		Status: http.StatusText(400),
	}
}

func Unauthorized(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   401,
		Detail: detail,
		Status: http.StatusText(401),
	}
}

func Forbidden(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   403,
		Detail: detail,
		Status: http.StatusText(403),
	}
}

func NotFound(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   404,
		Detail: detail,
		Status: http.StatusText(404),
	}
}

func InternalServerError(id, detail string) error {
	return &Error{
		Id:     id,
		Code:   500,
		Detail: detail,
		Status: http.StatusText(500),
	}
}
<|endoftext|>


<commit_before>package decoders

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/asaskevich/govalidator"
)

const minNameLength = 3
const minPasswordLength = 6

// store the errors as a map
type Errors map[string]string

func (e Errors) Error() string {
	return fmt.Sprintf("%v", e)
}

type Decoder interface {
	Decode() error
}

// Decode decodes JSON body of request and runs through validator
func Decode(r *http.Request, decoder Decoder) error {
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(decoder); err != nil {
		return err
	}
	return decoder.Decode()
}

type RecoverPassword struct {
	Identifier string `json:"identifier"`
}

func (r *RecoverPassword) Decode() error {

	if r.Identifier == "" {
		return Errors{
			"identifier": "Email or user name required",
		}
	}
	return nil
}

type Signup struct {
	Name     string `json:"name"`
	Email    string `json:"email"`
	Password string `json:"password"`
}

func (s *Signup) Decode() error {

	s.Name = strings.Trim(s.Name, " ")
	s.Email = strings.ToLower(strings.Trim(s.Email, " "))

	errors := make(Errors)

	if len(s.Name) < minNameLength {
		errors["name"] = fmt.Sprintf("Your name must be at least %d characters long", minNameLength)
	}
	if !govalidator.IsEmail(s.Email) {
		errors["email"] = "Email address is required"
	}
	if len(s.Password) < 6 {
		errors["password"] = fmt.Sprintf("Password must be at least %d characters long", minPasswordLength)
	}
	if len(errors) > 0 {
		return errors
	}

	return nil
}

type Login struct {
	Identifier string `json:"identifier"`
	Password   string `json:"password"`
}

func (l *Login) Decode() error {
	errors := make(Errors)
	if l.Identifier == "" {
		errors["identifier"] = "Email or user name missing"
	}
	if l.Password == "" {
		errors["password"] = "Password missing"
	}
	if len(errors) > 0 {
		return errors
	}
	return nil
}

type NewChannel struct {
	URL string `json:"url"`
}

func (n *NewChannel) Decode() error {
	n.URL = strings.Trim(n.URL, " ")
	if n.URL == "" || !govalidator.IsURL(n.URL) {
		return Errors{
			"url": "Valid URL is required",
		}
	}
	return nil
}

type NewEmail struct {
	Email string `json:"email"`
}

func (n *NewEmail) Decode() error {
	n.Email = strings.Trim(strings.ToLower(n.Email), " ")
	if n.Email == "" || !govalidator.IsEmail(n.Email) {
		return Errors{
			"email": "Valid email is required",
		}
	}
	return nil
}

type NewPassword struct {
	OldPassword string `json:"oldPassword"`
	NewPassword string `json:"newPassword"`
}

func (n *NewPassword) Decode() error {
	errors := make(Errors)
	if n.OldPassword == "" {
		errors["oldPassword"] = "Old password is required"
	}
	if len(n.NewPassword) < minPasswordLength {
		errors["newPassword"] = fmt.Sprintf("New password must be at least %d characters long", minPasswordLength)
	}
	if len(errors) > 0 {
		return errors
	}
	return nil
}
<commit_msg>Decoder fix<commit_after>package decoders

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/asaskevich/govalidator"
)

const minNameLength = 3
const minPasswordLength = 6

// store the errors as a map
type Errors map[string]string

func (e Errors) Error() string {
	return fmt.Sprintf("%v", e)
}

type Decoder interface {
	Decode() error
}

// Decode decodes JSON body of request and runs through validator
func Decode(r *http.Request, decoder Decoder) error {
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(decoder); err != nil {
		return err
	}
	return decoder.Decode()
}

type RecoverPassword struct {
	Identifier string `json:"identifier"`
}

func (r *RecoverPassword) Decode() error {

	if r.Identifier == "" {
		return Errors{
			"identifier": "Email or user name required",
		}
	}
	return nil
}

type Signup struct {
	Name     string `json:"name"`
	Email    string `json:"email"`
	Password string `json:"password"`
}

func (s *Signup) Decode() error {

	s.Name = strings.Trim(s.Name, " ")
	s.Email = strings.ToLower(strings.Trim(s.Email, " "))

	errors := make(Errors)

	if len(s.Name) < minNameLength {
		errors["name"] = fmt.Sprintf("Your name must be at least %d characters long", minNameLength)
	}
	if !govalidator.IsEmail(s.Email) {
		errors["email"] = "Email address is required"
	}
	if len(s.Password) < minPasswordLength {
		errors["password"] = fmt.Sprintf("Password must be at least %d characters long", minPasswordLength)
	}
	if len(errors) > 0 {
		return errors
	}

	return nil
}

type Login struct {
	Identifier string `json:"identifier"`
	Password   string `json:"password"`
}

func (l *Login) Decode() error {
	errors := make(Errors)
	if l.Identifier == "" {
		errors["identifier"] = "Email or user name missing"
	}
	if l.Password == "" {
		errors["password"] = "Password missing"
	}
	if len(errors) > 0 {
		return errors
	}
	return nil
}

type NewChannel struct {
	URL string `json:"url"`
}

func (n *NewChannel) Decode() error {
	n.URL = strings.Trim(n.URL, " ")
	if n.URL == "" || !govalidator.IsURL(n.URL) {
		return Errors{
			"url": "Valid URL is required",
		}
	}
	return nil
}

type NewEmail struct {
	Email string `json:"email"`
}

func (n *NewEmail) Decode() error {
	n.Email = strings.Trim(strings.ToLower(n.Email), " ")
	if n.Email == "" || !govalidator.IsEmail(n.Email) {
		return Errors{
			"email": "Valid email is required",
		}
	}
	return nil
}

type NewPassword struct {
	OldPassword string `json:"oldPassword"`
	NewPassword string `json:"newPassword"`
}

func (n *NewPassword) Decode() error {
	errors := make(Errors)
	if n.OldPassword == "" {
		errors["oldPassword"] = "Old password is required"
	}
	if len(n.NewPassword) < minPasswordLength {
		errors["newPassword"] = fmt.Sprintf("New password must be at least %d characters long", minPasswordLength)
	}
	if len(errors) > 0 {
		return errors
	}
	return nil
}
<|endoftext|>


<commit_before>package main

//
// Prepare PG Database:
//
// $ createdb testdb
// $ psql testdb
// testdb=#
// CREATE TABLE test
//     (id int, call_uuid text, dst text, callerid_name text, callerid_num text, duration int,
//      data jsonb, created timestamp );
//
// INSERT INTO cdr VALUES ("Outbound Call","123555555","123555555","default","2015-01-14 17:58:01","2015-01-14 17:58:01","2015-01-14 17:58:06",5,5,"NORMAL_CLEARING","2bbe83f7-5111-4b5b-9626-c5154608d4ee","","")
//

import (
	"fmt"
	"github.com/kr/pretty"
	"log"
	"time"
)

func main() {

	fmt.Printf("StartTime: %v\n", time.Now())

	// LoadConfig
	LoadConfig(Default_conf)
	log.Printf("Loaded Config:\n%# v\n\n", pretty.Formatter(config))

	f := new(SQLFetcher)

	if config.Storage_source == "sqlite" {
		f.Init(config.Db_file, config.Db_table, config.Max_push_batch, config.Cdr_fields)
		// Fetch CDRs from SQLite
		err := f.Fetch()
		if err != nil {
			log.Fatal(err)
		}
	} else {
		fmt.Println("Define a fetching method using the conf setting 'storage_source'!\n")
	}

	if config.Storage_destination == "postgres" {
		// Push CDRs to PostgreSQL
		p := new(PGPusher)
		p.Init(config.Pg_datasourcename, config.Cdr_fields, config.Switch_ip, config.Table_destination)
		err := p.Push(f.results)
		if err != nil {
			log.Fatal(err)
		}
	} else {
		fmt.Println("Define a pushing method using the conf setting 'storage_destination'!\n")
	}

	// 1. Create Go routine / Tick every x second: heartbeat
	// 2. Send Results through channels

	fmt.Printf("StopTime: %v\n", time.Now())
}
<commit_msg>add function to validate the conf<commit_after>package main

//
// Prepare PG Database:
//
// $ createdb testdb
// $ psql testdb
// testdb=#
// CREATE TABLE test
//     (id int, call_uuid text, dst text, callerid_name text, callerid_num text, duration int,
//      data jsonb, created timestamp );
//
// INSERT INTO cdr VALUES ("Outbound Call","123555555","123555555","default","2015-01-14 17:58:01","2015-01-14 17:58:01","2015-01-14 17:58:06",5,5,"NORMAL_CLEARING","2bbe83f7-5111-4b5b-9626-c5154608d4ee","","")
//

import (
	"errors"
	"fmt"
	"github.com/kr/pretty"
	"log"
	"time"
)

func go_fetcher(config Config) {

}

func validate_config(config Config) error {
	switch config.Storage_source {
	case "sqlite":
		// could check more settings
	default:
		return errors.New("Not a valid conf setting 'storage_source'")
	}
	switch config.Storage_destination {
	case "postgres":
		// could check more settings
	default:
		return errors.New("Not a valid conf setting 'storage_destination'")
	}
	return nil
}

func main() {

	fmt.Printf("StartTime: %v\n", time.Now())

	// LoadConfig
	LoadConfig(Default_conf)
	log.Printf("Loaded Config:\n%# v\n\n", pretty.Formatter(config))

	if err := validate_config(config); err != nil {
		panic(err)
	}

	f := new(SQLFetcher)

	if config.Storage_destination == "sqlite" {
		f.Init(config.Db_file, config.Db_table, config.Max_push_batch, config.Cdr_fields)
		// Fetch CDRs from SQLite
		err := f.Fetch()
		if err != nil {
			log.Fatal(err)
		}
	}

	if config.Storage_destination == "postgres" {
		// Push CDRs to PostgreSQL
		p := new(PGPusher)
		p.Init(config.Pg_datasourcename, config.Cdr_fields, config.Switch_ip, config.Table_destination)
		err := p.Push(f.results)
		if err != nil {
			log.Fatal(err)
		}
	}

	// 1. Create Go routine / Tick every x second: heartbeat
	// 2. Send Results through channels

	fmt.Printf("StopTime: %v\n", time.Now())
}
<|endoftext|>