<commit_before>/*
 * Copyright (c) 2013 Santiago Arias | Remy Jourde | Carlos Bernal
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package tournament

import (
	"errors"
	"fmt"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"

	"github.com/santiaago/purple-wing/helpers"
	"github.com/santiaago/purple-wing/helpers/log"
	tournamentinvidmdl "github.com/santiaago/purple-wing/models/tournamentInvertedIndex"
	tournamentrelmdl "github.com/santiaago/purple-wing/models/tournamentrel"
	tournamentteamrelmdl "github.com/santiaago/purple-wing/models/tournamentteamrel"
)

type Tournament struct {
	Id              int64
	KeyName         string
	Name            string
	Description     string
	Start           time.Time
	End             time.Time
	AdminId         int64
	Created         time.Time
	GroupIds        []int64
	Matches1stStage []int64
	Matches2ndStage []int64
}

type TournamentJson struct {
	Id              *int64     `json:",omitempty"`
	KeyName         *string    `json:",omitempty"`
	Name            *string    `json:",omitempty"`
	Description     *string    `json:",omitempty"`
	Start           *time.Time `json:",omitempty"`
	End             *time.Time `json:",omitempty"`
	AdminId         *int64     `json:",omitempty"`
	Created         *time.Time `json:",omitempty"`
	GroupIds        *[]int64   `json:",omitempty"`
	Matches1stStage *[]int64   `json:",omitempty"`
	Matches2ndStage *[]int64   `json:",omitempty"`
}

// Create tournament entity given a name and description.
func Create(c appengine.Context, name string, description string, start time.Time, end time.Time, adminId int64) (*Tournament, error) {

	tournamentID, _, err := datastore.AllocateIDs(c, "Tournament", nil, 1)
	if err != nil {
		return nil, err
	}

	key := datastore.NewKey(c, "Tournament", "", tournamentID, nil)

	// empty groups and tournaments for now
	groupIds := make([]int64, 0)
	matches1stStageIds := make([]int64, 0)
	matches2ndStageIds := make([]int64, 0)

	tournament := &Tournament{tournamentID, helpers.TrimLower(name), name, description, start, end, adminId, time.Now(), groupIds, matches1stStageIds, matches2ndStageIds}

	_, err = datastore.Put(c, key, tournament)
	if err != nil {
		return nil, err
	}

	tournamentinvidmdl.Add(c, helpers.TrimLower(name), tournamentID)
	return tournament, nil
}

// Destroy a tournament entity given a tournament id.
func Destroy(c appengine.Context, tournamentId int64) error {

	if tournament, err := ById(c, tournamentId); err != nil {
		return errors.New(fmt.Sprintf("Cannot find tournament with tournamentId=%d", tournamentId))
	} else {
		key := datastore.NewKey(c, "Tournament", "", tournament.Id, nil)
		return datastore.Delete(c, key)
	}
}

// Find all tournaments in datastore given a filter and value.
func Find(c appengine.Context, filter string, value interface{}) []*Tournament {

	q := datastore.NewQuery("Tournament").Filter(filter+" =", value)
	var tournaments []*Tournament
	if _, err := q.GetAll(c, &tournaments); err == nil {
		return tournaments
	} else {
		log.Errorf(c, " Tournament.Find, error occurred during GetAll: %v", err)
		return nil
	}
}

// Get a pointer to a tournament given a tournament id.
func ById(c appengine.Context, id int64) (*Tournament, error) {

	var t Tournament
	key := datastore.NewKey(c, "Tournament", "", id, nil)
	if err := datastore.Get(c, key, &t); err != nil {
		log.Errorf(c, " tournament not found : %v", err)
		return &t, err
	}
	return &t, nil
}

// Get a pointer to a tournament key given a tournament id.
func KeyById(c appengine.Context, id int64) *datastore.Key {

	key := datastore.NewKey(c, "Tournament", "", id, nil)
	return key
}

// Update a tournament given a tournament id and a tournament pointer.
func Update(c appengine.Context, id int64, t *Tournament) error {

	// update key name
	t.KeyName = helpers.TrimLower(t.Name)
	k := KeyById(c, id)
	oldTournament := new(Tournament)
	if err := datastore.Get(c, k, oldTournament); err == nil {
		if _, err = datastore.Put(c, k, t); err != nil {
			return err
		}
		// use name with trim lower as tournament inverted index stores lower key names.
		tournamentinvidmdl.Update(c, oldTournament.KeyName, t.KeyName, id)
	}
	return nil
}

// Find all tournaments in the datastore.
func FindAll(c appengine.Context) []*Tournament {

	q := datastore.NewQuery("Tournament")
	var tournaments []*Tournament
	if _, err := q.GetAll(c, &tournaments); err != nil {
		log.Errorf(c, " Tournament.FindAll, error occurred during GetAll call: %v", err)
	}
	return tournaments
}

// Find all tournaments with respect to array of ids.
func ByIds(c appengine.Context, ids []int64) []*Tournament {

	var tournaments []*Tournament
	for _, id := range ids {
		if tournament, err := ById(c, id); err == nil {
			tournaments = append(tournaments, tournament)
		} else {
			log.Errorf(c, " Tournament.ByIds, error occurred during ByIds call: %v", err)
		}
	}
	return tournaments
}

// Checks if a user has joined a tournament.
func Joined(c appengine.Context, tournamentId int64, userId int64) bool {
	tournamentRel := tournamentrelmdl.FindByTournamentIdAndUserId(c, tournamentId, userId)
	return tournamentRel != nil
}

// Makes a user join a tournament.
func Join(c appengine.Context, tournamentId int64, userId int64) error {
	if tournamentRel, err := tournamentrelmdl.Create(c, tournamentId, userId); tournamentRel == nil {
		return errors.New(fmt.Sprintf(" Tournament.Join, error during tournament relationship creation: %v", err))
	}

	return nil
}

// Makes a user leave a tournament.
// Todo: should we check that user is indeed a member of the tournament?
func Leave(c appengine.Context, tournamentId int64, userId int64) error {
	return tournamentrelmdl.Destroy(c, tournamentId, userId)
}

// Checks if user is admin of given tournament.
func IsTournamentAdmin(c appengine.Context, tournamentId int64, userId int64) bool {
	if tournament, err := ById(c, tournamentId); err == nil {
		return tournament.AdminId == userId
	}

	return false
}

// Check if a Team has joined the tournament.
func TeamJoined(c appengine.Context, tournamentId int64, teamId int64) bool {
	tournamentteamRel := tournamentteamrelmdl.FindByTournamentIdAndTeamId(c, tournamentId, teamId)
	return tournamentteamRel != nil
}

// Team joins the Tournament.
func TeamJoin(c appengine.Context, tournamentId int64, teamId int64) error {
	if tournamentteamRel, err := tournamentteamrelmdl.Create(c, tournamentId, teamId); tournamentteamRel == nil {
		return errors.New(fmt.Sprintf(" Tournament.TeamJoin, error during tournament team relationship creation: %v", err))
	}

	return nil
}

// Team leaves the Tournament.
func TeamLeave(c appengine.Context, tournamentId int64, teamId int64) error {
	return tournamentteamrelmdl.Destroy(c, tournamentId, teamId)
}

// Get the frequency of given word with respect to tournament id.
func GetWordFrequencyForTournament(c appengine.Context, id int64, word string) int64 {

	if tournaments := Find(c, "Id", id); tournaments != nil {
		return helpers.CountTerm(strings.Split(tournaments[0].KeyName, " "), word)
	}
	return 0
}

// Reset tournament values: Points, GoalsF, GoalsA to zero.
func Reset(c appengine.Context, t *Tournament) error {
	groups := Groups(c, t.GroupIds)
	for _, g := range groups {
		g.Points = make([]int64, len(g.Teams))
		g.GoalsF = make([]int64, len(g.Teams))
		g.GoalsA = make([]int64, len(g.Teams))
		for _, m := range g.Matches {
			m.Result1 = 0
			m.Result2 = 0
			if err := UpdateMatch(c, &m); err != nil {
				return err
			}
		}
		if err := UpdateGroup(c, g); err != nil {
			return err
		}
	}
	// reset all matches rules
	return nil
}
<commit_msg>Reset action now reset rules and results of second phase matches Fixes #293<commit_after>/*
 * Copyright (c) 2013 Santiago Arias | Remy Jourde | Carlos Bernal
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package tournament

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"time"

	"appengine"
	"appengine/datastore"

	"github.com/santiaago/purple-wing/helpers"
	"github.com/santiaago/purple-wing/helpers/log"
	tournamentinvidmdl "github.com/santiaago/purple-wing/models/tournamentInvertedIndex"
	tournamentrelmdl "github.com/santiaago/purple-wing/models/tournamentrel"
	tournamentteamrelmdl "github.com/santiaago/purple-wing/models/tournamentteamrel"
)

type Tournament struct {
	Id              int64
	KeyName         string
	Name            string
	Description     string
	Start           time.Time
	End             time.Time
	AdminId         int64
	Created         time.Time
	GroupIds        []int64
	Matches1stStage []int64
	Matches2ndStage []int64
}

type TournamentJson struct {
	Id              *int64     `json:",omitempty"`
	KeyName         *string    `json:",omitempty"`
	Name            *string    `json:",omitempty"`
	Description     *string    `json:",omitempty"`
	Start           *time.Time `json:",omitempty"`
	End             *time.Time `json:",omitempty"`
	AdminId         *int64     `json:",omitempty"`
	Created         *time.Time `json:",omitempty"`
	GroupIds        *[]int64   `json:",omitempty"`
	Matches1stStage *[]int64   `json:",omitempty"`
	Matches2ndStage *[]int64   `json:",omitempty"`
}

// Create tournament entity given a name and description.
func Create(c appengine.Context, name string, description string, start time.Time, end time.Time, adminId int64) (*Tournament, error) {

	tournamentID, _, err := datastore.AllocateIDs(c, "Tournament", nil, 1)
	if err != nil {
		return nil, err
	}

	key := datastore.NewKey(c, "Tournament", "", tournamentID, nil)

	// empty groups and tournaments for now
	groupIds := make([]int64, 0)
	matches1stStageIds := make([]int64, 0)
	matches2ndStageIds := make([]int64, 0)

	tournament := &Tournament{tournamentID, helpers.TrimLower(name), name, description, start, end, adminId, time.Now(), groupIds, matches1stStageIds, matches2ndStageIds}

	_, err = datastore.Put(c, key, tournament)
	if err != nil {
		return nil, err
	}

	tournamentinvidmdl.Add(c, helpers.TrimLower(name), tournamentID)
	return tournament, nil
}

// Destroy a tournament entity given a tournament id.
func Destroy(c appengine.Context, tournamentId int64) error {

	if tournament, err := ById(c, tournamentId); err != nil {
		return errors.New(fmt.Sprintf("Cannot find tournament with tournamentId=%d", tournamentId))
	} else {
		key := datastore.NewKey(c, "Tournament", "", tournament.Id, nil)
		return datastore.Delete(c, key)
	}
}

// Find all tournaments in datastore given a filter and value.
func Find(c appengine.Context, filter string, value interface{}) []*Tournament {

	q := datastore.NewQuery("Tournament").Filter(filter+" =", value)
	var tournaments []*Tournament
	if _, err := q.GetAll(c, &tournaments); err == nil {
		return tournaments
	} else {
		log.Errorf(c, " Tournament.Find, error occurred during GetAll: %v", err)
		return nil
	}
}

// Get a pointer to a tournament given a tournament id.
func ById(c appengine.Context, id int64) (*Tournament, error) {

	var t Tournament
	key := datastore.NewKey(c, "Tournament", "", id, nil)
	if err := datastore.Get(c, key, &t); err != nil {
		log.Errorf(c, " tournament not found : %v", err)
		return &t, err
	}
	return &t, nil
}

// Get a pointer to a tournament key given a tournament id.
func KeyById(c appengine.Context, id int64) *datastore.Key {

	key := datastore.NewKey(c, "Tournament", "", id, nil)
	return key
}

// Update a tournament given a tournament id and a tournament pointer.
func Update(c appengine.Context, id int64, t *Tournament) error {

	// update key name
	t.KeyName = helpers.TrimLower(t.Name)
	k := KeyById(c, id)
	oldTournament := new(Tournament)
	if err := datastore.Get(c, k, oldTournament); err == nil {
		if _, err = datastore.Put(c, k, t); err != nil {
			return err
		}
		// use name with trim lower as tournament inverted index stores lower key names.
		tournamentinvidmdl.Update(c, oldTournament.KeyName, t.KeyName, id)
	}
	return nil
}

// Find all tournaments in the datastore.
func FindAll(c appengine.Context) []*Tournament {

	q := datastore.NewQuery("Tournament")
	var tournaments []*Tournament
	if _, err := q.GetAll(c, &tournaments); err != nil {
		log.Errorf(c, " Tournament.FindAll, error occurred during GetAll call: %v", err)
	}
	return tournaments
}

// Find all tournaments with respect to array of ids.
func ByIds(c appengine.Context, ids []int64) []*Tournament {

	var tournaments []*Tournament
	for _, id := range ids {
		if tournament, err := ById(c, id); err == nil {
			tournaments = append(tournaments, tournament)
		} else {
			log.Errorf(c, " Tournament.ByIds, error occurred during ByIds call: %v", err)
		}
	}
	return tournaments
}

// Checks if a user has joined a tournament.
func Joined(c appengine.Context, tournamentId int64, userId int64) bool {
	tournamentRel := tournamentrelmdl.FindByTournamentIdAndUserId(c, tournamentId, userId)
	return tournamentRel != nil
}

// Makes a user join a tournament.
func Join(c appengine.Context, tournamentId int64, userId int64) error {
	if tournamentRel, err := tournamentrelmdl.Create(c, tournamentId, userId); tournamentRel == nil {
		return errors.New(fmt.Sprintf(" Tournament.Join, error during tournament relationship creation: %v", err))
	}

	return nil
}

// Makes a user leave a tournament.
// Todo: should we check that user is indeed a member of the tournament?
func Leave(c appengine.Context, tournamentId int64, userId int64) error {
	return tournamentrelmdl.Destroy(c, tournamentId, userId)
}

// Checks if user is admin of given tournament.
func IsTournamentAdmin(c appengine.Context, tournamentId int64, userId int64) bool {
	if tournament, err := ById(c, tournamentId); err == nil {
		return tournament.AdminId == userId
	}

	return false
}

// Check if a Team has joined the tournament.
func TeamJoined(c appengine.Context, tournamentId int64, teamId int64) bool {
	tournamentteamRel := tournamentteamrelmdl.FindByTournamentIdAndTeamId(c, tournamentId, teamId)
	return tournamentteamRel != nil
}

// Team joins the Tournament.
func TeamJoin(c appengine.Context, tournamentId int64, teamId int64) error {
	if tournamentteamRel, err := tournamentteamrelmdl.Create(c, tournamentId, teamId); tournamentteamRel == nil {
		return errors.New(fmt.Sprintf(" Tournament.TeamJoin, error during tournament team relationship creation: %v", err))
	}

	return nil
}

// Team leaves the Tournament.
func TeamLeave(c appengine.Context, tournamentId int64, teamId int64) error {
	return tournamentteamrelmdl.Destroy(c, tournamentId, teamId)
}

// Get the frequency of given word with respect to tournament id.
func GetWordFrequencyForTournament(c appengine.Context, id int64, word string) int64 {

	if tournaments := Find(c, "Id", id); tournaments != nil {
		return helpers.CountTerm(strings.Split(tournaments[0].KeyName, " "), word)
	}
	return 0
}

// Reset tournament values: Points, GoalsF, GoalsA to zero.
func Reset(c appengine.Context, t *Tournament) error {
	groups := Groups(c, t.GroupIds)
	for _, g := range groups {
		g.Points = make([]int64, len(g.Teams))
		g.GoalsF = make([]int64, len(g.Teams))
		g.GoalsA = make([]int64, len(g.Teams))
		for _, m := range g.Matches {
			m.Result1 = 0
			m.Result2 = 0
			if err := UpdateMatch(c, &m); err != nil {
				return err
			}
		}
		if err := UpdateGroup(c, g); err != nil {
			return err
		}
	}
	// reset all match rules
	mapMatches2ndRound := MapOf2ndRoundMatches()

	const (
		cMatchId       = 0
		cMatchDate     = 1
		cMatchTeam1    = 2
		cMatchTeam2    = 3
		cMatchLocation = 4
	)

	// build matches 2nd phase
	const shortForm = "Jan/02/2006"
	for _, roundMatches := range mapMatches2ndRound {
		for _, matchData := range roundMatches {
			matchInternalId, _ := strconv.Atoi(matchData[cMatchId])
			m := GetMatchByIdNumber(c, *t, int64(matchInternalId))
			rule := fmt.Sprintf("%s %s", matchData[cMatchTeam1], matchData[cMatchTeam2])
			m.Rule = rule
			m.Result1 = 0
			m.Result2 = 0
			if err := UpdateMatch(c, m); err != nil {
				log.Errorf(c, "Reset: unable to reset rule on match: %v", err)
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Cutehacks AS. All rights reserved.
// License can be found in the LICENSE file.

package commands

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"golang.org/x/net/context"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"qpm.io/common"
	msg "qpm.io/common/messages"
	"qpm.io/qpm/core"
	"strings"
	"text/template"
)

var packageFuncs = template.FuncMap{
	"relPriFile": func(vendorDir string, dep *common.PackageWrapper) string {
		abs := filepath.Join(dep.RootDir(), dep.PriFile())
		rel, err := filepath.Rel(vendorDir, abs)
		if err == nil {
			return rel
		} else {
			return abs
		}
	},
}

var (
	vendorPri = template.Must(template.New("vendorPri").Funcs(packageFuncs).Parse(`
DEFINES += QPM_INIT\\(E\\)=\"E.addImportPath(QStringLiteral(\\\"qrc:/\\\"));\"

{{$vendirDir := .VendorDir}}
{{range $dep := .Dependencies}}
include($$PWD/{{relPriFile $vendirDir $dep}})
{{end}}
`))
)

type ProgressProxyReader struct {
	io.Reader
	total    int64
	length   int64
	progress float64
}

func (r *ProgressProxyReader) Read(p []byte) (int, error) {
	n, err := r.Reader.Read(p)
	if n > 0 {
		r.total += int64(n)
		percentage := float64(r.total) / float64(r.length) * float64(100)
		i := int(percentage / float64(10))
		is := fmt.Sprintf("%v", i)
		if percentage-r.progress > 2 {
			fmt.Fprintf(os.Stderr, is)
			r.progress = percentage
		}
	}
	return n, err
}

type InstallCommand struct {
	BaseCommand
	pkg       *common.PackageWrapper
	fs        *flag.FlagSet
	vendorDir string
}

func NewInstallCommand(ctx core.Context) *InstallCommand {
	return &InstallCommand{
		BaseCommand: BaseCommand{
			Ctx: ctx,
		},
	}
}

func (i InstallCommand) Description() string {
	return "Installs a new package"
}

func (i *InstallCommand) RegisterFlags(flags *flag.FlagSet) {
	i.fs = flags

	// TODO: Support other directory names on the command line?
	var err error
	i.vendorDir, err = filepath.Abs(core.Vendor)
	if err != nil {
		i.vendorDir = core.Vendor
	}
}

func (i *InstallCommand) Run() error {

	packageName := i.fs.Arg(0)

	var err error
	i.pkg, err = common.LoadPackage("")
	if err != nil {
		// A missing package file is only an error if packageName is empty
		if os.IsNotExist(err) {
			if packageName == "" {
				err = fmt.Errorf("No %s file found", core.PackageFile)
				i.Error(err)
				return err
			} else {
				// Create a new package
				file, err := filepath.Abs(core.PackageFile)
				if err != nil {
					i.Error(err)
					return err
				}
				i.pkg = common.NewPackageWrapper(file)
			}
		} else {
			i.Error(err)
			return err
		}
	}

	var packageNames []string
	if packageName == "" {
		packageNames = i.pkg.Dependencies
	} else {
		packageNames = []string{packageName}
	}

	// Get list of dependencies from the server
	response, err := i.Ctx.Client.GetDependencies(context.Background(), &msg.DependencyRequest{packageNames})
	if err != nil {
		i.Error(err)
		return err
	}

	if len(response.Dependencies) == 0 {
		i.Info("No package(s) found")
		return nil
	}

	// create the vendor directory if needed
	if _, err = os.Stat(i.vendorDir); err != nil {
		err = os.Mkdir(i.vendorDir, 0755)
	}

	// Download and extract the packages
	packages := []*common.PackageWrapper{}
	for _, d := range response.Dependencies {
		p, err := i.install(d)
		if err != nil {
			return err
		}
		packages = append(packages, p)
	}

	// Save the dependencies in the package file
	err = i.save(packages)
	// FIXME: should we continue installing ?
	if err != nil {
		return err
	}

	err = i.postInstall(packages)
	// FIXME: should we continue installing ?
	if err != nil {
		return err
	}

	return nil
}

func (i *InstallCommand) install(d *msg.Dependency) (*common.PackageWrapper, error) {

	url := core.GitHub + "/" + d.Repository.Url + "/" + core.Tarball

	signature := strings.Join([]string{d.Name, d.Version.Label}, "@")
	fmt.Println("Installing", signature)

	fileName, err := i.download(url, i.vendorDir)
	if err != nil {
		return nil, err
	}

	pkg, err := i.extract(fileName, i.vendorDir)
	if err != nil {
		return nil, err
	}

	return pkg, os.Remove(fileName)
}

func (i *InstallCommand) save(newDeps []*common.PackageWrapper) error {

	existingDeps := i.pkg.ParseDependencies()

	for _, d := range newDeps {
		existingVersion, exists := existingDeps[d.Name]
		if exists {
			if d.Version.Label == existingVersion {
				i.Info("The package is already a dependency : " + d.GetDependencySignature())
			} else {
				// TODO: Handle conflicts
				err := fmt.Errorf("Conflict for package %s. Version %s != %s", d.Name, existingVersion, d.Version.Label)
				i.Error(err)
				return err
			}
		} else {
			i.pkg.Dependencies = append(i.pkg.Dependencies, d.GetDependencySignature())
		}
	}

	err := i.pkg.Save()
	if err != nil {
		i.Error(err)
		return err
	}

	return nil
}

func (i *InstallCommand) download(url string, destination string) (fileName string, err error) {

	tokens := strings.Split(url, "/")
	fileName = destination + string(filepath.Separator) + tokens[len(tokens)-2] + core.TarSuffix // FIXME: we assume it's a tarball

	var output *os.File
	output, err = os.Create(fileName)
	if err != nil {
		// TODO: check file existence first with os.IsExist(err)
		i.Error(err)
		return
	}
	defer output.Close()

	var response *http.Response
	response, err = http.Get(url)
	if err != nil {
		i.Error(err)
		return
	}
	defer response.Body.Close()

	//proxy := &ProgressProxyReader{ Reader: response.Body, length: response.ContentLength }
	//var written int64
	_, err = io.Copy(output, response.Body)
	if err != nil {
		i.Error(err)
		return
	}

	return
}

func (i *InstallCommand) extract(fileName string, destination string) (*common.PackageWrapper, error) {

	file, err := os.Open(fileName)

	if err != nil {
		i.Error(err)
		return nil, err
	}

	defer file.Close()

	var fileReader io.ReadCloser = file

	// add a filter to handle gzipped file
	if strings.HasSuffix(fileName, ".gz") {
		if fileReader, err = gzip.NewReader(file); err != nil {
			i.Error(err)
			return nil, err
		}
		defer fileReader.Close()
	}

	tarBallReader := tar.NewReader(fileReader)
	var topDir string

	for {
		header, err := tarBallReader.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			i.Error(err)
			return nil, err
		}

		filename := destination + string(filepath.Separator) + header.Name

		switch header.Typeflag {
		case tar.TypeDir:
			tokens := strings.Split(header.Name, string(filepath.Separator))
			topDir = tokens[0]
			err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755
			if err != nil {
				i.Error(err)
				return nil, err
			}

		case tar.TypeReg:
			writer, err := os.Create(filename)
			if err != nil {
				i.Error(err)
				return nil, err
			}
			io.Copy(writer, tarBallReader)
			err = os.Chmod(filename, os.FileMode(header.Mode))
			if err != nil {
				i.Error(err)
				return nil, err
			}
			writer.Close()

		case tar.TypeXGlobalHeader:
			// Ignore this

		default:
			//i.Info("Unable to extract type : %c in file %s\n", header.Typeflag, filename)
		}
	}

	if topDir != "" {

		src := filepath.Join(destination, topDir)

		pkg, err := common.LoadPackage(src)
		if err != nil {
			i.Error(err)
			return pkg, err
		}

		path := filepath.Join(destination, pkg.QrcPrefix())

		if err := os.MkdirAll(path, 0755); err != nil {
			i.Error(err)
			return pkg, err
		}

		os.RemoveAll(path)

		if err = os.Rename(src, path); err != nil {
			i.Error(err)
			return pkg, err
		}

		// Reload it from the new location
		pkg, err = common.LoadPackage(path)
		if err != nil {
			i.Error(err)
		}

		return pkg, err
	}

	return nil, nil
}

func (i *InstallCommand) postInstall(dependencies []*common.PackageWrapper) error {
	if err := GenerateVendorPri(i.vendorDir, i.pkg, dependencies); err != nil {
		i.Error(err)
		return err
	}
	return nil
}

// Generates a vendor.pri inside vendorDir using the information contained in the package file
// and the dependencies
func GenerateVendorPri(vendorDir string, pkg *common.PackageWrapper, deps []*common.PackageWrapper) error {

	vendorPriFile := filepath.Join(vendorDir, core.Vendor+".pri")

	var file *os.File
	var err error

	// re-create the .pri file
	file, err = os.Create(vendorPriFile)

	if err != nil {
		return err
	}
	defer file.Close()

	data := struct {
		VendorDir    string
		Package      *common.PackageWrapper
		Dependencies []*common.PackageWrapper
	}{
		vendorDir,
		pkg,
		deps,
	}

	if err := vendorPri.Execute(file, data); err != nil {
		return err
	}

	return nil
}
<commit_msg>Add better handling for Github errors. Fixes #5.<commit_after>// Copyright 2015 Cutehacks AS. All rights reserved.
// License can be found in the LICENSE file.

package commands

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"golang.org/x/net/context"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"qpm.io/common"
	msg "qpm.io/common/messages"
	"qpm.io/qpm/core"
	"strings"
	"text/template"
	"encoding/json"
)

var packageFuncs = template.FuncMap{
	"relPriFile": func(vendorDir string, dep *common.PackageWrapper) string {
		abs := filepath.Join(dep.RootDir(), dep.PriFile())
		rel, err := filepath.Rel(vendorDir, abs)
		if err == nil {
			return rel
		} else {
			return abs
		}
	},
}

var (
	vendorPri = template.Must(template.New("vendorPri").Funcs(packageFuncs).Parse(`
DEFINES += QPM_INIT\\(E\\)=\"E.addImportPath(QStringLiteral(\\\"qrc:/\\\"));\"

{{$vendirDir := .VendorDir}}
{{range $dep := .Dependencies}}
include($$PWD/{{relPriFile $vendirDir $dep}})
{{end}}
`))
)

type ProgressProxyReader struct {
	io.Reader
	total    int64
	length   int64
	progress float64
}

func (r *ProgressProxyReader) Read(p []byte) (int, error) {
	n, err := r.Reader.Read(p)
	if n > 0 {
		r.total += int64(n)
		percentage := float64(r.total) / float64(r.length) * float64(100)
		i := int(percentage / float64(10))
		is := fmt.Sprintf("%v", i)
		if percentage-r.progress > 2 {
			fmt.Fprintf(os.Stderr, is)
			r.progress = percentage
		}
	}
	return n, err
}

type InstallCommand struct {
	BaseCommand
	pkg       *common.PackageWrapper
	fs        *flag.FlagSet
	vendorDir string
}

func NewInstallCommand(ctx core.Context) *InstallCommand {
	return &InstallCommand{
		BaseCommand: BaseCommand{
			Ctx: ctx,
		},
	}
}

func (i InstallCommand) Description() string {
	return "Installs a new package"
}

func (i *InstallCommand) RegisterFlags(flags *flag.FlagSet) {
	i.fs = flags

	// TODO: Support other directory names on the command line?
	var err error
	i.vendorDir, err = filepath.Abs(core.Vendor)
	if err != nil {
		i.vendorDir = core.Vendor
	}
}

func (i *InstallCommand) Run() error {

	packageName := i.fs.Arg(0)

	var err error
	i.pkg, err = common.LoadPackage("")
	if err != nil {
		// A missing package file is only an error if packageName is empty
		if os.IsNotExist(err) {
			if packageName == "" {
				err = fmt.Errorf("No %s file found", core.PackageFile)
				i.Error(err)
				return err
			} else {
				// Create a new package
				file, err := filepath.Abs(core.PackageFile)
				if err != nil {
					i.Error(err)
					return err
				}
				i.pkg = common.NewPackageWrapper(file)
			}
		} else {
			i.Error(err)
			return err
		}
	}

	var packageNames []string
	if packageName == "" {
		packageNames = i.pkg.Dependencies
	} else {
		packageNames = []string{packageName}
	}

	// Get list of dependencies from the server
	response, err := i.Ctx.Client.GetDependencies(context.Background(), &msg.DependencyRequest{packageNames})
	if err != nil {
		i.Error(err)
		return err
	}

	if len(response.Dependencies) == 0 {
		i.Info("No package(s) found")
		return nil
	}

	// create the vendor directory if needed
	if _, err = os.Stat(i.vendorDir); err != nil {
		err = os.Mkdir(i.vendorDir, 0755)
	}

	// Download and extract the packages
	packages := []*common.PackageWrapper{}
	for _, d := range response.Dependencies {
		p, err := i.install(d)
		if err != nil {
			return err
		}
		packages = append(packages, p)
	}

	// Save the dependencies in the package file
	err = i.save(packages)
	// FIXME: should we continue installing ?
	if err != nil {
		return err
	}

	err = i.postInstall(packages)
	// FIXME: should we continue installing ?
	if err != nil {
		return err
	}

	return nil
}

func (i *InstallCommand) install(d *msg.Dependency) (*common.PackageWrapper, error) {

	url := core.GitHub + "/" + d.Repository.Url + "/" + core.Tarball

	signature := strings.Join([]string{d.Name, d.Version.Label}, "@")
	fmt.Println("Installing", signature)

	fileName, err := i.download(url, i.vendorDir)
	if err != nil {
		return nil, err
	}

	pkg, err := i.extract(fileName, i.vendorDir)
	if err != nil {
		return nil, err
	}

	return pkg, os.Remove(fileName)
}

func (i *InstallCommand) save(newDeps []*common.PackageWrapper) error {

	existingDeps := i.pkg.ParseDependencies()

	for _, d := range newDeps {
		existingVersion, exists := existingDeps[d.Name]
		if exists {
			if d.Version.Label == existingVersion {
				i.Info("The package is already a dependency : " + d.GetDependencySignature())
			} else {
				// TODO: Handle conflicts
				err := fmt.Errorf("Conflict for package %s. Version %s != %s", d.Name, existingVersion, d.Version.Label)
				i.Error(err)
				return err
			}
		} else {
			i.pkg.Dependencies = append(i.pkg.Dependencies, d.GetDependencySignature())
		}
	}

	err := i.pkg.Save()
	if err != nil {
		i.Error(err)
		return err
	}

	return nil
}

func (i *InstallCommand) download(url string, destination string) (fileName string, err error) {

	tokens := strings.Split(url, "/")
	fileName = destination + string(filepath.Separator) + tokens[len(tokens)-2] + core.TarSuffix // FIXME: we assume it's a tarball

	var output *os.File
	output, err = os.Create(fileName)
	if err != nil {
		// TODO: check file existence first with os.IsExist(err)
		i.Error(err)
		return
	}
	defer output.Close()

	var response *http.Response
	response, err = http.Get(url)
	if err != nil {
		i.Error(err)
		return
	}
	defer response.Body.Close()

	if response.StatusCode >= 400 {
		errResp := make(map[string]string)
		dec := json.NewDecoder(response.Body)
		if err = dec.Decode(&errResp); err != nil {
			i.Error(err)
			return
		}
		errMsg, ok := errResp["message"]
		if !ok {
			errMsg = response.Status
		}
		err = fmt.Errorf("Error fetching %s: %s", url, errMsg)
		i.Error(err)
		return
	}

	//proxy := &ProgressProxyReader{ Reader: response.Body, length: response.ContentLength }
	//var written int64
	_, err = io.Copy(output, response.Body)
	if err != nil {
		i.Error(err)
		return
	}

	return
}

func (i *InstallCommand) extract(fileName string, destination string) (*common.PackageWrapper, error) {

	file, err := os.Open(fileName)

	if err != nil {
		i.Error(err)
		return nil, err
	}

	defer file.Close()

	var fileReader io.ReadCloser = file

	// add a filter to handle gzipped file
	if strings.HasSuffix(fileName, ".gz") {
		if fileReader, err = gzip.NewReader(file); err != nil {
			i.Error(err)
			return nil, err
		}
		defer fileReader.Close()
	}

	tarBallReader := tar.NewReader(fileReader)
	var topDir string

	for {
		header, err := tarBallReader.Next()
		if err != nil {
			if err == io.EOF {
				break
			}
			i.Error(err)
			return nil, err
		}

		filename := destination + string(filepath.Separator) + header.Name

		switch header.Typeflag {
		case tar.TypeDir:
			tokens := strings.Split(header.Name, string(filepath.Separator))
			topDir = tokens[0]
			err = os.MkdirAll(filename, os.FileMode(header.Mode)) // or use 0755
			if err != nil {
				i.Error(err)
				return nil, err
			}

		case tar.TypeReg:
			writer, err := os.Create(filename)
			if err != nil {
				i.Error(err)
				return nil, err
			}
			io.Copy(writer, tarBallReader)
			err = os.Chmod(filename, os.FileMode(header.Mode))
			if err != nil {
				i.Error(err)
				return nil, err
			}
			writer.Close()

		case tar.TypeXGlobalHeader:
			// Ignore this

		default:
			//i.Info("Unable to extract type : %c in file %s\n", header.Typeflag, filename)
		}
	}

	if topDir != "" {

		src := filepath.Join(destination, topDir)

		pkg, err := common.LoadPackage(src)
		if err != nil {
			i.Error(err)
			return pkg, err
		}

		path := filepath.Join(destination, pkg.QrcPrefix())

		if err := os.MkdirAll(path, 0755); err != nil {
			i.Error(err)
			return pkg, err
		}

		os.RemoveAll(path)

		if err = os.Rename(src, path); err != nil {
			i.Error(err)
			return pkg, err
		}

		// Reload it from the new location
		pkg, err = common.LoadPackage(path)
		if err != nil {
			i.Error(err)
		}

		return pkg, err
	}

	return nil, nil
}

func (i *InstallCommand) postInstall(dependencies []*common.PackageWrapper) error {
	if err := GenerateVendorPri(i.vendorDir, i.pkg, dependencies); err != nil {
		i.Error(err)
		return err
	}
	return nil
}

// Generates a vendor.pri inside vendorDir using the information contained in the package file
// and the dependencies
func GenerateVendorPri(vendorDir string, pkg *common.PackageWrapper, deps []*common.PackageWrapper) error {

	vendorPriFile := filepath.Join(vendorDir, core.Vendor+".pri")

	var file *os.File
	var err error

	// re-create the .pri file
	file, err = os.Create(vendorPriFile)

	if err != nil {
		return err
	}
	defer file.Close()

	data := struct {
		VendorDir    string
		Package      *common.PackageWrapper
		Dependencies []*common.PackageWrapper
	}{
		vendorDir,
		pkg,
		deps,
	}

	if err := vendorPri.Execute(file, data); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	// "errors"
	"fmt"
	"os"
	// "strings"
)

//
func things(st *Status) string {
	if st.CRole == "primary" {
		return "secondary"
	}
	return "primary"
}

//
func configureHBAConf() error {

	self := myself()
	other, err := Whois(otherRole(self))
	if err != nil {
		log.Warn("I cant find another. disabling replication")
	}

	//
	entry := "host    all             all             all           trust"

	if other != nil {
		entry += fmt.Sprintf(`
host    replication     postgres        %s/32            trust`, other.Ip)
	}

	file := conf.DataDir + "pg_hba.conf"

	//
	// fi, err := stat(dataRoot"pg_hba.conf")

	//
	f, err := os.Create(file)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to create '%s'!\n%s\n", file, err)
		return err
	}

	//
	if _, err := f.WriteString(entry); err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

//
func configurePGConf(master bool) error {

	//
	entry := `
listen_addresses = '0.0.0.0'
max_connections = 100
shared_buffers = 128MB
log_timezone = 'UTC'
datestyle = 'iso, mdy'
timezone = 'UTC'
lc_messages = 'en_US.UTF-8'
lc_monetary = 'en_US.UTF-8'
lc_numeric = 'en_US.UTF-8'
lc_time = 'en_US.UTF-8'
default_text_search_config = 'pg_catalog.english'

wal_level = hot_standby
archive_mode = on
archive_command = 'exit 0'
max_wal_senders = 10
wal_keep_segments = 5000
hot_standby = on`

	// master only
	if master {
		entry += `
synchronous_standby_names = slave`
	}

	file := conf.DataDir + "postgresql.conf"

	//
	f, err := os.Create(file)
	if err != nil {
		log.Error("[pg_config.configurePGConf] Failed to create '%s'!\n%s\n", file, err)
		return err
	}

	//
	if _, err := f.WriteString(entry); err != nil {
		log.Error("[pg_config.configurePGConf] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	// #wal_level = minimal                    # minimal, archive, or hot_standby
	//                                         # (change requires restart)
	// #archive_mode = off             # allows archiving to be done
	//                                 # (change requires restart)
	// #archive_command = ''           # command to use to archive a logfile segment
	//                                 # placeholders: %p = path of file to archive
	//                                 #               %f = file name only
	//                                 # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
	// #max_wal_senders = 0            # max number of walsender processes
	//                                 # (change requires restart)
	// #wal_keep_segments = 0          # in logfile segments, 16MB each; 0 disables
	// #hot_standby = off                      # "on" allows queries during recovery
	//                                         # (change requires restart)

	//
	// opts := make(map[string]string)
	// opts["wal_level"] 								= "hot_standby"
	// opts["archive_mode"] 							= "on"
	// opts["archive_command"] 					= "exit 0"
	// opts["max_wal_senders"] 					= "10"
	// opts["wal_keep_segments"] 				= "5000"
	// opts["hot_standby"] 							= "on"
	// opts["synchronous_standby_names"] = "slave"

	return nil
}

//
func createRecovery() error {

	file := conf.DataDir + "recovery.conf"

	self := myself()
	other, err := Whois(otherRole(self))
	if err != nil {
		log.Error("NO OTHER! %s", err)
		os.Exit(1)
	}

	//
	f, err := os.Create(file)
	if err != nil {
		log.Error("[pg_config.createRecovery] Failed to create '%s'!\n%s\n", file, err)
		return err
	}

	//
	entry := fmt.Sprintf(`# -------------------------------------------------------
# PostgreSQL recovery config file generated by Pagoda Box
# -------------------------------------------------------

# When standby_mode is enabled, the PostgreSQL server will work as a standby. It
# tries to connect to the primary according to the connection settings
# primary_conninfo, and receives XLOG records continuously.
standby_mode = on
primary_conninfo = 'host=%s port=%d application_name=slave'

# restore_command specifies the shell command that is executed to copy log files
# back from archival storage. This parameter is *required* for an archive
# recovery, but optional for streaming replication. The given command satisfies
# the requirement without doing anything.
restore_command = 'exit 0'`, other.Ip, other.PGPort)

	//
	if _, err := f.WriteString(entry); err != nil {
		log.Error("[pg_config.createRecovery] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

//
func destroyRecovery() {

	file := conf.DataDir + "recovery.conf"

	//
	err := os.Remove(file)
	if err != nil {
		log.Warn("[pg_config.destroyRecovery] No recovery.conf found at '%s'", file)
	}
}

//
func stat(f string) (os.FileInfo, error) {
	fi, err := os.Stat(f)
	if err != nil {
		log.Fatal("[pg_config.readFile]", err)
		return nil, err
	}

	return fi, nil
}

// parseFile will parse a config file, returning a 'opts' map of the resulting
// config options.
func parseFile(file string) (map[string]string, error) {

	// attempt to open file
	f, err := os.Open(file)
	if err != nil {
		return nil, err
	}

	defer f.Close()

	m := make(map[string]string)
	s := make([]string, 0)

	scanner := bufio.NewScanner(f)
	readLine := 1

	// Read line by line, sending lines to parseLine
	for scanner.Scan() {
		if err := parseLine(scanner.Text(), m, s); err != nil {
			log.Error("[pg_config] Error reading line: %v\n", readLine)
			return nil, err
		}

		readLine++
	}

	fmt.Println("SLICE???", s)

	return m, nil
}

// parseLine reads each line of the config file, extracting a key/value pair to
// insert into an 'conf' map.
func parseLine(line string, m map[string]string, s []string) error {

	// if the line isn't already in the map add it
	if _, ok := m[line]; !ok {
		m[line] = line
		s = append(s, line)
	}

	return nil
}
<commit_msg>making pg_config actually work, rather than just hard coding everything for testing<commit_after>package main

import(
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
)

//
func configureHBAConf() error {

	//
  other, err := Whois(otherRole(myself()))
  if err != nil {
  	log.Warn("[pg_config.configureHBAConf] Unable to find another!\n%s\n", err)
  }

	//
	file := conf.DataDir+"pg_hba.conf"
	f, err := os.Open(file)
	if err != nil {
		log.Error("[pg_config.configureHBAConf] Failed to open '%s'!\n%s\n", file, err)
		return err
	}

	defer f.Close()

	//
	scanner := bufio.NewScanner(f)
	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(local|host)\s*(replication)`)
	readLine 	:= 1
	entry 		:= ""

	// Read file line by line
	for scanner.Scan() {

		// dont care about submatches, just if the string matches
		if reFindConfigOption.FindString(scanner.Text()) == "" {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}

		readLine++
	}

  //
	if other != nil {
		entry += fmt.Sprintf(`
#------------------------------------------------------------------------------
# PAGODA BOX
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

host    replication     postgres        %s/32            trust`, other.Ip)
	}

	//
	err = ioutil.WriteFile(file, []byte(entry), 0644)
  if err != nil {
  	log.Error("[pg_config.configureHBAConf] Failed to write to '%s'!\n%s\n", file, err)
  	return err
  }

	return nil
}

//
func configurePGConf(master bool) error {

	//
	file := conf.DataDir+"postgresql.conf"
	f, err := os.Open(file)
	if err != nil {
		return err
	}

	defer f.Close()

	reFindConfigOption := regexp.MustCompile(`^\s*#?\s*(listen_addresses|port|wal_level|archive_mode|archive_command|max_wal_senders|wal_keep_segments|hot_standby)\s*=\s*`)
	scanner := bufio.NewScanner(f)
	readLine := 1
	entry := ""

	// Read file line by line
	for scanner.Scan() {

		submatch := reFindConfigOption.FindStringSubmatch(scanner.Text())

		//
		if submatch == nil {
			entry += fmt.Sprintf("%s\n", scanner.Text())
		}

		readLine++
	}

	//
	entry += fmt.Sprintf(`
#------------------------------------------------------------------------------
# PAGODA BOX
#------------------------------------------------------------------------------

# these configuration options have been removed from their standard location and
# placed here so that Pagoda Box could override them with the neccessary values
# to configure redundancy.

# IMPORTANT: these settings will always be overriden when the server boots. They
# are set dynamically and so should never change.

listen_addresses = 0.0.0.0        # what IP address(es) to listen on;
                                  # comma-separated list of addresses;
                                  # defaults to 'localhost'; use '*' for all
                                  # (change requires restart)
port = %d                     # (change requires restart)
wal_level = hot_standy            # minimal, archive, or hot_standby
                                  # (change requires restart)
archive_mode = on                 # allows archiving to be done
                                  # (change requires restart)
archive_command = 'exit 0'        # command to use to archive a logfile segment
                                  # placeholders: %p = path of file to archive
                                  #               %f = file name only
                                  # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
max_wal_senders = 10              # max number of walsender processes
                                  # (change requires restart)
wal_keep_segments = 5000          # in logfile segments, 16MB each; 0 disables
hot_standby = on                  # "on" allows queries during recovery
                                  # (change requires restart)`, conf.PGPort)

	//
	if master {
		entry += `
synchronous_standby_names = slave # standby servers that provide sync rep
                                  # comma-separated list of application_name
                                  # from standby(s); '*' = all`
	}

	//
	err = ioutil.WriteFile(file, []byte(entry), 0644)
	if err != nil {
		log.Error("[pg_config.configurePGConf] Failed to write to '%s'!\n%s\n", file, err)
	}

	return nil
}

//
func createRecovery() error {

	file := conf.DataDir+"recovery.conf"

	self := myself()
  other, err := Whois(otherRole(self))
  if err != nil {
  	log.Fatal("[pg_config.createRecovery] Unable to find another... Exiting!\n%s\n", err)
  	os.Exit(1)
  }

	//
	f, err := os.Create(file)
	if err != nil {
		log.Error("[pg_config.createRecovery] Failed to create '%s'!\n%s\n", file, err)
		return err
	}

	//
	entry := fmt.Sprintf(`
# -------------------------------------------------------
# PAGODA BOX
# -------------------------------------------------------

# IMPORTANT: this config file is dynamically generated by Pagoda Box for redundancy
# any changes made here will be overriden.

# When standby_mode is enabled, the PostgreSQL server will work as a standby. It
# tries to connect to the primary according to the connection settings
# primary_conninfo, and receives XLOG records continuously.
standby_mode = on
primary_conninfo = 'host=%s port=%d application_name=slave'

# restore_command specifies the shell command that is executed to copy log files
# back from archival storage. This parameter is *required* for an archive
# recovery, but optional for streaming replication. The given command satisfies
# the requirement without doing anything.
restore_command = 'exit 0'`, other.Ip, other.PGPort)

	//
	if _, err := f.WriteString(entry); err != nil {
		log.Error("[pg_config.createRecovery] Failed to write to '%s'!\n%s\n", file, err)
		return err
	}

	return nil
}

//
func destroyRecovery() {

	file := conf.DataDir+"recovery.conf"

	//
	err := os.Remove(file)
	if err != nil {
		log.Warn("[pg_config.destroyRecovery] No recovery.conf found at '%s'", file)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	util "github.com/kopoli/go-util"
	"github.com/kopoliitti/paperless/lib"

	"encoding/json"
	"mime/multipart"

	"github.com/jmoiron/sqlx"
	_ "github.com/mattn/go-sqlite3"

	"github.com/codegangsta/cli"

	"net/http"
	"path/filepath"

	"github.com/gorilla/mux"
)

// Supplied information

var version = "development"
var instDatadir string
var instWebdir string

/// Configuration

var (
	uploaddir  = "images"
	dbfilename = "paperless.sqlite3"
)

type progPreferences struct {
	datadir string
	dbfile  string
	webdir  string
	server  string
	verbose bool
}

var preferences = progPreferences{
	datadir: "./target",
	webdir:  "./web/",
	server:  "http://localhost:8078",
	verbose: false,
}

// checks given error and panics if not nil
func check(err error) {
	if err == nil {
		return
	}
	pc := make([]uintptr, 10)
	runtime.Callers(2, pc)
	f := runtime.FuncForPC(pc[0])
	if preferences.verbose {
		file, line := f.FileLine(pc[0])
		log.Panic(file, ":", line, ": ", f.Name(), ": ", err)
	} else {
		log.Panic(f.Name(), ": ", err)
	}
}

/// Image structure
type img struct {
	// in img
	Id            int
	Checksum      string
	Fileid        string
	IsDiscarded   int
	ParentId      int
	TmScanned     int64
	TmProcessed   int64
	TmReinterpret int64
	ProcessLog    string
	Filename      string
	IsUploaded    int
	IsToBeOcrd    int

	// in imgtext
	Text    string
	Comment string

	// in tags
	Tags []string
}

func imagePath(image *img, body string) string {
	return path.Join(uploaddir,
		fmt.Sprintf("%05d-%s", image.Id, body))
}

func createUploadRequest(url string, path string) (req *http.Request) {
	fp, err := os.Open(path)
	check(err)
	defer fp.Close()
	// data, err := ioutil.ReadAll(fp)
	// check(err)

	var buf bytes.Buffer
	writer := multipart.NewWriter(&buf)
	part, err := writer.CreateFormFile("image", fp.Name())
	check(err)
	_, err = io.Copy(part, fp)
	// _, err = part.Write(data)
	check(err)
	err = writer.Close()
	check(err)

	req, err = http.NewRequest("POST", url, &buf)
	req.Header.Set("Content-type", writer.FormDataContentType())
	check(err)
	return
}

/// Process images and upload to server
func ocrAndUpload(fname string, url string) {

	var image img
	var err error

	// Get the checksum of the image
	image.Checksum, err = paperless.ChecksumFile(fname)
	check(err)

	// fmt.Println(path, image.Checksum)

	// Run through OCR
	fp, err := ioutil.TempFile("", "paperless-ocr")
	check(err)
	defer fp.Close()
	tmpname := fp.Name()

	cmd := exec.Command(path.Join(preferences.webdir, "process.sh"), "ocr", fname, fp.Name())
	output, err := cmd.CombinedOutput()
	if err != nil {
		log.Println("Command failed, output", output)
	}
	check(err)

	data, err := ioutil.ReadAll(fp)
	check(err)
	fp.Close()

	image.ProcessLog = string(output)
	image.Text = string(data)

	err = os.Remove(tmpname)
	check(err)

	// Marshal to JSON
	data, err = json.Marshal(image)
	check(err)
	fmt.Println(string(data))

	// Upload the data to server
	buf := bytes.NewBuffer(data)
	req, err := http.NewRequest("POST", url+"/images", buf)
	check(err)
	req.Header.Set("Content-type", "application/json")
	client := &http.Client{}
	resp, err := client.Do(req)
	check(err)
	defer resp.Body.Close()

	fmt.Println("Status:", resp.Status)
	fmt.Println("Headers:", resp.Header)
	data, err = ioutil.ReadAll(resp.Body)
	check(err)
	fmt.Println("Body:", string(data))

	// Uploading the picture
	err = json.Unmarshal(data, &image)
	check(err)

	uploadURL := fmt.Sprintf(url+"/images/upload/%d", image.Id)
	fmt.Println("Uploadurl on ", uploadURL)
	req = createUploadRequest(uploadURL, fname)
	_, err = client.Do(req)
	check(err)
}

func upload(fname string, url string, tags []string) {

	// Upload the image
	client := &http.Client{}
	uploadUrl := fmt.Sprintf(url + "/images/upload")
	req := createUploadRequest(uploadUrl, fname)
	resp, err := client.Do(req)
	check(err)

	// Update the information
	var image img
	data, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	check(err)

	if resp.StatusCode != http.StatusCreated {
		type jsonerror struct {
			Error string
		}
		var jerr jsonerror
		err = json.Unmarshal(data, &jerr)
		check(err)
		check(fmt.Errorf("Server error: %s", jerr.Error))
	}

	err = json.Unmarshal(data, &image)
	check(err)

	info, err := os.Stat(fname)
	image.TmScanned = info.ModTime().Unix()
	image.Tags = tags

	// Upload the updated information
	data, err = json.Marshal(image)
	check(err)

	buf := bytes.NewBuffer(data)
	req, err = http.NewRequest("POST", fmt.Sprintf("%s/images/%d", url, image.Id), buf)
	check(err)

	req.Header.Set("Content-type", "application/json")
	resp, err = client.Do(req)
	check(err)
}

/// Add command
func mainAdd(c *cli.Context) {
	preferences.verbose = c.GlobalBool("verbose")
	preferences.server = c.String("server-url")

	tags := strings.Split(c.String("tags"), ",")

	if len(tags) == 0 {
		log.Panic("Initial tags must be given for images.")
	}

	for _, fname := range c.Args() {
		if preferences.verbose {
			log.Println("Uploading image", fname)
		}
		func() {
			defer func() {
				if err := recover(); err != nil {
					log.Printf("Uploading image \"%s\" failed due to error: %s", fname, err)
				}
			}()
			upload(fname, preferences.server, tags)
		}()
	}
}

/// SQlite functionality
type database struct {
	*sqlx.DB
}
type databaseTx struct {
	*sqlx.Tx
}

func openDbFile(dbfile string) database {
	create := false

	if _, err := os.Stat(dbfile); os.IsNotExist(err) {
		create = true
	}

	db, err := sqlx.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc", dbfile))
	check(err)

	if create {
		log.Println("Initializing the database file", dbfile)
		_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS tag (
  id INTEGER PRIMARY KEY,
  name TEXT UNIQUE ON CONFLICT IGNORE
);

-- The image data
CREATE TABLE IF NOT EXISTS img (
 id INTEGER PRIMARY KEY ASC AUTOINCREMENT,
  checksum TEXT UNIQUE ON CONFLICT ABORT,	-- checksum of the file
  fileid TEXT DEFAULT "",                       -- used to construct the processed image,
						--   thumbnail and text files
  isdiscarded INTEGER DEFAULT 0,                -- Should the image be ignored on ocr
  parentid INTEGER DEFAULT -1,                  -- ID of a parent image
  tmscanned INTEGER DEFAULT 0,                  -- timestamp when it was scanned
  tmprocessed INTEGER DEFAULT 0,                -- timestamp when it was processed
  tmreinterpret INTEGER DEFAULT 0,              -- timestamp when it was reinterpret

  processlog TEXT DEFAULT "",                   -- Log of processing
  filename TEXT DEFAULT "",                     -- The original filename
  isuploaded INTEGER DEFAULT 0,                 -- has a real image been uploaded to paperless yet.

  istobeocrd INTEGER DEFAULT 0			-- Needs to be re-ocr'd
);

CREATE VIRTUAL TABLE IF NOT EXISTS imgtext USING fts4 (
  text DEFAULT "",				-- the OCR'd text
  comment DEFAULT ""				-- freeform comment
);

-- Tags for an image
CREATE TABLE IF NOT EXISTS imgtag (
  tagid INTEGER REFERENCES tag(id) NOT NULL,
  imgid INTEGER REFERENCES img(id) NOT NULL
);
`)
		if err != nil {
			db.Close()
			check(err)
		}

	}
	db.Exec("PRAGMA busy_timeout=2000")
	if err != nil {
		db.Close()
		check(err)
	}

	return database{db}
}

func openDb() database {
	return openDbFile(preferences.dbfile)
}

func (tx databaseTx) updateTags(image *img) {
	_, err := tx.Exec("DELETE FROM imgtag WHERE imgid = $1", image.Id)
	check(err)
	for i := range image.Tags {
		_, err = tx.Exec("INSERT OR IGNORE INTO tag(name) VALUES ($1)",
			image.Tags[i])
		check(err)
		_, err = tx.Exec("INSERT INTO imgtag(tagid,imgid) "+
			"SELECT tag.id, $1 FROM tag WHERE tag.name = $2", image.Id, image.Tags[i])
		check(err)
	}
}

func (db database) addImage(image *img) {
	image.IsUploaded = 1
	image.IsToBeOcrd = 0
	image.IsDiscarded = 0

	sqltx, err := db.Beginx()
	tx := databaseTx{sqltx}
	check(err)

	_, err = tx.NamedExec("INSERT INTO img(checksum,fileid,processlog,filename) "+
		"VALUES (:checksum,:fileid,:processlog,:filename)", image)
	check(err)
	tmp := img{}
	err = tx.Get(&tmp, "SELECT id from img where checksum=$1", image.Checksum)
	check(err)
	image.Id = tmp.Id
	_, err = tx.NamedExec("INSERT INTO imgtext(rowid,text,comment) VALUES (:id,:text,:comment)",
		image)
	check(err)
	tx.updateTags(image)

	err = tx.Commit()
	check(err)
}

func (db database) getTags(image *img) {
	err := db.Select(&image.Tags, "SELECT tag.name FROM imgtag, tag "+
		"WHERE imgtag.imgid = $1 AND tag.id = imgtag.tagid", image.Id)
	check(err)
}

func (db database) getImage(id int) (image img) {
	image.Id = id
	err := db.Get(&image, "SELECT * FROM img, imgtext WHERE img.id = $1 AND img.id = imgtext.rowid", id)
	check(err)

	db.getTags(&image)

	return image
}

func (db database) updateImage(image img) {
	sqltx, err := db.Beginx()
	tx := databaseTx{sqltx}
	check(err)
	_, err = tx.NamedExec(`UPDATE img SET
	fileid = :fileid,
	isdiscarded = :isdiscarded,
	parentid = :parentid,
	tmscanned = :tmscanned,
	tmprocessed = :tmprocessed,
	tmreinterpret = :tmreinterpret,
	processlog = :processlog,
	filename = :filename,
	isuploaded = :isuploaded,
	istobeocrd = :istobeocrd
	WHERE img.id = :id`, image)
	check(err)

	_, err = tx.NamedExec(`UPDATE imgtext SET
	text = :text,
	comment = :comment
	WHERE imgtext.rowid = :id`, image)
	check(err)
	tx.updateTags(&image)
	err = tx.Commit()
	check(err)
}

func (db database) getAllImages(limit int, offset int, search string) (count int, images []img) {
	var err error
	var ids []int

	if search != "" {
		err = db.Select(&ids, "SELECT rowid FROM imgtext WHERE imgtext.text MATCH ?", search)
		check(err)
		count = len(ids)
		if count == 0 {
			return
		}

		err = db.Select(&images, "SELECT * from img, imgtext "+
			"WHERE imgtext.text MATCH ? AND imgtext.rowid = img.id LIMIT ? OFFSET ?",
			search, limit, offset)
	} else {
		err = db.Select(&ids, "SELECT id from img")
		check(err)
		count = len(ids)
		err = db.Select(&images, "SELECT * from img, imgtext "+
			"WHERE imgtext.rowid = img.id LIMIT ? OFFSET ?",
			limit, offset)
	}
	check(err)

	for i := range images {
		db.getTags(&images[i])
		images[i].Fileid = filepath.Base(imagePath(&images[i], ""))
	}

	return
}

/// web handlers
func handleResponse(response interface{}, code int, w http.ResponseWriter,
	r *http.Request) {

	w.WriteHeader(code)
	if response != nil {
		bytes, err := json.Marshal(response)
		if err != nil {
			http.Error(w, "Internal error.",
				http.StatusInternalServerError)
		}

		w.Header().Set("Content-Type", "application/json")
		w.Write(bytes)
	}
	log.Printf("Responder %s %s %s %d", r.RemoteAddr, r.Method, r.URL, code)
}

func serveIndex(w http.ResponseWriter, r *http.Request) {
	http.ServeFile(w, r, path.Join(preferences.webdir, "paperless.html"))
}

type imageResponse struct {
	Limit  int
	Offset int
	Count  int
	Images []img
}

func listImages(w http.ResponseWriter, r *http.Request) {
	var err error
	limit := -1
	offset := 0
	search := ""
	args := r.URL.Query()
	if _, ok := args["limit"]; ok {
		limit, err = strconv.Atoi(args["limit"][0])
		check(err)
	}
	if _, ok := args["offset"]; ok {
		limit, err = strconv.Atoi(args["offset"][0])
		check(err)
	}
	if _, ok := args["q"]; ok {
		search = args["q"][0]
	}

	db := openDb()
	defer db.Close()
	count, images := db.getAllImages(limit, offset, search)

	response := imageResponse{
		Limit:  limit,
		Offset: offset,
		Count:  count,
		Images: images,
	}

	handleResponse(response, http.StatusOK, w, r)
}

func addImage(w http.ResponseWriter, r *http.Request) {
	text, err := ioutil.ReadAll(r.Body)
	check(err)

	var image img
	err = json.Unmarshal(text, &image)
	check(err)

	log.Println(string(text))
	db := openDb()
	defer db.Close()
	db.addImage(&image)
	handleResponse(image, http.StatusCreated, w, r)
}

func receiveFile(image *img, r *http.Request) {
	err := r.ParseMultipartForm(1 * 1024 * 1024)
	check(err)
	file, header, err := r.FormFile("image")
	check(err)

	image.Filename = header.Filename
	extension := strings.ToLower(path.Ext(header.Filename))
	fp, err := os.OpenFile(path.Join(preferences.datadir, imagePath(image, "original"+extension)),
		os.O_WRONLY|os.O_CREATE, 0666)
	check(err)
	defer fp.Close()

	_, err = io.Copy(fp, file)
	check(err)

	// TODO error out if checksums differ

	// KOMENTO kuvan uploadaamiseksi
	// curl -v -F image=@testdata/korkea_prioriteetti_2012-13/SCAN0022.JPG http://localhost:8078/images/upload/1
}

func receiveImage(w http.ResponseWriter, r *http.Request) {

	var err error
	vars := mux.Vars(r)
	strid := vars["id"]
	id, err := strconv.Atoi(strid)
	check(err)

	db := openDb()
	defer db.Close()
	image := db.getImage(id)

	if image.IsUploaded != 0 {
		handleResponse(nil, http.StatusConflict, w, r)
	}

	log.Println("Receiveimage headers", r.Header)

	receiveFile(&image, r)
	image.IsUploaded = 1
	log.Printf("Updating the image %+v", image)

	db.updateImage(image)

	handleResponse(nil, http.StatusAccepted, w, r)
}

func receiveNewImage(w http.ResponseWriter, r *http.Request) {

	// receive the file
	err := r.ParseMultipartForm(1 * 1024 * 1024)
	check(err)
	file, header, err := r.FormFile("image")
	check(err)

	data, err := ioutil.ReadAll(file)
	check(err)

	var image img
	image.Checksum = paperless.Checksum(data)
	image.Filename = header.Filename
	image.TmScanned = time.Now().Unix()

	db := openDb()
	defer db.Close()

	// add to db
	db.addImage(&image)

	// add to directory
	extension := strings.ToLower(path.Ext(header.Filename))
	origpath := path.Join(preferences.datadir, imagePath(&image, "original"+extension))
	tgtimgpath := path.Join(preferences.datadir, imagePath(&image, "processed.jpg"))
	thumbpath := path.Join(preferences.datadir, imagePath(&image, "thumb.jpg"))

	fp, err := os.OpenFile(origpath, os.O_WRONLY|os.O_CREATE, 0666)
	check(err)

	_, err = fp.Write(data)
	fp.Close()
	check(err)

	job := paperless.Job{
		Job: func() {
			// Run through OCR
			log.Println("Processing", origpath)
			cmd := exec.Command(path.Join(preferences.webdir, "process.sh"), origpath, tgtimgpath, thumbpath)
			output, err := cmd.CombinedOutput()
			if err != nil {
				log.Println("Command failed, output", string(output))
			}
			check(err)

			re, err := regexp.Compile("__LOG_ENDS_HERE__")
			check(err)
			strs := re.Split(string(output), 2)

			db := openDb()
			defer db.Close()
			image = db.getImage(image.Id)
			image.ProcessLog = string(strs[0])
			image.Text = string(strs[1])
			image.TmProcessed = time.Now().Unix()

			db.updateImage(image)
			log.Println("Processing successful", origpath)
		},
		Finalize: func() {},
	}

	paperless.Pool.Do(job)

	handleResponse(image, http.StatusCreated, w, r)
}

func updateImage(w http.ResponseWriter, r *http.Request) {

	var err error
	vars := mux.Vars(r)
	strid := vars["id"]
	id, err := strconv.Atoi(strid)
	check(err)

	log.Println("Updating image", id)

	db := openDb()
	defer db.Close()
	image := db.getImage(id)

	text, err := ioutil.ReadAll(r.Body)
	check(err)

	err = json.Unmarshal(text, &image)
	check(err)

	db.updateImage(image)

	log.Printf("%+v", image)

	handleResponse(nil, http.StatusAccepted, w, r)
}

func httpErrorWrap(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

		defer func() {
			if err := recover(); err != nil {
				w.WriteHeader(http.StatusBadRequest)
				w.Header().Set("Content-type", "application/json")
				fmt.Fprintf(w, `{"error" : "%s"}`, err)
				log.Printf("%s %s %s Error: %s", r.RemoteAddr, r.Method, r.URL, err)
			}
		}()
		handler.ServeHTTP(w, r)
	})
}

func httpLogWrap(handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		log.Printf("%s %s %s", r.RemoteAddr, r.Method, r.URL)
		handler.ServeHTTP(w, r)
	})
}

/// Web main
func mainStartWeb(c *cli.Context) {
	var err error

	if c.Bool("syslog") {
		paperless.SetupLogging()
	}

	preferences.webdir = c.String("web-directory")
	preferences.datadir = c.String("data-directory")
	preferences.dbfile = path.Join(preferences.datadir, dbfilename)
	preferences.verbose = c.GlobalBool("verbose")

	err = os.MkdirAll(preferences.datadir, 0755)
	check(err)
	err = os.MkdirAll(path.Join(preferences.datadir, uploaddir), 0755)
	check(err)

	db := openDb()
	db.Close()

	paperless.CreateDefaultPool(c.GlobalInt("jobs"))

	router := mux.NewRouter()
	router.HandleFunc("/", serveIndex)
	router.PathPrefix("/html/").Handler(http.StripPrefix("/html",
		http.FileServer(http.Dir(preferences.webdir))))
	router.PathPrefix("/static/").Handler(http.StripPrefix("/static",
		http.FileServer(http.Dir(path.Join(preferences.datadir, uploaddir)))))
	router.HandleFunc("/images", listImages).Methods("GET")
	router.HandleFunc("/images/upload", receiveNewImage).Methods("POST")
	router.HandleFunc("/images/{id:[0-9]+}", updateImage).Methods("POST")
	http.Handle("/", httpErrorWrap(httpLogWrap(router)))

	fmt.Println("Starting web!! port:", c.Int("port"))

	err = http.ListenAndServe(fmt.Sprintf(":%d", c.Int("port")), nil)
	check(err)
}

func printErr(err error, message string, arg ...string) {
	msg := ""
	if err != nil {
		msg = fmt.Sprintf(" (error: %s)", err)
	}
	fmt.Fprintf(os.Stderr, "Error: %s%s.%s\n", message, strings.Join(arg, " "), msg)
}

func fault(err error, message string, arg ...string) {
	printErr(err, message, arg...)
	os.Exit(1)
}

func main() {
	opts := util.NewOptions()

	err := paperless.Cli(opts, os.Args)
	if err != nil {
		fault(err, "Command line parsing failed")
	}

	err = paperless.StartWeb(opts)
	if err != nil {
		fault(err, "Starting paperless web server failed")
	}
}

func main2() {
	if instDatadir != "" {
		preferences.datadir = instDatadir
	}
	if instWebdir != "" {
		preferences.webdir = instWebdir
	}

	app := cli.NewApp()
	app.Name = "paperless"
	app.Usage = "Paperless office utility"
	app.Version = version
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:   "verbose, V",
			Usage:  "Verbose output",
			EnvVar: "PAPERLESS_VERBOSE",
		},
		cli.IntFlag{
			Name:   "jobs, j",
			Value:  runtime.NumCPU(),
			Usage:  "Number of jobs to process the images",
			EnvVar: "PAPERLESS_JOBS",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "add",
			Usage:  "Add a new picture",
			Action: mainAdd,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:   "preprocess, p",
					Usage:  "Preprocess added images through clipper",
					EnvVar: "PAPERLESS_PREPROCESS",
				},
				cli.StringFlag{
					Name:   "server-url, u",
					Value:  preferences.server,
					Usage:  "URL to upload to",
					EnvVar: "PAPERLESS_SERVER_URL",
				},
				cli.StringFlag{
					Name:   "tags, t",
					Value:  "",
					Usage:  "A comma separated list of tags",
					EnvVar: "PAPERLESS_TAGS",
				},
			},
		},
		{
			Name:   "start-web",
			Usage:  "Start web interface",
			Action: mainStartWeb,
			Flags: []cli.Flag{
				cli.IntFlag{
					Name:   "port, p",
					Value:  8078,
					Usage:  "Port to listen to",
					EnvVar: "PAPERLESS_PORT",
				},
				cli.StringFlag{
					Name:   "data-directory, d",
					Value:  preferences.datadir,
					Usage:  "Directory where the images and db are.",
					EnvVar: "PAPERLESS_DATADIR",
				},
				cli.StringFlag{
					Name:   "web-directory, w",
					Value:  preferences.webdir,
					Usage:  "Directory where the static web files are.",
					EnvVar: "PAPERLESS_WEBDIR",
				},
				cli.BoolFlag{
					Name:   "syslog, s",
					Usage:  "Print log through syslog",
					EnvVar: "PAPERLESS_SYSLOG",
				},
			},
		},
	}

	app.Run(os.Args)
}

// Local Variables:
// outline-regexp: "^////*\\|^func\\|^import"
// End:
<commit_msg>paperless: remove old functionality from the main file<commit_after>package main

import (
	"fmt"
	"os"
	"strings"

	util "github.com/kopoli/go-util"
	"github.com/kopoliitti/paperless/lib"
)

func printErr(err error, message string, arg ...string) {
	msg := ""
	if err != nil {
		msg = fmt.Sprintf(" (error: %s)", err)
	}
	fmt.Fprintf(os.Stderr, "Error: %s%s.%s\n", message, strings.Join(arg, " "), msg)
}

func fault(err error, message string, arg ...string) {
	printErr(err, message, arg...)
	os.Exit(1)
}

func main() {
	opts := util.NewOptions()

	err := paperless.Cli(opts, os.Args)
	if err != nil {
		fault(err, "Command line parsing failed")
	}

	err = paperless.StartWeb(opts)
	if err != nil {
		fault(err, "Starting paperless web server failed")
	}
}

// Local Variables:
// outline-regexp: "^////*\\|^func\\|^import"
// End:
<|endoftext|>


<commit_before>package kubernetes

import (
	"encoding/json"
	"fmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/intstr"
)

// pipelineMetadata struct for convert from pipelineVersion.MetaData
type piplineMetadata struct {
	name            string            `json:"name, omitempty"`
	namespace       string            `json:"namespace, omitempty"`
	selfLink        string            `json:"selflink, omitempty"`
	uid             types.UID         `json:"uid, omitempty"`
	createTimestamp unversioned.Time  `json:"createTimestamp, omitempty"`
	deleteTimestamp unversioned.Time  `json:"deleteTimestamp, omitempty"`
	timeoutDuration int64             `json:"timeoutDuration, omitempty"`
	labels          map[string]string `json:"labels, omitempty"`
	annotations     map[string]string `json:"annotations, omitempty"`
}

// pipelineSpec struct for convert from pipelineVersion.Spec
type piplineSpec struct {
	name                string `json:"name, omitempty"`
	replicas            int    `json:"replicas, omitempty"`
	dependencies        string `json:"dependencies, omitempty"`
	kind                string `json:"kind, omitempty"`
	statusCheckLink     string `json:"statusCheckLink, omitempty"`
	statusCheckInterval int64  `json:"statusCheckInterval, omitempty"`
	statusCheckCount    int64  `json:"statusCheckCount, omitempty"`
	imageName           string `json:"imagename, omitempty"`
	port                int    `json:"port, omitempty"`
}

type PipelineVersion struct {
	Id            int64    `json:"id"`
	WorkspaceId   int64    `json:"workspaceId"`
	ProjectId     int64    `json:"projectId"`
	PipelineId    int64    `json:"pipelineId"`
	Namespace     string   `json:"namespace"`
	SelfLink      string   `json:"selfLink" gorm:"type:varchar(255)"`
	Created       int64    `json:"created"`
	Updated       int64    `json:"updated"`
	Labels        string   `json:"labels"`
	Annotations   string   `json:"annotations"`
	Detail        string   `json:"detail" gorm:"type:text"`
	StageVersions []string `json:"stageVersions"`
	Log           string   `json:"log" gorm:"type:text"`
	Status        int64    `json:"state"` // 0 not start    1 working    2 success     3 failed
	MetaData      string   `json:"metadata"`
	Spec          string   `json:"spec"`
}

// unversioned.ReplicationController.ObjectMeta
// unversioned.ReplicationController.Spec

/*
api.ReplicationController{
				ObjectMeta: api.ObjectMeta{
					Name: "foo",
					Labels: map[string]string{
						"foo":  "bar",
						"name": "baz",
					},
				},
*/
/*
func StartK8SResource(pv *PipelineVersion) error {
	rc := &api.ReplicationController{}
	service := &api.Service{}

	var pvm piplineMetadata
	var pvs piplineSpec
	err := split(pv, &pvm, &pvs)
	if err != nil {
		return err
	}

	namespace := convert(pvm, pvs, rc, service)

	if _, err = CLIENT.ReplicationControllers(namespace).Create(rc); err != nil {
		fmt.Errorf("Create rc err : %v\n", err)
		return err
	}

	if _, err := CLIENT.Services(namespace).Create(service); err != nil {
		fmt.Errorf("Create service err : %v\n", err)
		return err
	}
	// writeBack(rcRes, serviceRes, &pvm, &pvs)
	return nil
}
*/

type PiplelineInterface interface {
	GetMetadata() string
	GetSpec() string
}

func StartK8SResource(pipeline PiplelineInterface) error {
	rc := &api.ReplicationController{}
	service := &api.Service{}

	var pvm piplineMetadata
	var pvs piplineSpec
	err := split(pipeline, &pvm, &pvs)
	if err != nil {
		return err
	}

	namespace := convert(pvm, pvs, rc, service)

	if _, err = CLIENT.ReplicationControllers(namespace).Create(rc); err != nil {
		fmt.Errorf("Create rc err : %v\n", err)
		return err
	}

	if _, err := CLIENT.Services(namespace).Create(service); err != nil {
		fmt.Errorf("Create service err : %v\n", err)
		return err
	}
	// writeBack(rcRes, serviceRes, &pvm, &pvs)
	return nil
}

/*
func split(pv *PipelineVersion, pvm *piplineMetadata, pvs *piplineSpec) error {
	err := json.Unmarshal([]byte(pv.MetaData), pvm)
	if err != nil {
		fmt.Errorf("Unmarshal PipelineVersion.ObjectMeta err : %v\n", err)
		return err
	}

	err = json.Unmarshal([]byte(pv.Spec), pvs)
	if err != nil {
		fmt.Errorf("Unmarshal PipelineVersion.Spec err : %v\n", err)
		return err
	}
	return nil
}
*/

func split(pipeline PiplelineInterface, pvm *piplineMetadata, pvs *piplineSpec) error {
	err := json.Unmarshal([]byte(pipeline.GetMetadata()), pvm)
	if err != nil {
		fmt.Errorf("Unmarshal pipeline.GetMetadata() err : %v\n", err)
		return err
	}

	err = json.Unmarshal([]byte(pipeline.GetSpec()), pvs)
	if err != nil {
		fmt.Errorf("Unmarshal pipeline.GetSpec() err : %v\n", err)
		return err
	}
	return nil
}

func convert(piplineMetadata piplineMetadata, piplineSpec piplineSpec,
	rcRes *api.ReplicationController, serviceRes *api.Service) string {
	rcRes.Name = piplineSpec.name
	rcRes.Namespace = piplineMetadata.namespace
	//Use map["rc"] = Spec.name for temprory
	rcRes.Labels["rc"] = piplineSpec.name
	rcRes.Spec.Replicas = piplineSpec.replicas
	rcRes.Spec.Template.Name = piplineSpec.name
	rcRes.Spec.Template.Labels["pod"] = piplineSpec.name
	rcRes.Spec.Template.Namespace = piplineMetadata.namespace
	rcRes.Spec.Template.Spec.Containers[0].Name = piplineSpec.name
	rcRes.Spec.Template.Spec.Containers[0].Image = piplineSpec.imageName
	rcRes.Spec.Template.Spec.Containers[0].Ports[0].Name = piplineSpec.name
	rcRes.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = piplineSpec.port
	rcRes.Spec.Selector["app"] = piplineSpec.name

	serviceRes.ObjectMeta.Name = piplineSpec.name
	serviceRes.ObjectMeta.Namespace = piplineMetadata.namespace
	serviceRes.ObjectMeta.Labels["service"] = piplineSpec.name
	serviceRes.Spec.Ports[0].Port = piplineSpec.port
	serviceRes.Spec.Ports[0].TargetPort = intstr.FromString(piplineSpec.name)
	serviceRes.Spec.Selector["app"] = piplineSpec.name

	return piplineMetadata.namespace
}
<commit_msg>update<commit_after>package kubernetes

import (
	"encoding/json"
	"fmt"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/types"
	"k8s.io/kubernetes/pkg/util/intstr"
)

// pipelineMetadata struct for convert from pipelineVersion.MetaData
type piplineMetadata struct {
	name            string            `json:"name, omitempty"`
	namespace       string            `json:"namespace, omitempty"`
	selfLink        string            `json:"selflink, omitempty"`
	uid             types.UID         `json:"uid, omitempty"`
	createTimestamp unversioned.Time  `json:"createTimestamp, omitempty"`
	deleteTimestamp unversioned.Time  `json:"deleteTimestamp, omitempty"`
	timeoutDuration int64             `json:"timeoutDuration, omitempty"`
	labels          map[string]string `json:"labels, omitempty"`
	annotations     map[string]string `json:"annotations, omitempty"`
}

// pipelineSpec struct for convert from pipelineVersion.Spec
type piplineSpec struct {
	name                string `json:"name, omitempty"`
	replicas            int    `json:"replicas, omitempty"`
	dependencies        string `json:"dependencies, omitempty"`
	kind                string `json:"kind, omitempty"`
	statusCheckLink     string `json:"statusCheckLink, omitempty"`
	statusCheckInterval int64  `json:"statusCheckInterval, omitempty"`
	statusCheckCount    int64  `json:"statusCheckCount, omitempty"`
	imageName           string `json:"imagename, omitempty"`
	port                int    `json:"port, omitempty"`
}

/*
type PipelineVersion struct {
	Id            int64    `json:"id"`
	WorkspaceId   int64    `json:"workspaceId"`
	ProjectId     int64    `json:"projectId"`
	PipelineId    int64    `json:"pipelineId"`
	Namespace     string   `json:"namespace"`
	SelfLink      string   `json:"selfLink" gorm:"type:varchar(255)"`
	Created       int64    `json:"created"`
	Updated       int64    `json:"updated"`
	Labels        string   `json:"labels"`
	Annotations   string   `json:"annotations"`
	Detail        string   `json:"detail" gorm:"type:text"`
	StageVersions []string `json:"stageVersions"`
	Log           string   `json:"log" gorm:"type:text"`
	Status        int64    `json:"state"` // 0 not start    1 working    2 success     3 failed
	MetaData      string   `json:"metadata"`
	Spec          string   `json:"spec"`
}
*/
// unversioned.ReplicationController.ObjectMeta
// unversioned.ReplicationController.Spec

/*
api.ReplicationController{
				ObjectMeta: api.ObjectMeta{
					Name: "foo",
					Labels: map[string]string{
						"foo":  "bar",
						"name": "baz",
					},
				},
*/
/*
func StartK8SResource(pv *PipelineVersion) error {
	rc := &api.ReplicationController{}
	service := &api.Service{}

	var pvm piplineMetadata
	var pvs piplineSpec
	err := split(pv, &pvm, &pvs)
	if err != nil {
		return err
	}

	namespace := convert(pvm, pvs, rc, service)

	if _, err = CLIENT.ReplicationControllers(namespace).Create(rc); err != nil {
		fmt.Errorf("Create rc err : %v\n", err)
		return err
	}

	if _, err := CLIENT.Services(namespace).Create(service); err != nil {
		fmt.Errorf("Create service err : %v\n", err)
		return err
	}
	// writeBack(rcRes, serviceRes, &pvm, &pvs)
	return nil
}
*/

type PiplelineInterface interface {
	GetMetadata() string
	GetSpec() string
}

func StartK8SResource(pipeline PiplelineInterface) error {
	rc := &api.ReplicationController{}
	service := &api.Service{}

	var pvm piplineMetadata
	var pvs piplineSpec
	err := split(pipeline, &pvm, &pvs)
	if err != nil {
		return err
	}

	namespace := convert(pvm, pvs, rc, service)

	if _, err = CLIENT.ReplicationControllers(namespace).Create(rc); err != nil {
		fmt.Errorf("Create rc err : %v\n", err)
		return err
	}

	if _, err := CLIENT.Services(namespace).Create(service); err != nil {
		fmt.Errorf("Create service err : %v\n", err)
		return err
	}
	// writeBack(rcRes, serviceRes, &pvm, &pvs)
	return nil
}

/*
func split(pv *PipelineVersion, pvm *piplineMetadata, pvs *piplineSpec) error {
	err := json.Unmarshal([]byte(pv.MetaData), pvm)
	if err != nil {
		fmt.Errorf("Unmarshal PipelineVersion.ObjectMeta err : %v\n", err)
		return err
	}

	err = json.Unmarshal([]byte(pv.Spec), pvs)
	if err != nil {
		fmt.Errorf("Unmarshal PipelineVersion.Spec err : %v\n", err)
		return err
	}
	return nil
}
*/

func split(pipeline PiplelineInterface, pvm *piplineMetadata, pvs *piplineSpec) error {
	err := json.Unmarshal([]byte(pipeline.GetMetadata()), pvm)
	if err != nil {
		fmt.Errorf("Unmarshal pipeline.GetMetadata() err : %v\n", err)
		return err
	}

	err = json.Unmarshal([]byte(pipeline.GetSpec()), pvs)
	if err != nil {
		fmt.Errorf("Unmarshal pipeline.GetSpec() err : %v\n", err)
		return err
	}
	return nil
}

func convert(piplineMetadata piplineMetadata, piplineSpec piplineSpec,
	rcRes *api.ReplicationController, serviceRes *api.Service) string {
	rcRes.Name = piplineSpec.name
	rcRes.Namespace = piplineMetadata.namespace
	//Use map["rc"] = Spec.name for temprory
	rcRes.Labels["rc"] = piplineSpec.name
	rcRes.Spec.Replicas = piplineSpec.replicas
	rcRes.Spec.Template.Name = piplineSpec.name
	rcRes.Spec.Template.Labels["pod"] = piplineSpec.name
	rcRes.Spec.Template.Namespace = piplineMetadata.namespace
	rcRes.Spec.Template.Spec.Containers[0].Name = piplineSpec.name
	rcRes.Spec.Template.Spec.Containers[0].Image = piplineSpec.imageName
	rcRes.Spec.Template.Spec.Containers[0].Ports[0].Name = piplineSpec.name
	rcRes.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = piplineSpec.port
	rcRes.Spec.Selector["app"] = piplineSpec.name

	serviceRes.ObjectMeta.Name = piplineSpec.name
	serviceRes.ObjectMeta.Namespace = piplineMetadata.namespace
	serviceRes.ObjectMeta.Labels["service"] = piplineSpec.name
	serviceRes.Spec.Ports[0].Port = piplineSpec.port
	serviceRes.Spec.Ports[0].TargetPort = intstr.FromString(piplineSpec.name)
	serviceRes.Spec.Selector["app"] = piplineSpec.name

	return piplineMetadata.namespace
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tp

// Packet types
const (
	TypeUndefined byte = 0
	TypePull      byte = 1
	TypeReply     byte = 2 // reply to pull
	TypePush      byte = 3
)

// TypeText returns the packet type text.
// If the type is undefined returns 'Undefined'.
func TypeText(typ byte) string {
	switch typ {
	case TypePull:
		return "PULL"
	case TypeReply:
		return "REPLY"
	case TypePush:
		return "PUSH"
	default:
		return "Undefined"
	}
}

// Internal Framework Rerror code.
// Note: Recommended custom code is greater than 1000.
const (
	CodeUnknownError   = -1
	CodeDialFailed     = 105
	CodeConnClosed     = 102
	CodeWriteFailed    = 104
	CodeBadPacket      = 400
	CodeNotFound       = 404
	CodeHandleTimeout  = 408
	CodeNotImplemented = 501

	// CodeConflict                      = 409
	// CodeUnsupportedTx                 = 410
	// CodeUnsupportedCodecType          = 415
	// CodeUnauthorized                  = 401
	// CodeInternalServerError           = 500
	// CodeBadGateway                    = 502
	// CodeServiceUnavailable            = 503
	// CodeGatewayTimeout                = 504
	// CodeVariantAlsoNegotiates         = 506
	// CodeInsufficientStorage           = 507
	// CodeLoopDetected                  = 508
	// CodeNotExtended                   = 510
	// CodeNetworkAuthenticationRequired = 511
)

// Internal Framework Rerror string.
var (
	rerrUnknownError       = NewRerror(-1, "Unknown error", "")
	rerrDialFailed         = NewRerror(CodeDialFailed, "Dial Failed", "")
	rerrConnClosed         = NewRerror(CodeConnClosed, "Connection Closed", "")
	rerrWriteFailed        = NewRerror(CodeWriteFailed, "Write Failed", "")
	rerrBadPacket          = NewRerror(CodeBadPacket, "Bad Packet", "")
	rerrNotFound           = NewRerror(CodeNotFound, "Not Found", "")
	rerrHandleTimeout      = NewRerror(CodeHandleTimeout, "Handle Timeout", "")
	rerrCodeNotImplemented = NewRerror(CodeNotImplemented, "Not Implemented", "")
)

// IsConnRerror determines whether the error is a connection error
func IsConnRerror(rerr *Rerror) bool {
	if rerr == nil {
		return false
	}
	if rerr.Code == CodeDialFailed || rerr.Code == CodeConnClosed {
		return true
	}
	return false
}

const (
	// MetaRerrorKey reply error metadata key
	MetaRerrorKey = "X-Reply-Error"
	// MetaRealId real ID metadata key
	MetaRealId = "X-Real-ID"
	// MetaRealIp real IP metadata key
	MetaRealIp = "X-Real-IP"
)
<commit_msg>Add CodeText<commit_after>// Copyright 2015-2017 HenryLee. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tp

// Packet types
const (
	TypeUndefined byte = 0
	TypePull      byte = 1
	TypeReply     byte = 2 // reply to pull
	TypePush      byte = 3
)

// TypeText returns the packet type text.
// If the type is undefined returns 'Undefined'.
func TypeText(typ byte) string {
	switch typ {
	case TypePull:
		return "PULL"
	case TypeReply:
		return "REPLY"
	case TypePush:
		return "PUSH"
	default:
		return "Undefined"
	}
}

// Internal Framework Rerror code.
// Note: Recommended custom code is greater than 1000.
const (
	CodeUnknownError   = -1
	CodeDialFailed     = 105
	CodeConnClosed     = 102
	CodeWriteFailed    = 104
	CodeBadPacket      = 400
	CodeNotFound       = 404
	CodeHandleTimeout  = 408
	CodeNotImplemented = 501

	// CodeConflict                      = 409
	// CodeUnsupportedTx                 = 410
	// CodeUnsupportedCodecType          = 415
	// CodeUnauthorized                  = 401
	// CodeInternalServerError           = 500
	// CodeBadGateway                    = 502
	// CodeServiceUnavailable            = 503
	// CodeGatewayTimeout                = 504
	// CodeVariantAlsoNegotiates         = 506
	// CodeInsufficientStorage           = 507
	// CodeLoopDetected                  = 508
	// CodeNotExtended                   = 510
	// CodeNetworkAuthenticationRequired = 511
)

// CodeText returns the reply error code text.
// If the type is undefined returns 'Unknown Error'.
func CodeText(rerrCode int32) string {
	switch rerrCode {
	case CodeDialFailed:
		return "Dial Failed"
	case CodeConnClosed:
		return "Connection Closed"
	case CodeWriteFailed:
		return "Write Failed"
	case CodeBadPacket:
		return "Bad Packet"
	case CodeNotFound:
		return "Not Found"
	case CodeHandleTimeout:
		return "Handle Timeout"
	case CodeNotImplemented:
		return "Not Implemented"
	case CodeUnknownError:
		fallthrough
	default:
		return "Unknown Error"
	}
}

// Internal Framework Rerror string.
var (
	rerrUnknownError       = NewRerror(CodeUnknownError, CodeText(CodeUnknownError), "")
	rerrDialFailed         = NewRerror(CodeDialFailed, CodeText(CodeDialFailed), "")
	rerrConnClosed         = NewRerror(CodeConnClosed, CodeText(CodeConnClosed), "")
	rerrWriteFailed        = NewRerror(CodeWriteFailed, CodeText(CodeWriteFailed), "")
	rerrBadPacket          = NewRerror(CodeBadPacket, CodeText(CodeBadPacket), "")
	rerrNotFound           = NewRerror(CodeNotFound, CodeText(CodeNotFound), "")
	rerrHandleTimeout      = NewRerror(CodeHandleTimeout, CodeText(CodeHandleTimeout), "")
	rerrCodeNotImplemented = NewRerror(CodeNotImplemented, CodeText(CodeNotImplemented), "")
)

// IsConnRerror determines whether the error is a connection error
func IsConnRerror(rerr *Rerror) bool {
	if rerr == nil {
		return false
	}
	if rerr.Code == CodeDialFailed || rerr.Code == CodeConnClosed {
		return true
	}
	return false
}

const (
	// MetaRerrorKey reply error metadata key
	MetaRerrorKey = "X-Reply-Error"
	// MetaRealId real ID metadata key
	MetaRealId = "X-Real-ID"
	// MetaRealIp real IP metadata key
	MetaRealIp = "X-Real-IP"
)
<|endoftext|>