<commit_before>package upgradeservice

/*
Copyright 2017 - 2020 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"time"

	"github.com/crunchydata/postgres-operator/internal/apiserver"
	"github.com/crunchydata/postgres-operator/internal/config"
	crv1 "github.com/crunchydata/postgres-operator/pkg/apis/crunchydata.com/v1"
	msgs "github.com/crunchydata/postgres-operator/pkg/apiservermsgs"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

// Currently supported version information for upgrades
const (
	REQUIRED_MAJOR_PGO_VERSION = 4
	MAXIMUM_MINOR_PGO_VERSION  = 4
	MINIMUM_MINOR_PGO_VERSION  = 1
)

// CreateUpgrade accepts the CreateUpgradeRequest performs the necessary validation checks and
// organizes the needed upgrade information before creating the required pgtask
// Command format: pgo upgrade mycluster
func CreateUpgrade(request *msgs.CreateUpgradeRequest, ns, pgouser string) msgs.CreateUpgradeResponse {
	response := msgs.CreateUpgradeResponse{}
	response.Status = msgs.Status{Code: msgs.Ok, Msg: ""}
	response.Results = make([]string, 0)

	log.Debugf("createUpgrade called %v", request)

	if request.Selector != "" {
		// use the selector instead of an argument list to filter on

		myselector, err := labels.Parse(request.Selector)
		if err != nil {
			log.Error("could not parse selector flag")
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}
		log.Debugf("myselector is %s", myselector.String())

		// get the clusters list

		clusterList, err := apiserver.Clientset.
			CrunchydataV1().Pgclusters(ns).
			List(metav1.ListOptions{LabelSelector: request.Selector})
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}

		// check that the cluster can be found
		if len(clusterList.Items) == 0 {
			log.Debug("no clusters found")
			response.Status.Msg = "no clusters found"
			return response
		} else {
			newargs := make([]string, 0)
			for _, cluster := range clusterList.Items {
				newargs = append(newargs, cluster.Spec.Name)
			}
			request.Args = newargs
		}
	}

	for _, clusterName := range request.Args {
		log.Debugf("create upgrade called for %s", clusterName)

		// build the pgtask for the upgrade
		spec := crv1.PgtaskSpec{}
		spec.TaskType = crv1.PgtaskUpgrade
		// set the status as created
		spec.Status = crv1.PgtaskUpgradeCreated
		spec.Parameters = make(map[string]string)
		spec.Parameters[config.LABEL_PG_CLUSTER] = clusterName
		spec.Parameters[crv1.PgtaskWorkflowSubmittedStatus] = time.Now().Format(time.RFC3339)

		u, err := ioutil.ReadFile("/proc/sys/kernel/random/uuid")
		if err != nil {
			log.Error(err)
			response.Status.Code = msgs.Error
			response.Status.Msg = fmt.Sprintf("Could not generate UUID for upgrade task. Error: %s", err.Error())
			return response
		}
		spec.Parameters[crv1.PgtaskWorkflowID] = string(u[:len(u)-1])

		if request.UpgradeCCPImageTag != "" {
			// pass the PostGIS CCP Image Tag provided with the upgrade command
			spec.Parameters[config.LABEL_CCP_IMAGE_KEY] = request.UpgradeCCPImageTag
		} else {
			// pass the CCP Image Tag from the apiserver
			spec.Parameters[config.LABEL_CCP_IMAGE_KEY] = apiserver.Pgo.Cluster.CCPImageTag
		}
		// pass the PGO version for the upgrade
		spec.Parameters[config.LABEL_PGO_VERSION] = msgs.PGO_VERSION
		// pass the PGO username for use in the updated CR if missing
		spec.Parameters[config.LABEL_PGOUSER] = pgouser

		spec.Name = clusterName + "-" + config.LABEL_UPGRADE
		spec.Namespace = ns
		labels := make(map[string]string)
		labels[config.LABEL_PG_CLUSTER] = clusterName
		labels[config.LABEL_PGOUSER] = pgouser
		labels[crv1.PgtaskWorkflowID] = spec.Parameters[crv1.PgtaskWorkflowID]

		newInstance := &crv1.Pgtask{
			ObjectMeta: metav1.ObjectMeta{
				Name:   spec.Name,
				Labels: labels,
			},
			Spec: spec,
		}

		// remove any existing pgtask for this upgrade
		task, err := apiserver.Clientset.CrunchydataV1().Pgtasks(ns).Get(spec.Name, metav1.GetOptions{})

		if err == nil && task.Spec.Status != crv1.CompletedStatus {
			response.Status.Code = msgs.Error
			response.Status.Msg = fmt.Sprintf("Could not upgrade cluster: there exists an ongoing upgrade task: [%s]. If you believe this is an error, try deleting this pgtask CR.", task.Spec.Name)
			return response
		}

		// validate the cluster name and ensure autofail is turned off for each cluster.
		cl, err := apiserver.Clientset.CrunchydataV1().Pgclusters(ns).Get(clusterName, metav1.GetOptions{})
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = clusterName + " is not a valid pgcluster"
			return response
		}

		// for the upgrade procedure, we only upgrade to the current image used by the
		// Postgres Operator. As such, we will validate that the Postgres Operator version is
		// is supported by the upgrade, unless the --ignore-validation flag is set.
		if !supportedOperatorVersion(cl.ObjectMeta.Labels[config.LABEL_PGO_VERSION]) && !request.IgnoreValidation {
			response.Status.Code = msgs.Error
			response.Status.Msg = "Cannot upgrade " + clusterName + " from Postgres Operator version " + cl.ObjectMeta.Labels[config.LABEL_PGO_VERSION]
			return response
		}

		// for the upgrade procedure, we only upgrade to the current image used by the
		// Postgres Operator. As such, we will validate that the Postgres Operator's configured
		// image tag (first value) is compatible (i.e. is the same Major PostgreSQL version) as the
		// existing cluster's PG value, unless the --ignore-validation flag is set or the --post-gis-image-tag
		// flag is used
		if !upgradeTagValid(cl.Spec.CCPImageTag, apiserver.Pgo.Cluster.CCPImageTag) && !request.IgnoreValidation && request.UpgradeCCPImageTag != "" {
			log.Debugf("Cannot upgrade from %s to %s. Image must be the same base OS and the upgrade must be within the same major PG version.", cl.Spec.CCPImageTag, apiserver.Pgo.Cluster.CCPImageTag)
			response.Status.Code = msgs.Error
			response.Status.Msg = fmt.Sprintf("cannot upgrade from %s to %s, upgrade task failed.", cl.Spec.CCPImageTag, apiserver.Pgo.Cluster.CCPImageTag)
			return response
		}

		// Create an instance of our CRD
		_, err = apiserver.Clientset.CrunchydataV1().Pgtasks(ns).Create(newInstance)
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			response.WorkflowID = spec.Parameters[crv1.PgtaskWorkflowID]
			return response
		}

		msg := "created upgrade task for " + clusterName
		response.Results = append(response.Results, msg)
		response.WorkflowID = spec.Parameters[crv1.PgtaskWorkflowID]
	}

	return response
}

// supportedOperatorVersion validates the Postgres Operator version
// information for the candidate pgcluster. If this value is in the
// required range, return true so that the upgrade may continue. Otherwise,
// return false.
func supportedOperatorVersion(version string) bool {
	// get the Operator version
	operatorVersionRegex := regexp.MustCompile(`^(\d)\.(\d)\.(\d)`)
	operatorVersion := operatorVersionRegex.FindStringSubmatch(version)

	// if this regex passes, the returned array should always contain
	// 4 values. At 0, the full match, then 1-3 are the three defined groups
	// If this is not true, the upgrade cannot continue (and we won't want to
	// reference potentially missing array items).
	if len(operatorVersion) != 4 {
		return false
	}

	// if the first group does not equal the current major version
	// then the upgrade cannot continue
	if major, err := strconv.Atoi(operatorVersion[1]); err != nil {
		log.Error(err)
		return false
	} else if major != REQUIRED_MAJOR_PGO_VERSION {
		return false
	}

	// if the second group does is not in the supported range,
	// then the upgrade cannot continue
	minor, err := strconv.Atoi(operatorVersion[2])
	if err != nil {
		log.Errorf("Cannot convert Postgres Operator's minor version to an integer. Error: %v", err)
		return false
	}
	if minor < MINIMUM_MINOR_PGO_VERSION || minor > MAXIMUM_MINOR_PGO_VERSION {
		return false
	}

	// If none of the above is true, the upgrade can continue
	return true

}

// upgradeTagValid compares and validates the PostgreSQL version values stored
// in the image tag of the existing pgcluster CR against the values set in the
// Postgres Operator's configuration
func upgradeTagValid(upgradeFrom, upgradeTo string) bool {

	log.Debugf("Validating upgrade from %s to %s", upgradeFrom, upgradeTo)

	versionRegex := regexp.MustCompile(`-(\d+)\.(\d+)(\.\d+)?-`)

	// get the PostgreSQL version values
	upgradeFromValue := versionRegex.FindStringSubmatch(upgradeFrom)
	upgradeToValue := versionRegex.FindStringSubmatch(upgradeTo)

	// if this regex passes, the returned array should always contain
	// 4 values. At 0, the full match, then 1-3 are the three defined groups
	// If this is not true, the upgrade cannot continue (and we won't want to
	// reference potentially missing array items).
	if len(upgradeFromValue) != 4 || len(upgradeToValue) != 4 {
		return false
	}

	// if the first group does not match (PG version 9, 10, 11, 12 etc), or if a value is
	// missing, then the upgrade cannot continue
	if upgradeFromValue[1] != upgradeToValue[1] && upgradeToValue[1] != "" {
		return false
	}

	// if the above check passed, and there is no fourth value, then the PG
	// version has only two digits (e.g. PG 10, 11 or 12), meaning this is a minor upgrade.
	// After validating the second value is at least equal (this is to allow for multiple executions of the
	// upgrade in case an error occurs), the upgrade can continue
	if upgradeFromValue[3] == "" && upgradeToValue[3] == "" && upgradeFromValue[2] <= upgradeToValue[2] {
		return true
	}

	// finally, if the second group matches and is not empty, then, based on the
	// possibilities remaining for Operator container image tags, this is either PG 9.5 or 9.6.
	// if the second group value matches, and the third group was already validated as not
	// empty, check that the third value is at least equal (this is to allow for multiple executions of the
	// upgrade in case an error occurs). If so, the upgrade can continue.
	if upgradeFromValue[2] == upgradeToValue[2] && upgradeToValue[2] != "" && upgradeFromValue[3] <= upgradeToValue[3] {
		return true
	}

	// if none of the above conditions are met, a two digit Major version upgrade is likely being
	// attempted, or a tag value or general error occurred, so we cannot continue
	return false

}
<commit_msg>Automated upgrade minor version bump<commit_after>package upgradeservice

/*
Copyright 2017 - 2020 Crunchy Data Solutions, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import (
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"time"

	"github.com/crunchydata/postgres-operator/internal/apiserver"
	"github.com/crunchydata/postgres-operator/internal/config"
	crv1 "github.com/crunchydata/postgres-operator/pkg/apis/crunchydata.com/v1"
	msgs "github.com/crunchydata/postgres-operator/pkg/apiservermsgs"

	log "github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

// Currently supported version information for upgrades
const (
	REQUIRED_MAJOR_PGO_VERSION = 4
	MAXIMUM_MINOR_PGO_VERSION  = 5
	MINIMUM_MINOR_PGO_VERSION  = 1
)

// CreateUpgrade accepts the CreateUpgradeRequest performs the necessary validation checks and
// organizes the needed upgrade information before creating the required pgtask
// Command format: pgo upgrade mycluster
func CreateUpgrade(request *msgs.CreateUpgradeRequest, ns, pgouser string) msgs.CreateUpgradeResponse {
	response := msgs.CreateUpgradeResponse{}
	response.Status = msgs.Status{Code: msgs.Ok, Msg: ""}
	response.Results = make([]string, 0)

	log.Debugf("createUpgrade called %v", request)

	if request.Selector != "" {
		// use the selector instead of an argument list to filter on

		myselector, err := labels.Parse(request.Selector)
		if err != nil {
			log.Error("could not parse selector flag")
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}
		log.Debugf("myselector is %s", myselector.String())

		// get the clusters list

		clusterList, err := apiserver.Clientset.
			CrunchydataV1().Pgclusters(ns).
			List(metav1.ListOptions{LabelSelector: request.Selector})
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			return response
		}

		// check that the cluster can be found
		if len(clusterList.Items) == 0 {
			log.Debug("no clusters found")
			response.Status.Msg = "no clusters found"
			return response
		} else {
			newargs := make([]string, 0)
			for _, cluster := range clusterList.Items {
				newargs = append(newargs, cluster.Spec.Name)
			}
			request.Args = newargs
		}
	}

	for _, clusterName := range request.Args {
		log.Debugf("create upgrade called for %s", clusterName)

		// build the pgtask for the upgrade
		spec := crv1.PgtaskSpec{}
		spec.TaskType = crv1.PgtaskUpgrade
		// set the status as created
		spec.Status = crv1.PgtaskUpgradeCreated
		spec.Parameters = make(map[string]string)
		spec.Parameters[config.LABEL_PG_CLUSTER] = clusterName
		spec.Parameters[crv1.PgtaskWorkflowSubmittedStatus] = time.Now().Format(time.RFC3339)

		u, err := ioutil.ReadFile("/proc/sys/kernel/random/uuid")
		if err != nil {
			log.Error(err)
			response.Status.Code = msgs.Error
			response.Status.Msg = fmt.Sprintf("Could not generate UUID for upgrade task. Error: %s", err.Error())
			return response
		}
		spec.Parameters[crv1.PgtaskWorkflowID] = string(u[:len(u)-1])

		if request.UpgradeCCPImageTag != "" {
			// pass the PostGIS CCP Image Tag provided with the upgrade command
			spec.Parameters[config.LABEL_CCP_IMAGE_KEY] = request.UpgradeCCPImageTag
		} else {
			// pass the CCP Image Tag from the apiserver
			spec.Parameters[config.LABEL_CCP_IMAGE_KEY] = apiserver.Pgo.Cluster.CCPImageTag
		}
		// pass the PGO version for the upgrade
		spec.Parameters[config.LABEL_PGO_VERSION] = msgs.PGO_VERSION
		// pass the PGO username for use in the updated CR if missing
		spec.Parameters[config.LABEL_PGOUSER] = pgouser

		spec.Name = clusterName + "-" + config.LABEL_UPGRADE
		spec.Namespace = ns
		labels := make(map[string]string)
		labels[config.LABEL_PG_CLUSTER] = clusterName
		labels[config.LABEL_PGOUSER] = pgouser
		labels[crv1.PgtaskWorkflowID] = spec.Parameters[crv1.PgtaskWorkflowID]

		newInstance := &crv1.Pgtask{
			ObjectMeta: metav1.ObjectMeta{
				Name:   spec.Name,
				Labels: labels,
			},
			Spec: spec,
		}

		// remove any existing pgtask for this upgrade
		task, err := apiserver.Clientset.CrunchydataV1().Pgtasks(ns).Get(spec.Name, metav1.GetOptions{})

		if err == nil && task.Spec.Status != crv1.CompletedStatus {
			response.Status.Code = msgs.Error
			response.Status.Msg = fmt.Sprintf("Could not upgrade cluster: there exists an ongoing upgrade task: [%s]. If you believe this is an error, try deleting this pgtask CR.", task.Spec.Name)
			return response
		}

		// validate the cluster name and ensure autofail is turned off for each cluster.
		cl, err := apiserver.Clientset.CrunchydataV1().Pgclusters(ns).Get(clusterName, metav1.GetOptions{})
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = clusterName + " is not a valid pgcluster"
			return response
		}

		// for the upgrade procedure, we only upgrade to the current image used by the
		// Postgres Operator. As such, we will validate that the Postgres Operator version is
		// is supported by the upgrade, unless the --ignore-validation flag is set.
		if !supportedOperatorVersion(cl.ObjectMeta.Labels[config.LABEL_PGO_VERSION]) && !request.IgnoreValidation {
			response.Status.Code = msgs.Error
			response.Status.Msg = "Cannot upgrade " + clusterName + " from Postgres Operator version " + cl.ObjectMeta.Labels[config.LABEL_PGO_VERSION]
			return response
		}

		// for the upgrade procedure, we only upgrade to the current image used by the
		// Postgres Operator. As such, we will validate that the Postgres Operator's configured
		// image tag (first value) is compatible (i.e. is the same Major PostgreSQL version) as the
		// existing cluster's PG value, unless the --ignore-validation flag is set or the --post-gis-image-tag
		// flag is used
		if !upgradeTagValid(cl.Spec.CCPImageTag, apiserver.Pgo.Cluster.CCPImageTag) && !request.IgnoreValidation && request.UpgradeCCPImageTag != "" {
			log.Debugf("Cannot upgrade from %s to %s. Image must be the same base OS and the upgrade must be within the same major PG version.", cl.Spec.CCPImageTag, apiserver.Pgo.Cluster.CCPImageTag)
			response.Status.Code = msgs.Error
			response.Status.Msg = fmt.Sprintf("cannot upgrade from %s to %s, upgrade task failed.", cl.Spec.CCPImageTag, apiserver.Pgo.Cluster.CCPImageTag)
			return response
		}

		// Create an instance of our CRD
		_, err = apiserver.Clientset.CrunchydataV1().Pgtasks(ns).Create(newInstance)
		if err != nil {
			response.Status.Code = msgs.Error
			response.Status.Msg = err.Error()
			response.WorkflowID = spec.Parameters[crv1.PgtaskWorkflowID]
			return response
		}

		msg := "created upgrade task for " + clusterName
		response.Results = append(response.Results, msg)
		response.WorkflowID = spec.Parameters[crv1.PgtaskWorkflowID]
	}

	return response
}

// supportedOperatorVersion validates the Postgres Operator version
// information for the candidate pgcluster. If this value is in the
// required range, return true so that the upgrade may continue. Otherwise,
// return false.
func supportedOperatorVersion(version string) bool {
	// get the Operator version
	operatorVersionRegex := regexp.MustCompile(`^(\d)\.(\d)\.(\d)`)
	operatorVersion := operatorVersionRegex.FindStringSubmatch(version)

	// if this regex passes, the returned array should always contain
	// 4 values. At 0, the full match, then 1-3 are the three defined groups
	// If this is not true, the upgrade cannot continue (and we won't want to
	// reference potentially missing array items).
	if len(operatorVersion) != 4 {
		return false
	}

	// if the first group does not equal the current major version
	// then the upgrade cannot continue
	if major, err := strconv.Atoi(operatorVersion[1]); err != nil {
		log.Error(err)
		return false
	} else if major != REQUIRED_MAJOR_PGO_VERSION {
		return false
	}

	// if the second group does is not in the supported range,
	// then the upgrade cannot continue
	minor, err := strconv.Atoi(operatorVersion[2])
	if err != nil {
		log.Errorf("Cannot convert Postgres Operator's minor version to an integer. Error: %v", err)
		return false
	}
	if minor < MINIMUM_MINOR_PGO_VERSION || minor > MAXIMUM_MINOR_PGO_VERSION {
		return false
	}

	// If none of the above is true, the upgrade can continue
	return true

}

// upgradeTagValid compares and validates the PostgreSQL version values stored
// in the image tag of the existing pgcluster CR against the values set in the
// Postgres Operator's configuration
func upgradeTagValid(upgradeFrom, upgradeTo string) bool {

	log.Debugf("Validating upgrade from %s to %s", upgradeFrom, upgradeTo)

	versionRegex := regexp.MustCompile(`-(\d+)\.(\d+)(\.\d+)?-`)

	// get the PostgreSQL version values
	upgradeFromValue := versionRegex.FindStringSubmatch(upgradeFrom)
	upgradeToValue := versionRegex.FindStringSubmatch(upgradeTo)

	// if this regex passes, the returned array should always contain
	// 4 values. At 0, the full match, then 1-3 are the three defined groups
	// If this is not true, the upgrade cannot continue (and we won't want to
	// reference potentially missing array items).
	if len(upgradeFromValue) != 4 || len(upgradeToValue) != 4 {
		return false
	}

	// if the first group does not match (PG version 9, 10, 11, 12 etc), or if a value is
	// missing, then the upgrade cannot continue
	if upgradeFromValue[1] != upgradeToValue[1] && upgradeToValue[1] != "" {
		return false
	}

	// if the above check passed, and there is no fourth value, then the PG
	// version has only two digits (e.g. PG 10, 11 or 12), meaning this is a minor upgrade.
	// After validating the second value is at least equal (this is to allow for multiple executions of the
	// upgrade in case an error occurs), the upgrade can continue
	if upgradeFromValue[3] == "" && upgradeToValue[3] == "" && upgradeFromValue[2] <= upgradeToValue[2] {
		return true
	}

	// finally, if the second group matches and is not empty, then, based on the
	// possibilities remaining for Operator container image tags, this is either PG 9.5 or 9.6.
	// if the second group value matches, and the third group was already validated as not
	// empty, check that the third value is at least equal (this is to allow for multiple executions of the
	// upgrade in case an error occurs). If so, the upgrade can continue.
	if upgradeFromValue[2] == upgradeToValue[2] && upgradeToValue[2] != "" && upgradeFromValue[3] <= upgradeToValue[3] {
		return true
	}

	// if none of the above conditions are met, a two digit Major version upgrade is likely being
	// attempted, or a tag value or general error occurred, so we cannot continue
	return false

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	ba "github.com/CSUNetSec/bgparchive"
	api "github.com/CSUNetSec/bgparchive/api"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	flag_refresh_minutes int
	flag_descpaths       descpaths
	flag_basepath        string
	flag_savepath        string
	flag_debug           bool
	flag_conffile        string
)

type descpath struct {
	Desc          string
	Path          string
	Delta_minutes int
	Basepath      string
	Collector     string
}

type descpaths []descpath

func (d *descpaths) String() string {
	var ret []string
	for _, dp := range *d {
		ret = append(ret, fmt.Sprintf("[Desc:%s->path:%s delta:%d basepath:%s collector:%s] ", dp.Desc, dp.Path, dp.Delta_minutes, dp.Basepath, dp.Collector))
	}
	return strings.Join(ret, "")
}

func (d *descpaths) Set(val string) error {
	strs := strings.Split(val, ",")
	for _, str := range strs {
		set := strings.Split(str, ":")
		if len(set) != 5 {
			return errors.New("syntax: fspath2:descriminator1:path1:delta_minutes1:collector1, fspath2:descriminator2:path2:delta_minutes2:collector2, ...")
		}
		dm, dmerr := strconv.Atoi(set[3])
		if dmerr != nil {
			return dmerr
		}
		*d = append(*d, descpath{Basepath: set[0], Desc: set[1], Path: set[2], Delta_minutes: dm, Collector: set[4]})
	}
	return nil
}

func init() {
	flag.IntVar(&flag_refresh_minutes, "refresh-minutes", 5, "rescan db every x minutes")
	flag.Var(&flag_descpaths, "descriminator-paths", "comma seperated list of fsbasepath:descriminator:urlpath:delta_minutes:collectorname quints")
	flag.StringVar(&flag_savepath, "savepath", ".", "directory to save the binary archive index files")
	flag.StringVar(&flag_conffile, "conf", "", "configuration file")
	flag.BoolVar(&flag_debug, "debug", false, "turn on debugging")
}

func main() {
	flag.Parse()
	if flag_conffile != "" { //the configuration file will overwrite any config from the command line
		file, err := os.Open(flag_conffile)
		if err != nil {
			log.Fatal(err)
		}
		decoder := json.NewDecoder(file)
		err = decoder.Decode(&flag_descpaths)
		if err != nil {
			log.Fatal(err)
		}
		file.Close()
	}
	var ars ba.MrtArchives
	if len(flag_descpaths) == 0 {
		log.Fatal("not descriminators and paths specified")
	}

	api := api.NewAPI()
	servewg := &sync.WaitGroup{}
	allscanwg := &sync.WaitGroup{}
	hmsg := new(ba.HelpMsg)
	for i, v := range flag_descpaths {
		ars = append(ars, ba.NewMRTArchive(v.Basepath, v.Desc, v.Collector, flag_refresh_minutes, flag_savepath, flag_debug))
		ars[i].SetTimeDelta(time.Duration(v.Delta_minutes) * time.Minute)
		statar := ba.NewFsarstat(ars[i].GetFsArchive())
		fsc := ba.NewFsarconf(ars[i].GetFsArchive())
		pbar := ba.NewPbArchive(ars[i].GetFsArchive())
		jsar := ba.NewJsonArchive(ars[i].GetFsArchive())
		api.AddResource(ars[i], fmt.Sprintf("/archive/mrt/%s%s", v.Collector, v.Path))
		api.AddResource(pbar, fmt.Sprintf("/archive/pb/%s%s", v.Collector, v.Path))
		api.AddResource(jsar, fmt.Sprintf("/archive/json/%s%s", v.Collector, v.Path))
		api.AddResource(fsc, fmt.Sprintf("/archive/mrt/%s%s/conf", v.Collector, v.Path))
		api.AddResource(statar, fmt.Sprintf("/archive/mrt/%s%s/stats", v.Collector, v.Path))
		mrtreqc := ars[i].Serve(servewg, allscanwg)
		errg := ars[i].Load(fmt.Sprintf("%s/%s-%s", flag_savepath, v.Desc, v.Collector))
		if errg != nil {
			log.Printf("failed to find serialized file. Scanning")
			mrtreqc <- "SCAN"
			//log.Printf("Entryfiles are:%s", ars[i].tempentryfiles)
			allscanwg.Wait()
			errg = ars[i].Save(fmt.Sprintf("%s/%s-%s", flag_savepath, v.Desc, v.Collector))
			if errg != nil {
				log.Println(errg)
			} else {
				log.Printf("created serialized file for archive:%v", v)
			}
		} else {
			//log.Printf("Found serialized file for archive:%s. entryfiles:%s", v, ars[i].entryfiles)
			log.Printf("Found serialized file for archive:%v.", v)
			ars[i].SetEntryFilesToTemp()
		}
		hmsg.AddArchive(fsc)
	}
	allscanwg.Wait()
	//the global help message
	api.AddResource(hmsg, "/archive/help")
	api.Start(80)
	for _, v := range ars {
		rc := v.GetReqChan()
		close(rc)
	}
	servewg.Wait()
	log.Print("all fsarchives stopped. exiting")
}
<commit_msg>implementing port flag for the server to bind to<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	ba "github.com/CSUNetSec/bgparchive"
	api "github.com/CSUNetSec/bgparchive/api"
	"log"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	flag_refresh_minutes int
	flag_descpaths       descpaths
	flag_basepath        string
	flag_savepath        string
	flag_debug           bool
	flag_conffile        string
	flag_port            int
)

type descpath struct {
	Desc          string
	Path          string
	Delta_minutes int
	Basepath      string
	Collector     string
}

type descpaths []descpath

func (d *descpaths) String() string {
	var ret []string
	for _, dp := range *d {
		ret = append(ret, fmt.Sprintf("[Desc:%s->path:%s delta:%d basepath:%s collector:%s] ", dp.Desc, dp.Path, dp.Delta_minutes, dp.Basepath, dp.Collector))
	}
	return strings.Join(ret, "")
}

func (d *descpaths) Set(val string) error {
	strs := strings.Split(val, ",")
	for _, str := range strs {
		set := strings.Split(str, ":")
		if len(set) != 5 {
			return errors.New("syntax: fspath2:descriminator1:path1:delta_minutes1:collector1, fspath2:descriminator2:path2:delta_minutes2:collector2, ...")
		}
		dm, dmerr := strconv.Atoi(set[3])
		if dmerr != nil {
			return dmerr
		}
		*d = append(*d, descpath{Basepath: set[0], Desc: set[1], Path: set[2], Delta_minutes: dm, Collector: set[4]})
	}
	return nil
}

func init() {
	flag.IntVar(&flag_refresh_minutes, "refresh-minutes", 5, "rescan db every x minutes")
	flag.Var(&flag_descpaths, "descriminator-paths", "comma seperated list of fsbasepath:descriminator:urlpath:delta_minutes:collectorname quints")
	flag.StringVar(&flag_savepath, "savepath", ".", "directory to save the binary archive index files")
	flag.StringVar(&flag_conffile, "conf", "", "configuration file")
	flag.BoolVar(&flag_debug, "debug", false, "turn on debugging")
	flag.IntVar(&flag_port, "port", 80, "default port for the HTTP server to bind to")
}

func main() {
	flag.Parse()
	if flag_conffile != "" { //the configuration file will overwrite any config from the command line
		file, err := os.Open(flag_conffile)
		if err != nil {
			log.Fatal(err)
		}
		decoder := json.NewDecoder(file)
		err = decoder.Decode(&flag_descpaths)
		if err != nil {
			log.Fatal(err)
		}
		file.Close()
	}
	var ars ba.MrtArchives
	if len(flag_descpaths) == 0 {
		log.Fatal("not descriminators and paths specified")
	}

	api := api.NewAPI()
	servewg := &sync.WaitGroup{}
	allscanwg := &sync.WaitGroup{}
	hmsg := new(ba.HelpMsg)
	for i, v := range flag_descpaths {
		ars = append(ars, ba.NewMRTArchive(v.Basepath, v.Desc, v.Collector, flag_refresh_minutes, flag_savepath, flag_debug))
		ars[i].SetTimeDelta(time.Duration(v.Delta_minutes) * time.Minute)
		statar := ba.NewFsarstat(ars[i].GetFsArchive())
		fsc := ba.NewFsarconf(ars[i].GetFsArchive())
		pbar := ba.NewPbArchive(ars[i].GetFsArchive())
		jsar := ba.NewJsonArchive(ars[i].GetFsArchive())
		api.AddResource(ars[i], fmt.Sprintf("/archive/mrt/%s%s", v.Collector, v.Path))
		api.AddResource(pbar, fmt.Sprintf("/archive/pb/%s%s", v.Collector, v.Path))
		api.AddResource(jsar, fmt.Sprintf("/archive/json/%s%s", v.Collector, v.Path))
		api.AddResource(fsc, fmt.Sprintf("/archive/mrt/%s%s/conf", v.Collector, v.Path))
		api.AddResource(statar, fmt.Sprintf("/archive/mrt/%s%s/stats", v.Collector, v.Path))
		mrtreqc := ars[i].Serve(servewg, allscanwg)
		errg := ars[i].Load(fmt.Sprintf("%s/%s-%s", flag_savepath, v.Desc, v.Collector))
		if errg != nil {
			log.Printf("failed to find serialized file. Scanning")
			mrtreqc <- "SCAN"
			//log.Printf("Entryfiles are:%s", ars[i].tempentryfiles)
			allscanwg.Wait()
			errg = ars[i].Save(fmt.Sprintf("%s/%s-%s", flag_savepath, v.Desc, v.Collector))
			if errg != nil {
				log.Println(errg)
			} else {
				log.Printf("created serialized file for archive:%v", v)
			}
		} else {
			//log.Printf("Found serialized file for archive:%s. entryfiles:%s", v, ars[i].entryfiles)
			log.Printf("Found serialized file for archive:%v.", v)
			ars[i].SetEntryFilesToTemp()
		}
		hmsg.AddArchive(fsc)
	}
	allscanwg.Wait()
	//the global help message
	api.AddResource(hmsg, "/archive/help")
	api.Start(flag_port)
	for _, v := range ars {
		rc := v.GetReqChan()
		close(rc)
	}
	servewg.Wait()
	log.Print("all fsarchives stopped. exiting")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sync"

	"github.com/bentranter/password"
	"golang.org/x/net/context"
)

var (
	db = newInMemDB()
)

type inMemDB struct {
	rwm *sync.RWMutex
	m   map[int]string
}

func newInMemDB() *inMemDB {
	return &inMemDB{
		rwm: &sync.RWMutex{},
		m:   make(map[int]string),
	}
}

func (db *inMemDB) Store(hashedPassword string) (string, error) {
	db.rwm.Lock()
	defer db.rwm.Unlock()
	db.m[1] = hashedPassword
	return "1", nil
}

func (db *inMemDB) Retrieve(id string) (string, error) {
	db.rwm.RLock()
	defer db.rwm.RUnlock()
	return db.m[1], nil
}

type user struct {
	username string `json:"username"`
	password string `json:"password"`
}

func createUser(w http.ResponseWriter, r *http.Request) {
	var u user
	json.NewDecoder(r.Body).Decode(&u)
	fmt.Printf("Fucking user: %+v\n", u)

	id, err := password.New(u.password, db)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	w.WriteHeader(http.StatusCreated)
	fmt.Fprintf(w, "{\n\tid: %s\n}\n", id)
}

func comparePwd(w http.ResponseWriter, r *http.Request) {
	var u user
	json.NewDecoder(r.Body).Decode(&u)
	fmt.Printf("Fucking user: %+v\n", u)

	password.Authenticate(u.username, u.password, w, db)
}

func authReq(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	user := ctx.Value("id")
	w.Write([]byte("User: " + user.(string)))
}

func main() {
	mux := http.NewServeMux()

	mux.HandleFunc("/", createUser)
	mux.HandleFunc("/auth", comparePwd)
	mux.Handle("/user", password.Protected(authReq))

	http.ListenAndServe(":3000", mux)
}
<commit_msg>Remove curse words<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"sync"

	"github.com/bentranter/password"
	"golang.org/x/net/context"
)

var (
	db = newInMemDB()
)

type inMemDB struct {
	rwm *sync.RWMutex
	m   map[int]string
}

func newInMemDB() *inMemDB {
	return &inMemDB{
		rwm: &sync.RWMutex{},
		m:   make(map[int]string),
	}
}

func (db *inMemDB) Store(hashedPassword string) (string, error) {
	db.rwm.Lock()
	defer db.rwm.Unlock()
	db.m[1] = hashedPassword
	return "1", nil
}

func (db *inMemDB) Retrieve(id string) (string, error) {
	db.rwm.RLock()
	defer db.rwm.RUnlock()
	return db.m[1], nil
}

type user struct {
	username string `json:"username"`
	password string `json:"password"`
}

func createUser(w http.ResponseWriter, r *http.Request) {
	var u user
	json.NewDecoder(r.Body).Decode(&u)
	fmt.Printf("User: %+v\n", u)

	id, err := password.New(u.password, db)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	w.WriteHeader(http.StatusCreated)
	fmt.Fprintf(w, "{\n\tid: %s\n}\n", id)
}

func comparePwd(w http.ResponseWriter, r *http.Request) {
	var u user
	json.NewDecoder(r.Body).Decode(&u)
	fmt.Printf("User: %+v\n", u)

	password.Authenticate(u.username, u.password, w, db)
}

func authReq(ctx context.Context, w http.ResponseWriter, r *http.Request) {
	user := ctx.Value("id")
	w.Write([]byte("User: " + user.(string)))
}

func main() {
	mux := http.NewServeMux()

	mux.HandleFunc("/", createUser)
	mux.HandleFunc("/auth", comparePwd)
	mux.Handle("/user", password.Protected(authReq))

	http.ListenAndServe(":3000", mux)
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"
	"path"

	"github.com/Masterminds/cookoo"
	"github.com/kylelemons/go-gypsy/yaml"
)

// Flatten recurses through all dependent packages and flattens to a top level.
//
// Flattening involves determining a tree's dependencies and flattening them
// into a single large list.
//
// Params:
//	- packages ([]string): The packages to read. If this is empty, it reads all
//		packages.
//	- force (bool): force git updates.
//	- conf (*Config): The configuration.
//
// Returns:
//
func Flatten(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
	conf := p.Get("conf", &Config{}).(*Config)
	skip := p.Get("skip", false).(bool)
	if skip {
		return conf, nil
	}
	packages := p.Get("packages", []string{}).([]string)
	force := p.Get("force", true).(bool)
	vend, _ := VendorPath(c)

	// If no packages are supplied, we do them all.
	if len(packages) == 0 {
		packages = make([]string, len(conf.Imports))
		for i, v := range conf.Imports {
			packages[i] = v.Name
		}
	}

	// Build an initial dependency map.
	deps := make(map[string]*Dependency, len(conf.Imports))
	for _, imp := range conf.Imports {
		deps[imp.Name] = imp
	}

	f := &flattening{conf, vend, vend, deps, packages}

	err := recFlatten(f, force)
	flattenSetRefs(f)
	Info("Project relies on %d dependencies.", len(deps))
	exportFlattenedDeps(conf, deps)

	return conf, err
}

func exportFlattenedDeps(conf *Config, in map[string]*Dependency) {
	out := make([]*Dependency, len(in))
	i := 0
	for _, v := range in {
		out[i] = v
		i++
	}
	conf.Imports = out
}

type flattening struct {
	conf *Config
	// Top vendor path, e.g. project/vendor
	top string
	// Current path
	curr string
	// Built list of dependencies
	deps map[string]*Dependency
	// Dependencies that need to be scanned.
	scan []string
}

// Hack: Cache record of updates so we don't have to keep doing git pulls.
var updateCache = map[string]bool{}

// refFlatten recursively flattens the vendor tree.
func recFlatten(f *flattening, force bool) error {
	Debug("---> Inspecting %s for changes (%d packages).\n", f.curr, len(f.scan))
	for _, imp := range f.scan {
		Debug("----> Scanning %s", imp)
		base := path.Join(f.top, imp)
		mod := []string{}
		if m, ok := mergeGlide(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGodep(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGPM(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGb(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGuess(base, imp, f.deps); ok {
			mod = m
		}

		if len(mod) > 0 {
			Debug("----> Updating all dependencies for %q (%d)", imp, len(mod))
			flattenGlideUp(f, base, force)
			f2 := &flattening{
				conf: f.conf,
				top:  f.top,
				curr: base,
				deps: f.deps,
				scan: mod}
			recFlatten(f2, force)
		}
	}

	return nil
}

// flattenGlideUp does a glide update in the middle of a flatten operation.
//
// While this is expensive, it is also necessary to make sure we have the
// correct version of all dependencies. We might be able to simplify by
// marking packages dirty when they are added.
func flattenGlideUp(f *flattening, base string, force bool) error {
	//vdir := path.Join(base, "vendor")
	for _, imp := range f.deps {
		wd := path.Join(f.top, imp.Name)
		if VcsExists(imp, wd) {
			if updateCache[imp.Name] {
				Debug("----> Already updated %s", imp.Name)
				continue
			}
			Debug("Updating project %s (%s)\n", imp.Name, wd)
			if err := VcsUpdate(imp, f.top, force); err != nil {
				// We can still go on just fine even if this fails.
				Warn("Skipped update %s: %s\n", imp.Name, err)
				continue
			}
			updateCache[imp.Name] = true
		} else {
			Debug("Importing %s to project %s\n", imp.Name, wd)
			if err := VcsGet(imp, wd); err != nil {
				Warn("Skipped getting %s: %v\n", imp.Name, err)
				continue
			}
		}

		// If a revision has been set use it.
		err := VcsVersion(imp, f.top)
		if err != nil {
			Warn("Problem setting version on %s: %s\n", imp.Name, err)
		}
	}

	return nil
}

// Set the references for all packages after a flatten is completed.
func flattenSetRefs(f *flattening) {
	Debug("Setting final version for %d dependencies.", len(f.deps))
	for _, imp := range f.deps {
		if err := VcsVersion(imp, f.top); err != nil {
			Warn("Problem setting version on %s: %s (flatten)\n", imp.Name, err)
		}
	}
}

func mergeGlide(dir, name string, deps map[string]*Dependency) ([]string, bool) {
	gp := path.Join(dir, "glide.yaml")
	if _, err := os.Stat(gp); err != nil {
		return []string{}, false
	}
	f, err := yaml.ReadFile(gp)
	if err != nil {
		Warn("Found glide file %q, but can't parse: %s", gp, err)
		return []string{}, false
	}

	conf, err := FromYaml(f.Root)
	if err != nil {
		Warn("Found glide file %q, but can't use it: %s", gp, err)
		return []string{}, false
	}

	Info("Found glide.yaml in %s", gp)

	return mergeDeps(deps, conf.Imports), true
}

// listGodep appends Godeps entries to the deps.
//
// It returns true if any dependencies were found (even if not added because
// they are duplicates).
func mergeGodep(dir, name string, deps map[string]*Dependency) ([]string, bool) {
	Debug("Looking in %s/Godeps/ for a Godeps.json file.\n", dir)
	d, err := parseGodepGodeps(dir)
	if err != nil {
		Warn("Looking for Godeps: %s\n", err)
		return []string{}, false
	} else if len(d) == 0 {
		return []string{}, false
	}

	Info("Found Godeps.json file for %q", name)
	return mergeDeps(deps, d), true
}

// listGb merges GB dependencies into the deps.
func mergeGb(dir, pkg string, deps map[string]*Dependency) ([]string, bool) {
	Debug("Looking in %s/vendor/ for a manifest file.\n", dir)
	d, err := parseGbManifest(dir)
	if err != nil || len(d) == 0 {
		return []string{}, false
	}
	Info("Found gb manifest file for %q", pkg)
	return mergeDeps(deps, d), true
}

// mergeGPM merges GPM Godeps files into deps.
func mergeGPM(dir, pkg string, deps map[string]*Dependency) ([]string, bool) {
	d, err := parseGPMGodeps(dir)
	if err != nil || len(d) == 0 {
		return []string{}, false
	}
	Info("Found GPM file for %q", pkg)
	return mergeDeps(deps, d), true
}

// mergeGuess guesses dependencies and merges.
//
// This always returns true because it always handles the job of searching
// for dependencies. So generally it should be the last merge strategy
// that you try.
func mergeGuess(dir, pkg string, deps map[string]*Dependency) ([]string, bool) {
	Info("Scanning %s for dependencies.", pkg)
	buildContext, err := GetBuildContext()
	if err != nil {
		Warn("Could not scan package %q: %s", pkg, err)
		return []string{}, false
	}

	res := []string{}

	if _, err := os.Stat(dir); err != nil {
		Warn("Directory is missing: %s", dir)
		return res, true
	}

	d := walkDeps(buildContext, dir, pkg)
	for _, name := range d {
		name, _ := NormalizeName(name)
		repo := getRepoRootFromPackage(name)
		if _, ok := deps[name]; ok {
			Debug("====> Seen %s already. Skipping", name)
			continue
		}

		found := findPkg(buildContext, name, dir)
		switch found.PType {
		case ptypeUnknown:
			Debug("✨☆ Undownloaded dependency: %s", name)
			nd := &Dependency{
				Name:       name,
				Repository: "https://" + repo,
			}
			deps[name] = nd
			res = append(res, name)
		case ptypeGoroot, ptypeCgo:
			break
		default:
			// We're looking for dependencies that might exist in $GOPATH
			// but not be on vendor. We add any that are on $GOPATH.
			if _, ok := deps[name]; !ok {
				Debug("✨☆ GOPATH dependency: %s", name)
				nd := &Dependency{Name: name}
				deps[name] = nd
				res = append(res, name)
			}
		}
	}

	return res, true
}

// mergeDeps merges any dependency array into deps.
func mergeDeps(orig map[string]*Dependency, add []*Dependency) []string {
	mod := []string{}
	for _, dd := range add {
		// Add it unless it's already there.
		if existing, ok := orig[dd.Name]; !ok {
			orig[dd.Name] = dd
			Debug("Adding %s to the scan list", dd.Name)
			mod = append(mod, dd.Name)
		} else if existing.Reference == "" && dd.Reference != "" {
			// If a nested dep has finer dependency references than outside,
			// set the reference.
			existing.Reference = dd.Reference
			mod = append(mod, dd.Name)
		} else if dd.Reference != "" && existing.Reference != "" && dd.Reference != existing.Reference {
			// We can detect version conflicts, but we can't really do
			// anything to correct, since we don't know the intentions of the
			// authors.
			Warn("Conflict: %s ref is %s, but also asked for %s", existing.Name, existing.Reference, dd.Reference)
			Info("Keeping %s %s", existing.Name, existing.Reference)
		}
	}
	return mod
}
<commit_msg>Stopped glide source code scanning to test #113.<commit_after>package cmd

import (
	"os"
	"path"

	"github.com/Masterminds/cookoo"
	"github.com/kylelemons/go-gypsy/yaml"
)

// Flatten recurses through all dependent packages and flattens to a top level.
//
// Flattening involves determining a tree's dependencies and flattening them
// into a single large list.
//
// Params:
//	- packages ([]string): The packages to read. If this is empty, it reads all
//		packages.
//	- force (bool): force git updates.
//	- conf (*Config): The configuration.
//
// Returns:
//
func Flatten(c cookoo.Context, p *cookoo.Params) (interface{}, cookoo.Interrupt) {
	conf := p.Get("conf", &Config{}).(*Config)
	skip := p.Get("skip", false).(bool)
	if skip {
		return conf, nil
	}
	packages := p.Get("packages", []string{}).([]string)
	force := p.Get("force", true).(bool)
	vend, _ := VendorPath(c)

	// If no packages are supplied, we do them all.
	if len(packages) == 0 {
		packages = make([]string, len(conf.Imports))
		for i, v := range conf.Imports {
			packages[i] = v.Name
		}
	}

	// Build an initial dependency map.
	deps := make(map[string]*Dependency, len(conf.Imports))
	for _, imp := range conf.Imports {
		deps[imp.Name] = imp
	}

	f := &flattening{conf, vend, vend, deps, packages}

	err := recFlatten(f, force)
	flattenSetRefs(f)
	Info("Project relies on %d dependencies.", len(deps))
	exportFlattenedDeps(conf, deps)

	return conf, err
}

func exportFlattenedDeps(conf *Config, in map[string]*Dependency) {
	out := make([]*Dependency, len(in))
	i := 0
	for _, v := range in {
		out[i] = v
		i++
	}
	conf.Imports = out
}

type flattening struct {
	conf *Config
	// Top vendor path, e.g. project/vendor
	top string
	// Current path
	curr string
	// Built list of dependencies
	deps map[string]*Dependency
	// Dependencies that need to be scanned.
	scan []string
}

// Hack: Cache record of updates so we don't have to keep doing git pulls.
var updateCache = map[string]bool{}

// refFlatten recursively flattens the vendor tree.
func recFlatten(f *flattening, force bool) error {
	Debug("---> Inspecting %s for changes (%d packages).\n", f.curr, len(f.scan))
	for _, imp := range f.scan {
		Debug("----> Scanning %s", imp)
		base := path.Join(f.top, imp)
		mod := []string{}
		if m, ok := mergeGlide(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGodep(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGPM(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGb(base, imp, f.deps); ok {
			mod = m
		} else if m, ok = mergeGuess(base, imp, f.deps); ok {
			mod = m
		}

		if len(mod) > 0 {
			Debug("----> Updating all dependencies for %q (%d)", imp, len(mod))
			flattenGlideUp(f, base, force)
			f2 := &flattening{
				conf: f.conf,
				top:  f.top,
				curr: base,
				deps: f.deps,
				scan: mod}
			recFlatten(f2, force)
		}
	}

	return nil
}

// flattenGlideUp does a glide update in the middle of a flatten operation.
//
// While this is expensive, it is also necessary to make sure we have the
// correct version of all dependencies. We might be able to simplify by
// marking packages dirty when they are added.
func flattenGlideUp(f *flattening, base string, force bool) error {
	//vdir := path.Join(base, "vendor")
	for _, imp := range f.deps {
		wd := path.Join(f.top, imp.Name)
		if VcsExists(imp, wd) {
			if updateCache[imp.Name] {
				Debug("----> Already updated %s", imp.Name)
				continue
			}
			Debug("Updating project %s (%s)\n", imp.Name, wd)
			if err := VcsUpdate(imp, f.top, force); err != nil {
				// We can still go on just fine even if this fails.
				Warn("Skipped update %s: %s\n", imp.Name, err)
				continue
			}
			updateCache[imp.Name] = true
		} else {
			Debug("Importing %s to project %s\n", imp.Name, wd)
			if err := VcsGet(imp, wd); err != nil {
				Warn("Skipped getting %s: %v\n", imp.Name, err)
				continue
			}
		}

		// If a revision has been set use it.
		err := VcsVersion(imp, f.top)
		if err != nil {
			Warn("Problem setting version on %s: %s\n", imp.Name, err)
		}
	}

	return nil
}

// Set the references for all packages after a flatten is completed.
func flattenSetRefs(f *flattening) {
	Debug("Setting final version for %d dependencies.", len(f.deps))
	for _, imp := range f.deps {
		if err := VcsVersion(imp, f.top); err != nil {
			Warn("Problem setting version on %s: %s (flatten)\n", imp.Name, err)
		}
	}
}

func mergeGlide(dir, name string, deps map[string]*Dependency) ([]string, bool) {
	gp := path.Join(dir, "glide.yaml")
	if _, err := os.Stat(gp); err != nil {
		return []string{}, false
	}
	f, err := yaml.ReadFile(gp)
	if err != nil {
		Warn("Found glide file %q, but can't parse: %s", gp, err)
		return []string{}, false
	}

	conf, err := FromYaml(f.Root)
	if err != nil {
		Warn("Found glide file %q, but can't use it: %s", gp, err)
		return []string{}, false
	}

	Info("Found glide.yaml in %s", gp)

	return mergeDeps(deps, conf.Imports), true
}

// listGodep appends Godeps entries to the deps.
//
// It returns true if any dependencies were found (even if not added because
// they are duplicates).
func mergeGodep(dir, name string, deps map[string]*Dependency) ([]string, bool) {
	Debug("Looking in %s/Godeps/ for a Godeps.json file.\n", dir)
	d, err := parseGodepGodeps(dir)
	if err != nil {
		Warn("Looking for Godeps: %s\n", err)
		return []string{}, false
	} else if len(d) == 0 {
		return []string{}, false
	}

	Info("Found Godeps.json file for %q", name)
	return mergeDeps(deps, d), true
}

// listGb merges GB dependencies into the deps.
func mergeGb(dir, pkg string, deps map[string]*Dependency) ([]string, bool) {
	Debug("Looking in %s/vendor/ for a manifest file.\n", dir)
	d, err := parseGbManifest(dir)
	if err != nil || len(d) == 0 {
		return []string{}, false
	}
	Info("Found gb manifest file for %q", pkg)
	return mergeDeps(deps, d), true
}

// mergeGPM merges GPM Godeps files into deps.
func mergeGPM(dir, pkg string, deps map[string]*Dependency) ([]string, bool) {
	d, err := parseGPMGodeps(dir)
	if err != nil || len(d) == 0 {
		return []string{}, false
	}
	Info("Found GPM file for %q", pkg)
	return mergeDeps(deps, d), true
}

// mergeGuess guesses dependencies and merges.
//
// This always returns true because it always handles the job of searching
// for dependencies. So generally it should be the last merge strategy
// that you try.
func mergeGuess(dir, pkg string, deps map[string]*Dependency) ([]string, bool) {
	/*
			Info("Scanning %s for dependencies.", pkg)
			buildContext, err := GetBuildContext()
			if err != nil {
				Warn("Could not scan package %q: %s", pkg, err)
				return []string{}, false
			}

			res := []string{}

			if _, err := os.Stat(dir); err != nil {
				Warn("Directory is missing: %s", dir)
				return res, true
			}

			d := walkDeps(buildContext, dir, pkg)
			for _, name := range d {
				name, _ := NormalizeName(name)
				repo := getRepoRootFromPackage(name)
				if _, ok := deps[name]; ok {
					Debug("====> Seen %s already. Skipping", name)
					continue
				}

				found := findPkg(buildContext, name, dir)
				switch found.PType {
				case ptypeUnknown:
					Debug("✨☆ Undownloaded dependency: %s", name)
					nd := &Dependency{
						Name:       name,
						Repository: "https://" + repo,
					}
					deps[name] = nd
					res = append(res, name)
				case ptypeGoroot, ptypeCgo:
					break
				default:
					// We're looking for dependencies that might exist in $GOPATH
					// but not be on vendor. We add any that are on $GOPATH.
					if _, ok := deps[name]; !ok {
						Debug("✨☆ GOPATH dependency: %s", name)
						nd := &Dependency{Name: name}
						deps[name] = nd
						res = append(res, name)
					}
				}
			}

		return res, true
	*/
	Info("Package %s manages its own dependencies", pkg)
	return []string{}, true
}

// mergeDeps merges any dependency array into deps.
func mergeDeps(orig map[string]*Dependency, add []*Dependency) []string {
	mod := []string{}
	for _, dd := range add {
		// Add it unless it's already there.
		if existing, ok := orig[dd.Name]; !ok {
			orig[dd.Name] = dd
			Debug("Adding %s to the scan list", dd.Name)
			mod = append(mod, dd.Name)
		} else if existing.Reference == "" && dd.Reference != "" {
			// If a nested dep has finer dependency references than outside,
			// set the reference.
			existing.Reference = dd.Reference
			mod = append(mod, dd.Name)
		} else if dd.Reference != "" && existing.Reference != "" && dd.Reference != existing.Reference {
			// We can detect version conflicts, but we can't really do
			// anything to correct, since we don't know the intentions of the
			// authors.
			Warn("Conflict: %s ref is %s, but also asked for %s", existing.Name, existing.Reference, dd.Reference)
			Info("Keeping %s %s", existing.Name, existing.Reference)
		}
	}
	return mod
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"os"

	"github.com/google/kf/pkg/kf/commands"
)

func main() {
	if err := commands.NewKfCommand().Execute(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Remove duplicate error message cobra Commands already print out errors<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"os"

	"github.com/google/kf/pkg/kf/commands"
)

func main() {
	if err := commands.NewKfCommand().Execute(); err != nil {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/BluePecker/JwtAuth/daemon"
	"os"
)

type Storage struct {
	Driver string
	Opts   string
}

type TLS struct {
	Key  string
	Cert string
}

type Args struct {
	PidFile  string
	LogFile  string
	LogLevel string
	Version  bool
	SockFile string
	Port     int
	Host     string
	Conf     string
	Secret   string
	Daemon   bool

	TLS     TLS
	Storage Storage
}

type RootCommand struct {
	Args  Args
	Cmd   *cobra.Command
	Viper *viper.Viper
}

var RootCmd *RootCommand = &RootCommand{}

func UsageTemplate() string {
	return `Usage:{{if .Runnable}}{{if .HasAvailableFlags}}
  {{appendIfNotPresent .Use}}[OPTIONS] COMMAND [arg...]{{else}}{{.UseLine}}{{end}}{{end}}{{if gt .Aliases 0}}
Aliases:{{.NameAndAliases}}
{{end}}{{if .HasExample}}
Examples:{{ .Example }}
{{end}}{{ if .HasAvailableLocalFlags}}
Options:
{{.LocalFlags.FlagUsages | trimRightSpace}}
{{end}}{{ if .HasAvailableSubCommands}}
Commands:{{range .Commands}}{{if .IsAvailableCommand}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}
{{end}}{{ if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsHelpCommand}}
  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableSubCommands }}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`
}

func init() {
	RootCmd.Viper = viper.GetViper()

	RootCmd.Cmd = &cobra.Command{
		Use:   "jwt",
		Short: "A self-sufficient runtime for json-web-token instance",
		RunE: func(cmd *cobra.Command, args []string) error {
			if _, err := os.Stat(RootCmd.Args.Conf); err == nil {
				RootCmd.Viper.SetConfigFile(RootCmd.Args.Conf)
				if err := RootCmd.Viper.ReadInConfig(); err != nil {
					return err
				}
			}

			RootCmd.Args.Port = RootCmd.Viper.GetInt("port")
			RootCmd.Args.Host = RootCmd.Viper.GetString("host")
			RootCmd.Args.PidFile = RootCmd.Viper.GetString("pid")
			RootCmd.Args.LogLevel = RootCmd.Viper.GetString("log-level")
			RootCmd.Args.LogFile = RootCmd.Viper.GetString("log")
			RootCmd.Args.SockFile = RootCmd.Viper.GetString("unix-sock")
			RootCmd.Args.Secret = RootCmd.Viper.GetString("secret")
			RootCmd.Args.Version = RootCmd.Viper.GetBool("version")
			RootCmd.Args.Daemon = RootCmd.Viper.GetBool("daemon")

			RootCmd.Args.Storage.Driver = RootCmd.Viper.GetString("storage.driver")
			RootCmd.Args.Storage.Opts = RootCmd.Viper.GetString("storage.opts")
			RootCmd.Args.TLS.Key = RootCmd.Viper.GetString("tls.key")
			RootCmd.Args.TLS.Cert = RootCmd.Viper.GetString("tls.cert")

			// 开启SERVER服务
			daemon.NewStart(daemon.Options{
				PidFile:  RootCmd.Args.PidFile,
				LogLevel: RootCmd.Args.LogLevel,
				LogFile:  RootCmd.Args.LogFile,
				SockFile: RootCmd.Args.SockFile,
				Port:     RootCmd.Args.Port,
				Host:     RootCmd.Args.Host,
				TLS: daemon.TLS{
					Cert: RootCmd.Args.TLS.Cert,
					Key:  RootCmd.Args.TLS.Key,
				},
				Version: RootCmd.Args.Version,
				Daemon:  RootCmd.Args.Daemon,
				Storage: daemon.Storage{
					Driver: RootCmd.Args.Storage.Driver,
					Opts:   RootCmd.Args.Storage.Opts,
				},
				Secret: RootCmd.Args.Secret,
			})

			return nil
		},
		SilenceUsage:  true,
		SilenceErrors: true,
	}
	RootCmd.Cmd.SetUsageTemplate(UsageTemplate())

	var PFlags *pflag.FlagSet = RootCmd.Cmd.Flags()

	PFlags.IntVarP(&RootCmd.Args.Port, "port", "p", 6010, "set the server listening port")
	PFlags.StringVarP(&RootCmd.Args.Host, "host", "", "127.0.0.1", "set the server bind host")
	PFlags.StringVarP(&RootCmd.Args.Conf, "config", "c", "/etc/jwt.json", "set configuration file")
	PFlags.BoolVarP(&RootCmd.Args.Version, "version", "v", false, "print version information and quit")
	PFlags.BoolVarP(&RootCmd.Args.Daemon, "daemon", "d", false, "enable daemon mode")
	PFlags.StringVarP(&RootCmd.Args.Secret, "secret", "s", "", "specify secret for jwt encode")
	PFlags.StringVarP(&RootCmd.Args.PidFile, "pid", "", "/var/run/jwt.pid", "path to use for daemon PID file")
	PFlags.StringVarP(&RootCmd.Args.LogLevel, "log-level", "l", "info", "set the logging level")
	PFlags.StringVarP(&RootCmd.Args.LogFile, "log", "", "/var/log/jwt.log", "path to use for log file")
	PFlags.StringVarP(&RootCmd.Args.SockFile, "unix-sock", "u", "/var/run/jwt.sock", "communication between the client and the daemon")
	PFlags.StringVarP(&RootCmd.Args.Storage.Driver, "storage-driver", "", "redis", "specify the storage driver")
	PFlags.StringVarP(&RootCmd.Args.Storage.Opts, "storage-opts", "", "redis://127.0.0.1:6379/1?PoolSize=20&MaxRetries=3&PoolTimeout=1000", "specify the storage uri")
	PFlags.StringVarP(&RootCmd.Args.TLS.Cert, "tlscert", "", "", "path to TLS certificate file")
	PFlags.StringVarP(&RootCmd.Args.TLS.Key, "tlskey", "", "", "path to TLS key file")

	RootCmd.Viper.BindPFlag("port", PFlags.Lookup("port"))
	RootCmd.Viper.BindPFlag("host", PFlags.Lookup("host"))
	RootCmd.Viper.BindPFlag("version", PFlags.Lookup("version"))
	RootCmd.Viper.BindPFlag("secret", PFlags.Lookup("secret"))
	RootCmd.Viper.BindPFlag("daemon", PFlags.Lookup("daemon"))
	RootCmd.Viper.BindPFlag("pid", PFlags.Lookup("pid"))
	RootCmd.Viper.BindPFlag("log", PFlags.Lookup("log"))
	RootCmd.Viper.BindPFlag("unix-sock", PFlags.Lookup("unix-sock"))
	RootCmd.Viper.BindPFlag("log-level", PFlags.Lookup("log-level"))
	RootCmd.Viper.BindPFlag("storage.driver", PFlags.Lookup("storage-driver"))
	RootCmd.Viper.BindPFlag("storage.opts", PFlags.Lookup("storage-opts"))
	RootCmd.Viper.BindPFlag("tls.cert", PFlags.Lookup("tlscert"))
	RootCmd.Viper.BindPFlag("tls.key", PFlags.Lookup("tlskey"))

	RootCmd.Cmd.AddCommand(StopCmd, TokenCmd, VersionCmd)
}
<commit_msg>fix bug<commit_after>package cmd

import (
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"github.com/BluePecker/JwtAuth/daemon"
	"os"
)

type Storage struct {
	Driver string
	Opts   string
}

type TLS struct {
	Key  string
	Cert string
}

type Args struct {
	PidFile  string
	LogFile  string
	LogLevel string
	Version  bool
	SockFile string
	Port     int
	Host     string
	Conf     string
	Secret   string
	Daemon   bool

	TLS     TLS
	Storage Storage
}

type RootCommand struct {
	Args  Args
	Cmd   *cobra.Command
	Viper *viper.Viper
}

var RootCmd *RootCommand = &RootCommand{}

func UsageTemplate() string {
	return `Usage:{{if .Runnable}}{{if .HasAvailableFlags}}
  {{appendIfNotPresent .Use [OPTIONS] COMMAND [arg...]}}{{else}}{{.UseLine}}{{end}}{{end}}{{if gt .Aliases 0}}
Aliases:{{.NameAndAliases}}
{{end}}{{if .HasExample}}
Examples:{{ .Example }}
{{end}}{{ if .HasAvailableLocalFlags}}
Options:
{{.LocalFlags.FlagUsages | trimRightSpace}}
{{end}}{{ if .HasAvailableSubCommands}}
Commands:{{range .Commands}}{{if .IsAvailableCommand}}
  {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}
{{end}}{{ if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimRightSpace}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsHelpCommand}}
  {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{ if .HasAvailableSubCommands }}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`
}

func init() {
	RootCmd.Viper = viper.GetViper()

	RootCmd.Cmd = &cobra.Command{
		Use:   "jwt",
		Short: "A self-sufficient runtime for json-web-token instance",
		RunE: func(cmd *cobra.Command, args []string) error {
			if _, err := os.Stat(RootCmd.Args.Conf); err == nil {
				RootCmd.Viper.SetConfigFile(RootCmd.Args.Conf)
				if err := RootCmd.Viper.ReadInConfig(); err != nil {
					return err
				}
			}

			RootCmd.Args.Port = RootCmd.Viper.GetInt("port")
			RootCmd.Args.Host = RootCmd.Viper.GetString("host")
			RootCmd.Args.PidFile = RootCmd.Viper.GetString("pid")
			RootCmd.Args.LogLevel = RootCmd.Viper.GetString("log-level")
			RootCmd.Args.LogFile = RootCmd.Viper.GetString("log")
			RootCmd.Args.SockFile = RootCmd.Viper.GetString("unix-sock")
			RootCmd.Args.Secret = RootCmd.Viper.GetString("secret")
			RootCmd.Args.Version = RootCmd.Viper.GetBool("version")
			RootCmd.Args.Daemon = RootCmd.Viper.GetBool("daemon")

			RootCmd.Args.Storage.Driver = RootCmd.Viper.GetString("storage.driver")
			RootCmd.Args.Storage.Opts = RootCmd.Viper.GetString("storage.opts")
			RootCmd.Args.TLS.Key = RootCmd.Viper.GetString("tls.key")
			RootCmd.Args.TLS.Cert = RootCmd.Viper.GetString("tls.cert")

			// 开启SERVER服务
			daemon.NewStart(daemon.Options{
				PidFile:  RootCmd.Args.PidFile,
				LogLevel: RootCmd.Args.LogLevel,
				LogFile:  RootCmd.Args.LogFile,
				SockFile: RootCmd.Args.SockFile,
				Port:     RootCmd.Args.Port,
				Host:     RootCmd.Args.Host,
				TLS: daemon.TLS{
					Cert: RootCmd.Args.TLS.Cert,
					Key:  RootCmd.Args.TLS.Key,
				},
				Version: RootCmd.Args.Version,
				Daemon:  RootCmd.Args.Daemon,
				Storage: daemon.Storage{
					Driver: RootCmd.Args.Storage.Driver,
					Opts:   RootCmd.Args.Storage.Opts,
				},
				Secret: RootCmd.Args.Secret,
			})

			return nil
		},
		SilenceUsage:  true,
		SilenceErrors: true,
	}
	RootCmd.Cmd.SetUsageTemplate(UsageTemplate())

	var PFlags *pflag.FlagSet = RootCmd.Cmd.Flags()

	PFlags.IntVarP(&RootCmd.Args.Port, "port", "p", 6010, "set the server listening port")
	PFlags.StringVarP(&RootCmd.Args.Host, "host", "", "127.0.0.1", "set the server bind host")
	PFlags.StringVarP(&RootCmd.Args.Conf, "config", "c", "/etc/jwt.json", "set configuration file")
	PFlags.BoolVarP(&RootCmd.Args.Version, "version", "v", false, "print version information and quit")
	PFlags.BoolVarP(&RootCmd.Args.Daemon, "daemon", "d", false, "enable daemon mode")
	PFlags.StringVarP(&RootCmd.Args.Secret, "secret", "s", "", "specify secret for jwt encode")
	PFlags.StringVarP(&RootCmd.Args.PidFile, "pid", "", "/var/run/jwt.pid", "path to use for daemon PID file")
	PFlags.StringVarP(&RootCmd.Args.LogLevel, "log-level", "l", "info", "set the logging level")
	PFlags.StringVarP(&RootCmd.Args.LogFile, "log", "", "/var/log/jwt.log", "path to use for log file")
	PFlags.StringVarP(&RootCmd.Args.SockFile, "unix-sock", "u", "/var/run/jwt.sock", "communication between the client and the daemon")
	PFlags.StringVarP(&RootCmd.Args.Storage.Driver, "storage-driver", "", "redis", "specify the storage driver")
	PFlags.StringVarP(&RootCmd.Args.Storage.Opts, "storage-opts", "", "redis://127.0.0.1:6379/1?PoolSize=20&MaxRetries=3&PoolTimeout=1000", "specify the storage uri")
	PFlags.StringVarP(&RootCmd.Args.TLS.Cert, "tlscert", "", "", "path to TLS certificate file")
	PFlags.StringVarP(&RootCmd.Args.TLS.Key, "tlskey", "", "", "path to TLS key file")

	RootCmd.Viper.BindPFlag("port", PFlags.Lookup("port"))
	RootCmd.Viper.BindPFlag("host", PFlags.Lookup("host"))
	RootCmd.Viper.BindPFlag("version", PFlags.Lookup("version"))
	RootCmd.Viper.BindPFlag("secret", PFlags.Lookup("secret"))
	RootCmd.Viper.BindPFlag("daemon", PFlags.Lookup("daemon"))
	RootCmd.Viper.BindPFlag("pid", PFlags.Lookup("pid"))
	RootCmd.Viper.BindPFlag("log", PFlags.Lookup("log"))
	RootCmd.Viper.BindPFlag("unix-sock", PFlags.Lookup("unix-sock"))
	RootCmd.Viper.BindPFlag("log-level", PFlags.Lookup("log-level"))
	RootCmd.Viper.BindPFlag("storage.driver", PFlags.Lookup("storage-driver"))
	RootCmd.Viper.BindPFlag("storage.opts", PFlags.Lookup("storage-opts"))
	RootCmd.Viper.BindPFlag("tls.cert", PFlags.Lookup("tlscert"))
	RootCmd.Viper.BindPFlag("tls.key", PFlags.Lookup("tlskey"))

	RootCmd.Cmd.AddCommand(StopCmd, TokenCmd, VersionCmd)
}
<|endoftext|>


<commit_before>package archive

// STATUS: 20%

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/martinlindhe/formats/parse"
)

const (
	arjBlockSizeMin  = 30
	arjBlockSizeMax  = 2600
	arjMaxSFX        = 500000 // size of self-extracting prefix
	arjHeaderIDHi    = 0xea
	arjHeaderIDLo    = 0x60
	arjFirstHdrSize  = 0x1e
	arjCommentMax    = 2048
	arjFileNameMax   = 512
	arjHeaderSizeMax = (arjFirstHdrSize + 10 + arjFileNameMax + arjCommentMax)
	arjCrcMask       = 0xffffffff
)

// ARJ parses the arj format
func ARJ(c *parse.Checker) (*parse.ParsedLayout, error) {

	if !isARJ(c.Header) {
		return nil, nil
	}

	mainHeader, err := parseARJMainHeader(c.File)

	// XXX rest of arj

	c.ParsedLayout.FileKind = parse.Archive
	c.ParsedLayout.MimeType = "application/x-arj"
	c.ParsedLayout.Layout = mainHeader

	return &c.ParsedLayout, err
}

func isARJ(b []byte) bool {

	return b[0] == 0x60 && b[1] == 0xea
}

// finds arj header and leaves file position at it
func findARJHeader(file *os.File) (int64, error) {

	reader := io.Reader(file)

	file.Seek(0, os.SEEK_SET)

	pos, _ := file.Seek(0, os.SEEK_CUR)
	lastpos, _ := file.Seek(0, os.SEEK_END)
	lastpos -= 2

	if lastpos > arjMaxSFX {
		lastpos = arjMaxSFX
	}
	// log.Println("starting", pos, lastpos)
	for ; pos < lastpos; pos++ {
		// log.Printf("setting pos to %04x\n", pos)
		pos2, _ := file.Seek(pos, os.SEEK_SET)
		if pos != pos2 {
			fmt.Printf("warning: expected %d, got %d\n", pos, pos2)
		}

		var c byte
		if err := binary.Read(reader, binary.LittleEndian, &c); err != nil {
			return 0, err
		}

		for pos < lastpos {
			if c != arjHeaderIDLo { // low order first
				if err := binary.Read(reader, binary.LittleEndian, &c); err != nil {
					return 0, err
				}
			} else {
				if err := binary.Read(reader, binary.LittleEndian, &c); err != nil {
					return 0, err
				}
				if c == arjHeaderIDHi {
					// log.Println("yes 1")
					break
				}
			}
			pos++
		}
		if pos >= lastpos {
			// log.Println("yes 2")
			break
		}

		var headerSize uint16
		if err := binary.Read(reader, binary.LittleEndian, &headerSize); err != nil {
			// log.Println("read err", err)
			return 0, err
		}

		log.Printf("header size %02x\n", headerSize)

		if headerSize <= arjHeaderSizeMax {
			log.Println("returning pos", pos)
			return pos, nil
		}
	}

	return 0, fmt.Errorf("could not find arj header in %s", file.Name())
}

var (
	arjHostOS = map[byte]string{
		0:  "MSDOS",
		1:  "PRIMOS",
		2:  "UNIX",
		3:  "AMIGA",
		4:  "MAC-OS",
		5:  "OS/2",
		6:  "APPLE GS",
		7:  "ATARI ST",
		8:  "NEXT",
		9:  "VAX VMS",
		10: "WIN95",
		11: "WIN32",
	}
)

func parseARJMainHeader(f *os.File) ([]parse.Layout, error) {

	pos, err := findARJHeader(f)
	if err != nil {
		return nil, err
	}

	hostOSName, _ := parse.ReadToMap(f, parse.Uint8, pos+7, arjHostOS)

	mainHeaderLen := int64(34)

	chunk := parse.Layout{
		Offset: pos,
		Type:   parse.Group,
		Info:   "main header",
		Childs: []parse.Layout{
			{Offset: pos, Length: 2, Type: parse.Uint16le, Info: "magic"},
			{Offset: pos + 2, Length: 2, Type: parse.Uint16le, Info: "basic header size"}, // excl. Magic+HdrSize
			{Offset: pos + 4, Length: 1, Type: parse.Uint8, Info: "size up to and including 'extra data'"},
			{Offset: pos + 5, Length: 1, Type: parse.Uint8, Info: "archiver version number"},
			{Offset: pos + 6, Length: 1, Type: parse.Uint8, Info: "minimum archiver version to extract"},
			{Offset: pos + 7, Length: 1, Type: parse.Uint8, Info: "host OS = " + hostOSName},
			{Offset: pos + 8, Length: 1, Type: parse.Uint8, Info: "arj flags"}, // XXX show bitfield
			{Offset: pos + 9, Length: 1, Type: parse.Uint8, Info: "security version"},
			{Offset: pos + 10, Length: 1, Type: parse.Uint8, Info: "file type"},
			{Offset: pos + 11, Length: 1, Type: parse.Uint8, Info: "reserved"},
			{Offset: pos + 12, Length: 4, Type: parse.ArjDateTime, Info: "created time"},
			{Offset: pos + 16, Length: 4, Type: parse.ArjDateTime, Info: "modified time"},
			{Offset: pos + 20, Length: 4, Type: parse.Uint32le, Info: "archive size for secured archive"},
			{Offset: pos + 24, Length: 4, Type: parse.Uint32le, Info: "security envelope file position"},
			{Offset: pos + 28, Length: 2, Type: parse.Uint16le, Info: "filespec position in filename"},
			{Offset: pos + 30, Length: 2, Type: parse.Uint16le, Info: "length in bytes of security envelope data"},
			{Offset: pos + 32, Length: 1, Type: parse.Uint8, Info: "encryption version"},
			{Offset: pos + 33, Length: 1, Type: parse.Uint8, Info: "last chapter"}, // XXX
		},
	}

	withExtData, _ := parse.ReadUint8(f, pos+4)
	if withExtData == 0x22 {
		chunk.Childs = append(chunk.Childs, []parse.Layout{
			{Offset: pos + 34, Length: 1, Type: parse.Uint8, Info: "protection factor"},
			{Offset: pos + 35, Length: 1, Type: parse.Uint8, Info: "flags (second series)"},
			{Offset: pos + 36, Length: 2, Type: parse.Uint8, Info: "spare bytes"},
		}...)
		mainHeaderLen += 4
	} else if withExtData == 0x1E {
		// no ext data
	} else {
		log.Fatalf("sample please. ext data = %02x", withExtData)
	}

	_, archiveNameLen, err := parse.ReadZeroTerminatedASCIIUntil(f, pos+mainHeaderLen, 255)
	if err != nil {
		return nil, err
	}

	_, commentLen, err := parse.ReadZeroTerminatedASCIIUntil(f, pos+mainHeaderLen+archiveNameLen, 4096)
	if err != nil {
		return nil, err
	}

	chunk.Length = mainHeaderLen + archiveNameLen + commentLen + 6

	pos += mainHeaderLen

	chunk.Childs = append(chunk.Childs, []parse.Layout{
		{Offset: pos, Length: archiveNameLen, Type: parse.ASCIIZ, Info: "archive name"},
	}...)
	pos += archiveNameLen

	chunk.Childs = append(chunk.Childs, []parse.Layout{
		{Offset: pos, Length: commentLen, Type: parse.ASCIIZ, Info: "comment"},
	}...)
	pos += commentLen

	chunk.Childs = append(chunk.Childs, []parse.Layout{
		{Offset: pos, Length: 4, Type: parse.Uint32le, Info: "crc32"},
		{Offset: pos + 4, Length: 2, Type: parse.Uint16le, Info: "ext header size"},
	}...)
	pos += 6

	// XXX if ext header size > 0, it should follow here! need sample

	return []parse.Layout{chunk}, nil
}
<commit_msg>arj: parse local file headers<commit_after>package archive

// STATUS: 60%

import (
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"

	"github.com/martinlindhe/formats/parse"
)

const (
	arjBlockSizeMin  = 30
	arjBlockSizeMax  = 2600
	arjMaxSFX        = 500000 // size of self-extracting prefix
	arjHeaderIDHi    = 0xea
	arjHeaderIDLo    = 0x60
	arjFirstHdrSize  = 0x1e
	arjCommentMax    = 2048
	arjFileNameMax   = 512
	arjHeaderSizeMax = (arjFirstHdrSize + 10 + arjFileNameMax + arjCommentMax)
	arjCrcMask       = 0xffffffff
)

// ARJ parses the arj format
func ARJ(c *parse.Checker) (*parse.ParsedLayout, error) {

	if !isARJ(c.Header) {
		return nil, nil
	}

	arj, err := parseARJ(c.File)

	c.ParsedLayout.FileKind = parse.Archive
	c.ParsedLayout.MimeType = "application/x-arj"
	c.ParsedLayout.Layout = arj

	return &c.ParsedLayout, err
}

func isARJ(b []byte) bool {

	return b[0] == 0x60 && b[1] == 0xea
}

// finds arj header and leaves file position at it
func findARJHeader(file *os.File) (int64, error) {

	reader := io.Reader(file)

	file.Seek(0, os.SEEK_SET)

	pos, _ := file.Seek(0, os.SEEK_CUR)
	lastpos, _ := file.Seek(0, os.SEEK_END)
	lastpos -= 2

	if lastpos > arjMaxSFX {
		lastpos = arjMaxSFX
	}
	// log.Println("starting", pos, lastpos)
	for ; pos < lastpos; pos++ {
		// log.Printf("setting pos to %04x\n", pos)
		pos2, _ := file.Seek(pos, os.SEEK_SET)
		if pos != pos2 {
			fmt.Printf("warning: expected %d, got %d\n", pos, pos2)
		}

		var c byte
		if err := binary.Read(reader, binary.LittleEndian, &c); err != nil {
			return 0, err
		}

		for pos < lastpos {
			if c != arjHeaderIDLo { // low order first
				if err := binary.Read(reader, binary.LittleEndian, &c); err != nil {
					return 0, err
				}
			} else {
				if err := binary.Read(reader, binary.LittleEndian, &c); err != nil {
					return 0, err
				}
				if c == arjHeaderIDHi {
					// log.Println("yes 1")
					break
				}
			}
			pos++
		}
		if pos >= lastpos {
			// log.Println("yes 2")
			break
		}

		var headerSize uint16
		if err := binary.Read(reader, binary.LittleEndian, &headerSize); err != nil {
			// log.Println("read err", err)
			return 0, err
		}

		// log.Printf("header size %02x\n", headerSize)
		if headerSize <= arjHeaderSizeMax {
			return pos, nil
		}
	}

	return 0, fmt.Errorf("could not find arj header in %s", file.Name())
}

var (
	arjHostOS = map[byte]string{
		0:  "MSDOS",
		1:  "PRIMOS",
		2:  "UNIX",
		3:  "AMIGA",
		4:  "MAC-OS",
		5:  "OS/2",
		6:  "APPLE GS",
		7:  "ATARI ST",
		8:  "NEXT",
		9:  "VAX VMS",
		10: "WIN95",
		11: "WIN32",
	}
	arjMethod = map[byte]string{
		0: "stored",
		1: "compressed most",
		2: "compressed 2",
		3: "compressed 3",
		4: "compressed fastest",
		8: "no data, no CRC",
		9: "no data",
	}
	arjFileType = map[byte]string{
		0: "binary",
		1: "7-bit text",
		3: "directory",
		4: "volume label",
		5: "chapter label",
	}
)

func parseARJ(f *os.File) ([]parse.Layout, error) {

	pos, err := findARJHeader(f)
	if err != nil {
		return nil, err
	}

	hostOSName, _ := parse.ReadToMap(f, parse.Uint8, pos+7, arjHostOS)

	mainHeaderLen := int64(34)

	chunk := parse.Layout{
		Offset: pos,
		Type:   parse.Group,
		Info:   "main header",
		Childs: []parse.Layout{
			{Offset: pos, Length: 2, Type: parse.Uint16le, Info: "magic"},
			{Offset: pos + 2, Length: 2, Type: parse.Uint16le, Info: "basic header size"}, // excl. Magic+HdrSize
			{Offset: pos + 4, Length: 1, Type: parse.Uint8, Info: "size up to and including 'extra data'"},
			{Offset: pos + 5, Length: 1, Type: parse.Uint8, Info: "archiver version number"},
			{Offset: pos + 6, Length: 1, Type: parse.Uint8, Info: "minimum archiver version to extract"},
			{Offset: pos + 7, Length: 1, Type: parse.Uint8, Info: "host OS = " + hostOSName},
			{Offset: pos + 8, Length: 1, Type: parse.Uint8, Info: "arj flags"}, // XXX show bitfield
			{Offset: pos + 9, Length: 1, Type: parse.Uint8, Info: "security version"},
			{Offset: pos + 10, Length: 1, Type: parse.Uint8, Info: "file type"},
			{Offset: pos + 11, Length: 1, Type: parse.Uint8, Info: "reserved"},
			{Offset: pos + 12, Length: 4, Type: parse.ArjDateTime, Info: "created time"},
			{Offset: pos + 16, Length: 4, Type: parse.ArjDateTime, Info: "modified time"},
			{Offset: pos + 20, Length: 4, Type: parse.Uint32le, Info: "archive size for secured archive"},
			{Offset: pos + 24, Length: 4, Type: parse.Uint32le, Info: "security envelope file position"},
			{Offset: pos + 28, Length: 2, Type: parse.Uint16le, Info: "filespec position in filename"},
			{Offset: pos + 30, Length: 2, Type: parse.Uint16le, Info: "length in bytes of security envelope data"},
			{Offset: pos + 32, Length: 1, Type: parse.Uint8, Info: "encryption version"},
			{Offset: pos + 33, Length: 1, Type: parse.Uint8, Info: "last chapter"},
		},
	}

	withExtData, _ := parse.ReadUint8(f, pos+4)
	if withExtData == 0x22 {
		chunk.Childs = append(chunk.Childs, []parse.Layout{
			{Offset: pos + 34, Length: 1, Type: parse.Uint8, Info: "protection factor"},
			{Offset: pos + 35, Length: 1, Type: parse.Uint8, Info: "flags (second series)"},
			{Offset: pos + 36, Length: 2, Type: parse.Uint8, Info: "spare bytes"},
		}...)
		mainHeaderLen += 4
	} else if withExtData == 0x1E {
		// no ext data
	} else {
		log.Fatalf("sample please. ext data = %02x", withExtData)
	}

	_, archiveNameLen, err := parse.ReadZeroTerminatedASCIIUntil(f, pos+mainHeaderLen, 255)
	if err != nil {
		return nil, err
	}

	_, commentLen, err := parse.ReadZeroTerminatedASCIIUntil(f, pos+mainHeaderLen+archiveNameLen, 4096)
	if err != nil {
		return nil, err
	}

	chunk.Length = mainHeaderLen + archiveNameLen + commentLen + 6

	pos += mainHeaderLen

	chunk.Childs = append(chunk.Childs, []parse.Layout{
		{Offset: pos, Length: archiveNameLen, Type: parse.ASCIIZ, Info: "archive name"},
	}...)
	pos += archiveNameLen

	chunk.Childs = append(chunk.Childs, []parse.Layout{
		{Offset: pos, Length: commentLen, Type: parse.ASCIIZ, Info: "archive comment"},
	}...)
	pos += commentLen

	chunk.Childs = append(chunk.Childs, []parse.Layout{
		{Offset: pos, Length: 4, Type: parse.Uint32le, Info: "crc32"},
		{Offset: pos + 4, Length: 2, Type: parse.Uint16le, Info: "ext header size"},
	}...)
	pos += 6
	// NOTE: if ext header size > 0, it should follow here. currently unused in file format

	res := []parse.Layout{chunk}

	// parse local file headers until one has size=0 == EOF
	for {
		magic, _ := parse.ReadUint16le(f, pos)
		if magic != 0xEA60 {
			log.Fatalf("Unexpected magic %04x at %04x", magic, pos)
		}
		length, _ := parse.ReadUint16le(f, pos+2)

		local := parse.Layout{
			Offset: pos,
			Length: 4,
			Type:   parse.Group,
			Info:   "local file header",
			Childs: []parse.Layout{
				{Offset: pos, Length: 2, Type: parse.Uint16le, Info: "magic"},
				{Offset: pos + 2, Length: 2, Type: parse.Uint16le, Info: "basic header size"},
			},
		}
		if length > 0 {
			localHostOSName, _ := parse.ReadToMap(f, parse.Uint8, pos+7, arjHostOS)
			methodName, _ := parse.ReadToMap(f, parse.Uint8, pos+9, arjMethod)
			fileTypeName, _ := parse.ReadToMap(f, parse.Uint8, pos+10, arjFileType)
			dataLength, _ := parse.ReadUint32le(f, pos+16)
			local.Childs = append(local.Childs, []parse.Layout{
				{Offset: pos + 4, Length: 1, Type: parse.Uint8, Info: "size up to and including 'extra data'"},
				{Offset: pos + 5, Length: 1, Type: parse.Uint8, Info: "archiver version number"},
				{Offset: pos + 6, Length: 1, Type: parse.Uint8, Info: "minimum archiver version to extract"},
				{Offset: pos + 7, Length: 1, Type: parse.Uint8, Info: "host OS = " + localHostOSName},
				{Offset: pos + 8, Length: 1, Type: parse.Uint8, Info: "arj flags"}, // XXX show bitfield
				{Offset: pos + 9, Length: 1, Type: parse.Uint8, Info: "method = " + methodName},
				{Offset: pos + 10, Length: 1, Type: parse.Uint8, Info: "file type = " + fileTypeName},
				{Offset: pos + 11, Length: 1, Type: parse.Uint8, Info: "reserved"},
				{Offset: pos + 12, Length: 4, Type: parse.ArjDateTime, Info: "modified time"},
				{Offset: pos + 16, Length: 4, Type: parse.Uint32le, Info: "compressed size"},
				{Offset: pos + 20, Length: 4, Type: parse.Uint32le, Info: "original size"},
				{Offset: pos + 24, Length: 4, Type: parse.Uint32le, Info: "original file's CRC"},
				{Offset: pos + 28, Length: 2, Type: parse.Uint16le, Info: "filespec position in filename"},
				{Offset: pos + 30, Length: 2, Type: parse.Uint16le, Info: "file access mode"},
				{Offset: pos + 32, Length: 1, Type: parse.Uint8, Info: "first chapter of file's lifespan"},
				{Offset: pos + 33, Length: 1, Type: parse.Uint8, Info: "last chapter of file's lifespan"},
			}...)
			local.Length += 30

			withExtData, _ := parse.ReadUint8(f, pos+4)
			if withExtData == 0x2E {
				local.Childs = append(local.Childs, []parse.Layout{
					{Offset: pos + 34, Length: 4, Type: parse.Uint32le, Info: "extended file position"},
					// XXX the following twelve bytes may be present in ARJ 2.62 and above:
					{Offset: pos + 38, Length: 4, Type: parse.ArjDateTime, Info: "accessed time"},
					{Offset: pos + 42, Length: 4, Type: parse.ArjDateTime, Info: "created time"},
					{Offset: pos + 46, Length: 4, Type: parse.Uint32le, Info: "original file size"},
				}...)
				local.Length += 16
			} else if withExtData == 0x1E {
				// no ext data
			} else {
				log.Fatalf("sample please. local file ext data = %02x", withExtData)
			}

			pos += int64(length)

			_, fileNameLen, err := parse.ReadZeroTerminatedASCIIUntil(f, pos, 255)
			if err != nil {
				return nil, err
			}
			local.Childs = append(local.Childs, []parse.Layout{
				{Offset: pos, Length: fileNameLen, Type: parse.ASCIIZ, Info: "file name"},
			}...)
			pos += fileNameLen
			local.Length += fileNameLen

			_, commentLen, err := parse.ReadZeroTerminatedASCIIUntil(f, pos, 4096)
			if err != nil {
				return nil, err
			}

			local.Childs = append(local.Childs, []parse.Layout{
				{Offset: pos, Length: commentLen, Type: parse.ASCIIZ, Info: "file comment"},
			}...)
			pos += commentLen
			local.Length += commentLen

			local.Childs = append(local.Childs, []parse.Layout{
				{Offset: pos, Length: 4, Type: parse.Uint32le, Info: "basic header crc32"},
				{Offset: pos + 4, Length: 2, Type: parse.Uint16le, Info: "ext header size"},
			}...)
			pos += 6
			local.Length += 6
			// NOTE: if ext header size > 0, it should follow here. currently unused in file format

			// XXX now follows compressed data
			local.Childs = append(local.Childs, []parse.Layout{
				{Offset: pos, Length: int64(dataLength), Type: parse.Bytes, Info: "compressed data"},
			}...)
			pos += int64(dataLength)
			local.Length += int64(dataLength)
		}
		res = append(res, local)

		if length == 0 {
			// log.Println("FOUND LAST ONE!!!")
			break
		}
	}

	return res, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

func VersionCmd() *cobra.Command {
	return &cobra.Command{
		Use:   "version",
		Short: "print the version",
		Long:  `Print the version.`,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("v1.0.0")
		},
	}
}
<commit_msg>Version up.<commit_after>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

func VersionCmd() *cobra.Command {
	return &cobra.Command{
		Use:   "version",
		Short: "print the version",
		Long:  `Print the version.`,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("v1.0.1")
		},
	}
}
<|endoftext|>