<commit_before>package main

import (
	"compress/gzip"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"time"

	"github.com/dickeyxxx/golock"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = "master"
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	Errln("updating cli...")
	Logln("updating")
	if err := golock.Lock(updateLockPath); err != nil {
		panic(err)
	}
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
		Logln("Timed out while updating")
	case <-done:
		touchAutoupdateFile()
	}
	Logln("done updating")
}

func updatePlugins() {
	Logln("updating plugins")
	b, _ := node.UpdatePackages()
	if len(b) > 0 {
		Logln("clearing plugins cache")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
	}
}

func updateCLI(channel string) {
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out.\n", Version, manifest.Version)
		return
	}
	Logln("updating from %s to %s (%s)", Version, manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	if err := os.Rename(binPath, binPath+".old"); err != nil {
		panic(err)
	}
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	reexecBin()
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 1*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := http.Get("https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json")
	if err != nil {
		panic(err)
	}
	var m manifest
	json.NewDecoder(res.Body).Decode(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+".gz", nil)
	if err != nil {
		return err
	}
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	uncompressed, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func reexecBin() {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(binPath, os.Args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		os.Exit(0)
	} else {
		if err := syscall.Exec(binPath, os.Args, os.Environ()); err != nil {
			panic(err)
		}
		os.Exit(99) // should never happen
	}
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	if err := exec.Command("./heroku-cli", "update").Start(); err != nil {
		panic(err)
	}
}
<commit_msg>added note for updates<commit_after>package main

import (
	"compress/gzip"
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"time"

	"github.com/dickeyxxx/golock"
)

var updateTopic = &Topic{
	Name:        "update",
	Description: "update heroku-cli",
}

var updateCmd = &Command{
	Topic:       "update",
	Description: "updates heroku-cli",
	Args:        []Arg{{Name: "channel", Optional: true}},
	Run: func(ctx *Context) {
		channel := ctx.Args.(map[string]string)["channel"]
		if channel == "" {
			channel = "master"
		}
		Update(channel)
	},
}

var binPath = filepath.Join(AppDir, "heroku-cli")
var updateLockPath = filepath.Join(AppDir, "updating.lock")
var autoupdateFile = filepath.Join(AppDir, "autoupdate")

func init() {
	if runtime.GOOS == "windows" {
		binPath = binPath + ".exe"
	}
}

// Update updates the CLI and plugins
func Update(channel string) {
	Errln("updating cli...")
	Logln("updating cli...")
	if err := golock.Lock(updateLockPath); err != nil {
		panic(err)
	}
	defer golock.Unlock(updateLockPath)
	done := make(chan bool)
	go func() {
		updateCLI(channel)
		updatePlugins()
		done <- true
	}()
	select {
	case <-time.After(time.Second * 120):
		Errln("Timed out while updating")
		Logln("Timed out while updating")
	case <-done:
		touchAutoupdateFile()
	}
	Errln("done updating")
	Logln("done updating")
}

func updatePlugins() {
	Errln("updating plugins")
	Logln("updating plugins")
	b, _ := node.UpdatePackages()
	if len(b) > 0 {
		Errln("clearing plugins cache")
		Logln("clearing plugins cache")
		ClearPluginCache()
		WritePluginCache(GetPlugins())
	}
}

func updateCLI(channel string) {
	manifest := getUpdateManifest(channel)
	if manifest.Version == Version && manifest.Channel == Channel {
		return
	}
	if !updatable() {
		Errf("Out of date: You are running %s but %s is out.\n", Version, manifest.Version)
		return
	}
	Logln("updating from %s to %s (%s)", Version, manifest.Version, manifest.Channel)
	Errln("updating from %s to %s (%s)", Version, manifest.Version, manifest.Channel)
	build := manifest.Builds[runtime.GOOS][runtime.GOARCH]
	// on windows we can't remove an existing file or remove the running binary
	// so we download the file to binName.new
	// move the running binary to binName.old (deleting any existing file first)
	// rename the downloaded file to binName
	if err := downloadBin(binPath+".new", build.URL); err != nil {
		panic(err)
	}
	if fileSha1(binPath+".new") != build.Sha1 {
		panic("SHA mismatch")
	}
	os.Remove(binPath + ".old")
	if err := os.Rename(binPath, binPath+".old"); err != nil {
		panic(err)
	}
	if err := os.Rename(binPath+".new", binPath); err != nil {
		panic(err)
	}
	reexecBin()
}

// IsUpdateNeeded checks if an update is available
func IsUpdateNeeded(t string) bool {
	f, err := os.Stat(autoupdateFile)
	if err != nil {
		return true
	}
	if t == "soft" {
		return time.Since(f.ModTime()) > 1*time.Hour
	}
	return time.Since(f.ModTime()) > 168*time.Hour
}

func touchAutoupdateFile() {
	out, err := os.OpenFile(autoupdateFile, os.O_WRONLY|os.O_CREATE, 0644)
	if err != nil {
		panic(err)
	}
	out.WriteString(time.Now().String())
}

type manifest struct {
	Channel, Version string
	Builds           map[string]map[string]struct {
		URL, Sha1 string
	}
}

func getUpdateManifest(channel string) manifest {
	res, err := http.Get("https://d1gvo455cekpjp.cloudfront.net/" + channel + "/manifest.json")
	if err != nil {
		panic(err)
	}
	var m manifest
	json.NewDecoder(res.Body).Decode(&m)
	return m
}

func updatable() bool {
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		Errln(err)
	}
	return path == binPath
}

func downloadBin(path, url string) error {
	out, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
	if err != nil {
		return err
	}
	client := &http.Client{}
	req, err := http.NewRequest("GET", url+".gz", nil)
	if err != nil {
		return err
	}
	req.Header.Add("Accept-Encoding", "gzip")
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	uncompressed, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	_, err = io.Copy(out, uncompressed)
	if err != nil {
		return err
	}
	return out.Close()
}

func fileSha1(path string) string {
	data, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("%x", sha1.Sum(data))
}

func reexecBin() {
	if runtime.GOOS == "windows" {
		cmd := exec.Command(binPath, os.Args[1:]...)
		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		os.Exit(0)
	} else {
		if err := syscall.Exec(binPath, os.Args, os.Environ()); err != nil {
			panic(err)
		}
		os.Exit(99) // should never happen
	}
}

// TriggerBackgroundUpdate will trigger an update to the client in the background
func TriggerBackgroundUpdate() {
	if err := exec.Command("./heroku-cli", "update").Start(); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"encoding/json"
	"io"
	"os"
	"strings"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	deploycmds "github.com/pachyderm/pachyderm/src/server/pkg/deploy/cmds"
	api "k8s.io/kubernetes/pkg/api/v1"
)

func TestMetrics(t *testing.T) {

	// Run deploy normally, should see METRICS=true
	old := os.Stdout
	r, w, _ := os.Pipe()
	os.Stdout = w

	os.Args = []string{"deploy", "--dry-run"}
	err := deploycmds.DeployCmd().Execute()
	require.NoError(t, err)
	outC := make(chan string)
	// copy the output in a separate goroutine so printing can't block indefinitely
	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf.String()
	}()

	// restore stdout
	w.Close()
	os.Stdout = old
	out := <-outC

	decoder := json.NewDecoder(strings.NewReader(out))
	foundPachdManifest := false
	for {
		var manifest *api.ReplicationController
		err = decoder.Decode(&manifest)
		if err == io.EOF {
			break
		}
		if err != nil {
			continue
		}
		require.NoError(t, err)

		if manifest.ObjectMeta.Name == "pachd" && manifest.Kind == "ReplicationController" {
			foundPachdManifest = true
			falseMetricEnvVar := api.EnvVar{
				Name:  "METRICS",
				Value: "true",
			}
			var env []interface{}
			require.Equal(t, 1, len(manifest.Spec.Template.Spec.Containers))
			for _, value := range manifest.Spec.Template.Spec.Containers[0].Env {
				env = append(env, value)
			}
			require.OneOfEquals(t, interface{}(falseMetricEnvVar), env)
		}
	}
	require.Equal(t, true, foundPachdManifest)

	// Run deploy w dev flag, should see METRICS=false
	r, w, _ = os.Pipe()
	os.Stdout = w

	os.Args = []string{"deploy", "-d", "--dry-run"}
	err = deploycmds.DeployCmd().Execute()
	require.NoError(t, err)
	outC = make(chan string)
	// copy the output in a separate goroutine so printing can't block indefinitely
	go func() {
		var buf bytes.Buffer
		io.Copy(&buf, r)
		outC <- buf.String()
	}()

	// restore stdout
	w.Close()
	os.Stdout = old
	out = <-outC

	decoder = json.NewDecoder(strings.NewReader(out))
	foundPachdManifest = false
	for {
		var manifest *api.ReplicationController
		err = decoder.Decode(&manifest)
		if err == io.EOF {
			break
		}
		if err != nil {
			continue
		}
		require.NoError(t, err)

		if manifest.ObjectMeta.Name == "pachd" && manifest.Kind == "ReplicationController" {
			foundPachdManifest = true
			falseMetricEnvVar := api.EnvVar{
				Name:  "METRICS",
				Value: "false",
			}
			var env []interface{}
			require.Equal(t, 1, len(manifest.Spec.Template.Spec.Containers))
			for _, value := range manifest.Spec.Template.Spec.Containers[0].Env {
				env = append(env, value)
			}
			require.OneOfEquals(t, interface{}(falseMetricEnvVar), env)
		}
	}
	require.Equal(t, true, foundPachdManifest)
}
<commit_msg>Remove channels, no need<commit_after>package cmd

import (
	"encoding/json"
	"io"
	"os"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	deploycmds "github.com/pachyderm/pachyderm/src/server/pkg/deploy/cmds"
	api "k8s.io/kubernetes/pkg/api/v1"
)

func TestMetrics(t *testing.T) {

	// Run deploy normally, should see METRICS=true
	old := os.Stdout
	r, w, _ := os.Pipe()
	os.Stdout = w

	os.Args = []string{"deploy", "--dry-run"}
	err := deploycmds.DeployCmd().Execute()
	require.NoError(t, err)
	require.NoError(t, w.Close())
	// restore stdout
	os.Stdout = old

	decoder := json.NewDecoder(r)
	foundPachdManifest := false
	for {
		var manifest *api.ReplicationController
		err = decoder.Decode(&manifest)
		if err == io.EOF {
			break
		}
		if err != nil {
			continue
		}
		require.NoError(t, err)

		if manifest.ObjectMeta.Name == "pachd" && manifest.Kind == "ReplicationController" {
			foundPachdManifest = true
			falseMetricEnvVar := api.EnvVar{
				Name:  "METRICS",
				Value: "true",
			}
			var env []interface{}
			require.Equal(t, 1, len(manifest.Spec.Template.Spec.Containers))
			for _, value := range manifest.Spec.Template.Spec.Containers[0].Env {
				env = append(env, value)
			}
			require.OneOfEquals(t, interface{}(falseMetricEnvVar), env)
		}
	}
	require.Equal(t, true, foundPachdManifest)

	// Run deploy w dev flag, should see METRICS=false
	r, w, _ = os.Pipe()
	os.Stdout = w

	os.Args = []string{"deploy", "-d", "--dry-run"}
	err = deploycmds.DeployCmd().Execute()
	require.NoError(t, err)
	require.NoError(t, w.Close())
	// restore stdout
	os.Stdout = old

	decoder = json.NewDecoder(r)
	foundPachdManifest = false
	for {
		var manifest *api.ReplicationController
		err = decoder.Decode(&manifest)
		if err == io.EOF {
			break
		}
		if err != nil {
			continue
		}

		if manifest.ObjectMeta.Name == "pachd" && manifest.Kind == "ReplicationController" {
			foundPachdManifest = true
			falseMetricEnvVar := api.EnvVar{
				Name:  "METRICS",
				Value: "false",
			}
			var env []interface{}
			require.Equal(t, 1, len(manifest.Spec.Template.Spec.Containers))
			for _, value := range manifest.Spec.Template.Spec.Containers[0].Env {
				env = append(env, value)
			}
			require.OneOfEquals(t, interface{}(falseMetricEnvVar), env)
		}
	}
	require.Equal(t, true, foundPachdManifest)
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"github.com/juju/errors"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/network"
	"github.com/juju/juju/provider/common"
)

// PrecheckInstance verifies that the provided series and constraints
// are valid for use in creating an instance in this environment.
func (env *environ) PrecheckInstance(series string, cons constraints.Value, placement string) error {
	if _, err := env.parsePlacement(placement); err != nil {
		return errors.Trace(err)
	}

	if cons.HasInstanceType() {
		if !checkInstanceType(cons) {
			return errors.Errorf("invalid GCE instance type %q", *cons.InstanceType)
		}
	}

	return nil
}

// SupportedArchitectures returns the image architectures which can
// be hosted by this environment.
func (env *environ) SupportedArchitectures() ([]string, error) {
	env.archLock.Lock()
	defer env.archLock.Unlock()

	if env.supportedArchitectures != nil {
		return env.supportedArchitectures, nil
	}

	archList, err := env.lookupArchitectures()
	if err != nil {
		return nil, errors.Trace(err)
	}
	env.supportedArchitectures = archList
	return archList, nil
}

var supportedArchitectures = common.SupportedArchitectures

func (env *environ) lookupArchitectures() ([]string, error) {
	// Create a filter to get all images from our region and for the
	// correct stream.
	cloudSpec, err := env.Region()
	if err != nil {
		return nil, errors.Trace(err)
	}
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: cloudSpec,
		Stream:    env.Config().ImageStream(),
	})
	archList, err := supportedArchitectures(env, imageConstraint)
	return archList, errors.Trace(err)
}

var unsupportedConstraints = []string{
	constraints.Tags,
	constraints.Networks,
}

// instanceTypeConstraints defines the fields defined on each of the
// instance types.  See instancetypes.go.
var instanceTypeConstraints = []string{
	constraints.Arch, // Arches
	constraints.CpuCores,
	constraints.CpuPower,
	constraints.Mem,
	constraints.Container, // VirtType
}

// ConstraintsValidator returns a Validator value which is used to
// validate and merge constraints.
func (env *environ) ConstraintsValidator() (constraints.Validator, error) {
	validator := constraints.NewValidator()

	// conflicts

	// TODO(ericsnow) Are these correct?
	validator.RegisterConflicts(
		[]string{constraints.InstanceType},
		instanceTypeConstraints,
	)

	// unsupported

	validator.RegisterUnsupported(unsupportedConstraints)

	// vocab

	supportedArches, err := env.SupportedArchitectures()
	if err != nil {
		return nil, errors.Trace(err)
	}
	validator.RegisterVocabulary(constraints.Arch, supportedArches)

	instTypeNames := make([]string, len(allInstanceTypes))
	for i, itype := range allInstanceTypes {
		instTypeNames[i] = itype.Name
	}
	validator.RegisterVocabulary(constraints.InstanceType, instTypeNames)

	validator.RegisterVocabulary(constraints.Container, []string{vtype})

	return validator, nil
}

// SupportsUnitPlacement implement via common.SupportsUnitPlacementPolicy

// SupportNetworks returns whether the environment has support to
// specify networks for services and machines.
func (env *environ) SupportNetworks() bool {
	return false
}

// SupportAddressAllocation takes a network.Id and returns a bool
// and an error. The bool indicates whether that network supports
// static ip address allocation.
func (env *environ) SupportAddressAllocation(netID network.Id) (bool, error) {
	return false, nil
}
<commit_msg>Clarify a comment.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"github.com/juju/errors"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/network"
	"github.com/juju/juju/provider/common"
)

// PrecheckInstance verifies that the provided series and constraints
// are valid for use in creating an instance in this environment.
func (env *environ) PrecheckInstance(series string, cons constraints.Value, placement string) error {
	if _, err := env.parsePlacement(placement); err != nil {
		return errors.Trace(err)
	}

	if cons.HasInstanceType() {
		if !checkInstanceType(cons) {
			return errors.Errorf("invalid GCE instance type %q", *cons.InstanceType)
		}
	}

	return nil
}

// SupportedArchitectures returns the image architectures which can
// be hosted by this environment.
func (env *environ) SupportedArchitectures() ([]string, error) {
	env.archLock.Lock()
	defer env.archLock.Unlock()

	if env.supportedArchitectures != nil {
		return env.supportedArchitectures, nil
	}

	archList, err := env.lookupArchitectures()
	if err != nil {
		return nil, errors.Trace(err)
	}
	env.supportedArchitectures = archList
	return archList, nil
}

var supportedArchitectures = common.SupportedArchitectures

func (env *environ) lookupArchitectures() ([]string, error) {
	// Create a filter to get all images from our region and for the
	// correct stream.
	cloudSpec, err := env.Region()
	if err != nil {
		return nil, errors.Trace(err)
	}
	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
		CloudSpec: cloudSpec,
		Stream:    env.Config().ImageStream(),
	})
	archList, err := supportedArchitectures(env, imageConstraint)
	return archList, errors.Trace(err)
}

var unsupportedConstraints = []string{
	constraints.Tags,
	constraints.Networks,
}

// instanceTypeConstraints defines the fields defined on each of the
// instance types.  See instancetypes.go.
var instanceTypeConstraints = []string{
	constraints.Arch, // Arches
	constraints.CpuCores,
	constraints.CpuPower,
	constraints.Mem,
	constraints.Container, // VirtType
}

// ConstraintsValidator returns a Validator value which is used to
// validate and merge constraints.
func (env *environ) ConstraintsValidator() (constraints.Validator, error) {
	validator := constraints.NewValidator()

	// conflicts

	// TODO(ericsnow) Are these correct?
	validator.RegisterConflicts(
		[]string{constraints.InstanceType},
		instanceTypeConstraints,
	)

	// unsupported

	validator.RegisterUnsupported(unsupportedConstraints)

	// vocab

	supportedArches, err := env.SupportedArchitectures()
	if err != nil {
		return nil, errors.Trace(err)
	}
	validator.RegisterVocabulary(constraints.Arch, supportedArches)

	instTypeNames := make([]string, len(allInstanceTypes))
	for i, itype := range allInstanceTypes {
		instTypeNames[i] = itype.Name
	}
	validator.RegisterVocabulary(constraints.InstanceType, instTypeNames)

	validator.RegisterVocabulary(constraints.Container, []string{vtype})

	return validator, nil
}

// environ provides SupportsUnitPlacement (a method of the
// state.EnvironCapatability interface) by embedding
// common.SupportsUnitPlacementPolicy.

// SupportNetworks returns whether the environment has support to
// specify networks for services and machines.
func (env *environ) SupportNetworks() bool {
	return false
}

// SupportAddressAllocation takes a network.Id and returns a bool
// and an error. The bool indicates whether that network supports
// static ip address allocation.
func (env *environ) SupportAddressAllocation(netID network.Id) (bool, error) {
	return false, nil
}
<|endoftext|>


<commit_before>package libkv

import (
	expire "github.com/jeffjen/go-libkv/timer"
	"sync"
	"time"
)

const (
	GET int = iota
	SET
	DEL
	EXPIRE
)

type Event struct {
	Action int
	Iden   string
}

type avent struct {
	c chan<- *Event
	h chan struct{}
}

type kv_avent struct {
	sync.RWMutex

	src     chan *Event
	list    map[int64]*avent
	halt    chan struct{}
	counter int64
}

type thing struct {
	x interface{}
	t *time.Time
}

type Store struct {
	sync.RWMutex

	e *expire.Timer
	m struct {
		store map[string]thing
		index map[string]int64
	}
	s *kv_avent
}

func (s *Store) init() {
	s.e.Tic()
	go func() {
		for yay := true; yay; {
			select {
			case <-s.s.halt:
				yay = false
			case one_event := <-s.s.src:
				s.s.Lock()
				for idx, ev := range s.s.list {
					select {
					case <-ev.h:
						delete(s.s.list, idx)
					case ev.c <- one_event:
						continue
					}
				}
				s.s.Unlock()
			}
		}
	}()
}

func NewStore() (s *Store) {
	s = &Store{
		e: expire.NewTimer(),
		m: struct {
			store map[string]thing
			index map[string]int64
		}{
			store: make(map[string]thing),
			index: make(map[string]int64),
		},
		s: &kv_avent{
			src:  make(chan *Event, 1),
			list: make(map[int64]*avent),
			halt: make(chan struct{}),
		},
	}
	s.init()
	return
}

func (s *Store) Close() {
	close(s.s.halt)
	s.e.Toc()
}

func (s *Store) del(iden string) {
	s.Lock()
	defer s.Unlock()
	delete(s.m.store, iden)
	delete(s.m.index, iden)
	s.s.src <- &Event{DEL, iden}
}

func (s *Store) set(iden string, x interface{}, exp *time.Time) bool {
	s.Lock()
	defer s.Unlock()
	if idx, ok := s.m.index[iden]; ok {
		s.e.Cancel(idx)
	}
	s.m.store[iden] = thing{x: x, t: exp}
	if exp != nil {
		id := iden
		s.m.index[iden] = s.e.SchedFunc(*exp, func() {
			s.del(id)
			s.s.src <- &Event{EXPIRE, iden}
		})
	}
	s.s.src <- &Event{SET, iden}
	return true
}

func (s *Store) Set(iden string, x interface{}) bool {
	return s.set(iden, x, nil)
}

func (s *Store) Setexp(iden string, x interface{}, exp time.Time) bool {
	return s.set(iden, x, &exp)
}

func (s *Store) Get(iden string) (x interface{}) {
	s.RLock()
	defer s.RUnlock()
	if obj, ok := s.m.store[iden]; ok {
		x = obj.x
		s.s.src <- &Event{GET, iden}
	}
	return
}

func (s *Store) Getset(iden string, x interface{}) (y interface{}) {
	y = s.Get(iden)
	s.Set(iden, x)
	return
}

func (s *Store) TTL(iden string) (in time.Duration) {
	s.RLock()
	defer s.RUnlock()
	if obj, ok := s.m.store[iden]; ok && obj.t != nil {
		in = obj.t.Sub(time.Now())
		if in < 0 {
			in = time.Duration(0)
		}
	}
	return
}

func (s *Store) Expire(iden string, exp time.Time) bool {
	x := s.Get(iden)
	if x != nil {
		return s.Setexp(iden, x, exp)
	}
	return false
}

func (s *Store) Del(iden string) {
	s.del(iden)
}

func (s *Store) register(inn *avent) {
	s.s.Lock()
	defer s.s.Unlock()
	r := s.s.counter
	s.s.counter = s.s.counter + 1
	s.s.list[r] = inn
}

func (s *Store) Watch(stop chan struct{}) <-chan *Event {
	output := make(chan *Event, 8)
	go func() {
		defer close(output)
		incoming := make(chan *Event, 1)
		end := make(chan struct{})
		s.register(&avent{incoming, end})
		defer close(end)
		for yay := true; yay; {
			select {
			case <-stop:
				yay = false
			case ev := <-incoming:
				select {
				default:
					yay = false
				case output <- ev:
					continue
				}
			}
		}
	}()
	return output
}

func (s *Store) List() (items []string) {
	s.RLock()
	defer s.RUnlock()
	items = make([]string, len(s.m.store))
	idx := 0
	for k, _ := range s.m.store {
		items[idx] = k
		idx++
	}
	return
}

func (s *Store) Listexp() (items []string) {
	s.RLock()
	defer s.RUnlock()
	items = make([]string, len(s.m.index))
	idx := 0
	for k, _ := range s.m.index {
		items[idx] = k
		idx++
	}
	return
}
<commit_msg>UPDATE: added godoc for go-libkv/libkv<commit_after>// Package libkv provides key value storage for embedded go application.
package libkv

import (
	expire "github.com/jeffjen/go-libkv/timer"
	"sync"
	"time"
)

const (
	GET int = iota
	SET
	DEL
	EXPIRE
)

// Event delivers keyspace changes to registerd Watcher
type Event struct {
	Action int    `desc: action taken e.g. GET, SET, EXPIRE`
	Iden   string `desc: key that was affected`
}

// avent is a control object for Store event hub to deliver Event
type avent struct {
	c chan<- *Event   `desc: channel to send the event into`
	h <-chan struct{} `desc: control channel to indicate unregister`
}

// kv_avent is holds metadata about how to distribute keyspace Event
type kv_avent struct {
	sync.RWMutex

	src     chan *Event      `desc: Event source channel from keyspace`
	list    map[int64]*avent `desc: list of registerd party for Event`
	halt    chan struct{}    `desc: control channel to stop Event distribution`
	counter int64            `desc: counter to tag Watcher`
}

// thing is an object stored in Store
type thing struct {
	x interface{} `desc: the object to store`
	t *time.Time  `desc: the expiration date on this thing`
}

// store is the actual KV store
type store struct {
	store map[string]thing `desc: the actual KV store`
	index map[string]int64 `desc: the index mapper for key to schdule index`
}

// Store is a simple key value in memory storage.
// Upon initialization, Store provides general get, set, del, list operations,
// as well as the ability to expire a key and watch a key change.
type Store struct {
	sync.RWMutex

	e *expire.Timer `desc: scheduler for keyspace expiration`
	m *store        `desc: the actual store`
	s *kv_avent     `desc: keyspace event hub`
}

// init setups the key value storage.
// Upon initialization, the Store spawns expiration scheduler and keyspace event hub.
func (s *Store) init() (ok <-chan bool) {
	ack := make(chan bool, 1)
	s.e.Tic()
	go func() {
		ack <- true
		for yay := true; yay; {
			select {
			case <-s.s.halt:
				yay = false
			case one_event := <-s.s.src:
				s.s.Lock()
				for idx, ev := range s.s.list {
					select {
					case <-ev.h:
						delete(s.s.list, idx)
					case ev.c <- one_event:
						continue
					}
				}
				s.s.Unlock()
			}
		}
	}()
	return ack
}

// NewStore creates a Store object.
// Store object is fully initialzed upon creation.
func NewStore() (s *Store) {
	s = &Store{
		e: expire.NewTimer(),
		m: &store{
			store: make(map[string]thing),
			index: make(map[string]int64),
		},
		s: &kv_avent{
			src:  make(chan *Event, 1),
			list: make(map[int64]*avent),
			halt: make(chan struct{}),
		},
	}
	<-s.init() // make sure both scheduler and event hub is running
	return
}

// Close stops the Store scheduler and event hub.
func (s *Store) Close() {
	close(s.s.halt)
	s.e.Toc()
}

// del removes an item from the Store keyspace
func (s *Store) del(iden string) {
	s.Lock()
	defer s.Unlock()
	delete(s.m.store, iden)
	delete(s.m.index, iden)
	s.s.src <- &Event{DEL, iden}
}

// set adds an item to the Store keyspace; sets expiration handler when
// appropriate
func (s *Store) set(iden string, x interface{}, exp *time.Time) bool {
	s.Lock()
	defer s.Unlock()
	if idx, ok := s.m.index[iden]; ok {
		s.e.Cancel(idx)
	}
	s.m.store[iden] = thing{x: x, t: exp}
	if exp != nil {
		id := iden
		s.m.index[iden] = s.e.SchedFunc(*exp, func() {
			s.del(id)
			s.s.src <- &Event{EXPIRE, iden}
		})
	}
	s.s.src <- &Event{SET, iden}
	return true
}

// Set puts an aribtrary item x into Store identified by iden
func (s *Store) Set(iden string, x interface{}) bool {
	return s.set(iden, x, nil)
}

// Set puts an aribtrary item x into Store identified by iden to be expired at
// exp
func (s *Store) Setexp(iden string, x interface{}, exp time.Time) bool {
	return s.set(iden, x, &exp)
}

// Get retrieves an item x identified by iden
func (s *Store) Get(iden string) (x interface{}) {
	s.RLock()
	defer s.RUnlock()
	if obj, ok := s.m.store[iden]; ok {
		x = obj.x
		s.s.src <- &Event{GET, iden}
	}
	return
}

// Getset retrieves an item y identified by iden and replace it with item x
func (s *Store) Getset(iden string, x interface{}) (y interface{}) {
	y = s.Get(iden)
	s.Set(iden, x)
	return
}

// TTL reports the life time left on the item identified by iden
func (s *Store) TTL(iden string) (in time.Duration) {
	s.RLock()
	defer s.RUnlock()
	if obj, ok := s.m.store[iden]; ok && obj.t != nil {
		in = obj.t.Sub(time.Now())
		if in < 0 {
			in = time.Duration(0)
		}
	}
	return
}

// Expire puts item identified by iden to expire at exp
func (s *Store) Expire(iden string, exp time.Time) bool {
	x := s.Get(iden)
	if x != nil {
		return s.Setexp(iden, x, exp)
	}
	return false
}

// Del removes the item identified by iden from Store
func (s *Store) Del(iden string) {
	s.del(iden)
}

// register takes a control object avent and place it into the Wather list in
// keyspace event hub.
func (s *Store) register(inn *avent) {
	s.s.Lock()
	defer s.s.Unlock()
	r := s.s.counter
	s.s.counter = s.s.counter + 1
	s.s.list[r] = inn
}

// Watch provides interested party to monitor keyspace changes.
// A Watcher (caller of Watch function) provides a stopping condition, and gets
// a channel for future Event in keyspace.
// Stop a Watcher by closeing the stop channel
func (s *Store) Watch(stop <-chan struct{}) <-chan *Event {
	output := make(chan *Event, 8)
	go func() {
		defer close(output)
		incoming := make(chan *Event, 1)
		end := make(chan struct{})
		s.register(&avent{incoming, end})
		defer close(end)
		for yay := true; yay; {
			select {
			case <-stop:
				yay = false
			case ev := <-incoming:
				select {
				default:
					yay = false
				case output <- ev:
					continue
				}
			}
		}
	}()
	return output
}

// List retrieves the full list of item key in Store.
func (s *Store) List() (items []string) {
	s.RLock()
	defer s.RUnlock()
	items = make([]string, len(s.m.store))
	idx := 0
	for k, _ := range s.m.store {
		items[idx] = k
		idx++
	}
	return
}

// Listexp retrieves the full list of item key in Store that has an expiration.
func (s *Store) Listexp() (items []string) {
	s.RLock()
	defer s.RUnlock()
	items = make([]string, len(s.m.index))
	idx := 0
	for k, _ := range s.m.index {
		items[idx] = k
		idx++
	}
	return
}
<|endoftext|>


<commit_before>package inbound

//go:generate go run $GOPATH/src/v2ray.com/core/common/errors/errorgen/main.go -pkg inbound -path Proxy,VMess,Inbound

import (
	"context"
	"io"
	"strings"
	"sync"
	"time"

	"v2ray.com/core"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/session"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/common/task"
	"v2ray.com/core/common/uuid"
	"v2ray.com/core/proxy/vmess"
	"v2ray.com/core/proxy/vmess/encoding"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/pipe"
)

type userByEmail struct {
	sync.Mutex
	cache           map[string]*protocol.MemoryUser
	defaultLevel    uint32
	defaultAlterIDs uint16
}

func newUserByEmail(config *DefaultConfig) *userByEmail {
	return &userByEmail{
		cache:           make(map[string]*protocol.MemoryUser),
		defaultLevel:    config.Level,
		defaultAlterIDs: uint16(config.AlterId),
	}
}

func (v *userByEmail) addNoLock(u *protocol.MemoryUser) bool {
	email := strings.ToLower(u.Email)
	user, found := v.cache[email]
	if found {
		return false
	}
	v.cache[email] = user
	return true
}

func (v *userByEmail) Add(u *protocol.MemoryUser) bool {
	v.Lock()
	defer v.Unlock()

	return v.addNoLock(u)
}

func (v *userByEmail) Get(email string) (*protocol.MemoryUser, bool) {
	email = strings.ToLower(email)

	v.Lock()
	defer v.Unlock()

	user, found := v.cache[email]
	if !found {
		id := uuid.New()
		rawAccount := &vmess.Account{
			Id:      id.String(),
			AlterId: uint32(v.defaultAlterIDs),
		}
		account, err := rawAccount.AsAccount()
		common.Must(err)
		user = &protocol.MemoryUser{
			Level:   v.defaultLevel,
			Email:   email,
			Account: account,
		}
		v.cache[email] = user
	}
	return user, found
}

func (v *userByEmail) Remove(email string) bool {
	email = strings.ToLower(email)

	v.Lock()
	defer v.Unlock()

	if _, found := v.cache[email]; !found {
		return false
	}
	delete(v.cache, email)
	return true
}

// Handler is an inbound connection handler that handles messages in VMess protocol.
type Handler struct {
	policyManager         core.PolicyManager
	inboundHandlerManager core.InboundHandlerManager
	clients               *vmess.TimedUserValidator
	usersByEmail          *userByEmail
	detours               *DetourConfig
	sessionHistory        *encoding.SessionHistory
	secure                bool
}

// New creates a new VMess inbound handler.
func New(ctx context.Context, config *Config) (*Handler, error) {
	v := core.MustFromContext(ctx)
	handler := &Handler{
		policyManager:         v.PolicyManager(),
		inboundHandlerManager: v.InboundHandlerManager(),
		clients:               vmess.NewTimedUserValidator(protocol.DefaultIDHash),
		detours:               config.Detour,
		usersByEmail:          newUserByEmail(config.GetDefaultValue()),
		sessionHistory:        encoding.NewSessionHistory(),
		secure:                config.SecureEncryptionOnly,
	}

	for _, user := range config.User {
		mUser, err := user.ToMemoryUser()
		if err != nil {
			return nil, newError("failed to get VMess user").Base(err)
		}

		if err := handler.AddUser(ctx, mUser); err != nil {
			return nil, newError("failed to initiate user").Base(err)
		}
	}

	return handler, nil
}

// Close implements common.Closable.
func (h *Handler) Close() error {
	return task.Run(
		task.SequentialAll(
			task.Close(h.clients), task.Close(h.sessionHistory), task.Close(h.usersByEmail)))()
}

// Network implements proxy.Inbound.Network().
func (*Handler) Network() net.NetworkList {
	return net.NetworkList{
		Network: []net.Network{net.Network_TCP},
	}
}

func (h *Handler) GetUser(email string) *protocol.MemoryUser {
	user, existing := h.usersByEmail.Get(email)
	if !existing {
		h.clients.Add(user)
	}
	return user
}

func (h *Handler) AddUser(ctx context.Context, user *protocol.MemoryUser) error {
	if len(user.Email) > 0 && !h.usersByEmail.Add(user) {
		return newError("User ", user.Email, " already exists.")
	}
	return h.clients.Add(user)
}

func (h *Handler) RemoveUser(ctx context.Context, email string) error {
	if len(email) == 0 {
		return newError("Email must not be empty.")
	}
	if !h.usersByEmail.Remove(email) {
		return newError("User ", email, " not found.")
	}
	h.clients.Remove(email)
	return nil
}

func transferResponse(timer signal.ActivityUpdater, session *encoding.ServerSession, request *protocol.RequestHeader, response *protocol.ResponseHeader, input buf.Reader, output *buf.BufferedWriter) error {
	session.EncodeResponseHeader(response, output)

	bodyWriter := session.EncodeResponseBody(request, output)

	{
		// Optimize for small response packet
		data, err := input.ReadMultiBuffer()
		if err != nil {
			return err
		}

		if err := bodyWriter.WriteMultiBuffer(data); err != nil {
			return err
		}
	}

	if err := output.SetBuffered(false); err != nil {
		return err
	}

	if err := buf.Copy(input, bodyWriter, buf.UpdateActivity(timer)); err != nil {
		return err
	}

	if request.Option.Has(protocol.RequestOptionChunkStream) {
		if err := bodyWriter.WriteMultiBuffer(buf.MultiBuffer{}); err != nil {
			return err
		}
	}

	return nil
}

func isInecureEncryption(s protocol.SecurityType) bool {
	return s == protocol.SecurityType_NONE || s == protocol.SecurityType_LEGACY || s == protocol.SecurityType_UNKNOWN
}

// Process implements proxy.Inbound.Process().
func (h *Handler) Process(ctx context.Context, network net.Network, connection internet.Connection, dispatcher core.Dispatcher) error {
	sessionPolicy := h.policyManager.ForLevel(0)
	if err := connection.SetReadDeadline(time.Now().Add(sessionPolicy.Timeouts.Handshake)); err != nil {
		return newError("unable to set read deadline").Base(err).AtWarning()
	}

	reader := &buf.BufferedReader{Reader: buf.NewReader(connection)}
	svrSession := encoding.NewServerSession(h.clients, h.sessionHistory)
	request, err := svrSession.DecodeRequestHeader(reader)

	if err != nil {
		if errors.Cause(err) != io.EOF {
			log.Record(&log.AccessMessage{
				From:   connection.RemoteAddr(),
				To:     "",
				Status: log.AccessRejected,
				Reason: err,
			})
			err = newError("invalid request from ", connection.RemoteAddr()).Base(err).AtInfo()
		}
		return err
	}

	if h.secure && isInecureEncryption(request.Security) {
		log.Record(&log.AccessMessage{
			From:   connection.RemoteAddr(),
			To:     "",
			Status: log.AccessRejected,
			Reason: "Insecure encryption",
		})
		return newError("client is using insecure encryption: ", request.Security)
	}

	if request.Command != protocol.RequestCommandMux {
		log.Record(&log.AccessMessage{
			From:   connection.RemoteAddr(),
			To:     request.Destination(),
			Status: log.AccessAccepted,
			Reason: "",
		})
	}

	newError("received request for ", request.Destination()).WriteToLog(session.ExportIDToError(ctx))

	if err := connection.SetReadDeadline(time.Time{}); err != nil {
		newError("unable to set back read deadline").Base(err).WriteToLog(session.ExportIDToError(ctx))
	}

	sessionPolicy = h.policyManager.ForLevel(request.User.Level)
	ctx = protocol.ContextWithUser(ctx, request.User)

	ctx, cancel := context.WithCancel(ctx)
	timer := signal.CancelAfterInactivity(ctx, cancel, sessionPolicy.Timeouts.ConnectionIdle)

	ctx = core.ContextWithBufferPolicy(ctx, sessionPolicy.Buffer)
	link, err := dispatcher.Dispatch(ctx, request.Destination())
	if err != nil {
		return newError("failed to dispatch request to ", request.Destination()).Base(err)
	}

	requestDone := func() error {
		defer timer.SetTimeout(sessionPolicy.Timeouts.DownlinkOnly)

		bodyReader := svrSession.DecodeRequestBody(request, reader)
		if err := buf.Copy(bodyReader, link.Writer, buf.UpdateActivity(timer)); err != nil {
			return newError("failed to transfer request").Base(err)
		}
		return nil
	}

	responseDone := func() error {
		writer := buf.NewBufferedWriter(buf.NewWriter(connection))
		defer writer.Flush()
		defer timer.SetTimeout(sessionPolicy.Timeouts.UplinkOnly)

		response := &protocol.ResponseHeader{
			Command: h.generateCommand(ctx, request),
		}
		return transferResponse(timer, svrSession, request, response, link.Reader, writer)
	}

	var requestDonePost = task.Single(requestDone, task.OnSuccess(task.Close(link.Writer)))
	if err := task.Run(task.WithContext(ctx), task.Parallel(requestDonePost, responseDone))(); err != nil {
		pipe.CloseError(link.Reader)
		pipe.CloseError(link.Writer)
		return newError("connection ends").Base(err)
	}

	return nil
}

func (h *Handler) generateCommand(ctx context.Context, request *protocol.RequestHeader) protocol.ResponseCommand {
	if h.detours != nil {
		tag := h.detours.To
		if h.inboundHandlerManager != nil {
			handler, err := h.inboundHandlerManager.GetHandler(ctx, tag)
			if err != nil {
				newError("failed to get detour handler: ", tag).Base(err).AtWarning().WriteToLog(session.ExportIDToError(ctx))
				return nil
			}
			proxyHandler, port, availableMin := handler.GetRandomInboundProxy()
			inboundHandler, ok := proxyHandler.(*Handler)
			if ok && inboundHandler != nil {
				if availableMin > 255 {
					availableMin = 255
				}

				newError("pick detour handler for port ", port, " for ", availableMin, " minutes.").AtDebug().WriteToLog(session.ExportIDToError(ctx))
				user := inboundHandler.GetUser(request.User.Email)
				if user == nil {
					return nil
				}
				account := user.Account.(*vmess.InternalAccount)
				return &protocol.CommandSwitchAccount{
					Port:     port,
					ID:       account.ID.UUID(),
					AlterIds: uint16(len(account.AlterIDs)),
					Level:    user.Level,
					ValidMin: byte(availableMin),
				}
			}
		}
	}

	return nil
}

func init() {
	common.Must(common.RegisterConfig((*Config)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return New(ctx, config.(*Config))
	}))
}
<commit_msg>fix a typo<commit_after>package inbound

//go:generate go run $GOPATH/src/v2ray.com/core/common/errors/errorgen/main.go -pkg inbound -path Proxy,VMess,Inbound

import (
	"context"
	"io"
	"strings"
	"sync"
	"time"

	"v2ray.com/core"
	"v2ray.com/core/common"
	"v2ray.com/core/common/buf"
	"v2ray.com/core/common/errors"
	"v2ray.com/core/common/log"
	"v2ray.com/core/common/net"
	"v2ray.com/core/common/protocol"
	"v2ray.com/core/common/session"
	"v2ray.com/core/common/signal"
	"v2ray.com/core/common/task"
	"v2ray.com/core/common/uuid"
	"v2ray.com/core/proxy/vmess"
	"v2ray.com/core/proxy/vmess/encoding"
	"v2ray.com/core/transport/internet"
	"v2ray.com/core/transport/pipe"
)

type userByEmail struct {
	sync.Mutex
	cache           map[string]*protocol.MemoryUser
	defaultLevel    uint32
	defaultAlterIDs uint16
}

func newUserByEmail(config *DefaultConfig) *userByEmail {
	return &userByEmail{
		cache:           make(map[string]*protocol.MemoryUser),
		defaultLevel:    config.Level,
		defaultAlterIDs: uint16(config.AlterId),
	}
}

func (v *userByEmail) addNoLock(u *protocol.MemoryUser) bool {
	email := strings.ToLower(u.Email)
	user, found := v.cache[email]
	if found {
		return false
	}
	v.cache[email] = user
	return true
}

func (v *userByEmail) Add(u *protocol.MemoryUser) bool {
	v.Lock()
	defer v.Unlock()

	return v.addNoLock(u)
}

func (v *userByEmail) Get(email string) (*protocol.MemoryUser, bool) {
	email = strings.ToLower(email)

	v.Lock()
	defer v.Unlock()

	user, found := v.cache[email]
	if !found {
		id := uuid.New()
		rawAccount := &vmess.Account{
			Id:      id.String(),
			AlterId: uint32(v.defaultAlterIDs),
		}
		account, err := rawAccount.AsAccount()
		common.Must(err)
		user = &protocol.MemoryUser{
			Level:   v.defaultLevel,
			Email:   email,
			Account: account,
		}
		v.cache[email] = user
	}
	return user, found
}

func (v *userByEmail) Remove(email string) bool {
	email = strings.ToLower(email)

	v.Lock()
	defer v.Unlock()

	if _, found := v.cache[email]; !found {
		return false
	}
	delete(v.cache, email)
	return true
}

// Handler is an inbound connection handler that handles messages in VMess protocol.
type Handler struct {
	policyManager         core.PolicyManager
	inboundHandlerManager core.InboundHandlerManager
	clients               *vmess.TimedUserValidator
	usersByEmail          *userByEmail
	detours               *DetourConfig
	sessionHistory        *encoding.SessionHistory
	secure                bool
}

// New creates a new VMess inbound handler.
func New(ctx context.Context, config *Config) (*Handler, error) {
	v := core.MustFromContext(ctx)
	handler := &Handler{
		policyManager:         v.PolicyManager(),
		inboundHandlerManager: v.InboundHandlerManager(),
		clients:               vmess.NewTimedUserValidator(protocol.DefaultIDHash),
		detours:               config.Detour,
		usersByEmail:          newUserByEmail(config.GetDefaultValue()),
		sessionHistory:        encoding.NewSessionHistory(),
		secure:                config.SecureEncryptionOnly,
	}

	for _, user := range config.User {
		mUser, err := user.ToMemoryUser()
		if err != nil {
			return nil, newError("failed to get VMess user").Base(err)
		}

		if err := handler.AddUser(ctx, mUser); err != nil {
			return nil, newError("failed to initiate user").Base(err)
		}
	}

	return handler, nil
}

// Close implements common.Closable.
func (h *Handler) Close() error {
	return task.Run(
		task.SequentialAll(
			task.Close(h.clients), task.Close(h.sessionHistory), task.Close(h.usersByEmail)))()
}

// Network implements proxy.Inbound.Network().
func (*Handler) Network() net.NetworkList {
	return net.NetworkList{
		Network: []net.Network{net.Network_TCP},
	}
}

func (h *Handler) GetUser(email string) *protocol.MemoryUser {
	user, existing := h.usersByEmail.Get(email)
	if !existing {
		h.clients.Add(user)
	}
	return user
}

func (h *Handler) AddUser(ctx context.Context, user *protocol.MemoryUser) error {
	if len(user.Email) > 0 && !h.usersByEmail.Add(user) {
		return newError("User ", user.Email, " already exists.")
	}
	return h.clients.Add(user)
}

func (h *Handler) RemoveUser(ctx context.Context, email string) error {
	if len(email) == 0 {
		return newError("Email must not be empty.")
	}
	if !h.usersByEmail.Remove(email) {
		return newError("User ", email, " not found.")
	}
	h.clients.Remove(email)
	return nil
}

func transferResponse(timer signal.ActivityUpdater, session *encoding.ServerSession, request *protocol.RequestHeader, response *protocol.ResponseHeader, input buf.Reader, output *buf.BufferedWriter) error {
	session.EncodeResponseHeader(response, output)

	bodyWriter := session.EncodeResponseBody(request, output)

	{
		// Optimize for small response packet
		data, err := input.ReadMultiBuffer()
		if err != nil {
			return err
		}

		if err := bodyWriter.WriteMultiBuffer(data); err != nil {
			return err
		}
	}

	if err := output.SetBuffered(false); err != nil {
		return err
	}

	if err := buf.Copy(input, bodyWriter, buf.UpdateActivity(timer)); err != nil {
		return err
	}

	if request.Option.Has(protocol.RequestOptionChunkStream) {
		if err := bodyWriter.WriteMultiBuffer(buf.MultiBuffer{}); err != nil {
			return err
		}
	}

	return nil
}

func isInsecureEncryption(s protocol.SecurityType) bool {
	return s == protocol.SecurityType_NONE || s == protocol.SecurityType_LEGACY || s == protocol.SecurityType_UNKNOWN
}

// Process implements proxy.Inbound.Process().
func (h *Handler) Process(ctx context.Context, network net.Network, connection internet.Connection, dispatcher core.Dispatcher) error {
	sessionPolicy := h.policyManager.ForLevel(0)
	if err := connection.SetReadDeadline(time.Now().Add(sessionPolicy.Timeouts.Handshake)); err != nil {
		return newError("unable to set read deadline").Base(err).AtWarning()
	}

	reader := &buf.BufferedReader{Reader: buf.NewReader(connection)}
	svrSession := encoding.NewServerSession(h.clients, h.sessionHistory)
	request, err := svrSession.DecodeRequestHeader(reader)

	if err != nil {
		if errors.Cause(err) != io.EOF {
			log.Record(&log.AccessMessage{
				From:   connection.RemoteAddr(),
				To:     "",
				Status: log.AccessRejected,
				Reason: err,
			})
			err = newError("invalid request from ", connection.RemoteAddr()).Base(err).AtInfo()
		}
		return err
	}

	if h.secure && isInsecureEncryption(request.Security) {
		log.Record(&log.AccessMessage{
			From:   connection.RemoteAddr(),
			To:     "",
			Status: log.AccessRejected,
			Reason: "Insecure encryption",
		})
		return newError("client is using insecure encryption: ", request.Security)
	}

	if request.Command != protocol.RequestCommandMux {
		log.Record(&log.AccessMessage{
			From:   connection.RemoteAddr(),
			To:     request.Destination(),
			Status: log.AccessAccepted,
			Reason: "",
		})
	}

	newError("received request for ", request.Destination()).WriteToLog(session.ExportIDToError(ctx))

	if err := connection.SetReadDeadline(time.Time{}); err != nil {
		newError("unable to set back read deadline").Base(err).WriteToLog(session.ExportIDToError(ctx))
	}

	sessionPolicy = h.policyManager.ForLevel(request.User.Level)
	ctx = protocol.ContextWithUser(ctx, request.User)

	ctx, cancel := context.WithCancel(ctx)
	timer := signal.CancelAfterInactivity(ctx, cancel, sessionPolicy.Timeouts.ConnectionIdle)

	ctx = core.ContextWithBufferPolicy(ctx, sessionPolicy.Buffer)
	link, err := dispatcher.Dispatch(ctx, request.Destination())
	if err != nil {
		return newError("failed to dispatch request to ", request.Destination()).Base(err)
	}

	requestDone := func() error {
		defer timer.SetTimeout(sessionPolicy.Timeouts.DownlinkOnly)

		bodyReader := svrSession.DecodeRequestBody(request, reader)
		if err := buf.Copy(bodyReader, link.Writer, buf.UpdateActivity(timer)); err != nil {
			return newError("failed to transfer request").Base(err)
		}
		return nil
	}

	responseDone := func() error {
		writer := buf.NewBufferedWriter(buf.NewWriter(connection))
		defer writer.Flush()
		defer timer.SetTimeout(sessionPolicy.Timeouts.UplinkOnly)

		response := &protocol.ResponseHeader{
			Command: h.generateCommand(ctx, request),
		}
		return transferResponse(timer, svrSession, request, response, link.Reader, writer)
	}

	var requestDonePost = task.Single(requestDone, task.OnSuccess(task.Close(link.Writer)))
	if err := task.Run(task.WithContext(ctx), task.Parallel(requestDonePost, responseDone))(); err != nil {
		pipe.CloseError(link.Reader)
		pipe.CloseError(link.Writer)
		return newError("connection ends").Base(err)
	}

	return nil
}

func (h *Handler) generateCommand(ctx context.Context, request *protocol.RequestHeader) protocol.ResponseCommand {
	if h.detours != nil {
		tag := h.detours.To
		if h.inboundHandlerManager != nil {
			handler, err := h.inboundHandlerManager.GetHandler(ctx, tag)
			if err != nil {
				newError("failed to get detour handler: ", tag).Base(err).AtWarning().WriteToLog(session.ExportIDToError(ctx))
				return nil
			}
			proxyHandler, port, availableMin := handler.GetRandomInboundProxy()
			inboundHandler, ok := proxyHandler.(*Handler)
			if ok && inboundHandler != nil {
				if availableMin > 255 {
					availableMin = 255
				}

				newError("pick detour handler for port ", port, " for ", availableMin, " minutes.").AtDebug().WriteToLog(session.ExportIDToError(ctx))
				user := inboundHandler.GetUser(request.User.Email)
				if user == nil {
					return nil
				}
				account := user.Account.(*vmess.InternalAccount)
				return &protocol.CommandSwitchAccount{
					Port:     port,
					ID:       account.ID.UUID(),
					AlterIds: uint16(len(account.AlterIDs)),
					Level:    user.Level,
					ValidMin: byte(availableMin),
				}
			}
		}
	}

	return nil
}

func init() {
	common.Must(common.RegisterConfig((*Config)(nil), func(ctx context.Context, config interface{}) (interface{}, error) {
		return New(ctx, config.(*Config))
	}))
}
<|endoftext|>


<commit_before>// Reddit audiences crawler
// Rémy Mathieu © 2016

package app

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/remeh/reddit-audiences/db"

	"github.com/PuerkitoBio/goquery"
)

const (
	REDDIT_SUBREDDIT_URL           = "https://www.reddit.com/r/"
	MIN_SECONDS_BETWEEN_EACH_CRAWL = 1
)

var subredditsToCrawl chan string

func StartCrawlingJob(a *App) {
	subredditsToCrawl = make(chan string)

	// starts the worker
	go Worker(a)

	// starts the main loop
	// regularly feeding the worker.
	if a.Config.Crawl {
		log.Println("info: starts crawling job.")
		ticker := time.NewTicker(time.Second * 30)
		for range ticker.C {
			log.Println("info: crawling job is running.")
			Feeder(a)
		}
		ticker.Stop()
	}
}

// Worker is the routine dealing with the HTTP
// call + reading hte DOM.
func Worker(a *App) {
	for subreddit := range subredditsToCrawl {
		if audience, subscribers, articles, err := readDOMData(subreddit); err == nil {
			// store the audience+subscribers and update the last crawl time
			// ----------------------
			if err := a.DB().InsertAudienceValue(subreddit, audience, subscribers); err != nil {
				log.Println("err:", err.Error())
			} else {
				log.Printf("info: subreddit %s has %d active users (%d subscribers)\n", subreddit, audience, subscribers)
			}

			// store each articles if not already present at this rank
			// ----------------------
			if err := storeArticles(a, articles); err != nil {
				log.Printf("err: %s", err.Error())
			}
		} else if err != nil {
			log.Println("err:", err.Error())
		}

		r := time.Duration(((rand.Int() % 2) + MIN_SECONDS_BETWEEN_EACH_CRAWL)) * time.Second

		time.Sleep(r) // wait a bit before the next crawl
	}
}

// Feeder retrieves the audience of subreddits for which
// the last crawl time is more than some minutes.
func Feeder(a *App) {
	// crawl each subreddit each 5 minutes
	five := time.Minute * 5
	t := time.Now().Add(-five)
	subreddits, err := a.DB().FindSubredditsToCrawl(t)

	if err != nil {
		log.Printf("err: can't retrieve subreddits to crawl: %s\n", err.Error())
	}

	for _, subreddit := range subreddits {
		log.Println("info: crawling", subreddit)
		subredditsToCrawl <- subreddit
	}
}

// readDOMData gets the subreddit page on reddit
// and gets the current audience, the subscribers and
// the article infos from the DOM.
// NOTE(remy): we stop as soon as we have a DOM error because
// it has great chances that the full DOM is corrupted/not retrieved.
func readDOMData(subreddit string) (int64, int64, []db.Article, error) {
	var audience int64
	var subscribers int64
	var err error

	doc, err := getSubredditPage(REDDIT_SUBREDDIT_URL + subreddit)
	if err != nil {
		return 0, 0, nil, fmt.Errorf("while crawling %s: %s", subreddit, err.Error())
	}

	// audience
	// ----------------------

	s := doc.Find("p.users-online span.number").First()

	value := s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s audience: no text value in the dom node.", subreddit)
	}

	if audience, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// subscribers
	// ----------------------

	s = doc.Find("span.subscribers span.number").First()

	value = s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s subscribers: no text value in the dom node.", subreddit)
	}

	if subscribers, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// articles
	// ----------------------

	now := time.Now()
	articles := make([]db.Article, 0)
	s = doc.Find(".link").Each(func(i int, selec *goquery.Selection) {
		l := selec.Find("p.title a.title")
		title := l.First()
		link, _ := selec.Find("a.comments").Attr("href")
		externalLink, _ := l.Attr("href")
		strPos := selec.ChildrenFiltered(".rank").First()
		articleId, _ := selec.Attr("data-fullname")
		author, _ := selec.Attr("data-author")

		// remove the t[1-3]_ from the article id
		for i := 0; i < 4; i++ {
			articleId = strings.TrimPrefix(articleId, fmt.Sprintf("t%d_", i))
		}

		rank, err := strconv.Atoi(strPos.Text())
		if err != nil {
			rank = 0 // it's probably a promoted or stickied article
		}

		promoted := false
		if selec.HasClass("promoted") {
			promoted = true
		}

		sticky := false
		if selec.HasClass("stickied") {
			sticky = true
		}

		articles = append(articles, db.Article{
			Subreddit:           subreddit,
			ArticleId:           articleId,
			ArticleTitle:        title.Text(),
			ArticleLink:         link,
			ArticleExternalLink: externalLink,
			Author:              author,
			Rank:                rank,
			CrawlTime:           now,
			Promoted:            promoted,
			Sticky:              sticky,
		})
	})

	return audience, subscribers, articles, err
}

// storeArticles checks for each article if the
// info isn't already present in database, if not,
// it stores it. If changed, it also stores it.
func storeArticles(a *App, articles []db.Article) error {
	if len(articles) == 0 {
		return nil
	}

	for _, article := range articles {
		id, rank, err := a.DB().FindArticleLastState(article.Subreddit, article.ArticleId)
		if err != nil {
			return fmt.Errorf("while retrieving article last state: %s", err.Error())
		}

		// already stored at this rank
		if article.ArticleId == id && rank == article.Rank {
			continue
		}

		// not already store, do it now
		if _, err := a.DB().InsertArticle(article); err != nil {
			return err
		}
	}

	return nil
}

func getSubredditPage(url string) (*goquery.Document, error) {
	r, err := NewRequest(url)
	if err != nil {
		return nil, err
	}

	client := http.Client{}
	resp, err := client.Do(r)

	if err != nil {
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
		return nil, err
	}

	if resp.StatusCode != 200 {
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
		return nil, fmt.Errorf(resp.Status)
	}

	return goquery.NewDocumentFromResponse(resp)
}

func cleanInt(str string) (int64, error) {
	// sometimes it starts with ~
	if strings.HasPrefix(str, "~") {
		str = str[1:]
	}
	// , for thousands etc.
	str = strings.Replace(str, ",", "", -1)
	// finally trim
	str = strings.Trim(str, " ")

	return strconv.ParseInt(str, 10, 64)
}
<commit_msg>app: add a timeout of 10s on crawl http clients.<commit_after>// Reddit audiences crawler
// Rémy Mathieu © 2016

package app

import (
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/remeh/reddit-audiences/db"

	"github.com/PuerkitoBio/goquery"
)

const (
	REDDIT_SUBREDDIT_URL           = "https://www.reddit.com/r/"
	MIN_SECONDS_BETWEEN_EACH_CRAWL = 1
)

var subredditsToCrawl chan string

func StartCrawlingJob(a *App) {
	subredditsToCrawl = make(chan string)

	// starts the worker
	go Worker(a)

	// starts the main loop
	// regularly feeding the worker.
	if a.Config.Crawl {
		log.Println("info: starts crawling job.")
		ticker := time.NewTicker(time.Second * 30)
		for range ticker.C {
			log.Println("info: crawling job is running.")
			Feeder(a)
		}
		ticker.Stop()
	}
}

// Worker is the routine dealing with the HTTP
// call + reading hte DOM.
func Worker(a *App) {
	for subreddit := range subredditsToCrawl {
		if audience, subscribers, articles, err := readDOMData(subreddit); err == nil {
			// store the audience+subscribers and update the last crawl time
			// ----------------------
			if err := a.DB().InsertAudienceValue(subreddit, audience, subscribers); err != nil {
				log.Println("err:", err.Error())
			} else {
				log.Printf("info: subreddit %s has %d active users (%d subscribers)\n", subreddit, audience, subscribers)
			}

			// store each articles if not already present at this rank
			// ----------------------
			if err := storeArticles(a, articles); err != nil {
				log.Printf("err: %s", err.Error())
			}
		} else if err != nil {
			log.Println("err:", err.Error())
		}

		r := time.Duration(((rand.Int() % 2) + MIN_SECONDS_BETWEEN_EACH_CRAWL)) * time.Second

		time.Sleep(r) // wait a bit before the next crawl
	}
}

// Feeder retrieves the audience of subreddits for which
// the last crawl time is more than some minutes.
func Feeder(a *App) {
	// crawl each subreddit each 5 minutes
	five := time.Minute * 5
	t := time.Now().Add(-five)
	subreddits, err := a.DB().FindSubredditsToCrawl(t)

	if err != nil {
		log.Printf("err: can't retrieve subreddits to crawl: %s\n", err.Error())
	}

	for _, subreddit := range subreddits {
		log.Println("info: crawling", subreddit)
		subredditsToCrawl <- subreddit
	}
}

// readDOMData gets the subreddit page on reddit
// and gets the current audience, the subscribers and
// the article infos from the DOM.
// NOTE(remy): we stop as soon as we have a DOM error because
// it has great chances that the full DOM is corrupted/not retrieved.
func readDOMData(subreddit string) (int64, int64, []db.Article, error) {
	var audience int64
	var subscribers int64
	var err error

	doc, err := getSubredditPage(REDDIT_SUBREDDIT_URL + subreddit)
	if err != nil {
		return 0, 0, nil, fmt.Errorf("while crawling %s: %s", subreddit, err.Error())
	}

	// audience
	// ----------------------

	s := doc.Find("p.users-online span.number").First()

	value := s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s audience: no text value in the dom node.", subreddit)
	}

	if audience, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// subscribers
	// ----------------------

	s = doc.Find("span.subscribers span.number").First()

	value = s.Text()
	if len(value) == 0 {
		return 0, 0, nil, fmt.Errorf("can't retrieve subreddit %s subscribers: no text value in the dom node.", subreddit)
	}

	if subscribers, err = cleanInt(value); err != nil {
		return 0, 0, nil, err
	}

	// articles
	// ----------------------

	now := time.Now()
	articles := make([]db.Article, 0)
	s = doc.Find(".link").Each(func(i int, selec *goquery.Selection) {
		l := selec.Find("p.title a.title")
		title := l.First()
		link, _ := selec.Find("a.comments").Attr("href")
		externalLink, _ := l.Attr("href")
		strPos := selec.ChildrenFiltered(".rank").First()
		articleId, _ := selec.Attr("data-fullname")
		author, _ := selec.Attr("data-author")

		// remove the t[1-3]_ from the article id
		for i := 0; i < 4; i++ {
			articleId = strings.TrimPrefix(articleId, fmt.Sprintf("t%d_", i))
		}

		rank, err := strconv.Atoi(strPos.Text())
		if err != nil {
			rank = 0 // it's probably a promoted or stickied article
		}

		promoted := false
		if selec.HasClass("promoted") {
			promoted = true
		}

		sticky := false
		if selec.HasClass("stickied") {
			sticky = true
		}

		articles = append(articles, db.Article{
			Subreddit:           subreddit,
			ArticleId:           articleId,
			ArticleTitle:        title.Text(),
			ArticleLink:         link,
			ArticleExternalLink: externalLink,
			Author:              author,
			Rank:                rank,
			CrawlTime:           now,
			Promoted:            promoted,
			Sticky:              sticky,
		})
	})

	return audience, subscribers, articles, err
}

// storeArticles checks for each article if the
// info isn't already present in database, if not,
// it stores it. If changed, it also stores it.
func storeArticles(a *App, articles []db.Article) error {
	if len(articles) == 0 {
		return nil
	}

	for _, article := range articles {
		id, rank, err := a.DB().FindArticleLastState(article.Subreddit, article.ArticleId)
		if err != nil {
			return fmt.Errorf("while retrieving article last state: %s", err.Error())
		}

		// already stored at this rank
		if article.ArticleId == id && rank == article.Rank {
			continue
		}

		// not already store, do it now
		if _, err := a.DB().InsertArticle(article); err != nil {
			return err
		}
	}

	return nil
}

func getSubredditPage(url string) (*goquery.Document, error) {
	r, err := NewRequest(url)
	if err != nil {
		return nil, err
	}

	client := http.Client{
		Timeout: time.Second * 10,
	}
	resp, err := client.Do(r)

	if err != nil {
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
		return nil, err
	}

	if resp.StatusCode != 200 {
		if resp != nil && resp.Body != nil {
			resp.Body.Close()
		}
		return nil, fmt.Errorf(resp.Status)
	}

	return goquery.NewDocumentFromResponse(resp)
}

func cleanInt(str string) (int64, error) {
	// sometimes it starts with ~
	if strings.HasPrefix(str, "~") {
		str = str[1:]
	}
	// , for thousands etc.
	str = strings.Replace(str, ",", "", -1)
	// finally trim
	str = strings.Trim(str, " ")

	return strconv.ParseInt(str, 10, 64)
}
<|endoftext|>


<commit_before>package archive

import (
	// Note - I'm using a patched version of the google-api-go-client library
	// because of this bug -
	// https://code.google.com/p/google-api-go-client/issues/detail?id=52
	bigquery "code.google.com/p/ox-google-api-go-client/bigquery/v2"
	"github.com/getlantern/statshub/statshub"
	"github.com/oxtoacart/oauther/oauth"
	"log"
	"os"
	"sort"
	"time"
)

const (
	OAUTH_CONFIG = "OAUTH_CONFIG"

	TIMESTAMP = "TIMESTAMP"
	RECORD    = "RECORD"
	INTEGER   = "INTEGER"
	STRING    = "STRING"
	global    = "global"
	counter   = "counter"
	gauge     = "gauge"
	_ts       = "_ts"
	_dim      = "_dim"
)

// StatsTable is a table that holds statistics from statshub
type StatsTable struct {
	service   *bigquery.Service
	tables    *bigquery.TablesService
	tabledata *bigquery.TabledataService
	dataset   *bigquery.Dataset
	table     *bigquery.Table
}

func NewStatsTable(projectId string, datasetId string, tableId string) (statsTable *StatsTable, err error) {
	statsTable = &StatsTable{
		table: &bigquery.Table{
			TableReference: &bigquery.TableReference{
				ProjectId: projectId,
				DatasetId: datasetId,
				TableId:   tableId,
			},
		},
	}
	var oauther *oauth.OAuther
	if oauther, err = oauth.FromJSON([]byte(os.Getenv(OAUTH_CONFIG))); err != nil {
		return
	} else if statsTable.service, err = bigquery.New(oauther.Transport().Client()); err != nil {
		return
	} else {
		statsTable.tables = bigquery.NewTablesService(statsTable.service)
		statsTable.tabledata = bigquery.NewTabledataService(statsTable.service)
		datasets := bigquery.NewDatasetsService(statsTable.service)
		statsTable.dataset, err = datasets.Get(projectId, datasetId).Do()
		return
	}
}

func (statsTable *StatsTable) WriteStats(dimStats map[string]*statshub.Stats, now time.Time) (err error) {
	if err = statsTable.createOrUpdateSchema(dimStats); err != nil {
		return
	}
	err = statsTable.insertRows(dimStats, now)
	return
}

func (statsTable *StatsTable) createOrUpdateSchema(dimStats map[string]*statshub.Stats) (err error) {
	var originalTable *bigquery.Table
	statsTable.table.Schema = schemaForStats(dimStats)
	if originalTable, err = statsTable.tables.Get(
		statsTable.table.TableReference.ProjectId,
		statsTable.table.TableReference.DatasetId,
		statsTable.table.TableReference.TableId,
	).Do(); err != nil {
		log.Printf("Creating table: %s", statsTable.table.TableReference.TableId)

		if statsTable.table, err = statsTable.tables.Insert(
			statsTable.table.TableReference.ProjectId,
			statsTable.table.TableReference.DatasetId,
			statsTable.table).Do(); err != nil {
			log.Printf("Error creating table: %s", err)
			return
		}
	} else {
		log.Printf("Patching table schema: %s", statsTable.table.TableReference.TableId)
		statsTable.mergeSchema(originalTable.Schema)

		if statsTable.table, err = statsTable.tables.Patch(
			statsTable.table.TableReference.ProjectId,
			statsTable.table.TableReference.DatasetId,
			statsTable.table.TableReference.TableId,
			statsTable.table).Do(); err != nil {
			log.Printf("Error patching table: %s", err)
			return
		}
	}

	return
}

func (statsTable *StatsTable) mergeSchema(schema *bigquery.TableSchema) {
	statsTable.table.Schema.Fields = consolidateFields(statsTable.table.Schema.Fields, schema.Fields)
}

func (statsTable *StatsTable) insertRows(dimStats map[string]*statshub.Stats, now time.Time) error {
	doInsert := func(rows []*bigquery.TableDataInsertAllRequestRows) error {
		insertRequest := &bigquery.TableDataInsertAllRequest{Rows: rows}
		_, err := statsTable.tabledata.InsertAll(
			statsTable.table.TableReference.ProjectId,
			statsTable.table.TableReference.DatasetId,
			statsTable.table.TableReference.TableId,
			insertRequest).Do()
		if err == nil {
			log.Printf("Inserted %d rows into: %s", len(rows), statsTable.table.TableReference.TableId)
		}
		return nil
	}

	rows := make([]*bigquery.TableDataInsertAllRequestRows, 1000)
	i := 0

	// Set up
	for dim, stats := range dimStats {
		rows[i] = &bigquery.TableDataInsertAllRequestRows{
			Json: rowFromStats(dim, stats, now),
		}
		i++
		if i == 1000 {
			// To deal with rate limiting, insert every 1000 rows
			if err := doInsert(rows); err != nil {
				return err
			}
			i = 0
		}
	}

	if i != 0 {
		// Insert the remaining rows
		return doInsert(rows[0:i])
	} else {
		return nil
	}
}

func schemaForStats(dimStats map[string]*statshub.Stats) *bigquery.TableSchema {
	fields := make([]*bigquery.TableFieldSchema, 2)
	fields[0] = &bigquery.TableFieldSchema{
		Type: TIMESTAMP,
		Name: _ts,
	}
	fields[1] = &bigquery.TableFieldSchema{
		Type: STRING,
		Name: _dim,
	}
	// Build fields based on stats for total
	dimFields := fieldsForStats(dimStats["total"])
	for _, dimField := range dimFields {
		fields = append(fields, dimField)
	}
	return &bigquery.TableSchema{
		Fields: fields,
	}
}

func fieldsForStats(stats *statshub.Stats) (fields []*bigquery.TableFieldSchema) {
	fields = make([]*bigquery.TableFieldSchema, 0)
	if len(stats.Counters) > 0 {
		fields = append(fields, &bigquery.TableFieldSchema{
			Type:   RECORD,
			Name:   counter,
			Fields: fieldsFor(stats.Counters),
		})
	}
	if len(stats.Gauges) > 0 {
		fields = append(fields, &bigquery.TableFieldSchema{
			Type:   RECORD,
			Name:   gauge,
			Fields: fieldsFor(stats.Gauges),
		})
	}
	return
}

func fieldsFor(m map[string]int64) (fields []*bigquery.TableFieldSchema) {
	keys := make([]string, len(m))
	i := 0
	for key, _ := range m {
		keys[i] = key
		i++
	}
	// Sort keys alphabetically
	sort.Strings(keys)
	fields = make([]*bigquery.TableFieldSchema, len(keys))
	for i, key := range keys {
		fields[i] = &bigquery.TableFieldSchema{
			Type: INTEGER,
			Name: key,
		}
	}
	return
}

// consolidateFields consolidates two lists of TableFieldSchemas into a single list
func consolidateFields(a []*bigquery.TableFieldSchema, b []*bigquery.TableFieldSchema) (consolidated []*bigquery.TableFieldSchema) {
	allFields := make(map[string]*bigquery.TableFieldSchema)

	for _, field := range a {
		allFields[field.Name] = field
	}
	for _, field := range b {
		matching, found := allFields[field.Name]
		if found {
			if matching.Type == RECORD {
				// For RECORD fields, consolidate their lists of fields
				matching.Fields = consolidateFields(field.Fields, matching.Fields)
			}
		} else {
			// No matching field found, add field
			allFields[field.Name] = field
		}
	}

	keys := make([]string, len(allFields))
	i := 0
	for key, _ := range allFields {
		keys[i] = key
		i++
	}

	// Sort keys alphabetically
	sort.Strings(keys)
	consolidated = make([]*bigquery.TableFieldSchema, len(keys))
	for i, key := range keys {
		consolidated[i] = allFields[key]
	}

	return
}

func rowFromStats(dim string, stats *statshub.Stats, now time.Time) (row map[string]interface{}) {
	row = statsAsMap(stats)
	row[_ts] = now.Unix()
	row[_dim] = dim
	return
}

func statsAsMap(stats *statshub.Stats) (m map[string]interface{}) {
	m = make(map[string]interface{})
	m[counter] = stats.Counters
	m[gauge] = stats.Gauges
	return
}
<commit_msg>Made table archiving work better with BigQuery limits<commit_after>package archive

import (
	// Note - I'm using a patched version of the google-api-go-client library
	// because of this bug -
	// https://code.google.com/p/google-api-go-client/issues/detail?id=52
	bigquery "code.google.com/p/ox-google-api-go-client/bigquery/v2"
	"github.com/getlantern/statshub/statshub"
	"github.com/oxtoacart/oauther/oauth"
	"log"
	"os"
	"sort"
	"time"
)

const (
	OAUTH_CONFIG = "OAUTH_CONFIG"

	TIMESTAMP = "TIMESTAMP"
	RECORD    = "RECORD"
	INTEGER   = "INTEGER"
	STRING    = "STRING"
	global    = "global"
	counter   = "counter"
	gauge     = "gauge"
	_ts       = "_ts"
	_dim      = "_dim"

	ROWS_PER_INSERT = 1000
)

// StatsTable is a table that holds statistics from statshub
type StatsTable struct {
	service   *bigquery.Service
	tables    *bigquery.TablesService
	tabledata *bigquery.TabledataService
	dataset   *bigquery.Dataset
	table     *bigquery.Table
}

func NewStatsTable(projectId string, datasetId string, tableId string) (statsTable *StatsTable, err error) {
	statsTable = &StatsTable{
		table: &bigquery.Table{
			TableReference: &bigquery.TableReference{
				ProjectId: projectId,
				DatasetId: datasetId,
				TableId:   tableId,
			},
		},
	}
	var oauther *oauth.OAuther
	if oauther, err = oauth.FromJSON([]byte(os.Getenv(OAUTH_CONFIG))); err != nil {
		return
	} else if statsTable.service, err = bigquery.New(oauther.Transport().Client()); err != nil {
		return
	} else {
		statsTable.tables = bigquery.NewTablesService(statsTable.service)
		statsTable.tabledata = bigquery.NewTabledataService(statsTable.service)
		datasets := bigquery.NewDatasetsService(statsTable.service)
		statsTable.dataset, err = datasets.Get(projectId, datasetId).Do()
		return
	}
}

func (statsTable *StatsTable) WriteStats(dimStats map[string]*statshub.Stats, now time.Time) (err error) {
	if err = statsTable.createOrUpdateSchema(dimStats); err != nil {
		return
	}
	err = statsTable.insertRows(dimStats, now)
	return
}

func (statsTable *StatsTable) createOrUpdateSchema(dimStats map[string]*statshub.Stats) (err error) {
	var originalTable *bigquery.Table
	statsTable.table.Schema = schemaForStats(dimStats)
	if originalTable, err = statsTable.tables.Get(
		statsTable.table.TableReference.ProjectId,
		statsTable.table.TableReference.DatasetId,
		statsTable.table.TableReference.TableId,
	).Do(); err != nil {
		log.Printf("Creating table: %s", statsTable.table.TableReference.TableId)

		if statsTable.table, err = statsTable.tables.Insert(
			statsTable.table.TableReference.ProjectId,
			statsTable.table.TableReference.DatasetId,
			statsTable.table).Do(); err != nil {
			log.Printf("Error creating table: %s", err)
			return
		}
	} else {
		log.Printf("Patching table schema: %s", statsTable.table.TableReference.TableId)
		statsTable.mergeSchema(originalTable.Schema)

		if statsTable.table, err = statsTable.tables.Patch(
			statsTable.table.TableReference.ProjectId,
			statsTable.table.TableReference.DatasetId,
			statsTable.table.TableReference.TableId,
			statsTable.table).Do(); err != nil {
			log.Printf("Error patching table: %s", err)
			return
		}
	}

	return
}

func (statsTable *StatsTable) mergeSchema(schema *bigquery.TableSchema) {
	statsTable.table.Schema.Fields = consolidateFields(statsTable.table.Schema.Fields, schema.Fields)
}

func (statsTable *StatsTable) insertRows(dimStats map[string]*statshub.Stats, now time.Time) error {
	doInsert := func(rows []*bigquery.TableDataInsertAllRequestRows) error {
		insertRequest := &bigquery.TableDataInsertAllRequest{Rows: rows}
		_, err := statsTable.tabledata.InsertAll(
			statsTable.table.TableReference.ProjectId,
			statsTable.table.TableReference.DatasetId,
			statsTable.table.TableReference.TableId,
			insertRequest).Do()
		if err == nil {
			log.Printf("Inserted %d rows into: %s", len(rows), statsTable.table.TableReference.TableId)
		}
		return nil
	}

	rows := make([]*bigquery.TableDataInsertAllRequestRows, ROWS_PER_INSERT)
	i := 0

	// Set up
	for dim, stats := range dimStats {
		rows[i] = &bigquery.TableDataInsertAllRequestRows{
			Json: rowFromStats(dim, stats, now),
		}
		i++
		if i == ROWS_PER_INSERT {
			// To deal with rate limiting, insert every 1000 rows
			if err := doInsert(rows); err != nil {
				return err
			}
			i = 0
		}
	}

	if i != 0 {
		// Insert the remaining rows
		return doInsert(rows[0:i])
	} else {
		return nil
	}
}

func schemaForStats(dimStats map[string]*statshub.Stats) *bigquery.TableSchema {
	fields := make([]*bigquery.TableFieldSchema, 2)
	fields[0] = &bigquery.TableFieldSchema{
		Type: TIMESTAMP,
		Name: _ts,
	}
	fields[1] = &bigquery.TableFieldSchema{
		Type: STRING,
		Name: _dim,
	}
	// Build fields based on stats for total
	dimFields := fieldsForStats(dimStats["total"])
	for _, dimField := range dimFields {
		fields = append(fields, dimField)
	}
	return &bigquery.TableSchema{
		Fields: fields,
	}
}

func fieldsForStats(stats *statshub.Stats) (fields []*bigquery.TableFieldSchema) {
	fields = make([]*bigquery.TableFieldSchema, 0)
	if len(stats.Counters) > 0 {
		fields = append(fields, &bigquery.TableFieldSchema{
			Type:   RECORD,
			Name:   counter,
			Fields: fieldsFor(stats.Counters),
		})
	}
	if len(stats.Gauges) > 0 {
		fields = append(fields, &bigquery.TableFieldSchema{
			Type:   RECORD,
			Name:   gauge,
			Fields: fieldsFor(stats.Gauges),
		})
	}
	return
}

func fieldsFor(m map[string]int64) (fields []*bigquery.TableFieldSchema) {
	keys := make([]string, len(m))
	i := 0
	for key, _ := range m {
		keys[i] = key
		i++
	}
	// Sort keys alphabetically
	sort.Strings(keys)
	fields = make([]*bigquery.TableFieldSchema, len(keys))
	for i, key := range keys {
		fields[i] = &bigquery.TableFieldSchema{
			Type: INTEGER,
			Name: key,
		}
	}
	return
}

// consolidateFields consolidates two lists of TableFieldSchemas into a single list
func consolidateFields(a []*bigquery.TableFieldSchema, b []*bigquery.TableFieldSchema) (consolidated []*bigquery.TableFieldSchema) {
	allFields := make(map[string]*bigquery.TableFieldSchema)

	for _, field := range a {
		allFields[field.Name] = field
	}
	for _, field := range b {
		matching, found := allFields[field.Name]
		if found {
			if matching.Type == RECORD {
				// For RECORD fields, consolidate their lists of fields
				matching.Fields = consolidateFields(field.Fields, matching.Fields)
			}
		} else {
			// No matching field found, add field
			allFields[field.Name] = field
		}
	}

	keys := make([]string, len(allFields))
	i := 0
	for key, _ := range allFields {
		keys[i] = key
		i++
	}

	// Sort keys alphabetically
	sort.Strings(keys)
	consolidated = make([]*bigquery.TableFieldSchema, len(keys))
	for i, key := range keys {
		consolidated[i] = allFields[key]
	}

	return
}

func rowFromStats(dim string, stats *statshub.Stats, now time.Time) (row map[string]interface{}) {
	row = statsAsMap(stats)
	row[_ts] = now.Unix()
	row[_dim] = dim
	return
}

func statsAsMap(stats *statshub.Stats) (m map[string]interface{}) {
	m = make(map[string]interface{})
	m[counter] = stats.Counters
	m[gauge] = stats.Gauges
	return
}
<|endoftext|>


<commit_before>package kill

import (
	"fmt"
	"os"

	"github.com/fatih/color"
	"github.com/gyuho/psn/ss"
	"github.com/spf13/cobra"
)

type Flags struct {
	Force   bool
	CleanUp bool
	Filter  *ss.Process
}

var (
	Command = &cobra.Command{
		Use:   "kill",
		Short: "kill kills programs using syscall. Make sure to specify the flags to find the program.",
		RunE:  CommandFunc,
	}
	cmdFlag = Flags{Filter: &ss.Process{}}
)

func init() {
	Command.PersistentFlags().BoolVarP(&cmdFlag.Force, "force", "f", false, "'true' to force kill any programs.")
	Command.PersistentFlags().BoolVar(&cmdFlag.CleanUp, "clean-up", false, "'true' to clean up deleted programs (this overwrites program flag).")
	Command.PersistentFlags().StringVarP(&cmdFlag.Filter.Program, "program", "s", "SPECIFY YOUR PROGRAM HERE", "Specify the program. Empty lists all programs.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.Protocol, "protocol", "", "'tcp' or 'tcp6'. Empty lists all protocols.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.LocalIP, "local-ip", "", "Specify the local IP. Empty lists all local IPs.")
	Command.PersistentFlags().StringVarP(&cmdFlag.Filter.LocalPort, "local-port", "l", "", "Specify the local port. Empty lists all local ports.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.RemoteIP, "remote-ip", "", "Specify the remote IP. Empty lists all remote IPs.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.RemotePort, "remote-port", "", "Specify the remote port. Empty lists all remote ports.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.State, "state", "", "Specify the state. Empty lists all states.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.User.Username, "username", "", "Specify the user name. Empty lists all user names.")
}

func CommandFunc(cmd *cobra.Command, args []string) error {
	if cmdFlag.CleanUp {
		cmdFlag.Filter.Program = "deleted)"
	}
	color.Set(color.FgRed)
	fmt.Fprintf(os.Stdout, "\npsn kill with %q (port %s)\n\n", cmdFlag.Filter.Program, cmdFlag.Filter.LocalPort)
	color.Unset()

	if cmdFlag.Filter.LocalIP != "" && cmdFlag.Filter.Program == "" {
		cmdFlag.Filter.Program = "SPECIFY YOUR PROGRAM HERE"
	}
	ssr, err := ss.List(cmdFlag.Filter, ss.TCP, ss.TCP6)
	if err != nil {
		return err
	}

	ss.WriteToTable(os.Stdout, ssr...)
	fmt.Fprintf(os.Stdout, "\n")

	ss.Kill(os.Stdout, cmdFlag.Force, ssr...)

	color.Set(color.FgGreen)
	fmt.Fprintf(os.Stdout, "\nDone.\n")
	color.Unset()

	return nil
}
<commit_msg>change order<commit_after>package kill

import (
	"fmt"
	"os"

	"github.com/fatih/color"
	"github.com/gyuho/psn/ss"
	"github.com/spf13/cobra"
)

type Flags struct {
	Force   bool
	CleanUp bool
	Filter  *ss.Process
}

var (
	Command = &cobra.Command{
		Use:   "kill",
		Short: "kill kills programs using syscall. Make sure to specify the flags to find the program.",
		RunE:  CommandFunc,
	}
	cmdFlag = Flags{Filter: &ss.Process{}}
)

func init() {
	Command.PersistentFlags().BoolVarP(&cmdFlag.Force, "force", "f", false, "'true' to force kill any programs.")
	Command.PersistentFlags().BoolVar(&cmdFlag.CleanUp, "clean-up", false, "'true' to clean up deleted programs (this overwrites program flag).")
	Command.PersistentFlags().StringVarP(&cmdFlag.Filter.Program, "program", "s", "SPECIFY YOUR PROGRAM HERE", "Specify the program. Empty lists all programs.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.Protocol, "protocol", "", "'tcp' or 'tcp6'. Empty lists all protocols.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.LocalIP, "local-ip", "", "Specify the local IP. Empty lists all local IPs.")
	Command.PersistentFlags().StringVarP(&cmdFlag.Filter.LocalPort, "local-port", "l", "", "Specify the local port. Empty lists all local ports.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.RemoteIP, "remote-ip", "", "Specify the remote IP. Empty lists all remote IPs.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.RemotePort, "remote-port", "", "Specify the remote port. Empty lists all remote ports.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.State, "state", "", "Specify the state. Empty lists all states.")
	Command.PersistentFlags().StringVar(&cmdFlag.Filter.User.Username, "username", "", "Specify the user name. Empty lists all user names.")
}

func CommandFunc(cmd *cobra.Command, args []string) error {
	if cmdFlag.CleanUp {
		cmdFlag.Filter.Program = "deleted)"
	}
	if cmdFlag.Filter.LocalIP != "" && cmdFlag.Filter.Program == "" {
		cmdFlag.Filter.Program = "SPECIFY YOUR PROGRAM HERE"
	}

	color.Set(color.FgRed)
	fmt.Fprintf(os.Stdout, "\npsn kill with %q (port %s)\n\n", cmdFlag.Filter.Program, cmdFlag.Filter.LocalPort)
	color.Unset()

	ssr, err := ss.List(cmdFlag.Filter, ss.TCP, ss.TCP6)
	if err != nil {
		return err
	}

	ss.WriteToTable(os.Stdout, ssr...)
	fmt.Fprintf(os.Stdout, "\n")

	ss.Kill(os.Stdout, cmdFlag.Force, ssr...)

	color.Set(color.FgGreen)
	fmt.Fprintf(os.Stdout, "\nDone.\n")
	color.Unset()

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"

	elektra "github.com/ElektraInitiative/go-elektra/kdb"
)

type KeyValueBody struct {
	Key   string  `json:"key"`
	Value *string `json:"value"`
}

func PostMetaHandler(w http.ResponseWriter, r *http.Request) {
	var meta KeyValueBody

	keyName := parseKeyNameFromURL(r)

	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(&meta); err != nil {
		writeError(w, err)
		return
	}

	if meta.Key == "" {
		badRequest(w)
	}

	kdb := getHandle(r)

	ks, err := elektra.CreateKeySet()

	if err != nil {
		writeError(w, err)
		return
	}

	parentKey, _ := elektra.CreateKey(keyName)
	_, err = kdb.Get(ks, parentKey)

	k, err := ks.Lookup(parentKey)

	if k == nil {
		err = ks.AppendKey(parentKey)
	}

	if err != nil {
		writeError(w, err)
		return
	}

	if meta.Value == nil {
		err = k.DeleteMeta(meta.Key)
	} else {
		err = k.SetMeta(meta.Key, *meta.Value)

	}

	if err != nil {
		writeError(w, err)
		return
	}

	noContent(w)
}

func DeleteMetaHandler(w http.ResponseWriter, r *http.Request) {
	// kdb := kdbHandle()

	// ks, err := elektra.CreateKeySet()

	// if err != nil {
	// 	writeError(w, err)
	// 	return
	// }

	// keyName := parseKeyNameFromURL(r)

	// key, _ := elektra.CreateKey(keyName)
	// _, err = kdb.Get(ks, key)

	// writeResponse(w, response)
}
<commit_msg>tool: elektrad - fix Post Meta handler, add Delete Meta Handler<commit_after>package main

import (
	"encoding/json"
	"net/http"

	elektra "github.com/ElektraInitiative/go-elektra/kdb"
)

type KeyValueBody struct {
	Key   string  `json:"key"`
	Value *string `json:"value"`
}

func PostMetaHandler(w http.ResponseWriter, r *http.Request) {
	var meta KeyValueBody

	keyName := parseKeyNameFromURL(r)

	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(&meta); err != nil {
		writeError(w, err)
		return
	}

	if meta.Key == "" {
		badRequest(w)
	}

	kdb := getHandle(r)

	parentKey, err := elektra.CreateKey(keyName)

	if err != nil {
		writeError(w, err)
		return
	}

	ks, err := getKeySet(kdb, parentKey)

	if err != nil {
		writeError(w, err)
		return
	}

	k, err := ks.Lookup(parentKey)

	if err != nil {
		writeError(w, err)
		return
	}

	if k == nil {
		err = ks.AppendKey(parentKey)
	}

	if err != nil {
		writeError(w, err)
		return
	}

	if meta.Value == nil {
		err = k.DeleteMeta(meta.Key)
	} else {
		err = k.SetMeta(meta.Key, *meta.Value)
	}

	if err != nil {
		writeError(w, err)
		return
	}

	_, err = kdb.Set(ks, parentKey)

	if err != nil {
		writeError(w, err)
		return
	}

	noContent(w)
}

func DeleteMetaHandler(w http.ResponseWriter, r *http.Request) {
	kdb := getHandle(r)

	var meta KeyValueBody

	keyName := parseKeyNameFromURL(r)

	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(&meta); err != nil {
		writeError(w, err)
		return
	}

	key, err := elektra.CreateKey(keyName)

	if err != nil {
		writeError(w, err)
		return
	}

	keySet, err := getKeySet(kdb, key)

	if err != nil {
		writeError(w, err)
		return
	}

	err = key.DeleteMeta(meta.Key)

	if err != nil {
		writeError(w, err)
		return
	}

	_, err = kdb.Set(keySet, key)

	if err != nil {
		writeError(w, err)
		return
	}

	noContent(w)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

//Licenselist holds short and long names of licenses
type Licenselist struct {
	Short string `json:"short"`
	Long  string `json:"long"`
}

//Licenseinfo holds information of licenses
type Licenseinfo struct {
	Name        string `json:"name"`
	Title       string `json:"title"`
	Category    string `json:"category"`
	Source      string `json:"source"`
	Description string `json:"description"`
	Tags        struct {
		Required  []string `json:"required"`
		Permitted []string `json:"permitted"`
		Forbidden []string `json:"forbidden"`
	} `json:"tags"`
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func getfilepath(filename string) string {
	gopath := os.Getenv("GOPATH")
	licensethispath := gopath + "/src/github.com/hasit/licensethis/"
	filepath := licensethispath + filename

	return filepath
}

func help() {
	helptext := `licensethis - Choose an OSS license for your project with ease.

Usage:
licensethis help
licensethis list
licensethis info <license-name>
licensethis generate <license-name>

Commands:
help		Show this help text.
list		Show a list of all available OSS licenses.
info		Show more information for a license.
generate	Generate LICENSE.txt file in current folder after asking for author name.

Examples:
licensethis help		Show this help text.
licensethis info mit		Show more information for MIT license.
licensethis generate mit	Geneate MIT license text file in current directory.`

	fmt.Printf("%v\n", helptext)
}

func list() {
	listfilepath := getfilepath("licenselist.json")
	file, err1 := ioutil.ReadFile(listfilepath)
	check(err1)

	var licenselist []Licenselist
	err2 := json.Unmarshal(file, &licenselist)
	check(err2)

	fmt.Println("List of all available OSS licenses:")
	for i := range licenselist {
		fmt.Printf("%d. %v - %v\n", i, licenselist[i].Short, licenselist[i].Long)
	}
}

// !!---FUTURE---!!
// alow substrings of license names as arguments too
func info(licensename string) {
	infofilepath := getfilepath("licenseinfo.json")
	file, err1 := ioutil.ReadFile(infofilepath)
	check(err1)

	var licenseinfo []Licenseinfo
	err2 := json.Unmarshal(file, &licenseinfo)
	check(err2)

	var closelicensenames []string

	for i := range licenseinfo {
		if licenseinfo[i].Name == licensename {
			fmt.Printf("Title: %v\n", licenseinfo[i].Title)
			fmt.Printf("Category: %v\n", licenseinfo[i].Category)
			fmt.Printf("Source: %v\n", licenseinfo[i].Source)
			fmt.Printf("Description: %v\n", licenseinfo[i].Description)
			fmt.Println("Tags: ")
			fmt.Println("  Required: ")
			for r := range licenseinfo[i].Tags.Required {
				fmt.Printf("    -%v\n", licenseinfo[i].Tags.Required[r])
			}
			fmt.Println("  Permitted: ")
			for r := range licenseinfo[i].Tags.Permitted {
				fmt.Printf("    -%v\n", licenseinfo[i].Tags.Permitted[r])
			}
			fmt.Println("  Forbidden: ")
			for r := range licenseinfo[i].Tags.Forbidden {
				fmt.Printf("    -%v\n", licenseinfo[i].Tags.Forbidden[r])
			}
			return
		} else if strings.Contains(licenseinfo[i].Name, licensename) {
			closelicensenames = append(closelicensenames, licenseinfo[i].Name)
		}
	}
	if len(closelicensenames) != 0 {
		fmt.Println("Did you mean any of these?")
		for i := range closelicensenames {
			fmt.Printf("%v\n", closelicensenames[i])
		}
	} else {
		fmt.Println("No such license was found")
		fmt.Println("Type `licensethis list` for a list of available licenses")
	}
}

func generate(licensename string) {
	fmt.Println(licensename)
}

// !!---FUTURE---!!
//might have to look into a way to accept more than one license name for `info`.
func parseArgs(args []string) {
	if len(args) == 1 {
		switch args[0] {
		case "help":
			help()
		case "list":
			list()
		default:
			fmt.Println("Incorrect usage!")
			fmt.Println("Type `licensethis help` for help on proper usage.")
		}
	} else if len(args) == 2 {
		switch args[0] {
		case "info":
			info(args[1])
		case "generate":
			generate(args[1])
		default:
			fmt.Println("Incorrect usage!")
			fmt.Println("Type `licensethis help` for help on proper usage.")
		}
	} else {
		fmt.Println("Incorrect usage!")
		fmt.Println("Type `licensethis help` for help on proper usage.")
	}
}

func main() {
	args := os.Args[1:]
	parseArgs(args)
}
<commit_msg>FUTURE removed<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
)

//Licenselist holds short and long names of licenses
type Licenselist struct {
	Short string `json:"short"`
	Long  string `json:"long"`
}

//Licenseinfo holds information of licenses
type Licenseinfo struct {
	Name        string `json:"name"`
	Title       string `json:"title"`
	Category    string `json:"category"`
	Source      string `json:"source"`
	Description string `json:"description"`
	Tags        struct {
		Required  []string `json:"required"`
		Permitted []string `json:"permitted"`
		Forbidden []string `json:"forbidden"`
	} `json:"tags"`
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func getfilepath(filename string) string {
	gopath := os.Getenv("GOPATH")
	licensethispath := gopath + "/src/github.com/hasit/licensethis/"
	filepath := licensethispath + filename

	return filepath
}

func help() {
	helptext := `licensethis - Choose an OSS license for your project with ease.

Usage:
licensethis help
licensethis list
licensethis info <license-name>
licensethis generate <license-name>

Commands:
help		Show this help text.
list		Show a list of all available OSS licenses.
info		Show more information for a license.
generate	Generate LICENSE.txt file in current folder after asking for author name.

Examples:
licensethis help		Show this help text.
licensethis info mit		Show more information for MIT license.
licensethis generate mit	Geneate MIT license text file in current directory.`

	fmt.Printf("%v\n", helptext)
}

func list() {
	listfilepath := getfilepath("licenselist.json")
	file, err1 := ioutil.ReadFile(listfilepath)
	check(err1)

	var licenselist []Licenselist
	err2 := json.Unmarshal(file, &licenselist)
	check(err2)

	fmt.Println("List of all available OSS licenses:")
	for i := range licenselist {
		fmt.Printf("%d. %v - %v\n", i, licenselist[i].Short, licenselist[i].Long)
	}
}

func info(licensename string) {
	infofilepath := getfilepath("licenseinfo.json")
	file, err1 := ioutil.ReadFile(infofilepath)
	check(err1)

	var licenseinfo []Licenseinfo
	err2 := json.Unmarshal(file, &licenseinfo)
	check(err2)

	var closelicensenames []string

	for i := range licenseinfo {
		if licenseinfo[i].Name == licensename {
			fmt.Printf("Title: %v\n", licenseinfo[i].Title)
			fmt.Printf("Category: %v\n", licenseinfo[i].Category)
			fmt.Printf("Source: %v\n", licenseinfo[i].Source)
			fmt.Printf("Description: %v\n", licenseinfo[i].Description)
			fmt.Println("Tags: ")
			fmt.Println("  Required: ")
			for r := range licenseinfo[i].Tags.Required {
				fmt.Printf("    -%v\n", licenseinfo[i].Tags.Required[r])
			}
			fmt.Println("  Permitted: ")
			for r := range licenseinfo[i].Tags.Permitted {
				fmt.Printf("    -%v\n", licenseinfo[i].Tags.Permitted[r])
			}
			fmt.Println("  Forbidden: ")
			for r := range licenseinfo[i].Tags.Forbidden {
				fmt.Printf("    -%v\n", licenseinfo[i].Tags.Forbidden[r])
			}
			return
		} else if strings.Contains(licenseinfo[i].Name, licensename) {
			closelicensenames = append(closelicensenames, licenseinfo[i].Name)
		}
	}
	if len(closelicensenames) != 0 {
		fmt.Println("Did you mean any of these?")
		for i := range closelicensenames {
			fmt.Printf("%v\n", closelicensenames[i])
		}
	} else {
		fmt.Println("No such license was found")
		fmt.Println("Type `licensethis list` for a list of available licenses")
	}
}

func generate(licensename string) {
	fmt.Println(licensename)
}

func parseArgs(args []string) {
	if len(args) == 1 {
		switch args[0] {
		case "help":
			help()
		case "list":
			list()
		default:
			fmt.Println("Incorrect usage!")
			fmt.Println("Type `licensethis help` for help on proper usage.")
		}
	} else if len(args) == 2 {
		switch args[0] {
		case "info":
			info(args[1])
		case "generate":
			generate(args[1])
		default:
			fmt.Println("Incorrect usage!")
			fmt.Println("Type `licensethis help` for help on proper usage.")
		}
	} else {
		fmt.Println("Incorrect usage!")
		fmt.Println("Type `licensethis help` for help on proper usage.")
	}
}

func main() {
	args := os.Args[1:]
	parseArgs(args)
}
<|endoftext|>