<commit_before>package repair

import (
	"fmt"
	"koding/klient/remote/req"
	"koding/klientctl/util"
	"os"
	"strings"

	"github.com/koding/logging"
)

// dncSuffix is the suffix of the error mesage returned by Golang when trying to
// open a not-configured device directory.
const dncSuffix = "device not configured"

// DeviceNotConfiguredRepair looks at the given mount directory and checks if
// is a device not configured. If so, remount.
type DeviceNotConfiguredRepair struct {
	Log logging.Logger

	Stdout *util.Fprint

	MountName string

	// The klient we will be communicating with.
	Klient interface {
		RemoteMountInfo(string) (req.MountInfoResponse, error)
		RemoteRemount(string) error
	}
}

func (r *DeviceNotConfiguredRepair) String() string {
	return "DeviceNotConfiguredRepair"
}

func (r *DeviceNotConfiguredRepair) getMountPath() (string, error) {
	// Get the mount dir.
	info, err := r.Klient.RemoteMountInfo(r.MountName)
	if err != nil {
		return "", err
	}

	return info.LocalPath, nil
}

// Status stats the mount directory that klient returns, and if it errors with
// Permission denied, return an error.
func (r *DeviceNotConfiguredRepair) Status() error {
	path, err := r.getMountPath()

	// If we can't even get the mount dir from klient, return the error
	// so that repair can fail with the same issue.
	//
	// TODO: Fix this behavior by adding bool to Status(), so that errors mean
	// bad things. ~LO
	if err != nil {
		r.Log.Warning("Encountered ignored error. err:%s", err)
		return nil
	}

	f, err := os.Open(path)

	// If open fails with device not configured, we return not okay.
	if err != nil && strings.HasSuffix(err.Error(), dncSuffix) {
		return fmt.Errorf("Mount %q at %q not configured", r.MountName, path)
	}

	// If it's another error, log it. In this case there's nothing to do,
	// we don't know what's wrong.
	if err != nil {
		r.Log.Warning("Encountered ignored error. err:%s", err)
		return nil
	}

	// Not returning an error for mount closing failure. Just logging it.
	if err := f.Close(); err != nil {
		r.Log.Warning(
			"Failed to close MountDir. mountName:%s, dir:%s, err:%s",
			r.MountName, path, err,
		)
	}

	return nil
}

// Repair simply remounts the given directory, and then checks the status one more
// time before exiting.
func (r *DeviceNotConfiguredRepair) Repair() error {
	r.Stdout.Printlnf("Mount is no longer valid, remounting...")

	if err := r.Klient.RemoteRemount(r.MountName); err != nil {
		r.Stdout.Printlnf("Unable to remount %s", r.MountName)
		return err
	}

	if err := r.Status(); err != nil {
		r.Stdout.Printlnf("Unable to repair mount issue.")
		return err
	}

	r.Stdout.Printlnf("Remounted successfully.")

	return nil
}
<commit_msg>klientctl: Corrected comment<commit_after>package repair

import (
	"fmt"
	"koding/klient/remote/req"
	"koding/klientctl/util"
	"os"
	"strings"

	"github.com/koding/logging"
)

// dncSuffix is the suffix of the error mesage returned by the OS when trying to
// open a not-configured device directory.
const dncSuffix = "device not configured"

// DeviceNotConfiguredRepair looks at the given mount directory and checks if
// is a device not configured. If so, remount.
type DeviceNotConfiguredRepair struct {
	Log logging.Logger

	Stdout *util.Fprint

	MountName string

	// The klient we will be communicating with.
	Klient interface {
		RemoteMountInfo(string) (req.MountInfoResponse, error)
		RemoteRemount(string) error
	}
}

func (r *DeviceNotConfiguredRepair) String() string {
	return "DeviceNotConfiguredRepair"
}

func (r *DeviceNotConfiguredRepair) getMountPath() (string, error) {
	// Get the mount dir.
	info, err := r.Klient.RemoteMountInfo(r.MountName)
	if err != nil {
		return "", err
	}

	return info.LocalPath, nil
}

// Status stats the mount directory that klient returns, and if it errors with
// Permission denied, return an error.
func (r *DeviceNotConfiguredRepair) Status() error {
	path, err := r.getMountPath()

	// If we can't even get the mount dir from klient, return the error
	// so that repair can fail with the same issue.
	//
	// TODO: Fix this behavior by adding bool to Status(), so that errors mean
	// bad things. ~LO
	if err != nil {
		r.Log.Warning("Encountered ignored error. err:%s", err)
		return nil
	}

	f, err := os.Open(path)

	// If open fails with device not configured, we return not okay.
	if err != nil && strings.HasSuffix(err.Error(), dncSuffix) {
		return fmt.Errorf("Mount %q at %q not configured", r.MountName, path)
	}

	// If it's another error, log it. In this case there's nothing to do,
	// we don't know what's wrong.
	if err != nil {
		r.Log.Warning("Encountered ignored error. err:%s", err)
		return nil
	}

	// Not returning an error for mount closing failure. Just logging it.
	if err := f.Close(); err != nil {
		r.Log.Warning(
			"Failed to close MountDir. mountName:%s, dir:%s, err:%s",
			r.MountName, path, err,
		)
	}

	return nil
}

// Repair simply remounts the given directory, and then checks the status one more
// time before exiting.
func (r *DeviceNotConfiguredRepair) Repair() error {
	r.Stdout.Printlnf("Mount is no longer valid, remounting...")

	if err := r.Klient.RemoteRemount(r.MountName); err != nil {
		r.Stdout.Printlnf("Unable to remount %s", r.MountName)
		return err
	}

	if err := r.Status(); err != nil {
		r.Stdout.Printlnf("Unable to repair mount issue.")
		return err
	}

	r.Stdout.Printlnf("Remounted successfully.")

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reverseproxy

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/caddyserver/caddy/v2"
)

func init() {
	caddy.RegisterModule(adminUpstreams{})
}

// adminUpstreams is a module that provides the
// /reverse_proxy/upstreams endpoint for the Caddy admin
// API. This allows for checking the health of configured
// reverse proxy upstreams in the pool.
type adminUpstreams struct{}

// upstreamResults holds the status of a particular upstream
type upstreamStatus struct {
	Address     string `json:"address"`
	Healthy     bool   `json:"healthy"`
	NumRequests int    `json:"num_requests"`
	Fails       int    `json:"fails"`
}

// CaddyModule returns the Caddy module information.
func (adminUpstreams) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		ID:  "admin.api.reverse_proxy",
		New: func() caddy.Module { return new(adminUpstreams) },
	}
}

// Routes returns a route for the /reverse_proxy/upstreams endpoint.
func (al adminUpstreams) Routes() []caddy.AdminRoute {
	return []caddy.AdminRoute{
		{
			Pattern: "/reverse_proxy/upstreams",
			Handler: caddy.AdminHandlerFunc(al.handleUpstreams),
		},
	}
}

// handleUpstreams reports the status of the reverse proxy
// upstream pool.
func (adminUpstreams) handleUpstreams(w http.ResponseWriter, r *http.Request) error {
	if r.Method != http.MethodGet {
		return caddy.APIError{
			HTTPStatus: http.StatusMethodNotAllowed,
			Err:        fmt.Errorf("method not allowed"),
		}
	}

	// Prep for a JSON response
	w.Header().Set("Content-Type", "application/json")
	enc := json.NewEncoder(w)

	// Collect the results to respond with
	results := []upstreamStatus{}

	// Iterate over the upstream pool (needs to be fast)
	var rangeErr error
	hosts.Range(func(key, val interface{}) bool {
		address, ok := key.(string)
		if !ok {
			rangeErr = caddy.APIError{
				HTTPStatus: http.StatusInternalServerError,
				Err:        fmt.Errorf("could not type assert upstream address"),
			}
			return false
		}

		upstream, ok := val.(*Host)
		if !ok {
			rangeErr = caddy.APIError{
				HTTPStatus: http.StatusInternalServerError,
				Err:        fmt.Errorf("could not type assert upstream struct"),
			}
			return false
		}

		results = append(results, upstreamStatus{
			Address:     address,
			NumRequests: upstream.NumRequests(),
			Fails:       upstream.Fails(),
		})
		return true
	})

	// If an error happened during the range, return it
	if rangeErr != nil {
		return rangeErr
	}

	err := enc.Encode(results)
	if err != nil {
		return caddy.APIError{
			HTTPStatus: http.StatusInternalServerError,
			Err:        err,
		}
	}

	return nil
}
<commit_msg>reverseproxy: api: Remove misleading 'healthy' value<commit_after>// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reverseproxy

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/caddyserver/caddy/v2"
)

func init() {
	caddy.RegisterModule(adminUpstreams{})
}

// adminUpstreams is a module that provides the
// /reverse_proxy/upstreams endpoint for the Caddy admin
// API. This allows for checking the health of configured
// reverse proxy upstreams in the pool.
type adminUpstreams struct{}

// upstreamResults holds the status of a particular upstream
type upstreamStatus struct {
	Address     string `json:"address"`
	NumRequests int    `json:"num_requests"`
	Fails       int    `json:"fails"`
}

// CaddyModule returns the Caddy module information.
func (adminUpstreams) CaddyModule() caddy.ModuleInfo {
	return caddy.ModuleInfo{
		ID:  "admin.api.reverse_proxy",
		New: func() caddy.Module { return new(adminUpstreams) },
	}
}

// Routes returns a route for the /reverse_proxy/upstreams endpoint.
func (al adminUpstreams) Routes() []caddy.AdminRoute {
	return []caddy.AdminRoute{
		{
			Pattern: "/reverse_proxy/upstreams",
			Handler: caddy.AdminHandlerFunc(al.handleUpstreams),
		},
	}
}

// handleUpstreams reports the status of the reverse proxy
// upstream pool.
func (adminUpstreams) handleUpstreams(w http.ResponseWriter, r *http.Request) error {
	if r.Method != http.MethodGet {
		return caddy.APIError{
			HTTPStatus: http.StatusMethodNotAllowed,
			Err:        fmt.Errorf("method not allowed"),
		}
	}

	// Prep for a JSON response
	w.Header().Set("Content-Type", "application/json")
	enc := json.NewEncoder(w)

	// Collect the results to respond with
	results := []upstreamStatus{}

	// Iterate over the upstream pool (needs to be fast)
	var rangeErr error
	hosts.Range(func(key, val interface{}) bool {
		address, ok := key.(string)
		if !ok {
			rangeErr = caddy.APIError{
				HTTPStatus: http.StatusInternalServerError,
				Err:        fmt.Errorf("could not type assert upstream address"),
			}
			return false
		}

		upstream, ok := val.(*Host)
		if !ok {
			rangeErr = caddy.APIError{
				HTTPStatus: http.StatusInternalServerError,
				Err:        fmt.Errorf("could not type assert upstream struct"),
			}
			return false
		}

		results = append(results, upstreamStatus{
			Address:     address,
			NumRequests: upstream.NumRequests(),
			Fails:       upstream.Fails(),
		})
		return true
	})

	// If an error happened during the range, return it
	if rangeErr != nil {
		return rangeErr
	}

	err := enc.Encode(results)
	if err != nil {
		return caddy.APIError{
			HTTPStatus: http.StatusInternalServerError,
			Err:        err,
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * Minio Client (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"io"
	"runtime"
	"sync"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/iodine"
)

// doCopy - Copy a singe file from source to destination
func doCopy(sourceURL string, sourceConfig *hostConfig, targetURL string, targetConfig *hostConfig, bar *barSend) error {
	if sourceURL == targetURL {
		return iodine.New(errSameURLs{source: sourceURL, target: targetURL}, nil)
	}
	readCloser, length, md5hex, err := getSourceReader(sourceURL, sourceConfig)
	if err != nil {
		return iodine.New(err, nil)
	}
	defer readCloser.Close()

	writeCloser, err := getTargetWriter(targetURL, targetConfig, md5hex, length)
	if err != nil {
		return iodine.New(err, nil)
	}

	var writers []io.Writer
	writers = append(writers, writeCloser)

	// set up progress bar
	writers = append(writers, bar)

	// write progress bar
	multiWriter := io.MultiWriter(writers...)
	// copy data to writers
	_, copyErr := io.CopyN(multiWriter, readCloser, int64(length))
	// close to see the error, verify it later
	err = writeCloser.Close()
	if copyErr != nil {
		return iodine.New(copyErr, nil)
	}
	if err != nil {
		return iodine.New(err, nil)
	}

	return nil
}

// runCopyCmd is bound to sub-command
func runCopyCmd(ctx *cli.Context) {
	if len(ctx.Args()) < 2 || ctx.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(ctx, "cp", 1) // last argument is exit code
	}
	if !isMcConfigExist() {
		console.Fatalln("\"mc\" is not configured.  Please run \"mc config generate\".")
	}
	config, err := getMcConfig()
	if err != nil {
		console.Debugln(iodine.New(err, nil))
		console.Fatalf("Unable to read config file [%s]. Reason: [%s].\n", mustGetMcConfigPath(), iodine.ToError(err))
	}

	// Convert arguments to URLs: expand alias, fix format...
	URLs, err := getExpandedURLs(ctx.Args(), config.Aliases)
	if err != nil {
		switch e := iodine.ToError(err).(type) {
		case errUnsupportedScheme:
			console.Debugln(iodine.New(err, nil))
			console.Fatalf("Unknown type of URL(s).\n")
		default:
			console.Debugln(iodine.New(err, nil))
			console.Fatalf("Unable to parse arguments. Reason: [%s].\n", e)
		}
	}

	// Separate source and target. 'cp' can take only one target,
	// but any number of sources, even the recursive URLs mixed in-between.
	targetURL := URLs[len(URLs)-1] // Last one is target
	sourceURLs := URLs[:len(URLs)-1]

	// set up progress bar
	bar := newCopyBar(globalQuietFlag)

	go func(sourceURLs []string, targetURL string) {
		for cpURLs := range prepareCopyURLs(sourceURLs, targetURL) {
			if cpURLs.Error != nil {
				// no need to print errors here, any error here
				// will be printed later during Copy()
				continue
			}
			bar.Extend(cpURLs.SourceContent.Size)
		}
	}(sourceURLs, targetURL)

	var cpQueue = make(chan bool, runtime.NumCPU()-1)
	var wg sync.WaitGroup

	for cpURLs := range prepareCopyURLs(sourceURLs, targetURL) {
		if cpURLs.Error != nil {
			console.Errorln(iodine.ToError(cpURLs.Error))
			continue
		}
		cpQueue <- true
		wg.Add(1)
		go func(cpURLs copyURLs) {
			defer wg.Done()
			srcConfig, err := getHostConfig(cpURLs.SourceContent.Name)
			if err != nil {
				console.Errorln(iodine.ToError(err))
				return
			}
			tgtConfig, err := getHostConfig(cpURLs.TargetContent.Name)
			if err != nil {
				console.Errorln(iodine.ToError(err))
				return
			}
			if err := doCopy(cpURLs.SourceContent.Name, srcConfig, cpURLs.TargetContent.Name, tgtConfig, &bar); err != nil {
				console.Errorln(iodine.ToError(err))
			}
			<-cpQueue
		}(*cpURLs)
	}
	wg.Wait()
	bar.Finish()
}
<commit_msg>Fix copy cmd when the host has only one CPU core<commit_after>/*
 * Minio Client (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"io"
	"runtime"
	"sync"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/iodine"
)

// doCopy - Copy a singe file from source to destination
func doCopy(sourceURL string, sourceConfig *hostConfig, targetURL string, targetConfig *hostConfig, bar *barSend) error {
	if sourceURL == targetURL {
		return iodine.New(errSameURLs{source: sourceURL, target: targetURL}, nil)
	}
	readCloser, length, md5hex, err := getSourceReader(sourceURL, sourceConfig)
	if err != nil {
		return iodine.New(err, nil)
	}
	defer readCloser.Close()

	writeCloser, err := getTargetWriter(targetURL, targetConfig, md5hex, length)
	if err != nil {
		return iodine.New(err, nil)
	}

	var writers []io.Writer
	writers = append(writers, writeCloser)

	// set up progress bar
	writers = append(writers, bar)

	// write progress bar
	multiWriter := io.MultiWriter(writers...)
	// copy data to writers
	_, copyErr := io.CopyN(multiWriter, readCloser, int64(length))
	// close to see the error, verify it later
	err = writeCloser.Close()
	if copyErr != nil {
		return iodine.New(copyErr, nil)
	}
	if err != nil {
		return iodine.New(err, nil)
	}

	return nil
}

// runCopyCmd is bound to sub-command
func runCopyCmd(ctx *cli.Context) {
	if len(ctx.Args()) < 2 || ctx.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(ctx, "cp", 1) // last argument is exit code
	}
	if !isMcConfigExist() {
		console.Fatalln("\"mc\" is not configured.  Please run \"mc config generate\".")
	}
	config, err := getMcConfig()
	if err != nil {
		console.Debugln(iodine.New(err, nil))
		console.Fatalf("Unable to read config file [%s]. Reason: [%s].\n", mustGetMcConfigPath(), iodine.ToError(err))
	}

	// Convert arguments to URLs: expand alias, fix format...
	URLs, err := getExpandedURLs(ctx.Args(), config.Aliases)
	if err != nil {
		switch e := iodine.ToError(err).(type) {
		case errUnsupportedScheme:
			console.Debugln(iodine.New(err, nil))
			console.Fatalf("Unknown type of URL(s).\n")
		default:
			console.Debugln(iodine.New(err, nil))
			console.Fatalf("Unable to parse arguments. Reason: [%s].\n", e)
		}
	}

	// Separate source and target. 'cp' can take only one target,
	// but any number of sources, even the recursive URLs mixed in-between.
	targetURL := URLs[len(URLs)-1] // Last one is target
	sourceURLs := URLs[:len(URLs)-1]

	// set up progress bar
	bar := newCopyBar(globalQuietFlag)

	go func(sourceURLs []string, targetURL string) {
		for cpURLs := range prepareCopyURLs(sourceURLs, targetURL) {
			if cpURLs.Error != nil {
				// no need to print errors here, any error here
				// will be printed later during Copy()
				continue
			}
			bar.Extend(cpURLs.SourceContent.Size)
		}
	}(sourceURLs, targetURL)

	var cpQueue = make(chan bool, intMax(runtime.NumCPU()-1, 1))
	var wg sync.WaitGroup

	for cpURLs := range prepareCopyURLs(sourceURLs, targetURL) {
		if cpURLs.Error != nil {
			console.Errorln(iodine.ToError(cpURLs.Error))
			continue
		}
		cpQueue <- true
		wg.Add(1)
		go func(cpURLs copyURLs) {
			defer wg.Done()
			srcConfig, err := getHostConfig(cpURLs.SourceContent.Name)
			if err != nil {
				console.Errorln(iodine.ToError(err))
				return
			}
			tgtConfig, err := getHostConfig(cpURLs.TargetContent.Name)
			if err != nil {
				console.Errorln(iodine.ToError(err))
				return
			}
			if err := doCopy(cpURLs.SourceContent.Name, srcConfig, cpURLs.TargetContent.Name, tgtConfig, &bar); err != nil {
				console.Errorln(iodine.ToError(err))
			}
			<-cpQueue
		}(*cpURLs)
	}
	wg.Wait()
	bar.Finish()
}
<|endoftext|>


<commit_before>package birc

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/lrstanley/girc"
	"github.com/missdeer/golib/ic"
	"github.com/paulrosania/go-charset/charset"
	"github.com/saintfish/chardet"

	// We need to import the 'data' package as an implicit dependency.
	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
	_ "github.com/paulrosania/go-charset/data"
)

func (b *Birc) handleCharset(msg *config.Message) error {
	if b.GetString("Charset") != "" {
		switch b.GetString("Charset") {
		case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
			msg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), msg.Text)
		default:
			buf := new(bytes.Buffer)
			w, err := charset.NewWriter(b.GetString("Charset"), buf)
			if err != nil {
				b.Log.Errorf("charset from utf-8 conversion failed: %s", err)
				return err
			}
			fmt.Fprint(w, msg.Text)
			w.Close()
			msg.Text = buf.String()
		}
	}
	return nil
}

// handleFiles returns true if we have handled the files, otherwise return false
func (b *Birc) handleFiles(msg *config.Message) bool {
	if msg.Extra == nil {
		return false
	}
	for _, rmsg := range helper.HandleExtra(msg, b.General) {
		b.Local <- rmsg
	}
	if len(msg.Extra["file"]) == 0 {
		return false
	}
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		if fi.Comment != "" {
			msg.Text += fi.Comment + " : "
		}
		if fi.URL != "" {
			msg.Text = fi.URL
			if fi.Comment != "" {
				msg.Text = fi.Comment + " : " + fi.URL
			}
		}
		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
	}
	return true
}

func (b *Birc) handleInvite(client *girc.Client, event girc.Event) {
	if len(event.Params) != 2 {
		return
	}

	channel := event.Params[1]

	b.Log.Debugf("got invite for %s", channel)

	if _, ok := b.channels[channel]; ok {
		b.i.Cmd.Join(channel)
	}
}

func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
	if len(event.Params) == 0 {
		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
		return
	}
	channel := strings.ToLower(event.Params[0])
	if event.Command == "KICK" && event.Params[1] == b.Nick {
		b.Log.Infof("Got kicked from %s by %s", channel, event.Source.Name)
		time.Sleep(time.Duration(b.GetInt("RejoinDelay")) * time.Second)
		b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: channel, Account: b.Account, Event: config.EventRejoinChannels}
		return
	}
	if event.Command == "QUIT" {
		if event.Source.Name == b.Nick && strings.Contains(event.Last(), "Ping timeout") {
			b.Log.Infof("%s reconnecting ..", b.Account)
			b.Remote <- config.Message{Username: "system", Text: "reconnect", Channel: channel, Account: b.Account, Event: config.EventFailure}
			return
		}
	}
	if event.Source.Name != b.Nick {
		if b.GetBool("nosendjoinpart") {
			return
		}
		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		if b.GetBool("verbosejoinpart") {
			b.Log.Debugf("<= Sending verbose JOIN_LEAVE event from %s to gateway", b.Account)
			msg = config.Message{Username: "system", Text: event.Source.Name + " (" + event.Source.Ident + "@" + event.Source.Host + ") " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		} else {
			b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
		}
		b.Log.Debugf("<= Message is %#v", msg)
		b.Remote <- msg
		return
	}
	b.Log.Debugf("handle %#v", event)
}

func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
	b.Log.Debug("Registering callbacks")
	i := b.i
	b.Nick = event.Params[0]

	i.Handlers.Add("PRIVMSG", b.handlePrivMsg)
	i.Handlers.Add("CTCP_ACTION", b.handlePrivMsg)
	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
	i.Handlers.Add(girc.NOTICE, b.handleNotice)
	i.Handlers.Add("JOIN", b.handleJoinPart)
	i.Handlers.Add("PART", b.handleJoinPart)
	i.Handlers.Add("QUIT", b.handleJoinPart)
	i.Handlers.Add("KICK", b.handleJoinPart)
	i.Handlers.Add("INVITE", b.handleInvite)
}

func (b *Birc) handleNickServ() {
	if !b.GetBool("UseSASL") && b.GetString("NickServNick") != "" && b.GetString("NickServPassword") != "" {
		b.Log.Debugf("Sending identify to nickserv %s", b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "IDENTIFY "+b.GetString("NickServPassword"))
	}
	if strings.EqualFold(b.GetString("NickServNick"), "Q@CServe.quakenet.org") {
		b.Log.Debugf("Authenticating %s against %s", b.GetString("NickServUsername"), b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "AUTH "+b.GetString("NickServUsername")+" "+b.GetString("NickServPassword"))
	}
	// give nickserv some slack
	time.Sleep(time.Second * 5)
	b.authDone = true
}

func (b *Birc) handleNotice(client *girc.Client, event girc.Event) {
	if strings.Contains(event.String(), "This nickname is registered") && event.Source.Name == b.GetString("NickServNick") {
		b.handleNickServ()
	} else {
		b.handlePrivMsg(client, event)
	}
}

func (b *Birc) handleOther(client *girc.Client, event girc.Event) {
	if b.GetInt("DebugLevel") == 1 {
		if event.Command != "CLIENT_STATE_UPDATED" &&
			event.Command != "CLIENT_GENERAL_UPDATED" {
			b.Log.Debugf("%#v", event.String())
		}
		return
	}
	switch event.Command {
	case "372", "375", "376", "250", "251", "252", "253", "254", "255", "265", "266", "002", "003", "004", "005":
		return
	}
	b.Log.Debugf("%#v", event.String())
}

func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
	b.handleNickServ()
	b.handleRunCommands()
	// we are now fully connected
	// only send on first connection
	if b.FirstConnection {
		b.connected <- nil
	}
}

func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
	if b.skipPrivMsg(event) {
		return
	}

	rmsg := config.Message{
		Username: event.Source.Name,
		Channel:  strings.ToLower(event.Params[0]),
		Account:  b.Account,
		UserID:   event.Source.Ident + "@" + event.Source.Host,
	}

	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)

	// set action event
	if event.IsAction() {
		rmsg.Event = config.EventUserAction
	}

	// set NOTICE event
	if event.Command == "NOTICE" {
		rmsg.Event = config.EventNoticeIRC
	}

	// strip action, we made an event if it was an action
	rmsg.Text += event.StripAction()

	// start detecting the charset
	mycharset := b.GetString("Charset")
	if mycharset == "" {
		// detect what were sending so that we convert it to utf-8
		detector := chardet.NewTextDetector()
		result, err := detector.DetectBest([]byte(rmsg.Text))
		if err != nil {
			b.Log.Infof("detection failed for rmsg.Text: %#v", rmsg.Text)
			return
		}
		b.Log.Debugf("detected %s confidence %#v", result.Charset, result.Confidence)
		mycharset = result.Charset
		// if we're not sure, just pick ISO-8859-1
		if result.Confidence < 80 {
			mycharset = "ISO-8859-1"
		}
	}
	switch mycharset {
	case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
		rmsg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), rmsg.Text)
	default:
		r, err := charset.NewReader(mycharset, strings.NewReader(rmsg.Text))
		if err != nil {
			b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
			return
		}
		output, _ := ioutil.ReadAll(r)
		rmsg.Text = string(output)
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", event.Params[0], b.Account)
	b.Remote <- rmsg
}

func (b *Birc) handleRunCommands() {
	for _, cmd := range b.GetStringSlice("RunCommands") {
		if err := b.i.Cmd.SendRaw(cmd); err != nil {
			b.Log.Errorf("RunCommands %s failed: %s", cmd, err)
		}
		time.Sleep(time.Second)
	}
}

func (b *Birc) handleTopicWhoTime(client *girc.Client, event girc.Event) {
	parts := strings.Split(event.Params[2], "!")
	t, err := strconv.ParseInt(event.Params[3], 10, 64)
	if err != nil {
		b.Log.Errorf("Invalid time stamp: %s", event.Params[3])
	}
	user := parts[0]
	if len(parts) > 1 {
		user += " [" + parts[1] + "]"
	}
	b.Log.Debugf("%s: Topic set by %s [%s]", event.Command, user, time.Unix(t, 0))
}
<commit_msg>Make handlers run async (irc) (#1325)<commit_after>package birc

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/lrstanley/girc"
	"github.com/missdeer/golib/ic"
	"github.com/paulrosania/go-charset/charset"
	"github.com/saintfish/chardet"

	// We need to import the 'data' package as an implicit dependency.
	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
	_ "github.com/paulrosania/go-charset/data"
)

func (b *Birc) handleCharset(msg *config.Message) error {
	if b.GetString("Charset") != "" {
		switch b.GetString("Charset") {
		case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
			msg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), msg.Text)
		default:
			buf := new(bytes.Buffer)
			w, err := charset.NewWriter(b.GetString("Charset"), buf)
			if err != nil {
				b.Log.Errorf("charset from utf-8 conversion failed: %s", err)
				return err
			}
			fmt.Fprint(w, msg.Text)
			w.Close()
			msg.Text = buf.String()
		}
	}
	return nil
}

// handleFiles returns true if we have handled the files, otherwise return false
func (b *Birc) handleFiles(msg *config.Message) bool {
	if msg.Extra == nil {
		return false
	}
	for _, rmsg := range helper.HandleExtra(msg, b.General) {
		b.Local <- rmsg
	}
	if len(msg.Extra["file"]) == 0 {
		return false
	}
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		if fi.Comment != "" {
			msg.Text += fi.Comment + " : "
		}
		if fi.URL != "" {
			msg.Text = fi.URL
			if fi.Comment != "" {
				msg.Text = fi.Comment + " : " + fi.URL
			}
		}
		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
	}
	return true
}

func (b *Birc) handleInvite(client *girc.Client, event girc.Event) {
	if len(event.Params) != 2 {
		return
	}

	channel := event.Params[1]

	b.Log.Debugf("got invite for %s", channel)

	if _, ok := b.channels[channel]; ok {
		b.i.Cmd.Join(channel)
	}
}

func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
	if len(event.Params) == 0 {
		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
		return
	}
	channel := strings.ToLower(event.Params[0])
	if event.Command == "KICK" && event.Params[1] == b.Nick {
		b.Log.Infof("Got kicked from %s by %s", channel, event.Source.Name)
		time.Sleep(time.Duration(b.GetInt("RejoinDelay")) * time.Second)
		b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: channel, Account: b.Account, Event: config.EventRejoinChannels}
		return
	}
	if event.Command == "QUIT" {
		if event.Source.Name == b.Nick && strings.Contains(event.Last(), "Ping timeout") {
			b.Log.Infof("%s reconnecting ..", b.Account)
			b.Remote <- config.Message{Username: "system", Text: "reconnect", Channel: channel, Account: b.Account, Event: config.EventFailure}
			return
		}
	}
	if event.Source.Name != b.Nick {
		if b.GetBool("nosendjoinpart") {
			return
		}
		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		if b.GetBool("verbosejoinpart") {
			b.Log.Debugf("<= Sending verbose JOIN_LEAVE event from %s to gateway", b.Account)
			msg = config.Message{Username: "system", Text: event.Source.Name + " (" + event.Source.Ident + "@" + event.Source.Host + ") " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		} else {
			b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
		}
		b.Log.Debugf("<= Message is %#v", msg)
		b.Remote <- msg
		return
	}
	b.Log.Debugf("handle %#v", event)
}

func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
	b.Log.Debug("Registering callbacks")
	i := b.i
	b.Nick = event.Params[0]

	i.Handlers.AddBg("PRIVMSG", b.handlePrivMsg)
	i.Handlers.AddBg("CTCP_ACTION", b.handlePrivMsg)
	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
	i.Handlers.AddBg(girc.NOTICE, b.handleNotice)
	i.Handlers.AddBg("JOIN", b.handleJoinPart)
	i.Handlers.AddBg("PART", b.handleJoinPart)
	i.Handlers.AddBg("QUIT", b.handleJoinPart)
	i.Handlers.AddBg("KICK", b.handleJoinPart)
	i.Handlers.Add("INVITE", b.handleInvite)
}

func (b *Birc) handleNickServ() {
	if !b.GetBool("UseSASL") && b.GetString("NickServNick") != "" && b.GetString("NickServPassword") != "" {
		b.Log.Debugf("Sending identify to nickserv %s", b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "IDENTIFY "+b.GetString("NickServPassword"))
	}
	if strings.EqualFold(b.GetString("NickServNick"), "Q@CServe.quakenet.org") {
		b.Log.Debugf("Authenticating %s against %s", b.GetString("NickServUsername"), b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "AUTH "+b.GetString("NickServUsername")+" "+b.GetString("NickServPassword"))
	}
	// give nickserv some slack
	time.Sleep(time.Second * 5)
	b.authDone = true
}

func (b *Birc) handleNotice(client *girc.Client, event girc.Event) {
	if strings.Contains(event.String(), "This nickname is registered") && event.Source.Name == b.GetString("NickServNick") {
		b.handleNickServ()
	} else {
		b.handlePrivMsg(client, event)
	}
}

func (b *Birc) handleOther(client *girc.Client, event girc.Event) {
	if b.GetInt("DebugLevel") == 1 {
		if event.Command != "CLIENT_STATE_UPDATED" &&
			event.Command != "CLIENT_GENERAL_UPDATED" {
			b.Log.Debugf("%#v", event.String())
		}
		return
	}
	switch event.Command {
	case "372", "375", "376", "250", "251", "252", "253", "254", "255", "265", "266", "002", "003", "004", "005":
		return
	}
	b.Log.Debugf("%#v", event.String())
}

func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
	b.handleNickServ()
	b.handleRunCommands()
	// we are now fully connected
	// only send on first connection
	if b.FirstConnection {
		b.connected <- nil
	}
}

func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
	if b.skipPrivMsg(event) {
		return
	}

	rmsg := config.Message{
		Username: event.Source.Name,
		Channel:  strings.ToLower(event.Params[0]),
		Account:  b.Account,
		UserID:   event.Source.Ident + "@" + event.Source.Host,
	}

	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)

	// set action event
	if event.IsAction() {
		rmsg.Event = config.EventUserAction
	}

	// set NOTICE event
	if event.Command == "NOTICE" {
		rmsg.Event = config.EventNoticeIRC
	}

	// strip action, we made an event if it was an action
	rmsg.Text += event.StripAction()

	// start detecting the charset
	mycharset := b.GetString("Charset")
	if mycharset == "" {
		// detect what were sending so that we convert it to utf-8
		detector := chardet.NewTextDetector()
		result, err := detector.DetectBest([]byte(rmsg.Text))
		if err != nil {
			b.Log.Infof("detection failed for rmsg.Text: %#v", rmsg.Text)
			return
		}
		b.Log.Debugf("detected %s confidence %#v", result.Charset, result.Confidence)
		mycharset = result.Charset
		// if we're not sure, just pick ISO-8859-1
		if result.Confidence < 80 {
			mycharset = "ISO-8859-1"
		}
	}
	switch mycharset {
	case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
		rmsg.Text = ic.ConvertString("utf-8", b.GetString("Charset"), rmsg.Text)
	default:
		r, err := charset.NewReader(mycharset, strings.NewReader(rmsg.Text))
		if err != nil {
			b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
			return
		}
		output, _ := ioutil.ReadAll(r)
		rmsg.Text = string(output)
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", event.Params[0], b.Account)
	b.Remote <- rmsg
}

func (b *Birc) handleRunCommands() {
	for _, cmd := range b.GetStringSlice("RunCommands") {
		if err := b.i.Cmd.SendRaw(cmd); err != nil {
			b.Log.Errorf("RunCommands %s failed: %s", cmd, err)
		}
		time.Sleep(time.Second)
	}
}

func (b *Birc) handleTopicWhoTime(client *girc.Client, event girc.Event) {
	parts := strings.Split(event.Params[2], "!")
	t, err := strconv.ParseInt(event.Params[3], 10, 64)
	if err != nil {
		b.Log.Errorf("Invalid time stamp: %s", event.Params[3])
	}
	user := parts[0]
	if len(parts) > 1 {
		user += " [" + parts[1] + "]"
	}
	b.Log.Debugf("%s: Topic set by %s [%s]", event.Command, user, time.Unix(t, 0))
}
<|endoftext|>


<commit_before>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"unicode/utf8"
)

var (
	pkg     = flag.String("p", "main", "package name")
	in      = flag.String("i", "", "input filename")
	out     = flag.String("o", "", "output filename")
	varname = flag.String("v", "httpdir.Default", "http dir variable name")
	help    = flag.Bool("h", false, "show help")
	comp    = flag.Bool("c", false, "compress using gzip")
)

func errHandler(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

type replacer struct {
	f *os.File
}

var hexArr = [16]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}

func (r replacer) Write(p []byte) (int, error) {
	n := 0
	toWrite := make([]byte, 0, 5)
	for len(p) > 0 {
		_, s := utf8.DecodeRune(p)
		if s > 1 || (p[0] > 0 && p[0] < 0x7f && p[0] != '\n' && p[0] != '\\' && p[0] != '"') {
			toWrite = append(toWrite[:0], p[:s]...)
		} else {
			switch p[0] {
			case '\n':
				toWrite = append(toWrite[:0], '\\', 'n')
			case '\\':
				toWrite = append(toWrite[:0], '\\', '\\')
			case '"':
				toWrite = append(toWrite[:0], '\\', '"')
			default:
				toWrite = append(toWrite[:0], '\\', 'x', hexArr[p[0]>>4], hexArr[p[0]&15])
			}
		}
		_, err := r.f.Write(toWrite)
		n += s
		if err != nil {
			return n, err
		}
		p = p[s:]
	}
	return n, nil
}

type tickReplacer struct {
	f *os.File
}

func (r tickReplacer) Write(p []byte) (int, error) {
	var (
		m, n int
		err  error
	)
	hexes := make([]byte, 0, 32)
	toWrite := make([]byte, 0, 1024)
	for len(p) > 0 {
		dr, s := utf8.DecodeRune(p)
		if dr == utf8.RuneError || dr == '`' || dr == '\r' {
			hexes = append(hexes, p[0])
		} else {
			if len(hexes) > 0 {
				toWrite = toWrite[:0]
				toWrite = append(toWrite, '`', '+', '"')
				for _, b := range hexes {
					if b == '`' {
						toWrite = append(toWrite, '`')
					} else if b == '\r' {
						toWrite = append(toWrite, '\\', 'r')
					} else {
						toWrite = append(toWrite, '\\', 'x', hexArr[b>>4], hexArr[b&15])
					}
				}
				toWrite = append(toWrite, '"', '+', '`')
				_, err = r.f.Write(toWrite)
				n += len(hexes)
				if err != nil {
					break
				}
				hexes = hexes[:0]
			}
			m, err = r.f.Write(p[:s])
			n += m
			if err != nil {
				break
			}
		}
		p = p[s:]
	}
	return n, err
}

func main() {
	flag.Parse()
	if *help {
		flag.Usage()
		return
	}
	if *in == "" || *out == "" {
		errHandler(errors.New("missing in/out file"))
	}
	fi, err := os.Open(*in)
	errHandler(err)
	defer fi.Close()
	stat, err := fi.Stat()
	errHandler(err)
	fo, err := os.Create(*out)
	errHandler(err)
	defer fo.Close()
	if *comp {
		_, err = fmt.Fprintf(fo, compressedStart, *pkg, *varname, *in)
		errHandler(err)
		w, err := gzip.NewWriterLevel(replacer{fo}, gzip.BestCompression)
		errHandler(err)
		_, err = io.Copy(w, fi)
		errHandler(err)
		errHandler(w.Close())
		_, err = fmt.Fprintf(fo, compressedEnd, stat.ModTime().Unix(), stat.Size())
		errHandler(err)
	} else {
		_, err = fmt.Fprintf(fo, uncompressedStart, *pkg, *varname, *in)
		errHandler(err)
		_, err = io.Copy(tickReplacer{fo}, fi)
		errHandler(err)
		_, err = fmt.Fprintf(fo, uncompressedEnd, stat.ModTime().Unix())
		errHandler(err)
	}
}

const (
	uncompressedStart = `package %s

import (
	"time"

	"github.com/MJKWoolnough/httpdir"
)

func init() {
	%s.Create(%q, httpdir.FileString(` + "`"
	uncompressedEnd = "`" + `, time.Unix(%d, 0)))
}
`

	compressedStart = `package %s

import (
	"time"

	"github.com/MJKWoolnough/httpdir"
)

func init() {
	if err := httpdir.Compressed(%s, %q, httpdir.FileString("`
	compressedEnd = `", time.Unix(%d, 0)), %d); err != nil {
		panic(err)
	}
}
`
)
<commit_msg>added flag to specify different path for web path<commit_after>package main

import (
	"compress/gzip"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"unicode/utf8"
)

var (
	pkg     = flag.String("p", "main", "package name")
	in      = flag.String("i", "", "input filename")
	out     = flag.String("o", "", "output filename")
	path    = flag.String("w", "", "http path")
	varname = flag.String("v", "httpdir.Default", "http dir variable name")
	help    = flag.Bool("h", false, "show help")
	comp    = flag.Bool("c", false, "compress using gzip")
)

func errHandler(err error) {
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}
}

type replacer struct {
	f *os.File
}

var hexArr = [16]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}

func (r replacer) Write(p []byte) (int, error) {
	n := 0
	toWrite := make([]byte, 0, 5)
	for len(p) > 0 {
		_, s := utf8.DecodeRune(p)
		if s > 1 || (p[0] > 0 && p[0] < 0x7f && p[0] != '\n' && p[0] != '\\' && p[0] != '"') {
			toWrite = append(toWrite[:0], p[:s]...)
		} else {
			switch p[0] {
			case '\n':
				toWrite = append(toWrite[:0], '\\', 'n')
			case '\\':
				toWrite = append(toWrite[:0], '\\', '\\')
			case '"':
				toWrite = append(toWrite[:0], '\\', '"')
			default:
				toWrite = append(toWrite[:0], '\\', 'x', hexArr[p[0]>>4], hexArr[p[0]&15])
			}
		}
		_, err := r.f.Write(toWrite)
		n += s
		if err != nil {
			return n, err
		}
		p = p[s:]
	}
	return n, nil
}

type tickReplacer struct {
	f *os.File
}

func (r tickReplacer) Write(p []byte) (int, error) {
	var (
		m, n int
		err  error
	)
	hexes := make([]byte, 0, 32)
	toWrite := make([]byte, 0, 1024)
	for len(p) > 0 {
		dr, s := utf8.DecodeRune(p)
		if dr == utf8.RuneError || dr == '`' || dr == '\r' {
			hexes = append(hexes, p[0])
		} else {
			if len(hexes) > 0 {
				toWrite = toWrite[:0]
				toWrite = append(toWrite, '`', '+', '"')
				for _, b := range hexes {
					if b == '`' {
						toWrite = append(toWrite, '`')
					} else if b == '\r' {
						toWrite = append(toWrite, '\\', 'r')
					} else {
						toWrite = append(toWrite, '\\', 'x', hexArr[b>>4], hexArr[b&15])
					}
				}
				toWrite = append(toWrite, '"', '+', '`')
				_, err = r.f.Write(toWrite)
				n += len(hexes)
				if err != nil {
					break
				}
				hexes = hexes[:0]
			}
			m, err = r.f.Write(p[:s])
			n += m
			if err != nil {
				break
			}
		}
		p = p[s:]
	}
	return n, err
}

func main() {
	flag.Parse()
	if *help {
		flag.Usage()
		return
	}
	if *in == "" || *out == "" {
		errHandler(errors.New("missing in/out file"))
	}
	fi, err := os.Open(*in)
	errHandler(err)
	defer fi.Close()
	stat, err := fi.Stat()
	errHandler(err)
	fo, err := os.Create(*out)
	errHandler(err)
	defer fo.Close()
	if *path == "" {
		path = in
	}
	if *comp {
		_, err = fmt.Fprintf(fo, compressedStart, *pkg, *varname, *path)
		errHandler(err)
		w, err := gzip.NewWriterLevel(replacer{fo}, gzip.BestCompression)
		errHandler(err)
		_, err = io.Copy(w, fi)
		errHandler(err)
		errHandler(w.Close())
		_, err = fmt.Fprintf(fo, compressedEnd, stat.ModTime().Unix(), stat.Size())
		errHandler(err)
	} else {
		_, err = fmt.Fprintf(fo, uncompressedStart, *pkg, *varname, *path)
		errHandler(err)
		_, err = io.Copy(tickReplacer{fo}, fi)
		errHandler(err)
		_, err = fmt.Fprintf(fo, uncompressedEnd, stat.ModTime().Unix())
		errHandler(err)
	}
}

const (
	uncompressedStart = `package %s

import (
	"time"

	"github.com/MJKWoolnough/httpdir"
)

func init() {
	%s.Create(%q, httpdir.FileString(` + "`"
	uncompressedEnd = "`" + `, time.Unix(%d, 0)))
}
`

	compressedStart = `package %s

import (
	"time"

	"github.com/MJKWoolnough/httpdir"
)

func init() {
	if err := httpdir.Compressed(%s, %q, httpdir.FileString("`
	compressedEnd = `", time.Unix(%d, 0)), %d); err != nil {
		panic(err)
	}
}
`
)
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "asami",
	Short: "a simple image corruptor",
	Long:  ``,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
}
<commit_msg>fix typo<commit_after>package cmd

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "asami",
	Short: "simple image corrupter",
	Long:  ``,
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"
	"strconv"

	"github.com/spf13/cobra"
	"github.com/srinandan/apigeecli/apiclient"
	"github.com/srinandan/apigeecli/cmd/apis"
	"github.com/srinandan/apigeecli/cmd/apps"
	cache "github.com/srinandan/apigeecli/cmd/cache"
	"github.com/srinandan/apigeecli/cmd/developers"
	"github.com/srinandan/apigeecli/cmd/env"
	flowhooks "github.com/srinandan/apigeecli/cmd/flowhooks"
	"github.com/srinandan/apigeecli/cmd/iam"
	"github.com/srinandan/apigeecli/cmd/keyaliases"
	"github.com/srinandan/apigeecli/cmd/keystores"
	"github.com/srinandan/apigeecli/cmd/kvm"
	"github.com/srinandan/apigeecli/cmd/org"
	"github.com/srinandan/apigeecli/cmd/products"
	"github.com/srinandan/apigeecli/cmd/projects"
	"github.com/srinandan/apigeecli/cmd/references"
	res "github.com/srinandan/apigeecli/cmd/res"
	"github.com/srinandan/apigeecli/cmd/sharedflows"
	"github.com/srinandan/apigeecli/cmd/sync"
	targetservers "github.com/srinandan/apigeecli/cmd/targetservers"
	"github.com/srinandan/apigeecli/cmd/token"
)

//RootCmd to manage apigeecli
var RootCmd = &cobra.Command{
	Use:   "apigeecli",
	Short: "Utility to work with Apigee APIs.",
	Long:  "This command lets you interact with Apigee APIs.",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		apiclient.SetServiceAccount(serviceAccount)
		apiclient.SetApigeeToken(accessToken)

		err := apiclient.SetAccessToken()
		if err != nil {
			return err
		}
		return nil
	},
}

func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

var accessToken, serviceAccount string

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVarP(&accessToken, "token", "t",
		"", "Google OAuth Token")

	RootCmd.PersistentFlags().StringVarP(&serviceAccount, "account", "a",
		"", "Path Service Account private key in JSON")

	RootCmd.AddCommand(apis.Cmd)
	RootCmd.AddCommand(org.Cmd)
	RootCmd.AddCommand(sync.Cmd)
	RootCmd.AddCommand(env.Cmd)
	RootCmd.AddCommand(products.Cmd)
	RootCmd.AddCommand(developers.Cmd)
	RootCmd.AddCommand(apps.Cmd)
	RootCmd.AddCommand(sharedflows.Cmd)
	RootCmd.AddCommand(kvm.Cmd)
	RootCmd.AddCommand(flowhooks.Cmd)
	RootCmd.AddCommand(targetservers.Cmd)
	RootCmd.AddCommand(token.Cmd)
	RootCmd.AddCommand(keystores.Cmd)
	RootCmd.AddCommand(keyaliases.Cmd)
	RootCmd.AddCommand(cache.Cmd)
	RootCmd.AddCommand(references.Cmd)
	RootCmd.AddCommand(res.Cmd)
	RootCmd.AddCommand(projects.Cmd)
	RootCmd.AddCommand(iam.Cmd)
}

func initConfig() {
	var skipLogInfo, skipCache = true, false

	if os.Getenv("APIGEECLI_SKIPLOG") == "false" {
		skipLogInfo = false
	}

	skipCache, _ = strconv.ParseBool(os.Getenv("APIGEECLI_SKIPCACHE"))

	apiclient.NewApigeeClient(apiclient.ApigeeClientOptions{
		SkipCheck:   true,
		PrintOutput: true,
		SkipLogInfo: skipLogInfo,
		SkipCache:   skipCache,
	})
}

// GetRootCmd returns the root of the cobra command-tree.
func GetRootCmd() *cobra.Command {
	return RootCmd
}
<commit_msg>golint<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"
	"strconv"

	"github.com/spf13/cobra"
	"github.com/srinandan/apigeecli/apiclient"
	"github.com/srinandan/apigeecli/cmd/apis"
	"github.com/srinandan/apigeecli/cmd/apps"
	cache "github.com/srinandan/apigeecli/cmd/cache"
	"github.com/srinandan/apigeecli/cmd/developers"
	"github.com/srinandan/apigeecli/cmd/env"
	flowhooks "github.com/srinandan/apigeecli/cmd/flowhooks"
	"github.com/srinandan/apigeecli/cmd/iam"
	"github.com/srinandan/apigeecli/cmd/keyaliases"
	"github.com/srinandan/apigeecli/cmd/keystores"
	"github.com/srinandan/apigeecli/cmd/kvm"
	"github.com/srinandan/apigeecli/cmd/org"
	"github.com/srinandan/apigeecli/cmd/products"
	"github.com/srinandan/apigeecli/cmd/projects"
	"github.com/srinandan/apigeecli/cmd/references"
	res "github.com/srinandan/apigeecli/cmd/res"
	"github.com/srinandan/apigeecli/cmd/sharedflows"
	"github.com/srinandan/apigeecli/cmd/sync"
	targetservers "github.com/srinandan/apigeecli/cmd/targetservers"
	"github.com/srinandan/apigeecli/cmd/token"
)

//RootCmd to manage apigeecli
var RootCmd = &cobra.Command{
	Use:   "apigeecli",
	Short: "Utility to work with Apigee APIs.",
	Long:  "This command lets you interact with Apigee APIs.",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		apiclient.SetServiceAccount(serviceAccount)
		apiclient.SetApigeeToken(accessToken)

		err := apiclient.SetAccessToken()
		if err != nil {
			return err
		}
		return nil
	},
}

func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

var accessToken, serviceAccount string

func init() {
	cobra.OnInitialize(initConfig)

	RootCmd.PersistentFlags().StringVarP(&accessToken, "token", "t",
		"", "Google OAuth Token")

	RootCmd.PersistentFlags().StringVarP(&serviceAccount, "account", "a",
		"", "Path Service Account private key in JSON")

	RootCmd.AddCommand(apis.Cmd)
	RootCmd.AddCommand(org.Cmd)
	RootCmd.AddCommand(sync.Cmd)
	RootCmd.AddCommand(env.Cmd)
	RootCmd.AddCommand(products.Cmd)
	RootCmd.AddCommand(developers.Cmd)
	RootCmd.AddCommand(apps.Cmd)
	RootCmd.AddCommand(sharedflows.Cmd)
	RootCmd.AddCommand(kvm.Cmd)
	RootCmd.AddCommand(flowhooks.Cmd)
	RootCmd.AddCommand(targetservers.Cmd)
	RootCmd.AddCommand(token.Cmd)
	RootCmd.AddCommand(keystores.Cmd)
	RootCmd.AddCommand(keyaliases.Cmd)
	RootCmd.AddCommand(cache.Cmd)
	RootCmd.AddCommand(references.Cmd)
	RootCmd.AddCommand(res.Cmd)
	RootCmd.AddCommand(projects.Cmd)
	RootCmd.AddCommand(iam.Cmd)
}

func initConfig() {
	var skipLogInfo = true
	var skipCache bool

	if os.Getenv("APIGEECLI_SKIPLOG") == "false" {
		skipLogInfo = false
	}

	skipCache, _ = strconv.ParseBool(os.Getenv("APIGEECLI_SKIPCACHE"))

	apiclient.NewApigeeClient(apiclient.ApigeeClientOptions{
		SkipCheck:   true,
		PrintOutput: true,
		SkipLogInfo: skipLogInfo,
		SkipCache:   skipCache,
	})
}

// GetRootCmd returns the root of the cobra command-tree.
func GetRootCmd() *cobra.Command {
	return RootCmd
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/stormforger/cli/api"
)

var (
	RootCmd = &cobra.Command{
		Use:   "forge",
		Short: "Command line client to StormForger (https://stormforger.com)",
		Long: `The command line client "forge" to StormForger offers a interface
to the StormForger API and several convinience methods
to work on load and performance tests.

Happy Load Testing :)`,
	}

	rootOpts struct {
		APIEndpoint string
		JWTToken    string
	}
)

const (
	CONFIG_FILENAME = ".stormforger"
	ENV_PREFIX      = "stormforger"
)

type BuildInfo struct {
	Version string
	Time    string
	Commit  string
}

func (buildInfo BuildInfo) String() string {
	return fmt.Sprintf("%v %v (%v - %v) - https://stormforger.com", RootCmd.Use, buildInfo.Version, buildInfo.Time, buildInfo.Commit)
}

func (buildInfo BuildInfo) ShortString() string {
	return buildInfo.Version
}

var buildInfo BuildInfo

// Execute
// searchs for JWT-token,
// adds all child commands to the root command
// and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute(version string, buildTime string, buildCommit string) {
	buildInfo = BuildInfo{version, buildTime, buildCommit}

	// most important thing is the jwt
	//viper.SetDefault("jwt", "")
	findJwt()

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}

	if viper.GetString("jwt") == "" {
		color.Yellow("\nNo JWT token in config file, environment or via command line flag!\nPlease provide JWT token to talk to the StormForger API. See README.md.\n")
	} else {
		color.Green("\nUsing token: %s\n", viper.GetString("jwt"))
	}
}

func NewClient() *api.Client {
	return api.NewClient(rootOpts.APIEndpoint, viper.GetString("jwt"))
}

// JWT token must be either in one of these locations
// .stormforger.toml
// $HOMR/.stormforger.toml
// $ENV["STORMFORGER_JWT"]
// via cli flag --jwt-token
func findJwt() bool {
	// ENV overrides config
	viper.SetEnvPrefix(ENV_PREFIX)
	viper.BindEnv("jwt")

	// config
	viper.SetConfigName(CONFIG_FILENAME) // name of config file (without extension)
	viper.AddConfigPath("$HOME")         // call multiple times to add many search paths
	viper.AddConfigPath(".")             // optionally look for config in the working directory
	err := viper.ReadInConfig()          // Find and read the config file
	if err != nil {
		// Handle errors reading the config file
		// no config
		// TODO I don't think that we inform the user at this stage about a missing config file
		// log.Println("No config file: %s \n", err)
	}

	// command line flag overrides config and ENV
	viper.BindPFlag("jwt", RootCmd.Flags().Lookup("jwt-token"))

	return true
}

func init() {
	RootCmd.PersistentFlags().StringVar(&rootOpts.APIEndpoint, "api-endpoint", "https://api.stormforger.com", "API Endpoint")
	RootCmd.Flags().StringVar(&rootOpts.JWTToken, "jwt-token", "", "JWT Token")
}
<commit_msg>convinience ->  convenience<commit_after>package cmd

import (
	"fmt"
	"os"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	"github.com/stormforger/cli/api"
)

var (
	RootCmd = &cobra.Command{
		Use:   "forge",
		Short: "Command line client to StormForger (https://stormforger.com)",
		Long: `The command line client "forge" to StormForger offers a interface
to the StormForger API and several convenience methods
to handle load and performance tests.

Happy Load Testing :)`,
	}

	rootOpts struct {
		APIEndpoint string
		JWTToken    string
	}
)

const (
	CONFIG_FILENAME = ".stormforger"
	ENV_PREFIX      = "stormforger"
)

type BuildInfo struct {
	Version string
	Time    string
	Commit  string
}

func (buildInfo BuildInfo) String() string {
	return fmt.Sprintf("%v %v (%v - %v) - https://stormforger.com", RootCmd.Use, buildInfo.Version, buildInfo.Time, buildInfo.Commit)
}

func (buildInfo BuildInfo) ShortString() string {
	return buildInfo.Version
}

var buildInfo BuildInfo

// Execute
// searchs for JWT-token,
// adds all child commands to the root command
// and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute(version string, buildTime string, buildCommit string) {
	buildInfo = BuildInfo{version, buildTime, buildCommit}

	// most important thing is the jwt
	//viper.SetDefault("jwt", "")
	findJwt()

	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}

	if viper.GetString("jwt") == "" {
		color.Yellow("\nNo JWT token in config file, environment or via command line flag!\nPlease provide JWT token to talk to the StormForger API. See README.md.\n")
	} else {
		color.Green("\nUsing token: %s\n", viper.GetString("jwt"))
	}
}

func NewClient() *api.Client {
	return api.NewClient(rootOpts.APIEndpoint, viper.GetString("jwt"))
}

// JWT token must be either in one of these locations
// .stormforger.toml
// $HOMR/.stormforger.toml
// $ENV["STORMFORGER_JWT"]
// via cli flag --jwt-token
func findJwt() bool {
	// ENV overrides config
	viper.SetEnvPrefix(ENV_PREFIX)
	viper.BindEnv("jwt")

	// config
	viper.SetConfigName(CONFIG_FILENAME) // name of config file (without extension)
	viper.AddConfigPath("$HOME")         // call multiple times to add many search paths
	viper.AddConfigPath(".")             // optionally look for config in the working directory
	err := viper.ReadInConfig()          // Find and read the config file
	if err != nil {
		// Handle errors reading the config file
		// no config
		// TODO I don't think that we inform the user at this stage about a missing config file
		// log.Println("No config file: %s \n", err)
	}

	// command line flag overrides config and ENV
	viper.BindPFlag("jwt", RootCmd.Flags().Lookup("jwt-token"))

	return true
}

func init() {
	RootCmd.PersistentFlags().StringVar(&rootOpts.APIEndpoint, "api-endpoint", "https://api.stormforger.com", "API Endpoint")
	RootCmd.Flags().StringVar(&rootOpts.JWTToken, "jwt-token", "", "JWT Token")
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Unknwon/com"
	"github.com/urfave/cli"
	log "gopkg.in/clog.v1"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/models/errors"
	"github.com/gogits/gogs/pkg/setting"
	http "github.com/gogits/gogs/routes/repo"
)

const (
	_ACCESS_DENIED_MESSAGE = "Repository does not exist or you do not have access"
)

var Serv = cli.Command{
	Name:        "serv",
	Usage:       "This command should only be called by SSH shell",
	Description: `Serv provide access auth for repositories`,
	Action:      runServ,
	Flags: []cli.Flag{
		stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
	},
}

func fail(userMessage, logMessage string, args ...interface{}) {
	fmt.Fprintln(os.Stderr, "Gogs:", userMessage)

	if len(logMessage) > 0 {
		if !setting.ProdMode {
			fmt.Fprintf(os.Stderr, logMessage+"\n", args...)
		}
		log.Fatal(3, logMessage, args...)
	}

	os.Exit(1)
}

func setup(c *cli.Context, logPath string, connectDB bool) {
	if c.IsSet("config") {
		setting.CustomConf = c.String("config")
	} else if c.GlobalIsSet("config") {
		setting.CustomConf = c.GlobalString("config")
	}

	setting.NewContext()

	level := log.TRACE
	if setting.ProdMode {
		level = log.ERROR
	}
	log.New(log.FILE, log.FileConfig{
		Level:    level,
		Filename: filepath.Join(setting.LogRootPath, logPath),
		FileRotationConfig: log.FileRotationConfig{
			Rotate:  true,
			Daily:   true,
			MaxDays: 3,
		},
	})
	log.Delete(log.CONSOLE) // Remove primary logger

	if !connectDB {
		return
	}

	models.LoadConfigs()

	if setting.UseSQLite3 {
		workDir, _ := setting.WorkDir()
		os.Chdir(workDir)
	}

	if err := models.SetEngine(); err != nil {
		fail("Internal error", "SetEngine: %v", err)
	}
}

func parseSSHCmd(cmd string) (string, string, []string) {
	ss := strings.Split(cmd, " ")
	if len(ss) < 2 {
		return "", "", nil
	}
	return ss[0], strings.Replace(ss[len(ss)-1], "/", "'", 1), ss
}

func checkDeployKey(key *models.PublicKey, repo *models.Repository) {
	// Check if this deploy key belongs to current repository.
	if !models.HasDeployKey(key.ID, repo.ID) {
		fail("Key access denied", "Deploy key access denied: [key_id: %d, repo_id: %d]", key.ID, repo.ID)
	}

	// Update deploy key activity.
	deployKey, err := models.GetDeployKeyByRepo(key.ID, repo.ID)
	if err != nil {
		fail("Internal error", "GetDeployKey: %v", err)
	}

	deployKey.Updated = time.Now()
	if err = models.UpdateDeployKey(deployKey); err != nil {
		fail("Internal error", "UpdateDeployKey: %v", err)
	}
}

var (
	allowedCommands = map[string]models.AccessMode{
		"git-upload-pack":    models.ACCESS_MODE_READ,
		"git-upload-archive": models.ACCESS_MODE_READ,
		"git-receive-pack":   models.ACCESS_MODE_WRITE,
		"git-annex-shell":    models.ACCESS_MODE_WRITE,
	}
)

func runServ(c *cli.Context) error {
	setup(c, "serv.log", true)

	if setting.SSH.Disabled {
		println("Gogs: SSH has been disabled")
		return nil
	}

	if len(c.Args()) < 1 {
		fail("Not enough arguments", "Not enough arguments")
	}

	sshCmd := strings.Replace(os.Getenv("SSH_ORIGINAL_COMMAND"), "'", "", -1)
	log.Info("SSH commadn:%s", sshCmd)
	if len(sshCmd) == 0 {
		println("Hi there, You've successfully authenticated, but Gogs does not provide shell access.")
		println("If this is unexpected, please log in with password and setup Gogs under another user.")
		return nil
	}

	verb, path, args := parseSSHCmd(sshCmd)
	repoFullName := strings.ToLower(strings.Trim(path, "'"))
	repoFields := strings.SplitN(repoFullName, "/", 2)
	if len(repoFields) != 2 {
		fail("Invalid repository path", "Invalid repository path: %v", path)
	}
	ownerName := strings.ToLower(repoFields[0])
	repoName := strings.TrimSuffix(strings.ToLower(repoFields[1]), ".git")
	repoName = strings.TrimSuffix(repoName, ".wiki")

	owner, err := models.GetUserByName(ownerName)
	if err != nil {
		if errors.IsUserNotExist(err) {
			fail("Repository owner does not exist", "Unregistered owner: %s", ownerName)
		}
		fail("Internal error", "Fail to get repository owner '%s': %v", ownerName, err)
	}

	repo, err := models.GetRepositoryByName(owner.ID, repoName)
	if err != nil {
		if errors.IsRepoNotExist(err) {
			fail(_ACCESS_DENIED_MESSAGE, "Repository does not exist: %s/%s", owner.Name, repoName)
		}
		fail("Internal error", "Fail to get repository: %v", err)
	}
	repo.Owner = owner

	requestMode, ok := allowedCommands[verb]
	if !ok {
		fail("Unknown git command", "Unknown git command '%s'", verb)
	}

	// Prohibit push to mirror repositories.
	if requestMode > models.ACCESS_MODE_READ && repo.IsMirror {
		fail("Mirror repository is read-only", "")
	}

	// Allow anonymous (user is nil) clone for public repositories.
	var user *models.User

	key, err := models.GetPublicKeyByID(com.StrTo(strings.TrimPrefix(c.Args()[0], "key-")).MustInt64())
	if err != nil {
		fail("Invalid key ID", "Invalid key ID '%s': %v", c.Args()[0], err)
	}

	if requestMode == models.ACCESS_MODE_WRITE || repo.IsPrivate {
		// Check deploy key or user key.
		if key.IsDeployKey() {
			if key.Mode < requestMode {
				fail("Key permission denied", "Cannot push with deployment key: %d", key.ID)
			}
			checkDeployKey(key, repo)
		} else {
			user, err = models.GetUserByKeyID(key.ID)
			if err != nil {
				fail("Internal error", "Fail to get user by key ID '%d': %v", key.ID, err)
			}

			mode, err := models.AccessLevel(user.ID, repo)
			if err != nil {
				fail("Internal error", "Fail to check access: %v", err)
			}

			if mode < requestMode {
				clientMessage := _ACCESS_DENIED_MESSAGE
				if mode >= models.ACCESS_MODE_READ {
					clientMessage = "You do not have sufficient authorization for this action"
				}
				fail(clientMessage,
					"User '%s' does not have level '%v' access to repository '%s'",
					user.Name, requestMode, repoFullName)
			}
		}
	} else {
		setting.NewService()
		// Check if the key can access to the repository in case of it is a deploy key (a deploy keys != user key).
		// A deploy key doesn't represent a signed in user, so in a site with Service.RequireSignInView activated
		// we should give read access only in repositories where this deploy key is in use. In other case, a server
		// or system using an active deploy key can get read access to all the repositories in a Gogs service.
		if key.IsDeployKey() && setting.Service.RequireSignInView {
			checkDeployKey(key, repo)
		}
	}

	// Update user key activity.
	if key.ID > 0 {
		key, err := models.GetPublicKeyByID(key.ID)
		if err != nil {
			fail("Internal error", "GetPublicKeyByID: %v", err)
		}

		key.Updated = time.Now()
		if err = models.UpdatePublicKey(key); err != nil {
			fail("Internal error", "UpdatePublicKey: %v", err)
		}
	}

	// Special handle for Windows.
	// Todo will break with annex
	if setting.IsWindows {
		verb = strings.Replace(verb, "-", " ", 1)
	}
	verbs := strings.Split(verb, " ")
	var cmd []string
	if len(verbs) == 2 {
		cmd = []string{verbs[0], verbs[1], repoFullName}
	} else if (verb == "git-annex-shell") {
		repoAbsPath := setting.RepoRootPath + "/" + repoFullName
		if err := secureGitAnnex(repoAbsPath, requestMode); err != nil {
			fail("Git annex failed", "Git annex failed: %s", err)
		}
		cmd = args
		// Setting full path to repo as git-annex-shell requires it
		cmd[len(cmd)-1] = repoAbsPath
	} else {
		cmd = []string{verb, repoFullName}
	}
	return runGit(cmd, requestMode, user, owner, repo)

}

func runGit(cmd [] string, requestMode models.AccessMode, user *models.User, owner *models.User,
	repo *models.Repository) error {
	log.Info("will exectute:%s", cmd)
	gitCmd := exec.Command(cmd[0], cmd[1:]...)
	if requestMode == models.ACCESS_MODE_WRITE {
		gitCmd.Env = append(os.Environ(), http.ComposeHookEnvs(http.ComposeHookEnvsOptions{
			AuthUser:  user,
			OwnerName: owner.Name,
			OwnerSalt: owner.Salt,
			RepoID:    repo.ID,
			RepoName:  repo.Name,
			RepoPath:  repo.RepoPath(),
		})...)
	}
	gitCmd.Dir = setting.RepoRootPath
	gitCmd.Stdout = os.Stdout
	gitCmd.Stdin = os.Stdin
	gitCmd.Stderr = os.Stderr
	log.Info("args:%s", gitCmd.Args)
	if err := gitCmd.Run(); err != nil {
		fail("Internal error", "Fail to execute git command: %v", err)
	}

	return nil
}

// Make sure git-annex-shell does not make "bad" changes (refectored from repo)
func secureGitAnnex(path string, requestMode models.AccessMode) error {
	// "If set, disallows running git-shell to handle unknown commands."
	err := os.Setenv("GIT_ANNEX_SHELL_LIMITED", "True")
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell to be limited.")
	}
	// "If set, git-annex-shell will refuse to run commands
	//  that do not operate on the specified directory."
	err = os.Setenv("GIT_ANNEX_SHELL_DIRECTORY", path)
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell directory.")
	}
	if ! (requestMode > models.ACCESS_MODE_READ) {
		err = os.Setenv("GIT_ANNEX_SHELL_READONLY", "True")
		if err != nil {
			return fmt.Errorf("ERROR: Could set annex shell to read only.")
		}
	}
	return nil
}
<commit_msg>[annex] allow for more complex git annexes<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/Unknwon/com"
	"github.com/urfave/cli"
	log "gopkg.in/clog.v1"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/models/errors"
	"github.com/gogits/gogs/pkg/setting"
	http "github.com/gogits/gogs/routers/repo"
	"syscall"
)

const (
	_ACCESS_DENIED_MESSAGE = "Repository does not exist or you do not have access"
)

var Serv = cli.Command{
	Name:        "serv",
	Usage:       "This command should only be called by SSH shell",
	Description: `Serv provide access auth for repositories`,
	Action:      runServ,
	Flags: []cli.Flag{
		stringFlag("config, c", "custom/conf/app.ini", "Custom configuration file path"),
	},
}

func fail(userMessage, logMessage string, args ...interface{}) {
	fmt.Fprintln(os.Stderr, "Gogs:", userMessage)

	if len(logMessage) > 0 {
		if !setting.ProdMode {
			fmt.Fprintf(os.Stderr, logMessage+"\n", args...)
		}
		log.Fatal(3, logMessage, args...)
	}

	os.Exit(1)
}

func setup(c *cli.Context, logPath string, connectDB bool) {
	if c.IsSet("config") {
		setting.CustomConf = c.String("config")
	} else if c.GlobalIsSet("config") {
		setting.CustomConf = c.GlobalString("config")
	}

	setting.NewContext()

	level := log.TRACE
	if setting.ProdMode {
		level = log.ERROR
	}
	log.New(log.FILE, log.FileConfig{
		Level:    level,
		Filename: filepath.Join(setting.LogRootPath, logPath),
		FileRotationConfig: log.FileRotationConfig{
			Rotate:  true,
			Daily:   true,
			MaxDays: 3,
		},
	})
	log.Delete(log.CONSOLE) // Remove primary logger

	if !connectDB {
		return
	}

	models.LoadConfigs()

	if setting.UseSQLite3 {
		workDir, _ := setting.WorkDir()
		os.Chdir(workDir)
	}

	if err := models.SetEngine(); err != nil {
		fail("Internal error", "SetEngine: %v", err)
	}
}

func isAnnexShell(cmd string) bool {
	return cmd == "git-annex-shell"
}

func parseSSHCmd(cmd string) (string, string, []string) {
	ss := strings.Split(cmd, " ")
	if len(ss) < 2 {
		return "", "", nil
	}
	if isAnnexShell(ss[0]) {
		return ss[0], strings.Replace(ss[2], "/", "'", 1), ss
	} else {
		return ss[0], strings.Replace(ss[1], "/", "'", 1), ss
	}
}

func checkDeployKey(key *models.PublicKey, repo *models.Repository) {
	// Check if this deploy key belongs to current repository.
	if !models.HasDeployKey(key.ID, repo.ID) {
		fail("Key access denied", "Deploy key access denied: [key_id: %d, repo_id: %d]", key.ID, repo.ID)
	}

	// Update deploy key activity.
	deployKey, err := models.GetDeployKeyByRepo(key.ID, repo.ID)
	if err != nil {
		fail("Internal error", "GetDeployKey: %v", err)
	}

	deployKey.Updated = time.Now()
	if err = models.UpdateDeployKey(deployKey); err != nil {
		fail("Internal error", "UpdateDeployKey: %v", err)
	}
}

var (
	allowedCommands = map[string]models.AccessMode{
		"git-upload-pack":    models.ACCESS_MODE_READ,
		"git-upload-archive": models.ACCESS_MODE_READ,
		"git-receive-pack":   models.ACCESS_MODE_WRITE,
		"git-annex-shell":    models.ACCESS_MODE_WRITE,
	}
)

func runServ(c *cli.Context) error {
	setup(c, "serv.log", true)

	if setting.SSH.Disabled {
		println("Gogs: SSH has been disabled")
		return nil
	}

	if len(c.Args()) < 1 {
		fail("Not enough arguments", "Not enough arguments")
	}

	sshCmd := strings.Replace(os.Getenv("SSH_ORIGINAL_COMMAND"), "'", "", -1)
	log.Info("SSH commadn:%s", sshCmd)
	if len(sshCmd) == 0 {
		println("Hi there, You've successfully authenticated, but Gogs does not provide shell access.")
		println("If this is unexpected, please log in with password and setup Gogs under another user.")
		return nil
	}

	verb, path, args := parseSSHCmd(sshCmd)
	repoFullName := strings.ToLower(strings.Trim(path, "'"))
	repoFields := strings.SplitN(repoFullName, "/", 2)
	if len(repoFields) != 2 {
		fail("Invalid repository path", "Invalid repository path: %v", path)
	}
	ownerName := strings.ToLower(repoFields[0])
	repoName := strings.TrimSuffix(strings.ToLower(repoFields[1]), ".git")
	repoName = strings.TrimSuffix(repoName, ".wiki")

	owner, err := models.GetUserByName(ownerName)
	if err != nil {
		if errors.IsUserNotExist(err) {
			fail("Repository owner does not exist", "Unregistered owner: %s", ownerName)
		}
		fail("Internal error", "Fail to get repository owner '%s': %v", ownerName, err)
	}

	repo, err := models.GetRepositoryByName(owner.ID, repoName)
	if err != nil {
		if errors.IsRepoNotExist(err) {
			fail(_ACCESS_DENIED_MESSAGE, "Repository does not exist: %s/%s", owner.Name, repoName)
		}
		fail("Internal error", "Fail to get repository: %v", err)
	}
	repo.Owner = owner

	requestMode, ok := allowedCommands[verb]
	if !ok {
		fail("Unknown git command", "Unknown git command '%s'", verb)
	}

	// Prohibit push to mirror repositories.
	if requestMode > models.ACCESS_MODE_READ && repo.IsMirror {
		fail("Mirror repository is read-only", "")
	}

	// Allow anonymous (user is nil) clone for public repositories.
	var user *models.User

	key, err := models.GetPublicKeyByID(com.StrTo(strings.TrimPrefix(c.Args()[0], "key-")).MustInt64())
	if err != nil {
		fail("Invalid key ID", "Invalid key ID '%s': %v", c.Args()[0], err)
	}

	if requestMode == models.ACCESS_MODE_WRITE || repo.IsPrivate {
		// Check deploy key or user key.
		if key.IsDeployKey() {
			if key.Mode < requestMode {
				fail("Key permission denied", "Cannot push with deployment key: %d", key.ID)
			}
			checkDeployKey(key, repo)
		} else {
			user, err = models.GetUserByKeyID(key.ID)
			if err != nil {
				fail("Internal error", "Fail to get user by key ID '%d': %v", key.ID, err)
			}

			mode, err := models.AccessLevel(user.ID, repo)
			if err != nil {
				fail("Internal error", "Fail to check access: %v", err)
			}

			if mode < requestMode {
				clientMessage := _ACCESS_DENIED_MESSAGE
				if mode >= models.ACCESS_MODE_READ {
					clientMessage = "You do not have sufficient authorization for this action"
				}
				fail(clientMessage,
					"User '%s' does not have level '%v' access to repository '%s'",
					user.Name, requestMode, repoFullName)
			}
		}
	} else {
		setting.NewService()
		// Check if the key can access to the repository in case of it is a deploy key (a deploy keys != user key).
		// A deploy key doesn't represent a signed in user, so in a site with Service.RequireSignInView activated
		// we should give read access only in repositories where this deploy key is in use. In other case, a server
		// or system using an active deploy key can get read access to all the repositories in a Gogs service.
		if key.IsDeployKey() && setting.Service.RequireSignInView {
			checkDeployKey(key, repo)
		}
	}

	// Update user key activity.
	if key.ID > 0 {
		key, err := models.GetPublicKeyByID(key.ID)
		if err != nil {
			fail("Internal error", "GetPublicKeyByID: %v", err)
		}

		key.Updated = time.Now()
		if err = models.UpdatePublicKey(key); err != nil {
			fail("Internal error", "UpdatePublicKey: %v", err)
		}
	}

	// Special handle for Windows.
	// Todo will break with annex
	if setting.IsWindows {
		verb = strings.Replace(verb, "-", " ", 1)
	}
	verbs := strings.Split(verb, " ")
	var cmd []string
	if len(verbs) == 2 {
		cmd = []string{verbs[0], verbs[1], repoFullName}
	} else if isAnnexShell(verb) {
		repoAbsPath := setting.RepoRootPath + "/" + repoFullName
		//		if err := secureGitAnnex(repoAbsPath, requestMode); err != nil {
		//			fail("Git annex failed", "Git annex failed: %s", err)
		//}
		cmd = args
		// Setting full path to repo as git-annex-shell requires it
		cmd[2] = repoAbsPath
	} else {
		cmd = []string{verb, repoFullName}
	}
	runGit(cmd, requestMode, user, owner, repo)
	return nil

}

func runGit(cmd [] string, requestMode models.AccessMode, user *models.User, owner *models.User,
	repo *models.Repository) error {
	log.Info("will exectute:%s", cmd)
	gitCmd := exec.Command(cmd[0], cmd[1:]...)
	if requestMode == models.ACCESS_MODE_WRITE {
		gitCmd.Env = append(os.Environ(), http.ComposeHookEnvs(http.ComposeHookEnvsOptions{
			AuthUser:  user,
			OwnerName: owner.Name,
			OwnerSalt: owner.Salt,
			RepoID:    repo.ID,
			RepoName:  repo.Name,
			RepoPath:  repo.RepoPath(),
		})...)
	}
	gitCmd.Dir = setting.RepoRootPath
	gitCmd.Stdout = os.Stdout
	gitCmd.Stdin = os.Stdin
	gitCmd.Stderr = os.Stderr
	log.Info("args:%s", gitCmd.Args)
	err := gitCmd.Run()
	log.Info("err:%s", err)
	if t, ok := err.(*exec.ExitError); ok {
		log.Info("t:%s", t)
		os.Exit(t.Sys().(syscall.WaitStatus).ExitStatus())
	}

	return nil
}

// Make sure git-annex-shell does not make "bad" changes (refectored from repo)
func secureGitAnnex(path string, requestMode models.AccessMode) error {
	// "If set, disallows running git-shell to handle unknown commands."
	err := os.Setenv("GIT_ANNEX_SHELL_LIMITED", "True")
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell to be limited.")
	}
	// "If set, git-annex-shell will refuse to run commands
	//  that do not operate on the specified directory."
	err = os.Setenv("GIT_ANNEX_SHELL_DIRECTORY", path)
	if err != nil {
		return fmt.Errorf("ERROR: Could set annex shell directory.")
	}
	if ! (requestMode > models.ACCESS_MODE_READ) {
		err = os.Setenv("GIT_ANNEX_SHELL_READONLY", "True")
		if err != nil {
			return fmt.Errorf("ERROR: Could set annex shell to read only.")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package stringsutil

import (
	"regexp"
	"sort"
	"strconv"
	"strings"
)

// Unshift adds an element at the first position
// of the slice.
func Unshift(a []string, x string) []string {
	return append([]string{x}, a...)
}

// SliceCondenseSpace trims space from lines and removes
// empty lines. `unique` dedupes lines and `sort` preforms
// a sort on the results.
func SliceCondenseSpace(lines []string, dedupeResults, sortResults bool) []string {
	results := SliceTrimSpace(lines, true)
	if dedupeResults {
		results = Dedupe(results)
	}
	if sortResults {
		sort.Strings(results)
	}
	return results
}

// SliceTrimSpace removes leading and trailing spaces per
// string and optionally removes empty strings.
func SliceTrimSpace(lines []string, condense bool) []string {
	return SliceTrim(lines, " ", condense)
}

/*
// SliceTrimSpace removes leading and trailing spaces per
// string and optionally removes empty strings.
func SliceTrimSpaceMore(lines []string, condense bool) []string {
	return SliceTrim(lines, " ", condense)
}*/

// SliceTrim trims each line in a slice of lines using a
// provided cut string.
func SliceTrim(lines []string, cutstr string, condense bool) []string {
	newLines := []string{}
	for _, line := range lines {
		line = strings.Trim(line, cutstr)
		if condense && len(line) == 0 {
			continue
		}
		newLines = append(newLines, line)
	}
	return newLines
}

// SliceIndexOrEmpty returns the element at the index
// provided or an empty string.
func SliceIndexOrEmpty(s []string, index uint64) string {
	if int(index) >= len(s) {
		return ""
	}
	return s[index]
}

/*
// JoinAny takes an array of interface{} and converts
// each value to a string using fmt.Sprintf("%v")
func JoinAny(a []interface{}, sep string) string {
	strs := []string{}
	for _, item := range a {
		strs = append(strs, fmt.Sprintf("%v", item))
	}
	return strings.Join(strs, sep)
}

func JoinInt(a []int, sep string) string {
	strs := []string{}
	for _, i := range a {
		strs = append(strs, strconv.Itoa(i))
	}
	return strings.Join(strs, sep)
}

func JoinCondenseTrimSpace(slice []string, sep string) string {
	return strings.Join(SliceTrimSpace(slice, true), sep)
}
*/

func SliceCondenseRegexps(texts []string, regexps []*regexp.Regexp, replacement string) []string {
	parts := []string{}
	for _, part := range texts {
		for _, rx := range regexps {
			part = rx.ReplaceAllString(part, replacement)
		}
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondensePunctuation(texts []string) []string {
	parts := []string{}
	for _, part := range texts {
		part = regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(part, " ")
		part = regexp.MustCompile(`\s+`).ReplaceAllString(part, " ")
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondenseAndQuoteSpace(items []string, quoteLeft, quoteRight string) []string {
	return SliceCondenseAndQuote(items, " ", " ", quoteLeft, quoteRight)
}

func SliceCondenseAndQuote(items []string, trimLeft, trimRight, quoteLeft, quoteRight string) []string {
	newItems := []string{}
	for _, item := range items {
		item = strings.TrimLeft(item, trimLeft)
		item = strings.TrimRight(item, trimRight)
		if len(item) > 0 {
			item = quoteLeft + item + quoteRight
			newItems = append(newItems, item)
		}
	}
	return newItems
}

// SplitTrimSpace splits a string and trims spaces on
// remaining elements.
func SplitTrimSpace(s, sep string) []string {
	split := strings.Split(s, sep)
	strs := []string{}
	for _, str := range split {
		strs = append(strs, strings.TrimSpace(str))
	}
	return strs
}

var rxSplitLines = regexp.MustCompile(`(\r\n|\r|\n)`)

// SplitTextLines splits a string on the regxp
// `(\r\n|\r|\n)`.
func SplitTextLines(text string) []string {
	return rxSplitLines.Split(text, -1)
}

// SliceToSingleIntOrNeg converts a single element slice
// with a string to an integer or `-1`
func SliceToSingleIntOrNeg(vals []string) int {
	if len(vals) != 1 {
		return -1
	}
	num, err := strconv.Atoi(vals[0])
	if err != nil {
		return -1
	}
	return num
}

// Dedupe returns a string slice with duplicate values
// removed. First observance is kept.
func Dedupe(vals []string) []string {
	deduped := []string{}
	seen := map[string]int{}
	for _, val := range vals {
		if _, ok := seen[val]; ok {
			continue
		}
		seen[val] = 1
		deduped = append(deduped, val)
	}
	return deduped
}

// SliceIndex returns the index of an element in a
// string slice. Returns -1 if not found.
func SliceIndex(haystack []string, needle string, trimSpace, toLower bool, matchType MatchType) int {
	if trimSpace {
		needle = strings.TrimSpace(needle)
	}
	if toLower {
		needle = strings.ToLower(needle)
	}
	for idx, hay := range haystack {
		if trimSpace {
			hay = strings.TrimSpace(hay)
		}
		if toLower {
			hay = strings.ToLower(hay)
		}
		if matchType == MatchHasSuffix {
			if strings.HasSuffix(needle, hay) {
				return idx
			}
		} else if matchType == MatchHasPrefix {
			if strings.HasPrefix(needle, hay) {
				return idx
			}
		} else {
			if needle == hay {
				return idx
			}
		}
	}
	return -1
}

func SliceChooseOnePreferredLowerTrimSpace(options, preferenceOrder []string) string {
	if len(options) == 0 {
		return ""
	} else if len(preferenceOrder) == 0 {
		return strings.ToLower(strings.TrimSpace(options[0]))
	}
	optMap := map[string]int{}
	for _, opt := range options {
		opt = strings.ToLower(strings.TrimSpace(opt))
		if len(opt) > 0 {
			optMap[opt] = 1
		}
	}
	for _, pref := range preferenceOrder {
		pref = strings.ToLower(strings.TrimSpace(pref))
		if _, ok := optMap[pref]; ok {
			return pref
		}
	}
	return strings.ToLower(strings.TrimSpace(options[0]))
}

/*
func SliceJoinQuoteMaxLength(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if maxLength > 0 && curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

type JoinCustomConfig struct {
	QuoteBegin string
	QuoteEnd   string
	Separator  string
	MaxLength  int
	TrimSpace  bool
	SkipEmpty  bool
}

func JoinCustom(slice []string, cfg JoinCustomConfig) []string {
	lines := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if cfg.TrimSpace {
			word = strings.TrimSpace(word)
		}
		if cfg.SkipEmpty && len(word) == 0 {
			continue
		}
		if cfg.MaxLength > 0 {
			if curLength+len(cfg.QuoteBegin+word+cfg.QuoteEnd+cfg.Separator) > cfg.MaxLength {
				lines = append(lines, strings.Join(curWords, cfg.Separator))
				curWords = []string{}
				curLength = 0
			} else {
				curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
				curLength += len(cfg.QuoteBegin + word + cfg.QuoteEnd + cfg.Separator)
			}
		} else {
			curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
		}
	}
	if len(curWords) > 0 {
		lines = append(lines, strings.Join(curWords, cfg.Separator))
	}
	return lines
}

func SliceJoinQuoteMaxLengthTrimSpaceSkipEmpty(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if maxLength > 0 && curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

func SliceJoinQuoted(slice []string, begQuote, endQuote, sep string) string {
	words := []string{}
	for _, word := range slice {
		words = append(words, begQuote+word+endQuote)
	}
	return strings.Join(words, sep)
}
*/

func SliceSubtract(real, filter []string) []string {
	filtered := []string{}
	filterMap := map[string]int{}
	for _, f := range filter {
		filterMap[f] = 1
	}
	for _, r := range real {
		if _, ok := filterMap[r]; !ok {
			filtered = append(filtered, r)
		}
	}
	return filtered
}

func SliceToMap(strs []string) map[string]int {
	strmap := map[string]int{}
	for _, s := range strs {
		if _, ok := strmap[s]; !ok {
			strmap[s] = 0
		}
		strmap[s]++
	}
	return strmap
}

func SliceIntersection(list1, list2 []string) []string {
	map1 := map[string]int{}
	map2 := map[string]int{}
	for _, item1 := range list1 {
		map1[item1] = 1
	}
	for _, item2 := range list2 {
		if _, ok := map1[item2]; ok {
			map2[item2] = 1
		}
	}
	intersection := []string{}
	for key := range map2 {
		intersection = append(intersection, key)
	}
	return intersection
}

func SliceIntersectionCondenseSpace(slice1, slice2 []string) []string {
	return SliceIntersection(
		SliceCondenseSpace(slice1, true, false),
		SliceCondenseSpace(slice2, true, false))
}
<commit_msg>enhance: add comment for stringsutil.SliceSubtract()<commit_after>package stringsutil

import (
	"regexp"
	"sort"
	"strconv"
	"strings"
)

// Unshift adds an element at the first position
// of the slice.
func Unshift(a []string, x string) []string {
	return append([]string{x}, a...)
}

// SliceCondenseSpace trims space from lines and removes
// empty lines. `unique` dedupes lines and `sort` preforms
// a sort on the results.
func SliceCondenseSpace(lines []string, dedupeResults, sortResults bool) []string {
	results := SliceTrimSpace(lines, true)
	if dedupeResults {
		results = Dedupe(results)
	}
	if sortResults {
		sort.Strings(results)
	}
	return results
}

// SliceTrimSpace removes leading and trailing spaces per
// string and optionally removes empty strings.
func SliceTrimSpace(lines []string, condense bool) []string {
	return SliceTrim(lines, " ", condense)
}

/*
// SliceTrimSpace removes leading and trailing spaces per
// string and optionally removes empty strings.
func SliceTrimSpaceMore(lines []string, condense bool) []string {
	return SliceTrim(lines, " ", condense)
}*/

// SliceTrim trims each line in a slice of lines using a
// provided cut string.
func SliceTrim(lines []string, cutstr string, condense bool) []string {
	newLines := []string{}
	for _, line := range lines {
		line = strings.Trim(line, cutstr)
		if condense && len(line) == 0 {
			continue
		}
		newLines = append(newLines, line)
	}
	return newLines
}

// SliceIndexOrEmpty returns the element at the index
// provided or an empty string.
func SliceIndexOrEmpty(s []string, index uint64) string {
	if int(index) >= len(s) {
		return ""
	}
	return s[index]
}

/*
// JoinAny takes an array of interface{} and converts
// each value to a string using fmt.Sprintf("%v")
func JoinAny(a []interface{}, sep string) string {
	strs := []string{}
	for _, item := range a {
		strs = append(strs, fmt.Sprintf("%v", item))
	}
	return strings.Join(strs, sep)
}

func JoinInt(a []int, sep string) string {
	strs := []string{}
	for _, i := range a {
		strs = append(strs, strconv.Itoa(i))
	}
	return strings.Join(strs, sep)
}

func JoinCondenseTrimSpace(slice []string, sep string) string {
	return strings.Join(SliceTrimSpace(slice, true), sep)
}
*/

func SliceCondenseRegexps(texts []string, regexps []*regexp.Regexp, replacement string) []string {
	parts := []string{}
	for _, part := range texts {
		for _, rx := range regexps {
			part = rx.ReplaceAllString(part, replacement)
		}
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondensePunctuation(texts []string) []string {
	parts := []string{}
	for _, part := range texts {
		part = regexp.MustCompile(`[^a-zA-Z0-9]+`).ReplaceAllString(part, " ")
		part = regexp.MustCompile(`\s+`).ReplaceAllString(part, " ")
		part = strings.TrimSpace(part)
		if len(part) > 0 {
			parts = append(parts, part)
		}
	}
	return parts
}

func SliceCondenseAndQuoteSpace(items []string, quoteLeft, quoteRight string) []string {
	return SliceCondenseAndQuote(items, " ", " ", quoteLeft, quoteRight)
}

func SliceCondenseAndQuote(items []string, trimLeft, trimRight, quoteLeft, quoteRight string) []string {
	newItems := []string{}
	for _, item := range items {
		item = strings.TrimLeft(item, trimLeft)
		item = strings.TrimRight(item, trimRight)
		if len(item) > 0 {
			item = quoteLeft + item + quoteRight
			newItems = append(newItems, item)
		}
	}
	return newItems
}

// SplitTrimSpace splits a string and trims spaces on
// remaining elements.
func SplitTrimSpace(s, sep string) []string {
	split := strings.Split(s, sep)
	strs := []string{}
	for _, str := range split {
		strs = append(strs, strings.TrimSpace(str))
	}
	return strs
}

var rxSplitLines = regexp.MustCompile(`(\r\n|\r|\n)`)

// SplitTextLines splits a string on the regxp
// `(\r\n|\r|\n)`.
func SplitTextLines(text string) []string {
	return rxSplitLines.Split(text, -1)
}

// SliceToSingleIntOrNeg converts a single element slice
// with a string to an integer or `-1`
func SliceToSingleIntOrNeg(vals []string) int {
	if len(vals) != 1 {
		return -1
	}
	num, err := strconv.Atoi(vals[0])
	if err != nil {
		return -1
	}
	return num
}

// Dedupe returns a string slice with duplicate values
// removed. First observance is kept.
func Dedupe(vals []string) []string {
	deduped := []string{}
	seen := map[string]int{}
	for _, val := range vals {
		if _, ok := seen[val]; ok {
			continue
		}
		seen[val] = 1
		deduped = append(deduped, val)
	}
	return deduped
}

// SliceIndex returns the index of an element in a
// string slice. Returns -1 if not found.
func SliceIndex(haystack []string, needle string, trimSpace, toLower bool, matchType MatchType) int {
	if trimSpace {
		needle = strings.TrimSpace(needle)
	}
	if toLower {
		needle = strings.ToLower(needle)
	}
	for idx, hay := range haystack {
		if trimSpace {
			hay = strings.TrimSpace(hay)
		}
		if toLower {
			hay = strings.ToLower(hay)
		}
		if matchType == MatchHasSuffix {
			if strings.HasSuffix(needle, hay) {
				return idx
			}
		} else if matchType == MatchHasPrefix {
			if strings.HasPrefix(needle, hay) {
				return idx
			}
		} else {
			if needle == hay {
				return idx
			}
		}
	}
	return -1
}

func SliceChooseOnePreferredLowerTrimSpace(options, preferenceOrder []string) string {
	if len(options) == 0 {
		return ""
	} else if len(preferenceOrder) == 0 {
		return strings.ToLower(strings.TrimSpace(options[0]))
	}
	optMap := map[string]int{}
	for _, opt := range options {
		opt = strings.ToLower(strings.TrimSpace(opt))
		if len(opt) > 0 {
			optMap[opt] = 1
		}
	}
	for _, pref := range preferenceOrder {
		pref = strings.ToLower(strings.TrimSpace(pref))
		if _, ok := optMap[pref]; ok {
			return pref
		}
	}
	return strings.ToLower(strings.TrimSpace(options[0]))
}

/*
func SliceJoinQuoteMaxLength(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if maxLength > 0 && curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

type JoinCustomConfig struct {
	QuoteBegin string
	QuoteEnd   string
	Separator  string
	MaxLength  int
	TrimSpace  bool
	SkipEmpty  bool
}

func JoinCustom(slice []string, cfg JoinCustomConfig) []string {
	lines := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if cfg.TrimSpace {
			word = strings.TrimSpace(word)
		}
		if cfg.SkipEmpty && len(word) == 0 {
			continue
		}
		if cfg.MaxLength > 0 {
			if curLength+len(cfg.QuoteBegin+word+cfg.QuoteEnd+cfg.Separator) > cfg.MaxLength {
				lines = append(lines, strings.Join(curWords, cfg.Separator))
				curWords = []string{}
				curLength = 0
			} else {
				curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
				curLength += len(cfg.QuoteBegin + word + cfg.QuoteEnd + cfg.Separator)
			}
		} else {
			curWords = append(curWords, cfg.QuoteBegin+word+cfg.QuoteEnd)
		}
	}
	if len(curWords) > 0 {
		lines = append(lines, strings.Join(curWords, cfg.Separator))
	}
	return lines
}

func SliceJoinQuoteMaxLengthTrimSpaceSkipEmpty(slice []string, begQuote, endQuote, sep string, maxLength int) []string {
	words := []string{}
	curWords := []string{}
	curLength := 0
	for _, word := range slice {
		if maxLength > 0 && curLength+len(begQuote+word+endQuote+sep) > maxLength {
			words = append(words, strings.Join(curWords, sep))
			curWords = []string{}
			curLength = 0
		} else {
			curWords = append(curWords, begQuote+word+endQuote)
			curLength += len(begQuote + word + endQuote + sep)
		}
	}
	if len(curWords) > 0 {
		words = append(words, strings.Join(curWords, sep))
	}
	return words
}

func SliceJoinQuoted(slice []string, begQuote, endQuote, sep string) string {
	words := []string{}
	for _, word := range slice {
		words = append(words, begQuote+word+endQuote)
	}
	return strings.Join(words, sep)
}
*/

// SliceSubtract uses Set math to remove elements of filter
// from real.
func SliceSubtract(real, filter []string) []string {
	filtered := []string{}
	filterMap := map[string]int{}
	for _, f := range filter {
		filterMap[f] = 1
	}
	for _, r := range real {
		if _, ok := filterMap[r]; !ok {
			filtered = append(filtered, r)
		}
	}
	return filtered
}

func SliceToMap(strs []string) map[string]int {
	strmap := map[string]int{}
	for _, s := range strs {
		if _, ok := strmap[s]; !ok {
			strmap[s] = 0
		}
		strmap[s]++
	}
	return strmap
}

func SliceIntersection(list1, list2 []string) []string {
	map1 := map[string]int{}
	map2 := map[string]int{}
	for _, item1 := range list1 {
		map1[item1] = 1
	}
	for _, item2 := range list2 {
		if _, ok := map1[item2]; ok {
			map2[item2] = 1
		}
	}
	intersection := []string{}
	for key := range map2 {
		intersection = append(intersection, key)
	}
	return intersection
}

func SliceIntersectionCondenseSpace(slice1, slice2 []string) []string {
	return SliceIntersection(
		SliceCondenseSpace(slice1, true, false),
		SliceCondenseSpace(slice2, true, false))
}
<|endoftext|>