<commit_before>/*
Package ikaring provides http client Api for SplatNet; web service for Splatoon by Nintendo.
*/
package ikaring

import (
	"errors"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"

	"github.com/PuerkitoBio/goquery"
	"github.com/bitly/go-simplejson"
)

// IkaClient is a http client for SplatNet.
// it includes http.Client.
type IkaClient struct {
	hc *http.Client
}

const (
	splatoonCookieName = "_wag_session"
	splatoonDomainURL  = "https://splatoon.nintendo.net/"

	splatoonOauthURL = "https://splatoon.nintendo.net/users/auth/nintendo"
	nintendoOauthURL = "https://id.nintendo.net/oauth/authorize"

	splatoonScheduleAPI   = "https://splatoon.nintendo.net/schedule.json"
	splatoonRankingAPI    = "https://splatoon.nintendo.net/ranking.json"
	splatoonFriendListAPI = "https://splatoon.nintendo.net/friend_list/index.json"
)

// CreateClient generates ikaClient, http client object for Splatnet.
// It provides a http client with empty cookiejar.
func CreateClient() (*IkaClient, error) {
	jar, err := cookiejar.New(nil)
	if err != nil {
		return nil, err
	}
	hc := &http.Client{Jar: jar}
	client := &IkaClient{hc: hc}
	return client, nil
}

// SetSession sets session cookie to receiver IkaClient.
func (c *IkaClient) SetSession(session string) {
	uri, _ := url.Parse(splatoonDomainURL)
	c.hc.Jar.SetCookies(uri, []*http.Cookie{
		&http.Cookie{
			Secure:   true,
			HttpOnly: true,
			Name:     splatoonCookieName,
			Value:    session,
		}})
}

// Login sends http request to authorize Nintendo Network.
// it require NNID and password and return session cookie.
func (c *IkaClient) Login(name string, password string) (string, error) {
	query, err := getOauthQuery(splatoonOauthURL, name, password)
	if err != nil {
		return "", err
	}

	resp, err := c.hc.PostForm(nintendoOauthURL, query)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return "", errors.New(resp.Status)
	}

	session := getSessionFromCookie(resp.Cookies())

	return session, nil
}

// Authorized judges wheather the client authorized
// It checks cookies for session that used for authorization
func (c *IkaClient) Authorized() bool {
	uri, _ := url.Parse(splatoonDomainURL)
	session := getSessionFromCookie(c.hc.Jar.Cookies(uri))
	return len(session) != 0
}

// GetStageInfo get Stage Info from SplatNet.
// this API send GET request and parse stage schedules from JSON.
func (c *IkaClient) GetStageInfo() (*StageInfo, error) {

	resp, err := c.hc.Get(splatoonScheduleAPI)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if err = checkJSONError(body); err != nil {
		return nil, err
	}

	return decodeJSONSchedule(body)
}

// GetRanking get Ranking of Friends from SplatNet.
// this API send GET request and parse ranking from JSON.
func (c *IkaClient) GetRanking() (*RankingInfo, error) {
	resp, err := c.hc.Get(splatoonRankingAPI)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if err = checkJSONError(body); err != nil {
		return nil, err
	}

	return decodeJSONRanking(body)
}

// GetFriendList get Friend List form SplatNet.
// this API send GET request and parse friend online status from JSON
func (c *IkaClient) GetFriendList() ([]Friend, error) {
	resp, err := c.hc.Get(splatoonFriendListAPI)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if err = checkJSONError(body); err != nil {
		return nil, err
	}
	return decodeJSONFriendList(body)
}

// GetWeaponMap get Weapon Set from SplatNet.
// this API send GET request and parse weapon map by scraping HTML
func (c *IkaClient) GetWeaponMap() (map[string]string, error) {
	resp, err := c.hc.Get(splatoonDomainURL)
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, err
	}

	weapons := map[string]string{}
	doc.Find("#user_intention_weapon").Children().Each(func(_ int, s *goquery.Selection) {
		key, ok := s.Attr("value")
		if ok {
			weapons[key] = s.Text()
		}
	})
	return weapons, nil
}

func checkJSONError(data []byte) error {
	js, err := simplejson.NewJson(data)
	if err != nil {
		return err
	}

	if info := js.Get("error").MustString(); len(info) != 0 {
		return errors.New(info)
	}
	return nil
}
<commit_msg>add BaseURL and Logger to IkaClient struct<commit_after>/*
Package ikaring provides http client Api for SplatNet; web service for Splatoon by Nintendo.
*/
package ikaring

import (
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"

	"log"

	"github.com/PuerkitoBio/goquery"
	"github.com/bitly/go-simplejson"
	"github.com/pkg/errors"
)

// IkaClient is a http client for SplatNet.
// it includes http.Client.
type IkaClient struct {
	hc      *http.Client
	BaseURL *url.URL    // Splatnet Domain URL
	Logger  *log.Logger // Logger
}

const (
	splatoonCookieName = "_wag_session"
	splatoonDomainURL  = "https://splatoon.nintendo.net/"

	splatoonOauthURL = "https://splatoon.nintendo.net/users/auth/nintendo"
	nintendoOauthURL = "https://id.nintendo.net/oauth/authorize"

	splatoonScheduleAPI   = "https://splatoon.nintendo.net/schedule.json"
	splatoonRankingAPI    = "https://splatoon.nintendo.net/ranking.json"
	splatoonFriendListAPI = "https://splatoon.nintendo.net/friend_list/index.json"
)

// CreateClient generates ikaClient, http client object for Splatnet.
// It provides a http client with empty cookiejar.
func CreateClient() (*IkaClient, error) {
	return newClient(splatoonDomainURL, nil)
}

// newCleint generates ikaClient, http client object for Splatnet.
// this is inner implement for CreateClient() and used for tests.
func newClient(urlStr string, logger *log.Logger) (*IkaClient, error) {
	// cookie
	jar, err := cookiejar.New(nil)
	if err != nil {
		return nil, errors.Wrap(err, "failed to create cookie jar")
	}
	// http client
	hc := &http.Client{Jar: jar}
	client := &IkaClient{
		hc: hc,
	}
	// base URL
	client.BaseURL, err = url.ParseRequestURI(urlStr)
	if err != nil {
		return nil, errors.Wrapf(err, "failed to parse url: %s", urlStr)
	}
	// logger
	client.Logger = logger
	if logger == nil {
		var discardLogger = log.New(ioutil.Discard, "", log.LstdFlags)
		client.Logger = discardLogger
	} else {
		client.Logger = logger
	}
	return client, nil
}

// SetSession sets session cookie to receiver IkaClient.
func (c *IkaClient) SetSession(session string) {
	c.hc.Jar.SetCookies(c.BaseURL, []*http.Cookie{
		&http.Cookie{
			Secure:   true,
			HttpOnly: true,
			Name:     splatoonCookieName,
			Value:    session,
		}})
}

// Login sends http request to authorize Nintendo Network.
// it require NNID and password and return session cookie.
func (c *IkaClient) Login(name string, password string) (string, error) {
	query, err := getOauthQuery(splatoonOauthURL, name, password)
	if err != nil {
		return "", err
	}

	resp, err := c.hc.PostForm(nintendoOauthURL, query)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return "", errors.New(resp.Status)
	}

	session := getSessionFromCookie(resp.Cookies())

	return session, nil
}

// Authorized judges wheather the client authorized
// It checks cookies for session that used for authorization
func (c *IkaClient) Authorized() bool {
	uri := c.BaseURL
	session := getSessionFromCookie(c.hc.Jar.Cookies(uri))
	return len(session) != 0
}

// GetStageInfo get Stage Info from SplatNet.
// this API send GET request and parse stage schedules from JSON.
func (c *IkaClient) GetStageInfo() (*StageInfo, error) {

	resp, err := c.hc.Get(splatoonScheduleAPI)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if err = checkJSONError(body); err != nil {
		return nil, err
	}

	return decodeJSONSchedule(body)
}

// GetRanking get Ranking of Friends from SplatNet.
// this API send GET request and parse ranking from JSON.
func (c *IkaClient) GetRanking() (*RankingInfo, error) {
	resp, err := c.hc.Get(splatoonRankingAPI)

	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	if err = checkJSONError(body); err != nil {
		return nil, err
	}

	return decodeJSONRanking(body)
}

// GetFriendList get Friend List form SplatNet.
// this API send GET request and parse friend online status from JSON
func (c *IkaClient) GetFriendList() ([]Friend, error) {
	resp, err := c.hc.Get(splatoonFriendListAPI)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	if err = checkJSONError(body); err != nil {
		return nil, err
	}
	return decodeJSONFriendList(body)
}

// GetWeaponMap get Weapon Set from SplatNet.
// this API send GET request and parse weapon map by scraping HTML
func (c *IkaClient) GetWeaponMap() (map[string]string, error) {
	resp, err := c.hc.Get(splatoonDomainURL)
	if err != nil {
		return nil, err
	}

	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return nil, err
	}

	weapons := map[string]string{}
	doc.Find("#user_intention_weapon").Children().Each(func(_ int, s *goquery.Selection) {
		key, ok := s.Attr("value")
		if ok {
			weapons[key] = s.Text()
		}
	})
	return weapons, nil
}

func checkJSONError(data []byte) error {
	js, err := simplejson.NewJson(data)
	if err != nil {
		return err
	}

	if info := js.Get("error").MustString(); len(info) != 0 {
		return errors.New(info)
	}
	return nil
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"

	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/bitrise-io/go-utils/colorstring"
	"github.com/bitrise-io/go-utils/fileutil"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/goinp/goinp"
	"github.com/bitrise-tools/codesigndoc/osxkeychain"
	"github.com/bitrise-tools/codesigndoc/provprofile"
	"github.com/bitrise-tools/codesigndoc/utils"
	"github.com/bitrise-tools/codesigndoc/xcode"
	"github.com/codegangsta/cli"
)

const (
	confExportOutputDirPath = "./codesigndoc_exports"
)

func printFinished() {
	fmt.Println()
	fmt.Println(colorstring.Green("That's all."))
	fmt.Println("You just have to upload the found code signing files and you'll be good to go!")
}

func failWithError(format string, args ...interface{}) {
	log.Errorf(colorstring.Red("Error: ")+format, args...)
	fmt.Println()
	fmt.Println("------------------------------")
	fmt.Println(colorstring.Red("Please create an issue") + " on GitHub at: https://github.com/bitrise-tools/codesigndoc/issues")
	fmt.Println("with as many details & logs as you can share!")
	fmt.Println("------------------------------")
	fmt.Println()
	os.Exit(1)
}

func scan(c *cli.Context) {
	absExportOutputDirPath, err := pathutil.AbsPath(confExportOutputDirPath)
	log.Debugf("absExportOutputDirPath: %s", absExportOutputDirPath)
	if err != nil {
		failWithError("Failed to determin Absolute path of export dir: %s", confExportOutputDirPath)
	}
	if exist, err := pathutil.IsDirExists(absExportOutputDirPath); err != nil {
		failWithError("Failed to determin whether the export directory already exists: %s", err)
	} else if !exist {
		if err := os.Mkdir(absExportOutputDirPath, 0777); err != nil {
			failWithError("Failed to create export output directory at path: %s | error: %s", absExportOutputDirPath, err)
		}
	} else {
		log.Infof("Export output dir already exists at path: %s", absExportOutputDirPath)
	}

	projectPath := c.String(FileParamKey)
	if projectPath == "" {
		askText := `Please drag-and-drop your Xcode Project (` + colorstring.Green(".xcodeproj") + `)
   or Workspace (` + colorstring.Green(".xcworkspace") + `) file, the one you usually open in Xcode,
   then hit Enter.

  (Note: if you have a Workspace file you should most likely use that)`
		fmt.Println()
		projpth, err := goinp.AskForPath(askText)
		if err != nil {
			failWithError("Failed to read input: %s", err)
		}
		projectPath = projpth
	}
	log.Debugf("projectPath: %s", projectPath)
	xcodeCmd := xcode.CommandModel{
		ProjectFilePath: projectPath,
	}

	schemeToUse := c.String(SchemeParamKey)
	if schemeToUse == "" {
		log.Println("🔦  Scanning Schemes ...")
		schemes, err := xcodeCmd.ScanSchemes()
		if err != nil {
			failWithError("Failed to scan Schemes: %s", err)
		}
		log.Debugf("schemes: %v", schemes)

		fmt.Println()
		selectedScheme, err := goinp.SelectFromStrings("Select the Scheme you usually use in Xcode", schemes)
		if err != nil {
			failWithError("Failed to select Scheme: %s", err)
		}
		log.Debugf("selected scheme: %v", selectedScheme)
		schemeToUse = selectedScheme
	}
	xcodeCmd.Scheme = schemeToUse

	fmt.Println()
	fmt.Println()
	log.Println("🔦  Running an Xcode Archive, to get all the required code signing settings...")
	codeSigningSettings, xcodebuildOutput, err := xcodeCmd.ScanCodeSigningSettings()
	// save the xcodebuild output into a debug log file
	{
		xcodebuildOutputFilePath := filepath.Join(absExportOutputDirPath, "xcodebuild-output.log")
		log.Infof("  💡  Saving xcodebuild output into file: %s", xcodebuildOutputFilePath)
		if err := fileutil.WriteStringToFile(xcodebuildOutputFilePath, xcodebuildOutput); err != nil {
			log.Errorf("Failed to save xcodebuild output into file (%s), error: %s", xcodebuildOutputFilePath, err)
		}
	}
	if err != nil {
		failWithError("Failed to detect code signing settings: %s", err)
	}
	log.Debugf("codeSigningSettings: %#v", codeSigningSettings)

	fmt.Println()
	fmt.Println()
	utils.Printlnf("=== Required Identities/Certificates (%d) ===", len(codeSigningSettings.Identities))
	for idx, anIdentity := range codeSigningSettings.Identities {
		utils.Printlnf(" * (%d): %s", idx+1, anIdentity.Title)
	}
	fmt.Println("============================================")

	fmt.Println()
	utils.Printlnf("=== Required Provisioning Profiles (%d) ===", len(codeSigningSettings.ProvProfiles))
	for idx, aProvProfile := range codeSigningSettings.ProvProfiles {
		utils.Printlnf(" * (%d): %s (UUID: %s)", idx+1, aProvProfile.Title, aProvProfile.UUID)
	}
	fmt.Println("==========================================")

	fmt.Println()
	utils.Printlnf("=== Team IDs (%d) ===", len(codeSigningSettings.TeamIDs))
	for idx, aTeamID := range codeSigningSettings.TeamIDs {
		utils.Printlnf(" * (%d): %s", idx+1, aTeamID)
	}
	fmt.Println("==========================================")

	fmt.Println()
	utils.Printlnf("=== App/Bundle IDs (%d) ===", len(codeSigningSettings.AppBundleIDs))
	for idx, anAppBundleID := range codeSigningSettings.AppBundleIDs {
		utils.Printlnf(" * (%d): %s", idx+1, anAppBundleID)
	}
	fmt.Println("==========================================")
	fmt.Println()

	//
	// --- Code Signing issue checks / report
	//

	if len(codeSigningSettings.Identities) < 1 {
		failWithError("No Code Signing Identity detected!")
	}
	if len(codeSigningSettings.Identities) > 1 {
		log.Warning(colorstring.Yellow("More than one Code Signing Identity (certificate) is required to sign your app!"))
		log.Warning("You should check your settings and make sure a single Identity/Certificate can be used")
		log.Warning(" for Archiving your app!")
	}

	if len(codeSigningSettings.ProvProfiles) < 1 {
		failWithError("No Provisioning Profiles detected!")
	}

	//
	// --- Export
	//

	if !c.Bool(AllowExportParamKey) {
		isShouldExport, err := goinp.AskForBoolWithDefault("Do you want to export these files?", true)
		if err != nil {
			failWithError("Failed to process your input: %s", err)
		}
		if !isShouldExport {
			printFinished()
			return
		}
	} else {
		log.Debug("Allow Export flag was set - doing export without asking")
	}

	fmt.Println()
	log.Println("Collecting the required Identities (Certificates) for a base Xcode Archive ...")
	fmt.Println()

	identitiesWithKeychainRefs := []osxkeychain.IdentityWithRefModel{}
	defer osxkeychain.ReleaseIdentityWithRefList(identitiesWithKeychainRefs)

	for _, aIdentity := range codeSigningSettings.Identities {
		log.Infof(" * "+colorstring.Blue("Searching for Identity")+": %s", aIdentity.Title)
		validIdentityRefs, err := osxkeychain.FindAndValidateIdentity(aIdentity.Title, true)
		if err != nil {
			failWithError("Failed to export, error: %s", err)
		}

		if len(validIdentityRefs) < 1 {
			failWithError("Identity not found in the keychain, or it was invalid (expired)!")
		}
		if len(validIdentityRefs) > 1 {
			log.Warning(colorstring.Yellow("Multiple matching Identities found in Keychain! Most likely you have duplicated identities in separate Keychains, e.g. one in System.keychain and one in your Login.keychain, or you have revoked versions of the Certificate."))
		}

		identitiesWithKeychainRefs = append(identitiesWithKeychainRefs, validIdentityRefs...)
	}

	fmt.Println()
	log.Println("Collecting additional identities, for Distribution builds ...")
	fmt.Println()

	for _, aTeamID := range codeSigningSettings.TeamIDs {
		log.Infof(" * "+colorstring.Blue("Searching for Identities with Team ID")+": %s", aTeamID)
		validIdentityRefs, err := osxkeychain.FindAndValidateIdentity(fmt.Sprintf("(%s)", aTeamID), false)
		if err != nil {
			failWithError("Failed to export, error: %s", err)
		}

		if len(validIdentityRefs) < 1 {
			log.Infoln("No valid identity found for this Team ID")
		}

		identitiesWithKeychainRefs = append(identitiesWithKeychainRefs, validIdentityRefs...)
	}

	fmt.Println()
	log.Println(colorstring.Green("Exporting the Identities") + " (Certificates):")
	fmt.Println()

	identityKechainRefs := osxkeychain.CreateEmptyCFTypeRefSlice()
	for _, aIdentityWithRefItm := range identitiesWithKeychainRefs {
		fmt.Println(" * "+colorstring.Blue("Identity")+":", aIdentityWithRefItm.Label)
		identityKechainRefs = append(identityKechainRefs, aIdentityWithRefItm.KeychainRef)
	}

	fmt.Println()
	log.Infoln(colorstring.Blue("Exporting from Keychain") + ", " + colorstring.Yellow("using empty Passphrase") + " ...")
	log.Info(" This means that " + colorstring.Yellow("if you want to import the file the passphrase at import should be left empty") + ",")
	log.Info(" you don't have to type in anything, just leave the passphrase input empty.")
	fmt.Println()
	log.Info(colorstring.Blue("You'll most likely see popups") + " (one for each Identity) from Keychain,")
	log.Info(colorstring.Yellow(" you will have to accept (Allow)") + " those to be able to export the Identities!")
	fmt.Println()
	if err := osxkeychain.ExportFromKeychain(identityKechainRefs, filepath.Join(absExportOutputDirPath, "Identities.p12")); err != nil {
		failWithError("Failed to export from Keychain: %s", err)
	}

	fmt.Println()
	log.Println(colorstring.Green("Exporting base Provisioning Profile(s)"), "...")
	fmt.Println()

	for _, aProvProfile := range codeSigningSettings.ProvProfiles {
		log.Infof(" * "+colorstring.Blue("Exporting Provisioning Profile")+": %s (UUID: %s)", aProvProfile.Title, aProvProfile.UUID)
		filePth, err := provprofile.FindProvProfileFileByUUID(aProvProfile.UUID)
		if err != nil {
			failWithError("Failed to find Provisioning Profile: %s", err)
		}
		log.Infof("   File found at: %s", filePth)

		exportFileName := provProfileExportFileName(aProvProfile.UUID, aProvProfile.Title)
		exportPth := filepath.Join(absExportOutputDirPath, exportFileName)
		if err := cmdex.RunCommand("cp", filePth, exportPth); err != nil {
			failWithError("Failed to copy the Provisioning Profile into the export directory: %s", err)
		}
	}

	fmt.Println()
	log.Println(colorstring.Green("Exporting additinal, Distribution Provisioning Profile(s)"), "...")
	fmt.Println()
	for _, aAppBundleID := range codeSigningSettings.AppBundleIDs {
		log.Infof(" * "+colorstring.Blue("Searching for Provisioning Profiles with Bundle ID")+": %s", aAppBundleID)
		filePths, err := provprofile.FindProvProfilesFileByAppID(aAppBundleID)
		if err != nil {
			failWithError("Failed to find Provisioning Profile: %s", err)
		}
		if len(filePths) < 1 {
			log.Warn("   No Provisioning Profile found for this Bundle ID")
			continue
		}

		for _, aFilePth := range filePths {
			log.Info("   " + colorstring.Green("Exporting Provisioning Profile:") + " " + aFilePth)
			exportFileName := provProfileExportFileName(
				strings.TrimSuffix(filepath.Base(aFilePth), ".mobileprovision"),
				aAppBundleID,
			)
			exportPth := filepath.Join(absExportOutputDirPath, exportFileName)
			if err := cmdex.RunCommand("cp", aFilePth, exportPth); err != nil {
				failWithError("Failed to copy the Provisioning Profile into the export directory: %s", err)
			}
		}
	}

	fmt.Println()
	fmt.Printf(colorstring.Green("Exports finished")+" you can find the exported files at: %s\n", absExportOutputDirPath)
	if err := cmdex.RunCommand("open", absExportOutputDirPath); err != nil {
		log.Errorf("Failed to open the export directory in Finder: %s", absExportOutputDirPath)
	}
	fmt.Println("Opened the directory in Finder.")
	fmt.Println()

	printFinished()
}

func provProfileExportFileName(provProfileUUID, title string) string {
	replaceRexp, err := regexp.Compile("[^A-Za-z0-9_.-]")
	if err != nil {
		log.Warn("Invalid regex, error: %s", err)
		return ""
	}
	safeTitle := replaceRexp.ReplaceAllString(title, "")

	return provProfileUUID + "_" + safeTitle + ".mobileprovision"
}
<commit_msg>error report log minor enhancement<commit_after>package cli

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"

	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/bitrise-io/go-utils/colorstring"
	"github.com/bitrise-io/go-utils/fileutil"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/goinp/goinp"
	"github.com/bitrise-tools/codesigndoc/osxkeychain"
	"github.com/bitrise-tools/codesigndoc/provprofile"
	"github.com/bitrise-tools/codesigndoc/utils"
	"github.com/bitrise-tools/codesigndoc/xcode"
	"github.com/codegangsta/cli"
)

const (
	confExportOutputDirPath = "./codesigndoc_exports"
)

func printFinished() {
	fmt.Println()
	fmt.Println(colorstring.Green("That's all."))
	fmt.Println("You just have to upload the found code signing files and you'll be good to go!")
}

func failWithError(format string, args ...interface{}) {
	log.Errorf(colorstring.Red("Error: ")+format, args...)
	fmt.Println()
	fmt.Println("------------------------------")
	fmt.Println("First of all " + colorstring.Red("please make sure that you can Archive your app from Xcode."))
	fmt.Println("codesigndoc only works if you can archive your app from Xcode.")
	fmt.Println("If you can, and you get a valid IPA file if you export from Xcode,")
	fmt.Println(colorstring.Red("please create an issue") + " on GitHub at: https://github.com/bitrise-tools/codesigndoc/issues")
	fmt.Println("with as many details & logs as you can share!")
	fmt.Println("------------------------------")
	fmt.Println()
	os.Exit(1)
}

func scan(c *cli.Context) {
	absExportOutputDirPath, err := pathutil.AbsPath(confExportOutputDirPath)
	log.Debugf("absExportOutputDirPath: %s", absExportOutputDirPath)
	if err != nil {
		failWithError("Failed to determin Absolute path of export dir: %s", confExportOutputDirPath)
	}
	if exist, err := pathutil.IsDirExists(absExportOutputDirPath); err != nil {
		failWithError("Failed to determin whether the export directory already exists: %s", err)
	} else if !exist {
		if err := os.Mkdir(absExportOutputDirPath, 0777); err != nil {
			failWithError("Failed to create export output directory at path: %s | error: %s", absExportOutputDirPath, err)
		}
	} else {
		log.Infof("Export output dir already exists at path: %s", absExportOutputDirPath)
	}

	projectPath := c.String(FileParamKey)
	if projectPath == "" {
		askText := `Please drag-and-drop your Xcode Project (` + colorstring.Green(".xcodeproj") + `)
   or Workspace (` + colorstring.Green(".xcworkspace") + `) file, the one you usually open in Xcode,
   then hit Enter.

  (Note: if you have a Workspace file you should most likely use that)`
		fmt.Println()
		projpth, err := goinp.AskForPath(askText)
		if err != nil {
			failWithError("Failed to read input: %s", err)
		}
		projectPath = projpth
	}
	log.Debugf("projectPath: %s", projectPath)
	xcodeCmd := xcode.CommandModel{
		ProjectFilePath: projectPath,
	}

	schemeToUse := c.String(SchemeParamKey)
	if schemeToUse == "" {
		log.Println("🔦  Scanning Schemes ...")
		schemes, err := xcodeCmd.ScanSchemes()
		if err != nil {
			failWithError("Failed to scan Schemes: %s", err)
		}
		log.Debugf("schemes: %v", schemes)

		fmt.Println()
		selectedScheme, err := goinp.SelectFromStrings("Select the Scheme you usually use in Xcode", schemes)
		if err != nil {
			failWithError("Failed to select Scheme: %s", err)
		}
		log.Debugf("selected scheme: %v", selectedScheme)
		schemeToUse = selectedScheme
	}
	xcodeCmd.Scheme = schemeToUse

	fmt.Println()
	fmt.Println()
	log.Println("🔦  Running an Xcode Archive, to get all the required code signing settings...")
	codeSigningSettings, xcodebuildOutput, err := xcodeCmd.ScanCodeSigningSettings()
	// save the xcodebuild output into a debug log file
	{
		xcodebuildOutputFilePath := filepath.Join(absExportOutputDirPath, "xcodebuild-output.log")
		log.Infof("  💡  Saving xcodebuild output into file: %s", xcodebuildOutputFilePath)
		if err := fileutil.WriteStringToFile(xcodebuildOutputFilePath, xcodebuildOutput); err != nil {
			log.Errorf("Failed to save xcodebuild output into file (%s), error: %s", xcodebuildOutputFilePath, err)
		}
	}
	if err != nil {
		failWithError("Failed to detect code signing settings: %s", err)
	}
	log.Debugf("codeSigningSettings: %#v", codeSigningSettings)

	fmt.Println()
	fmt.Println()
	utils.Printlnf("=== Required Identities/Certificates (%d) ===", len(codeSigningSettings.Identities))
	for idx, anIdentity := range codeSigningSettings.Identities {
		utils.Printlnf(" * (%d): %s", idx+1, anIdentity.Title)
	}
	fmt.Println("============================================")

	fmt.Println()
	utils.Printlnf("=== Required Provisioning Profiles (%d) ===", len(codeSigningSettings.ProvProfiles))
	for idx, aProvProfile := range codeSigningSettings.ProvProfiles {
		utils.Printlnf(" * (%d): %s (UUID: %s)", idx+1, aProvProfile.Title, aProvProfile.UUID)
	}
	fmt.Println("==========================================")

	fmt.Println()
	utils.Printlnf("=== Team IDs (%d) ===", len(codeSigningSettings.TeamIDs))
	for idx, aTeamID := range codeSigningSettings.TeamIDs {
		utils.Printlnf(" * (%d): %s", idx+1, aTeamID)
	}
	fmt.Println("==========================================")

	fmt.Println()
	utils.Printlnf("=== App/Bundle IDs (%d) ===", len(codeSigningSettings.AppBundleIDs))
	for idx, anAppBundleID := range codeSigningSettings.AppBundleIDs {
		utils.Printlnf(" * (%d): %s", idx+1, anAppBundleID)
	}
	fmt.Println("==========================================")
	fmt.Println()

	//
	// --- Code Signing issue checks / report
	//

	if len(codeSigningSettings.Identities) < 1 {
		failWithError("No Code Signing Identity detected!")
	}
	if len(codeSigningSettings.Identities) > 1 {
		log.Warning(colorstring.Yellow("More than one Code Signing Identity (certificate) is required to sign your app!"))
		log.Warning("You should check your settings and make sure a single Identity/Certificate can be used")
		log.Warning(" for Archiving your app!")
	}

	if len(codeSigningSettings.ProvProfiles) < 1 {
		failWithError("No Provisioning Profiles detected!")
	}

	//
	// --- Export
	//

	if !c.Bool(AllowExportParamKey) {
		isShouldExport, err := goinp.AskForBoolWithDefault("Do you want to export these files?", true)
		if err != nil {
			failWithError("Failed to process your input: %s", err)
		}
		if !isShouldExport {
			printFinished()
			return
		}
	} else {
		log.Debug("Allow Export flag was set - doing export without asking")
	}

	fmt.Println()
	log.Println("Collecting the required Identities (Certificates) for a base Xcode Archive ...")
	fmt.Println()

	identitiesWithKeychainRefs := []osxkeychain.IdentityWithRefModel{}
	defer osxkeychain.ReleaseIdentityWithRefList(identitiesWithKeychainRefs)

	for _, aIdentity := range codeSigningSettings.Identities {
		log.Infof(" * "+colorstring.Blue("Searching for Identity")+": %s", aIdentity.Title)
		validIdentityRefs, err := osxkeychain.FindAndValidateIdentity(aIdentity.Title, true)
		if err != nil {
			failWithError("Failed to export, error: %s", err)
		}

		if len(validIdentityRefs) < 1 {
			failWithError("Identity not found in the keychain, or it was invalid (expired)!")
		}
		if len(validIdentityRefs) > 1 {
			log.Warning(colorstring.Yellow("Multiple matching Identities found in Keychain! Most likely you have duplicated identities in separate Keychains, e.g. one in System.keychain and one in your Login.keychain, or you have revoked versions of the Certificate."))
		}

		identitiesWithKeychainRefs = append(identitiesWithKeychainRefs, validIdentityRefs...)
	}

	fmt.Println()
	log.Println("Collecting additional identities, for Distribution builds ...")
	fmt.Println()

	for _, aTeamID := range codeSigningSettings.TeamIDs {
		log.Infof(" * "+colorstring.Blue("Searching for Identities with Team ID")+": %s", aTeamID)
		validIdentityRefs, err := osxkeychain.FindAndValidateIdentity(fmt.Sprintf("(%s)", aTeamID), false)
		if err != nil {
			failWithError("Failed to export, error: %s", err)
		}

		if len(validIdentityRefs) < 1 {
			log.Infoln("No valid identity found for this Team ID")
		}

		identitiesWithKeychainRefs = append(identitiesWithKeychainRefs, validIdentityRefs...)
	}

	fmt.Println()
	log.Println(colorstring.Green("Exporting the Identities") + " (Certificates):")
	fmt.Println()

	identityKechainRefs := osxkeychain.CreateEmptyCFTypeRefSlice()
	for _, aIdentityWithRefItm := range identitiesWithKeychainRefs {
		fmt.Println(" * "+colorstring.Blue("Identity")+":", aIdentityWithRefItm.Label)
		identityKechainRefs = append(identityKechainRefs, aIdentityWithRefItm.KeychainRef)
	}

	fmt.Println()
	log.Infoln(colorstring.Blue("Exporting from Keychain") + ", " + colorstring.Yellow("using empty Passphrase") + " ...")
	log.Info(" This means that " + colorstring.Yellow("if you want to import the file the passphrase at import should be left empty") + ",")
	log.Info(" you don't have to type in anything, just leave the passphrase input empty.")
	fmt.Println()
	log.Info(colorstring.Blue("You'll most likely see popups") + " (one for each Identity) from Keychain,")
	log.Info(colorstring.Yellow(" you will have to accept (Allow)") + " those to be able to export the Identities!")
	fmt.Println()
	if err := osxkeychain.ExportFromKeychain(identityKechainRefs, filepath.Join(absExportOutputDirPath, "Identities.p12")); err != nil {
		failWithError("Failed to export from Keychain: %s", err)
	}

	fmt.Println()
	log.Println(colorstring.Green("Exporting base Provisioning Profile(s)"), "...")
	fmt.Println()

	for _, aProvProfile := range codeSigningSettings.ProvProfiles {
		log.Infof(" * "+colorstring.Blue("Exporting Provisioning Profile")+": %s (UUID: %s)", aProvProfile.Title, aProvProfile.UUID)
		filePth, err := provprofile.FindProvProfileFileByUUID(aProvProfile.UUID)
		if err != nil {
			failWithError("Failed to find Provisioning Profile: %s", err)
		}
		log.Infof("   File found at: %s", filePth)

		exportFileName := provProfileExportFileName(aProvProfile.UUID, aProvProfile.Title)
		exportPth := filepath.Join(absExportOutputDirPath, exportFileName)
		if err := cmdex.RunCommand("cp", filePth, exportPth); err != nil {
			failWithError("Failed to copy the Provisioning Profile into the export directory: %s", err)
		}
	}

	fmt.Println()
	log.Println(colorstring.Green("Exporting additinal, Distribution Provisioning Profile(s)"), "...")
	fmt.Println()
	for _, aAppBundleID := range codeSigningSettings.AppBundleIDs {
		log.Infof(" * "+colorstring.Blue("Searching for Provisioning Profiles with Bundle ID")+": %s", aAppBundleID)
		filePths, err := provprofile.FindProvProfilesFileByAppID(aAppBundleID)
		if err != nil {
			failWithError("Failed to find Provisioning Profile: %s", err)
		}
		if len(filePths) < 1 {
			log.Warn("   No Provisioning Profile found for this Bundle ID")
			continue
		}

		for _, aFilePth := range filePths {
			log.Info("   " + colorstring.Green("Exporting Provisioning Profile:") + " " + aFilePth)
			exportFileName := provProfileExportFileName(
				strings.TrimSuffix(filepath.Base(aFilePth), ".mobileprovision"),
				aAppBundleID,
			)
			exportPth := filepath.Join(absExportOutputDirPath, exportFileName)
			if err := cmdex.RunCommand("cp", aFilePth, exportPth); err != nil {
				failWithError("Failed to copy the Provisioning Profile into the export directory: %s", err)
			}
		}
	}

	fmt.Println()
	fmt.Printf(colorstring.Green("Exports finished")+" you can find the exported files at: %s\n", absExportOutputDirPath)
	if err := cmdex.RunCommand("open", absExportOutputDirPath); err != nil {
		log.Errorf("Failed to open the export directory in Finder: %s", absExportOutputDirPath)
	}
	fmt.Println("Opened the directory in Finder.")
	fmt.Println()

	printFinished()
}

func provProfileExportFileName(provProfileUUID, title string) string {
	replaceRexp, err := regexp.Compile("[^A-Za-z0-9_.-]")
	if err != nil {
		log.Warn("Invalid regex, error: %s", err)
		return ""
	}
	safeTitle := replaceRexp.ReplaceAllString(title, "")

	return provProfileUUID + "_" + safeTitle + ".mobileprovision"
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage_test

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/state"
)

type storageAddSuite struct {
	baseStorageSuite
}

var _ = gc.Suite(&storageAddSuite{})

func (s *storageAddSuite) assertStorageAddedNoErrors(c *gc.C, args params.StorageAddParams) {
	s.assertStoragesAddedNoErrors(c,
		params.StoragesAddParams{[]params.StorageAddParams{args}},
	)
}

func (s *storageAddSuite) assertStoragesAddedNoErrors(c *gc.C, args params.StoragesAddParams) {
	failures, err := s.api.AddToUnit(args)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, len(args.Storages))
}

func (s *storageAddSuite) TestStorageAddEmpty(c *gc.C) {
	s.assertStoragesAddedNoErrors(c, params.StoragesAddParams{Storages: nil})
	s.assertStoragesAddedNoErrors(c, params.StoragesAddParams{Storages: []params.StorageAddParams{}})
}

func (s *storageAddSuite) TestStorageAddUnit(c *gc.C) {
	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	s.assertStorageAddedNoErrors(c, args)
	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitBlocked(c *gc.C) {
	s.blockAllChanges(c, "TestStorageAddUnitBlocked")

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	_, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	s.assertBlocked(c, err, "TestStorageAddUnitBlocked")
}

func (s *storageAddSuite) TestStorageAddUnitDestroyIgnored(c *gc.C) {
	s.blockDestroyEnvironment(c, "TestStorageAddUnitDestroyIgnored")
	s.blockRemoveObject(c, "TestStorageAddUnitDestroyIgnored")

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	s.assertStorageAddedNoErrors(c, args)
	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitError(c *gc.C) {
	args := params.StorageAddParams{
		StorageName: "data",
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 1)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, ".*is not a valid tag.*")

	expectedCalls := []string{getBlockForTypeCall}
	s.assertCalls(c, expectedCalls)
}

func (s *storageAddSuite) TestStorageAddUnitStateError(c *gc.C) {
	msg := "add test directive error"
	s.state.addStorageForUnit = func(u names.UnitTag, name string, cons state.StorageConstraints) error {
		s.calls = append(s.calls, addStorageForUnitCall)
		return errors.Errorf(msg)
	}

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 1)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, fmt.Sprintf(".*%v.*", msg))

	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitPermError(c *gc.C) {
	msg := "add test directive error"
	s.state.addStorageForUnit = func(u names.UnitTag, name string, cons state.StorageConstraints) error {
		s.calls = append(s.calls, addStorageForUnitCall)
		return errors.NotFoundf(msg)
	}

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 1)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, ".*permission denied.*")

	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitResultOrder(c *gc.C) {
	wrong0 := params.StorageAddParams{
		StorageName: "data",
	}
	right := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	wrong1 := params.StorageAddParams{
		UnitTag: s.unitTag.String(),
	}
	msg := "storage name missing error"
	s.state.addStorageForUnit = func(u names.UnitTag, name string, cons state.StorageConstraints) error {
		s.calls = append(s.calls, addStorageForUnitCall)
		if name == "" {
			return errors.Errorf(msg)
		}
		return nil
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{
		[]params.StorageAddParams{
			wrong0,
			right,
			wrong1}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 3)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, ".*is not a valid tag.*")
	c.Assert(failures.Results[1].Error, gc.IsNil)
	c.Assert(failures.Results[2].Error.Error(), gc.Matches, fmt.Sprintf(".*%v.*", msg))

	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall, addStorageForUnitCall})
}
<commit_msg>Added assertion to check individual errors.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage_test

import (
	"fmt"

	"github.com/juju/errors"
	"github.com/juju/names"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/state"
)

type storageAddSuite struct {
	baseStorageSuite
}

var _ = gc.Suite(&storageAddSuite{})

func (s *storageAddSuite) assertStorageAddedNoErrors(c *gc.C, args params.StorageAddParams) {
	s.assertStoragesAddedNoErrors(c,
		params.StoragesAddParams{[]params.StorageAddParams{args}},
	)
}

func (s *storageAddSuite) assertStoragesAddedNoErrors(c *gc.C, args params.StoragesAddParams) {
	failures, err := s.api.AddToUnit(args)
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, len(args.Storages))
	for _, one := range failures.Results {
		c.Assert(one.Error, gc.IsNil)
	}
}

func (s *storageAddSuite) TestStorageAddEmpty(c *gc.C) {
	s.assertStoragesAddedNoErrors(c, params.StoragesAddParams{Storages: nil})
	s.assertStoragesAddedNoErrors(c, params.StoragesAddParams{Storages: []params.StorageAddParams{}})
}

func (s *storageAddSuite) TestStorageAddUnit(c *gc.C) {
	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	s.assertStorageAddedNoErrors(c, args)
	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitBlocked(c *gc.C) {
	s.blockAllChanges(c, "TestStorageAddUnitBlocked")

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	_, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	s.assertBlocked(c, err, "TestStorageAddUnitBlocked")
}

func (s *storageAddSuite) TestStorageAddUnitDestroyIgnored(c *gc.C) {
	s.blockDestroyEnvironment(c, "TestStorageAddUnitDestroyIgnored")
	s.blockRemoveObject(c, "TestStorageAddUnitDestroyIgnored")

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	s.assertStorageAddedNoErrors(c, args)
	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitError(c *gc.C) {
	args := params.StorageAddParams{
		StorageName: "data",
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 1)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, ".*is not a valid tag.*")

	expectedCalls := []string{getBlockForTypeCall}
	s.assertCalls(c, expectedCalls)
}

func (s *storageAddSuite) TestStorageAddUnitStateError(c *gc.C) {
	msg := "add test directive error"
	s.state.addStorageForUnit = func(u names.UnitTag, name string, cons state.StorageConstraints) error {
		s.calls = append(s.calls, addStorageForUnitCall)
		return errors.Errorf(msg)
	}

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 1)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, fmt.Sprintf(".*%v.*", msg))

	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitPermError(c *gc.C) {
	msg := "add test directive error"
	s.state.addStorageForUnit = func(u names.UnitTag, name string, cons state.StorageConstraints) error {
		s.calls = append(s.calls, addStorageForUnitCall)
		return errors.NotFoundf(msg)
	}

	args := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{[]params.StorageAddParams{args}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 1)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, ".*permission denied.*")

	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall})
}

func (s *storageAddSuite) TestStorageAddUnitResultOrder(c *gc.C) {
	wrong0 := params.StorageAddParams{
		StorageName: "data",
	}
	right := params.StorageAddParams{
		UnitTag:     s.unitTag.String(),
		StorageName: "data",
	}
	wrong1 := params.StorageAddParams{
		UnitTag: s.unitTag.String(),
	}
	msg := "storage name missing error"
	s.state.addStorageForUnit = func(u names.UnitTag, name string, cons state.StorageConstraints) error {
		s.calls = append(s.calls, addStorageForUnitCall)
		if name == "" {
			return errors.Errorf(msg)
		}
		return nil
	}
	failures, err := s.api.AddToUnit(params.StoragesAddParams{
		[]params.StorageAddParams{
			wrong0,
			right,
			wrong1}})
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(failures.Results, gc.HasLen, 3)
	c.Assert(failures.Results[0].Error.Error(), gc.Matches, ".*is not a valid tag.*")
	c.Assert(failures.Results[1].Error, gc.IsNil)
	c.Assert(failures.Results[2].Error.Error(), gc.Matches, fmt.Sprintf(".*%v.*", msg))

	s.assertCalls(c, []string{getBlockForTypeCall, addStorageForUnitCall, addStorageForUnitCall})
}
<|endoftext|>


<commit_before>package metrics_prometheus

import (
	//"gopkg.in/cfchou/go-gentle.v1/gentle"
	"../../gentle"
	prom "github.com/prometheus/client_golang/prometheus"
)

type promObeservation struct {
	name    string
	histVec *prom.HistogramVec
}

func (p *promObeservation) Observe(value float64, labels map[string]string) {
	m := map[string]string{"name": p.name, "result": labels["result"]}
	h := p.histVec.With(m)
	h.Observe(value)
}

type promCounter struct {
	name       string
	counterVec *prom.CounterVec
}

func (p *promCounter) Add(value float64, labels map[string]string) {
	m := map[string]string{"name": p.name, "err": labels["err"]}
	c := p.counterVec.With(m)
	c.Add(value)
}

// namespace_s_rate_get_seconds{name, result}
func RegisterRateLimitedStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_RATELIMITED, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_RATELIMITED,
			Name:      "get_seconds",
			Help:      "Duration of RateLimitedStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_retry_get_seconds{name, result}
// namespace_s_retry_tries_total{name, result}
// Given N = RetryStream's len(backoffs)+1, so that N is the maximum of tries.
// tryBuckets have buckets sensibly grouping the range [1, N]. The total number
// of buckets should not be too large. For example,
// if backoffs = [1, 2, 4, 8, 16], then tryBuckets may be [1, 2, 3, 4, 5, 6]
// which makes one try one bucket.
// If backoffs is a large list of 30 elements, then tryBuckets may be
// [1, 2, 4, 8, 16, 24, 32].
func RegisterRetryStreamMetrics(namespace, name string, tryBuckets []float64) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_RETRY, "get"}
	if gentle.GetObservation(key) == nil {
		histVec := prom.NewHistogramVec(
			prom.HistogramOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_RETRY,
				Name:      "get_seconds",
				Help:      "Duration of RetryStream.Get() in seconds",
				Buckets:   prom.DefBuckets,
			},
			[]string{"name", "result"})
		ob := &promObeservation{
			name:    name,
			histVec: histVec,
		}
		gentle.RegisterObservation(key, ob)
	}
	key = &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_RETRY, "try"}
	if gentle.GetObservation(key) == nil {
		histVec := prom.NewHistogramVec(
			prom.HistogramOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_RETRY,
				Name:      "tries_total",
				Help:      "Number of tries of RetryStream.Get()",
				Buckets:   tryBuckets,
			},
			[]string{"name", "result"})
		ob := &promObeservation{
			name:    name,
			histVec: histVec,
		}
		gentle.RegisterObservation(key, ob)
	}
}

// namespace_s_bulk_get_seconds{name, result}
func RegisterBulkStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_BULKHEAD, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_BULKHEAD,
			Name:      "get_seconds",
			Help:      "Duration of BulkheadStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_circuit_get_seconds{name, result}
// namespace_s_circuit_errors_total{name, err}
func RegisterCircuitBreakerStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CIRCUITBREAKER, "get"}
	if gentle.GetObservation(key) == nil {
		histVec := prom.NewHistogramVec(
			prom.HistogramOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_CIRCUITBREAKER,
				Name:      "get_seconds",
				Help:      "Duration of CircuitBreakerStream.Get() in seconds",
				Buckets:   prom.DefBuckets,
			},
			[]string{"name", "result"})
		ob := &promObeservation{
			name:    name,
			histVec: histVec,
		}
		gentle.RegisterObservation(key, ob)
	}
	key = &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CIRCUITBREAKER, "hystrix_err"}
	if gentle.GetCounter(key) == nil {
		counterVec := prom.NewCounterVec(
			prom.CounterOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_CIRCUITBREAKER,
				Name:      "errors_total",
				Help:      "Number of errors from hystrix.Do() in CircuitBreakerStream",
			},
			[]string{"name", "err"})
		counter := &promCounter{
			name:       name,
			counterVec: counterVec,
		}
		gentle.RegisterCounter(key, counter)
	}
}

// namespace_s_chan_get_seconds{name, result}
func RegisterChannelStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CHANNEL, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_CHANNEL,
			Name:      "get_seconds",
			Help:      "Duration of ChannelStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_con_get_seconds{name, result}
func RegisterConcurrentFetchStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CONCURRENTFETCH, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_CONCURRENTFETCH,
			Name:      "get_seconds",
			Help:      "Duration of ConcurrentFetchStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_map_get_seconds{name, result}
func RegisterMappedStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_MAPPED, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_MAPPED,
			Name:      "get_seconds",
			Help:      "Duration of MappedStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}
<commit_msg>prom.MustRegister<commit_after>package metrics_prometheus

import (
	//"gopkg.in/cfchou/go-gentle.v1/gentle"
	"../../gentle"
	prom "github.com/prometheus/client_golang/prometheus"
)

type promObeservation struct {
	name    string
	histVec *prom.HistogramVec
}

func (p *promObeservation) Observe(value float64, labels map[string]string) {
	m := map[string]string{"name": p.name}
	for k, v := range labels {
		m[k] = v
	}
	h := p.histVec.With(m)
	h.Observe(value)
}

type promCounter struct {
	name       string
	counterVec *prom.CounterVec
}

func (p *promCounter) Add(value float64, labels map[string]string) {
	m := map[string]string{"name": p.name}
	for k, v := range labels {
		m[k] = v
	}
	c := p.counterVec.With(m)
	c.Add(value)
}

// namespace_s_rate_get_seconds{name, result}
func RegisterRateLimitedStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_RATELIMITED, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_RATELIMITED,
			Name:      "get_seconds",
			Help:      "Duration of RateLimitedStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	prom.MustRegister(histVec)
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_retry_get_seconds{name, result}
// namespace_s_retry_tries_total{name, result}
// Given N = RetryStream's len(backoffs)+1, so that N is the maximum of tries.
// tryBuckets have buckets sensibly grouping the range [1, N]. The total number
// of buckets should not be too large. For example,
// if backoffs = [1, 2, 4, 8, 16], then tryBuckets may be [1, 2, 3, 4, 5, 6]
// which makes one try one bucket.
// If backoffs is a large list of 30 elements, then tryBuckets may be
// [1, 2, 4, 8, 16, 24, 32].
func RegisterRetryStreamMetrics(namespace, name string, tryBuckets []float64) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_RETRY, "get"}
	if gentle.GetObservation(key) == nil {
		histVec := prom.NewHistogramVec(
			prom.HistogramOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_RETRY,
				Name:      "get_seconds",
				Help:      "Duration of RetryStream.Get() in seconds",
				Buckets:   prom.DefBuckets,
			},
			[]string{"name", "result"})
		prom.MustRegister(histVec)
		ob := &promObeservation{
			name:    name,
			histVec: histVec,
		}
		gentle.RegisterObservation(key, ob)
	}
	key = &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_RETRY, "try"}
	if gentle.GetObservation(key) == nil {
		histVec := prom.NewHistogramVec(
			prom.HistogramOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_RETRY,
				Name:      "tries_total",
				Help:      "Number of tries of RetryStream.Get()",
				Buckets:   tryBuckets,
			},
			[]string{"name", "result"})
		prom.MustRegister(histVec)
		ob := &promObeservation{
			name:    name,
			histVec: histVec,
		}
		gentle.RegisterObservation(key, ob)
	}
}

// namespace_s_bulk_get_seconds{name, result}
func RegisterBulkStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_BULKHEAD, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_BULKHEAD,
			Name:      "get_seconds",
			Help:      "Duration of BulkheadStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	prom.MustRegister(histVec)
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_circuit_get_seconds{name, result}
// namespace_s_circuit_errors_total{name, err}
func RegisterCircuitBreakerStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CIRCUITBREAKER, "get"}
	if gentle.GetObservation(key) == nil {
		histVec := prom.NewHistogramVec(
			prom.HistogramOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_CIRCUITBREAKER,
				Name:      "get_seconds",
				Help:      "Duration of CircuitBreakerStream.Get() in seconds",
				Buckets:   prom.DefBuckets,
			},
			[]string{"name", "result"})
		prom.MustRegister(histVec)
		ob := &promObeservation{
			name:    name,
			histVec: histVec,
		}
		gentle.RegisterObservation(key, ob)
	}
	key = &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CIRCUITBREAKER, "hystrix_err"}
	if gentle.GetCounter(key) == nil {
		counterVec := prom.NewCounterVec(
			prom.CounterOpts{
				Namespace: namespace,
				Subsystem: gentle.MIXIN_STREAM_CIRCUITBREAKER,
				Name:      "errors_total",
				Help:      "Number of errors from hystrix.Do() in CircuitBreakerStream",
			},
			[]string{"name", "err"})
		prom.MustRegister(counterVec)
		counter := &promCounter{
			name:       name,
			counterVec: counterVec,
		}
		gentle.RegisterCounter(key, counter)
	}
}

// namespace_s_chan_get_seconds{name, result}
func RegisterChannelStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CHANNEL, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_CHANNEL,
			Name:      "get_seconds",
			Help:      "Duration of ChannelStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	prom.MustRegister(histVec)
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_con_get_seconds{name, result}
func RegisterConcurrentFetchStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_CONCURRENTFETCH, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_CONCURRENTFETCH,
			Name:      "get_seconds",
			Help:      "Duration of ConcurrentFetchStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	prom.MustRegister(histVec)
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}

// namespace_s_map_get_seconds{name, result}
func RegisterMappedStreamMetrics(namespace, name string) {
	key := &gentle.RegistryKey{namespace, name,
		gentle.MIXIN_STREAM_MAPPED, "get"}
	if gentle.GetObservation(key) != nil {
		// registered
		return
	}
	histVec := prom.NewHistogramVec(
		prom.HistogramOpts{
			Namespace: namespace,
			Subsystem: gentle.MIXIN_STREAM_MAPPED,
			Name:      "get_seconds",
			Help:      "Duration of MappedStream.Get() in seconds",
			Buckets:   prom.DefBuckets,
		},
		[]string{"name", "result"})
	prom.MustRegister(histVec)
	ob := &promObeservation{
		name:    name,
		histVec: histVec,
	}
	gentle.RegisterObservation(key, ob)
}
<|endoftext|>


<commit_before>// Copyright 2018 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"

	"github.com/open-policy-agent/opa/dependencies"
	"github.com/open-policy-agent/opa/internal/presentation"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"github.com/open-policy-agent/opa/ast"
	"github.com/open-policy-agent/opa/loader"
	"github.com/open-policy-agent/opa/util"
)

type depsCommandParams struct {
	dataPaths   repeatedStringFlag
	format      *util.EnumFlag
	ignore      []string
	bundlePaths repeatedStringFlag
}

const (
	depsFormatPretty = "pretty"
	depsFormatJSON   = "json"
)

func init() {

	var params depsCommandParams

	params.format = util.NewEnumFlag(depsFormatPretty, []string{
		depsFormatPretty, depsFormatJSON,
	})

	depsCommand := &cobra.Command{
		Use:   "deps <query>",
		Short: "Analyze Rego query dependencies",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			if len(args) != 1 {
				return errors.New("specify exactly one query argument")
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			if err := deps(args, params); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
		},
	}

	depsCommand.Flags().VarP(params.format, "format", "f", "set output format")
	depsCommand.Flags().VarP(&params.dataPaths, "data", "d", "set policy or data file(s). This flag can be repeated.")
	depsCommand.Flags().VarP(&params.bundlePaths, "bundle", "b", "set bundle file(s) or directory path(s). This flag can be repeated.")
	addIgnoreFlag(depsCommand.Flags(), &params.ignore)

	RootCommand.AddCommand(depsCommand)
}

func deps(args []string, params depsCommandParams) error {

	query, err := ast.ParseBody(args[0])
	if err != nil {
		return err
	}

	modules := map[string]*ast.Module{}

	if len(params.dataPaths.v) > 0 {
		f := loaderFilter{
			Ignore: params.ignore,
		}

		result, err := loader.NewFileLoader().Filtered(params.dataPaths.v, f.Apply)
		if err != nil {
			return err
		}

		for _, m := range result.Modules {
			modules[m.Name] = m.Parsed
		}
	}

	if len(params.bundlePaths.v) > 0 {
		for _, path := range params.bundlePaths.v {
			b, err := loader.NewFileLoader().WithSkipBundleVerification(true).AsBundle(path)
			if err != nil {
				return err
			}

			for name, mod := range b.ParsedModules(path) {
				modules[name] = mod
			}
		}
	}

	compiler := ast.NewCompiler()
	compiler.Compile(modules)

	if compiler.Failed() {
		return compiler.Errors
	}

	brs, err := dependencies.Base(compiler, query)
	if err != nil {
		return err
	}

	vrs, err := dependencies.Virtual(compiler, query)
	if err != nil {
		return err
	}

	output := presentation.DepAnalysisOutput{
		Base:    brs,
		Virtual: vrs,
	}

	switch params.format.String() {
	case depsFormatJSON:
		return presentation.JSON(os.Stdout, output)
	default:
		return output.Pretty(os.Stdout)
	}
}
<commit_msg>Updating deps command to use addDataFlag and addBundleFlag<commit_after>// Copyright 2018 The OPA Authors.  All rights reserved.
// Use of this source code is governed by an Apache2
// license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"os"

	"github.com/open-policy-agent/opa/dependencies"
	"github.com/open-policy-agent/opa/internal/presentation"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"github.com/open-policy-agent/opa/ast"
	"github.com/open-policy-agent/opa/loader"
	"github.com/open-policy-agent/opa/util"
)

type depsCommandParams struct {
	dataPaths   repeatedStringFlag
	format      *util.EnumFlag
	ignore      []string
	bundlePaths repeatedStringFlag
}

const (
	depsFormatPretty = "pretty"
	depsFormatJSON   = "json"
)

func init() {

	var params depsCommandParams

	params.format = util.NewEnumFlag(depsFormatPretty, []string{
		depsFormatPretty, depsFormatJSON,
	})

	depsCommand := &cobra.Command{
		Use:   "deps <query>",
		Short: "Analyze Rego query dependencies",
		PreRunE: func(cmd *cobra.Command, args []string) error {
			if len(args) != 1 {
				return errors.New("specify exactly one query argument")
			}
			return nil
		},
		Run: func(cmd *cobra.Command, args []string) {
			if err := deps(args, params); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(1)
			}
		},
	}

	depsCommand.Flags().VarP(params.format, "format", "f", "set output format")
	addIgnoreFlag(depsCommand.Flags(), &params.ignore)
	addDataFlag(depsCommand.Flags(), &params.dataPaths)
	addBundleFlag(depsCommand.Flags(), &params.bundlePaths)

	RootCommand.AddCommand(depsCommand)
}

func deps(args []string, params depsCommandParams) error {

	query, err := ast.ParseBody(args[0])
	if err != nil {
		return err
	}

	modules := map[string]*ast.Module{}

	if len(params.dataPaths.v) > 0 {
		f := loaderFilter{
			Ignore: params.ignore,
		}

		result, err := loader.NewFileLoader().Filtered(params.dataPaths.v, f.Apply)
		if err != nil {
			return err
		}

		for _, m := range result.Modules {
			modules[m.Name] = m.Parsed
		}
	}

	if len(params.bundlePaths.v) > 0 {
		for _, path := range params.bundlePaths.v {
			b, err := loader.NewFileLoader().WithSkipBundleVerification(true).AsBundle(path)
			if err != nil {
				return err
			}

			for name, mod := range b.ParsedModules(path) {
				modules[name] = mod
			}
		}
	}

	compiler := ast.NewCompiler()
	compiler.Compile(modules)

	if compiler.Failed() {
		return compiler.Errors
	}

	brs, err := dependencies.Base(compiler, query)
	if err != nil {
		return err
	}

	vrs, err := dependencies.Virtual(compiler, query)
	if err != nil {
		return err
	}

	output := presentation.DepAnalysisOutput{
		Base:    brs,
		Virtual: vrs,
	}

	switch params.format.String() {
	case depsFormatJSON:
		return presentation.JSON(os.Stdout, output)
	default:
		return output.Pretty(os.Stdout)
	}
}
<|endoftext|>


<commit_before>package numgo

import (
	"fmt"
	"math"
	"runtime"
)

// Flatten reshapes the data to a 1-D array.
func (a *Array64) Flatten() *Array64 {
	if a.HasErr() {
		return a
	}
	return a.Reshape(int(a.strides[0]))
}

// C will return a deep copy of the source array.
func (a *Array64) C() (b *Array64) {
	if a.HasErr() {
		return a
	}

	b = &Array64{
		shape:   make([]int, len(a.shape)),
		strides: make([]int, len(a.strides)),
		data:    make([]float64, a.strides[0]),
		err:     nil,
		debug:   "",
		stack:   "",
	}

	copy(b.shape, a.shape)
	copy(b.strides, a.strides)
	copy(b.data, a.data)
	return b
}

// Shape returns a copy of the array shape
func (a *Array64) Shape() []int {
	if a.HasErr() {
		return nil
	}

	res := make([]int, 0, len(a.shape))
	for _, v := range a.shape {
		res = append(res, int(v))
	}

	return res
}

// At returns the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) At(index ...int) float64 {
	idx := a.valIdx(index, "At")
	if a.HasErr() {
		return math.NaN()
	}

	return a.data[idx]
}

func (a *Array64) at(index []int) float64 {
	var idx int
	for i, v := range index {
		idx += v * a.strides[i+1]
	}
	return a.data[idx]
}

func (a *Array64) valIdx(index []int, mthd string) (idx int) {
	if a.HasErr() {
		return 0
	}
	if len(index) > len(a.shape) {
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by %s().  Shape: %v  Index: %v", mthd, a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return 0
	}
	for i, v := range index {
		if int(v) >= a.shape[i] || v < 0 {
			a.err = IndexError
			if debug {
				a.debug = fmt.Sprintf("Index received by %s() does not exist shape: %v index: %v", mthd, a.shape, index)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return 0
		}
		idx += int(v) * a.strides[i+1]
	}
	return
}

// SliceElement returns the element group at one axis above the leaf elements.
// Data is returned as a copy  in a float slice.
func (a *Array64) SliceElement(index ...int) (ret []float64) {
	idx := a.valIdx(index, "SliceElement")
	switch {
	case a.HasErr():
		return nil
	case len(a.shape)-1 != len(index):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return nil
	}

	return append(ret, a.data[idx:idx+a.strides[len(a.strides)-2]]...)
}

// SubArr slices the array at a given index.
func (a *Array64) SubArr(index ...int) (ret *Array64) {
	idx := a.valIdx(index, "SubArr")
	if a.HasErr() {
		return a
	}

	ret = newArray64(a.shape[len(index):]...)
	copy(ret.data, a.data[idx:idx+a.strides[len(index)]])

	return
}

// Set sets the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) Set(val float64, index ...int) *Array64 {
	idx := a.valIdx(index, "Set")
	if a.HasErr() {
		return a
	}

	a.data[idx] = val
	return a
}

// SetSliceElement sets the element group at one axis above the leaf elements.
// Source Array is returned, for function-chaining design.
func (a *Array64) SetSliceElement(vals []float64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSliceElement")
	switch {
	case a.HasErr():
		return a
	case len(a.shape)-1 != len(index):
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SetSliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		fallthrough
	case int(len(vals)) != a.shape[len(a.shape)-1]:
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect slice length received by SetSliceElement().  Shape: %v  Index: %v", a.shape, len(index))
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	copy(a.data[idx:idx+a.strides[len(a.strides)-2]], vals[:a.strides[len(a.strides)-2]])
	return a
}

// SetSubArr sets the array below a given index to the values in vals.
// Values will be broadcast up multiple axes if the shapes match.
func (a *Array64) SetSubArr(vals *Array64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSubArr")
	switch {
	case a.HasErr():
		return a
	case vals.HasErr():
		a.err = vals.getErr()
		if debug {
			a.debug = "Array received by SetSubArr() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(vals.shape)+len(index) > len(a.shape):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Array received by SetSubArr() cant be broadcast.  Shape: %v  Vals shape: %v index: %v", a.shape, vals.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for i, j := len(a.shape)-1, len(vals.shape)-1; j >= 0; i, j = i-1, j-1 {
		if a.shape[i] != vals.shape[j] {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Shape of array recieved by SetSubArr() doesn't match receiver.  Shape: %v  Vals Shape: %v", a.shape, vals.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	if len(a.shape)-len(index)-len(vals.shape) == 0 {
		copy(a.data[idx:idx+int(len(vals.data))], vals.data)
		return a
	}

	reps := int(1)
	for i := len(index); i < len(a.shape)-len(vals.shape); i++ {
		reps *= a.shape[i]
	}

	ln := int(len(vals.data))
	for i := int(1); i <= reps; i++ {
		copy(a.data[idx+ln*(i-1):idx+ln*i], vals.data)
	}
	return a
}

// Resize will change the underlying array size.
//
// Make a copy C() if the original array needs to remain unchanged.
// Element location in the underlying slice will not be adjusted to the new shape.
func (a *Array64) Resize(shape ...int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case len(shape) == 0:
		tmp := newArray64(0)
		a.shape, a.strides = tmp.shape, tmp.strides
		a.data = tmp.data
		return a
	}

	var sz int = 1
	for _, v := range shape {
		if v >= 0 {
			sz *= int(v)
			continue
		}

		a.err = NegativeAxis
		if debug {
			a.debug = fmt.Sprintf("Negative axis length received by Resize.  Shape: %v", shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	ln, cp := len(shape), cap(a.shape)
	if ln > cp {
		a.shape = append(a.shape[:cp], make([]int, ln-cp)...)
	} else {
		a.shape = a.shape[:ln]
	}

	ln, cp = ln+1, cap(a.strides)
	if ln > cp {
		a.strides = append(a.strides[:cp], make([]int, ln-cp)...)
	} else {
		a.strides = a.strides[:ln]
	}

	a.strides[ln-1] = 1
	for i := ln - 2; i >= 0; i-- {
		a.shape[i] = int(shape[i])
		a.strides[i] = a.shape[i] * a.strides[i+1]
	}

	cp = cap(a.data)
	if sz > int(cp) {
		a.data = append(a.data[:cp], make([]float64, sz-int(cp))...)
	} else {
		a.data = a.data[:sz]
	}

	return a
}

// Append will concatenate a and val at the given axis.
//
// Source array will be changed, so use C() if the original data is needed.
// All axes must be the same except the appending axis.
func (a *Array64) Append(val *Array64, axis int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case axis >= len(a.shape), axis < 0:
		a.err = IndexError
		if debug {
			a.debug = fmt.Sprintf("Axis received by Append() out of range.  Shape: %v  Axis: %v", a.shape, axis)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case val.HasErr():
		a.err = val.GetErr()
		if debug {
			a.debug = "Array received by Append() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(a.shape) != len(val.shape):
		a.err = ShapeError
		if debug {
			a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for k, v := range a.shape {
		if v != val.shape[k] && k != axis {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	ln := len(a.data) + len(val.data)
	var dat []float64
	cp := cap(a.data)
	if ln > cp {
		dat = make([]float64, ln)
	} else {
		dat = a.data[:ln]
	}

	as, vs := a.strides[axis], val.strides[axis]
	for i, j := a.strides[0], val.strides[0]; i > 0; i, j = i-as, j-vs {
		copy(dat[i+j-vs:i+j], val.data[j-vs:j])
		copy(dat[i+j-as-vs:i+j-vs], a.data[i-as:i])
	}

	a.data = dat
	a.shape[axis] += val.shape[axis]

	for i := axis; i >= 0; i-- {
		a.strides[i] = a.strides[i+1] * a.shape[i]
	}

	return a
}
<commit_msg>removed unnessesary int casts from accessors.go<commit_after>package numgo

import (
	"fmt"
	"math"
	"runtime"
)

// Flatten reshapes the data to a 1-D array.
func (a *Array64) Flatten() *Array64 {
	if a.HasErr() {
		return a
	}
	return a.Reshape(a.strides[0])
}

// C will return a deep copy of the source array.
func (a *Array64) C() (b *Array64) {
	if a.HasErr() {
		return a
	}

	b = &Array64{
		shape:   make([]int, len(a.shape)),
		strides: make([]int, len(a.strides)),
		data:    make([]float64, a.strides[0]),
		err:     nil,
		debug:   "",
		stack:   "",
	}

	copy(b.shape, a.shape)
	copy(b.strides, a.strides)
	copy(b.data, a.data)
	return b
}

// Shape returns a copy of the array shape
func (a *Array64) Shape() []int {
	if a.HasErr() {
		return nil
	}

	res := make([]int, 0, len(a.shape))
	for _, v := range a.shape {
		res = append(res, v)
	}

	return res
}

// At returns the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) At(index ...int) float64 {
	idx := a.valIdx(index, "At")
	if a.HasErr() {
		return math.NaN()
	}

	return a.data[idx]
}

func (a *Array64) at(index []int) float64 {
	var idx int
	for i, v := range index {
		idx += v * a.strides[i+1]
	}
	return a.data[idx]
}

func (a *Array64) valIdx(index []int, mthd string) (idx int) {
	if a.HasErr() {
		return 0
	}
	if len(index) > len(a.shape) {
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by %s().  Shape: %v  Index: %v", mthd, a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return 0
	}
	for i, v := range index {
		if v >= a.shape[i] || v < 0 {
			a.err = IndexError
			if debug {
				a.debug = fmt.Sprintf("Index received by %s() does not exist shape: %v index: %v", mthd, a.shape, index)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return 0
		}
		idx += v * a.strides[i+1]
	}
	return
}

// SliceElement returns the element group at one axis above the leaf elements.
// Data is returned as a copy  in a float slice.
func (a *Array64) SliceElement(index ...int) (ret []float64) {
	idx := a.valIdx(index, "SliceElement")
	switch {
	case a.HasErr():
		return nil
	case len(a.shape)-1 != len(index):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return nil
	}

	return append(ret, a.data[idx:idx+a.strides[len(a.strides)-2]]...)
}

// SubArr slices the array at a given index.
func (a *Array64) SubArr(index ...int) (ret *Array64) {
	idx := a.valIdx(index, "SubArr")
	if a.HasErr() {
		return a
	}

	ret = newArray64(a.shape[len(index):]...)
	copy(ret.data, a.data[idx:idx+a.strides[len(index)]])

	return
}

// Set sets the element at the given index.
// There should be one index per axis.  Generates a ShapeError if incorrect index.
func (a *Array64) Set(val float64, index ...int) *Array64 {
	idx := a.valIdx(index, "Set")
	if a.HasErr() {
		return a
	}

	a.data[idx] = val
	return a
}

// SetSliceElement sets the element group at one axis above the leaf elements.
// Source Array is returned, for function-chaining design.
func (a *Array64) SetSliceElement(vals []float64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSliceElement")
	switch {
	case a.HasErr():
		return a
	case len(a.shape)-1 != len(index):
		if debug {
			a.debug = fmt.Sprintf("Incorrect number of indicies received by SetSliceElement().  Shape: %v  Index: %v", a.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		fallthrough
	case len(vals) != a.shape[len(a.shape)-1]:
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Incorrect slice length received by SetSliceElement().  Shape: %v  Index: %v", a.shape, len(index))
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	copy(a.data[idx:idx+a.strides[len(a.strides)-2]], vals[:a.strides[len(a.strides)-2]])
	return a
}

// SetSubArr sets the array below a given index to the values in vals.
// Values will be broadcast up multiple axes if the shapes match.
func (a *Array64) SetSubArr(vals *Array64, index ...int) *Array64 {
	idx := a.valIdx(index, "SetSubArr")
	switch {
	case a.HasErr():
		return a
	case vals.HasErr():
		a.err = vals.getErr()
		if debug {
			a.debug = "Array received by SetSubArr() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(vals.shape)+len(index) > len(a.shape):
		a.err = InvIndexError
		if debug {
			a.debug = fmt.Sprintf("Array received by SetSubArr() cant be broadcast.  Shape: %v  Vals shape: %v index: %v", a.shape, vals.shape, index)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for i, j := len(a.shape)-1, len(vals.shape)-1; j >= 0; i, j = i-1, j-1 {
		if a.shape[i] != vals.shape[j] {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Shape of array recieved by SetSubArr() doesn't match receiver.  Shape: %v  Vals Shape: %v", a.shape, vals.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	if len(a.shape)-len(index)-len(vals.shape) == 0 {
		copy(a.data[idx:idx+len(vals.data)], vals.data)
		return a
	}

	reps := 1
	for i := len(index); i < len(a.shape)-len(vals.shape); i++ {
		reps *= a.shape[i]
	}

	ln := len(vals.data)
	for i := 1; i <= reps; i++ {
		copy(a.data[idx+ln*(i-1):idx+ln*i], vals.data)
	}
	return a
}

// Resize will change the underlying array size.
//
// Make a copy C() if the original array needs to remain unchanged.
// Element location in the underlying slice will not be adjusted to the new shape.
func (a *Array64) Resize(shape ...int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case len(shape) == 0:
		tmp := newArray64(0)
		a.shape, a.strides = tmp.shape, tmp.strides
		a.data = tmp.data
		return a
	}

	var sz int = 1
	for _, v := range shape {
		if v >= 0 {
			sz *= v
			continue
		}

		a.err = NegativeAxis
		if debug {
			a.debug = fmt.Sprintf("Negative axis length received by Resize.  Shape: %v", shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	ln, cp := len(shape), cap(a.shape)
	if ln > cp {
		a.shape = append(a.shape[:cp], make([]int, ln-cp)...)
	} else {
		a.shape = a.shape[:ln]
	}

	ln, cp = ln+1, cap(a.strides)
	if ln > cp {
		a.strides = append(a.strides[:cp], make([]int, ln-cp)...)
	} else {
		a.strides = a.strides[:ln]
	}

	a.strides[ln-1] = 1
	for i := ln - 2; i >= 0; i-- {
		a.shape[i] = shape[i]
		a.strides[i] = a.shape[i] * a.strides[i+1]
	}

	cp = cap(a.data)
	if sz > cp {
		a.data = append(a.data[:cp], make([]float64, sz-cp)...)
	} else {
		a.data = a.data[:sz]
	}

	return a
}

// Append will concatenate a and val at the given axis.
//
// Source array will be changed, so use C() if the original data is needed.
// All axes must be the same except the appending axis.
func (a *Array64) Append(val *Array64, axis int) *Array64 {
	switch {
	case a.HasErr():
		return a
	case axis >= len(a.shape), axis < 0:
		a.err = IndexError
		if debug {
			a.debug = fmt.Sprintf("Axis received by Append() out of range.  Shape: %v  Axis: %v", a.shape, axis)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case val.HasErr():
		a.err = val.GetErr()
		if debug {
			a.debug = "Array received by Append() is in error."
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	case len(a.shape) != len(val.shape):
		a.err = ShapeError
		if debug {
			a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
			a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
		}
		return a
	}

	for k, v := range a.shape {
		if v != val.shape[k] && k != axis {
			a.err = ShapeError
			if debug {
				a.debug = fmt.Sprintf("Array received by Append() can not be matched.  Shape: %v  Val shape: %v", a.shape, val.shape)
				a.stack = string(stackBuf[:runtime.Stack(stackBuf, false)])
			}
			return a
		}
	}

	ln := len(a.data) + len(val.data)
	var dat []float64
	cp := cap(a.data)
	if ln > cp {
		dat = make([]float64, ln)
	} else {
		dat = a.data[:ln]
	}

	as, vs := a.strides[axis], val.strides[axis]
	for i, j := a.strides[0], val.strides[0]; i > 0; i, j = i-as, j-vs {
		copy(dat[i+j-vs:i+j], val.data[j-vs:j])
		copy(dat[i+j-as-vs:i+j-vs], a.data[i-as:i])
	}

	a.data = dat
	a.shape[axis] += val.shape[axis]

	for i := axis; i >= 0; i-- {
		a.strides[i] = a.strides[i+1] * a.shape[i]
	}

	return a
}
<|endoftext|>


<commit_before>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package cmd

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os/exec"
)

// ExecSh exec command shell
func ExecSh(str string, args ...string) (string, error) {
	var (
		cmdName = "/bin/bash"
		params  = "-c"
	)

	if len(args) > 0 {
		cmdName = args[0]
	}

	if len(args) > 1 {
		params = args[1]
	}

	cmd := exec.Command(cmdName, params, str)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	return out.String(), err
}

// ExecCmd exex command stdout
func ExecCmd(cmdName string, params []string) bool {
	cmd := exec.Command(cmdName, params...)

	stdout, err := cmd.StdoutPipe()

	if err != nil {
		log.Println("cmd.StdoutPipe error: ", err)
		return false
	}
	cmd.Start()

	reader := bufio.NewReader(stdout)
	for {
		line, err2 := reader.ReadString('\n')
		if err2 != nil || io.EOF == err2 {
			break
		}
		fmt.Println(line)
	}

	cmd.Wait()
	return true
}
<commit_msg>update exec code and add windows support<commit_after>// Copyright 2016 The go-vgo Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// https://github.com/go-ego/ego/blob/master/LICENSE
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

package cmd

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os/exec"
	"runtime"
)

func getName(args ...string) (string, string) {
	var (
		cmdName = "/bin/bash"
		// cmdName = os.Getenv("SHELL")
		params = "-c"
	)

	if runtime.GOOS == "windows" {
		cmdName = "cmd"
		params = "/C"
	}

	if len(args) > 0 {
		cmdName = args[0]
	}

	if len(args) > 1 {
		params = args[1]
	}

	return cmdName, params
}

// Run run cmd shell
func Run(str string, args ...string) (string, error) {
	cmdName, params := getName(args...)

	fmt.Println("cmd run: ", cmdName, params, ": ", str)
	cmd := exec.Command(cmdName, params, str)

	var out bytes.Buffer
	cmd.Stdout = &out

	err := cmd.Run()
	return out.String(), err
}

// Exec exex command stdout
func Exec(cmdName string, params []string) bool {
	cmd := exec.Command(cmdName, params...)

	stdout, err := cmd.StdoutPipe()

	if err != nil {
		log.Println("cmd.StdoutPipe error: ", err)
		return false
	}
	cmd.Start()

	reader := bufio.NewReader(stdout)
	for {
		line, err2 := reader.ReadString('\n')
		if err2 != nil || io.EOF == err2 {
			break
		}
		fmt.Println(line)
	}

	cmd.Wait()
	return true
}
<|endoftext|>


<commit_before>package fastconnect

import (
	"github.com/Jeffail/gabs"
	"github.com/TobiEiss/fiGo"
)

// RetrieveAllBankAccounts retrieves all bankAccounts
func RetrieveAllBankAccounts(connection fiGo.IConnection, accessToken string) ([]map[string]interface{}, error) {
	var transactions []map[string]interface{}

	// get transactions
	answerByte, err := connection.RetrieveAllBankAccounts(accessToken)
	if err != nil {
		return transactions, err
	}

	// try to get accessToken
	jsonParsed, err := gabs.ParseJSON(answerByte)
	accounts, ok := jsonParsed.Path("accounts").Data().([]map[string]interface{})
	if !ok {
		return accounts, err
	}
	return accounts, nil
}
<commit_msg>fix: return interface after retrieveAllBankAccounts<commit_after>package fastconnect

import (
	"log"

	"github.com/Jeffail/gabs"
	"github.com/TobiEiss/fiGo"
)

// RetrieveAllBankAccounts retrieves all bankAccounts
func RetrieveAllBankAccounts(connection fiGo.IConnection, accessToken string) (interface{}, error) {
	// get accounts
	answerByte, err := connection.RetrieveAllBankAccounts(accessToken)
	if err != nil {
		return nil, err
	}

	// try to get accessToken
	log.Println(string(answerByte))
	jsonParsed, err := gabs.ParseJSON(answerByte)
	accounts, ok := jsonParsed.Path("accounts").Data().(interface{})
	if !ok {
		return accounts, err
	}
	return accounts, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/buth/stocker/backend"
	"github.com/buth/stocker/crypto"
	"log"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"syscall"
)

var Exec = &Command{
	UsageLine: "exec [OPTIONS] COMMAND [ARG...]",
	Short:     "execute a command with the given environment",
}

type StringAcumulator []string

func (s *StringAcumulator) Set(value string) error {
	*s = append(*s, value)
	return nil
}

func (s *StringAcumulator) String() string {
	return fmt.Sprintf("%s", *s)
}

var execConfig struct {
	SecretFilepath, Backend, BackendProtocol, BackendHost, Group, User string
	EnvVars                                                            StringAcumulator
}

func init() {
	Exec.Run = execRun
	Exec.Flag.StringVar(&execConfig.SecretFilepath, "k", "", "path to encryption key")
	Exec.Flag.StringVar(&execConfig.Backend, "b", "redis", "backend to use")
	Exec.Flag.StringVar(&execConfig.BackendProtocol, "t", "tcp", "backend connection protocol")
	Exec.Flag.StringVar(&execConfig.BackendHost, "h", ":6379", "backend connection host (optionally including port)")
	Exec.Flag.StringVar(&execConfig.Group, "g", "", "group to use for storing and retrieving data")
	Exec.Flag.StringVar(&execConfig.User, "u", "", "user to execute the command as")
	Exec.Flag.Var(&execConfig.EnvVars, "e", "environment variables")
}

func execRun(cmd *Command, args []string) {

	// Check the number of args.
	if len(args) < 1 {
		cmd.Usage()
	}

	key, err := crypto.NewKeyFromFile(execConfig.SecretFilepath)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	c, err := crypto.NewCrypter(key)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	b, err := backend.NewBackend(execConfig.Backend, execConfig.BackendProtocol, execConfig.BackendHost)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Find the expanded path to cmd.
	command, err := exec.LookPath(args[0])
	if err != nil {
		log.Fatalf("%s: command not found", args[0])
	}

	// Set the args.
	commandArgs := args[1:]

	// Create a map of environment variables to be passed to cmd and
	// initialize it with the current environment.
	env := make(map[string]string)
	for _, variable := range os.Environ() {
		components := strings.Split(variable, "=")
		env[components[0]] = components[1]
	}

	// Loop through the provided environment variables, looking for values
	// first in the environment, and secondarally in the backend store.
	// All errors are fatal.
	for _, variable := range execConfig.EnvVars {

		// Set the key to use with the backend.
		value := os.Getenv(variable)

		// Check if we should search for a value.
		if value == "" {

			cryptedValue, err := b.GetVariable(execConfig.Group, variable)
			if err != nil {
				log.Fatalf("%s: %s", variable, err)
			}

			decryptedValue, err := c.DecryptString(cryptedValue)
			if err != nil {
				log.Fatalf("%s: %s", variable, err)
			}

			value = decryptedValue
		}

		// Format the statement.
		env[variable] = value
	}

	// Create a list of environment key/value pairs and write the
	// flattened environment variables map to it.
	commandEnv := make([]string, 0, len(env))
	for key, value := range env {
		commandEnv = commandEnv[:len(commandEnv)+1]
		commandEnv[len(commandEnv)-1] = fmt.Sprintf("%s=%s", key, value)
	}

	// Handle user.
	if execConfig.User != "" {

		u, err := user.Lookup(execConfig.User)
		if err != nil {
			log.Fatal(err)
		}

		uid, err := strconv.Atoi(u.Uid)
		if err != nil {
			log.Fatal(err)
		}

		if err := syscall.Setuid(uid); err != nil {
			log.Fatal(err)
		}
	}

	// Exec the new command.
	syscall.Exec(command, commandArgs, commandEnv)
}
<commit_msg>arguments must still contain a valid argv[0]<commit_after>package cmd

import (
	"fmt"
	"github.com/buth/stocker/backend"
	"github.com/buth/stocker/crypto"
	"log"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"strings"
	"syscall"
)

var Exec = &Command{
	UsageLine: "exec [OPTIONS] COMMAND [ARG...]",
	Short:     "execute a command with the given environment",
}

type StringAcumulator []string

func (s *StringAcumulator) Set(value string) error {
	*s = append(*s, value)
	return nil
}

func (s *StringAcumulator) String() string {
	return fmt.Sprintf("%s", *s)
}

var execConfig struct {
	SecretFilepath, Backend, BackendProtocol, BackendHost, Group, User string
	EnvVars                                                            StringAcumulator
}

func init() {
	Exec.Run = execRun
	Exec.Flag.StringVar(&execConfig.SecretFilepath, "k", "", "path to encryption key")
	Exec.Flag.StringVar(&execConfig.Backend, "b", "redis", "backend to use")
	Exec.Flag.StringVar(&execConfig.BackendProtocol, "t", "tcp", "backend connection protocol")
	Exec.Flag.StringVar(&execConfig.BackendHost, "h", ":6379", "backend connection host (optionally including port)")
	Exec.Flag.StringVar(&execConfig.Group, "g", "", "group to use for storing and retrieving data")
	Exec.Flag.StringVar(&execConfig.User, "u", "", "user to execute the command as")
	Exec.Flag.Var(&execConfig.EnvVars, "e", "environment variables")
}

func execRun(cmd *Command, args []string) {

	// Check the number of args.
	if len(args) < 1 {
		cmd.Usage()
	}

	key, err := crypto.NewKeyFromFile(execConfig.SecretFilepath)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	c, err := crypto.NewCrypter(key)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	b, err := backend.NewBackend(execConfig.Backend, execConfig.BackendProtocol, execConfig.BackendHost)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Find the expanded path to cmd.
	command, err := exec.LookPath(args[0])
	if err != nil {
		log.Fatalf("%s: command not found", args[0])
	}

	// Create a map of environment variables to be passed to cmd and
	// initialize it with the current environment.
	env := make(map[string]string)
	for _, variable := range os.Environ() {
		components := strings.Split(variable, "=")
		env[components[0]] = components[1]
	}

	// Loop through the provided environment variables, looking for values
	// first in the environment, and secondarally in the backend store.
	// All errors are fatal.
	for _, variable := range execConfig.EnvVars {

		// Set the key to use with the backend.
		value := os.Getenv(variable)

		// Check if we should search for a value.
		if value == "" {

			cryptedValue, err := b.GetVariable(execConfig.Group, variable)
			if err != nil {
				log.Fatalf("%s: %s", variable, err)
			}

			decryptedValue, err := c.DecryptString(cryptedValue)
			if err != nil {
				log.Fatalf("%s: %s", variable, err)
			}

			value = decryptedValue
		}

		// Format the statement.
		env[variable] = value
	}

	// Create a list of environment key/value pairs and write the
	// flattened environment variables map to it.
	commandEnv := make([]string, 0, len(env))
	for key, value := range env {
		commandEnv = commandEnv[:len(commandEnv)+1]
		commandEnv[len(commandEnv)-1] = fmt.Sprintf("%s=%s", key, value)
	}

	// Handle user.
	if execConfig.User != "" {

		u, err := user.Lookup(execConfig.User)
		if err != nil {
			log.Fatal(err)
		}

		uid, err := strconv.Atoi(u.Uid)
		if err != nil {
			log.Fatal(err)
		}

		if err := syscall.Setuid(uid); err != nil {
			log.Fatal(err)
		}
	}

	// Exec the new command.
	syscall.Exec(command, args, commandEnv)
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/crazy-max/cron"
	"github.com/ftpgrab/ftpgrab/internal/app"
	"github.com/ftpgrab/ftpgrab/internal/config"
	"github.com/ftpgrab/ftpgrab/internal/logging"
	"github.com/ftpgrab/ftpgrab/internal/model"
	"github.com/rs/zerolog/log"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	ftpgrab *app.FtpGrab
	flags   *model.Flags
	c       *cron.Cron
	version = "dev"
)

func main() {
	// Parse command line
	kingpin.Flag("config", "Yaml configuration file.").Envar("CONFIG").Required().StringVar(&flags.Cfgfile)
	kingpin.Flag("output", "Output destination folder.").Envar("OUTPUT").Required().StringVar(&flags.Output)
	kingpin.Flag("schedule", "CRON expression format.").Envar("SCHEDULE").StringVar(&flags.Schedule)
	kingpin.Flag("log-level", "Set log level.").Envar("LOG_LEVEL").Default("info").StringVar(&flags.LogLevel)
	kingpin.Flag("log-file", "Enable logging to file.").Envar("LOG_FILE").Default("false").BoolVar(&flags.LogFile)
	kingpin.Flag("log-nocolor", "Disable the colorized output.").Envar("LOG_NOCOLOR").Default("false").BoolVar(&flags.LogNocolor)
	kingpin.Flag("log-ftp", "Enable FTP log.").Envar("LOG_FTP").Default("false").BoolVar(&flags.LogFtp)
	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(version).Author("CrazyMax")
	kingpin.CommandLine.Name = "ftpgrab"
	kingpin.CommandLine.Help = `Grab your files from a remote FTP server easily. More info : https://ftpgrab.github.io`
	kingpin.Parse()

	// Init
	logging.Configure(flags)
	log.Info().Msgf("Starting FTPGrab %s", version)

	// Handle os signals
	channel := make(chan os.Signal)
	signal.Notify(channel, os.Interrupt, syscall.SIGTERM)
	go func() {
		sig := <-channel
		if c != nil {
			c.Stop()
		}
		ftpgrab.Close()
		log.Warn().Msgf("Caught signal %v", sig)
		os.Exit(0)
	}()

	// Load and check configuration
	cfg, err := config.Load(flags, version)
	if err != nil {
		log.Fatal().Err(err).Msg("Cannot load configuration")
	}
	if err := cfg.Check(); err != nil {
		log.Fatal().Err(err).Msg("Improper configuration")
	}

	// Init
	if ftpgrab, err = app.New(cfg); err != nil {
		log.Fatal().Err(err).Msg("Cannot initialize FTPGrab")
	}

	// Run immediately if schedule is not defined
	if flags.Schedule == "" {
		ftpgrab.Run()
		return
	}

	// Start cronjob
	c = cron.NewWithLocation(cfg.Location)
	log.Info().Msgf("Add cronjob with schedule %s", flags.Schedule)
	if err := c.AddJob(flags.Schedule, ftpgrab); err != nil {
		log.Fatal().Err(err).Msg("Cannot create cron task")
	}
	c.Start()

	for {
		runtime.Gosched()
	}
}
<commit_msg>Fix flags issue<commit_after>package main

import (
	"os"
	"os/signal"
	"runtime"
	"syscall"

	"github.com/crazy-max/cron"
	"github.com/ftpgrab/ftpgrab/internal/app"
	"github.com/ftpgrab/ftpgrab/internal/config"
	"github.com/ftpgrab/ftpgrab/internal/logging"
	"github.com/ftpgrab/ftpgrab/internal/model"
	"github.com/rs/zerolog/log"
	"gopkg.in/alecthomas/kingpin.v2"
)

var (
	ftpgrab *app.FtpGrab
	flags   model.Flags
	c       *cron.Cron
	version = "dev"
)

func main() {
	// Parse command line
	kingpin.Flag("config", "Yaml configuration file.").Envar("CONFIG").Required().StringVar(&flags.Cfgfile)
	kingpin.Flag("output", "Output destination folder.").Envar("OUTPUT").Required().StringVar(&flags.Output)
	kingpin.Flag("schedule", "CRON expression format.").Envar("SCHEDULE").StringVar(&flags.Schedule)
	kingpin.Flag("log-level", "Set log level.").Envar("LOG_LEVEL").Default("info").StringVar(&flags.LogLevel)
	kingpin.Flag("log-file", "Enable logging to file.").Envar("LOG_FILE").Default("false").BoolVar(&flags.LogFile)
	kingpin.Flag("log-nocolor", "Disable the colorized output.").Envar("LOG_NOCOLOR").Default("false").BoolVar(&flags.LogNocolor)
	kingpin.Flag("log-ftp", "Enable FTP log.").Envar("LOG_FTP").Default("false").BoolVar(&flags.LogFtp)
	kingpin.UsageTemplate(kingpin.CompactUsageTemplate).Version(version).Author("CrazyMax")
	kingpin.CommandLine.Name = "ftpgrab"
	kingpin.CommandLine.Help = `Grab your files from a remote FTP server easily. More info : https://ftpgrab.github.io`
	kingpin.Parse()

	// Init
	logging.Configure(&flags)
	log.Info().Msgf("Starting FTPGrab %s", version)

	// Handle os signals
	channel := make(chan os.Signal)
	signal.Notify(channel, os.Interrupt, syscall.SIGTERM)
	go func() {
		sig := <-channel
		if c != nil {
			c.Stop()
		}
		ftpgrab.Close()
		log.Warn().Msgf("Caught signal %v", sig)
		os.Exit(0)
	}()

	// Load and check configuration
	cfg, err := config.Load(&flags, version)
	if err != nil {
		log.Fatal().Err(err).Msg("Cannot load configuration")
	}
	if err := cfg.Check(); err != nil {
		log.Fatal().Err(err).Msg("Improper configuration")
	}

	// Init
	if ftpgrab, err = app.New(cfg); err != nil {
		log.Fatal().Err(err).Msg("Cannot initialize FTPGrab")
	}

	// Run immediately if schedule is not defined
	if flags.Schedule == "" {
		ftpgrab.Run()
		return
	}

	// Start cronjob
	c = cron.NewWithLocation(cfg.Location)
	log.Info().Msgf("Add cronjob with schedule %s", flags.Schedule)
	if err := c.AddJob(flags.Schedule, ftpgrab); err != nil {
		log.Fatal().Err(err).Msg("Cannot create cron task")
	}
	c.Start()

	for {
		runtime.Gosched()
	}
}
<|endoftext|>